diff options
186 files changed, 5570 insertions, 4248 deletions
diff --git a/appveyor.yml b/appveyor.yml deleted file mode 100644 index 2b1eab5d0..000000000 --- a/appveyor.yml +++ /dev/null @@ -1,36 +0,0 @@ -clone_depth: 5 - -artifacts: - - path: bin\nim.exe - -platform: - - x64 - -before_build: - - git log -1 - - C:\msys64\usr\bin\bash -lc "pacman --noconfirm -S zlib-devel" - - appveyor DownloadFile http://nim-lang.org/download/dlls.zip - - 7z e dlls.zip -odlls - - del dlls\libcurl.dll - - appveyor DownloadFile http://flatassembler.net/fasmw17139.zip - - 7z e fasmw17139.zip -obin fasm.exe - -build_script: - - SET PATH=C:\msys64\mingw64\bin;dlls;bin;%PATH% - - gcc -v - - git clone -q --depth 1 https://github.com/nim-lang/csources - - cd csources - - build64.bat - - cd .. - - nim c koch - - koch boot - - koch boot -d:release - -before_test: - - nim e install_nimble.nims - - nimble update - - nimble install zip - -test_script: - - nim c --taintMode:on tests/testament/tester - - tests\testament\tester --pedantic all diff --git a/compiler.nimble b/compiler.nimble index cb0a08a3f..a98981a97 100644 --- a/compiler.nimble +++ b/compiler.nimble @@ -1,6 +1,6 @@ [Package] name = "compiler" -version = "0.12.0" +version = "0.13.0" author = "Andreas Rumpf" description = "Compiler package providing the compiler sources as a library." license = "MIT" diff --git a/compiler/ast.nim b/compiler/ast.nim index e073ce3e1..2ce0afcc3 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -500,8 +500,7 @@ type skResult, # special 'result' variable skProc, # a proc skMethod, # a method - skIterator, # an inline iterator - skClosureIterator, # a resumable closure iterator + skIterator, # an iterator skConverter, # a type converter skMacro, # a macro skTemplate, # a template; currently also misused for user-defined @@ -518,7 +517,7 @@ type TSymKinds* = set[TSymKind] const - routineKinds* = {skProc, skMethod, skIterator, skClosureIterator, + routineKinds* = {skProc, skMethod, skIterator, skConverter, skMacro, skTemplate} tfIncompleteStruct* = tfVarargs tfUncheckedArray* = tfVarargs @@ -905,7 +904,7 @@ type # the poor naming choices in the standard library. const - OverloadableSyms* = {skProc, skMethod, skIterator, skClosureIterator, + OverloadableSyms* = {skProc, skMethod, skIterator, skConverter, skModule, skTemplate, skMacro} GenericTypes*: TTypeKinds = {tyGenericInvocation, tyGenericBody, @@ -929,11 +928,11 @@ const NilableTypes*: TTypeKinds = {tyPointer, tyCString, tyRef, tyPtr, tySequence, tyProc, tyString, tyError} ExportableSymKinds* = {skVar, skConst, skProc, skMethod, skType, - skIterator, skClosureIterator, + skIterator, skMacro, skTemplate, skConverter, skEnumField, skLet, skStub, skAlias} PersistentNodeFlags*: TNodeFlags = {nfBase2, nfBase8, nfBase16, nfDotSetter, nfDotField, - nfIsRef, nfIsCursor} + nfIsRef, nfIsCursor, nfLL} namePos* = 0 patternPos* = 1 # empty except for term rewriting macros genericParamsPos* = 2 @@ -958,11 +957,9 @@ const nkStrKinds* = {nkStrLit..nkTripleStrLit} skLocalVars* = {skVar, skLet, skForVar, skParam, skResult} - skProcKinds* = {skProc, skTemplate, skMacro, skIterator, skClosureIterator, + skProcKinds* = {skProc, skTemplate, skMacro, skIterator, skMethod, skConverter} - skIterators* = {skIterator, skClosureIterator} - var ggDebug* {.deprecated.}: bool ## convenience switch for trying out things proc isCallExpr*(n: PNode): bool = @@ -1013,6 +1010,10 @@ proc newNode*(kind: TNodeKind): PNode = writeStackTrace() inc gNodeId +proc newTree*(kind: TNodeKind; children: varargs[PNode]): PNode = + result = newNode(kind) + result.sons = @children + proc newIntNode*(kind: TNodeKind, intVal: BiggestInt): PNode = result = newNode(kind) result.intVal = intVal @@ -1554,12 +1555,13 @@ proc isGenericRoutine*(s: PSym): bool = else: discard proc skipGenericOwner*(s: PSym): PSym = - internalAssert s.kind in skProcKinds ## Generic instantiations are owned by their originating generic ## symbol. This proc skips such owners and goes straight to the owner ## of the generic itself (the module or the enclosing proc). - result = if sfFromGeneric in s.flags: s.owner.owner - else: s.owner + result = if s.kind in skProcKinds and sfFromGeneric in s.flags: + s.owner.owner + else: + s.owner proc originatingModule*(s: PSym): PSym = result = s.owner diff --git a/compiler/ccgcalls.nim b/compiler/ccgcalls.nim index 86ecc9db8..bd17f85e4 100644 --- a/compiler/ccgcalls.nim +++ b/compiler/ccgcalls.nim @@ -118,6 +118,14 @@ proc openArrayLoc(p: BProc, n: PNode): Rope = result = "$1->data, $1->$2" % [a.rdLoc, lenField(p)] of tyArray, tyArrayConstr: result = "$1, $2" % [rdLoc(a), rope(lengthOrd(a.t))] + of tyPtr, tyRef: + case lastSon(a.t).kind + of tyString, tySequence: + result = "(*$1)->data, (*$1)->$2" % [a.rdLoc, lenField(p)] + of tyArray, tyArrayConstr: + result = "$1, $2" % [rdLoc(a), rope(lengthOrd(lastSon(a.t)))] + else: + internalError("openArrayLoc: " & typeToString(a.t)) else: internalError("openArrayLoc: " & typeToString(a.t)) proc genArgStringToCString(p: BProc, n: PNode): Rope {.inline.} = @@ -515,7 +523,7 @@ proc genNamedParamCall(p: BProc, ri: PNode, d: var TLoc) = line(p, cpsStmts, pl) proc genCall(p: BProc, e: PNode, d: var TLoc) = - if e.sons[0].typ.callConv == ccClosure: + if e.sons[0].typ.skipTypes({tyGenericInst}).callConv == ccClosure: genClosureCall(p, nil, e, d) elif e.sons[0].kind == nkSym and sfInfixCall in e.sons[0].sym.flags: genInfixCall(p, nil, e, d) @@ -528,7 +536,7 @@ proc genCall(p: BProc, e: PNode, d: var TLoc) = if d.s == onStack and containsGarbageCollectedRef(d.t): keepAlive(p, d) proc genAsgnCall(p: BProc, le, ri: PNode, d: var TLoc) = - if ri.sons[0].typ.callConv == ccClosure: + if ri.sons[0].typ.skipTypes({tyGenericInst}).callConv == ccClosure: genClosureCall(p, le, ri, d) elif ri.sons[0].kind == nkSym and sfInfixCall in ri.sons[0].sym.flags: genInfixCall(p, le, ri, d) diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index c9ff9d4f0..3607f347e 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -959,6 +959,7 @@ proc genEcho(p: BProc, n: PNode) = addf(args, ", $1? ($1)->data:\"nil\"", [rdLoc(a)]) linefmt(p, cpsStmts, "printf($1$2);$n", makeCString(repeat("%s", n.len) & tnl), args) + linefmt(p, cpsStmts, "fflush(stdout);$n") proc gcUsage(n: PNode) = if gSelectedGC == gcNone: message(n.info, warnGcMem, n.renderTree) @@ -1415,11 +1416,11 @@ proc binaryExprIn(p: BProc, e: PNode, a, b, d: var TLoc, frmt: string) = proc genInExprAux(p: BProc, e: PNode, a, b, d: var TLoc) = case int(getSize(skipTypes(e.sons[1].typ, abstractVar))) - of 1: binaryExprIn(p, e, a, b, d, "(($1 &(1<<(($2)&7)))!=0)") - of 2: binaryExprIn(p, e, a, b, d, "(($1 &(1<<(($2)&15)))!=0)") - of 4: binaryExprIn(p, e, a, b, d, "(($1 &(1<<(($2)&31)))!=0)") - of 8: binaryExprIn(p, e, a, b, d, "(($1 &(IL64(1)<<(($2)&IL64(63))))!=0)") - else: binaryExprIn(p, e, a, b, d, "(($1[$2/8] &(1<<($2%8)))!=0)") + of 1: binaryExprIn(p, e, a, b, d, "(($1 &(1U<<((NU)($2)&7U)))!=0)") + of 2: binaryExprIn(p, e, a, b, d, "(($1 &(1U<<((NU)($2)&15U)))!=0)") + of 4: binaryExprIn(p, e, a, b, d, "(($1 &(1U<<((NU)($2)&31U)))!=0)") + of 8: binaryExprIn(p, e, a, b, d, "(($1 &((NU64)1<<((NU)($2)&63U)))!=0)") + else: binaryExprIn(p, e, a, b, d, "(($1[(NU)($2)>>3] &(1U<<((NU)($2)&7U)))!=0)") proc binaryStmtInExcl(p: BProc, e: PNode, d: var TLoc, frmt: string) = var a, b: TLoc @@ -1500,8 +1501,8 @@ proc genSetOp(p: BProc, e: PNode, d: var TLoc, op: TMagic) = else: internalError(e.info, "genSetOp()") else: case op - of mIncl: binaryStmtInExcl(p, e, d, "$1[$2/8] |=(1<<($2%8));$n") - of mExcl: binaryStmtInExcl(p, e, d, "$1[$2/8] &= ~(1<<($2%8));$n") + of mIncl: binaryStmtInExcl(p, e, d, "$1[(NU)($2)>>3] |=(1U<<($2&7U));$n") + of mExcl: binaryStmtInExcl(p, e, d, "$1[(NU)($2)>>3] &= ~(1U<<($2&7U));$n") of mCard: unaryExprChar(p, e, d, "#cardSet($1, " & $size & ')') of mLtSet, mLeSet: getTemp(p, getSysType(tyInt), i) # our counter @@ -1733,8 +1734,6 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) = of mEcho: genEcho(p, e[1].skipConv) of mArrToSeq: genArrToSeq(p, e, d) of mNLen..mNError, mSlurp..mQuoteAst: - echo "from here ", p.prc.name.s, " ", p.prc.info - writestacktrace() localError(e.info, errXMustBeCompileTime, e.sons[0].sym.name.s) of mSpawn: let n = lowerings.wrapProcForSpawn(p.module.module, e, e.typ, nil, nil) @@ -1788,11 +1787,11 @@ proc genSetConstr(p: BProc, e: PNode, d: var TLoc) = initLocExpr(p, e.sons[i].sons[0], a) initLocExpr(p, e.sons[i].sons[1], b) lineF(p, cpsStmts, "for ($1 = $3; $1 <= $4; $1++) $n" & - "$2[$1/8] |=(1<<($1%8));$n", [rdLoc(idx), rdLoc(d), + "$2[(NU)($1)>>3] |=(1U<<((NU)($1)&7U));$n", [rdLoc(idx), rdLoc(d), rdSetElemLoc(a, e.typ), rdSetElemLoc(b, e.typ)]) else: initLocExpr(p, e.sons[i], a) - lineF(p, cpsStmts, "$1[$2/8] |=(1<<($2%8));$n", + lineF(p, cpsStmts, "$1[(NU)($2)>>3] |=(1U<<((NU)($2)&7U));$n", [rdLoc(d), rdSetElemLoc(a, e.typ)]) else: # small set @@ -1839,15 +1838,17 @@ proc genClosure(p: BProc, n: PNode, d: var TLoc) = assert n.kind == nkClosure if isConstClosure(n): - inc(p.labels) - var tmp = "LOC" & rope(p.labels) - addf(p.module.s[cfsData], "NIM_CONST $1 $2 = $3;$n", + inc(p.module.labels) + var tmp = "CNSTCLOSURE" & rope(p.module.labels) + addf(p.module.s[cfsData], "static NIM_CONST $1 $2 = $3;$n", [getTypeDesc(p.module, n.typ), tmp, genConstExpr(p, n)]) putIntoDest(p, d, n.typ, tmp, OnStatic) else: var tmp, a, b: TLoc initLocExpr(p, n.sons[0], a) initLocExpr(p, n.sons[1], b) + if n.sons[0].skipConv.kind == nkClosure: + internalError(n.info, "closure to closure created") getTemp(p, n.typ, tmp) linefmt(p, cpsStmts, "$1.ClPrc = $2; $1.ClEnv = $3;$n", tmp.rdLoc, a.rdLoc, b.rdLoc) @@ -1966,7 +1967,9 @@ proc expr(p: BProc, n: PNode, d: var TLoc) = else: genProc(p.module, sym) putLocIntoDest(p, d, sym.loc) - of skProc, skConverter, skIterators: + of skProc, skConverter, skIterator: + #if sym.kind == skIterator: + # echo renderTree(sym.getBody, {renderIds}) if sfCompileTime in sym.flags: localError(n.info, "request to generate code for .compileTime proc: " & sym.name.s) @@ -1988,6 +1991,7 @@ proc expr(p: BProc, n: PNode, d: var TLoc) = if sfGlobal in sym.flags: genVarPrototype(p.module, sym) if sym.loc.r == nil or sym.loc.t == nil: #echo "FAILED FOR PRCO ", p.prc.name.s + #echo renderTree(p.prc.ast, {renderIds}) internalError n.info, "expr: var not init " & sym.name.s & "_" & $sym.id if sfThread in sym.flags: accessThreadLocalVar(p, sym) @@ -2005,9 +2009,9 @@ proc expr(p: BProc, n: PNode, d: var TLoc) = putLocIntoDest(p, d, sym.loc) of skParam: if sym.loc.r == nil or sym.loc.t == nil: - #echo "FAILED FOR PRCO ", p.prc.name.s - #debug p.prc.typ.n - #echo renderTree(p.prc.ast, {renderIds}) + # echo "FAILED FOR PRCO ", p.prc.name.s + # debug p.prc.typ.n + # echo renderTree(p.prc.ast, {renderIds}) internalError(n.info, "expr: param not init " & sym.name.s & "_" & $sym.id) putLocIntoDest(p, d, sym.loc) else: internalError(n.info, "expr(" & $sym.kind & "); unknown symbol") diff --git a/compiler/ccgstmts.nim b/compiler/ccgstmts.nim index f4a7c4400..529e59273 100644 --- a/compiler/ccgstmts.nim +++ b/compiler/ccgstmts.nim @@ -16,13 +16,13 @@ const # above X strings a hash-switch for strings is generated proc registerGcRoot(p: BProc, v: PSym) = - if gSelectedGC in {gcMarkAndSweep, gcGenerational} and + if gSelectedGC in {gcMarkAndSweep, gcGenerational, gcV2} and containsGarbageCollectedRef(v.loc.t): # we register a specialized marked proc here; this has the advantage # that it works out of the box for thread local storage then :-) let prc = genTraverseProcForGlobal(p.module, v) - linefmt(p.module.initProc, cpsStmts, - "#nimRegisterGlobalMarker($1);$n", prc) + appcg(p.module, p.module.initProc.procSec(cpsStmts), + "#nimRegisterGlobalMarker($1);$n", [prc]) proc isAssignedImmediately(n: PNode): bool {.inline.} = if n.kind == nkEmpty: return false @@ -955,7 +955,7 @@ proc genAsmOrEmitStmt(p: BProc, t: PNode, isAsmStmt=false): Rope = res.add(t.sons[i].strVal) of nkSym: var sym = t.sons[i].sym - if sym.kind in {skProc, skIterator, skClosureIterator, skMethod}: + if sym.kind in {skProc, skIterator, skMethod}: var a: TLoc initLocExpr(p, t.sons[i], a) res.add($rdLoc(a)) diff --git a/compiler/cgen.nim b/compiler/cgen.nim index f63134b66..db376821c 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -594,7 +594,7 @@ proc cgsym(m: BModule, name: string): Rope = var sym = magicsys.getCompilerProc(name) if sym != nil: case sym.kind - of skProc, skMethod, skConverter, skIterators: genProc(m, sym) + of skProc, skMethod, skConverter, skIterator: genProc(m, sym) of skVar, skResult, skLet: genVarPrototype(m, sym) of skType: discard getTypeDesc(m, sym.typ) else: internalError("cgsym: " & name & ": " & $sym.kind) diff --git a/compiler/cgmeth.nim b/compiler/cgmeth.nim index 3c2c51b76..312afec1a 100644 --- a/compiler/cgmeth.nim +++ b/compiler/cgmeth.nim @@ -18,8 +18,10 @@ proc genConv(n: PNode, d: PType, downcast: bool): PNode = var source = skipTypes(n.typ, abstractPtrs) if (source.kind == tyObject) and (dest.kind == tyObject): var diff = inheritanceDiff(dest, source) - if diff == high(int): internalError(n.info, "cgmeth.genConv") - if diff < 0: + if diff == high(int): + # no subtype relation, nothing to do + result = n + elif diff < 0: result = newNodeIT(nkObjUpConv, n.info, d) addSon(result, n) if downcast: internalError(n.info, "cgmeth.genConv: no upcast allowed") diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 8536cc619..8555ec4f0 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -149,10 +149,10 @@ proc ropeFormatNamedVars(frmt: FormatStr, varnames: openArray[string], proc genComment(d: PDoc, n: PNode): string = result = "" var dummyHasToc: bool - if n.comment != nil and startsWith(n.comment, "##"): + if n.comment != nil: renderRstToOut(d[], parseRst(n.comment, toFilename(n.info), toLinenumber(n.info), toColumn(n.info), - dummyHasToc, d.options + {roSkipPounds}), result) + dummyHasToc, d.options), result) proc genRecComment(d: PDoc, n: PNode): Rope = if n == nil: return nil @@ -537,7 +537,7 @@ proc generateJson(d: PDoc, n: PNode, jArray: JsonNode = nil): JsonNode = proc genSection(d: PDoc, kind: TSymKind) = const sectionNames: array[skModule..skTemplate, string] = [ "Imports", "Types", "Vars", "Lets", "Consts", "Vars", "Procs", "Methods", - "Iterators", "Iterators", "Converters", "Macros", "Templates" + "Iterators", "Converters", "Macros", "Templates" ] if d.section[kind] == nil: return var title = sectionNames[kind].rope diff --git a/compiler/evaltempl.nim b/compiler/evaltempl.nim index 863aa696e..a5a132005 100644 --- a/compiler/evaltempl.nim +++ b/compiler/evaltempl.nim @@ -38,7 +38,8 @@ proc evalTemplateAux(templ, actual: PNode, c: var TemplCtx, result: PNode) = if s.owner.id == c.owner.id: if s.kind == skParam and sfGenSym notin s.flags: handleParam actual.sons[s.position] - elif s.kind == skGenericParam: + elif s.kind == skGenericParam or + s.kind == skType and s.typ != nil and s.typ.kind == tyGenericParam: handleParam actual.sons[s.owner.typ.len + s.position - 1] else: internalAssert sfGenSym in s.flags diff --git a/compiler/installer.ini b/compiler/installer.ini index 778cf75bf..0ec6e52f1 100644 --- a/compiler/installer.ini +++ b/compiler/installer.ini @@ -62,7 +62,7 @@ Files: "icons/koch_icon.o" Files: "compiler/readme.txt" Files: "compiler/installer.ini" -Files: "compiler/nim.nim.cfg" +Files: "compiler/*.cfg" Files: "compiler/*.nim" Files: "doc/*.txt" Files: "doc/manual/*.txt" @@ -84,6 +84,8 @@ Files: "tools/niminst/*.nsh" Files: "web/website.ini" Files: "web/*.nim" Files: "web/*.txt" +Files: "bin/nimblepkg/*.nim" +Files: "bin/nimblepkg/*.cfg" [Lib] Files: "lib/nimbase.h" @@ -107,9 +109,6 @@ Files: "lib/wrappers/readline/*.nim" Files: "lib/wrappers/linenoise/*.nim" Files: "lib/wrappers/linenoise/*.c" Files: "lib/wrappers/linenoise/*.h" -Files: "lib/wrappers/sdl/*.nim" -Files: "lib/wrappers/zip/*.nim" -Files: "lib/wrappers/zip/libzip_all.c" Files: "lib/windows/*.nim" Files: "lib/posix/*.nim" diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index f8bf35ed6..14cdf0174 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -931,7 +931,7 @@ proc isIndirect(v: PSym): bool = result = {sfAddrTaken, sfGlobal} * v.flags != {} and #(mapType(v.typ) != etyObject) and {sfImportc, sfVolatile, sfExportc} * v.flags == {} and - v.kind notin {skProc, skConverter, skMethod, skIterator, skClosureIterator, + v.kind notin {skProc, skConverter, skMethod, skIterator, skConst, skTemp, skLet} proc genAddr(p: PProc, n: PNode, r: var TCompRes) = @@ -1636,7 +1636,10 @@ proc gen(p: PProc, n: PNode, r: var TCompRes) = of nkSym: genSym(p, n, r) of nkCharLit..nkInt64Lit: - r.res = rope(n.intVal) + if n.typ.kind == tyBool: + r.res = if n.intVal == 0: rope"false" else: rope"true" + else: + r.res = rope(n.intVal) r.kind = resExpr of nkNilLit: if isEmptyType(n.typ): diff --git a/compiler/jstypes.nim b/compiler/jstypes.nim index 367c173ea..611f50eaf 100644 --- a/compiler/jstypes.nim +++ b/compiler/jstypes.nim @@ -116,7 +116,7 @@ proc genEnumInfo(p: PProc, typ: PType, name: Rope) = [name, genTypeInfo(p, typ.sons[0])]) proc genTypeInfo(p: PProc, typ: PType): Rope = - let t = typ.skipTypes({tyGenericInst}) + let t = typ.skipTypes({tyGenericInst, tyDistinct}) result = "NTI$1" % [rope(t.id)] if containsOrIncl(p.g.typeInfoGenerated, t.id): return case t.kind diff --git a/compiler/lambdalifting.nim b/compiler/lambdalifting.nim index cccc94756..1c0c2494a 100644 --- a/compiler/lambdalifting.nim +++ b/compiler/lambdalifting.nim @@ -11,7 +11,7 @@ import intsets, strutils, lists, options, ast, astalgo, trees, treetab, msgs, os, - idents, renderer, types, magicsys, rodread, lowerings + idents, renderer, types, magicsys, rodread, lowerings, tables discard """ The basic approach is that captured vars need to be put on the heap and @@ -113,43 +113,19 @@ discard """ # local storage requirements for efficiency. This means closure iterators # have slightly different semantics from ordinary closures. +# ---------------- essential helpers ------------------------------------- const upName* = ":up" # field name for the 'up' reference paramName* = ":envP" envName* = ":env" -type - POuterContext = ref TOuterContext - - TIter = object - fn, closureParam, state, resultSym: PSym # most are only valid if - # fn.kind == skClosureIterator - obj: PType - - PEnv = ref TEnv - TEnv {.final.} = object of RootObj - attachedNode, replacementNode: PNode - createdVar: PNode # if != nil it is a used environment; for closure - # iterators this can be 'envParam.env' - createdVarComesFromIter: bool - capturedVars: seq[PSym] # captured variables in this environment - up, next: PEnv # outer scope and next to keep all in a list - upField: PSym # if != nil the dependency to the outer scope is used - obj: PType - fn: PSym # function that belongs to this scope; - # if up.fn != fn then we cross function boundaries. - # This is an important case to consider. - vars: IntSet # variables belonging to this environment - - TOuterContext = object - fn: PSym # may also be a module! - head: PEnv - capturedVars, processed: IntSet - localsToAccess: TIdNodeTable - lambdasToEnv: TIdTable # PSym->PEnv mapping - -proc getStateType(iter: PSym): PType = +proc newCall(a: PSym, b: PNode): PNode = + result = newNodeI(nkCall, a.info) + result.add newSymNode(a) + result.add b + +proc createStateType(iter: PSym): PType = var n = newNodeI(nkRange, iter.info) addSon(n, newIntNode(nkIntLit, -1)) addSon(n, newIntNode(nkIntLit, 0)) @@ -161,7 +137,7 @@ proc getStateType(iter: PSym): PType = proc createStateField(iter: PSym): PSym = result = newSym(skField, getIdent(":state"), iter, iter.info) - result.typ = getStateType(iter) + result.typ = createStateType(iter) proc createEnvObj(owner: PSym): PType = # YYY meh, just add the state field for every closure for now, it's too @@ -169,7 +145,7 @@ proc createEnvObj(owner: PSym): PType = result = createObj(owner, owner.info) rawAddField(result, createStateField(owner)) -proc newIterResult(iter: PSym): PSym = +proc getIterResult(iter: PSym): PSym = if resultPos < iter.ast.len: result = iter.ast.sons[resultPos].sym else: @@ -186,513 +162,445 @@ proc addHiddenParam(routine: PSym, param: PSym) = # some nkEffect node: param.position = routine.typ.n.len-1 addSon(params, newSymNode(param)) - incl(routine.typ.flags, tfCapturesEnv) + #incl(routine.typ.flags, tfCapturesEnv) assert sfFromGeneric in param.flags - #echo "produced environment: ", param.id, " for ", routine.name.s + #echo "produced environment: ", param.id, " for ", routine.id proc getHiddenParam(routine: PSym): PSym = let params = routine.ast.sons[paramsPos] let hidden = lastSon(params) - internalAssert hidden.kind == nkSym and hidden.sym.kind == skParam - result = hidden.sym - assert sfFromGeneric in result.flags + if hidden.kind == nkSym and hidden.sym.kind == skParam and hidden.sym.name.s == paramName: + result = hidden.sym + assert sfFromGeneric in result.flags + else: + # writeStackTrace() + localError(routine.info, "internal error: could not find env param for " & routine.name.s) + result = routine -proc getEnvParam(routine: PSym): PSym = +proc getEnvParam*(routine: PSym): PSym = let params = routine.ast.sons[paramsPos] let hidden = lastSon(params) if hidden.kind == nkSym and hidden.sym.name.s == paramName: result = hidden.sym assert sfFromGeneric in result.flags -proc initIter(iter: PSym): TIter = - result.fn = iter - if iter.kind == skClosureIterator: - var cp = getEnvParam(iter) - if cp == nil: - result.obj = createEnvObj(iter) - - cp = newSym(skParam, getIdent(paramName), iter, iter.info) - incl(cp.flags, sfFromGeneric) - cp.typ = newType(tyRef, iter) - rawAddSon(cp.typ, result.obj) - addHiddenParam(iter, cp) - else: - result.obj = cp.typ.sons[0] - assert result.obj.kind == tyObject - internalAssert result.obj.n.len > 0 - result.state = result.obj.n[0].sym - result.closureParam = cp - if iter.typ.sons[0] != nil: - result.resultSym = newIterResult(iter) - #iter.ast.add(newSymNode(c.resultSym)) - -proc newOuterContext(fn: PSym): POuterContext = - new(result) - result.fn = fn - result.capturedVars = initIntSet() - result.processed = initIntSet() - initIdNodeTable(result.localsToAccess) - initIdTable(result.lambdasToEnv) - -proc newEnv(o: POuterContext; up: PEnv, n: PNode; owner: PSym): PEnv = - new(result) - result.capturedVars = @[] - result.up = up - result.attachedNode = n - result.fn = owner - result.vars = initIntSet() - result.next = o.head - o.head = result - if owner.kind != skModule and (up == nil or up.fn != owner): - let param = getEnvParam(owner) - if param != nil: - result.obj = param.typ.sons[0] - assert result.obj.kind == tyObject - if result.obj.isNil: - result.obj = createEnvObj(owner) - -proc addCapturedVar(e: PEnv, v: PSym) = - for x in e.capturedVars: - if x == v: return - e.capturedVars.add(v) - addField(e.obj, v) - -proc newCall(a: PSym, b: PNode): PNode = - result = newNodeI(nkCall, a.info) - result.add newSymNode(a) - result.add b - -proc isInnerProc(s, outerProc: PSym): bool = - if s.kind in {skProc, skMethod, skConverter, skClosureIterator}: - var owner = s.skipGenericOwner - while true: - if owner.isNil: return false - if owner == outerProc: return true - owner = owner.owner - #s.typ.callConv == ccClosure - -proc addClosureParam(fn: PSym; e: PEnv) = - var cp = getEnvParam(fn) - if cp == nil: - cp = newSym(skParam, getIdent(paramName), fn, fn.info) - incl(cp.flags, sfFromGeneric) - cp.typ = newType(tyRef, fn) - rawAddSon(cp.typ, e.obj) - addHiddenParam(fn, cp) - #else: - #cp.typ.sons[0] = e.obj - #assert e.obj.kind == tyObject +proc interestingVar(s: PSym): bool {.inline.} = + result = s.kind in {skVar, skLet, skTemp, skForVar, skParam, skResult} and + sfGlobal notin s.flags proc illegalCapture(s: PSym): bool {.inline.} = result = skipTypes(s.typ, abstractInst).kind in {tyVar, tyOpenArray, tyVarargs} or s.kind == skResult -proc interestingVar(s: PSym): bool {.inline.} = - result = s.kind in {skVar, skLet, skTemp, skForVar, skParam, skResult} and - sfGlobal notin s.flags +proc isInnerProc(s: PSym): bool = + if s.kind in {skProc, skMethod, skConverter, skIterator} and s.magic == mNone: + result = s.skipGenericOwner.kind in routineKinds -proc nestedAccess(top: PEnv; local: PSym): PNode = - # Parts after the transformation are in []: - # - # proc main = - # var [:env.]foo = 23 - # proc outer(:paramO) = - # [var :envO; createClosure(:envO); :envO.up = paramO] - # proc inner(:paramI) = - # echo [:paramI.up.]foo - # inner([:envO]) - # outer([:env]) - if not interestingVar(local) or top.fn == local.owner: - return nil - # check it's in fact a captured variable: - var it = top - while it != nil: - if it.vars.contains(local.id): break - it = it.up - if it == nil: return nil - let envParam = top.fn.getEnvParam - internalAssert(not envParam.isNil) - var access = newSymNode(envParam) - it = top.up - while it != nil: - if it.vars.contains(local.id): - access = indirectAccess(access, local, local.info) - return access - internalAssert it.upField != nil - access = indirectAccess(access, it.upField, local.info) - it = it.up - when false: - # Type based expression construction works too, but turned out to hide - # other bugs: - while true: - let obj = access.typ.sons[0] - let field = getFieldFromObj(obj, local) - if field != nil: - return rawIndirectAccess(access, field, local.info) - let upField = lookupInRecord(obj.n, getIdent(upName)) - if upField == nil: break - access = rawIndirectAccess(access, upField, local.info) - return nil - -proc createUpField(obj, fieldType: PType): PSym = - let pos = obj.n.len - result = newSym(skField, getIdent(upName), obj.owner, obj.owner.info) - result.typ = newType(tyRef, obj.owner) - result.position = pos - rawAddSon(result.typ, fieldType) - #rawAddField(obj, result) - addField(obj, result) - -proc captureVar(o: POuterContext; top: PEnv; local: PSym; - info: TLineInfo): bool = - # first check if we should be concerned at all: - var it = top - while it != nil: - if it.vars.contains(local.id): break - it = it.up - if it == nil: return false - # yes, so mark every 'up' pointer as taken: - if illegalCapture(local) or top.fn.typ.callConv notin {ccClosure, ccDefault}: - localError(info, errIllegalCaptureX, local.name.s) - it = top - while it != nil: - if it.vars.contains(local.id): break - # keep in mind that the first element of the chain belong to top.fn itself - # and these don't need any upFields - if it.upField == nil and it.up != nil and it.fn != top.fn: - it.upField = createUpField(it.obj, it.up.obj) - - if it.fn != local.owner: - it.fn.typ.callConv = ccClosure - incl(it.fn.typ.flags, tfCapturesEnv) - - var u = it.up - while u != nil and u.fn == it.fn: u = u.up - addClosureParam(it.fn, u) - - if idTableGet(o.lambdasToEnv, it.fn) == nil: - if u != nil: idTablePut(o.lambdasToEnv, it.fn, u) - - it = it.up - # don't do this: 'top' might not require a closure: - #if idTableGet(o.lambdasToEnv, it.fn) == nil: - # idTablePut(o.lambdasToEnv, it.fn, top) - - # mark as captured: - #if top.iter != nil: - # if not containsOrIncl(o.capturedVars, local.id): - # #addField(top.iter.obj, local) - # addCapturedVar(it, local) - #else: - incl(o.capturedVars, local.id) - addCapturedVar(it, local) - result = true - -proc semCaptureSym*(s, owner: PSym) = - if interestingVar(s) and owner.id != s.owner.id and s.kind != skResult: - if owner.typ != nil and not isGenericRoutine(owner): - # XXX: is this really safe? - # if we capture a var from another generic routine, - # it won't be consider captured. - owner.typ.callConv = ccClosure - #echo "semCaptureSym ", owner.name.s, owner.id, " ", s.name.s, s.id - # since the analysis is not entirely correct, we don't set 'tfCapturesEnv' - # here +proc newAsgnStmt(le, ri: PNode, info: TLineInfo): PNode = + # Bugfix: unfortunately we cannot use 'nkFastAsgn' here as that would + # mean to be able to capture string literals which have no GC header. + # However this can only happen if the capture happens through a parameter, + # which is however the only case when we generate an assignment in the first + # place. + result = newNodeI(nkAsgn, info, 2) + result.sons[0] = le + result.sons[1] = ri -proc gatherVars(o: POuterContext; e: PEnv; n: PNode): int = - # gather used vars for closure generation; returns number of captured vars - if n == nil: return 0 - case n.kind - of nkSym: - var s = n.sym - if interestingVar(s) and e.fn != s.owner: - if captureVar(o, e, s, n.info): result = 1 - of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit, nkClosure, nkProcDef, - nkMethodDef, nkConverterDef, nkMacroDef, nkTemplateDef, nkTypeSection: - discard - else: - for k in countup(0, sonsLen(n) - 1): - result += gatherVars(o, e, n.sons[k]) - -proc generateThunk(prc: PNode, dest: PType): PNode = - ## Converts 'prc' into '(thunk, nil)' so that it's compatible with - ## a closure. - - # we cannot generate a proper thunk here for GC-safety reasons (see internal - # documentation): - if gCmd == cmdCompileToJS: return prc - result = newNodeIT(nkClosure, prc.info, dest) - var conv = newNodeIT(nkHiddenStdConv, prc.info, dest) - conv.add(emptyNode) - conv.add(prc) - result.add(conv) - result.add(newNodeIT(nkNilLit, prc.info, getSysType(tyNil))) - -proc transformOuterConv(n: PNode): PNode = - # numeric types need range checks: - var dest = skipTypes(n.typ, abstractVarRange) - var source = skipTypes(n.sons[1].typ, abstractVarRange) - if dest.kind == tyProc: - if dest.callConv == ccClosure and source.callConv == ccDefault: - result = generateThunk(n.sons[1], dest) - -proc makeClosure(prc: PSym; env: PNode; info: TLineInfo): PNode = +proc makeClosure*(prc: PSym; env: PNode; info: TLineInfo): PNode = result = newNodeIT(nkClosure, info, prc.typ) result.add(newSymNode(prc)) if env == nil: result.add(newNodeIT(nkNilLit, info, getSysType(tyNil))) else: + if env.skipConv.kind == nkClosure: + localError(info, "internal error: taking closure of closure") result.add(env) -proc newClosureCreationVar(e: PEnv): PNode = - var v = newSym(skVar, getIdent(envName), e.fn, e.attachedNode.info) - incl(v.flags, sfShadowed) - v.typ = newType(tyRef, e.fn) - v.typ.rawAddSon(e.obj) - if e.fn.kind == skClosureIterator: - let it = initIter(e.fn) - addUniqueField(it.obj, v) - result = indirectAccess(newSymNode(it.closureParam), v, v.info) +proc interestingIterVar(s: PSym): bool {.inline.} = + # XXX optimization: Only lift the variable if it lives across + # yield/return boundaries! This can potentially speed up + # closure iterators quite a bit. + result = s.kind in {skVar, skLet, skTemp, skForVar} and sfGlobal notin s.flags + +template isIterator*(owner: PSym): bool = + owner.kind == skIterator and owner.typ.callConv == ccClosure + +proc liftIterSym*(n: PNode; owner: PSym): PNode = + # transforms (iter) to (let env = newClosure[iter](); (iter, env)) + let iter = n.sym + assert iter.isIterator + + result = newNodeIT(nkStmtListExpr, n.info, n.typ) + + let hp = getHiddenParam(iter) + let env = newSym(skLet, iter.name, owner, n.info) + env.typ = hp.typ + env.flags = hp.flags + var v = newNodeI(nkVarSection, n.info) + addVar(v, newSymNode(env)) + result.add(v) + # add 'new' statement: + let envAsNode = env.newSymNode + result.add newCall(getSysSym"internalNew", envAsNode) + result.add makeClosure(iter, envAsNode, n.info) + +proc freshVarForClosureIter*(s, owner: PSym): PNode = + let envParam = getHiddenParam(owner) + let obj = envParam.typ.lastSon + addField(obj, s) + + var access = newSymNode(envParam) + assert obj.kind == tyObject + let field = getFieldFromObj(obj, s) + if field != nil: + result = rawIndirectAccess(access, field, s.info) else: - result = newSymNode(v) + localError(s.info, "internal error: cannot generate fresh variable") + result = access + +# ------------------ new stuff ------------------------------------------- + +proc markAsClosure(owner: PSym; n: PNode) = + let s = n.sym + if illegalCapture(s) or owner.typ.callConv notin {ccClosure, ccDefault}: + localError(n.info, errIllegalCaptureX, s.name.s) + incl(owner.typ.flags, tfCapturesEnv) + owner.typ.callConv = ccClosure -proc getClosureVar(e: PEnv): PNode = - if e.createdVar == nil: - result = newClosureCreationVar(e) - e.createdVar = result +type + DetectionPass = object + processed, capturedVars: IntSet + ownerToType: Table[int, PType] + somethingToDo: bool + +proc initDetectionPass(fn: PSym): DetectionPass = + result.processed = initIntSet() + result.capturedVars = initIntSet() + result.ownerToType = initTable[int, PType]() + result.processed.incl(fn.id) + +discard """ +proc outer = + var a, b: int + proc innerA = use(a) + proc innerB = use(b); innerA() +# --> innerA and innerB need to *share* the closure type! +This is why need to store the 'ownerToType' table and use it +during .closure'fication. +""" + +proc getEnvTypeForOwner(c: var DetectionPass; owner: PSym): PType = + result = c.ownerToType.getOrDefault(owner.id) + if result.isNil: + result = newType(tyRef, owner) + let obj = createEnvObj(owner) + rawAddSon(result, obj) + c.ownerToType[owner.id] = result + +proc createUpField(c: var DetectionPass; dest, dep: PSym) = + let refObj = c.getEnvTypeForOwner(dest) # getHiddenParam(dest).typ + let obj = refObj.lastSon + let fieldType = c.getEnvTypeForOwner(dep) #getHiddenParam(dep).typ + if refObj == fieldType: + localError(dep.info, "internal error: invalid up reference computed") + + let upIdent = getIdent(upName) + let upField = lookupInRecord(obj.n, upIdent) + if upField != nil: + if upField.typ != fieldType: + localError(dep.info, "internal error: up references do not agree") else: - result = e.createdVar - -proc findEnv(o: POuterContext; s: PSym): PEnv = - var env = o.head - while env != nil: - if env.fn == s: break - env = env.next - internalAssert env != nil and env.up != nil - result = env.up - while result.fn == s: result = result.up - -proc transformInnerProc(o: POuterContext; e: PEnv, n: PNode): PNode = + let result = newSym(skField, upIdent, obj.owner, obj.owner.info) + result.typ = fieldType + rawAddField(obj, result) + +discard """ +There are a couple of possibilities of how to implement closure +iterators that capture outer variables in a traditional sense +(aka closure closure iterators). + +1. Transform iter() to iter(state, capturedEnv). So use 2 hidden + parameters. +2. Add the captured vars directly to 'state'. +3. Make capturedEnv an up-reference of 'state'. + +We do (3) here because (2) is obviously wrong and (1) is wrong too. +Consider: + + proc outer = + var xx = 9 + + iterator foo() = + var someState = 3 + + proc bar = echo someState + proc baz = someState = 0 + baz() + bar() + +""" + +proc addClosureParam(c: var DetectionPass; fn: PSym) = + var cp = getEnvParam(fn) + let owner = if fn.kind == skIterator: fn else: fn.skipGenericOwner + let t = c.getEnvTypeForOwner(owner) + if cp == nil: + cp = newSym(skParam, getIdent(paramName), fn, fn.info) + incl(cp.flags, sfFromGeneric) + cp.typ = t + addHiddenParam(fn, cp) + elif cp.typ != t and fn.kind != skIterator: + localError(fn.info, "internal error: inconsistent environment type") + #echo "adding closure to ", fn.name.s + +proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) = case n.kind - of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit: discard of nkSym: let s = n.sym - if s == e.fn: - # recursive calls go through (lambda, hiddenParam): - result = makeClosure(s, getEnvParam(s).newSymNode, n.info) - elif isInnerProc(s, o.fn) and s.typ.callConv == ccClosure: - # ugh: call to some other inner proc; - result = makeClosure(s, findEnv(o, s).getClosureVar, n.info) - else: - # captured symbol? - result = nestedAccess(e, n.sym) - #result = idNodeTableGet(i.localsToAccess, n.sym) - #of nkLambdaKinds, nkIteratorDef: - # if n.typ != nil: - # result = transformInnerProc(o, e, n.sons[namePos]) - #of nkClosure: - # let x = transformInnerProc(o, e, n.sons[0]) - # if x != nil: n.sons[0] = x - of nkProcDef, nkMethodDef, nkConverterDef, nkMacroDef, nkTemplateDef, - nkLambdaKinds, nkIteratorDef, nkClosure: - # don't recurse here: + if s.kind in {skProc, skMethod, skConverter, skIterator} and s.typ != nil and s.typ.callConv == ccClosure: + # this handles the case that the inner proc was declared as + # .closure but does not actually capture anything: + addClosureParam(c, s) + c.somethingToDo = true + + let innerProc = isInnerProc(s) + if innerProc: + if s.isIterator: c.somethingToDo = true + if not c.processed.containsOrIncl(s.id): + detectCapturedVars(s.getBody, s, c) + let ow = s.skipGenericOwner + if ow == owner: + if owner.isIterator: + c.somethingToDo = true + addClosureParam(c, owner) + if interestingIterVar(s): + if not c.capturedVars.containsOrIncl(s.id): + let obj = getHiddenParam(owner).typ.lastSon + #let obj = c.getEnvTypeForOwner(s.owner).lastSon + addField(obj, s) + # but always return because the rest of the proc is only relevant when + # ow != owner: + return + # direct or indirect dependency: + if (innerProc and s.typ.callConv == ccClosure) or interestingVar(s): + discard """ + proc outer() = + var x: int + proc inner() = + proc innerInner() = + echo x + innerInner() + inner() + # inner() takes a closure too! + """ + # mark 'owner' as taking a closure: + c.somethingToDo = true + markAsClosure(owner, n) + addClosureParam(c, owner) + #echo "capturing ", n.info + # variable 's' is actually captured: + if interestingVar(s) and not c.capturedVars.containsOrIncl(s.id): + let obj = c.getEnvTypeForOwner(ow).lastSon + #getHiddenParam(owner).typ.lastSon + addField(obj, s) + # create required upFields: + var w = owner.skipGenericOwner + if isInnerProc(w) or owner.isIterator: + if owner.isIterator: w = owner + let last = if ow.isIterator: ow.skipGenericOwner else: ow + while w != nil and w.kind != skModule and last != w: + discard """ + proc outer = + var a, b: int + proc outerB = + proc innerA = use(a) + proc innerB = use(b); innerA() + # --> make outerB of calling convention .closure and + # give it the same env type that outer's env var gets: + """ + let up = w.skipGenericOwner + #echo "up for ", w.name.s, " up ", up.name.s + markAsClosure(w, n) + addClosureParam(c, w) # , ow + createUpField(c, w, up) + w = up + of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit, + nkTemplateDef, nkTypeSection: discard - else: - for j in countup(0, sonsLen(n) - 1): - let x = transformInnerProc(o, e, n.sons[j]) - if x != nil: n.sons[j] = x - -proc closureCreationPoint(n: PNode): PNode = - if n.kind == nkStmtList and n.len >= 1 and n[0].kind == nkEmpty: - # we already have a free slot - result = n - else: - result = newNodeI(nkStmtList, n.info) - result.add(emptyNode) - result.add(n) - #result.flags.incl nfLL - -proc addParamsToEnv(fn: PSym; env: PEnv) = - let params = fn.typ.n - for i in 1.. <params.len: - if params.sons[i].kind != nkSym: - internalError(params.info, "liftLambdas: strange params") - let param = params.sons[i].sym - env.vars.incl(param.id) - # put the 'result' into the environment so it can be captured: - let ast = fn.ast - if resultPos < sonsLen(ast) and ast.sons[resultPos].kind == nkSym: - env.vars.incl(ast.sons[resultPos].sym.id) - -proc searchForInnerProcs(o: POuterContext, n: PNode, env: PEnv) = - if n == nil: return - case n.kind - of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit: + of nkProcDef, nkMethodDef, nkConverterDef, nkMacroDef: discard - of nkSym: - let fn = n.sym - if isInnerProc(fn, o.fn) and not containsOrIncl(o.processed, fn.id): - let body = fn.getBody - - # handle deeply nested captures: - let ex = closureCreationPoint(body) - let envB = newEnv(o, env, ex, fn) - addParamsToEnv(fn, envB) - searchForInnerProcs(o, body, envB) - fn.ast.sons[bodyPos] = ex - - let capturedCounter = gatherVars(o, envB, body) - # dummy closure param needed? - if capturedCounter == 0 and fn.typ.callConv == ccClosure: - #assert tfCapturesEnv notin n.sym.typ.flags - if idTableGet(o.lambdasToEnv, fn) == nil: - idTablePut(o.lambdasToEnv, fn, env) - addClosureParam(fn, env) - - elif fn.getEnvParam != nil: - # only transform if it really needs a closure: - let ti = transformInnerProc(o, envB, body) - if ti != nil: fn.ast.sons[bodyPos] = ti of nkLambdaKinds, nkIteratorDef: if n.typ != nil: - searchForInnerProcs(o, n.sons[namePos], env) - of nkWhileStmt, nkForStmt, nkParForStmt, nkBlockStmt: - # some nodes open a new scope, so they are candidates for the insertion - # of closure creation; however for simplicity we merge closures between - # branches, in fact, only loop bodies are of interest here as only they - # yield observable changes in semantics. For Zahary we also - # include ``nkBlock``. We don't do this for closure iterators because - # 'yield' can produce wrong code otherwise (XXX show example): - if env.fn.kind != skClosureIterator: - var body = n.len-1 - for i in countup(0, body - 1): searchForInnerProcs(o, n.sons[i], env) - # special handling for the loop body: - let ex = closureCreationPoint(n.sons[body]) - searchForInnerProcs(o, n.sons[body], newEnv(o, env, ex, env.fn)) - n.sons[body] = ex - else: - for i in countup(0, sonsLen(n) - 1): - searchForInnerProcs(o, n.sons[i], env) - of nkVarSection, nkLetSection: - # we need to compute a mapping var->declaredBlock. Note: The definition - # counts, not the block where it is captured! - for i in countup(0, sonsLen(n) - 1): - var it = n.sons[i] - if it.kind == nkCommentStmt: discard - elif it.kind == nkIdentDefs: - var L = sonsLen(it) - if it.sons[0].kind == nkSym: - # this can be false for recursive invocations that already - # transformed it into 'env.varName': - env.vars.incl(it.sons[0].sym.id) - searchForInnerProcs(o, it.sons[L-1], env) - elif it.kind == nkVarTuple: - var L = sonsLen(it) - for j in countup(0, L-3): - #echo "set: ", it.sons[j].sym.name.s, " ", o.currentBlock == nil - if it.sons[j].kind == nkSym: - env.vars.incl(it.sons[j].sym.id) - searchForInnerProcs(o, it.sons[L-1], env) - else: - internalError(it.info, "searchForInnerProcs") - of nkClosure: - searchForInnerProcs(o, n.sons[0], env) - of nkProcDef, nkMethodDef, nkConverterDef, nkMacroDef, nkTemplateDef, - nkTypeSection: - # don't recurse here: - discard + detectCapturedVars(n[namePos], owner, c) else: - for i in countup(0, sonsLen(n) - 1): - searchForInnerProcs(o, n.sons[i], env) + for i in 0..<n.len: + detectCapturedVars(n[i], owner, c) -proc newAsgnStmt(le, ri: PNode, info: TLineInfo): PNode = - # Bugfix: unfortunately we cannot use 'nkFastAsgn' here as that would - # mean to be able to capture string literals which have no GC header. - # However this can only happen if the capture happens through a parameter, - # which is however the only case when we generate an assignment in the first - # place. - result = newNodeI(nkAsgn, info, 2) - result.sons[0] = le - result.sons[1] = ri +type + LiftingPass = object + processed: IntSet + envVars: Table[int, PNode] -proc rawClosureCreation(o: POuterContext, scope: PEnv; env: PNode): PNode = - result = newNodeI(nkStmtList, env.info) - if env.kind == nkSym: - var v = newNodeI(nkVarSection, env.info) - addVar(v, env) - result.add(v) - # add 'new' statement: - result.add(newCall(getSysSym"internalNew", env)) - - # add assignment statements: - for local in scope.capturedVars: - let fieldAccess = indirectAccess(env, local, env.info) - if local.kind == skParam: - # maybe later: (sfByCopy in local.flags) - # add ``env.param = param`` - result.add(newAsgnStmt(fieldAccess, newSymNode(local), env.info)) - # it can happen that we already captured 'local' in some other environment - # then we capture by copy for now. This is not entirely correct but better - # than nothing: - let existing = idNodeTableGet(o.localsToAccess, local) - if existing.isNil: - idNodeTablePut(o.localsToAccess, local, fieldAccess) +proc initLiftingPass(fn: PSym): LiftingPass = + result.processed = initIntSet() + result.processed.incl(fn.id) + result.envVars = initTable[int, PNode]() + +proc accessViaEnvParam(n: PNode; owner: PSym): PNode = + let s = n.sym + # Type based expression construction for simplicity: + let envParam = getHiddenParam(owner) + if not envParam.isNil: + var access = newSymNode(envParam) + while true: + let obj = access.typ.sons[0] + assert obj.kind == tyObject + let field = getFieldFromObj(obj, s) + if field != nil: + return rawIndirectAccess(access, field, n.info) + let upField = lookupInRecord(obj.n, getIdent(upName)) + if upField == nil: break + access = rawIndirectAccess(access, upField, n.info) + localError(n.info, "internal error: environment misses: " & s.name.s) + result = n + +proc newEnvVar(owner: PSym; typ: PType): PNode = + var v = newSym(skVar, getIdent(envName), owner, owner.info) + incl(v.flags, sfShadowed) + v.typ = typ + result = newSymNode(v) + when false: + if owner.kind == skIterator and owner.typ.callConv == ccClosure: + let it = getHiddenParam(owner) + addUniqueField(it.typ.sons[0], v) + result = indirectAccess(newSymNode(it), v, v.info) + else: + result = newSymNode(v) + +proc setupEnvVar(owner: PSym; d: DetectionPass; + c: var LiftingPass): PNode = + if owner.isIterator: + return getHiddenParam(owner).newSymNode + result = c.envvars.getOrDefault(owner.id) + if result.isNil: + let envVarType = d.ownerToType.getOrDefault(owner.id) + if envVarType.isNil: + localError owner.info, "internal error: could not determine closure type" + result = newEnvVar(owner, envVarType) + c.envVars[owner.id] = result + +proc getUpViaParam(owner: PSym): PNode = + let p = getHiddenParam(owner) + result = p.newSymNode + if owner.isIterator: + let upField = lookupInRecord(p.typ.lastSon.n, getIdent(upName)) + if upField == nil: + localError(owner.info, "could not find up reference for closure iter") else: - result.add(newAsgnStmt(fieldAccess, existing, env.info)) - if scope.upField != nil: - # "up" chain has been used: - if scope.up.fn != scope.fn: - # crosses function boundary: - result.add(newAsgnStmt(indirectAccess(env, scope.upField, env.info), - newSymNode(getEnvParam(scope.fn)), env.info)) + result = rawIndirectAccess(result, upField, p.info) + +proc rawClosureCreation(owner: PSym; + d: DetectionPass; c: var LiftingPass): PNode = + result = newNodeI(nkStmtList, owner.info) + + var env: PNode + if owner.isIterator: + env = getHiddenParam(owner).newSymNode + else: + env = setupEnvVar(owner, d, c) + if env.kind == nkSym: + var v = newNodeI(nkVarSection, env.info) + addVar(v, env) + result.add(v) + # add 'new' statement: + result.add(newCall(getSysSym"internalNew", env)) + # add assignment statements for captured parameters: + for i in 1..<owner.typ.n.len: + let local = owner.typ.n[i].sym + if local.id in d.capturedVars: + let fieldAccess = indirectAccess(env, local, env.info) + # add ``env.param = param`` + result.add(newAsgnStmt(fieldAccess, newSymNode(local), env.info)) + + let upField = lookupInRecord(env.typ.lastSon.n, getIdent(upName)) + if upField != nil: + let up = getUpViaParam(owner) + if up != nil and upField.typ == up.typ: + result.add(newAsgnStmt(rawIndirectAccess(env, upField, env.info), + up, env.info)) + #elif oldenv != nil and oldenv.typ == upField.typ: + # result.add(newAsgnStmt(rawIndirectAccess(env, upField, env.info), + # oldenv, env.info)) else: - result.add(newAsgnStmt(indirectAccess(env, scope.upField, env.info), - getClosureVar(scope.up), env.info)) - -proc generateClosureCreation(o: POuterContext, scope: PEnv): PNode = - var env = getClosureVar(scope) - result = rawClosureCreation(o, scope, env) - -proc generateIterClosureCreation(o: POuterContext; env: PEnv; - scope: PNode): PNode = - if env.createdVarComesFromIter or env.createdVar.isNil: - # we have to create a new closure: - result = newClosureCreationVar(env) - let cc = rawClosureCreation(o, env, result) - var insertPoint = scope.sons[0] - if insertPoint.kind == nkEmpty: scope.sons[0] = cc + localError(env.info, "internal error: cannot create up reference") + +proc closureCreationForIter(iter: PNode; + d: DetectionPass; c: var LiftingPass): PNode = + result = newNodeIT(nkStmtListExpr, iter.info, iter.sym.typ) + let owner = iter.sym.skipGenericOwner + var v = newSym(skVar, getIdent(envName), owner, iter.info) + incl(v.flags, sfShadowed) + v.typ = getHiddenParam(iter.sym).typ + var vnode: PNode + if owner.isIterator: + let it = getHiddenParam(owner) + addUniqueField(it.typ.sons[0], v) + vnode = indirectAccess(newSymNode(it), v, v.info) + else: + vnode = v.newSymNode + var vs = newNodeI(nkVarSection, iter.info) + addVar(vs, vnode) + result.add(vs) + result.add(newCall(getSysSym"internalNew", vnode)) + + let upField = lookupInRecord(v.typ.lastSon.n, getIdent(upName)) + if upField != nil: + let u = setupEnvVar(owner, d, c) + if u.typ == upField.typ: + result.add(newAsgnStmt(rawIndirectAccess(vnode, upField, iter.info), + u, iter.info)) else: - assert cc.kind == nkStmtList and insertPoint.kind == nkStmtList - for x in cc: insertPoint.add(x) - if env.createdVar == nil: env.createdVar = result + localError(iter.info, "internal error: cannot create up reference for iter") + result.add makeClosure(iter.sym, vnode, iter.info) + +proc accessViaEnvVar(n: PNode; owner: PSym; d: DetectionPass; + c: var LiftingPass): PNode = + let access = setupEnvVar(owner, d, c) + let obj = access.typ.sons[0] + let field = getFieldFromObj(obj, n.sym) + if field != nil: + result = rawIndirectAccess(access, field, n.info) else: - result = env.createdVar - env.createdVarComesFromIter = true + localError(n.info, "internal error: not part of closure object type") + result = n -proc interestingIterVar(s: PSym): bool {.inline.} = - result = s.kind in {skVar, skLet, skTemp, skForVar} and sfGlobal notin s.flags +proc getStateField(owner: PSym): PSym = + getHiddenParam(owner).typ.sons[0].n.sons[0].sym -proc transformOuterProc(o: POuterContext, n: PNode, it: TIter): PNode +proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass; + c: var LiftingPass): PNode -proc transformYield(c: POuterContext, n: PNode, it: TIter): PNode = - assert it.state != nil - assert it.state.typ != nil - assert it.state.typ.n != nil - inc it.state.typ.n.sons[1].intVal - let stateNo = it.state.typ.n.sons[1].intVal +proc transformYield(n: PNode; owner: PSym; d: DetectionPass; + c: var LiftingPass): PNode = + let state = getStateField(owner) + assert state != nil + assert state.typ != nil + assert state.typ.n != nil + inc state.typ.n.sons[1].intVal + let stateNo = state.typ.n.sons[1].intVal var stateAsgnStmt = newNodeI(nkAsgn, n.info) - stateAsgnStmt.add(rawIndirectAccess(newSymNode(it.closureParam), - it.state, n.info)) + stateAsgnStmt.add(rawIndirectAccess(newSymNode(getEnvParam(owner)), + state, n.info)) stateAsgnStmt.add(newIntTypeNode(nkIntLit, stateNo, getSysType(tyInt))) var retStmt = newNodeI(nkReturnStmt, n.info) if n.sons[0].kind != nkEmpty: var a = newNodeI(nkAsgn, n.sons[0].info) - var retVal = transformOuterProc(c, n.sons[0], it) - addSon(a, newSymNode(it.resultSym)) - addSon(a, if retVal.isNil: n.sons[0] else: retVal) + var retVal = liftCapturedVars(n.sons[0], owner, d, c) + addSon(a, newSymNode(getIterResult(owner))) + addSon(a, retVal) retStmt.add(a) else: retStmt.add(emptyNode) @@ -705,251 +613,159 @@ proc transformYield(c: POuterContext, n: PNode, it: TIter): PNode = result.add(retStmt) result.add(stateLabelStmt) -proc transformReturn(c: POuterContext, n: PNode, it: TIter): PNode = +proc transformReturn(n: PNode; owner: PSym; d: DetectionPass; + c: var LiftingPass): PNode = + let state = getStateField(owner) result = newNodeI(nkStmtList, n.info) var stateAsgnStmt = newNodeI(nkAsgn, n.info) - stateAsgnStmt.add(rawIndirectAccess(newSymNode(it.closureParam), it.state, - n.info)) + stateAsgnStmt.add(rawIndirectAccess(newSymNode(getEnvParam(owner)), + state, n.info)) stateAsgnStmt.add(newIntTypeNode(nkIntLit, -1, getSysType(tyInt))) result.add(stateAsgnStmt) result.add(n) -proc outerProcSons(o: POuterContext, n: PNode, it: TIter) = - for i in countup(0, sonsLen(n) - 1): - let x = transformOuterProc(o, n.sons[i], it) - if x != nil: n.sons[i] = x - -proc liftIterSym(n: PNode; owner: PSym): PNode = - # transforms (iter) to (let env = newClosure[iter](); (iter, env)) - let iter = n.sym - assert iter.kind == skClosureIterator - - result = newNodeIT(nkStmtListExpr, n.info, n.typ) - - let hp = getHiddenParam(iter) - let env = newSym(skLet, iter.name, owner, n.info) - env.typ = hp.typ - env.flags = hp.flags - var v = newNodeI(nkVarSection, n.info) - addVar(v, newSymNode(env)) - result.add(v) - # add 'new' statement: - let envAsNode = env.newSymNode - result.add newCall(getSysSym"internalNew", envAsNode) - result.add makeClosure(iter, envAsNode, n.info) - -when false: - proc transformRemainingLocals(n: PNode; it: TIter): PNode = - assert it.fn.kind == skClosureIterator - result = n - case n.kind - of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit: discard - of nkSym: - let local = n.sym - if interestingIterVar(local) and it.fn == local.owner: - addUniqueField(it.obj, local) - result = indirectAccess(newSymNode(it.closureParam), local, n.info) - else: - result = newNodeI(n.kind, n.info, n.len) - for i in 0.. <n.safeLen: - result.sons[i] = transformRemainingLocals(n.sons[i], it) - -template envActive(env): expr = - (env.capturedVars.len > 0 or env.upField != nil) - -# We have to split up environment creation in 2 steps: -# 1. Generate it and store it in env.replacementNode -# 2. Insert replacementNode into its forseen slot. -# This split is necessary so that assignments belonging to closure -# creation like 'env.param = param' are not transformed -# into 'env.param = env.param'. -proc createEnvironments(o: POuterContext) = - var env = o.head - while env != nil: - if envActive(env): - var scope = env.attachedNode - assert scope.kind == nkStmtList - if scope.sons[0].kind == nkEmpty: - # prepare for closure construction: - env.replacementNode = generateClosureCreation(o, env) - env = env.next - -proc finishEnvironments(o: POuterContext) = - var env = o.head - while env != nil: - if env.replacementNode != nil: - var scope = env.attachedNode - assert scope.kind == nkStmtList - if scope.sons[0].kind == nkEmpty: - # change the empty node to contain the closure construction: - scope.sons[0] = env.replacementNode - when false: - if env.fn.kind == skClosureIterator: - scope.sons[0] = transformRemainingLocals(env.replacementNode, - initIter(env.fn)) - else: - scope.sons[0] = env.replacementNode - env = env.next - -proc transformOuterProcBody(o: POuterContext, n: PNode; it: TIter): PNode = - if nfLL in n.flags: - result = nil - elif it.fn.kind == skClosureIterator: +proc wrapIterBody(n: PNode; owner: PSym): PNode = + if not owner.isIterator: return n + when false: # unfortunately control flow is still convoluted and we can end up # multiple times here for the very same iterator. We shield against this # with some rather primitive check for now: if n.kind == nkStmtList and n.len > 0: - if n.sons[0].kind == nkGotoState: return nil + if n.sons[0].kind == nkGotoState: return n if n.len > 1 and n[1].kind == nkStmtList and n[1].len > 0 and n[1][0].kind == nkGotoState: - return nil - result = newNodeI(nkStmtList, it.fn.info) - var gs = newNodeI(nkGotoState, it.fn.info) - assert it.closureParam != nil - assert it.state != nil - gs.add(rawIndirectAccess(newSymNode(it.closureParam), it.state, it.fn.info)) - result.add(gs) - var state0 = newNodeI(nkState, it.fn.info) - state0.add(newIntNode(nkIntLit, 0)) - result.add(state0) - - let newBody = transformOuterProc(o, n, it) - if newBody != nil: - result.add(newBody) - else: - result.add(n) - - var stateAsgnStmt = newNodeI(nkAsgn, it.fn.info) - stateAsgnStmt.add(rawIndirectAccess(newSymNode(it.closureParam), - it.state, it.fn.info)) - stateAsgnStmt.add(newIntTypeNode(nkIntLit, -1, getSysType(tyInt))) - result.add(stateAsgnStmt) - result.flags.incl nfLL - else: - result = transformOuterProc(o, n, it) - if result != nil: result.flags.incl nfLL + return n + let info = n.info + result = newNodeI(nkStmtList, info) + var gs = newNodeI(nkGotoState, info) + gs.add(rawIndirectAccess(newSymNode(owner.getHiddenParam), getStateField(owner), info)) + result.add(gs) + var state0 = newNodeI(nkState, info) + state0.add(newIntNode(nkIntLit, 0)) + result.add(state0) + + result.add(n) + + var stateAsgnStmt = newNodeI(nkAsgn, info) + stateAsgnStmt.add(rawIndirectAccess(newSymNode(owner.getHiddenParam), + getStateField(owner), info)) + stateAsgnStmt.add(newIntTypeNode(nkIntLit, -1, getSysType(tyInt))) + result.add(stateAsgnStmt) -proc transformOuterProc(o: POuterContext, n: PNode; it: TIter): PNode = - if n == nil or nfLL in n.flags: return nil +proc symToClosure(n: PNode; owner: PSym; d: DetectionPass; + c: var LiftingPass): PNode = + let s = n.sym + if s == owner: + # recursive calls go through (lambda, hiddenParam): + let available = getHiddenParam(owner) + result = makeClosure(s, available.newSymNode, n.info) + elif s.isIterator: + result = closureCreationForIter(n, d, c) + elif s.skipGenericOwner == owner: + # direct dependency, so use the outer's env variable: + result = makeClosure(s, setupEnvVar(owner, d, c), n.info) + else: + let available = getHiddenParam(owner) + let wanted = getHiddenParam(s).typ + # ugh: call through some other inner proc; + var access = newSymNode(available) + while true: + if access.typ == wanted: + return makeClosure(s, access, n.info) + let obj = access.typ.sons[0] + let upField = lookupInRecord(obj.n, getIdent(upName)) + if upField == nil: + localError(n.info, "internal error: no environment found") + return n + access = rawIndirectAccess(access, upField, n.info) + +proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass; + c: var LiftingPass): PNode = + result = n case n.kind - of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit: discard of nkSym: - var local = n.sym - - if isInnerProc(local, o.fn) and o.processed.contains(local.id): - o.processed.excl(local.id) - let body = local.getBody - let newBody = transformOuterProcBody(o, body, initIter(local)) - if newBody != nil: - local.ast.sons[bodyPos] = newBody - - if it.fn.kind == skClosureIterator and interestingIterVar(local) and - it.fn == local.owner: - # every local goes through the closure: - #if not containsOrIncl(o.capturedVars, local.id): - # addField(it.obj, local) - if contains(o.capturedVars, local.id): - # change 'local' to 'closure.local', unless it's a 'byCopy' variable: - # if sfByCopy notin local.flags: - result = idNodeTableGet(o.localsToAccess, local) - assert result != nil, "cannot find: " & local.name.s - return result - else: - addUniqueField(it.obj, local) - return indirectAccess(newSymNode(it.closureParam), local, n.info) - - if local.kind == skClosureIterator: - # bug #3354; allow for - #iterator iter(): int {.closure.}= - # s.add(iter) - # yield 1 - - #if local == o.fn or local == it.fn: - # message(n.info, errRecursiveDependencyX, local.name.s) - - # consider: [i1, i2, i1] Since we merged the iterator's closure - # with the captured owning variables, we need to generate the - # closure generation code again: - # XXX why doesn't this work? - var closure = PEnv(idTableGet(o.lambdasToEnv, local)) - if closure.isNil: - return liftIterSym(n, o.fn) - else: - let createdVar = generateIterClosureCreation(o, closure, - closure.attachedNode) - let lpt = getHiddenParam(local).typ - if lpt != createdVar.typ: - assert lpt.kind == tyRef and createdVar.typ.kind == tyRef - # fix bug 'tshallowcopy_closures' but report if this gets any weirder: - if createdVar.typ.sons[0].len == 1 and lpt.sons[0].len >= 1: - createdVar.typ = lpt - if createdVar.kind == nkSym: createdVar.sym.typ = lpt - closure.obj = lpt.sons[0] - else: - internalError(n.info, "environment computation failed") - return makeClosure(local, createdVar, n.info) - - var closure = PEnv(idTableGet(o.lambdasToEnv, local)) - if closure != nil: - # we need to replace the lambda with '(lambda, env)': - let a = closure.createdVar - if a != nil: - return makeClosure(local, a, n.info) + let s = n.sym + if isInnerProc(s): + if not c.processed.containsOrIncl(s.id): + #if s.name.s == "temp": + # echo renderTree(s.getBody, {renderIds}) + let body = wrapIterBody(liftCapturedVars(s.getBody, s, d, c), s) + if c.envvars.getOrDefault(s.id).isNil: + s.ast.sons[bodyPos] = body + else: + s.ast.sons[bodyPos] = newTree(nkStmtList, rawClosureCreation(s, d, c), body) + if s.typ.callConv == ccClosure: + result = symToClosure(n, owner, d, c) + elif s.id in d.capturedVars: + if s.owner != owner: + result = accessViaEnvParam(n, owner) + elif owner.isIterator and interestingIterVar(s): + result = accessViaEnvParam(n, owner) else: - # can happen for dummy closures: - var scope = closure.attachedNode - assert scope.kind == nkStmtList - if scope.sons[0].kind == nkEmpty: - # change the empty node to contain the closure construction: - scope.sons[0] = generateClosureCreation(o, closure) - let x = closure.createdVar - assert x != nil - return makeClosure(local, x, n.info) - - if not contains(o.capturedVars, local.id): return - # change 'local' to 'closure.local', unless it's a 'byCopy' variable: - # if sfByCopy notin local.flags: - result = idNodeTableGet(o.localsToAccess, local) - assert result != nil, "cannot find: " & local.name.s - # else it is captured by copy and this means that 'outer' should continue - # to access the local as a local. - of nkLambdaKinds, nkIteratorDef: - if n.typ != nil: - result = transformOuterProc(o, n.sons[namePos], it) - of nkProcDef, nkMethodDef, nkConverterDef, nkMacroDef, nkTemplateDef: - # don't recurse here: + result = accessViaEnvVar(n, owner, d, c) + of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit, + nkTemplateDef, nkTypeSection: + discard + of nkProcDef, nkMethodDef, nkConverterDef, nkMacroDef: discard of nkClosure: - if n.sons[0].kind == nkSym: - var local = n.sons[0].sym - if isInnerProc(local, o.fn) and o.processed.contains(local.id): - o.processed.excl(local.id) - let body = local.getBody - let newBody = transformOuterProcBody(o, body, initIter(local)) - if newBody != nil: - local.ast.sons[bodyPos] = newBody - when false: - if n.sons[1].kind == nkSym: - var local = n.sons[1].sym - if it.fn.kind == skClosureIterator and interestingIterVar(local) and - it.fn == local.owner: - # every local goes through the closure: - addUniqueField(it.obj, local) - n.sons[1] = indirectAccess(newSymNode(it.closureParam), local, n.info) - of nkHiddenStdConv, nkHiddenSubConv, nkConv: - let x = transformOuterProc(o, n.sons[1], it) - if x != nil: n.sons[1] = x - result = transformOuterConv(n) - of nkYieldStmt: - if it.fn.kind == skClosureIterator: result = transformYield(o, n, it) - else: outerProcSons(o, n, it) - of nkReturnStmt: - if it.fn.kind == skClosureIterator: result = transformReturn(o, n, it) - else: outerProcSons(o, n, it) + if n[1].kind == nkNilLit: + n.sons[0] = liftCapturedVars(n[0], owner, d, c) + let x = n.sons[0].skipConv + if x.kind == nkClosure: + #localError(n.info, "internal error: closure to closure created") + # now we know better, so patch it: + n.sons[0] = x.sons[0] + of nkLambdaKinds, nkIteratorDef: + if n.typ != nil and n[namePos].kind == nkSym: + let m = newSymNode(n[namePos].sym) + m.typ = n.typ + result = liftCapturedVars(m, owner, d, c) else: - outerProcSons(o, n, it) + if owner.isIterator: + if n.kind == nkYieldStmt: + return transformYield(n, owner, d, c) + elif n.kind == nkReturnStmt: + return transformReturn(n, owner, d, c) + elif nfLL in n.flags: + # special case 'when nimVm' due to bug #3636: + n.sons[1] = liftCapturedVars(n[1], owner, d, c) + return + for i in 0..<n.len: + n.sons[i] = liftCapturedVars(n[i], owner, d, c) + +# ------------------ old stuff ------------------------------------------- + +proc semCaptureSym*(s, owner: PSym) = + if interestingVar(s) and s.kind != skResult: + if owner.typ != nil and not isGenericRoutine(owner): + # XXX: is this really safe? + # if we capture a var from another generic routine, + # it won't be consider captured. + var o = owner.skipGenericOwner + while o.kind != skModule and o != nil: + if s.owner == o: + owner.typ.callConv = ccClosure + #echo "computing .closure for ", owner.name.s, " ", owner.info, " because of ", s.name.s + o = o.skipGenericOwner + # since the analysis is not entirely correct, we don't set 'tfCapturesEnv' + # here -proc liftLambdas*(fn: PSym, body: PNode): PNode = +proc liftIterToProc*(fn: PSym; body: PNode; ptrType: PType): PNode = + var d = initDetectionPass(fn) + var c = initLiftingPass(fn) + # pretend 'fn' is a closure iterator for the analysis: + let oldKind = fn.kind + let oldCC = fn.typ.callConv + fn.kind = skIterator + fn.typ.callConv = ccClosure + d.ownerToType[fn.id] = ptrType + detectCapturedVars(body, fn, d) + result = wrapIterBody(liftCapturedVars(body, fn, d, c), fn) + fn.kind = oldKind + fn.typ.callConv = oldCC + +proc liftLambdas*(fn: PSym, body: PNode; tooEarly: var bool): PNode = # XXX gCmd == cmdCompileToJS does not suffice! The compiletime stuff needs # the transformation even when compiling to JS ... @@ -960,40 +776,35 @@ proc liftLambdas*(fn: PSym, body: PNode): PNode = fn.skipGenericOwner.kind != skModule: # ignore forward declaration: result = body + tooEarly = true else: - #if fn.name.s == "sort": - # echo rendertree(fn.ast, {renderIds}) - var o = newOuterContext(fn) - let ex = closureCreationPoint(body) - let env = newEnv(o, nil, ex, fn) - addParamsToEnv(fn, env) - searchForInnerProcs(o, body, env) - createEnvironments(o) - if fn.kind == skClosureIterator: - result = transformOuterProcBody(o, body, initIter(fn)) + var d = initDetectionPass(fn) + detectCapturedVars(body, fn, d) + if not d.somethingToDo and fn.isIterator: + addClosureParam(d, fn) + d.somethingToDo = true + if d.somethingToDo: + var c = initLiftingPass(fn) + var newBody = liftCapturedVars(body, fn, d, c) + if c.envvars.getOrDefault(fn.id) != nil: + newBody = newTree(nkStmtList, rawClosureCreation(fn, d, c), newBody) + result = wrapIterBody(newBody, fn) else: - discard transformOuterProcBody(o, body, initIter(fn)) - result = ex - finishEnvironments(o) - #if fn.name.s == "parseLong": - # echo rendertree(result, {renderIds}) + result = body + #if fn.name.s == "get2": + # echo "had something to do ", d.somethingToDo + # echo renderTree(result, {renderIds}) proc liftLambdasForTopLevel*(module: PSym, body: PNode): PNode = if body.kind == nkEmpty or gCmd == cmdCompileToJS: result = body else: - var o = newOuterContext(module) - let ex = closureCreationPoint(body) - let env = newEnv(o, nil, ex, module) - searchForInnerProcs(o, body, env) - createEnvironments(o) - discard transformOuterProc(o, body, initIter(module)) - finishEnvironments(o) - result = ex + # XXX implement it properly + result = body # ------------------- iterator transformation -------------------------------- -proc liftForLoop*(body: PNode): PNode = +proc liftForLoop*(body: PNode; owner: PSym): PNode = # problem ahead: the iterator could be invoked indirectly, but then # we don't know what environment to create here: # @@ -1031,17 +842,27 @@ proc liftForLoop*(body: PNode): PNode = # static binding? var env: PSym - if call[0].kind == nkSym and call[0].sym.kind == skClosureIterator: + let op = call[0] + if op.kind == nkSym and op.sym.isIterator: # createClosure() - let iter = call[0].sym - assert iter.kind == skClosureIterator - env = copySym(getHiddenParam(iter)) + let iter = op.sym + + let hp = getHiddenParam(iter) + env = newSym(skLet, iter.name, owner, body.info) + env.typ = hp.typ + env.flags = hp.flags var v = newNodeI(nkVarSection, body.info) addVar(v, newSymNode(env)) result.add(v) # add 'new' statement: result.add(newCall(getSysSym"internalNew", env.newSymNode)) + elif op.kind == nkStmtListExpr: + let closure = op.lastSon + if closure.kind == nkClosure: + call.sons[0] = closure + for i in 0 .. op.len-2: + result.add op[i] var loopBody = newNodeI(nkStmtList, body.info, 3) var whileLoop = newNodeI(nkWhileStmt, body.info, 2) @@ -1054,8 +875,8 @@ proc liftForLoop*(body: PNode): PNode = var v2 = newNodeI(nkLetSection, body.info) var vpart = newNodeI(if L == 3: nkIdentDefs else: nkVarTuple, body.info) for i in 0 .. L-3: - assert body[i].kind == nkSym - body[i].sym.kind = skLet + if body[i].kind == nkSym: + body[i].sym.kind = skLet addSon(vpart, body[i]) addSon(vpart, ast.emptyNode) # no explicit type diff --git a/compiler/lexer.nim b/compiler/lexer.nim index cea42ad1e..fc43f8d6a 100644 --- a/compiler/lexer.nim +++ b/compiler/lexer.nim @@ -662,6 +662,7 @@ proc getString(L: var TLexer, tok: var TToken, rawMode: bool) = L.lineNumber = line lexMessagePos(L, errClosingTripleQuoteExpected, L.lineStart) L.lineNumber = line2 + L.bufpos = pos break else: add(tok.literal, buf[pos]) @@ -768,24 +769,88 @@ proc getOperator(L: var TLexer, tok: var TToken) = if buf[pos] in {CR, LF, nimlexbase.EndOfFile}: tok.strongSpaceB = -1 +proc skipMultiLineComment(L: var TLexer; tok: var TToken; start: int; + isDoc: bool) = + var pos = start + var buf = L.buf + var toStrip = 0 + # detect the amount of indentation: + if isDoc: + toStrip = getColNumber(L, pos) + while buf[pos] == ' ': inc pos + if buf[pos] in {CR, LF}: + pos = handleCRLF(L, pos) + buf = L.buf + toStrip = 0 + while buf[pos] == ' ': + inc pos + inc toStrip + var nesting = 0 + while true: + case buf[pos] + of '#': + if isDoc: + if buf[pos+1] == '#' and buf[pos+2] == '[': + inc nesting + tok.literal.add '#' + elif buf[pos+1] == '[': + inc nesting + inc pos + of ']': + if isDoc: + if buf[pos+1] == '#' and buf[pos+2] == '#': + if nesting == 0: + inc(pos, 3) + break + dec nesting + tok.literal.add ']' + elif buf[pos+1] == '#': + if nesting == 0: + inc(pos, 2) + break + dec nesting + inc pos + of '\t': + lexMessagePos(L, errTabulatorsAreNotAllowed, pos) + inc(pos) + if isDoc: tok.literal.add '\t' + of CR, LF: + pos = handleCRLF(L, pos) + buf = L.buf + # strip leading whitespace: + if isDoc: + tok.literal.add "\n" + inc tok.iNumber + var c = toStrip + while buf[pos] == ' ' and c > 0: + inc pos + dec c + of nimlexbase.EndOfFile: + lexMessagePos(L, errGenerated, pos, "end of multiline comment expected") + break + else: + if isDoc: tok.literal.add buf[pos] + inc(pos) + L.bufpos = pos + proc scanComment(L: var TLexer, tok: var TToken) = var pos = L.bufpos var buf = L.buf + tok.tokType = tkComment + # iNumber contains the number of '\n' in the token + tok.iNumber = 0 when not defined(nimfix): assert buf[pos+1] == '#' if buf[pos+2] == '[': - if buf[pos+3] == ']': - # ##[] is the (rather complex) "cursor token" for idetools - tok.tokType = tkComment - tok.literal = "[]" - inc(L.bufpos, 4) - return - else: - lexMessagePos(L, warnDeprecated, pos, "use '## [' instead; '##['") + skipMultiLineComment(L, tok, pos+3, true) + return + inc(pos, 2) + + var toStrip = 0 + while buf[pos] == ' ': + inc pos + inc toStrip - tok.tokType = tkComment - # iNumber contains the number of '\n' in the token - tok.iNumber = 0 when defined(nimfix): var col = getColNumber(L, pos) while true: @@ -819,6 +884,12 @@ proc scanComment(L: var TLexer, tok: var TToken) = if doContinue(): tok.literal.add "\n" when defined(nimfix): col = indent + else: + inc(pos, 2) + var c = toStrip + while buf[pos] == ' ' and c > 0: + inc pos + dec c inc tok.iNumber else: if buf[pos] > ' ': @@ -842,9 +913,16 @@ proc skip(L: var TLexer, tok: var TToken) = pos = handleCRLF(L, pos) buf = L.buf var indent = 0 - while buf[pos] == ' ': - inc(pos) - inc(indent) + while true: + if buf[pos] == ' ': + inc(pos) + inc(indent) + elif buf[pos] == '#' and buf[pos+1] == '[': + skipMultiLineComment(L, tok, pos+2, false) + pos = L.bufpos + buf = L.buf + else: + break tok.strongSpaceA = 0 when defined(nimfix): template doBreak(): expr = buf[pos] > ' ' @@ -862,8 +940,11 @@ proc skip(L: var TLexer, tok: var TToken) = # do not skip documentation comment: if buf[pos+1] == '#': break if buf[pos+1] == '[': - lexMessagePos(L, warnDeprecated, pos, "use '# [' instead; '#['") - while buf[pos] notin {CR, LF, nimlexbase.EndOfFile}: inc(pos) + skipMultiLineComment(L, tok, pos+2, false) + pos = L.bufpos + buf = L.buf + else: + while buf[pos] notin {CR, LF, nimlexbase.EndOfFile}: inc(pos) else: break # EndOfFile also leaves the loop L.bufpos = pos diff --git a/compiler/lookups.nim b/compiler/lookups.nim index e88589c3e..a337bf0f3 100644 --- a/compiler/lookups.nim +++ b/compiler/lookups.nim @@ -133,7 +133,7 @@ type proc getSymRepr*(s: PSym): string = case s.kind - of skProc, skMethod, skConverter, skIterators: result = getProcHeader(s) + of skProc, skMethod, skConverter, skIterator: result = getProcHeader(s) else: result = s.name.s proc ensureNoMissingOrUnusedSymbols(scope: PScope) = diff --git a/compiler/lowerings.nim b/compiler/lowerings.nim index 20800b809..7a5c7f44b 100644 --- a/compiler/lowerings.nim +++ b/compiler/lowerings.nim @@ -165,9 +165,10 @@ proc indirectAccess*(a: PNode, b: string, info: TLineInfo): PNode = deref.typ = a.typ.skipTypes(abstractInst).sons[0] var t = deref.typ.skipTypes(abstractInst) var field: PSym + let bb = getIdent(b) while true: assert t.kind == tyObject - field = getSymFromList(t.n, getIdent(b)) + field = getSymFromList(t.n, bb) if field != nil: break t = t.sons[0] if t == nil: break @@ -585,7 +586,7 @@ proc wrapProcForSpawn*(owner: PSym; spawnExpr: PNode; retType: PType; objType.addField(field) result.add newFastAsgnStmt(newDotExpr(scratchObj, field), n[0]) fn = indirectAccess(castExpr, field, n.info) - elif fn.kind == nkSym and fn.sym.kind in {skClosureIterator, skIterator}: + elif fn.kind == nkSym and fn.sym.kind == skIterator: localError(n.info, "iterator in spawn environment is not allowed") elif fn.typ.callConv == ccClosure: localError(n.info, "closure in spawn environment is not allowed") diff --git a/compiler/nimsets.nim b/compiler/nimsets.nim index 055bae909..f15ad6368 100644 --- a/compiler/nimsets.nim +++ b/compiler/nimsets.nim @@ -106,13 +106,17 @@ proc toTreeSet(s: TBitSet, settype: PType, info: TLineInfo): PNode = inc(b) if (b >= len(s) * ElemSize) or not bitSetIn(s, b): break dec(b) + let aa = newIntTypeNode(nkIntLit, a + first, elemType) + aa.info = info if a == b: - addSon(result, newIntTypeNode(nkIntLit, a + first, elemType)) + addSon(result, aa) else: n = newNodeI(nkRange, info) n.typ = elemType - addSon(n, newIntTypeNode(nkIntLit, a + first, elemType)) - addSon(n, newIntTypeNode(nkIntLit, b + first, elemType)) + addSon(n, aa) + let bb = newIntTypeNode(nkIntLit, b + first, elemType) + bb.info = info + addSon(n, bb) addSon(result, n) e = b inc(e) diff --git a/compiler/parser.nim b/compiler/parser.nim index dbf9706ea..c4681a5cd 100644 --- a/compiler/parser.nim +++ b/compiler/parser.nim @@ -112,12 +112,7 @@ proc rawSkipComment(p: var TParser, node: PNode) = if p.tok.tokType == tkComment: if node != nil: if node.comment == nil: node.comment = "" - if p.tok.literal == "[]": - node.flags.incl nfIsCursor - #echo "parser: " - #debug node - else: - add(node.comment, p.tok.literal) + add(node.comment, p.tok.literal) else: parMessage(p, errInternal, "skipComment") getTok(p) @@ -250,12 +245,14 @@ proc isUnary(p: TParser): bool = if p.tok.tokType in {tkOpr, tkDotDot} and p.tok.strongSpaceB == 0 and p.tok.strongSpaceA > 0: - # XXX change this after 0.10.4 is out - if p.strongSpaces: result = true - else: - parMessage(p, warnDeprecated, - "will be parsed as unary operator; inconsistent spacing") + # versions prior to 0.13.0 used to do this: + when false: + if p.strongSpaces: + result = true + else: + parMessage(p, warnDeprecated, + "will be parsed as unary operator; inconsistent spacing") proc checkBinary(p: TParser) {.inline.} = ## Check if the current parser token is a binary operator. @@ -991,7 +988,7 @@ proc isExprStart(p: TParser): bool = of tkSymbol, tkAccent, tkOpr, tkNot, tkNil, tkCast, tkIf, tkProc, tkIterator, tkBind, tkAddr, tkParLe, tkBracketLe, tkCurlyLe, tkIntLit..tkCharLit, tkVar, tkRef, tkPtr, - tkTuple, tkObject, tkType, tkWhen, tkCase: + tkTuple, tkObject, tkType, tkWhen, tkCase, tkOut: result = true else: result = false @@ -1038,7 +1035,7 @@ proc parseObject(p: var TParser): PNode proc parseTypeClass(p: var TParser): PNode proc primary(p: var TParser, mode: TPrimaryMode): PNode = - #| typeKeyw = 'var' | 'ref' | 'ptr' | 'shared' | 'tuple' + #| typeKeyw = 'var' | 'out' | 'ref' | 'ptr' | 'shared' | 'tuple' #| | 'proc' | 'iterator' | 'distinct' | 'object' | 'enum' #| primary = typeKeyw typeDescK #| / prefixOperator* identOrLiteral primarySuffix* @@ -1112,6 +1109,7 @@ proc primary(p: var TParser, mode: TPrimaryMode): PNode = optInd(p, result) addSon(result, primary(p, pmNormal)) of tkVar: result = parseTypeDescKAux(p, nkVarTy, mode) + of tkOut: result = parseTypeDescKAux(p, nkVarTy, mode) of tkRef: result = parseTypeDescKAux(p, nkRefTy, mode) of tkPtr: result = parseTypeDescKAux(p, nkPtrTy, mode) of tkDistinct: result = parseTypeDescKAux(p, nkDistinctTy, mode) @@ -1763,7 +1761,7 @@ proc parseObject(p: var TParser): PNode = addSon(result, parseObjectPart(p)) proc parseTypeClassParam(p: var TParser): PNode = - if p.tok.tokType == tkVar: + if p.tok.tokType in {tkOut, tkVar}: result = newNodeP(nkVarTy, p) getTok(p) result.addSon(p.parseSymbol) @@ -1771,7 +1769,7 @@ proc parseTypeClassParam(p: var TParser): PNode = result = p.parseSymbol proc parseTypeClass(p: var TParser): PNode = - #| typeClassParam = ('var')? symbol + #| typeClassParam = ('var' | 'out')? symbol #| typeClass = typeClassParam ^* ',' (pragma)? ('of' typeDesc ^* ',')? #| &IND{>} stmt result = newNodeP(nkTypeClassTy, p) diff --git a/compiler/plugins/active.nim b/compiler/plugins/active.nim index e9c11c2ea..7b6411178 100644 --- a/compiler/plugins/active.nim +++ b/compiler/plugins/active.nim @@ -10,4 +10,4 @@ ## Include file that imports all plugins that are active. import - locals.locals + locals.locals, itersgen diff --git a/compiler/plugins/itersgen.nim b/compiler/plugins/itersgen.nim new file mode 100644 index 000000000..f44735b77 --- /dev/null +++ b/compiler/plugins/itersgen.nim @@ -0,0 +1,51 @@ +# +# +# The Nim Compiler +# (c) Copyright 2015 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Plugin to transform an inline iterator into a data structure. + +import compiler/pluginsupport, compiler/ast, compiler/astalgo, + compiler/magicsys, compiler/lookups, compiler/semdata, + compiler/lambdalifting, compiler/rodread, compiler/msgs + + +proc iterToProcImpl(c: PContext, n: PNode): PNode = + result = newNodeI(nkStmtList, n.info) + let iter = n[1] + if iter.kind != nkSym or iter.sym.kind != skIterator: + localError(iter.info, "first argument needs to be an iterator") + return + if n[2].typ.isNil: + localError(n[2].info, "second argument needs to be a type") + return + if n[3].kind != nkIdent: + localError(n[3].info, "third argument needs to be an identifier") + return + + let t = n[2].typ.skipTypes({tyTypeDesc, tyGenericInst}) + if t.kind notin {tyRef, tyPtr} or t.lastSon.kind != tyObject: + localError(n[2].info, + "type must be a non-generic ref|ptr to object with state field") + return + let body = liftIterToProc(iter.sym, iter.sym.getBody, t) + + let prc = newSym(skProc, n[3].ident, iter.sym.owner, iter.sym.info) + prc.typ = copyType(iter.sym.typ, prc, false) + excl prc.typ.flags, tfCapturesEnv + prc.typ.n.add newSymNode(getEnvParam(iter.sym)) + prc.typ.rawAddSon t + let orig = iter.sym.ast + prc.ast = newProcNode(nkProcDef, n.info, + name = newSymNode(prc), + params = orig[paramsPos], + pragmas = orig[pragmasPos], + body = body) + prc.ast.add iter.sym.ast.sons[resultPos] + addInterfaceDecl(c, prc) + +registerPlugin("stdlib", "system", "iterToProc", iterToProcImpl) diff --git a/compiler/plugins/locals/locals.nim b/compiler/plugins/locals/locals.nim index 59e3d677d..8a3f67dd4 100644 --- a/compiler/plugins/locals/locals.nim +++ b/compiler/plugins/locals/locals.nim @@ -9,8 +9,8 @@ ## The builtin 'system.locals' implemented as a plugin. -import compiler/plugins, compiler/ast, compiler/astalgo, compiler/magicsys, - compiler/lookups, compiler/semdata, compiler/lowerings +import compiler/pluginsupport, compiler/ast, compiler/astalgo, + compiler/magicsys, compiler/lookups, compiler/semdata, compiler/lowerings proc semLocals(c: PContext, n: PNode): PNode = var counter = 0 diff --git a/compiler/plugins.nim b/compiler/pluginsupport.nim index 1c9b7b77b..19a0bc84d 100644 --- a/compiler/plugins.nim +++ b/compiler/pluginsupport.nim @@ -7,7 +7,7 @@ # distribution, for details about the copyright. # -## Plugin support for the Nim compiler. Right now there are no plugins and they +## Plugin support for the Nim compiler. Right now they ## need to be build with the compiler, no DLL support. import ast, semdata, idents @@ -20,13 +20,16 @@ type next: Plugin proc pluginMatches(p: Plugin; s: PSym): bool = - if s.name.id != p.fn.id: return false - let module = s.owner + if s.name.id != p.fn.id: + return false + let module = s.skipGenericOwner if module == nil or module.kind != skModule or - module.name.id != p.module.id: return false + module.name.id != p.module.id: + return false let package = module.owner if package == nil or package.kind != skPackage or - package.name.id != p.package.id: return false + package.name.id != p.package.id: + return false return true var head: Plugin diff --git a/compiler/renderer.nim b/compiler/renderer.nim index 7cd8e25ee..8e4aa1831 100644 --- a/compiler/renderer.nim +++ b/compiler/renderer.nim @@ -167,33 +167,24 @@ proc makeNimString(s: string): string = proc putComment(g: var TSrcGen, s: string) = if s.isNil: return var i = 0 - var comIndent = 1 var isCode = (len(s) >= 2) and (s[1] != ' ') var ind = g.lineLen - var com = "" + var com = "## " while true: case s[i] of '\0': break of '\x0D': put(g, tkComment, com) - com = "" + com = "## " inc(i) if s[i] == '\x0A': inc(i) optNL(g, ind) of '\x0A': put(g, tkComment, com) - com = "" + com = "## " inc(i) optNL(g, ind) - of '#': - add(com, s[i]) - inc(i) - comIndent = 0 - while s[i] == ' ': - add(com, s[i]) - inc(i) - inc(comIndent) of ' ', '\x09': add(com, s[i]) inc(i) @@ -206,7 +197,7 @@ proc putComment(g: var TSrcGen, s: string) = if not isCode and (g.lineLen + (j - i) > MaxLineLen): put(g, tkComment, com) optNL(g, ind) - com = '#' & spaces(comIndent) + com = "## " while s[i] > ' ': add(com, s[i]) inc(i) @@ -283,7 +274,7 @@ proc shouldRenderComment(g: var TSrcGen, n: PNode): bool = result = false if n.comment != nil: result = (renderNoComments notin g.flags) or - (renderDocComments in g.flags) and startsWith(n.comment, "##") + (renderDocComments in g.flags) proc gcom(g: var TSrcGen, n: PNode) = assert(n != nil) @@ -1330,6 +1321,8 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) = initContext c putWithSpace g, tkSymbol, if n.kind == nkState: "state" else: "goto" gsons(g, n, c) + of nkBreakState: + put(g, tkTuple, "breakstate") of nkTypeClassTy: gTypeClassTy(g, n) else: diff --git a/compiler/sem.nim b/compiler/sem.nim index c6db6fbd3..cddd763ce 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -16,7 +16,7 @@ import procfind, lookups, rodread, pragmas, passes, semdata, semtypinst, sigmatch, intsets, transf, vmdef, vm, idgen, aliases, cgmeth, lambdalifting, evaltempl, patterns, parampatterns, sempass2, nimfix.pretty, semmacrosanity, - semparallel, lowerings, plugins, plugins.active + semparallel, lowerings, pluginsupport, plugins.active when defined(nimfix): import nimfix.prettybase @@ -186,6 +186,8 @@ proc newSymG*(kind: TSymKind, n: PNode, c: PContext): PSym = result.owner = getCurrOwner() else: result = newSym(kind, considerQuotedIdent(n), getCurrOwner(), n.info) + #if kind in {skForVar, skLet, skVar} and result.owner.kind == skModule: + # incl(result.flags, sfGlobal) proc semIdentVis(c: PContext, kind: TSymKind, n: PNode, allowed: TSymFlags): PSym @@ -202,7 +204,7 @@ proc typeAllowedCheck(info: TLineInfo; typ: PType; kind: TSymKind) = "' in this context: '" & typeToString(typ) & "'") proc paramsTypeCheck(c: PContext, typ: PType) {.inline.} = - typeAllowedCheck(typ.n.info, typ, skConst) + typeAllowedCheck(typ.n.info, typ, skProc) proc expectMacroOrTemplateCall(c: PContext, n: PNode): PSym proc semDirectOp(c: PContext, n: PNode, flags: TExprFlags): PNode @@ -485,4 +487,3 @@ proc myClose(context: PPassContext, n: PNode): PNode = popProcCon(c) const semPass* = makePass(myOpen, myOpenCached, myProcess, myClose) - diff --git a/compiler/semcall.nim b/compiler/semcall.nim index d8838e347..8445b24d9 100644 --- a/compiler/semcall.nim +++ b/compiler/semcall.nim @@ -75,7 +75,7 @@ proc pickBestCandidate(c: PContext, headSymbol: PNode, errors.add(err) if z.state == csMatch: # little hack so that iterators are preferred over everything else: - if sym.kind in skIterators: inc(z.exactMatches, 200) + if sym.kind == skIterator: inc(z.exactMatches, 200) case best.state of csEmpty, csNoMatch: best = z of csMatch: @@ -395,7 +395,7 @@ proc explicitGenericInstantiation(c: PContext, n: PNode, s: PSym): PNode = for i in countup(0, len(a)-1): var candidate = a.sons[i].sym if candidate.kind in {skProc, skMethod, skConverter, - skIterator, skClosureIterator}: + skIterator}: # it suffices that the candidate has the proper number of generic # type parameters: if safeLen(candidate.ast.sons[genericParamsPos]) == n.len-1: diff --git a/compiler/semdata.nim b/compiler/semdata.nim index 9b2f2e2ce..656bfc449 100644 --- a/compiler/semdata.nim +++ b/compiler/semdata.nim @@ -315,7 +315,7 @@ proc makeRangeType*(c: PContext; first, last: BiggestInt; addSonSkipIntLit(result, intType) # basetype of range proc markIndirect*(c: PContext, s: PSym) {.inline.} = - if s.kind in {skProc, skConverter, skMethod, skIterator, skClosureIterator}: + if s.kind in {skProc, skConverter, skMethod, skIterator}: incl(s.flags, sfAddrTaken) # XXX add to 'c' for global analysis diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 58c42d410..87d7764a2 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -385,7 +385,8 @@ proc isOpImpl(c: PContext, n: PNode): PNode = result = newIntNode(nkIntLit, ord(t.kind == tyProc and t.callConv == ccClosure and tfIterator notin t.flags)) - else: discard + else: + result = newIntNode(nkIntLit, 0) else: var t2 = n[2].typ.skipTypes({tyTypeDesc}) maybeLiftType(t2, c, n.info) @@ -752,11 +753,11 @@ proc semOverloadedCallAnalyseEffects(c: PContext, n: PNode, nOrig: PNode, flags: TExprFlags): PNode = if flags*{efInTypeof, efWantIterator} != {}: # consider: 'for x in pReturningArray()' --> we don't want the restriction - # to 'skIterators' anymore; skIterators are preferred in sigmatch already + # to 'skIterator' anymore; skIterator is preferred in sigmatch already # for typeof support. # for ``type(countup(1,3))``, see ``tests/ttoseq``. result = semOverloadedCall(c, n, nOrig, - {skProc, skMethod, skConverter, skMacro, skTemplate}+skIterators) + {skProc, skMethod, skConverter, skMacro, skTemplate, skIterator}) else: result = semOverloadedCall(c, n, nOrig, {skProc, skMethod, skConverter, skMacro, skTemplate}) @@ -769,7 +770,7 @@ proc semOverloadedCallAnalyseEffects(c: PContext, n: PNode, nOrig: PNode, case callee.kind of skMacro, skTemplate: discard else: - if callee.kind in skIterators and callee.id == c.p.owner.id: + if callee.kind == skIterator and callee.id == c.p.owner.id: localError(n.info, errRecursiveDependencyX, callee.name.s) # error correction, prevents endless for loop elimination in transf. # See bug #2051: @@ -1200,7 +1201,7 @@ proc semSubscript(c: PContext, n: PNode, flags: TExprFlags): PNode = let s = if n.sons[0].kind == nkSym: n.sons[0].sym elif n[0].kind in nkSymChoices: n.sons[0][0].sym else: nil - if s != nil and s.kind in {skProc, skMethod, skConverter}+skIterators: + if s != nil and s.kind in {skProc, skMethod, skConverter, skIterator}: # type parameters: partial generic specialization n.sons[0] = semSymGenericInstantiation(c, n.sons[0], s) result = explicitGenericInstantiation(c, n, s) @@ -1348,8 +1349,8 @@ proc semAsgn(c: PContext, n: PNode; mode=asgnNormal): PNode = proc semReturn(c: PContext, n: PNode): PNode = result = n checkSonsLen(n, 1) - if c.p.owner.kind in {skConverter, skMethod, skProc, skMacro, - skClosureIterator}: + if c.p.owner.kind in {skConverter, skMethod, skProc, skMacro} or ( + c.p.owner.kind == skIterator and c.p.owner.typ.callConv == ccClosure): if n.sons[0].kind != nkEmpty: # transform ``return expr`` to ``result = expr; return`` if c.p.resultSym != nil: @@ -1425,7 +1426,7 @@ proc semYieldVarResult(c: PContext, n: PNode, restype: PType) = proc semYield(c: PContext, n: PNode): PNode = result = n checkSonsLen(n, 1) - if c.p.owner == nil or c.p.owner.kind notin skIterators: + if c.p.owner == nil or c.p.owner.kind != skIterator: localError(n.info, errYieldNotAllowedHere) elif c.p.inTryStmt > 0 and c.p.owner.typ.callConv != ccInline: localError(n.info, errYieldNotAllowedInTryStmt) @@ -1434,20 +1435,15 @@ proc semYield(c: PContext, n: PNode): PNode = var iterType = c.p.owner.typ let restype = iterType.sons[0] if restype != nil: - let adjustedRes = if restype.kind == tyIter: restype.base - else: restype - if adjustedRes.kind != tyExpr: - n.sons[0] = fitNode(c, adjustedRes, n.sons[0]) + if restype.kind != tyExpr: + n.sons[0] = fitNode(c, restype, n.sons[0]) if n.sons[0].typ == nil: internalError(n.info, "semYield") - if resultTypeIsInferrable(adjustedRes): + if resultTypeIsInferrable(restype): let inferred = n.sons[0].typ - if restype.kind == tyIter: - restype.sons[0] = inferred - else: - iterType.sons[0] = inferred + iterType.sons[0] = inferred - semYieldVarResult(c, n, adjustedRes) + semYieldVarResult(c, n, restype) else: localError(n.info, errCannotReturnExpr) elif c.p.owner.typ.sons[0] != nil: @@ -1780,7 +1776,24 @@ proc semMagic(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode = result = setMs(n, s) result.sons[1] = semExpr(c, n.sons[1]) result.typ = n[1].typ - else: result = semDirectOp(c, n, flags) + of mPlugin: + # semDirectOp with conditional 'afterCallActions': + let nOrig = n.copyTree + #semLazyOpAux(c, n) + result = semOverloadedCallAnalyseEffects(c, n, nOrig, flags) + if result == nil: + result = errorNode(c, n) + else: + let callee = result.sons[0].sym + if callee.magic == mNone: + semFinishOperands(c, result) + activate(c, result) + fixAbstractType(c, result) + analyseIfAddressTakenInCall(c, result) + if callee.magic != mNone: + result = magicsAfterOverloadResolution(c, result, flags) + else: + result = semDirectOp(c, n, flags) proc semWhen(c: PContext, n: PNode, semCheck = true): PNode = # If semCheck is set to false, ``when`` will return the verbatim AST of @@ -1804,6 +1817,7 @@ proc semWhen(c: PContext, n: PNode, semCheck = true): PNode = whenNimvm = lookUp(c, exprNode).magic == mNimvm elif exprNode.kind == nkSym: whenNimvm = exprNode.sym.magic == mNimvm + if whenNimvm: n.flags.incl nfLL for i in countup(0, sonsLen(n) - 1): var it = n.sons[i] @@ -2111,7 +2125,7 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode = var s = lookUp(c, n) if c.inTypeClass == 0: semCaptureSym(s, c.p.owner) result = semSym(c, n, s, flags) - if s.kind in {skProc, skMethod, skConverter}+skIterators: + if s.kind in {skProc, skMethod, skConverter, skIterator}: #performProcvarCheck(c, n, s) result = symChoice(c, n, s, scClosed) if result.kind == nkSym: @@ -2167,7 +2181,7 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode = message(n.info, warnDeprecated, "bind") result = semExpr(c, n.sons[0], flags) of nkTypeOfExpr, nkTupleTy, nkTupleClassTy, nkRefTy..nkEnumTy, nkStaticTy: - var typ = semTypeNode(c, n, nil).skipTypes({tyTypeDesc, tyIter}) + var typ = semTypeNode(c, n, nil).skipTypes({tyTypeDesc}) result.typ = makeTypeDesc(c, typ) #result = symNodeFromType(c, typ, n.info) of nkCall, nkInfix, nkPrefix, nkPostfix, nkCommand, nkCallStrLit: @@ -2199,7 +2213,7 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode = localError(n.info, errUseQualifier, s.name.s) elif s.magic == mNone: result = semDirectOp(c, n, flags) else: result = semMagic(c, n, s, flags) - of skProc, skMethod, skConverter, skIterators: + of skProc, skMethod, skConverter, skIterator: if s.magic == mNone: result = semDirectOp(c, n, flags) else: result = semMagic(c, n, s, flags) else: @@ -2240,7 +2254,7 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode = var tupexp = semTuplePositionsConstr(c, n, flags) if isTupleType(tupexp): # reinterpret as type - var typ = semTypeNode(c, n, nil).skipTypes({tyTypeDesc, tyIter}) + var typ = semTypeNode(c, n, nil).skipTypes({tyTypeDesc}) result.typ = makeTypeDesc(c, typ) else: result = tupexp diff --git a/compiler/semgnrc.nim b/compiler/semgnrc.nim index 620453277..6651de78e 100644 --- a/compiler/semgnrc.nim +++ b/compiler/semgnrc.nim @@ -58,7 +58,7 @@ proc semGenericStmtSymbol(c: PContext, n: PNode, s: PSym, of skUnknown: # Introduced in this pass! Leave it as an identifier. result = n - of skProc, skMethod, skIterators, skConverter, skModule: + of skProc, skMethod, skIterator, skConverter, skModule: result = symChoice(c, n, s, scOpen) of skTemplate: if macroToExpand(s): @@ -226,7 +226,7 @@ proc semGenericStmt(c: PContext, n: PNode, of skUnknown, skParam: # Leave it as an identifier. discard - of skProc, skMethod, skIterators, skConverter, skModule: + of skProc, skMethod, skIterator, skConverter, skModule: result.sons[0] = symChoice(c, fn, s, scOption) # do not check of 's.magic==mRoof' here because it might be some # other '^' but after overload resolution the proper one: diff --git a/compiler/semmagic.nim b/compiler/semmagic.nim index deef38ae3..f98ff0266 100644 --- a/compiler/semmagic.nim +++ b/compiler/semmagic.nim @@ -207,7 +207,7 @@ proc magicsAfterOverloadResolution(c: PContext, n: PNode, result = n.sons[1] else: result = newNodeIT(nkCall, n.info, getSysType(tyInt)) - result.add newSymNode(createMagic("-", mSubI), n.info) + result.add newSymNode(getSysMagic("-", mSubI), n.info) result.add lenExprB result.add n.sons[1] of mPlugin: diff --git a/compiler/sempass2.nim b/compiler/sempass2.nim index 29cce9247..c3a9e01a0 100644 --- a/compiler/sempass2.nim +++ b/compiler/sempass2.nim @@ -504,7 +504,8 @@ proc notNilCheck(tracked: PEffects, n: PNode, paramType: PType) = if n.kind == nkAddr: # addr(x[]) can't be proven, but addr(x) can: if not containsNode(n, {nkDerefExpr, nkHiddenDeref}): return - elif (n.kind == nkSym and n.sym.kind in routineKinds) or n.kind in procDefs: + elif (n.kind == nkSym and n.sym.kind in routineKinds) or + n.kind in procDefs+{nkObjConstr}: # 'p' is not nil obviously: return case impliesNotNil(tracked.guards, n) @@ -704,7 +705,15 @@ proc track(tracked: PEffects, n: PNode) = for i in 1 .. <len(n): trackOperand(tracked, n.sons[i], paramType(op, i)) if a.kind == nkSym and a.sym.magic in {mNew, mNewFinalize, mNewSeq}: # may not look like an assignment, but it is: - initVarViaNew(tracked, n.sons[1]) + let arg = n.sons[1] + initVarViaNew(tracked, arg) + if {tfNeedsInit} * arg.typ.lastSon.flags != {}: + if a.sym.magic == mNewSeq and n[2].kind in {nkCharLit..nkUInt64Lit} and + n[2].intVal == 0: + # var s: seq[notnil]; newSeq(s, 0) is a special case! + discard + else: + message(arg.info, warnProveInit, $arg) for i in 0 .. <safeLen(n): track(tracked, n.sons[i]) of nkDotExpr: @@ -875,7 +884,8 @@ proc trackProc*(s: PSym, body: PNode) = var t: TEffects initEffects(effects, s, t) track(t, body) - if not isEmptyType(s.typ.sons[0]) and tfNeedsInit in s.typ.sons[0].flags and + if not isEmptyType(s.typ.sons[0]) and + {tfNeedsInit, tfNotNil} * s.typ.sons[0].flags != {} and s.kind in {skProc, skConverter, skMethod}: var res = s.ast.sons[resultPos].sym # get result symbol if res.id notin t.init: diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index adb1c81c1..fdf147a2e 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -84,7 +84,7 @@ proc performProcvarCheck(c: PContext, n: PNode, s: PSym) = proc semProcvarCheck(c: PContext, n: PNode) = let n = n.skipConv if n.kind == nkSym and n.sym.kind in {skProc, skMethod, skConverter, - skIterator, skClosureIterator}: + skIterator}: performProcvarCheck(c, n, n.sym) proc semProc(c: PContext, n: PNode): PNode @@ -326,11 +326,14 @@ proc semIdentDef(c: PContext, n: PNode, kind: TSymKind): PSym = incl(result.flags, sfGlobal) else: result = semIdentWithPragma(c, kind, n, {}) + if result.owner.kind == skModule: + incl(result.flags, sfGlobal) suggestSym(n.info, result) styleCheckDef(result) proc checkNilable(v: PSym) = - if sfGlobal in v.flags and {tfNotNil, tfNeedsInit} * v.typ.flags != {}: + if {sfGlobal, sfImportC} * v.flags == {sfGlobal} and + {tfNotNil, tfNeedsInit} * v.typ.flags != {}: if v.ast.isNil: message(v.info, warnProveInit, v.name.s) elif tfNotNil in v.typ.flags and tfNotNil notin v.ast.typ.flags: @@ -539,7 +542,7 @@ proc symForVar(c: PContext, n: PNode): PSym = proc semForVars(c: PContext, n: PNode): PNode = result = n var length = sonsLen(n) - let iterBase = n.sons[length-2].typ.skipTypes({tyIter}) + let iterBase = n.sons[length-2].typ var iter = skipTypes(iterBase, {tyGenericInst}) # length == 3 means that there is one for loop variable # and thus no tuple unpacking: @@ -593,12 +596,11 @@ proc semFor(c: PContext, n: PNode): PNode = result.kind = nkParForStmt else: result = semForFields(c, n, call.sons[0].sym.magic) - elif (isCallExpr and call.sons[0].typ.callConv == ccClosure) or - call.typ.kind == tyIter: + elif isCallExpr and call.sons[0].typ.callConv == ccClosure: # first class iterator: result = semForVars(c, n) elif not isCallExpr or call.sons[0].kind != nkSym or - call.sons[0].sym.kind notin skIterators: + call.sons[0].sym.kind != skIterator: if length == 3: n.sons[length-2] = implicitIterator(c, "items", n.sons[length-2]) elif length == 4: @@ -958,15 +960,17 @@ proc semInferredLambda(c: PContext, pt: TIdTable, n: PNode): PNode = var n = n let original = n.sons[namePos].sym - let s = copySym(original, false) - incl(s.flags, sfFromGeneric) + let s = original #copySym(original, false) + #incl(s.flags, sfFromGeneric) + #s.owner = original n = replaceTypesInBody(c, pt, n, original) result = n s.ast = result n.sons[namePos].sym = s n.sons[genericParamsPos] = emptyNode - let params = n.typ.n + # for LL we need to avoid wrong aliasing + let params = copyTree n.typ.n n.sons[paramsPos] = params s.typ = n.typ for i in 1..<params.len: @@ -974,6 +978,7 @@ proc semInferredLambda(c: PContext, pt: TIdTable, n: PNode): PNode = tyFromExpr, tyFieldAccessor}+tyTypeClasses: localError(params[i].info, "cannot infer type of parameter: " & params[i].sym.name.s) + #params[i].sym.owner = s openScope(c) pushOwner(s) addParams(c, params, skProc) @@ -1006,7 +1011,8 @@ proc activate(c: PContext, n: PNode) = discard proc maybeAddResult(c: PContext, s: PSym, n: PNode) = - if s.typ.sons[0] != nil and s.kind != skIterator: + if s.typ.sons[0] != nil and not + (s.kind == skIterator and s.typ.callConv != ccClosure): addResult(c, s.typ.sons[0], n.info, s.kind) addResultNode(c, n) @@ -1143,13 +1149,15 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind, if tfTriggersCompileTime in s.typ.flags: incl(s.flags, sfCompileTime) if n.sons[patternPos].kind != nkEmpty: n.sons[patternPos] = semPattern(c, n.sons[patternPos]) - if s.kind in skIterators: + if s.kind == skIterator: s.typ.flags.incl(tfIterator) var proto = searchForProc(c, oldScope, s) if proto == nil: - if s.kind == skClosureIterator: s.typ.callConv = ccClosure - else: s.typ.callConv = lastOptionEntry(c).defaultCC + if s.kind == skIterator and s.typ.callConv == ccClosure: + discard + else: + s.typ.callConv = lastOptionEntry(c).defaultCC # add it here, so that recursive procs are possible: if sfGenSym in s.flags: discard elif kind in OverloadableSyms: @@ -1209,7 +1217,7 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind, n.sons[bodyPos] = transformBody(c.module, semBody, s) popProcCon(c) else: - if s.typ.sons[0] != nil and kind notin skIterators: + if s.typ.sons[0] != nil and kind != skIterator: addDecl(c, newSym(skUnknown, getIdent"result", nil, n.info)) openScope(c) n.sons[bodyPos] = semGenericStmt(c, n.sons[bodyPos]) @@ -1230,9 +1238,9 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind, if n.sons[patternPos].kind != nkEmpty: c.patterns.add(s) if isAnon: result.typ = s.typ - if isTopLevel(c) and s.kind != skClosureIterator and + if isTopLevel(c) and s.kind != skIterator and s.typ.callConv == ccClosure: - message(s.info, warnDeprecated, "top level '.closure' calling convention") + localError(s.info, "'.closure' calling convention for top level routines is invalid") proc determineType(c: PContext, s: PSym) = if s.typ != nil: return @@ -1240,15 +1248,12 @@ proc determineType(c: PContext, s: PSym) = discard semProcAux(c, s.ast, s.kind, {}, stepDetermineType) proc semIterator(c: PContext, n: PNode): PNode = - let kind = if hasPragma(n[pragmasPos], wClosure) or - n[namePos].kind == nkEmpty: skClosureIterator - else: skIterator # gensym'ed iterator? if n[namePos].kind == nkSym: # gensym'ed iterators might need to become closure iterators: n[namePos].sym.owner = getCurrOwner() - n[namePos].sym.kind = kind - result = semProcAux(c, n, kind, iteratorPragmas) + n[namePos].sym.kind = skIterator + result = semProcAux(c, n, skIterator, iteratorPragmas) var s = result.sons[namePos].sym var t = s.typ if t.sons[0] == nil and s.typ.callConv != ccClosure: diff --git a/compiler/semtempl.nim b/compiler/semtempl.nim index 2dda8276d..a4498a3ae 100644 --- a/compiler/semtempl.nim +++ b/compiler/semtempl.nim @@ -228,10 +228,7 @@ proc semTemplSymbol(c: PContext, n: PNode, s: PSym): PNode = of skParam: result = n of skType: - if (s.typ != nil) and (s.typ.kind != tyGenericParam): - result = newSymNodeTypeDesc(s, n.info) - else: - result = n + result = newSymNodeTypeDesc(s, n.info) else: result = newSymNode(s, n.info) @@ -456,9 +453,7 @@ proc semTemplBody(c: var TemplCtx, n: PNode): PNode = of nkMethodDef: result = semRoutineInTemplBody(c, n, skMethod) of nkIteratorDef: - let kind = if hasPragma(n[pragmasPos], wClosure): skClosureIterator - else: skIterator - result = semRoutineInTemplBody(c, n, kind) + result = semRoutineInTemplBody(c, n, skIterator) of nkTemplateDef: result = semRoutineInTemplBody(c, n, skTemplate) of nkMacroDef: diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index 65cb9421b..ac425ba15 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -135,13 +135,19 @@ proc semAnyRef(c: PContext; n: PNode; kind: TTypeKind; prev: PType): PType = checkMinSonsLen(n, 1) var base = semTypeNode(c, n.lastSon, nil) result = newOrPrevType(kind, prev, c) + var isNilable = false # check every except the last is an object: for i in isCall .. n.len-2: - let region = semTypeNode(c, n[i], nil) - if region.skipTypes({tyGenericInst}).kind notin {tyError, tyObject}: - message n[i].info, errGenerated, "region needs to be an object type" - addSonSkipIntLit(result, region) + let ni = n[i] + if ni.kind == nkNilLit: + isNilable = true + else: + let region = semTypeNode(c, ni, nil) + if region.skipTypes({tyGenericInst}).kind notin {tyError, tyObject}: + message n[i].info, errGenerated, "region needs to be an object type" + addSonSkipIntLit(result, region) addSonSkipIntLit(result, base) + #if not isNilable: result.flags.incl tfNotNil proc semVarType(c: PContext, n: PNode, prev: PType): PType = if sonsLen(n) == 1: @@ -826,15 +832,6 @@ proc liftParamType(c: PContext, procKind: TSymKind, genericParams: PNode, result = newTypeWithSons(c, tyCompositeTypeClass, @[paramType, result]) result = addImplicitGeneric(result) - of tyIter: - if paramType.callConv == ccInline: - if procKind notin {skTemplate, skMacro, skIterator}: - localError(info, errInlineIteratorsAsProcParams) - if paramType.len == 1: - let lifted = liftingWalk(paramType.base) - if lifted != nil: paramType.sons[0] = lifted - result = addImplicitGeneric(paramType) - of tyGenericInst: if paramType.lastSon.kind == tyUserTypeClass: var cp = copyType(paramType, getCurrOwner(), false) @@ -865,11 +862,6 @@ proc liftParamType(c: PContext, procKind: TSymKind, genericParams: PNode, of tyUserTypeClass, tyBuiltInTypeClass, tyAnd, tyOr, tyNot: result = addImplicitGeneric(copyType(paramType, getCurrOwner(), true)) - of tyExpr: - if procKind notin {skMacro, skTemplate}: - result = addImplicitGeneric(newTypeS(tyAnything, c)) - #result = addImplicitGenericImpl(newTypeS(tyGenericParam, c), nil) - of tyGenericParam: markUsed(info, paramType.sym) styleCheckUse(info, paramType.sym) @@ -968,10 +960,6 @@ proc semProcTypeNode(c: PContext, n, genericParams: PNode, var r: PType if n.sons[0].kind != nkEmpty: r = semTypeNode(c, n.sons[0], nil) - elif kind == skIterator: - # XXX This is special magic we should likely get rid of - r = newTypeS(tyExpr, c) - message(n.info, warnDeprecated, "implicit return type for 'iterator'") if r != nil: # turn explicit 'void' return type into 'nil' because the rest of the @@ -996,7 +984,8 @@ proc semProcTypeNode(c: PContext, n, genericParams: PNode, # see tchainediterators # in cases like iterator foo(it: iterator): type(it) # we don't need to change the return type to iter[T] - if not r.isInlineIterator: r = newTypeWithSons(c, tyIter, @[r]) + result.flags.incl tfIterator + # XXX Would be nice if we could get rid of this result.sons[0] = r result.n.typ = r @@ -1151,7 +1140,7 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType = # for ``type(countup(1,3))``, see ``tests/ttoseq``. checkSonsLen(n, 1) let typExpr = semExprWithType(c, n.sons[0], {efInTypeof}) - result = typExpr.typ.skipTypes({tyIter}) + result = typExpr.typ of nkPar: if sonsLen(n) == 1: result = semTypeNode(c, n.sons[0], prev) else: @@ -1169,6 +1158,14 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType = result = semTypeNode(c, b, prev) elif ident != nil and ident.id == ord(wDotDot): result = semRangeAux(c, n, prev) + elif n[0].kind == nkNilLit and n.len == 2: + result = semTypeNode(c, n.sons[1], prev) + if result.skipTypes({tyGenericInst}).kind in NilableTypes+GenericTypes: + if tfNotNil in result.flags: + result = freshType(result, prev) + result.flags.excl(tfNotNil) + else: + localError(n.info, errGenerated, "invalid type") elif n[0].kind notin nkIdentKinds: result = semTypeExpr(c, n) else: @@ -1209,7 +1206,7 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType = elif op.id == ord(wType): checkSonsLen(n, 2) let typExpr = semExprWithType(c, n.sons[1], {efInTypeof}) - result = typExpr.typ.skipTypes({tyIter}) + result = typExpr.typ else: result = semTypeExpr(c, n) of nkWhenStmt: @@ -1290,14 +1287,16 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType = result.flags.incl tfHasStatic of nkIteratorTy: if n.sonsLen == 0: - result = newConstraint(c, tyIter) + result = newTypeS(tyBuiltInTypeClass, c) + let child = newTypeS(tyProc, c) + child.flags.incl tfIterator + result.addSonSkipIntLit(child) else: - result = semProcTypeWithScope(c, n, prev, skClosureIterator) + result = semProcTypeWithScope(c, n, prev, skIterator) + result.flags.incl(tfIterator) if n.lastSon.kind == nkPragma and hasPragma(n.lastSon, wInline): - result.kind = tyIter result.callConv = ccInline else: - result.flags.incl(tfIterator) result.callConv = ccClosure of nkProcTy: if n.sonsLen == 0: diff --git a/compiler/semtypinst.nim b/compiler/semtypinst.nim index 7957ac50a..20b60a88d 100644 --- a/compiler/semtypinst.nim +++ b/compiler/semtypinst.nim @@ -77,7 +77,7 @@ proc cacheTypeInst*(inst: PType) = # update the refcount let gt = inst.sons[0] let t = if gt.kind == tyGenericBody: gt.lastSon else: gt - if t.kind in {tyStatic, tyGenericParam, tyIter} + tyTypeClasses: + if t.kind in {tyStatic, tyGenericParam} + tyTypeClasses: return gt.sym.typeInstCache.safeAdd(inst) @@ -390,7 +390,7 @@ proc replaceTypeVarsTAux(cl: var TReplTypeVars, t: PType): PType = result = t if t == nil: return - if t.kind in {tyStatic, tyGenericParam, tyIter} + tyTypeClasses: + if t.kind in {tyStatic, tyGenericParam} + tyTypeClasses: let lookup = PType(idTableGet(cl.typeMap, t)) if lookup != nil: return lookup diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index 9fda8c860..96df0c5c6 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -167,12 +167,12 @@ proc sumGeneric(t: PType): int = t = t.lastSon if t.kind == tyEmpty: break inc result - of tyGenericInvocation, tyTuple: + of tyGenericInvocation, tyTuple, tyProc: result += ord(t.kind == tyGenericInvocation) for i in 0 .. <t.len: result += t.sons[i].sumGeneric break of tyGenericParam, tyExpr, tyStatic, tyStmt: break - of tyBool, tyChar, tyEnum, tyObject, tyProc, tyPointer, + of tyBool, tyChar, tyEnum, tyObject, tyPointer, tyString, tyCString, tyInt..tyInt64, tyFloat..tyFloat128, tyUInt..tyUInt64: return isvar @@ -1256,10 +1256,6 @@ proc localConvMatch(c: PContext, m: var TCandidate, f, a: PType, result.typ = getInstantiatedType(c, arg, m, base(f)) m.baseTypeMatch = true -proc isInlineIterator*(t: PType): bool = - result = t.kind == tyIter or - (t.kind == tyBuiltInTypeClass and t.base.kind == tyIter) - proc incMatches(m: var TCandidate; r: TTypeRelation; convMatch = 1) = case r of isConvertible, isIntConv: inc(m.convMatches, convMatch) @@ -1323,13 +1319,6 @@ proc paramTypesMatchAux(m: var TCandidate, f, argType: PType, else: return argSemantized # argOrig - if r != isNone and f.isInlineIterator: - var inlined = newTypeS(tyStatic, c) - inlined.sons = @[argType] - inlined.n = argSemantized - put(m.bindings, f, inlined) - return argSemantized - # If r == isBothMetaConvertible then we rerun typeRel. # bothMetaCounter is for safety to avoid any infinite loop, # I don't have any example when it is needed. @@ -1453,7 +1442,7 @@ proc paramTypesMatch*(m: var TCandidate, f, a: PType, z.calleeSym = m.calleeSym var best = -1 for i in countup(0, sonsLen(arg) - 1): - if arg.sons[i].sym.kind in {skProc, skMethod, skConverter}+skIterators: + if arg.sons[i].sym.kind in {skProc, skMethod, skConverter, skIterator}: copyCandidate(z, m) z.callee = arg.sons[i].typ z.calleeSym = arg.sons[i].sym @@ -1646,6 +1635,7 @@ proc matchesAux(c: PContext, n, nOrig: PNode, if arg != nil and m.baseTypeMatch and container != nil: addSon(container, arg) incrIndexType(container.typ) + checkConstraint(n.sons[a]) else: m.state = csNoMatch return @@ -1686,7 +1676,7 @@ proc matchesAux(c: PContext, n, nOrig: PNode, setSon(m.call, formal.position + 1, arg) inc(f) container = nil - checkConstraint(n.sons[a]) + checkConstraint(n.sons[a]) inc(a) proc semFinishOperands*(c: PContext, n: PNode) = diff --git a/compiler/transf.nim b/compiler/transf.nim index 3d78a6b92..b2bbdcec3 100644 --- a/compiler/transf.nim +++ b/compiler/transf.nim @@ -45,7 +45,7 @@ type inlining: int # > 0 if we are in inlining context (copy vars) nestedProcs: int # > 0 if we are in a nested proc contSyms, breakSyms: seq[PSym] # to transform 'continue' and 'break' - deferDetected: bool + deferDetected, tooEarly: bool PTransf = ref TTransfContext proc newTransNode(a: PNode): PTransNode {.inline.} = @@ -93,10 +93,15 @@ proc getCurrOwner(c: PTransf): PSym = if c.transCon != nil: result = c.transCon.owner else: result = c.module -proc newTemp(c: PTransf, typ: PType, info: TLineInfo): PSym = - result = newSym(skTemp, getIdent(genPrefix), getCurrOwner(c), info) - result.typ = skipTypes(typ, {tyGenericInst}) - incl(result.flags, sfFromGeneric) +proc newTemp(c: PTransf, typ: PType, info: TLineInfo): PNode = + let r = newSym(skTemp, getIdent(genPrefix), getCurrOwner(c), info) + r.typ = skipTypes(typ, {tyGenericInst}) + incl(r.flags, sfFromGeneric) + let owner = getCurrOwner(c) + if owner.isIterator and not c.tooEarly: + result = freshVarForClosureIter(r, owner) + else: + result = newSymNode(r) proc transform(c: PTransf, n: PNode): PTransNode @@ -111,13 +116,22 @@ proc newAsgnStmt(c: PTransf, le: PNode, ri: PTransNode): PTransNode = result[1] = ri proc transformSymAux(c: PTransf, n: PNode): PNode = - #if n.sym.kind == skClosureIterator: - # return liftIterSym(n) + let s = n.sym + if s.typ != nil and s.typ.callConv == ccClosure: + if s.kind == skIterator: + if c.tooEarly: return n + else: return liftIterSym(n, getCurrOwner(c)) + elif s.kind in {skProc, skConverter, skMethod} and not c.tooEarly: + # top level .closure procs are still somewhat supported for 'Nake': + return makeClosure(s, nil, n.info) + #elif n.sym.kind in {skVar, skLet} and n.sym.typ.callConv == ccClosure: + # echo n.info, " come heer for ", c.tooEarly + # if not c.tooEarly: var b: PNode var tc = c.transCon - if sfBorrow in n.sym.flags and n.sym.kind in routineKinds: + if sfBorrow in s.flags and s.kind in routineKinds: # simply exchange the symbol: - b = n.sym.getBody + b = s.getBody if b.kind != nkSym: internalError(n.info, "wrong AST for borrowed symbol") b = newSymNode(b.sym) b.info = n.info @@ -132,6 +146,16 @@ proc transformSymAux(c: PTransf, n: PNode): PNode = proc transformSym(c: PTransf, n: PNode): PTransNode = result = PTransNode(transformSymAux(c, n)) +proc freshVar(c: PTransf; v: PSym): PNode = + let owner = getCurrOwner(c) + if owner.isIterator and not c.tooEarly: + result = freshVarForClosureIter(v, owner) + else: + var newVar = copySym(v) + incl(newVar.flags, sfFromGeneric) + newVar.owner = owner + result = newSymNode(newVar) + proc transformVarSection(c: PTransf, v: PNode): PTransNode = result = newTransNode(v) for i in countup(0, sonsLen(v)-1): @@ -141,35 +165,30 @@ proc transformVarSection(c: PTransf, v: PNode): PTransNode = elif it.kind == nkIdentDefs: if it.sons[0].kind == nkSym: internalAssert(it.len == 3) - var newVar = copySym(it.sons[0].sym) - incl(newVar.flags, sfFromGeneric) - # fixes a strange bug for rodgen: - #include(it.sons[0].sym.flags, sfFromGeneric); - newVar.owner = getCurrOwner(c) - idNodeTablePut(c.transCon.mapping, it.sons[0].sym, newSymNode(newVar)) + let x = freshVar(c, it.sons[0].sym) + idNodeTablePut(c.transCon.mapping, it.sons[0].sym, x) var defs = newTransNode(nkIdentDefs, it.info, 3) if importantComments(): # keep documentation information: PNode(defs).comment = it.comment - defs[0] = newSymNode(newVar).PTransNode + defs[0] = x.PTransNode defs[1] = it.sons[1].PTransNode defs[2] = transform(c, it.sons[2]) - newVar.ast = defs[2].PNode + if x.kind == nkSym: x.sym.ast = defs[2].PNode result[i] = defs else: - # has been transformed into 'param.x' for closure iterators, so keep it: - result[i] = PTransNode(it) + # has been transformed into 'param.x' for closure iterators, so just + # transform it: + result[i] = transform(c, it) else: if it.kind != nkVarTuple: internalError(it.info, "transformVarSection: not nkVarTuple") var L = sonsLen(it) var defs = newTransNode(it.kind, it.info, L) for j in countup(0, L-3): - var newVar = copySym(it.sons[j].sym) - incl(newVar.flags, sfFromGeneric) - newVar.owner = getCurrOwner(c) - idNodeTablePut(c.transCon.mapping, it.sons[j].sym, newSymNode(newVar)) - defs[j] = newSymNode(newVar).PTransNode + let x = freshVar(c, it.sons[j].sym) + idNodeTablePut(c.transCon.mapping, it.sons[j].sym, x) + defs[j] = x.PTransNode assert(it.sons[L-2].kind == nkEmpty) defs[L-2] = ast.emptyNode.PTransNode defs[L-1] = transform(c, it.sons[L-1]) @@ -294,10 +313,18 @@ proc introduceNewLocalVars(c: PTransf, n: PNode): PTransNode = result = PTransNode(n) of nkVarSection, nkLetSection: result = transformVarSection(c, n) + of nkClosure: + # it can happen that for-loop-inlining produced a fresh + # set of variables, including some computed environment + # (bug #2604). We need to patch this environment here too: + let a = n[1] + if a.kind == nkSym: + n.sons[1] = transformSymAux(c, a) + return PTransNode(n) else: result = newTransNode(n) for i in countup(0, sonsLen(n)-1): - result[i] = introduceNewLocalVars(c, n.sons[i]) + result[i] = introduceNewLocalVars(c, n.sons[i]) proc transformYield(c: PTransf, n: PNode): PTransNode = result = newTransNode(nkStmtList, n.info, 0) @@ -348,6 +375,22 @@ proc transformAddrDeref(c: PTransf, n: PNode, a, b: TNodeKind): PTransNode = # addr ( deref ( x )) --> x result = PTransNode(n.sons[0].sons[0]) +proc generateThunk(prc: PNode, dest: PType): PNode = + ## Converts 'prc' into '(thunk, nil)' so that it's compatible with + ## a closure. + + # we cannot generate a proper thunk here for GC-safety reasons + # (see internal documentation): + if gCmd == cmdCompileToJS: return prc + result = newNodeIT(nkClosure, prc.info, dest) + var conv = newNodeIT(nkHiddenSubConv, prc.info, dest) + conv.add(emptyNode) + conv.add(prc) + if prc.kind == nkClosure: + internalError(prc.info, "closure to closure created") + result.add(conv) + result.add(newNodeIT(nkNilLit, prc.info, getSysType(tyNil))) + proc transformConv(c: PTransf, n: PNode): PTransNode = # numeric types need range checks: var dest = skipTypes(n.typ, abstractVarRange) @@ -428,6 +471,10 @@ proc transformConv(c: PTransf, n: PNode): PTransNode = of tyGenericParam, tyOrdinal: result = transform(c, n.sons[1]) # happens sometimes for generated assignments, etc. + of tyProc: + result = transformSons(c, n) + if dest.callConv == ccClosure and source.callConv == ccDefault: + result = generateThunk(result[1].PNode, dest).PTransNode else: result = transformSons(c, n) @@ -478,11 +525,14 @@ proc transformFor(c: PTransf, n: PNode): PTransNode = result[1] = newNode(nkEmpty).PTransNode return result c.breakSyms.add(labl) - if call.typ.kind != tyIter and - (call.kind notin nkCallKinds or call.sons[0].kind != nkSym or - call.sons[0].sym.kind != skIterator): + if call.kind notin nkCallKinds or call.sons[0].kind != nkSym or + call.sons[0].typ.callConv == ccClosure: n.sons[length-1] = transformLoopBody(c, n.sons[length-1]).PNode - result[1] = lambdalifting.liftForLoop(n).PTransNode + if not c.tooEarly: + n.sons[length-2] = transform(c, n.sons[length-2]).PNode + result[1] = lambdalifting.liftForLoop(n, getCurrOwner(c)).PTransNode + else: + result[1] = newNode(nkEmpty).PTransNode discard c.breakSyms.pop return result @@ -512,16 +562,15 @@ proc transformFor(c: PTransf, n: PNode): PTransNode = for i in countup(1, sonsLen(call) - 1): var arg = transform(c, call.sons[i]).PNode var formal = skipTypes(iter.typ, abstractInst).n.sons[i].sym - if arg.typ.kind == tyIter: continue case putArgInto(arg, formal.typ) of paDirectMapping: idNodeTablePut(newC.mapping, formal, arg) of paFastAsgn: # generate a temporary and produce an assignment statement: var temp = newTemp(c, formal.typ, formal.info) - addVar(v, newSymNode(temp)) - add(stmtList, newAsgnStmt(c, newSymNode(temp), arg.PTransNode)) - idNodeTablePut(newC.mapping, formal, newSymNode(temp)) + addVar(v, temp) + add(stmtList, newAsgnStmt(c, temp, arg.PTransNode)) + idNodeTablePut(newC.mapping, formal, temp) of paVarAsgn: assert(skipTypes(formal.typ, abstractInst).kind == tyVar) idNodeTablePut(newC.mapping, formal, arg) @@ -702,18 +751,13 @@ proc transform(c: PTransf, n: PNode): PTransNode = result = PTransNode(n) of nkBracketExpr: result = transformArrayAccess(c, n) of procDefs: - when false: - if n.sons[genericParamsPos].kind == nkEmpty: - var s = n.sons[namePos].sym - n.sons[bodyPos] = PNode(transform(c, s.getBody)) - if s.ast.sons[bodyPos] != n.sons[bodyPos]: - # somehow this can happen ... :-/ - s.ast.sons[bodyPos] = n.sons[bodyPos] - #n.sons[bodyPos] = liftLambdas(s, n) - #if n.kind == nkMethodDef: methodDef(s, false) - #if n.kind == nkIteratorDef and n.typ != nil: - # return liftIterSym(n.sons[namePos]).PTransNode - result = PTransNode(n) + var s = n.sons[namePos].sym + if n.typ != nil and s.typ.callConv == ccClosure: + result = transformSym(c, n.sons[namePos]) + # use the same node as before if still a symbol: + if result.PNode.kind == nkSym: result = PTransNode(n) + else: + result = PTransNode(n) of nkMacroDef: # XXX no proper closure support yet: when false: @@ -750,7 +794,7 @@ proc transform(c: PTransf, n: PNode): PTransNode = result = newTransNode(nkCommentStmt, n.info, 0) tryStmt.addSon(deferPart) # disable the original 'defer' statement: - n.kind = nkCommentStmt + n.kind = nkEmpty of nkContinueStmt: result = PTransNode(newNodeI(nkBreakStmt, n.info)) var labl = c.contSyms[c.contSyms.high] @@ -796,7 +840,14 @@ proc transform(c: PTransf, n: PNode): PTransNode = # XXX comment handling really sucks: if importantComments(): PNode(result).comment = n.comment - of nkClosure: return PTransNode(n) + of nkClosure: + # it can happen that for-loop-inlining produced a fresh + # set of variables, including some computed environment + # (bug #2604). We need to patch this environment here too: + let a = n[1] + if a.kind == nkSym: + n.sons[1] = transformSymAux(c, a) + return PTransNode(n) else: result = transformSons(c, n) when false: @@ -868,11 +919,11 @@ proc transformBody*(module: PSym, n: PNode, prc: PSym): PNode = result = n else: var c = openTransf(module, "") - result = processTransf(c, n, prc) + result = liftLambdas(prc, n, c.tooEarly) + #result = n + result = processTransf(c, result, prc) liftDefer(c, result) - result = liftLambdas(prc, result) - #if prc.kind == skClosureIterator: - # result = lambdalifting.liftIterator(prc, result) + #result = liftLambdas(prc, result) incl(result.flags, nfTransf) when useEffectSystem: trackProc(prc, result) #if prc.name.s == "testbody": @@ -885,9 +936,11 @@ proc transformStmt*(module: PSym, n: PNode): PNode = var c = openTransf(module, "") result = processTransf(c, n, module) liftDefer(c, result) - result = liftLambdasForTopLevel(module, result) + #result = liftLambdasForTopLevel(module, result) incl(result.flags, nfTransf) when useEffectSystem: trackTopLevelStmt(module, result) + #if n.info ?? "temp.nim": + # echo renderTree(result, {renderIds}) proc transformExpr*(module: PSym, n: PNode): PNode = if nfTransf in n.flags: diff --git a/compiler/types.nim b/compiler/types.nim index 3846be8a0..71ab84022 100644 --- a/compiler/types.nim +++ b/compiler/types.nim @@ -1061,7 +1061,8 @@ proc typeAllowedNode(marker: var IntSet, n: PNode, kind: TSymKind, else: for i in countup(0, sonsLen(n) - 1): let it = n.sons[i] - if it.kind == nkRecCase and kind == skConst: return n.typ + if it.kind == nkRecCase and kind in {skProc, skConst}: + return n.typ result = typeAllowedNode(marker, it, kind, flags) if result != nil: break @@ -1076,7 +1077,7 @@ proc matchType*(a: PType, pattern: openArray[tuple[k:TTypeKind, i:int]], proc typeAllowedAux(marker: var IntSet, typ: PType, kind: TSymKind, flags: TTypeAllowedFlags = {}): PType = - assert(kind in {skVar, skLet, skConst, skParam, skResult}) + assert(kind in {skVar, skLet, skConst, skProc, skParam, skResult}) # if we have already checked the type, return true, because we stop the # evaluation if something is wrong: result = nil @@ -1085,7 +1086,7 @@ proc typeAllowedAux(marker: var IntSet, typ: PType, kind: TSymKind, var t = skipTypes(typ, abstractInst-{tyTypeDesc}) case t.kind of tyVar: - if kind == skConst: return t + if kind in {skProc, skConst}: return t var t2 = skipTypes(t.sons[0], abstractInst-{tyTypeDesc}) case t2.kind of tyVar: @@ -1097,6 +1098,7 @@ proc typeAllowedAux(marker: var IntSet, typ: PType, kind: TSymKind, if kind notin {skParam, skResult}: result = t else: result = typeAllowedAux(marker, t2, kind, flags) of tyProc: + if kind == skConst and t.callConv == ccClosure: return t for i in countup(1, sonsLen(t) - 1): result = typeAllowedAux(marker, t.sons[i], skParam, flags) if result != nil: break @@ -1144,7 +1146,8 @@ proc typeAllowedAux(marker: var IntSet, typ: PType, kind: TSymKind, result = typeAllowedAux(marker, t.sons[i], kind, flags) if result != nil: break of tyObject, tyTuple: - if kind == skConst and t.kind == tyObject and t.sons[0] != nil: return t + if kind in {skProc, skConst} and + t.kind == tyObject and t.sons[0] != nil: return t let flags = flags+{taField} for i in countup(0, sonsLen(t) - 1): result = typeAllowedAux(marker, t.sons[i], kind, flags) diff --git a/compiler/vm.nim b/compiler/vm.nim index 495b0c747..80c2c0fbf 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -255,9 +255,12 @@ proc cleanUpOnException(c: PCtx; tos: PStackFrame): nextExceptOrFinally = pc2 + c.code[pc2].regBx - wordExcess inc pc2 while c.code[pc2].opcode == opcExcept: - let exceptType = c.types[c.code[pc2].regBx-wordExcess].skipTypes( + let excIndex = c.code[pc2].regBx-wordExcess + let exceptType = if excIndex > 0: c.types[excIndex].skipTypes( abstractPtrs) - if inheritanceDiff(exceptType, raisedType) <= 0: + else: nil + #echo typeToString(exceptType), " ", typeToString(raisedType) + if exceptType.isNil or inheritanceDiff(exceptType, raisedType) <= 0: # mark exception as handled but keep it in B for # the getCurrentException() builtin: c.currentExceptionB = c.currentExceptionA @@ -356,7 +359,14 @@ proc opConv*(dest: var TFullReg, src: TFullReg, desttyp, srctyp: PType): bool = of tyFloat..tyFloat64: dest.intVal = int(src.floatVal) else: - dest.intVal = src.intVal and ((1 shl (desttyp.size*8))-1) + let srcDist = (sizeof(src.intVal) - srctyp.size) * 8 + let destDist = (sizeof(dest.intVal) - desttyp.size) * 8 + when system.cpuEndian == bigEndian: + dest.intVal = (src.intVal shr srcDist) shl srcDist + dest.intVal = (dest.intVal shr destDist) shl destDist + else: + dest.intVal = (src.intVal shl srcDist) shr srcDist + dest.intVal = (dest.intVal shl destDist) shr destDist of tyFloat..tyFloat64: if dest.kind != rkFloat: myreset(dest); dest.kind = rkFloat @@ -608,7 +618,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = addSon(regs[ra].node, r.copyTree) of opcExcl: decodeB(rkNode) - var b = newNodeIT(nkCurly, regs[rb].node.info, regs[rb].node.typ) + var b = newNodeIT(nkCurly, regs[ra].node.info, regs[ra].node.typ) addSon(b, regs[rb].regToNode) var r = diffSets(regs[ra].node, b) discardSons(regs[ra].node) @@ -1190,6 +1200,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = createStr regs[ra] let a = regs[rb].node if a.kind in {nkStrLit..nkTripleStrLit}: regs[ra].node.strVal = a.strVal + elif a.kind == nkCommentStmt: regs[ra].node.strVal = a.comment else: stackTrace(c, tos, pc, errFieldXNotFound, "strVal") of opcSlurp: decodeB(rkNode) diff --git a/compiler/vmdeps.nim b/compiler/vmdeps.nim index 2cc4a107b..a4f02092d 100644 --- a/compiler/vmdeps.nim +++ b/compiler/vmdeps.nim @@ -70,7 +70,7 @@ proc atomicTypeX(name: string; t: PType; info: TLineInfo): PNode = proc mapTypeToAst(t: PType, info: TLineInfo; allowRecursion=false): PNode proc mapTypeToBracket(name: string; t: PType; info: TLineInfo): PNode = - result = newNodeIT(nkBracketExpr, info, t) + result = newNodeIT(nkBracketExpr, if t.n.isNil: info else: t.n.info, t) result.add atomicTypeX(name, t, info) for i in 0 .. < t.len: if t.sons[i] == nil: @@ -92,19 +92,19 @@ proc mapTypeToAst(t: PType, info: TLineInfo; allowRecursion=false): PNode = of tyStmt: result = atomicType("stmt") of tyEmpty: result = atomicType"void" of tyArrayConstr, tyArray: - result = newNodeIT(nkBracketExpr, info, t) + result = newNodeIT(nkBracketExpr, if t.n.isNil: info else: t.n.info, t) result.add atomicType("array") result.add mapTypeToAst(t.sons[0], info) result.add mapTypeToAst(t.sons[1], info) of tyTypeDesc: if t.base != nil: - result = newNodeIT(nkBracketExpr, info, t) + result = newNodeIT(nkBracketExpr, if t.n.isNil: info else: t.n.info, t) result.add atomicType("typeDesc") result.add mapTypeToAst(t.base, info) else: result = atomicType"typeDesc" of tyGenericInvocation: - result = newNodeIT(nkBracketExpr, info, t) + result = newNodeIT(nkBracketExpr, if t.n.isNil: info else: t.n.info, t) for i in 0 .. < t.len: result.add mapTypeToAst(t.sons[i], info) of tyGenericInst, tyGenericBody, tyOrdinal, tyUserTypeClassInst: @@ -117,7 +117,7 @@ proc mapTypeToAst(t: PType, info: TLineInfo; allowRecursion=false): PNode = of tyGenericParam, tyForward: result = atomicType(t.sym.name.s) of tyObject: if allowRecursion: - result = newNodeIT(nkObjectTy, info, t) + result = newNodeIT(nkObjectTy, if t.n.isNil: info else: t.n.info, t) if t.sons[0] == nil: result.add ast.emptyNode else: @@ -126,7 +126,7 @@ proc mapTypeToAst(t: PType, info: TLineInfo; allowRecursion=false): PNode = else: result = atomicType(t.sym.name.s) of tyEnum: - result = newNodeIT(nkEnumTy, info, t) + result = newNodeIT(nkEnumTy, if t.n.isNil: info else: t.n.info, t) result.add copyTree(t.n) of tyTuple: result = mapTypeToBracket("tuple", t, info) of tySet: result = mapTypeToBracket("set", t, info) @@ -137,7 +137,7 @@ proc mapTypeToAst(t: PType, info: TLineInfo; allowRecursion=false): PNode = of tyProc: result = mapTypeToBracket("proc", t, info) of tyOpenArray: result = mapTypeToBracket("openArray", t, info) of tyRange: - result = newNodeIT(nkBracketExpr, info, t) + result = newNodeIT(nkBracketExpr, if t.n.isNil: info else: t.n.info, t) result.add atomicType("range") result.add t.n.sons[0].copyTree result.add t.n.sons[1].copyTree @@ -174,7 +174,7 @@ proc mapTypeToAst(t: PType, info: TLineInfo; allowRecursion=false): PNode = of tyNot: result = mapTypeToBracket("not", t, info) of tyAnything: result = atomicType"anything" of tyStatic, tyFromExpr, tyFieldAccessor: - result = newNodeIT(nkBracketExpr, info, t) + result = newNodeIT(nkBracketExpr, if t.n.isNil: info else: t.n.info, t) result.add atomicType("static") if t.n != nil: result.add t.n.copyTree diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index 97c6a5580..75c1378e5 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -1209,7 +1209,7 @@ proc checkCanEval(c: PCtx; n: PNode) = not s.isOwnedBy(c.prc.sym) and s.owner != c.module and c.mode != emRepl: cannotEval(n) elif s.kind in {skProc, skConverter, skMethod, - skIterator, skClosureIterator} and sfForward in s.flags: + skIterator} and sfForward in s.flags: cannotEval(n) proc isTemp(c: PCtx; dest: TDest): bool = @@ -1604,7 +1604,8 @@ proc matches(s: PSym; x: string): bool = var s = s var L = y.len-1 while L >= 0: - if s == nil or y[L].cmpIgnoreStyle(s.name.s) != 0: return false + if s == nil or (y[L].cmpIgnoreStyle(s.name.s) != 0 and y[L] != "*"): + return false s = s.owner dec L result = true @@ -1613,7 +1614,8 @@ proc matches(s: PSym; y: varargs[string]): bool = var s = s var L = y.len-1 while L >= 0: - if s == nil or y[L].cmpIgnoreStyle(s.name.s) != 0: return false + if s == nil or (y[L].cmpIgnoreStyle(s.name.s) != 0 and y[L] != "*"): + return false s = if sfFromGeneric in s.flags: s.owner.owner else: s.owner dec L result = true @@ -1636,7 +1638,7 @@ proc gen(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}) = case s.kind of skVar, skForVar, skTemp, skLet, skParam, skResult: genRdVar(c, n, dest, flags) - of skProc, skConverter, skMacro, skTemplate, skMethod, skIterators: + of skProc, skConverter, skMacro, skTemplate, skMethod, skIterator: # 'skTemplate' is only allowed for 'getAst' support: if procIsCallback(c, s): discard elif sfImportc in s.flags: c.importcSym(n.info, s) diff --git a/compiler/vmhooks.nim b/compiler/vmhooks.nim index 576b0565f..3456e893b 100644 --- a/compiler/vmhooks.nim +++ b/compiler/vmhooks.nim @@ -55,9 +55,16 @@ template getX(k, field) {.immediate, dirty.} = result = s[i+a.rb+1].field proc getInt*(a: VmArgs; i: Natural): BiggestInt = getX(rkInt, intVal) +proc getBool*(a: VmArgs; i: Natural): bool = getInt(a, i) != 0 proc getFloat*(a: VmArgs; i: Natural): BiggestFloat = getX(rkFloat, floatVal) proc getString*(a: VmArgs; i: Natural): string = doAssert i < a.rc-1 let s = cast[seq[TFullReg]](a.slots) doAssert s[i+a.rb+1].kind == rkNode result = s[i+a.rb+1].node.strVal + +proc getNode*(a: VmArgs; i: Natural): PNode = + doAssert i < a.rc-1 + let s = cast[seq[TFullReg]](a.slots) + doAssert s[i+a.rb+1].kind == rkNode + result = s[i+a.rb+1].node diff --git a/compiler/vmops.nim b/compiler/vmops.nim index e1a0dfef8..e40e05eff 100644 --- a/compiler/vmops.nim +++ b/compiler/vmops.nim @@ -13,7 +13,7 @@ from math import sqrt, ln, log10, log2, exp, round, arccos, arcsin, arctan, arctan2, cos, cosh, hypot, sinh, sin, tan, tanh, pow, trunc, floor, ceil, fmod -from os import getEnv, existsEnv, dirExists, fileExists +from os import getEnv, existsEnv, dirExists, fileExists, walkDir template mathop(op) {.immediate, dirty.} = registerCallback(c, "stdlib.math." & astToStr(op), `op Wrapper`) @@ -48,6 +48,12 @@ proc getCurrentExceptionMsgWrapper(a: VmArgs) {.nimcall.} = setResult(a, if a.currentException.isNil: "" else: a.currentException.sons[3].skipColon.strVal) +proc staticWalkDirImpl(path: string, relative: bool): PNode = + result = newNode(nkBracket) + for k, f in walkDir(path, relative): + result.add newTree(nkPar, newIntNode(nkIntLit, k.ord), + newStrNode(nkStrLit, f)) + proc registerAdditionalOps*(c: PCtx) = wrap1f(sqrt) wrap1f(ln) @@ -78,3 +84,5 @@ proc registerAdditionalOps*(c: PCtx) = wrap1s(fileExists) wrap2svoid(writeFile) systemop getCurrentExceptionMsg + registerCallback c, "stdlib.*.staticWalkDir", proc (a: VmArgs) {.nimcall.} = + setResult(a, staticWalkDirImpl(getString(a, 0), getBool(a, 1))) diff --git a/config/nimdoc.cfg b/config/nimdoc.cfg index 23151b275..d973b922a 100644 --- a/config/nimdoc.cfg +++ b/config/nimdoc.cfg @@ -102,8 +102,8 @@ doc.file = """<?xml version="1.0" encoding="utf-8" ?> <link rel="shortcut icon" href="data:image/x-icon;base64,AAABAAEAEBAAAAEAIABoBAAAFgAAACgAAAAQAAAAIAAAAAEAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AAAAAAUAAAAF////AP///wD///8A////AP///wD///8A////AP///wD///8A////AAAAAAIAAABbAAAAlQAAAKIAAACbAAAAmwAAAKIAAACVAAAAWwAAAAL///8A////AP///wD///8A////AAAAABQAAADAAAAAYwAAAA3///8A////AP///wD///8AAAAADQAAAGMAAADAAAAAFP///wD///8A////AP///wAAAACdAAAAOv///wD///8A////AP///wD///8A////AP///wD///8AAAAAOgAAAJ3///8A////AP///wAAAAAnAAAAcP///wAAAAAoAAAASv///wD///8A////AP///wAAAABKAAAAKP///wAAAABwAAAAJ////wD///8AAAAAgQAAABwAAACIAAAAkAAAAJMAAACtAAAAFQAAABUAAACtAAAAkwAAAJAAAACIAAAAHAAAAIH///8A////AAAAAKQAAACrAAAAaP///wD///8AAAAARQAAANIAAADSAAAARf///wD///8AAAAAaAAAAKsAAACk////AAAAADMAAACcAAAAnQAAABj///8A////AP///wAAAAAYAAAAGP///wD///8A////AAAAABgAAACdAAAAnAAAADMAAAB1AAAAwwAAAP8AAADpAAAAsQAAAE4AAAAb////AP///wAAAAAbAAAATgAAALEAAADpAAAA/wAAAMMAAAB1AAAAtwAAAOkAAAD/AAAA/wAAAP8AAADvAAAA3gAAAN4AAADeAAAA3gAAAO8AAAD/AAAA/wAAAP8AAADpAAAAtwAAAGUAAAA/AAAA3wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAADfAAAAPwAAAGX///8A////AAAAAEgAAADtAAAAvwAAAL0AAADGAAAA7wAAAO8AAADGAAAAvQAAAL8AAADtAAAASP///wD///8A////AP///wD///8AAAAAO////wD///8A////AAAAAIcAAACH////AP///wD///8AAAAAO////wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A//8AAP//AAD4HwAA7/cAAN/7AAD//wAAoYUAAJ55AACf+QAAh+EAAAAAAADAAwAA4AcAAP5/AAD//wAA//8AAA=="/> <!-- Google fonts --> -<link href='http://fonts.googleapis.com/css?family=Raleway:400,600,900' rel='stylesheet' type='text/css'> -<link href='http://fonts.googleapis.com/css?family=Source+Code+Pro:400,500,600' rel='stylesheet' type='text/css'> +<link href='http://fonts.googleapis.com/css?family=Raleway:400,600,900' rel='stylesheet' type='text/css'/> +<link href='http://fonts.googleapis.com/css?family=Source+Code+Pro:400,500,600' rel='stylesheet' type='text/css'/> <!-- CSS --> <title>$title</title> @@ -1246,7 +1246,7 @@ dt pre > span.Operator ~ span.Identifier, dt pre > span.Operator ~ span.Operator <div class="row"> <div class="twelve-columns footer"> <span class="nim-sprite"></span> - <br> + <br/> <small>Made with Nim. Generated: $date $time UTC</small> </div> </div> diff --git a/doc/astspec.txt b/doc/astspec.txt index c84fad8e8..f235e2984 100644 --- a/doc/astspec.txt +++ b/doc/astspec.txt @@ -924,9 +924,11 @@ AST: .. code-block:: nim nnkLetSection( - nnkIdentDefs(!"v"), - nnkEmpty(), # for the type - nnkIntLit(3) + nnkIdentDefs( + nnkIdent(!"a"), + nnkEmpty(), # or nnkIdent(...) for the type + nnkIntLit(3), + ) ) Const section diff --git a/doc/grammar.txt b/doc/grammar.txt index 72dc6c974..d967bf938 100644 --- a/doc/grammar.txt +++ b/doc/grammar.txt @@ -35,10 +35,13 @@ castExpr = 'cast' '[' optInd typeDesc optPar ']' '(' optInd expr optPar ')' parKeyw = 'discard' | 'include' | 'if' | 'while' | 'case' | 'try' | 'finally' | 'except' | 'for' | 'block' | 'const' | 'let' | 'when' | 'var' | 'mixin' -par = '(' optInd (&parKeyw complexOrSimpleStmt ^+ ';' - | simpleExpr ('=' expr (';' complexOrSimpleStmt ^+ ';' )? )? - | (':' expr)? (',' (exprColonEqExpr comma?)*)? )? - optPar ')' +par = '(' optInd + ( &parKeyw complexOrSimpleStmt ^+ ';' + | ';' complexOrSimpleStmt ^+ ';' + | pragmaStmt + | simpleExpr ( ('=' expr (';' complexOrSimpleStmt ^+ ';' )? ) + | (':' expr (',' exprColonEqExpr ^+ ',' )? ) ) ) + optPar ')' literal = | INT_LIT | INT8_LIT | INT16_LIT | INT32_LIT | INT64_LIT | UINT_LIT | UINT8_LIT | UINT16_LIT | UINT32_LIT | UINT64_LIT | FLOAT_LIT | FLOAT32_LIT | FLOAT64_LIT @@ -86,7 +89,7 @@ expr = (ifExpr | caseExpr | tryExpr) / simpleExpr -typeKeyw = 'var' | 'ref' | 'ptr' | 'shared' | 'tuple' +typeKeyw = 'var' | 'out' | 'ref' | 'ptr' | 'shared' | 'tuple' | 'proc' | 'iterator' | 'distinct' | 'object' | 'enum' primary = typeKeyw typeDescK / prefixOperator* identOrLiteral primarySuffix* @@ -165,7 +168,7 @@ objectCase = 'case' identWithPragma ':' typeDesc ':'? COMMENT? objectPart = IND{>} objectPart^+IND{=} DED / objectWhen / objectCase / 'nil' / 'discard' / declColonEquals object = 'object' pragma? ('of' typeDesc)? COMMENT? objectPart -typeClassParam = ('var')? symbol +typeClassParam = ('var' | 'out')? symbol typeClass = typeClassParam ^* ',' (pragma)? ('of' typeDesc ^* ',')? &IND{>} stmt typeDef = identWithPragma genericParamList? '=' optInd typeDefAux diff --git a/doc/lib.txt b/doc/lib.txt index 3dc58eebf..90cf36240 100644 --- a/doc/lib.txt +++ b/doc/lib.txt @@ -84,7 +84,7 @@ Collections and algorithms * `sequtils <sequtils.html>`_ This module implements operations for the built-in seq type which were inspired by functional programming languages. - + String handling --------------- @@ -165,6 +165,8 @@ Generic Operating System Services This module implements the ability to monitor a directory/file for changes using Posix's inotify API. + **Warning:** This module will likely be moved out to a Nimble package soon. + * `asyncfile <asyncfile.html>`_ This module implements asynchronous file reading and writing using ``asyncdispatch``. @@ -191,6 +193,11 @@ Math libraries * `basic3d <basic3d.html>`_ Basic 3d support with vectors, points, matrices and some basic utilities. +* `mersenne <mersenne.html>`_ + Mersenne twister random number generator. + +* `stats <stats.html>`_ + Statistical analysis Internet Protocols and Support ------------------------------ @@ -209,7 +216,8 @@ Internet Protocols and Support This module implements a simple HTTP server. * `httpclient <httpclient.html>`_ - This module implements a simple HTTP client. + This module implements a simple HTTP client which supports both synchronous + and asynchronous retrieval of web pages. * `smtp <smtp.html>`_ This module implement a simple SMTP client. @@ -226,19 +234,17 @@ Internet Protocols and Support * `asyncdispatch <asyncdispatch.html>`_ This module implements an asynchronous dispatcher for IO operations. - **Note:** This module is still largely experimental. - * `asyncnet <asyncnet.html>`_ This module implements asynchronous sockets based on the ``asyncdispatch`` module. - **Note:** This module is still largely experimental. - * `asynchttpserver <asynchttpserver.html>`_ This module implements an asynchronous HTTP server using the ``asyncnet`` module. - **Note:** This module is still largely experimental. +* `asyncftpclient <asyncftpclient.html>`_ + This module implements an asynchronous FTP client using the ``asyncnet`` + module. * `net <net.html>`_ This module implements a high-level sockets API. It will replace the @@ -346,6 +352,8 @@ Cryptography and Hashing * `base64 <base64.html>`_ This module implements a base64 encoder and decoder. +* `securehash <securehash.html>`_ + This module implements a sha1 encoder and decoder. Multimedia support ------------------ @@ -374,10 +382,18 @@ Miscellaneous * `logging <logging.html>`_ This module implements a simple logger. +* `options <options.html>`_ + Types which encapsulate an optional value. + * `future <future.html>`_ This module implements new experimental features. Currently the syntax sugar for anonymous procedures. +* `coro <coro.html>`_ + This module implements experimental coroutines in Nim. + +* `unittest <unittest.html>`_ + Implements a Unit testing DSL. Modules for JS backend --------------------------- diff --git a/doc/manual/generics.txt b/doc/manual/generics.txt index a73e22988..07d98b289 100644 --- a/doc/manual/generics.txt +++ b/doc/manual/generics.txt @@ -213,7 +213,7 @@ Concepts are written in the following form: Container[T] = concept c c.len is Ordinal - items(c) is iterator + items(c) is T for value in c: type(value) is T diff --git a/doc/manual/lexing.txt b/doc/manual/lexing.txt index 7f81ab422..5990dff07 100644 --- a/doc/manual/lexing.txt +++ b/doc/manual/lexing.txt @@ -69,6 +69,34 @@ Documentation comments are tokens; they are only allowed at certain places in the input file as they belong to the syntax tree! +Multiline comments +------------------ + +Starting with version 0.13.0 of the language Nim supports multiline comments. +They look like: + +.. code-block:: nim + #[Comment here. + Multiple lines + are not a problem.]# + +Multiline comments support nesting: + +.. code-block:: nim + #[ #[ Multiline comment in already + commented out code. ]# + proc p[T](x: T) = discard + ]# + +Multiline documentation comments look like and support nesting too: + +.. code-block:: nim + proc foo = + ##[Long documentation comment + here. + ]## + + Identifiers & Keywords ---------------------- diff --git a/doc/manual/modules.txt b/doc/manual/modules.txt index e6a08b5ce..ac47d89dd 100644 --- a/doc/manual/modules.txt +++ b/doc/manual/modules.txt @@ -152,9 +152,11 @@ In module related statements, if any part of the module name / path begins with a number, you may have to quote it in double quotes. In the following example, it would be seen as a literal number '3.0' of type 'float64' if not quoted, if uncertain - quote it: + .. code-block:: nim import "gfx/3d/somemodule" + Scope rules ----------- Identifiers are valid from the point of their declaration until the end of diff --git a/doc/manual/procs.txt b/doc/manual/procs.txt index ee74b2ea6..654893286 100644 --- a/doc/manual/procs.txt +++ b/doc/manual/procs.txt @@ -236,8 +236,6 @@ executable code. Do notation ----------- -**Note:** The future of the ``do`` notation is uncertain. - As a special more convenient notation, proc expressions involved in procedure calls can use the ``do`` keyword: @@ -251,10 +249,12 @@ calls can use the ``do`` keyword: ``do`` is written after the parentheses enclosing the regular proc params. The proc expression represented by the do block is appended to them. -More than one ``do`` block can appear in a single call: +``do`` with parentheses is an anonymous ``proc``; however a ``do`` without +parentheses is just a block of code. The ``do`` notation can be used to +pass multiple blocks to a macro: .. code-block:: nim - proc performWithUndo(task: proc(), undo: proc()) = ... + macro performWithUndo(task, undo: untyped) = ... performWithUndo do: # multiple-line block of code diff --git a/doc/manual/syntax.txt b/doc/manual/syntax.txt index c444a3995..ca3b582ca 100644 --- a/doc/manual/syntax.txt +++ b/doc/manual/syntax.txt @@ -64,6 +64,14 @@ Precedence level Operators First charact ================ =============================================== ================== =============== +Whether an operator is used a prefix operator is also affected by preceeding whitespace (this parsing change was introduced with version 0.13.0): + +.. code-block:: nim + echo $foo + # is parsed as + echo($foo) + + Strong spaces ------------- diff --git a/doc/manual/templates.txt b/doc/manual/templates.txt index 092d65ea2..b60fe632e 100644 --- a/doc/manual/templates.txt +++ b/doc/manual/templates.txt @@ -10,7 +10,7 @@ The syntax to *invoke* a template is the same as calling a procedure. Example: .. code-block:: nim - template `!=` (a, b: expr): expr = + template `!=` (a, b: untyped): untyped = # this definition exists in the System module not (a == b) @@ -23,50 +23,56 @@ templates: | ``a in b`` is transformed into ``contains(b, a)``. | ``notin`` and ``isnot`` have the obvious meanings. -The "types" of templates can be the symbols ``expr`` (stands for *expression*), -``stmt`` (stands for *statement*) or ``typedesc`` (stands for *type +The "types" of templates can be the symbols ``untyped``, +``typed`` or ``typedesc`` (stands for *type description*). These are "meta types", they can only be used in certain -contexts. Real types can be used too; this implies that expressions are -expected. +contexts. Real types can be used too; this implies that ``typed`` expressions +are expected. -Ordinary vs immediate templates -------------------------------- +Typed vs untyped parameters +--------------------------- -There are two different kinds of templates: immediate templates and -ordinary templates. Ordinary templates take part in overloading resolution. As -such their arguments need to be type checked before the template is invoked. -So ordinary templates cannot receive undeclared identifiers: +An ``untyped`` parameter means that symbol lookups and type resolution is not +performed before the expression is passed to the template. This means that for +example *undeclared* identifiers can be passed to the template: .. code-block:: nim - template declareInt(x: expr) = + template declareInt(x: untyped) = var x: int - declareInt(x) # error: unknown identifier: 'x' + declareInt(x) # valid + x = 3 -An ``immediate`` template does not participate in overload resolution and so -its arguments are not checked for semantics before invocation. So they can -receive undeclared identifiers: .. code-block:: nim - template declareInt(x: expr) {.immediate.} = + template declareInt(x: typed) = var x: int - declareInt(x) # valid + declareInt(x) # invalid, because x has not been declared and so has no type + +A template where every parameter is ``untyped`` is called an `immediate`:idx: +template. For historical reasons templates can be explicitly annotated with +an ``immediate`` pragma and then these templates do not take part in +overloading resolution and the parameters' types are *ignored* by the +compiler. Explicit immediate templates are about to be deprecated in later +versions of the compiler. + +**Note**: For historical reasons ``stmt`` is an alias for ``typed`` and +``expr`` an alias for ``untyped``, but new code should use the newer, +clearer names. Passing a code block to a template ---------------------------------- -If there is a ``stmt`` parameter it should be the last in the template -declaration, because statements are passed to a template via a +You can pass a block of statements as a last parameter to a template via a special ``:`` syntax: .. code-block:: nim - - template withFile(f, fn, mode: expr, actions: stmt): stmt {.immediate.} = + template withFile(f, fn, mode, actions: untyped): untyped = var f: File if open(f, fn, mode): try: @@ -84,6 +90,64 @@ In the example the two ``writeLine`` statements are bound to the ``actions`` parameter. +Usually to pass a block of code to a template the parameter that accepts +the block needs to be of type ``untyped``. Because symbol lookups are then +delayed until template instantiation time: + +.. code-block:: nim + template t(body: typed) = + block: + body + + t: + var i = 1 + echo i + + t: + var i = 2 # fails with 'attempt to redeclare i' + echo i + +The above code fails with the mysterious error message that ``i`` has already +been declared. The reason for this is that the ``var i = ...`` bodies need to +be type-checked before they are passed to the ``body`` parameter and type +checking in Nim implies symbol lookups. For the symbol lookups to succeed +``i`` needs to be added to the current (i.e. outer) scope. After type checking +these additions to the symbol table are not rolled back (for better or worse). +The same code works with ``untyped`` as the passed body is not required to be +type-checked: + +.. code-block:: nim + template t(body: untyped) = + block: + body + + t: + var i = 1 + echo i + + t: + var i = 2 # compiles + echo i + + +Varargs of untyped +------------------ + +In addition to the ``untyped`` meta-type that prevents type checking there is +also ``varargs[untyped]`` so that not even the number of parameters is fixed: + +.. code-block:: nim + template hideIdentifiers(x: varargs[untyped]) = discard + + hideIdentifiers(undeclared1, undeclared2) + +However, since a template cannot iterate over varargs, this feature is +generally much more useful for macros. + +**Note**: For historical reasons ``varargs[expr]`` is not equivalent +to ``varargs[untyped]``. + + Symbol binding in templates --------------------------- diff --git a/doc/tools.txt b/doc/tools.txt index bad603925..7c9aed7ad 100644 --- a/doc/tools.txt +++ b/doc/tools.txt @@ -4,7 +4,7 @@ Tools available with Nim The standard distribution ships with the following tools: -- | `Documentation generator <docs/docgen.html>`_ +- | `Documentation generator <docgen.html>`_ | The builtin document generator ``nim doc2`` generates HTML documentation from ``.nim`` source files. diff --git a/doc/tut1.txt b/doc/tut1.txt index 7dce8a218..747c1a3ff 100644 --- a/doc/tut1.txt +++ b/doc/tut1.txt @@ -758,19 +758,18 @@ However, this cannot be done for mutually recursive procedures: # forward declaration: proc even(n: int): bool -proc even(n: int): bool - -proc odd(n: int): bool = - assert(n >= 0) # makes sure we don't run into negative recursion - if n == 0: false - else: - n == 1 or even(n-1) +.. code-block:: nim + proc odd(n: int): bool = + assert(n >= 0) # makes sure we don't run into negative recursion + if n == 0: false + else: + n == 1 or even(n-1) -proc even(n: int): bool = - assert(n >= 0) # makes sure we don't run into negative recursion - if n == 1: false - else: - n == 0 or odd(n-1) + proc even(n: int): bool = + assert(n >= 0) # makes sure we don't run into negative recursion + if n == 1: false + else: + n == 0 or odd(n-1) Here ``odd`` depends on ``even`` and vice versa. Thus ``even`` needs to be introduced to the compiler before it is completely defined. The syntax for diff --git a/doc/tut2.txt b/doc/tut2.txt index db9e4cd58..563344570 100644 --- a/doc/tut2.txt +++ b/doc/tut2.txt @@ -112,7 +112,7 @@ Example: Sym = object # a symbol name: string # the symbol's name line: int # the line the symbol was declared in - code: PNode # the symbol's abstract syntax tree + code: Node # the symbol's abstract syntax tree Type conversions @@ -162,11 +162,11 @@ An example: of nkFloat: floatVal: float of nkString: strVal: string of nkAdd, nkSub: - leftOp, rightOp: PNode + leftOp, rightOp: Node of nkIf: - condition, thenPart, elsePart: PNode + condition, thenPart, elsePart: Node - var n = PNode(kind: nkFloat, floatVal: 1.0) + var n = Node(kind: nkFloat, floatVal: 1.0) # the following statement raises an `FieldError` exception, because # n.kind's value does not fit: n.strVal = "" @@ -990,3 +990,17 @@ generated by `treeRepr <macros.html#treeRepr>`_. If at the end of the this example you add ``echo treeRepr(result)`` you should get the same output as using the ``dumpTree`` macro, but of course you can call that at any point of the macro where you might be having troubles. + + +Compilation to JavaScript +========================= + +Nim code can be compiled to JavaScript. However in order to write +JavaScript-compatible code you should remember the following: +- ``addr`` and ``ptr`` have slightly different semantic meaning in JavaScript. + It is recommended to avoid those if you're not sure how they are translated + to JavaScript. +- ``cast[T](x)`` in JavaScript is translated to ``(x)``. +- ``cstring`` in JavaScript means JavaScript string. It is a good practice to + use ``cstring`` only when it is semantically appropriate. E.g. don't use + ``cstring`` as a binary data buffer. diff --git a/examples/statcsv.nim b/examples/statcsv.nim index f2cf809e2..983cd555f 100644 --- a/examples/statcsv.nim +++ b/examples/statcsv.nim @@ -3,7 +3,7 @@ # the standard deviation of its columns. # The CSV file can have a header which is then used for the output. -import os, streams, parsecsv, strutils, math +import os, streams, parsecsv, strutils, math, stats if paramCount() < 1: quit("Usage: statcsv filename[.csv]") diff --git a/install_nimble.nims b/install_nimble.nims index 5e363c689..5d028726b 100644 --- a/install_nimble.nims +++ b/install_nimble.nims @@ -1,4 +1,6 @@ +import ospaths + mode = ScriptMode.Verbose var id = 0 @@ -10,4 +12,8 @@ exec "git clone https://github.com/nim-lang/nimble.git nimble" & $id withDir "nimble" & $id & "/src": exec "nim c nimble" +mkDir "bin/nimblepkg" +for file in listFiles("nimble" & $id & "/src/nimblepkg/"): + cpFile file, "bin/nimblepkg/" & file.extractFilename + mvFile "nimble" & $id & "/src/nimble".toExe, "bin/nimble".toExe diff --git a/lib/core/macros.nim b/lib/core/macros.nim index 552c0dbff..872d4848d 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -97,7 +97,7 @@ type nskUnknown, nskConditional, nskDynLib, nskParam, nskGenericParam, nskTemp, nskModule, nskType, nskVar, nskLet, nskConst, nskResult, - nskProc, nskMethod, nskIterator, nskClosureIterator, + nskProc, nskMethod, nskIterator, nskConverter, nskMacro, nskTemplate, nskField, nskEnumField, nskForVar, nskLabel, nskStub @@ -416,8 +416,7 @@ proc newLit*(i: BiggestInt): NimNode {.compileTime.} = proc newLit*(b: bool): NimNode {.compileTime.} = ## produces a new boolean literal node. - result = newNimNode(nnkIntLit) - result.intVal = ord(b) + result = if b: bindSym"true" else: bindSym"false" proc newLit*(f: BiggestFloat): NimNode {.compileTime.} = ## produces a new float literal node. diff --git a/lib/core/typeinfo.nim b/lib/core/typeinfo.nim index 1f9fb1072..db5a83755 100644 --- a/lib/core/typeinfo.nim +++ b/lib/core/typeinfo.nim @@ -61,7 +61,10 @@ type ## wrapped value and **must not** live longer than ## its wrapped value. value: pointer - rawType: PNimType + when defined(js): + rawType: PNimType + else: + rawTypePtr: pointer ppointer = ptr pointer pbyteArray = ptr array[0.. 0xffff, int8] @@ -71,6 +74,14 @@ type when defined(gogc): elemSize: int PGenSeq = ptr TGenericSeq + +when not defined(js): + template rawType(x: Any): PNimType = + cast[PNimType](x.rawTypePtr) + + template `rawType=`(x: var Any, p: PNimType) = + x.rawTypePtr = cast[pointer](p) + {.deprecated: [TAny: Any, TAnyKind: AnyKind].} when defined(gogc): @@ -108,7 +119,7 @@ proc selectBranch(aa: pointer, n: ptr TNimNode): ptr TNimNode = else: result = n.sons[n.len] -proc newAny(value: pointer, rawType: PNimType): Any = +proc newAny(value: pointer, rawType: PNimType): Any {.inline.} = result.value = value result.rawType = rawType @@ -126,8 +137,7 @@ proc toAny*[T](x: var T): Any {.inline.} = ## constructs a ``Any`` object from `x`. This captures `x`'s address, so ## `x` can be modified with its ``Any`` wrapper! The client needs to ensure ## that the wrapper **does not** live longer than `x`! - result.value = addr(x) - result.rawType = cast[PNimType](getTypeInfo(x)) + newAny(addr(x), cast[PNimType](getTypeInfo(x))) proc kind*(x: Any): AnyKind {.inline.} = ## get the type kind @@ -345,7 +355,7 @@ proc `[]`*(x: Any, fieldName: string): Any = result.value = x.value +!! n.offset result.rawType = n.typ elif x.rawType.kind == tyObject and x.rawType.base != nil: - return `[]`(Any(value: x.value, rawType: x.rawType.base), fieldName) + return `[]`(newAny(x.value, x.rawType.base), fieldName) else: raise newException(ValueError, "invalid field name: " & fieldName) diff --git a/lib/impure/db_mysql.nim b/lib/impure/db_mysql.nim index 7f7511264..1b7f1de61 100644 --- a/lib/impure/db_mysql.nim +++ b/lib/impure/db_mysql.nim @@ -1,7 +1,7 @@ # # # Nim's Runtime Library -# (c) Copyright 2012 Andreas Rumpf +# (c) Copyright 2015 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -10,7 +10,49 @@ ## A higher level `mySQL`:idx: database wrapper. The same interface is ## implemented for other databases too. ## -## Example: +## See also: `db_odbc <db_odbc.html>`_, `db_sqlite <db_sqlite.html>`_, +## `db_postgres <db_postgres.html>`_. +## +## Parameter substitution +## ---------------------- +## +## All ``db_*`` modules support the same form of parameter substitution. +## That is, using the ``?`` (question mark) to signify the place where a +## value should be placed. For example: +## +## .. code-block:: Nim +## sql"INSERT INTO myTable (colA, colB, colC) VALUES (?, ?, ?)" +## +## +## Examples +## -------- +## +## Opening a connection to a database +## ================================== +## +## .. code-block:: Nim +## import db_mysql +## let db = open("localhost", "user", "password", "dbname") +## db.close() +## +## Creating a table +## ================ +## +## .. code-block:: Nim +## db.exec(sql"DROP TABLE IF EXISTS myTable") +## db.exec(sql("""CREATE TABLE myTable ( +## id integer, +## name varchar(50) not null)""")) +## +## Inserting data +## ============== +## +## .. code-block:: Nim +## db.exec(sql"INSERT INTO myTable (id, name) VALUES (0, ?)", +## "Dominik") +## +## Larger example +## ============== ## ## .. code-block:: Nim ## @@ -43,45 +85,26 @@ import strutils, mysql +import db_common +export db_common + type - DbConn* = PMySQL ## encapsulates a database connection + DbConn* = PMySQL ## encapsulates a database connection Row* = seq[string] ## a row of a dataset. NULL database values will be - ## transformed always to the empty string. - InstantRow* = tuple[row: cstringArray, len: int] ## a handle that can be - ## used to get a row's - ## column text on demand - EDb* = object of IOError ## exception that is raised if a database error occurs - - SqlQuery* = distinct string ## an SQL query string - - FDb* = object of IOEffect ## effect that denotes a database operation - FReadDb* = object of FDb ## effect that denotes a read operation - FWriteDb* = object of FDb ## effect that denotes a write operation -{.deprecated: [TRow: Row, TSqlQuery: SqlQuery, TDbConn: DbConn].} - -proc sql*(query: string): SqlQuery {.noSideEffect, inline.} = - ## constructs a SqlQuery from the string `query`. This is supposed to be - ## used as a raw-string-literal modifier: - ## ``sql"update user set counter = counter + 1"`` - ## - ## If assertions are turned off, it does nothing. If assertions are turned - ## on, later versions will check the string for valid syntax. - result = SqlQuery(query) + ## converted to nil. + InstantRow* = object ## a handle that can be used to get a row's + ## column text on demand + row: cstringArray + len: int +{.deprecated: [TRow: Row, TDbConn: DbConn].} -proc dbError(db: DbConn) {.noreturn.} = - ## raises an EDb exception. - var e: ref EDb +proc dbError*(db: DbConn) {.noreturn.} = + ## raises a DbError exception. + var e: ref DbError new(e) e.msg = $mysql.error(db) raise e -proc dbError*(msg: string) {.noreturn.} = - ## raises an EDb exception with message `msg`. - var e: ref EDb - new(e) - e.msg = msg - raise e - when false: proc dbQueryOpt*(db: DbConn, query: string, args: varargs[string, `$`]) = var stmt = mysql_stmt_init(db) @@ -114,7 +137,7 @@ proc dbFormat(formatstr: SqlQuery, args: varargs[string]): string = add(result, c) proc tryExec*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): bool {. - tags: [FReadDB, FWriteDb].} = + tags: [ReadDbEffect, WriteDbEffect].} = ## tries to execute the query and returns true if successful, false otherwise. var q = dbFormat(query, args) return mysql.realQuery(db, q, q.len) == 0'i32 @@ -124,7 +147,7 @@ proc rawExec(db: DbConn, query: SqlQuery, args: varargs[string, `$`]) = if mysql.realQuery(db, q, q.len) != 0'i32: dbError(db) proc exec*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]) {. - tags: [FReadDB, FWriteDb].} = + tags: [ReadDbEffect, WriteDbEffect].} = ## executes the query and raises EDB if not successful. var q = dbFormat(query, args) if mysql.realQuery(db, q, q.len) != 0'i32: dbError(db) @@ -139,7 +162,7 @@ proc properFreeResult(sqlres: mysql.PRES, row: cstringArray) = mysql.freeResult(sqlres) iterator fastRows*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): Row {.tags: [FReadDB].} = + args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} = ## executes the query and iterates over the result dataset. ## ## This is very fast, but potentially dangerous. Use this iterator only @@ -167,9 +190,9 @@ iterator fastRows*(db: DbConn, query: SqlQuery, iterator instantRows*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): InstantRow - {.tags: [FReadDb].} = - ## same as fastRows but returns a handle that can be used to get column text - ## on demand using []. Returned handle is valid only within the interator body. + {.tags: [ReadDbEffect].} = + ## Same as fastRows but returns a handle that can be used to get column text + ## on demand using []. Returned handle is valid only within the iterator body. rawExec(db, query, args) var sqlres = mysql.useResult(db) if sqlres != nil: @@ -178,20 +201,102 @@ iterator instantRows*(db: DbConn, query: SqlQuery, while true: row = mysql.fetchRow(sqlres) if row == nil: break - yield (row: row, len: L) + yield InstantRow(row: row, len: L) properFreeResult(sqlres, row) +proc setTypeName(t: var DbType; f: PFIELD) = + shallowCopy(t.name, $f.name) + t.maxReprLen = Natural(f.max_length) + if (NOT_NULL_FLAG and f.flags) != 0: t.notNull = true + case f.ftype + of TYPE_DECIMAL: + t.kind = dbDecimal + of TYPE_TINY: + t.kind = dbInt + t.size = 1 + of TYPE_SHORT: + t.kind = dbInt + t.size = 2 + of TYPE_LONG: + t.kind = dbInt + t.size = 4 + of TYPE_FLOAT: + t.kind = dbFloat + t.size = 4 + of TYPE_DOUBLE: + t.kind = dbFloat + t.size = 8 + of TYPE_NULL: + t.kind = dbNull + of TYPE_TIMESTAMP: + t.kind = dbTimestamp + of TYPE_LONGLONG: + t.kind = dbInt + t.size = 8 + of TYPE_INT24: + t.kind = dbInt + t.size = 3 + of TYPE_DATE: + t.kind = dbDate + of TYPE_TIME: + t.kind = dbTime + of TYPE_DATETIME: + t.kind = dbDatetime + of TYPE_YEAR: + t.kind = dbDate + of TYPE_NEWDATE: + t.kind = dbDate + of TYPE_VARCHAR, TYPE_VAR_STRING, TYPE_STRING: + t.kind = dbVarchar + of TYPE_BIT: + t.kind = dbBit + of TYPE_NEWDECIMAL: + t.kind = dbDecimal + of TYPE_ENUM: t.kind = dbEnum + of TYPE_SET: t.kind = dbSet + of TYPE_TINY_BLOB, TYPE_MEDIUM_BLOB, TYPE_LONG_BLOB, + TYPE_BLOB: t.kind = dbBlob + of TYPE_GEOMETRY: + t.kind = dbGeometry + +proc setColumnInfo(columns: var DbColumns; res: PRES; L: int) = + setLen(columns, L) + for i in 0..<L: + let fp = mysql.fetch_field_direct(res, cint(i)) + setTypeName(columns[i].typ, fp) + columns[i].name = $fp.name + columns[i].tableName = $fp.table + columns[i].primaryKey = (fp.flags and PRI_KEY_FLAG) != 0 + #columns[i].foreignKey = there is no such thing in mysql + +iterator instantRows*(db: DbConn; columns: var DbColumns; query: SqlQuery; + args: varargs[string, `$`]): InstantRow = + ## Same as fastRows but returns a handle that can be used to get column text + ## on demand using []. Returned handle is valid only within the iterator body. + rawExec(db, query, args) + var sqlres = mysql.useResult(db) + if sqlres != nil: + let L = int(mysql.numFields(sqlres)) + setColumnInfo(columns, sqlres, L) + var row: cstringArray + while true: + row = mysql.fetchRow(sqlres) + if row == nil: break + yield InstantRow(row: row, len: L) + properFreeResult(sqlres, row) + + proc `[]`*(row: InstantRow, col: int): string {.inline.} = - ## returns text for given column of the row + ## Returns text for given column of the row. $row.row[col] proc len*(row: InstantRow): int {.inline.} = - ## returns number of columns in the row + ## Returns number of columns in the row. row.len proc getRow*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): Row {.tags: [FReadDB].} = - ## retrieves a single row. If the query doesn't return any rows, this proc + args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} = + ## Retrieves a single row. If the query doesn't return any rows, this proc ## will return a Row with empty strings for each column. rawExec(db, query, args) var sqlres = mysql.useResult(db) @@ -209,7 +314,7 @@ proc getRow*(db: DbConn, query: SqlQuery, properFreeResult(sqlres, row) proc getAllRows*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): seq[Row] {.tags: [FReadDB].} = + args: varargs[string, `$`]): seq[Row] {.tags: [ReadDbEffect].} = ## executes the query and returns the whole result dataset. result = @[] rawExec(db, query, args) @@ -232,19 +337,19 @@ proc getAllRows*(db: DbConn, query: SqlQuery, mysql.freeResult(sqlres) iterator rows*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): Row {.tags: [FReadDB].} = + args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} = ## same as `fastRows`, but slower and safe. for r in items(getAllRows(db, query, args)): yield r proc getValue*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): string {.tags: [FReadDB].} = + args: varargs[string, `$`]): string {.tags: [ReadDbEffect].} = ## executes the query and returns the first column of the first row of the ## result dataset. Returns "" if the dataset contains no rows or the database ## value is NULL. result = getRow(db, query, args)[0] proc tryInsertId*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): int64 {.tags: [FWriteDb].} = + args: varargs[string, `$`]): int64 {.tags: [WriteDbEffect].} = ## executes the query (typically "INSERT") and returns the ## generated ID for the row or -1 in case of an error. var q = dbFormat(query, args) @@ -254,7 +359,7 @@ proc tryInsertId*(db: DbConn, query: SqlQuery, result = mysql.insertId(db) proc insertId*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): int64 {.tags: [FWriteDb].} = + args: varargs[string, `$`]): int64 {.tags: [WriteDbEffect].} = ## executes the query (typically "INSERT") and returns the ## generated ID for the row. result = tryInsertID(db, query, args) @@ -262,18 +367,18 @@ proc insertId*(db: DbConn, query: SqlQuery, proc execAffectedRows*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): int64 {. - tags: [FReadDB, FWriteDb].} = + tags: [ReadDbEffect, WriteDbEffect].} = ## runs the query (typically "UPDATE") and returns the ## number of affected rows rawExec(db, query, args) result = mysql.affectedRows(db) -proc close*(db: DbConn) {.tags: [FDb].} = +proc close*(db: DbConn) {.tags: [DbEffect].} = ## closes the database connection. if db != nil: mysql.close(db) proc open*(connection, user, password, database: string): DbConn {. - tags: [FDb].} = + tags: [DbEffect].} = ## opens a database connection. Raises `EDb` if the connection could not ## be established. result = mysql.init(nil) @@ -291,7 +396,7 @@ proc open*(connection, user, password, database: string): DbConn {. dbError(errmsg) proc setEncoding*(connection: DbConn, encoding: string): bool {. - tags: [FDb].} = + tags: [DbEffect].} = ## sets the encoding of a database connection, returns true for ## success, false for failure. result = mysql.set_character_set(connection, encoding) == 0 diff --git a/lib/impure/db_odbc.nim b/lib/impure/db_odbc.nim new file mode 100644 index 000000000..4f0b0469d --- /dev/null +++ b/lib/impure/db_odbc.nim @@ -0,0 +1,505 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2015 Nim Contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## A higher level `ODBC` database wrapper. +## +## This is the same interface that is implemented for other databases. +## +## This has NOT yet been (extensively) tested against ODBC drivers for +## Teradata, Oracle, Sybase, MSSqlvSvr, et. al. databases. +## +## Currently all queries are ANSI calls, not Unicode. +## +## See also: `db_postgres <db_postgres.html>`_, `db_sqlite <db_sqlite.html>`_, +## `db_mysql <db_mysql.html>`_. +## +## Parameter substitution +## ---------------------- +## +## All ``db_*`` modules support the same form of parameter substitution. +## That is, using the ``?`` (question mark) to signify the place where a +## value should be placed. For example: +## +## .. code-block:: Nim +## sql"INSERT INTO myTable (colA, colB, colC) VALUES (?, ?, ?)" +## +## +## Examples +## -------- +## +## Opening a connection to a database +## ================================== +## +## .. code-block:: Nim +## import db_odbc +## let db = open("localhost", "user", "password", "dbname") +## db.close() +## +## Creating a table +## ================ +## +## .. code-block:: Nim +## db.exec(sql"DROP TABLE IF EXISTS myTable") +## db.exec(sql("""CREATE TABLE myTable ( +## id integer, +## name varchar(50) not null)""")) +## +## Inserting data +## ============== +## +## .. code-block:: Nim +## db.exec(sql"INSERT INTO myTable (id, name) VALUES (0, ?)", +## "Andreas") +## +## Large example +## ============= +## +## .. code-block:: Nim +## +## import db_odbc, math +## +## let theDb = open("localhost", "nim", "nim", "test") +## +## theDb.exec(sql"Drop table if exists myTestTbl") +## theDb.exec(sql("create table myTestTbl (" & +## " Id INT(11) NOT NULL AUTO_INCREMENT PRIMARY KEY, " & +## " Name VARCHAR(50) NOT NULL, " & +## " i INT(11), " & +## " f DECIMAL(18,10))")) +## +## theDb.exec(sql"START TRANSACTION") +## for i in 1..1000: +## theDb.exec(sql"INSERT INTO myTestTbl (name,i,f) VALUES (?,?,?)", +## "Item#" & $i, i, sqrt(i.float)) +## theDb.exec(sql"COMMIT") +## +## for x in theDb.fastRows(sql"select * from myTestTbl"): +## echo x +## +## let id = theDb.tryInsertId(sql"INSERT INTO myTestTbl (name,i,f) VALUES (?,?,?)", +## "Item#1001", 1001, sqrt(1001.0)) +## echo "Inserted item: ", theDb.getValue(sql"SELECT name FROM myTestTbl WHERE id=?", id) +## +## theDb.close() + + +import strutils, odbcsql + +import db_common +export db_common + +type + OdbcConnTyp = tuple[hDb: SqlHDBC, env: SqlHEnv, stmt: SqlHStmt] + DbConn* = OdbcConnTyp ## encapsulates a database connection + Row* = seq[string] ## a row of a dataset. NULL database values will be + ## converted to nil. + InstantRow* = tuple[row: seq[string], len: int] ## a handle that can be + ## used to get a row's + ## column text on demand + +{.deprecated: [TRow: Row, TSqlQuery: SqlQuery, TDbConn: DbConn].} + +var + buf: array[0..4096, char] + +proc properFreeResult(hType: int, sqlres: var SqlHandle) {. + tags: [WriteDbEffect], raises: [].} = + try: + discard SQLFreeHandle(hType.TSqlSmallInt, sqlres) + sqlres = nil + except: discard + +proc getErrInfo(db: var DbConn): tuple[res: int, ss, ne, msg: string] {. + tags: [ReadDbEffect], raises: [].} = + ## Returns ODBC error information + var + sqlState: array[0..512, char] + nativeErr: array[0..512, char] + errMsg: array[0..512, char] + retSz: TSqlSmallInt = 0 + res: TSqlSmallInt = 0 + try: + sqlState[0] = '\0' + nativeErr[0] = '\0' + errMsg[0] = '\0' + res = SQLErr(db.env, db.hDb, db.stmt, + cast[PSQLCHAR](sqlState.addr), + cast[PSQLCHAR](nativeErr.addr), + cast[PSQLCHAR](errMsg.addr), + 511.TSqlSmallInt, retSz.addr.PSQLSMALLINT) + except: + discard + return (res.int, $sqlState, $nativeErr, $errMsg) + +proc dbError*(db: var DbConn) {. + tags: [ReadDbEffect, WriteDbEffect], raises: [DbError] .} = + ## Raises an `[DbError]` exception with ODBC error information + var + e: ref DbError + ss, ne, msg: string = "" + isAnError = false + res: int = 0 + prevSs = "" + while true: + prevSs = ss + (res, ss, ne, msg) = db.getErrInfo() + if prevSs == ss: + break + # sqlState of 00000 is not an error + elif ss == "00000": + break + elif ss == "01000": + echo "\nWarning: ", ss, " ", msg + continue + else: + isAnError = true + echo "\nError: ", ss, " ", msg + if isAnError: + new(e) + e.msg = "ODBC Error" + if db.stmt != nil: + properFreeResult(SQL_HANDLE_STMT, db.stmt) + properFreeResult(SQL_HANDLE_DBC, db.hDb) + properFreeResult(SQL_HANDLE_ENV, db.env) + raise e + +proc SqlCheck(db: var DbConn, resVal: TSqlSmallInt) {.raises: [DbError]} = + ## Wrapper that checks if ``resVal`` is not SQL_SUCCESS and if so, raises [EDb] + if resVal != SQL_SUCCESS: dbError(db) + +proc SqlGetDBMS(db: var DbConn): string {. + tags: [ReadDbEffect, WriteDbEffect], raises: [] .} = + ## Returns the ODBC SQL_DBMS_NAME string + const + SQL_DBMS_NAME = 17.SqlUSmallInt + var + sz: TSqlSmallInt = 0 + buf[0] = '\0' + try: + db.SqlCheck(SQLGetInfo(db.hDb, SQL_DBMS_NAME, cast[SqlPointer](buf.addr), + 4095.TSqlSmallInt, sz.addr)) + except: discard + return $buf.cstring + +proc dbQuote*(s: string): string {.noSideEffect.} = + ## DB quotes the string. + result = "'" + for c in items(s): + if c == '\'': add(result, "''") + else: add(result, c) + add(result, '\'') + +proc dbFormat(formatstr: SqlQuery, args: varargs[string]): string {. + noSideEffect.} = + ## Replace any ``?`` placeholders with `args`, + ## and quotes the arguments + result = "" + var a = 0 + for c in items(string(formatstr)): + if c == '?': + if args[a] == nil: + add(result, "NULL") + else: + add(result, dbQuote(args[a])) + inc(a) + else: + add(result, c) + +proc prepareFetch(db: var DbConn, query: SqlQuery, + args: varargs[string, `$`]) {. + tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} = + # Prepare a statement, execute it and fetch the data to the driver + # ready for retrieval of the data + # Used internally by iterators and retrieval procs + # requires calling + # properFreeResult(SQL_HANDLE_STMT, db.stmt) + # when finished + db.SqlCheck(SQLAllocHandle(SQL_HANDLE_STMT, db.hDb, db.stmt)) + var q = dbFormat(query, args) + db.SqlCheck(SQLPrepare(db.stmt, q.PSQLCHAR, q.len.TSqlSmallInt)) + db.SqlCheck(SQLExecute(db.stmt)) + db.SqlCheck(SQLFetch(db.stmt)) + +proc prepareFetchDirect(db: var DbConn, query: SqlQuery, + args: varargs[string, `$`]) {. + tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} = + # Prepare a statement, execute it and fetch the data to the driver + # ready for retrieval of the data + # Used internally by iterators and retrieval procs + # requires calling + # properFreeResult(SQL_HANDLE_STMT, db.stmt) + # when finished + db.SqlCheck(SQLAllocHandle(SQL_HANDLE_STMT, db.hDb, db.stmt)) + var q = dbFormat(query, args) + db.SqlCheck(SQLExecDirect(db.stmt, q.PSQLCHAR, q.len.TSqlSmallInt)) + db.SqlCheck(SQLFetch(db.stmt)) + +proc tryExec*(db: var DbConn, query: SqlQuery, args: varargs[string, `$`]): bool {. + tags: [ReadDbEffect, WriteDbEffect], raises: [].} = + ## Tries to execute the query and returns true if successful, false otherwise. + var + res:TSqlSmallInt = -1 + try: + db.prepareFetchDirect(query, args) + var + rCnt = -1 + res = SQLRowCount(db.stmt, rCnt) + if res != SQL_SUCCESS: dbError(db) + properFreeResult(SQL_HANDLE_STMT, db.stmt) + except: discard + return res == SQL_SUCCESS + +proc rawExec(db: var DbConn, query: SqlQuery, args: varargs[string, `$`]) {. + tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} = + db.prepareFetchDirect(query, args) + +proc exec*(db: var DbConn, query: SqlQuery, args: varargs[string, `$`]) {. + tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} = + ## Executes the query and raises EDB if not successful. + db.prepareFetchDirect(query, args) + properFreeResult(SQL_HANDLE_STMT, db.stmt) + +proc newRow(L: int): Row {.noSideEFfect.} = + newSeq(result, L) + for i in 0..L-1: result[i] = "" + +iterator fastRows*(db: var DbConn, query: SqlQuery, + args: varargs[string, `$`]): Row {. + tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} = + ## Executes the query and iterates over the result dataset. + ## + ## This is very fast, but potentially dangerous. Use this iterator only + ## if you require **ALL** the rows. + ## + ## Breaking the fastRows() iterator during a loop may cause a driver error + ## for subsequenct queries + ## + ## Rows are retrieved from the server at each iteration. + var + rowRes: Row + sz: TSqlSmallInt = 0 + cCnt: TSqlSmallInt = 0.TSqlSmallInt + rCnt = -1 + + db.prepareFetch(query, args) + db.SqlCheck(SQLNumResultCols(db.stmt, cCnt)) + db.SqlCheck(SQLRowCount(db.stmt, rCnt)) + rowRes = newRow(cCnt) + for rNr in 1..rCnt: + for colId in 1..cCnt: + buf[0] = '\0' + db.SqlCheck(SQLGetData(db.stmt, colId.SqlUSmallInt, SQL_C_CHAR, + cast[cstring](buf.addr), 4095.TSqlSmallInt, sz.addr)) + rowRes[colId-1] = $buf.cstring + db.SqlCheck(SQLFetchScroll(db.stmt, SQL_FETCH_NEXT, 1)) + yield rowRes + properFreeResult(SQL_HANDLE_STMT, db.stmt) + +iterator instantRows*(db: var DbConn, query: SqlQuery, + args: varargs[string, `$`]): InstantRow + {.tags: [ReadDbEffect, WriteDbEffect].} = + ## Same as fastRows but returns a handle that can be used to get column text + ## on demand using []. Returned handle is valid only within the interator body. + var + rowRes: Row + sz: TSqlSmallInt = 0 + cCnt: TSqlSmallInt = 0.TSqlSmallInt + rCnt = -1 + db.prepareFetch(query, args) + db.SqlCheck(SQLNumResultCols(db.stmt, cCnt)) + db.SqlCheck(SQLRowCount(db.stmt, rCnt)) + rowRes = newRow(cCnt) + for rNr in 1..rCnt: + for colId in 1..cCnt: + buf[0] = '\0' + db.SqlCheck(SQLGetData(db.stmt, colId.SqlUSmallInt, SQL_C_CHAR, + cast[cstring](buf.addr), 4095.TSqlSmallInt, sz.addr)) + rowRes[colId-1] = $buf.cstring + db.SqlCheck(SQLFetchScroll(db.stmt, SQL_FETCH_NEXT, 1)) + yield (row: rowRes, len: cCnt.int) + properFreeResult(SQL_HANDLE_STMT, db.stmt) + +proc `[]`*(row: InstantRow, col: int): string {.inline.} = + ## Returns text for given column of the row + row.row[col] + +proc len*(row: InstantRow): int {.inline.} = + ## Returns number of columns in the row + row.len + +proc getRow*(db: var DbConn, query: SqlQuery, + args: varargs[string, `$`]): Row {. + tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} = + ## Retrieves a single row. If the query doesn't return any rows, this proc + ## will return a Row with empty strings for each column. + var + sz: TSqlSmallInt = 0.TSqlSmallInt + cCnt: TSqlSmallInt = 0.TSqlSmallInt + rCnt = -1 + result = @[] + db.prepareFetch(query, args) + db.SqlCheck(SQLNumResultCols(db.stmt, cCnt)) + + db.SqlCheck(SQLRowCount(db.stmt, rCnt)) + for colId in 1..cCnt: + db.SqlCheck(SQLGetData(db.stmt, colId.SqlUSmallInt, SQL_C_CHAR, + cast[cstring](buf.addr), 4095.TSqlSmallInt, sz.addr)) + result.add($buf.cstring) + db.SqlCheck(SQLFetchScroll(db.stmt, SQL_FETCH_NEXT, 1)) + properFreeResult(SQL_HANDLE_STMT, db.stmt) + +proc getAllRows*(db: var DbConn, query: SqlQuery, + args: varargs[string, `$`]): seq[Row] {. + tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} = + ## Executes the query and returns the whole result dataset. + var + rowRes: Row + sz: TSqlSmallInt = 0 + cCnt: TSqlSmallInt = 0.TSqlSmallInt + rCnt = -1 + db.prepareFetch(query, args) + db.SqlCheck(SQLNumResultCols(db.stmt, cCnt)) + db.SqlCheck(SQLRowCount(db.stmt, rCnt)) + result = @[] + for rNr in 1..rCnt: + rowRes = @[] + buf[0] = '\0' + for colId in 1..cCnt: + db.SqlCheck(SQLGetData(db.stmt, colId.SqlUSmallInt, SQL_C_CHAR, + cast[SqlPointer](buf.addr), 4095.TSqlSmallInt, sz.addr)) + rowRes.add($buf.cstring) + db.SqlCheck(SQLFetchScroll(db.stmt, SQL_FETCH_NEXT, 1)) + result.add(rowRes) + properFreeResult(SQL_HANDLE_STMT, db.stmt) + +iterator rows*(db: var DbConn, query: SqlQuery, + args: varargs[string, `$`]): Row {. + tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} = + ## Same as `fastRows`, but slower and safe. + ## + ## This retrieves ALL rows into memory before + ## iterating through the rows. + ## Large dataset queries will impact on memory usage. + for r in items(getAllRows(db, query, args)): yield r + +proc getValue*(db: var DbConn, query: SqlQuery, + args: varargs[string, `$`]): string {. + tags: [ReadDbEffect, WriteDbEffect], raises: [].} = + ## Executes the query and returns the first column of the first row of the + ## result dataset. Returns "" if the dataset contains no rows or the database + ## value is NULL. + result = "" + try: + result = getRow(db, query, args)[0] + except: discard + +proc tryInsertId*(db: var DbConn, query: SqlQuery, + args: varargs[string, `$`]): int64 {. + tags: [ReadDbEffect, WriteDbEffect], raises: [].} = + ## Executes the query (typically "INSERT") and returns the + ## generated ID for the row or -1 in case of an error. + if not tryExec(db, query, args): + result = -1'i64 + else: + echo "DBMS: ",SqlGetDBMS(db).toLower() + result = -1'i64 + try: + case SqlGetDBMS(db).toLower(): + of "postgresql": + result = getValue(db, sql"SELECT LASTVAL();", []).parseInt + of "mysql": + result = getValue(db, sql"SELECT LAST_INSERT_ID();", []).parseInt + of "sqlite": + result = getValue(db, sql"SELECT LAST_INSERT_ROWID();", []).parseInt + of "microsoft sql server": + result = getValue(db, sql"SELECT SCOPE_IDENTITY();", []).parseInt + of "oracle": + result = getValue(db, sql"SELECT id.currval FROM DUAL;", []).parseInt + else: result = -1'i64 + except: discard + +proc insertId*(db: var DbConn, query: SqlQuery, + args: varargs[string, `$`]): int64 {. + tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} = + ## Executes the query (typically "INSERT") and returns the + ## generated ID for the row. + result = tryInsertID(db, query, args) + if result < 0: dbError(db) + +proc execAffectedRows*(db: var DbConn, query: SqlQuery, + args: varargs[string, `$`]): int64 {. + tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} = + ## Runs the query (typically "UPDATE") and returns the + ## number of affected rows + result = -1 + var res = SQLAllocHandle(SQL_HANDLE_STMT, db.hDb, db.stmt.SqlHandle) + if res != SQL_SUCCESS: dbError(db) + var q = dbFormat(query, args) + res = SQLPrepare(db.stmt, q.PSQLCHAR, q.len.TSqlSmallInt) + if res != SQL_SUCCESS: dbError(db) + rawExec(db, query, args) + var rCnt = -1 + result = SQLRowCount(db.hDb, rCnt) + if res != SQL_SUCCESS: dbError(db) + properFreeResult(SQL_HANDLE_STMT, db.stmt) + result = rCnt + +proc close*(db: var DbConn) {. + tags: [WriteDbEffect], raises: [].} = + ## Closes the database connection. + if db.hDb != nil: + try: + var res = SQLDisconnect(db.hDb) + if db.stmt != nil: + res = SQLFreeHandle(SQL_HANDLE_STMT, db.stmt) + res = SQLFreeHandle(SQL_HANDLE_DBC, db.hDb) + res = SQLFreeHandle(SQL_HANDLE_ENV, db.env) + db = (hDb: nil, env: nil, stmt: nil) + except: + discard + +proc open*(connection, user, password, database: string): DbConn {. + tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} = + ## Opens a database connection. + ## + ## Raises `EDb` if the connection could not be established. + ## + ## Currently the database parameter is ignored, + ## but included to match ``open()`` in the other db_xxxxx library modules. + var + val: TSqlInteger = SQL_OV_ODBC3 + resLen = 0 + result = (hDb: nil, env: nil, stmt: nil) + # allocate environment handle + var res = SQLAllocHandle(SQL_HANDLE_ENV, result.env, result.env) + if res != SQL_SUCCESS: dbError("Error: unable to initialise ODBC environment.") + res = SQLSetEnvAttr(result.env, + SQL_ATTR_ODBC_VERSION.TSqlInteger, + val, resLen.TSqlInteger) + if res != SQL_SUCCESS: dbError("Error: unable to set ODBC driver version.") + # allocate hDb handle + res = SQLAllocHandle(SQL_HANDLE_DBC, result.env, result.hDb) + if res != SQL_SUCCESS: dbError("Error: unable to allocate connection handle.") + + # Connect: connection = dsn str, + res = SQLConnect(result.hDb, + connection.PSQLCHAR , connection.len.TSqlSmallInt, + user.PSQLCHAR, user.len.TSqlSmallInt, + password.PSQLCHAR, password.len.TSqlSmallInt) + if res != SQL_SUCCESS: + result.dbError() + +proc setEncoding*(connection: DbConn, encoding: string): bool {. + tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} = + ## Currently not implemented for ODBC. + ## + ## Sets the encoding of a database connection, returns true for + ## success, false for failure. + #result = set_character_set(connection, encoding) == 0 + dbError("setEncoding() is currently not implemented by the db_odbc module") diff --git a/lib/impure/db_postgres.nim b/lib/impure/db_postgres.nim index 7e6219465..60bd1f081 100644 --- a/lib/impure/db_postgres.nim +++ b/lib/impure/db_postgres.nim @@ -10,6 +10,9 @@ ## A higher level `PostgreSQL`:idx: database wrapper. This interface ## is implemented for other databases also. ## +## See also: `db_odbc <db_odbc.html>`_, `db_sqlite <db_sqlite.html>`_, +## `db_mysql <db_mysql.html>`_. +## ## Parameter substitution ## ---------------------- ## @@ -27,7 +30,7 @@ ## ## 2. ``SqlPrepared`` using ``$1, $2, $3, ...`` ## -## .. code-block:: Nim +## .. code-block:: Nim ## prepare(db, "myExampleInsert", ## sql"""INSERT INTO myTable ## (colA, colB, colC) @@ -62,47 +65,28 @@ ## "Dominik") import strutils, postgres +import db_common +export db_common + type DbConn* = PPGconn ## encapsulates a database connection Row* = seq[string] ## a row of a dataset. NULL database values will be - ## transformed always to the empty string. + ## converted to nil. InstantRow* = tuple[res: PPGresult, line: int32] ## a handle that can be ## used to get a row's ## column text on demand - EDb* = object of IOError ## exception that is raised if a database error occurs - - SqlQuery* = distinct string ## an SQL query string SqlPrepared* = distinct string ## a identifier for the prepared queries - FDb* = object of IOEffect ## effect that denotes a database operation - FReadDb* = object of FDb ## effect that denotes a read operation - FWriteDb* = object of FDb ## effect that denotes a write operation -{.deprecated: [TRow: Row, TSqlQuery: SqlQuery, TDbConn: DbConn, +{.deprecated: [TRow: Row, TDbConn: DbConn, TSqlPrepared: SqlPrepared].} -proc sql*(query: string): SqlQuery {.noSideEffect, inline.} = - ## constructs a SqlQuery from the string `query`. This is supposed to be - ## used as a raw-string-literal modifier: - ## ``sql"update user set counter = counter + 1"`` - ## - ## If assertions are turned off, it does nothing. If assertions are turned - ## on, later versions will check the string for valid syntax. - result = SqlQuery(query) - proc dbError*(db: DbConn) {.noreturn.} = - ## raises an EDb exception. - var e: ref EDb + ## raises a DbError exception. + var e: ref DbError new(e) e.msg = $pqErrorMessage(db) raise e -proc dbError*(msg: string) {.noreturn.} = - ## raises an EDb exception with message `msg`. - var e: ref EDb - new(e) - e.msg = msg - raise e - proc dbQuote*(s: string): string = ## DB quotes the string. result = "'" @@ -127,7 +111,7 @@ proc dbFormat(formatstr: SqlQuery, args: varargs[string]): string = add(result, c) proc tryExec*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): bool {.tags: [FReadDB, FWriteDb].} = + args: varargs[string, `$`]): bool {.tags: [ReadDbEffect, WriteDbEffect].} = ## tries to execute the query and returns true if successful, false otherwise. var res = pqexecParams(db, dbFormat(query, args), 0, nil, nil, nil, nil, 0) @@ -135,7 +119,8 @@ proc tryExec*(db: DbConn, query: SqlQuery, pqclear(res) proc tryExec*(db: DbConn, stmtName: SqlPrepared, - args: varargs[string, `$`]): bool {.tags: [FReadDB, FWriteDb].} = + args: varargs[string, `$`]): bool {.tags: [ + ReadDbEffect, WriteDbEffect].} = ## tries to execute the query and returns true if successful, false otherwise. var arr = allocCStringArray(args) var res = pqexecPrepared(db, stmtName.string, int32(args.len), arr, @@ -145,7 +130,7 @@ proc tryExec*(db: DbConn, stmtName: SqlPrepared, pqclear(res) proc exec*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]) {. - tags: [FReadDB, FWriteDb].} = + tags: [ReadDbEffect, WriteDbEffect].} = ## executes the query and raises EDB if not successful. var res = pqexecParams(db, dbFormat(query, args), 0, nil, nil, nil, nil, 0) @@ -153,7 +138,7 @@ proc exec*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]) {. pqclear(res) proc exec*(db: DbConn, stmtName: SqlPrepared, - args: varargs[string]) {.tags: [FReadDB, FWriteDb].} = + args: varargs[string]) {.tags: [ReadDbEffect, WriteDbEffect].} = var arr = allocCStringArray(args) var res = pqexecPrepared(db, stmtName.string, int32(args.len), arr, nil, nil, 0) @@ -167,11 +152,7 @@ proc newRow(L: int): Row = proc setupQuery(db: DbConn, query: SqlQuery, args: varargs[string]): PPGresult = - # s is a dummy unique id str for each setupQuery query - let s = "setupQuery_Query_" & string(query) - var res = pqprepare(db, s, dbFormat(query, args), 0, nil) - result = pqexecPrepared(db, s, 0, nil, - nil, nil, 0) + result = pqexec(db, dbFormat(query, args)) if pqResultStatus(result) != PGRES_TUPLES_OK: dbError(db) proc setupQuery(db: DbConn, stmtName: SqlPrepared, @@ -184,8 +165,10 @@ proc setupQuery(db: DbConn, stmtName: SqlPrepared, proc prepare*(db: DbConn; stmtName: string, query: SqlQuery; nParams: int): SqlPrepared = + ## Creates a new ``SqlPrepared`` statement. Parameter substitution is done + ## via ``$1``, ``$2``, ``$3``, etc. if nParams > 0 and not string(query).contains("$1"): - dbError("""parameter substitution expects "$1" """) + dbError("parameter substitution expects \"$1\"") var res = pqprepare(db, stmtName, query.string, int32(nParams), nil) if pqResultStatus(res) != PGRES_COMMAND_OK: dbError(db) return SqlPrepared(stmtName) @@ -200,7 +183,7 @@ proc setRow(res: PPGresult, r: var Row, line, cols: int32) = add(r[col], x) iterator fastRows*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): Row {.tags: [FReadDB].} = + args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} = ## executes the query and iterates over the result dataset. This is very ## fast, but potenially dangerous: If the for-loop-body executes another ## query, the results can be undefined. For Postgres it is safe though. @@ -213,7 +196,7 @@ iterator fastRows*(db: DbConn, query: SqlQuery, pqclear(res) iterator fastRows*(db: DbConn, stmtName: SqlPrepared, - args: varargs[string, `$`]): Row {.tags: [FReadDB].} = + args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} = ## executes the prepared query and iterates over the result dataset. var res = setupQuery(db, stmtName, args) var L = pqNfields(res) @@ -225,9 +208,9 @@ iterator fastRows*(db: DbConn, stmtName: SqlPrepared, iterator instantRows*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): InstantRow - {.tags: [FReadDb].} = + {.tags: [ReadDbEffect].} = ## same as fastRows but returns a handle that can be used to get column text - ## on demand using []. Returned handle is valid only within interator body. + ## on demand using []. Returned handle is valid only within iterator body. var res = setupQuery(db, query, args) for i in 0..pqNtuples(res)-1: yield (res: res, line: i) @@ -235,9 +218,9 @@ iterator instantRows*(db: DbConn, query: SqlQuery, iterator instantRows*(db: DbConn, stmtName: SqlPrepared, args: varargs[string, `$`]): InstantRow - {.tags: [FReadDb].} = + {.tags: [ReadDbEffect].} = ## same as fastRows but returns a handle that can be used to get column text - ## on demand using []. Returned handle is valid only within interator body. + ## on demand using []. Returned handle is valid only within iterator body. var res = setupQuery(db, stmtName, args) for i in 0..pqNtuples(res)-1: yield (res: res, line: i) @@ -252,7 +235,7 @@ proc len*(row: InstantRow): int32 {.inline.} = pqNfields(row.res) proc getRow*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): Row {.tags: [FReadDB].} = + args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} = ## retrieves a single row. If the query doesn't return any rows, this proc ## will return a Row with empty strings for each column. var res = setupQuery(db, query, args) @@ -262,7 +245,7 @@ proc getRow*(db: DbConn, query: SqlQuery, pqclear(res) proc getRow*(db: DbConn, stmtName: SqlPrepared, - args: varargs[string, `$`]): Row {.tags: [FReadDB].} = + args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} = var res = setupQuery(db, stmtName, args) var L = pqNfields(res) result = newRow(L) @@ -270,39 +253,52 @@ proc getRow*(db: DbConn, stmtName: SqlPrepared, pqClear(res) proc getAllRows*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): seq[Row] {.tags: [FReadDB].} = + args: varargs[string, `$`]): seq[Row] {. + tags: [ReadDbEffect].} = ## executes the query and returns the whole result dataset. result = @[] for r in fastRows(db, query, args): result.add(r) proc getAllRows*(db: DbConn, stmtName: SqlPrepared, - args: varargs[string, `$`]): seq[Row] {.tags: [FReadDB].} = + args: varargs[string, `$`]): seq[Row] {.tags: + [ReadDbEffect].} = ## executes the prepared query and returns the whole result dataset. result = @[] for r in fastRows(db, stmtName, args): result.add(r) iterator rows*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): Row {.tags: [FReadDB].} = + args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} = ## same as `fastRows`, but slower and safe. for r in items(getAllRows(db, query, args)): yield r iterator rows*(db: DbConn, stmtName: SqlPrepared, - args: varargs[string, `$`]): Row {.tags: [FReadDB].} = + args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} = ## same as `fastRows`, but slower and safe. for r in items(getAllRows(db, stmtName, args)): yield r proc getValue*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): string {.tags: [FReadDB].} = + args: varargs[string, `$`]): string {. + tags: [ReadDbEffect].} = ## executes the query and returns the first column of the first row of the ## result dataset. Returns "" if the dataset contains no rows or the database ## value is NULL. var x = pqgetvalue(setupQuery(db, query, args), 0, 0) result = if isNil(x): "" else: $x +proc getValue*(db: DbConn, stmtName: SqlPrepared, + args: varargs[string, `$`]): string {. + tags: [ReadDbEffect].} = + ## executes the query and returns the first column of the first row of the + ## result dataset. Returns "" if the dataset contains no rows or the database + ## value is NULL. + var x = pqgetvalue(setupQuery(db, stmtName, args), 0, 0) + result = if isNil(x): "" else: $x + proc tryInsertID*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): int64 {.tags: [FWriteDb].}= + args: varargs[string, `$`]): int64 {. + tags: [WriteDbEffect].}= ## executes the query (typically "INSERT") and returns the ## generated ID for the row or -1 in case of an error. For Postgre this adds ## ``RETURNING id`` to the query, so it only works if your primary key is @@ -315,7 +311,8 @@ proc tryInsertID*(db: DbConn, query: SqlQuery, result = -1 proc insertID*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): int64 {.tags: [FWriteDb].} = + args: varargs[string, `$`]): int64 {. + tags: [WriteDbEffect].} = ## executes the query (typically "INSERT") and returns the ## generated ID for the row. For Postgre this adds ## ``RETURNING id`` to the query, so it only works if your primary key is @@ -325,7 +322,7 @@ proc insertID*(db: DbConn, query: SqlQuery, proc execAffectedRows*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): int64 {.tags: [ - FReadDB, FWriteDb].} = + ReadDbEffect, WriteDbEffect].} = ## executes the query (typically "UPDATE") and returns the ## number of affected rows. var q = dbFormat(query, args) @@ -336,7 +333,7 @@ proc execAffectedRows*(db: DbConn, query: SqlQuery, proc execAffectedRows*(db: DbConn, stmtName: SqlPrepared, args: varargs[string, `$`]): int64 {.tags: [ - FReadDB, FWriteDb].} = + ReadDbEffect, WriteDbEffect].} = ## executes the query (typically "UPDATE") and returns the ## number of affected rows. var arr = allocCStringArray(args) @@ -347,12 +344,12 @@ proc execAffectedRows*(db: DbConn, stmtName: SqlPrepared, result = parseBiggestInt($pqcmdTuples(res)) pqclear(res) -proc close*(db: DbConn) {.tags: [FDb].} = +proc close*(db: DbConn) {.tags: [DbEffect].} = ## closes the database connection. if db != nil: pqfinish(db) proc open*(connection, user, password, database: string): DbConn {. - tags: [FDb].} = + tags: [DbEffect].} = ## opens a database connection. Raises `EDb` if the connection could not ## be established. ## @@ -374,10 +371,10 @@ proc open*(connection, user, password, database: string): DbConn {. if pqStatus(result) != CONNECTION_OK: dbError(result) # result = nil proc setEncoding*(connection: DbConn, encoding: string): bool {. - tags: [FDb].} = + tags: [DbEffect].} = ## sets the encoding of a database connection, returns true for ## success, false for failure. return pqsetClientEncoding(connection, encoding) == 0 -# Tests are in ../../tests/untestable/tpostgres. \ No newline at end of file +# Tests are in ../../tests/untestable/tpostgres. diff --git a/lib/impure/db_sqlite.nim b/lib/impure/db_sqlite.nim index 8366fdadc..1633d48f7 100644 --- a/lib/impure/db_sqlite.nim +++ b/lib/impure/db_sqlite.nim @@ -1,7 +1,7 @@ # # # Nim's Runtime Library -# (c) Copyright 2012 Andreas Rumpf +# (c) Copyright 2015 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -10,7 +10,48 @@ ## A higher level `SQLite`:idx: database wrapper. This interface ## is implemented for other databases too. ## -## Example: +## See also: `db_odbc <db_odbc.html>`_, `db_postgres <db_postgres.html>`_, +## `db_mysql <db_mysql.html>`_. +## +## Parameter substitution +## ---------------------- +## +## All ``db_*`` modules support the same form of parameter substitution. +## That is, using the ``?`` (question mark) to signify the place where a +## value should be placed. For example: +## +## .. code-block:: Nim +## sql"INSERT INTO myTable (colA, colB, colC) VALUES (?, ?, ?)" +## +## Examples +## -------- +## +## Opening a connection to a database +## ================================== +## +## .. code-block:: Nim +## import db_sqlite +## let db = open("localhost", "user", "password", "dbname") +## db.close() +## +## Creating a table +## ================ +## +## .. code-block:: Nim +## db.exec(sql"DROP TABLE IF EXISTS myTable") +## db.exec(sql("""CREATE TABLE myTable ( +## id integer, +## name varchar(50) not null)""")) +## +## Inserting data +## ============== +## +## .. code-block:: Nim +## db.exec(sql"INSERT INTO myTable (id, name) VALUES (0, ?)", +## "Jack") +## +## Larger example +## ============== ## ## .. code-block:: nim ## @@ -40,47 +81,30 @@ ## ## theDb.close() +{.deadCodeElim:on.} + import strutils, sqlite3 +import db_common +export db_common + type DbConn* = PSqlite3 ## encapsulates a database connection Row* = seq[string] ## a row of a dataset. NULL database values will be - ## transformed always to the empty string. + ## converted to nil. InstantRow* = Pstmt ## a handle that can be used to get a row's column ## text on demand - EDb* = object of IOError ## exception that is raised if a database error occurs - - SqlQuery* = distinct string ## an SQL query string - - FDb* = object of IOEffect ## effect that denotes a database operation - FReadDb* = object of FDb ## effect that denotes a read operation - FWriteDb* = object of FDb ## effect that denotes a write operation -{.deprecated: [TRow: Row, TSqlQuery: SqlQuery, TDbConn: DbConn].} +{.deprecated: [TRow: Row, TDbConn: DbConn].} -proc sql*(query: string): SqlQuery {.noSideEffect, inline.} = - ## constructs a SqlQuery from the string `query`. This is supposed to be - ## used as a raw-string-literal modifier: - ## ``sql"update user set counter = counter + 1"`` - ## - ## If assertions are turned off, it does nothing. If assertions are turned - ## on, later versions will check the string for valid syntax. - result = SqlQuery(query) - -proc dbError(db: DbConn) {.noreturn.} = - ## raises an EDb exception. - var e: ref EDb +proc dbError*(db: DbConn) {.noreturn.} = + ## raises a DbError exception. + var e: ref DbError new(e) e.msg = $sqlite3.errmsg(db) raise e -proc dbError*(msg: string) {.noreturn.} = - ## raises an EDb exception with message `msg`. - var e: ref EDb - new(e) - e.msg = msg - raise e - -proc dbQuote(s: string): string = +proc dbQuote*(s: string): string = + ## DB quotes the string. if s.isNil: return "NULL" result = "'" for c in items(s): @@ -99,7 +123,8 @@ proc dbFormat(formatstr: SqlQuery, args: varargs[string]): string = add(result, c) proc tryExec*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): bool {.tags: [FReadDb, FWriteDb].} = + args: varargs[string, `$`]): bool {. + tags: [ReadDbEffect, WriteDbEffect].} = ## tries to execute the query and returns true if successful, false otherwise. var q = dbFormat(query, args) var stmt: sqlite3.Pstmt @@ -108,8 +133,8 @@ proc tryExec*(db: DbConn, query: SqlQuery, result = finalize(stmt) == SQLITE_OK proc exec*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]) {. - tags: [FReadDb, FWriteDb].} = - ## executes the query and raises EDB if not successful. + tags: [ReadDbEffect, WriteDbEffect].} = + ## executes the query and raises DbError if not successful. if not tryExec(db, query, args): dbError(db) proc newRow(L: int): Row = @@ -129,14 +154,14 @@ proc setRow(stmt: Pstmt, r: var Row, cols: cint) = if not isNil(x): add(r[col], x) iterator fastRows*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): Row {.tags: [FReadDb].} = + args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} = ## Executes the query and iterates over the result dataset. ## ## This is very fast, but potentially dangerous. Use this iterator only ## if you require **ALL** the rows. ## ## Breaking the fastRows() iterator during a loop will cause the next - ## database query to raise an [EDb] exception ``unable to close due to ...``. + ## database query to raise a DbError exception ``unable to close due to ...``. var stmt = setupQuery(db, query, args) var L = (column_count(stmt)) var result = newRow(L) @@ -147,10 +172,43 @@ iterator fastRows*(db: DbConn, query: SqlQuery, iterator instantRows*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): InstantRow - {.tags: [FReadDb].} = + {.tags: [ReadDbEffect].} = + ## same as fastRows but returns a handle that can be used to get column text + ## on demand using []. Returned handle is valid only within the iterator body. + var stmt = setupQuery(db, query, args) + while step(stmt) == SQLITE_ROW: + yield stmt + if finalize(stmt) != SQLITE_OK: dbError(db) + +proc toTypeKind(t: var DbType; x: int32) = + case x + of SQLITE_INTEGER: + t.kind = dbInt + t.size = 8 + of SQLITE_FLOAT: + t.kind = dbFloat + t.size = 8 + of SQLITE_BLOB: t.kind = dbBlob + of SQLITE_NULL: t.kind = dbNull + of SQLITE_TEXT: t.kind = dbVarchar + else: t.kind = dbUnknown + +proc setColumns(columns: var DbColumns; x: PStmt) = + let L = column_count(x) + setLen(columns, L) + for i in 0'i32 ..< L: + columns[i].name = $column_name(x, i) + columns[i].typ.name = $column_decltype(x, i) + toTypeKind(columns[i].typ, column_type(x, i)) + columns[i].tableName = $column_table_name(x, i) + +iterator instantRows*(db: DbConn; columns: var DbColumns; query: SqlQuery, + args: varargs[string, `$`]): InstantRow + {.tags: [ReadDbEffect].} = ## same as fastRows but returns a handle that can be used to get column text - ## on demand using []. Returned handle is valid only within the interator body. + ## on demand using []. Returned handle is valid only within the iterator body. var stmt = setupQuery(db, query, args) + setColumns(columns, stmt) while step(stmt) == SQLITE_ROW: yield stmt if finalize(stmt) != SQLITE_OK: dbError(db) @@ -164,7 +222,7 @@ proc len*(row: InstantRow): int32 {.inline.} = column_count(row) proc getRow*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): Row {.tags: [FReadDb].} = + args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} = ## retrieves a single row. If the query doesn't return any rows, this proc ## will return a Row with empty strings for each column. var stmt = setupQuery(db, query, args) @@ -175,19 +233,19 @@ proc getRow*(db: DbConn, query: SqlQuery, if finalize(stmt) != SQLITE_OK: dbError(db) proc getAllRows*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): seq[Row] {.tags: [FReadDb].} = + args: varargs[string, `$`]): seq[Row] {.tags: [ReadDbEffect].} = ## executes the query and returns the whole result dataset. result = @[] for r in fastRows(db, query, args): result.add(r) iterator rows*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): Row {.tags: [FReadDb].} = + args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} = ## same as `FastRows`, but slower and safe. for r in fastRows(db, query, args): yield r proc getValue*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): string {.tags: [FReadDb].} = + args: varargs[string, `$`]): string {.tags: [ReadDbEffect].} = ## executes the query and returns the first column of the first row of the ## result dataset. Returns "" if the dataset contains no rows or the database ## value is NULL. @@ -205,7 +263,7 @@ proc getValue*(db: DbConn, query: SqlQuery, proc tryInsertID*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): int64 - {.tags: [FWriteDb], raises: [].} = + {.tags: [WriteDbEffect], raises: [].} = ## executes the query (typically "INSERT") and returns the ## generated ID for the row or -1 in case of an error. var q = dbFormat(query, args) @@ -218,7 +276,7 @@ proc tryInsertID*(db: DbConn, query: SqlQuery, result = -1 proc insertID*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): int64 {.tags: [FWriteDb].} = + args: varargs[string, `$`]): int64 {.tags: [WriteDbEffect].} = ## executes the query (typically "INSERT") and returns the ## generated ID for the row. For Postgre this adds ## ``RETURNING id`` to the query, so it only works if your primary key is @@ -228,18 +286,18 @@ proc insertID*(db: DbConn, query: SqlQuery, proc execAffectedRows*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): int64 {. - tags: [FReadDb, FWriteDb].} = + tags: [ReadDbEffect, WriteDbEffect].} = ## executes the query (typically "UPDATE") and returns the ## number of affected rows. exec(db, query, args) result = changes(db) -proc close*(db: DbConn) {.tags: [FDb].} = +proc close*(db: DbConn) {.tags: [DbEffect].} = ## closes the database connection. if sqlite3.close(db) != SQLITE_OK: dbError(db) proc open*(connection, user, password, database: string): DbConn {. - tags: [FDb].} = + tags: [DbEffect].} = ## opens a database connection. Raises `EDb` if the connection could not ## be established. Only the ``connection`` parameter is used for ``sqlite``. var db: DbConn @@ -249,7 +307,7 @@ proc open*(connection, user, password, database: string): DbConn {. dbError(db) proc setEncoding*(connection: DbConn, encoding: string): bool {. - tags: [FDb].} = + tags: [DbEffect].} = ## sets the encoding of a database connection, returns true for ## success, false for failure. ## diff --git a/lib/js/dom.nim b/lib/js/dom.nim index b063fa838..11df959d7 100644 --- a/lib/js/dom.nim +++ b/lib/js/dom.nim @@ -14,36 +14,35 @@ when not defined(js) and not defined(Nimdoc): {.error: "This module only works on the JavaScript platform".} type - TEventHandlers* {.importc.} = object of RootObj - onabort*: proc (event: ref TEvent) {.nimcall.} - onblur*: proc (event: ref TEvent) {.nimcall.} - onchange*: proc (event: ref TEvent) {.nimcall.} - onclick*: proc (event: ref TEvent) {.nimcall.} - ondblclick*: proc (event: ref TEvent) {.nimcall.} - onerror*: proc (event: ref TEvent) {.nimcall.} - onfocus*: proc (event: ref TEvent) {.nimcall.} - onkeydown*: proc (event: ref TEvent) {.nimcall.} - onkeypress*: proc (event: ref TEvent) {.nimcall.} - onkeyup*: proc (event: ref TEvent) {.nimcall.} - onload*: proc (event: ref TEvent) {.nimcall.} - onmousedown*: proc (event: ref TEvent) {.nimcall.} - onmousemove*: proc (event: ref TEvent) {.nimcall.} - onmouseout*: proc (event: ref TEvent) {.nimcall.} - onmouseover*: proc (event: ref TEvent) {.nimcall.} - onmouseup*: proc (event: ref TEvent) {.nimcall.} - onreset*: proc (event: ref TEvent) {.nimcall.} - onselect*: proc (event: ref TEvent) {.nimcall.} - onsubmit*: proc (event: ref TEvent) {.nimcall.} - onunload*: proc (event: ref TEvent) {.nimcall.} - - addEventListener*: proc(ev: cstring, cb: proc(ev: ref TEvent), useCapture: bool = false) {.nimcall.} + EventTarget* = ref EventTargetObj + EventTargetObj {.importc.} = object of RootObj + onabort*: proc (event: Event) {.nimcall.} + onblur*: proc (event: Event) {.nimcall.} + onchange*: proc (event: Event) {.nimcall.} + onclick*: proc (event: Event) {.nimcall.} + ondblclick*: proc (event: Event) {.nimcall.} + onerror*: proc (event: Event) {.nimcall.} + onfocus*: proc (event: Event) {.nimcall.} + onkeydown*: proc (event: Event) {.nimcall.} + onkeypress*: proc (event: Event) {.nimcall.} + onkeyup*: proc (event: Event) {.nimcall.} + onload*: proc (event: Event) {.nimcall.} + onmousedown*: proc (event: Event) {.nimcall.} + onmousemove*: proc (event: Event) {.nimcall.} + onmouseout*: proc (event: Event) {.nimcall.} + onmouseover*: proc (event: Event) {.nimcall.} + onmouseup*: proc (event: Event) {.nimcall.} + onreset*: proc (event: Event) {.nimcall.} + onselect*: proc (event: Event) {.nimcall.} + onsubmit*: proc (event: Event) {.nimcall.} + onunload*: proc (event: Event) {.nimcall.} Window* = ref WindowObj - WindowObj {.importc.} = object of TEventHandlers + WindowObj {.importc.} = object of EventTargetObj document*: Document - event*: ref TEvent - history*: ref THistory - location*: ref TLocation + event*: Event + history*: History + location*: Location closed*: bool defaultStatus*: cstring innerHeight*, innerWidth*: int @@ -57,50 +56,15 @@ type statusbar*: ref TStatusBar status*: cstring toolbar*: ref TToolBar - - alert*: proc (msg: cstring) {.nimcall.} - back*: proc () {.nimcall.} - blur*: proc () {.nimcall.} - captureEvents*: proc (eventMask: int) {.nimcall.} - clearInterval*: proc (interval: ref TInterval) {.nimcall.} - clearTimeout*: proc (timeout: ref TTimeOut) {.nimcall.} - close*: proc () {.nimcall.} - confirm*: proc (msg: cstring): bool {.nimcall.} - disableExternalCapture*: proc () {.nimcall.} - enableExternalCapture*: proc () {.nimcall.} - find*: proc (text: cstring, caseSensitive = false, - backwards = false) {.nimcall.} - focus*: proc () {.nimcall.} - forward*: proc () {.nimcall.} - handleEvent*: proc (e: ref TEvent) {.nimcall.} - home*: proc () {.nimcall.} - moveBy*: proc (x, y: int) {.nimcall.} - moveTo*: proc (x, y: int) {.nimcall.} - open*: proc (uri, windowname: cstring, - properties: cstring = nil): Window {.nimcall.} - print*: proc () {.nimcall.} - prompt*: proc (text, default: cstring): cstring {.nimcall.} - releaseEvents*: proc (eventMask: int) {.nimcall.} - resizeBy*: proc (x, y: int) {.nimcall.} - resizeTo*: proc (x, y: int) {.nimcall.} - routeEvent*: proc (event: ref TEvent) {.nimcall.} - scrollBy*: proc (x, y: int) {.nimcall.} - scrollTo*: proc (x, y: int) {.nimcall.} - setInterval*: proc (code: cstring, pause: int): ref TInterval {.nimcall.} - setTimeout*: proc (code: cstring, pause: int): ref TTimeOut {.nimcall.} - stop*: proc () {.nimcall.} frames*: seq[TFrame] Frame* = ref FrameObj FrameObj {.importc.} = object of WindowObj - ClassList* {.importc.} = object of RootObj - add*: proc (class: cstring) {.nimcall.} - remove*: proc (class: cstring) {.nimcall.} - contains*: proc (class: cstring):bool {.nimcall.} - toggle*: proc (class: cstring) {.nimcall.} + ClassList* = ref ClassListObj + ClassListObj {.importc.} = object of RootObj - TNodeType* = enum + NodeType* = enum ElementNode = 1, AttributeNode, TextNode, @@ -115,7 +79,7 @@ type NotationNode Node* = ref NodeObj - NodeObj {.importc.} = object of TEventHandlers + NodeObj {.importc.} = object of EventTargetObj attributes*: seq[Node] childNodes*: seq[Node] children*: seq[Node] @@ -124,29 +88,12 @@ type lastChild*: Node nextSibling*: Node nodeName*: cstring - nodeType*: TNodeType + nodeType*: NodeType nodeValue*: cstring parentNode*: Node previousSibling*: Node - appendChild*: proc (child: Node) {.nimcall.} - appendData*: proc (data: cstring) {.nimcall.} - cloneNode*: proc (copyContent: bool): Node {.nimcall.} - deleteData*: proc (start, len: int) {.nimcall.} - getAttribute*: proc (attr: cstring): cstring {.nimcall.} - getAttributeNode*: proc (attr: cstring): Node {.nimcall.} - hasChildNodes*: proc (): bool {.nimcall.} innerHTML*: cstring - insertBefore*: proc (newNode, before: Node) {.nimcall.} - insertData*: proc (position: int, data: cstring) {.nimcall.} - removeAttribute*: proc (attr: cstring) {.nimcall.} - removeAttributeNode*: proc (attr: Node) {.nimcall.} - removeChild*: proc (child: Node) {.nimcall.} - replaceChild*: proc (newNode, oldNode: Node) {.nimcall.} - replaceData*: proc (start, len: int, text: cstring) {.nimcall.} - scrollIntoView*: proc () {.nimcall.} - setAttribute*: proc (name, value: cstring) {.nimcall.} - setAttributeNode*: proc (attr: Node) {.nimcall.} - style*: ref TStyle + style*: Style Document* = ref DocumentObj DocumentObj {.importc.} = object of NodeObj @@ -164,31 +111,16 @@ type title*: cstring URL*: cstring vlinkColor*: cstring - captureEvents*: proc (eventMask: int) {.nimcall.} - createAttribute*: proc (identifier: cstring): Node {.nimcall.} - createElement*: proc (identifier: cstring): Element {.nimcall.} - createTextNode*: proc (identifier: cstring): Node {.nimcall.} - getElementById*: proc (id: cstring): Element {.nimcall.} - getElementsByName*: proc (name: cstring): seq[Element] {.nimcall.} - getElementsByTagName*: proc (name: cstring): seq[Element] {.nimcall.} - getElementsByClassName*: proc (name: cstring): seq[Element] {.nimcall.} - getSelection*: proc (): cstring {.nimcall.} - handleEvent*: proc (event: ref TEvent) {.nimcall.} - open*: proc () {.nimcall.} - releaseEvents*: proc (eventMask: int) {.nimcall.} - routeEvent*: proc (event: ref TEvent) {.nimcall.} - write*: proc (text: cstring) {.nimcall.} - writeln*: proc (text: cstring) {.nimcall.} anchors*: seq[AnchorElement] forms*: seq[FormElement] images*: seq[ImageElement] - applets*: seq[ref TApplet] + applets*: seq[Element] embeds*: seq[EmbedElement] links*: seq[LinkElement] Element* = ref ElementObj ElementObj {.importc.} = object of NodeObj - classList*: ref Classlist + classList*: Classlist checked*: bool defaultChecked*: bool defaultValue*: cstring @@ -196,14 +128,7 @@ type form*: FormElement name*: cstring readOnly*: bool - blur*: proc () {.nimcall.} - click*: proc () {.nimcall.} - focus*: proc () {.nimcall.} - handleEvent*: proc (event: ref TEvent) {.nimcall.} - select*: proc () {.nimcall.} options*: seq[OptionElement] - getElementsByTagName*: proc (name: cstring): seq[Element] {.nimcall.} - getElementsByClassName*: proc (name: cstring): seq[Element] {.nimcall.} LinkElement* = ref LinkObj LinkObj {.importc.} = object of ElementObj @@ -220,16 +145,12 @@ type width*: int `type`*: cstring vspace*: int - play*: proc () {.nimcall.} - stop*: proc () {.nimcall.} AnchorElement* = ref AnchorObj AnchorObj {.importc.} = object of ElementObj text*: cstring x*, y*: int - TApplet* {.importc.} = object of RootObj - OptionElement* = ref OptionObj OptionObj {.importc.} = object of ElementObj defaultSelected*: bool @@ -244,8 +165,6 @@ type encoding*: cstring `method`*: cstring target*: cstring - reset*: proc () {.nimcall.} - submit*: proc () {.nimcall.} elements*: seq[Element] ImageElement* = ref ImageObj @@ -259,8 +178,8 @@ type vspace*: int width*: int - - TStyle* {.importc.} = object of RootObj + Style = ref StyleObj + StyleObj {.importc.} = object of RootObj background*: cstring backgroundAttachment*: cstring backgroundColor*: cstring @@ -350,11 +269,9 @@ type width*: cstring wordSpacing*: cstring zIndex*: int - getAttribute*: proc (attr: cstring, caseSensitive=false): cstring {.nimcall.} - removeAttribute*: proc (attr: cstring, caseSensitive=false) {.nimcall.} - setAttribute*: proc (attr, value: cstring, caseSensitive=false) {.nimcall.} - TEvent* {.importc.} = object of RootObj + Event* = ref EventObj + EventObj {.importc.} = object of RootObj target*: Node altKey*, ctrlKey*, shiftKey*: bool button*: int @@ -393,7 +310,8 @@ type SUBMIT*: int UNLOAD*: int - TLocation* {.importc.} = object of RootObj + Location* = ref LocationObj + LocationObj {.importc.} = object of RootObj hash*: cstring host*: cstring hostname*: cstring @@ -402,16 +320,13 @@ type port*: cstring protocol*: cstring search*: cstring - reload*: proc () {.nimcall.} - replace*: proc (s: cstring) {.nimcall.} - THistory* {.importc.} = object of RootObj + History* = ref HistoryObj + HistoryObj {.importc.} = object of RootObj length*: int - back*: proc () {.nimcall.} - forward*: proc () {.nimcall.} - go*: proc (pagesToJump: int) {.nimcall.} - TNavigator* {.importc.} = object of RootObj + Navigator* = ref NavigatorObj + NavigatorObj {.importc.} = object of RootObj appCodeName*: cstring appName*: cstring appVersion*: cstring @@ -419,7 +334,6 @@ type language*: cstring platform*: cstring userAgent*: cstring - javaEnabled*: proc (): bool {.nimcall.} mimeTypes*: seq[ref TMimeType] TPlugin* {.importc.} = object of RootObj @@ -441,7 +355,8 @@ type TToolBar* = TLocationBar TStatusBar* = TLocationBar - TScreen* {.importc.} = object of RootObj + Screen = ref ScreenObj + ScreenObj {.importc.} = object of RootObj availHeight*: int availWidth*: int colorDepth*: int @@ -452,11 +367,127 @@ type TTimeOut* {.importc.} = object of RootObj TInterval* {.importc.} = object of RootObj +{.push importcpp.} + +# EventTarget "methods" +proc addEventListener*(et: EventTarget, ev: cstring, cb: proc(ev: Event), useCapture: bool = false) + +# Window "methods" +proc alert*(w: Window, msg: cstring) +proc back*(w: Window) +proc blur*(w: Window) +proc captureEvents*(w: Window, eventMask: int) {.deprecated.} +proc clearInterval*(w: Window, interval: ref TInterval) +proc clearTimeout*(w: Window, timeout: ref TTimeOut) +proc close*(w: Window) +proc confirm*(w: Window, msg: cstring): bool +proc disableExternalCapture*(w: Window) +proc enableExternalCapture*(w: Window) +proc find*(w: Window, text: cstring, caseSensitive = false, + backwards = false) +proc focus*(w: Window) +proc forward*(w: Window) +proc handleEvent*(w: Window, e: Event) +proc home*(w: Window) +proc moveBy*(w: Window, x, y: int) +proc moveTo*(w: Window, x, y: int) +proc open*(w: Window, uri, windowname: cstring, + properties: cstring = nil): Window +proc print*(w: Window) +proc prompt*(w: Window, text, default: cstring): cstring +proc releaseEvents*(w: Window, eventMask: int) {.deprecated.} +proc resizeBy*(w: Window, x, y: int) +proc resizeTo*(w: Window, x, y: int) +proc routeEvent*(w: Window, event: Event) +proc scrollBy*(w: Window, x, y: int) +proc scrollTo*(w: Window, x, y: int) +proc setInterval*(w: Window, code: cstring, pause: int): ref TInterval +proc setTimeout*(w: Window, code: cstring, pause: int): ref TTimeOut +proc stop*(w: Window) + +# Node "methods" +proc appendChild*(n, child: Node) +proc appendData*(n: Node, data: cstring) +proc cloneNode*(n: Node, copyContent: bool): Node +proc deleteData*(n: Node, start, len: int) +proc getAttribute*(n: Node, attr: cstring): cstring +proc getAttributeNode*(n: Node, attr: cstring): Node +proc hasChildNodes*(n: Node): bool +proc insertBefore*(n, newNode, before: Node) +proc insertData*(n: Node, position: int, data: cstring) +proc removeAttribute*(n: Node, attr: cstring) +proc removeAttributeNode*(n, attr: Node) +proc removeChild*(n, child: Node) +proc replaceChild*(n, newNode, oldNode: Node) +proc replaceData*(n: Node, start, len: int, text: cstring) +proc scrollIntoView*(n: Node) +proc setAttribute*(n: Node, name, value: cstring) +proc setAttributeNode*(n: Node, attr: Node) + +# Document "methods" +proc captureEvents*(d: Document, eventMask: int) {.deprecated.} +proc createAttribute*(d: Document, identifier: cstring): Node +proc createElement*(d: Document, identifier: cstring): Element +proc createTextNode*(d: Document, identifier: cstring): Node +proc getElementById*(d: Document, id: cstring): Element +proc getElementsByName*(d: Document, name: cstring): seq[Element] +proc getElementsByTagName*(d: Document, name: cstring): seq[Element] +proc getElementsByClassName*(d: Document, name: cstring): seq[Element] +proc getSelection*(d: Document): cstring +proc handleEvent*(d: Document, event: Event) +proc open*(d: Document) +proc releaseEvents*(d: Document, eventMask: int) {.deprecated.} +proc routeEvent*(d: Document, event: Event) +proc write*(d: Document, text: cstring) +proc writeln*(d: Document, text: cstring) + +# Element "methods" +proc blur*(e: Element) +proc click*(e: Element) +proc focus*(e: Element) +proc handleEvent*(e: Element, event: Event) +proc select*(e: Element) +proc getElementsByTagName*(e: Element, name: cstring): seq[Element] +proc getElementsByClassName*(e: Element, name: cstring): seq[Element] + +# FormElement "methods" +proc reset*(f: FormElement) +proc submit*(f: FormElement) + +# EmbedElement "methods" +proc play*(e: EmbedElement) +proc stop*(e: EmbedElement) + +# Location "methods" +proc reload*(loc: Location) +proc replace*(loc: Location, s: cstring) + +# History "methods" +proc back*(h: History) +proc forward*(h: History) +proc go*(h: History, pagesToJump: int) + +# Navigator "methods" +proc javaEnabled*(h: Navigator): bool + +# ClassList "methods" +proc add*(c: ClassList, class: cstring) +proc remove*(c: ClassList, class: cstring) +proc contains*(c: ClassList, class: cstring):bool +proc toggle*(c: ClassList, class: cstring) + +# Style "methods" +proc getAttribute*(s: Style, attr: cstring, caseSensitive=false): cstring +proc removeAttribute*(s: Style, attr: cstring, caseSensitive=false) +proc setAttribute*(s: Style, attr, value: cstring, caseSensitive=false) + +{.pop.} + var window* {.importc, nodecl.}: Window document* {.importc, nodecl.}: Document - navigator* {.importc, nodecl.}: ref TNavigator - screen* {.importc, nodecl.}: ref TScreen + navigator* {.importc, nodecl.}: Navigator + screen* {.importc, nodecl.}: Screen proc decodeURI*(uri: cstring): cstring {.importc, nodecl.} proc encodeURI*(uri: cstring): cstring {.importc, nodecl.} @@ -474,6 +505,7 @@ proc parseInt*(s: cstring, radix: int):int {.importc, nodecl.} type + TEventHandlers* {.deprecated.} = EventTargetObj TWindow* {.deprecated.} = WindowObj TFrame* {.deprecated.} = FrameObj TNode* {.deprecated.} = NodeObj @@ -485,3 +517,11 @@ type TOption* {.deprecated.} = OptionObj TForm* {.deprecated.} = FormObj TImage* {.deprecated.} = ImageObj + TNodeType* {.deprecated.} = NodeType + TEvent* {.deprecated.} = EventObj + TLocation* {.deprecated.} = LocationObj + THistory* {.deprecated.} = HistoryObj + TNavigator* {.deprecated.} = NavigatorObj + TStyle* {.deprecated.} = StyleObj + TScreen* {.deprecated.} = ScreenObj + TApplet* {.importc, deprecated.} = object of RootObj diff --git a/lib/nimbase.h b/lib/nimbase.h index 0946b9a1f..bba5ac023 100644 --- a/lib/nimbase.h +++ b/lib/nimbase.h @@ -418,10 +418,6 @@ typedef int assert_numbits[sizeof(NI) == sizeof(void*) && NIM_INTBITS == sizeof( # define NIM_EXTERNC #endif -/* we have to tinker with TNimType as it's both part of system.nim and - typeinfo.nim but system.nim doesn't export it cleanly... */ -typedef struct TNimType TNimType; - /* ---------------- platform specific includes ----------------------- */ /* VxWorks related includes */ diff --git a/lib/packages/docutils/highlite.nim b/lib/packages/docutils/highlite.nim index 640b8cd5a..1bc0af1b6 100644 --- a/lib/packages/docutils/highlite.nim +++ b/lib/packages/docutils/highlite.nim @@ -173,7 +173,41 @@ proc nimNextToken(g: var GeneralTokenizer) = while g.buf[pos] in {' ', '\x09'..'\x0D'}: inc(pos) of '#': g.kind = gtComment - while not (g.buf[pos] in {'\0', '\x0A', '\x0D'}): inc(pos) + inc(pos) + var isDoc = false + if g.buf[pos] == '#': + inc(pos) + isDoc = true + if g.buf[pos] == '[': + g.kind = gtLongComment + var nesting = 0 + while true: + case g.buf[pos] + of '\0': break + of '#': + if isDoc: + if g.buf[pos+1] == '#' and g.buf[pos+2] == '[': + inc nesting + elif g.buf[pos+1] == '[': + inc nesting + inc pos + of ']': + if isDoc: + if g.buf[pos+1] == '#' and g.buf[pos+2] == '#': + if nesting == 0: + inc(pos, 3) + break + dec nesting + elif g.buf[pos+1] == '#': + if nesting == 0: + inc(pos, 2) + break + dec nesting + inc pos + else: + inc pos + else: + while g.buf[pos] notin {'\0', '\x0A', '\x0D'}: inc(pos) of 'a'..'z', 'A'..'Z', '_', '\x80'..'\xFF': var id = "" while g.buf[pos] in SymChars + {'_'}: diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim index 4a0304a7c..22d944597 100644 --- a/lib/packages/docutils/rstgen.nim +++ b/lib/packages/docutils/rstgen.nim @@ -534,7 +534,7 @@ proc generateDocumentationJumps(docs: IndexedDocs): string = for title in titles: chunks.add("<a href=\"" & title.link & "\">" & title.keyword & "</a>") - result.add(chunks.join(", ") & ".<br>") + result.add(chunks.join(", ") & ".<br/>") proc generateModuleJumps(modules: seq[string]): string = ## Returns a plain list of hyperlinks to the list of modules. @@ -544,7 +544,7 @@ proc generateModuleJumps(modules: seq[string]): string = for name in modules: chunks.add("<a href=\"" & name & ".html\">" & name & "</a>") - result.add(chunks.join(", ") & ".<br>") + result.add(chunks.join(", ") & ".<br/>") proc readIndexDir(dir: string): tuple[modules: seq[string], symbols: seq[IndexEntry], docs: IndexedDocs] = diff --git a/lib/posix/posix.nim b/lib/posix/posix.nim index 5f1dfcfcd..40b48f992 100644 --- a/lib/posix/posix.nim +++ b/lib/posix/posix.nim @@ -2165,6 +2165,10 @@ proc pwrite*(a1: cint, a2: pointer, a3: int, a4: Off): int {. importc, header: "<unistd.h>".} proc read*(a1: cint, a2: pointer, a3: int): int {.importc, header: "<unistd.h>".} proc readlink*(a1, a2: cstring, a3: int): int {.importc, header: "<unistd.h>".} +proc ioctl*(f: FileHandle, device: uint): int {.importc: "ioctl", + header: "<sys/ioctl.h>", varargs, tags: [WriteIOEffect].} + ## A system call for device-specific input/output operations and other + ## operations which cannot be expressed by regular system calls proc rmdir*(a1: cstring): cint {.importc, header: "<unistd.h>".} proc setegid*(a1: Gid): cint {.importc, header: "<unistd.h>".} diff --git a/lib/pure/asyncftpclient.nim b/lib/pure/asyncftpclient.nim index b806f4235..fd899e080 100644 --- a/lib/pure/asyncftpclient.nim +++ b/lib/pure/asyncftpclient.nim @@ -288,7 +288,7 @@ proc defaultOnProgressChanged*(total, progress: BiggestInt, result.complete() proc retrFile*(ftp: AsyncFtpClient, file, dest: string, - onProgressChanged = defaultOnProgressChanged) {.async.} = + onProgressChanged: ProgressChangedProc = defaultOnProgressChanged) {.async.} = ## Downloads ``file`` and saves it to ``dest``. ## The ``EvRetr`` event is passed to the specified ``handleEvent`` function ## when the download is finished. The event's ``filename`` field will be equal @@ -339,7 +339,7 @@ proc doUpload(ftp: AsyncFtpClient, file: File, await countdownFut or sendFut proc store*(ftp: AsyncFtpClient, file, dest: string, - onProgressChanged = defaultOnProgressChanged) {.async.} = + onProgressChanged: ProgressChangedProc = defaultOnProgressChanged) {.async.} = ## Uploads ``file`` to ``dest`` on the remote FTP server. Usage of this ## function asynchronously is recommended to view the progress of ## the download. diff --git a/lib/pure/collections/critbits.nim b/lib/pure/collections/critbits.nim index 8c507d4fb..bb234565b 100644 --- a/lib/pure/collections/critbits.nim +++ b/lib/pure/collections/critbits.nim @@ -232,7 +232,7 @@ iterator mpairs*[T](c: var CritBitTree[T]): tuple[key: string, val: var T] = ## yields all (key, value)-pairs of `c`. The yielded values can be modified. for x in leaves(c.root): yield (x.key, x.val) -proc allprefixedAux[T](c: CritBitTree[T], key: string): Node[T] = +proc allprefixedAux[T](c: CritBitTree[T], key: string; longestMatch: bool): Node[T] = var p = c.root var top = p if p != nil: @@ -242,43 +242,51 @@ proc allprefixedAux[T](c: CritBitTree[T], key: string): Node[T] = let dir = (1 + (ch.ord or p.otherBits.ord)) shr 8 p = p.child[dir] if q.byte < key.len: top = p - for i in 0 .. <key.len: - if p.key[i] != key[i]: return + if not longestMatch: + for i in 0 .. <key.len: + if p.key[i] != key[i]: return result = top -iterator itemsWithPrefix*[T](c: CritBitTree[T], prefix: string): string = - ## yields all keys starting with `prefix`. - let top = allprefixedAux(c, prefix) +iterator itemsWithPrefix*[T](c: CritBitTree[T], prefix: string; + longestMatch=false): string = + ## yields all keys starting with `prefix`. If `longestMatch` is true, + ## the longest match is returned, it doesn't have to be a complete match then. + let top = allprefixedAux(c, prefix, longestMatch) for x in leaves(top): yield x.key -iterator keysWithPrefix*[T](c: CritBitTree[T], prefix: string): string = +iterator keysWithPrefix*[T](c: CritBitTree[T], prefix: string; + longestMatch=false): string = ## yields all keys starting with `prefix`. - let top = allprefixedAux(c, prefix) + let top = allprefixedAux(c, prefix, longestMatch) for x in leaves(top): yield x.key -iterator valuesWithPrefix*[T](c: CritBitTree[T], prefix: string): T = +iterator valuesWithPrefix*[T](c: CritBitTree[T], prefix: string; + longestMatch=false): T = ## yields all values of `c` starting with `prefix` of the ## corresponding keys. - let top = allprefixedAux(c, prefix) + let top = allprefixedAux(c, prefix, longestMatch) for x in leaves(top): yield x.val -iterator mvaluesWithPrefix*[T](c: var CritBitTree[T], prefix: string): var T = +iterator mvaluesWithPrefix*[T](c: var CritBitTree[T], prefix: string; + longestMatch=false): var T = ## yields all values of `c` starting with `prefix` of the ## corresponding keys. The values can be modified. - let top = allprefixedAux(c, prefix) + let top = allprefixedAux(c, prefix, longestMatch) for x in leaves(top): yield x.val iterator pairsWithPrefix*[T](c: CritBitTree[T], - prefix: string): tuple[key: string, val: T] = + prefix: string; + longestMatch=false): tuple[key: string, val: T] = ## yields all (key, value)-pairs of `c` starting with `prefix`. - let top = allprefixedAux(c, prefix) + let top = allprefixedAux(c, prefix, longestMatch) for x in leaves(top): yield (x.key, x.val) iterator mpairsWithPrefix*[T](c: var CritBitTree[T], - prefix: string): tuple[key: string, val: var T] = + prefix: string; + longestMatch=false): tuple[key: string, val: var T] = ## yields all (key, value)-pairs of `c` starting with `prefix`. ## The yielded values can be modified. - let top = allprefixedAux(c, prefix) + let top = allprefixedAux(c, prefix, longestMatch) for x in leaves(top): yield (x.key, x.val) proc `$`*[T](c: CritBitTree[T]): string = diff --git a/lib/pure/collections/sequtils.nim b/lib/pure/collections/sequtils.nim index 71babe93b..b72face91 100644 --- a/lib/pure/collections/sequtils.nim +++ b/lib/pure/collections/sequtils.nim @@ -10,12 +10,9 @@ ## :Author: Alexander Mitchell-Robinson (Amrykid) ## ## This module implements operations for the built-in `seq`:idx: type which -## were inspired by functional programming languages. If you are looking for -## the typical `map` function which applies a function to every element in a -## sequence, it already exists in the `system <system.html>`_ module in both -## mutable and immutable styles. +## were inspired by functional programming languages. ## -## Also, for functional style programming you may want to pass `anonymous procs +## For functional style programming you may want to pass `anonymous procs ## <manual.html#anonymous-procs>`_ to procs like ``filter`` to reduce typing. ## Anonymous procs can use `the special do notation <manual.html#do-notation>`_ ## which is more convenient in certain situations. @@ -471,7 +468,7 @@ template toSeq*(iter: expr): expr {.immediate.} = ## if x mod 2 == 1: ## result = true) ## assert odd_numbers == @[1, 3, 5, 7, 9] - + when compiles(iter.len): var i = 0 var result = newSeq[type(iter)](iter.len) diff --git a/lib/pure/collections/tables.nim b/lib/pure/collections/tables.nim index 329b2a1cb..2ed0d2034 100644 --- a/lib/pure/collections/tables.nim +++ b/lib/pure/collections/tables.nim @@ -887,7 +887,7 @@ proc mget*[A](t: CountTableRef[A], key: A): var int {.deprecated.} = result = t[][key] proc getOrDefault*[A](t: CountTableRef[A], key: A): int = - getOrDefaultImpl(t, key) + result = t[].getOrDefault(key) proc hasKey*[A](t: CountTableRef[A], key: A): bool = ## returns true iff `key` is in the table `t`. @@ -1028,3 +1028,15 @@ when isMainModule: assert(merged["foo"] == 5) assert(merged["bar"] == 3) assert(merged["baz"] == 14) + + block: + const testKey = "TESTKEY" + let t: CountTableRef[string] = newCountTable[string]() + + # Before, does not compile with error message: + #test_counttable.nim(7, 43) template/generic instantiation from here + #lib/pure/collections/tables.nim(117, 21) template/generic instantiation from here + #lib/pure/collections/tableimpl.nim(32, 27) Error: undeclared field: 'hcode + doAssert 0 == t.getOrDefault(testKey) + t.inc(testKey,3) + doAssert 3 == t.getOrDefault(testKey) diff --git a/lib/pure/db_common.nim b/lib/pure/db_common.nim new file mode 100644 index 000000000..957389605 --- /dev/null +++ b/lib/pure/db_common.nim @@ -0,0 +1,103 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2015 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Common datatypes and definitions for all ``db_*.nim`` ( +## `db_mysql <db_mysql.html>`_, `db_postgres <db_postgres.html>`_, +## and `db_sqlite <db_sqlite.html>`_) modules. + +type + DbError* = object of IOError ## exception that is raised if a database error occurs + + SqlQuery* = distinct string ## an SQL query string + + + DbEffect* = object of IOEffect ## effect that denotes a database operation + ReadDbEffect* = object of DbEffect ## effect that denotes a read operation + WriteDbEffect* = object of DbEffect ## effect that denotes a write operation + + DbTypeKind* = enum ## a superset of datatypes that might be supported. + dbUnknown, ## unknown datatype + dbSerial, ## datatype used for primary auto-increment keys + dbNull, ## datatype used for the NULL value + dbBit, ## bit datatype + dbBool, ## boolean datatype + dbBlob, ## blob datatype + dbFixedChar, ## string of fixed length + dbVarchar, ## string datatype + dbJson, ## JSON datatype + dbXml, ## XML datatype + dbInt, ## some integer type + dbUInt, ## some unsigned integer type + dbDecimal, ## decimal numbers (fixed-point number) + dbFloat, ## some floating point type + dbDate, ## a year-month-day description + dbTime, ## HH:MM:SS information + dbDatetime, ## year-month-day and HH:MM:SS information, + ## plus optional time or timezone information + dbTimestamp, ## Timestamp values are stored as the number of seconds + ## since the epoch ('1970-01-01 00:00:00' UTC). + dbTimeInterval, ## an interval [a,b] of times + dbEnum, ## some enum + dbSet, ## set of enum values + dbArray, ## an array of values + dbComposite, ## composite type (record, struct, etc) + dbUrl, ## a URL + dbUuid, ## a UUID + dbInet, ## an IP address + dbMacAddress, ## a MAC address + dbGeometry, ## some geometric type + dbPoint, ## Point on a plane (x,y) + dbLine, ## Infinite line ((x1,y1),(x2,y2)) + dbLseg, ## Finite line segment ((x1,y1),(x2,y2)) + dbBox, ## Rectangular box ((x1,y1),(x2,y2)) + dbPath, ## Closed or open path (similar to polygon) ((x1,y1),...) + dbPolygon, ## Polygon (similar to closed path) ((x1,y1),...) + dbCircle, ## Circle <(x,y),r> (center point and radius) + dbUser1, ## user definable datatype 1 (for unknown extensions) + dbUser2, ## user definable datatype 2 (for unknown extensions) + dbUser3, ## user definable datatype 3 (for unknown extensions) + dbUser4, ## user definable datatype 4 (for unknown extensions) + dbUser5 ## user definable datatype 5 (for unknown extensions) + + DbType* = object ## describes a database type + kind*: DbTypeKind ## the kind of the described type + notNull*: bool ## does the type contain NULL? + name*: string ## the name of the type + size*: Natural ## the size of the datatype; 0 if of variable size + maxReprLen*: Natural ## maximal length required for the representation + precision*, scale*: Natural ## precision and scale of the number + min*, max*: BiggestInt ## the minimum and maximum of allowed values + validValues*: seq[string] ## valid values of an enum or a set + + DbColumn* = object ## information about a database column + name*: string ## name of the column + tableName*: string ## name of the table the column belongs to (optional) + typ*: DbType ## type of the column + primaryKey*: bool ## is this a primary key? + foreignKey*: bool ## is this a foreign key? + DbColumns* = seq[DbColumn] + +{.deprecated: [EDb: DbError, TSqlQuery: SqlQuery, FDb: DbEffect, + FReadDb: ReadDbEffect, FWriteDb: WriteDbEffect].} + +template sql*(query: string): SqlQuery = + ## constructs a SqlQuery from the string `query`. This is supposed to be + ## used as a raw-string-literal modifier: + ## ``sql"update user set counter = counter + 1"`` + ## + ## If assertions are turned off, it does nothing. If assertions are turned + ## on, later versions will check the string for valid syntax. + SqlQuery(query) + +proc dbError*(msg: string) {.noreturn, noinline.} = + ## raises an DbError exception with message `msg`. + var e: ref DbError + new(e) + e.msg = msg + raise e diff --git a/lib/pure/events.nim b/lib/pure/events.nim index 62800c5c8..23a8a2c58 100644 --- a/lib/pure/events.nim +++ b/lib/pure/events.nim @@ -57,7 +57,7 @@ proc addHandler*(handler: var EventHandler, fn: proc(e: EventArgs) {.closure.}) proc removeHandler*(handler: var EventHandler, fn: proc(e: EventArgs) {.closure.}) = ## Removes the callback from the specified event handler. - for i in countup(0, len(handler.handlers) -1): + for i in countup(0, len(handler.handlers)-1): if fn == handler.handlers[i]: handler.handlers.del(i) break diff --git a/lib/pure/fsmonitor.nim b/lib/pure/fsmonitor.nim index 787acb5d4..b22e84f44 100644 --- a/lib/pure/fsmonitor.nim +++ b/lib/pure/fsmonitor.nim @@ -10,6 +10,9 @@ ## This module allows you to monitor files or directories for changes using ## asyncio. ## +## **Warning**: This module will likely disappear soon and be moved into a +## new Nimble package. +## ## Windows support is not yet implemented. ## ## **Note:** This module uses ``inotify`` on Linux (Other Unixes are not yet @@ -34,8 +37,8 @@ type MonitorEventType* = enum ## Monitor event type MonitorAccess, ## File was accessed. MonitorAttrib, ## Metadata changed. - MonitorCloseWrite, ## Writtable file was closed. - MonitorCloseNoWrite, ## Unwrittable file closed. + MonitorCloseWrite, ## Writable file was closed. + MonitorCloseNoWrite, ## Non-writable file closed. MonitorCreate, ## Subfile was created. MonitorDelete, ## Subfile was deleted. MonitorDeleteSelf, ## Watched file/directory was itself deleted. @@ -78,21 +81,21 @@ proc add*(monitor: FSMonitor, target: string, ## watched paths of ``monitor``. ## You can specify the events to report using the ``filters`` parameter. - var INFilter = -1 + var INFilter = 0 for f in filters: case f - of MonitorAccess: INFilter = INFilter and IN_ACCESS - of MonitorAttrib: INFilter = INFilter and IN_ATTRIB - of MonitorCloseWrite: INFilter = INFilter and IN_CLOSE_WRITE - of MonitorCloseNoWrite: INFilter = INFilter and IN_CLOSE_NO_WRITE - of MonitorCreate: INFilter = INFilter and IN_CREATE - of MonitorDelete: INFilter = INFilter and IN_DELETE - of MonitorDeleteSelf: INFilter = INFilter and IN_DELETE_SELF - of MonitorModify: INFilter = INFilter and IN_MODIFY - of MonitorMoveSelf: INFilter = INFilter and IN_MOVE_SELF - of MonitorMoved: INFilter = INFilter and IN_MOVED_FROM and IN_MOVED_TO - of MonitorOpen: INFilter = INFilter and IN_OPEN - of MonitorAll: INFilter = INFilter and IN_ALL_EVENTS + of MonitorAccess: INFilter = INFilter or IN_ACCESS + of MonitorAttrib: INFilter = INFilter or IN_ATTRIB + of MonitorCloseWrite: INFilter = INFilter or IN_CLOSE_WRITE + of MonitorCloseNoWrite: INFilter = INFilter or IN_CLOSE_NO_WRITE + of MonitorCreate: INFilter = INFilter or IN_CREATE + of MonitorDelete: INFilter = INFilter or IN_DELETE + of MonitorDeleteSelf: INFilter = INFilter or IN_DELETE_SELF + of MonitorModify: INFilter = INFilter or IN_MODIFY + of MonitorMoveSelf: INFilter = INFilter or IN_MOVE_SELF + of MonitorMoved: INFilter = INFilter or IN_MOVED_FROM or IN_MOVED_TO + of MonitorOpen: INFilter = INFilter or IN_OPEN + of MonitorAll: INFilter = INFilter or IN_ALL_EVENTS result = inotifyAddWatch(monitor.fd, target, INFilter.uint32) if result < 0: @@ -200,9 +203,18 @@ proc register*(d: Dispatcher, monitor: FSMonitor, when not defined(testing) and isMainModule: proc main = - var disp = newDispatcher() - var monitor = newMonitor() - echo monitor.add("/home/dom/inotifytests/") + var + disp = newDispatcher() + monitor = newMonitor() + n = 0 + n = monitor.add("/tmp") + assert n == 1 + n = monitor.add("/tmp", {MonitorAll}) + assert n == 1 + n = monitor.add("/tmp", {MonitorCloseWrite, MonitorCloseNoWrite}) + assert n == 1 + n = monitor.add("/tmp", {MonitorMoved, MonitorOpen, MonitorAccess}) + assert n == 1 disp.register(monitor, proc (m: FSMonitor, ev: MonitorEvent) = echo("Got event: ", ev.kind) diff --git a/lib/pure/gentabs.nim b/lib/pure/gentabs.nim index e6a05ec63..928ff8fe0 100644 --- a/lib/pure/gentabs.nim +++ b/lib/pure/gentabs.nim @@ -11,6 +11,8 @@ ## key-value mapping. The keys are required to be strings, but the values ## may be any Nim or user defined type. This module supports matching ## of keys in case-sensitive, case-insensitive and style-insensitive modes. +## +## **Warning:** This module is deprecated, new code shouldn't use it! {.deprecated.} diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index 8e182e274..1b91132db 100644 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -110,7 +110,7 @@ type EInvalidProtocol: ProtocolError, EHttpRequestErr: HttpRequestError ].} -const defUserAgent* = "Nim httpclient/0.1" +const defUserAgent* = "Nim httpclient/" & NimVersion proc httpError(msg: string) = var e: ref ProtocolError @@ -389,6 +389,7 @@ proc request*(url: string, httpMethod: string, extraHeaders = "", ## | An optional timeout can be specified in milliseconds, if reading from the ## server takes longer than specified an ETimeout exception will be raised. var r = if proxy == nil: parseUri(url) else: proxy.url + var hostUrl = if proxy == nil: r else: parseUri(url) var headers = substr(httpMethod, len("http")) # TODO: Use generateHeaders further down once it supports proxies. if proxy == nil: @@ -402,10 +403,10 @@ proc request*(url: string, httpMethod: string, extraHeaders = "", headers.add(" HTTP/1.1\c\L") - if r.port == "": - add(headers, "Host: " & r.hostname & "\c\L") + if hostUrl.port == "": + add(headers, "Host: " & hostUrl.hostname & "\c\L") else: - add(headers, "Host: " & r.hostname & ":" & r.port & "\c\L") + add(headers, "Host: " & hostUrl.hostname & ":" & hostUrl.port & "\c\L") if userAgent != "": add(headers, "User-Agent: " & userAgent & "\c\L") @@ -414,7 +415,6 @@ proc request*(url: string, httpMethod: string, extraHeaders = "", add(headers, "Proxy-Authorization: basic " & auth & "\c\L") add(headers, extraHeaders) add(headers, "\c\L") - var s = newSocket() if s == nil: raiseOSError(osLastError()) var port = net.Port(80) diff --git a/lib/pure/httpserver.nim b/lib/pure/httpserver.nim index 71ba04991..632eb198a 100644 --- a/lib/pure/httpserver.nim +++ b/lib/pure/httpserver.nim @@ -9,6 +9,9 @@ ## This module implements a simple HTTP-Server. ## +## **Warning**: This module will soon be deprecated in favour of +## the ``asyncdispatch`` module, you should use it instead. +## ## Example: ## ## .. code-block:: nim diff --git a/lib/pure/lexbase.nim b/lib/pure/lexbase.nim index bfecf6a58..cf2e8bb89 100644 --- a/lib/pure/lexbase.nim +++ b/lib/pure/lexbase.nim @@ -28,7 +28,10 @@ type BaseLexer* = object of RootObj ## the base lexer. Inherit your lexer from ## this object. bufpos*: int ## the current position within the buffer - buf*: cstring ## the buffer itself + when defined(js): ## the buffer itself + buf*: string + else: + buf*: cstring bufLen*: int ## length of buffer in characters input: Stream ## the input stream lineNumber*: int ## the current line number @@ -43,7 +46,8 @@ const proc close*(L: var BaseLexer) = ## closes the base lexer. This closes `L`'s associated stream too. - dealloc(L.buf) + when not defined(js): + dealloc(L.buf) close(L.input) proc fillBuffer(L: var BaseLexer) = @@ -58,8 +62,11 @@ proc fillBuffer(L: var BaseLexer) = toCopy = L.bufLen - L.sentinel - 1 assert(toCopy >= 0) if toCopy > 0: - moveMem(L.buf, addr(L.buf[L.sentinel + 1]), toCopy * chrSize) - # "moveMem" handles overlapping regions + when defined(js): + for i in 0 ..< toCopy: L.buf[i] = L.buf[L.sentinel + 1 + i] + else: + # "moveMem" handles overlapping regions + moveMem(L.buf, addr L.buf[L.sentinel + 1], toCopy * chrSize) charsRead = readData(L.input, addr(L.buf[toCopy]), (L.sentinel + 1) * chrSize) div chrSize s = toCopy + charsRead @@ -81,7 +88,10 @@ proc fillBuffer(L: var BaseLexer) = # double the buffer's size and try again: oldBufLen = L.bufLen L.bufLen = L.bufLen * 2 - L.buf = cast[cstring](realloc(L.buf, L.bufLen * chrSize)) + when defined(js): + L.buf.setLen(L.bufLen) + else: + L.buf = cast[cstring](realloc(L.buf, L.bufLen * chrSize)) assert(L.bufLen - oldBufLen == oldBufLen) charsRead = readData(L.input, addr(L.buf[oldBufLen]), oldBufLen * chrSize) div chrSize @@ -139,7 +149,10 @@ proc open*(L: var BaseLexer, input: Stream, bufLen: int = 8192; L.bufpos = 0 L.bufLen = bufLen L.refillChars = refillChars - L.buf = cast[cstring](alloc(bufLen * chrSize)) + when defined(js): + L.buf = newString(bufLen) + else: + L.buf = cast[cstring](alloc(bufLen * chrSize)) L.sentinel = bufLen - 1 L.lineStart = 0 L.lineNumber = 1 # lines start at 1 diff --git a/lib/pure/math.nim b/lib/pure/math.nim index 391a880ae..b0104336e 100644 --- a/lib/pure/math.nim +++ b/lib/pure/math.nim @@ -118,26 +118,6 @@ proc sum*[T](x: openArray[T]): T {.noSideEffect.} = ## If `x` is empty, 0 is returned. for i in items(x): result = result + i -template toFloat(f: float): float = f - -proc mean*[T](x: openArray[T]): float {.noSideEffect.} = - ## Computes the mean of the elements in `x`, which are first converted to floats. - ## If `x` is empty, NaN is returned. - ## ``toFloat(x: T): float`` must be defined. - for i in items(x): result = result + toFloat(i) - result = result / toFloat(len(x)) - -proc variance*[T](x: openArray[T]): float {.noSideEffect.} = - ## Computes the variance of the elements in `x`. - ## If `x` is empty, NaN is returned. - ## ``toFloat(x: T): float`` must be defined. - result = 0.0 - var m = mean(x) - for i in items(x): - var diff = toFloat(i) - m - result = result + diff*diff - result = result / toFloat(len(x)) - proc random*(max: int): int {.benign.} ## Returns a random number in the range 0..max-1. The sequence of ## random number is always the same, unless `randomize` is called @@ -376,48 +356,6 @@ proc random*[T](a: openArray[T]): T = ## returns a random element from the openarray `a`. result = a[random(a.low..a.len)] -type - RunningStat* = object ## an accumulator for statistical data - n*: int ## number of pushed data - sum*, min*, max*, mean*: float ## self-explaining - oldM, oldS, newS: float - -{.deprecated: [TFloatClass: FloatClass, TRunningStat: RunningStat].} - -proc push*(s: var RunningStat, x: float) = - ## pushes a value `x` for processing - inc(s.n) - # See Knuth TAOCP vol 2, 3rd edition, page 232 - if s.n == 1: - s.min = x - s.max = x - s.oldM = x - s.mean = x - s.oldS = 0.0 - else: - if s.min > x: s.min = x - if s.max < x: s.max = x - s.mean = s.oldM + (x - s.oldM)/toFloat(s.n) - s.newS = s.oldS + (x - s.oldM)*(x - s.mean) - - # set up for next iteration: - s.oldM = s.mean - s.oldS = s.newS - s.sum = s.sum + x - -proc push*(s: var RunningStat, x: int) = - ## pushes a value `x` for processing. `x` is simply converted to ``float`` - ## and the other push operation is called. - push(s, toFloat(x)) - -proc variance*(s: RunningStat): float = - ## computes the current variance of `s` - if s.n > 1: result = s.newS / (toFloat(s.n - 1)) - -proc standardDeviation*(s: RunningStat): float = - ## computes the current standard deviation of `s` - result = sqrt(variance(s)) - {.pop.} {.pop.} diff --git a/lib/pure/nativesockets.nim b/lib/pure/nativesockets.nim index c9e067a3e..b5a8d5777 100644 --- a/lib/pure/nativesockets.nim +++ b/lib/pure/nativesockets.nim @@ -203,9 +203,12 @@ proc getAddrInfo*(address: string, port: Port, domain: Domain = AF_INET, hints.ai_family = toInt(domain) hints.ai_socktype = toInt(sockType) hints.ai_protocol = toInt(protocol) + # OpenBSD doesn't support AI_V4MAPPED and doesn't define the macro AI_V4MAPPED. + # FreeBSD doesn't support AI_V4MAPPED but defines the macro. # https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=198092 - when not defined(freebsd): - hints.ai_flags = AI_V4MAPPED + when not defined(freebsd) and not defined(openbsd) and not defined(netbsd): + if domain == AF_INET6: + hints.ai_flags = AI_V4MAPPED var gaiResult = getaddrinfo(address, $port, addr(hints), result) if gaiResult != 0'i32: when useWinVersion: diff --git a/lib/pure/nimprof.nim b/lib/pure/nimprof.nim index cfe6bc40d..e2397b91c 100644 --- a/lib/pure/nimprof.nim +++ b/lib/pure/nimprof.nim @@ -1,7 +1,7 @@ # # # Nim's Runtime Library -# (c) Copyright 2012 Andreas Rumpf +# (c) Copyright 2015 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -117,24 +117,38 @@ when defined(memProfiler): var gTicker {.threadvar.}: int - proc hook(st: StackTrace, size: int) {.nimcall.} = + proc requestedHook(): bool {.nimcall.} = if gTicker == 0: - gTicker = -1 - when defined(ignoreAllocationSize): - hookAux(st, 1) - else: - hookAux(st, size) gTicker = SamplingInterval + result = true dec gTicker + proc hook(st: StackTrace, size: int) {.nimcall.} = + when defined(ignoreAllocationSize): + hookAux(st, 1) + else: + hookAux(st, size) + else: var t0 {.threadvar.}: Ticks + gTicker: int # we use an additional counter to + # avoid calling 'getTicks' too frequently + + proc requestedHook(): bool {.nimcall.} = + if interval == 0: result = true + elif gTicker == 0: + gTicker = 500 + if getTicks() - t0 > interval: + result = true + else: + dec gTicker proc hook(st: StackTrace) {.nimcall.} = + #echo "profiling! ", interval if interval == 0: hookAux(st, 1) - elif int64(t0) == 0 or getTicks() - t0 > interval: + else: hookAux(st, 1) t0 = getTicks() @@ -145,9 +159,10 @@ proc cmpEntries(a, b: ptr ProfileEntry): int = result = b.getTotal - a.getTotal proc `//`(a, b: int): string = - result = format("$1/$2 = $3%", a, b, formatFloat(a / b * 100.0, ffDefault, 2)) + result = format("$1/$2 = $3%", a, b, formatFloat(a / b * 100.0, ffDecimal, 2)) proc writeProfile() {.noconv.} = + system.profilingRequestedHook = nil when declared(system.StackTrace): system.profilerHook = nil const filename = "profile_results.txt" @@ -193,14 +208,15 @@ var proc disableProfiling*() = when declared(system.StackTrace): atomicDec disabled - system.profilerHook = nil + system.profilingRequestedHook = nil proc enableProfiling*() = when declared(system.StackTrace): if atomicInc(disabled) >= 0: - system.profilerHook = hook + system.profilingRequestedHook = requestedHook when declared(system.StackTrace): + system.profilingRequestedHook = requestedHook system.profilerHook = hook addQuitProc(writeProfile) diff --git a/lib/pure/numeric.nim b/lib/pure/numeric.nim index 71adf19b3..ccda3a146 100644 --- a/lib/pure/numeric.nim +++ b/lib/pure/numeric.nim @@ -7,6 +7,9 @@ # distribution, for details about the copyright. # +## **Warning:** This module will be moved out of the stdlib and into a +## Nimble package, don't use it. + type OneVarFunction* = proc (x: float): float {.deprecated: [TOneVarFunction: OneVarFunction].} diff --git a/lib/pure/options.nim b/lib/pure/options.nim index 3122d58b1..2abb80016 100644 --- a/lib/pure/options.nim +++ b/lib/pure/options.nim @@ -28,7 +28,7 @@ ## ## .. code-block:: nim ## -## import optionals +## import options ## ## proc find(haystack: string, needle: char): Option[int] = ## for i, c in haystack: @@ -156,7 +156,7 @@ proc `$`*[T]( self: Option[T] ): string = when isMainModule: import unittest, sequtils - suite "optionals": + suite "options": # work around a bug in unittest let intNone = none(int) let stringNone = none(string) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index c01228563..1e00f92b1 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -810,6 +810,10 @@ type {.deprecated: [TPathComponent: PathComponent].} +proc staticWalkDir(dir: string; relative: bool): seq[ + tuple[kind: PathComponent, path: string]] = + discard + iterator walkDir*(dir: string; relative=false): tuple[kind: PathComponent, path: string] {. tags: [ReadDirEffect].} = ## walks over the directory `dir` and yields for each directory or file in @@ -833,49 +837,53 @@ iterator walkDir*(dir: string; relative=false): tuple[kind: PathComponent, path: ## dirA/dirC ## dirA/fileA1.txt ## dirA/fileA2.txt - when defined(windows): - var f: WIN32_FIND_DATA - var h = findFirstFile(dir / "*", f) - if h != -1: - while true: - var k = pcFile - if not skipFindData(f): - if (f.dwFileAttributes and FILE_ATTRIBUTE_DIRECTORY) != 0'i32: - k = pcDir - if (f.dwFileAttributes and FILE_ATTRIBUTE_REPARSE_POINT) != 0'i32: - k = succ(k) - let xx = if relative: extractFilename(getFilename(f)) - else: dir / extractFilename(getFilename(f)) - yield (k, xx) - if findNextFile(h, f) == 0'i32: break - findClose(h) + when nimvm: + for k, v in items(staticWalkDir(dir, relative)): + yield (k, v) else: - var d = opendir(dir) - if d != nil: - while true: - var x = readdir(d) - if x == nil: break - var y = $x.d_name - if y != "." and y != "..": - var s: Stat - if not relative: - y = dir / y + when defined(windows): + var f: WIN32_FIND_DATA + var h = findFirstFile(dir / "*", f) + if h != -1: + while true: var k = pcFile - - when defined(linux) or defined(macosx) or defined(bsd): - if x.d_type != DT_UNKNOWN: - if x.d_type == DT_DIR: k = pcDir - if x.d_type == DT_LNK: - if dirExists(y): k = pcLinkToDir - else: k = succ(k) - yield (k, y) - continue - - if lstat(y, s) < 0'i32: break - if S_ISDIR(s.st_mode): k = pcDir - if S_ISLNK(s.st_mode): k = succ(k) - yield (k, y) - discard closedir(d) + if not skipFindData(f): + if (f.dwFileAttributes and FILE_ATTRIBUTE_DIRECTORY) != 0'i32: + k = pcDir + if (f.dwFileAttributes and FILE_ATTRIBUTE_REPARSE_POINT) != 0'i32: + k = succ(k) + let xx = if relative: extractFilename(getFilename(f)) + else: dir / extractFilename(getFilename(f)) + yield (k, xx) + if findNextFile(h, f) == 0'i32: break + findClose(h) + else: + var d = opendir(dir) + if d != nil: + while true: + var x = readdir(d) + if x == nil: break + var y = $x.d_name + if y != "." and y != "..": + var s: Stat + if not relative: + y = dir / y + var k = pcFile + + when defined(linux) or defined(macosx) or defined(bsd): + if x.d_type != DT_UNKNOWN: + if x.d_type == DT_DIR: k = pcDir + if x.d_type == DT_LNK: + if dirExists(y): k = pcLinkToDir + else: k = succ(k) + yield (k, y) + continue + + if lstat(y, s) < 0'i32: break + if S_ISDIR(s.st_mode): k = pcDir + if S_ISLNK(s.st_mode): k = succ(k) + yield (k, y) + discard closedir(d) iterator walkDirRec*(dir: string, filter={pcFile, pcDir}): string {. tags: [ReadDirEffect].} = @@ -1353,7 +1361,7 @@ proc getAppFilename*(): string {.rtl, extern: "nos$1", tags: [ReadIOEffect].} = # /proc/<pid>/file when defined(windows): when useWinUnicode: - var buf = cast[WideCString](alloc(256*2)) + var buf = newWideCString("", 256) var len = getModuleFileNameW(0, buf, 256) result = buf$len else: diff --git a/lib/pure/osproc.nim b/lib/pure/osproc.nim index de9e63909..8560c3ee4 100644 --- a/lib/pure/osproc.nim +++ b/lib/pure/osproc.nim @@ -886,7 +886,7 @@ elif not defined(useNimRtl): discard write(data.pErrorPipe[writeIdx], addr error, sizeof(error)) exitnow(1) - when defined(macosx) or defined(freebsd): + when defined(macosx) or defined(freebsd) or defined(netbsd) or defined(android): var environ {.importc.}: cstringArray proc startProcessAfterFork(data: ptr StartProcessData) = @@ -916,7 +916,7 @@ elif not defined(useNimRtl): discard fcntl(data.pErrorPipe[writeIdx], F_SETFD, FD_CLOEXEC) if data.optionPoUsePath: - when defined(macosx) or defined(freebsd): + when defined(macosx) or defined(freebsd) or defined(netbsd) or defined(android): # MacOSX doesn't have execvpe, so we need workaround. # On MacOSX we can arrive here only from fork, so this is safe: environ = data.sysEnv @@ -937,9 +937,10 @@ elif not defined(useNimRtl): if p.inStream != nil: close(p.inStream) if p.outStream != nil: close(p.outStream) if p.errStream != nil: close(p.errStream) - discard close(p.inHandle) - discard close(p.outHandle) - discard close(p.errHandle) + if poParentStreams notin p.options: + discard close(p.inHandle) + discard close(p.outHandle) + discard close(p.errHandle) proc suspend(p: Process) = if kill(p.id, SIGSTOP) != 0'i32: raiseOsError(osLastError()) diff --git a/lib/pure/oswalkdir.nim b/lib/pure/oswalkdir.nim new file mode 100644 index 000000000..000fe25a3 --- /dev/null +++ b/lib/pure/oswalkdir.nim @@ -0,0 +1,27 @@ + +## Compile-time only version for walkDir if you need it at compile-time +## for JavaScript. + +type + PathComponent* = enum ## Enumeration specifying a path component. + pcFile, ## path refers to a file + pcLinkToFile, ## path refers to a symbolic link to a file + pcDir, ## path refers to a directory + pcLinkToDir ## path refers to a symbolic link to a directory + +proc staticWalkDir(dir: string; relative: bool): seq[ + tuple[kind: PathComponent, path: string]] = + discard + +iterator walkDir*(dir: string; relative=false): tuple[kind: PathComponent, path: string] = + for k, v in items(staticWalkDir(dir, relative)): + yield (k, v) + +iterator walkDirRec*(dir: string, filter={pcFile, pcDir}): string = + var stack = @[dir] + while stack.len > 0: + for k,p in walkDir(stack.pop()): + if k in filter: + case k + of pcFile, pcLinkToFile: yield p + of pcDir, pcLinkToDir: stack.add(p) diff --git a/lib/pure/parseopt2.nim b/lib/pure/parseopt2.nim index 73b498fe0..7fd9c60fe 100644 --- a/lib/pure/parseopt2.nim +++ b/lib/pure/parseopt2.nim @@ -70,7 +70,7 @@ when not defined(createNimRtl): ## Initializes option parser from current command line arguments. return initOptParser(commandLineParams()) -proc next*(p: var OptParser) {.rtl, extern: "npo$1".} +proc next*(p: var OptParser) {.rtl, extern: "npo2$1".} proc nextOption(p: var OptParser, token: string, allowEmpty: bool) = for splitchar in [':', '=']: @@ -113,7 +113,7 @@ proc next(p: var OptParser) = p.key = token p.val = "" -proc cmdLineRest*(p: OptParser): TaintedString {.rtl, extern: "npo$1", deprecated.} = +proc cmdLineRest*(p: OptParser): TaintedString {.rtl, extern: "npo2$1", deprecated.} = ## Returns part of command line string that has not been parsed yet. ## Do not use - does not correctly handle whitespace. return p.cmd[p.pos..p.cmd.len-1].join(" ") diff --git a/lib/pure/parseutils.nim b/lib/pure/parseutils.nim index b3708838a..698bde42a 100644 --- a/lib/pure/parseutils.nim +++ b/lib/pure/parseutils.nim @@ -25,7 +25,7 @@ const proc toLower(c: char): char {.inline.} = result = if c in {'A'..'Z'}: chr(ord(c)-ord('A')+ord('a')) else: c -proc parseHex*(s: string, number: var int, start = 0): int {. +proc parseHex*(s: string, number: var int, start = 0; maxLen = 0): int {. rtl, extern: "npuParseHex", noSideEffect.} = ## Parses a hexadecimal number and stores its value in ``number``. ## @@ -45,11 +45,14 @@ proc parseHex*(s: string, number: var int, start = 0): int {. ## discard parseHex("0x38", value) ## assert value == -200 ## + ## If 'maxLen==0' the length of the hexadecimal number has no + ## upper bound. Not more than ```maxLen`` characters are parsed. var i = start var foundDigit = false if s[i] == '0' and (s[i+1] == 'x' or s[i+1] == 'X'): inc(i, 2) elif s[i] == '#': inc(i) - while true: + let last = if maxLen == 0: s.len else: i+maxLen + while i < last: case s[i] of '_': discard of '0'..'9': diff --git a/lib/pure/poly.nim b/lib/pure/poly.nim index c52300400..b20e9f9d0 100644 --- a/lib/pure/poly.nim +++ b/lib/pure/poly.nim @@ -7,6 +7,9 @@ # distribution, for details about the copyright. # +## **Warning:** This module will be moved out of the stdlib and into a +## Nimble package, don't use it. + import math import strutils import numeric diff --git a/lib/pure/redis.nim b/lib/pure/redis.nim deleted file mode 100644 index e3f18a496..000000000 --- a/lib/pure/redis.nim +++ /dev/null @@ -1,1096 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2012 Dominik Picheta -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## This module implements a redis client. It allows you to connect to a -## redis-server instance, send commands and receive replies. -## -## **Beware**: Most (if not all) functions that return a ``RedisString`` may -## return ``redisNil``, and functions which return a ``RedisList`` -## may return ``nil``. - -import sockets, os, strutils, parseutils - -const - redisNil* = "\0\0" - -type - Pipeline = ref object - enabled: bool - buffer: string - expected: int ## number of replies expected if pipelined - -type - SendMode = enum - normal, pipelined, multiple - -type - Redis* = object - socket: Socket - connected: bool - pipeline: Pipeline - - RedisStatus* = string - RedisInteger* = BiggestInt - RedisString* = string ## Bulk reply - RedisList* = seq[RedisString] ## Multi-bulk reply - - ReplyError* = object of IOError ## Invalid reply from redis - RedisError* = object of IOError ## Error in redis - -{.deprecated: [TSendMode: SendMode, TRedis: Redis, TRedisStatus: RedisStatus, - TRedisInteger: RedisInteger, TRedisString: RedisString, - TRedisList: RedisList, EInvalidReply: ReplyError, ERedis: RedisError].} - -proc newPipeline(): Pipeline = - new(result) - result.buffer = "" - result.enabled = false - result.expected = 0 - -proc open*(host = "localhost", port = 6379.Port): Redis = - ## Opens a connection to the redis server. - result.socket = socket(buffered = false) - if result.socket == invalidSocket: - raiseOSError(osLastError()) - result.socket.connect(host, port) - result.pipeline = newPipeline() - -proc raiseInvalidReply(expected, got: char) = - raise newException(ReplyError, - "Expected '$1' at the beginning of a status reply got '$2'" % - [$expected, $got]) - -proc raiseNoOK(status: string, pipelineEnabled: bool) = - if pipelineEnabled and not (status == "QUEUED" or status == "PIPELINED"): - raise newException(ReplyError, "Expected \"QUEUED\" or \"PIPELINED\" got \"$1\"" % status) - elif not pipelineEnabled and status != "OK": - raise newException(ReplyError, "Expected \"OK\" got \"$1\"" % status) - -template readSocket(r: Redis, dummyVal:expr): stmt = - var line {.inject.}: TaintedString = "" - if r.pipeline.enabled: - return dummyVal - else: - readLine(r.socket, line) - -proc parseStatus(r: Redis, line: string = ""): RedisStatus = - if r.pipeline.enabled: - return "PIPELINED" - - if line == "": - raise newException(RedisError, "Server closed connection prematurely") - - if line[0] == '-': - raise newException(RedisError, strip(line)) - if line[0] != '+': - raiseInvalidReply('+', line[0]) - - return line.substr(1) # Strip '+' - -proc readStatus(r:Redis): RedisStatus = - r.readSocket("PIPELINED") - return r.parseStatus(line) - -proc parseInteger(r: Redis, line: string = ""): RedisInteger = - if r.pipeline.enabled: return -1 - - #if line == "+QUEUED": # inside of multi - # return -1 - - if line == "": - raise newException(RedisError, "Server closed connection prematurely") - - if line[0] == '-': - raise newException(RedisError, strip(line)) - if line[0] != ':': - raiseInvalidReply(':', line[0]) - - # Strip ':' - if parseBiggestInt(line, result, 1) == 0: - raise newException(ReplyError, "Unable to parse integer.") - -proc readInteger(r: Redis): RedisInteger = - r.readSocket(-1) - return r.parseInteger(line) - -proc recv(sock: Socket, size: int): TaintedString = - result = newString(size).TaintedString - if sock.recv(cstring(result), size) != size: - raise newException(ReplyError, "recv failed") - -proc parseSingleString(r: Redis, line:string, allowMBNil = false): RedisString = - if r.pipeline.enabled: return "" - - # Error. - if line[0] == '-': - raise newException(RedisError, strip(line)) - - # Some commands return a /bulk/ value or a /multi-bulk/ nil. Odd. - if allowMBNil: - if line == "*-1": - return redisNil - - if line[0] != '$': - raiseInvalidReply('$', line[0]) - - var numBytes = parseInt(line.substr(1)) - if numBytes == -1: - return redisNil - - var s = r.socket.recv(numBytes+2) - result = strip(s.string) - -proc readSingleString(r: Redis): RedisString = - r.readSocket("") - return r.parseSingleString(line) - -proc readNext(r: Redis): RedisList - -proc parseArrayLines(r: Redis, countLine:string): RedisList = - if countLine.string[0] != '*': - raiseInvalidReply('*', countLine.string[0]) - - var numElems = parseInt(countLine.string.substr(1)) - if numElems == -1: return nil - result = @[] - - for i in 1..numElems: - var parsed = r.readNext() - if not isNil(parsed): - for item in parsed: - result.add(item) - -proc readArrayLines(r: Redis): RedisList = - r.readSocket(nil) - return r.parseArrayLines(line) - -proc parseBulkString(r: Redis, allowMBNil = false, line:string = ""): RedisString = - if r.pipeline.enabled: return "" - - return r.parseSingleString(line, allowMBNil) - -proc readBulkString(r: Redis, allowMBNil = false): RedisString = - r.readSocket("") - return r.parseBulkString(allowMBNil, line) - -proc readArray(r: Redis): RedisList = - r.readSocket(@[]) - return r.parseArrayLines(line) - -proc readNext(r: Redis): RedisList = - r.readSocket(@[]) - - var res = case line[0] - of '+', '-': @[r.parseStatus(line)] - of ':': @[$(r.parseInteger(line))] - of '$': @[r.parseBulkString(true,line)] - of '*': r.parseArrayLines(line) - else: - raise newException(ReplyError, "readNext failed on line: " & line) - nil - r.pipeline.expected -= 1 - return res - -proc flushPipeline*(r: Redis, wasMulti = false): RedisList = - ## Send buffered commands, clear buffer, return results - if r.pipeline.buffer.len > 0: - r.socket.send(r.pipeline.buffer) - r.pipeline.buffer = "" - - r.pipeline.enabled = false - result = @[] - - var tot = r.pipeline.expected - - for i in 0..tot-1: - var ret = r.readNext() - for item in ret: - if not (item.contains("OK") or item.contains("QUEUED")): - result.add(item) - - r.pipeline.expected = 0 - -proc startPipelining*(r: Redis) = - ## Enable command pipelining (reduces network roundtrips). - ## Note that when enabled, you must call flushPipeline to actually send commands, except - ## for multi/exec() which enable and flush the pipeline automatically. - ## Commands return immediately with dummy values; actual results returned from - ## flushPipeline() or exec() - r.pipeline.expected = 0 - r.pipeline.enabled = true - -proc sendCommand(r: Redis, cmd: string, args: varargs[string]) = - var request = "*" & $(1 + args.len()) & "\c\L" - request.add("$" & $cmd.len() & "\c\L") - request.add(cmd & "\c\L") - for i in items(args): - request.add("$" & $i.len() & "\c\L") - request.add(i & "\c\L") - - if r.pipeline.enabled: - r.pipeline.buffer.add(request) - r.pipeline.expected += 1 - else: - r.socket.send(request) - -proc sendCommand(r: Redis, cmd: string, arg1: string, - args: varargs[string]) = - var request = "*" & $(2 + args.len()) & "\c\L" - request.add("$" & $cmd.len() & "\c\L") - request.add(cmd & "\c\L") - request.add("$" & $arg1.len() & "\c\L") - request.add(arg1 & "\c\L") - for i in items(args): - request.add("$" & $i.len() & "\c\L") - request.add(i & "\c\L") - - if r.pipeline.enabled: - r.pipeline.expected += 1 - r.pipeline.buffer.add(request) - else: - r.socket.send(request) - -# Keys - -proc del*(r: Redis, keys: varargs[string]): RedisInteger = - ## Delete a key or multiple keys - r.sendCommand("DEL", keys) - return r.readInteger() - -proc exists*(r: Redis, key: string): bool = - ## Determine if a key exists - r.sendCommand("EXISTS", key) - return r.readInteger() == 1 - -proc expire*(r: Redis, key: string, seconds: int): bool = - ## Set a key's time to live in seconds. Returns `false` if the key could - ## not be found or the timeout could not be set. - r.sendCommand("EXPIRE", key, $seconds) - return r.readInteger() == 1 - -proc expireAt*(r: Redis, key: string, timestamp: int): bool = - ## Set the expiration for a key as a UNIX timestamp. Returns `false` - ## if the key could not be found or the timeout could not be set. - r.sendCommand("EXPIREAT", key, $timestamp) - return r.readInteger() == 1 - -proc keys*(r: Redis, pattern: string): RedisList = - ## Find all keys matching the given pattern - r.sendCommand("KEYS", pattern) - return r.readArray() - -proc scan*(r: Redis, cursor: var BiggestInt): RedisList = - ## Find all keys matching the given pattern and yield it to client in portions - ## using default Redis values for MATCH and COUNT parameters - r.sendCommand("SCAN", $cursor) - let reply = r.readArray() - cursor = strutils.parseBiggestInt(reply[0]) - return reply[1..high(reply)] - -proc scan*(r: Redis, cursor: var BiggestInt, pattern: string): RedisList = - ## Find all keys matching the given pattern and yield it to client in portions - ## using cursor as a client query identifier. Using default Redis value for COUNT argument - r.sendCommand("SCAN", $cursor, ["MATCH", pattern]) - let reply = r.readArray() - cursor = strutils.parseBiggestInt(reply[0]) - return reply[1..high(reply)] - -proc scan*(r: Redis, cursor: var BiggestInt, pattern: string, count: int): RedisList = - ## Find all keys matching the given pattern and yield it to client in portions - ## using cursor as a client query identifier. - r.sendCommand("SCAN", $cursor, ["MATCH", pattern, "COUNT", $count]) - let reply = r.readArray() - cursor = strutils.parseBiggestInt(reply[0]) - return reply[1..high(reply)] - -proc move*(r: Redis, key: string, db: int): bool = - ## Move a key to another database. Returns `true` on a successful move. - r.sendCommand("MOVE", key, $db) - return r.readInteger() == 1 - -proc persist*(r: Redis, key: string): bool = - ## Remove the expiration from a key. - ## Returns `true` when the timeout was removed. - r.sendCommand("PERSIST", key) - return r.readInteger() == 1 - -proc randomKey*(r: Redis): RedisString = - ## Return a random key from the keyspace - r.sendCommand("RANDOMKEY") - return r.readBulkString() - -proc rename*(r: Redis, key, newkey: string): RedisStatus = - ## Rename a key. - ## - ## **WARNING:** Overwrites `newkey` if it exists! - r.sendCommand("RENAME", key, newkey) - raiseNoOK(r.readStatus(), r.pipeline.enabled) - -proc renameNX*(r: Redis, key, newkey: string): bool = - ## Same as ``rename`` but doesn't continue if `newkey` exists. - ## Returns `true` if key was renamed. - r.sendCommand("RENAMENX", key, newkey) - return r.readInteger() == 1 - -proc ttl*(r: Redis, key: string): RedisInteger = - ## Get the time to live for a key - r.sendCommand("TTL", key) - return r.readInteger() - -proc keyType*(r: Redis, key: string): RedisStatus = - ## Determine the type stored at key - r.sendCommand("TYPE", key) - return r.readStatus() - - -# Strings - -proc append*(r: Redis, key, value: string): RedisInteger = - ## Append a value to a key - r.sendCommand("APPEND", key, value) - return r.readInteger() - -proc decr*(r: Redis, key: string): RedisInteger = - ## Decrement the integer value of a key by one - r.sendCommand("DECR", key) - return r.readInteger() - -proc decrBy*(r: Redis, key: string, decrement: int): RedisInteger = - ## Decrement the integer value of a key by the given number - r.sendCommand("DECRBY", key, $decrement) - return r.readInteger() - -proc get*(r: Redis, key: string): RedisString = - ## Get the value of a key. Returns `redisNil` when `key` doesn't exist. - r.sendCommand("GET", key) - return r.readBulkString() - -proc getBit*(r: Redis, key: string, offset: int): RedisInteger = - ## Returns the bit value at offset in the string value stored at key - r.sendCommand("GETBIT", key, $offset) - return r.readInteger() - -proc getRange*(r: Redis, key: string, start, stop: int): RedisString = - ## Get a substring of the string stored at a key - r.sendCommand("GETRANGE", key, $start, $stop) - return r.readBulkString() - -proc getSet*(r: Redis, key: string, value: string): RedisString = - ## Set the string value of a key and return its old value. Returns `redisNil` - ## when key doesn't exist. - r.sendCommand("GETSET", key, value) - return r.readBulkString() - -proc incr*(r: Redis, key: string): RedisInteger = - ## Increment the integer value of a key by one. - r.sendCommand("INCR", key) - return r.readInteger() - -proc incrBy*(r: Redis, key: string, increment: int): RedisInteger = - ## Increment the integer value of a key by the given number - r.sendCommand("INCRBY", key, $increment) - return r.readInteger() - -proc setk*(r: Redis, key, value: string) = - ## Set the string value of a key. - ## - ## NOTE: This function had to be renamed due to a clash with the `set` type. - r.sendCommand("SET", key, value) - raiseNoOK(r.readStatus(), r.pipeline.enabled) - -proc setNX*(r: Redis, key, value: string): bool = - ## Set the value of a key, only if the key does not exist. Returns `true` - ## if the key was set. - r.sendCommand("SETNX", key, value) - return r.readInteger() == 1 - -proc setBit*(r: Redis, key: string, offset: int, - value: string): RedisInteger = - ## Sets or clears the bit at offset in the string value stored at key - r.sendCommand("SETBIT", key, $offset, value) - return r.readInteger() - -proc setEx*(r: Redis, key: string, seconds: int, value: string): RedisStatus = - ## Set the value and expiration of a key - r.sendCommand("SETEX", key, $seconds, value) - raiseNoOK(r.readStatus(), r.pipeline.enabled) - -proc setRange*(r: Redis, key: string, offset: int, - value: string): RedisInteger = - ## Overwrite part of a string at key starting at the specified offset - r.sendCommand("SETRANGE", key, $offset, value) - return r.readInteger() - -proc strlen*(r: Redis, key: string): RedisInteger = - ## Get the length of the value stored in a key. Returns 0 when key doesn't - ## exist. - r.sendCommand("STRLEN", key) - return r.readInteger() - -# Hashes -proc hDel*(r: Redis, key, field: string): bool = - ## Delete a hash field at `key`. Returns `true` if the field was removed. - r.sendCommand("HDEL", key, field) - return r.readInteger() == 1 - -proc hExists*(r: Redis, key, field: string): bool = - ## Determine if a hash field exists. - r.sendCommand("HEXISTS", key, field) - return r.readInteger() == 1 - -proc hGet*(r: Redis, key, field: string): RedisString = - ## Get the value of a hash field - r.sendCommand("HGET", key, field) - return r.readBulkString() - -proc hGetAll*(r: Redis, key: string): RedisList = - ## Get all the fields and values in a hash - r.sendCommand("HGETALL", key) - return r.readArray() - -proc hIncrBy*(r: Redis, key, field: string, incr: int): RedisInteger = - ## Increment the integer value of a hash field by the given number - r.sendCommand("HINCRBY", key, field, $incr) - return r.readInteger() - -proc hKeys*(r: Redis, key: string): RedisList = - ## Get all the fields in a hash - r.sendCommand("HKEYS", key) - return r.readArray() - -proc hLen*(r: Redis, key: string): RedisInteger = - ## Get the number of fields in a hash - r.sendCommand("HLEN", key) - return r.readInteger() - -proc hMGet*(r: Redis, key: string, fields: varargs[string]): RedisList = - ## Get the values of all the given hash fields - r.sendCommand("HMGET", key, fields) - return r.readArray() - -proc hMSet*(r: Redis, key: string, - fieldValues: openArray[tuple[field, value: string]]) = - ## Set multiple hash fields to multiple values - var args = @[key] - for field, value in items(fieldValues): - args.add(field) - args.add(value) - r.sendCommand("HMSET", args) - raiseNoOK(r.readStatus(), r.pipeline.enabled) - -proc hSet*(r: Redis, key, field, value: string): RedisInteger = - ## Set the string value of a hash field - r.sendCommand("HSET", key, field, value) - return r.readInteger() - -proc hSetNX*(r: Redis, key, field, value: string): RedisInteger = - ## Set the value of a hash field, only if the field does **not** exist - r.sendCommand("HSETNX", key, field, value) - return r.readInteger() - -proc hVals*(r: Redis, key: string): RedisList = - ## Get all the values in a hash - r.sendCommand("HVALS", key) - return r.readArray() - -# Lists - -proc bLPop*(r: Redis, keys: varargs[string], timeout: int): RedisList = - ## Remove and get the *first* element in a list, or block until - ## one is available - var args: seq[string] = @[] - for i in items(keys): args.add(i) - args.add($timeout) - r.sendCommand("BLPOP", args) - return r.readArray() - -proc bRPop*(r: Redis, keys: varargs[string], timeout: int): RedisList = - ## Remove and get the *last* element in a list, or block until one - ## is available. - var args: seq[string] = @[] - for i in items(keys): args.add(i) - args.add($timeout) - r.sendCommand("BRPOP", args) - return r.readArray() - -proc bRPopLPush*(r: Redis, source, destination: string, - timeout: int): RedisString = - ## Pop a value from a list, push it to another list and return it; or - ## block until one is available. - ## - ## http://redis.io/commands/brpoplpush - r.sendCommand("BRPOPLPUSH", source, destination, $timeout) - return r.readBulkString(true) # Multi-Bulk nil allowed. - -proc lIndex*(r: Redis, key: string, index: int): RedisString = - ## Get an element from a list by its index - r.sendCommand("LINDEX", key, $index) - return r.readBulkString() - -proc lInsert*(r: Redis, key: string, before: bool, pivot, value: string): - RedisInteger = - ## Insert an element before or after another element in a list - var pos = if before: "BEFORE" else: "AFTER" - r.sendCommand("LINSERT", key, pos, pivot, value) - return r.readInteger() - -proc lLen*(r: Redis, key: string): RedisInteger = - ## Get the length of a list - r.sendCommand("LLEN", key) - return r.readInteger() - -proc lPop*(r: Redis, key: string): RedisString = - ## Remove and get the first element in a list - r.sendCommand("LPOP", key) - return r.readBulkString() - -proc lPush*(r: Redis, key, value: string, create: bool = true): RedisInteger = - ## Prepend a value to a list. Returns the length of the list after the push. - ## The ``create`` param specifies whether a list should be created if it - ## doesn't exist at ``key``. More specifically if ``create`` is true, `LPUSH` - ## will be used, otherwise `LPUSHX`. - if create: - r.sendCommand("LPUSH", key, value) - else: - r.sendCommand("LPUSHX", key, value) - return r.readInteger() - -proc lRange*(r: Redis, key: string, start, stop: int): RedisList = - ## Get a range of elements from a list. Returns `nil` when `key` - ## doesn't exist. - r.sendCommand("LRANGE", key, $start, $stop) - return r.readArray() - -proc lRem*(r: Redis, key: string, value: string, count: int = 0): RedisInteger = - ## Remove elements from a list. Returns the number of elements that have been - ## removed. - r.sendCommand("LREM", key, $count, value) - return r.readInteger() - -proc lSet*(r: Redis, key: string, index: int, value: string) = - ## Set the value of an element in a list by its index - r.sendCommand("LSET", key, $index, value) - raiseNoOK(r.readStatus(), r.pipeline.enabled) - -proc lTrim*(r: Redis, key: string, start, stop: int) = - ## Trim a list to the specified range - r.sendCommand("LTRIM", key, $start, $stop) - raiseNoOK(r.readStatus(), r.pipeline.enabled) - -proc rPop*(r: Redis, key: string): RedisString = - ## Remove and get the last element in a list - r.sendCommand("RPOP", key) - return r.readBulkString() - -proc rPopLPush*(r: Redis, source, destination: string): RedisString = - ## Remove the last element in a list, append it to another list and return it - r.sendCommand("RPOPLPUSH", source, destination) - return r.readBulkString() - -proc rPush*(r: Redis, key, value: string, create: bool = true): RedisInteger = - ## Append a value to a list. Returns the length of the list after the push. - ## The ``create`` param specifies whether a list should be created if it - ## doesn't exist at ``key``. More specifically if ``create`` is true, `RPUSH` - ## will be used, otherwise `RPUSHX`. - if create: - r.sendCommand("RPUSH", key, value) - else: - r.sendCommand("RPUSHX", key, value) - return r.readInteger() - -# Sets - -proc sadd*(r: Redis, key: string, member: string): RedisInteger = - ## Add a member to a set - r.sendCommand("SADD", key, member) - return r.readInteger() - -proc scard*(r: Redis, key: string): RedisInteger = - ## Get the number of members in a set - r.sendCommand("SCARD", key) - return r.readInteger() - -proc sdiff*(r: Redis, keys: varargs[string]): RedisList = - ## Subtract multiple sets - r.sendCommand("SDIFF", keys) - return r.readArray() - -proc sdiffstore*(r: Redis, destination: string, - keys: varargs[string]): RedisInteger = - ## Subtract multiple sets and store the resulting set in a key - r.sendCommand("SDIFFSTORE", destination, keys) - return r.readInteger() - -proc sinter*(r: Redis, keys: varargs[string]): RedisList = - ## Intersect multiple sets - r.sendCommand("SINTER", keys) - return r.readArray() - -proc sinterstore*(r: Redis, destination: string, - keys: varargs[string]): RedisInteger = - ## Intersect multiple sets and store the resulting set in a key - r.sendCommand("SINTERSTORE", destination, keys) - return r.readInteger() - -proc sismember*(r: Redis, key: string, member: string): RedisInteger = - ## Determine if a given value is a member of a set - r.sendCommand("SISMEMBER", key, member) - return r.readInteger() - -proc smembers*(r: Redis, key: string): RedisList = - ## Get all the members in a set - r.sendCommand("SMEMBERS", key) - return r.readArray() - -proc smove*(r: Redis, source: string, destination: string, - member: string): RedisInteger = - ## Move a member from one set to another - r.sendCommand("SMOVE", source, destination, member) - return r.readInteger() - -proc spop*(r: Redis, key: string): RedisString = - ## Remove and return a random member from a set - r.sendCommand("SPOP", key) - return r.readBulkString() - -proc srandmember*(r: Redis, key: string): RedisString = - ## Get a random member from a set - r.sendCommand("SRANDMEMBER", key) - return r.readBulkString() - -proc srem*(r: Redis, key: string, member: string): RedisInteger = - ## Remove a member from a set - r.sendCommand("SREM", key, member) - return r.readInteger() - -proc sunion*(r: Redis, keys: varargs[string]): RedisList = - ## Add multiple sets - r.sendCommand("SUNION", keys) - return r.readArray() - -proc sunionstore*(r: Redis, destination: string, - key: varargs[string]): RedisInteger = - ## Add multiple sets and store the resulting set in a key - r.sendCommand("SUNIONSTORE", destination, key) - return r.readInteger() - -# Sorted sets - -proc zadd*(r: Redis, key: string, score: int, member: string): RedisInteger = - ## Add a member to a sorted set, or update its score if it already exists - r.sendCommand("ZADD", key, $score, member) - return r.readInteger() - -proc zcard*(r: Redis, key: string): RedisInteger = - ## Get the number of members in a sorted set - r.sendCommand("ZCARD", key) - return r.readInteger() - -proc zcount*(r: Redis, key: string, min: string, max: string): RedisInteger = - ## Count the members in a sorted set with scores within the given values - r.sendCommand("ZCOUNT", key, min, max) - return r.readInteger() - -proc zincrby*(r: Redis, key: string, increment: string, - member: string): RedisString = - ## Increment the score of a member in a sorted set - r.sendCommand("ZINCRBY", key, increment, member) - return r.readBulkString() - -proc zinterstore*(r: Redis, destination: string, numkeys: string, - keys: openArray[string], weights: openArray[string] = [], - aggregate: string = ""): RedisInteger = - ## Intersect multiple sorted sets and store the resulting sorted set in - ## a new key - var args = @[destination, numkeys] - for i in items(keys): args.add(i) - - if weights.len != 0: - args.add("WITHSCORE") - for i in items(weights): args.add(i) - if aggregate.len != 0: - args.add("AGGREGATE") - args.add(aggregate) - - r.sendCommand("ZINTERSTORE", args) - - return r.readInteger() - -proc zrange*(r: Redis, key: string, start: string, stop: string, - withScores: bool): RedisList = - ## Return a range of members in a sorted set, by index - if not withScores: - r.sendCommand("ZRANGE", key, start, stop) - else: - r.sendCommand("ZRANGE", "WITHSCORES", key, start, stop) - return r.readArray() - -proc zrangebyscore*(r: Redis, key: string, min: string, max: string, - withScore: bool = false, limit: bool = false, - limitOffset: int = 0, limitCount: int = 0): RedisList = - ## Return a range of members in a sorted set, by score - var args = @[key, min, max] - - if withScore: args.add("WITHSCORE") - if limit: - args.add("LIMIT") - args.add($limitOffset) - args.add($limitCount) - - r.sendCommand("ZRANGEBYSCORE", args) - return r.readArray() - -proc zrank*(r: Redis, key: string, member: string): RedisString = - ## Determine the index of a member in a sorted set - r.sendCommand("ZRANK", key, member) - return r.readBulkString() - -proc zrem*(r: Redis, key: string, member: string): RedisInteger = - ## Remove a member from a sorted set - r.sendCommand("ZREM", key, member) - return r.readInteger() - -proc zremrangebyrank*(r: Redis, key: string, start: string, - stop: string): RedisInteger = - ## Remove all members in a sorted set within the given indexes - r.sendCommand("ZREMRANGEBYRANK", key, start, stop) - return r.readInteger() - -proc zremrangebyscore*(r: Redis, key: string, min: string, - max: string): RedisInteger = - ## Remove all members in a sorted set within the given scores - r.sendCommand("ZREMRANGEBYSCORE", key, min, max) - return r.readInteger() - -proc zrevrange*(r: Redis, key: string, start: string, stop: string, - withScore: bool): RedisList = - ## Return a range of members in a sorted set, by index, - ## with scores ordered from high to low - if withScore: - r.sendCommand("ZREVRANGE", "WITHSCORE", key, start, stop) - else: r.sendCommand("ZREVRANGE", key, start, stop) - return r.readArray() - -proc zrevrangebyscore*(r: Redis, key: string, min: string, max: string, - withScore: bool = false, limit: bool = false, - limitOffset: int = 0, limitCount: int = 0): RedisList = - ## Return a range of members in a sorted set, by score, with - ## scores ordered from high to low - var args = @[key, min, max] - - if withScore: args.add("WITHSCORE") - if limit: - args.add("LIMIT") - args.add($limitOffset) - args.add($limitCount) - - r.sendCommand("ZREVRANGEBYSCORE", args) - return r.readArray() - -proc zrevrank*(r: Redis, key: string, member: string): RedisString = - ## Determine the index of a member in a sorted set, with - ## scores ordered from high to low - r.sendCommand("ZREVRANK", key, member) - return r.readBulkString() - -proc zscore*(r: Redis, key: string, member: string): RedisString = - ## Get the score associated with the given member in a sorted set - r.sendCommand("ZSCORE", key, member) - return r.readBulkString() - -proc zunionstore*(r: Redis, destination: string, numkeys: string, - keys: openArray[string], weights: openArray[string] = [], - aggregate: string = ""): RedisInteger = - ## Add multiple sorted sets and store the resulting sorted set in a new key - var args = @[destination, numkeys] - for i in items(keys): args.add(i) - - if weights.len != 0: - args.add("WEIGHTS") - for i in items(weights): args.add(i) - if aggregate.len != 0: - args.add("AGGREGATE") - args.add(aggregate) - - r.sendCommand("ZUNIONSTORE", args) - - return r.readInteger() - -# HyperLogLog - -proc pfadd*(r: Redis, key: string, elements: varargs[string]): RedisInteger = - ## Add variable number of elements into special 'HyperLogLog' set type - r.sendCommand("PFADD", key, elements) - return r.readInteger() - -proc pfcount*(r: Redis, key: string): RedisInteger = - ## Count approximate number of elements in 'HyperLogLog' - r.sendCommand("PFCOUNT", key) - return r.readInteger() - -proc pfmerge*(r: Redis, destination: string, sources: varargs[string]) = - ## Merge several source HyperLogLog's into one specified by destKey - r.sendCommand("PFMERGE", destination, sources) - raiseNoOK(r.readStatus(), r.pipeline.enabled) - -# Pub/Sub - -# TODO: pub/sub -- I don't think this will work synchronously. -discard """ -proc psubscribe*(r: Redis, pattern: openarray[string]): ???? = - ## Listen for messages published to channels matching the given patterns - r.socket.send("PSUBSCRIBE $#\c\L" % pattern) - return ??? - -proc publish*(r: Redis, channel: string, message: string): RedisInteger = - ## Post a message to a channel - r.socket.send("PUBLISH $# $#\c\L" % [channel, message]) - return r.readInteger() - -proc punsubscribe*(r: Redis, [pattern: openarray[string], : string): ???? = - ## Stop listening for messages posted to channels matching the given patterns - r.socket.send("PUNSUBSCRIBE $# $#\c\L" % [[pattern.join(), ]) - return ??? - -proc subscribe*(r: Redis, channel: openarray[string]): ???? = - ## Listen for messages published to the given channels - r.socket.send("SUBSCRIBE $#\c\L" % channel.join) - return ??? - -proc unsubscribe*(r: Redis, [channel: openarray[string], : string): ???? = - ## Stop listening for messages posted to the given channels - r.socket.send("UNSUBSCRIBE $# $#\c\L" % [[channel.join(), ]) - return ??? - -""" - -# Transactions - -proc discardMulti*(r: Redis) = - ## Discard all commands issued after MULTI - r.sendCommand("DISCARD") - raiseNoOK(r.readStatus(), r.pipeline.enabled) - -proc exec*(r: Redis): RedisList = - ## Execute all commands issued after MULTI - r.sendCommand("EXEC") - r.pipeline.enabled = false - # Will reply with +OK for MULTI/EXEC and +QUEUED for every command - # between, then with the results - return r.flushPipeline(true) - - -proc multi*(r: Redis) = - ## Mark the start of a transaction block - r.startPipelining() - r.sendCommand("MULTI") - raiseNoOK(r.readStatus(), r.pipeline.enabled) - -proc unwatch*(r: Redis) = - ## Forget about all watched keys - r.sendCommand("UNWATCH") - raiseNoOK(r.readStatus(), r.pipeline.enabled) - -proc watch*(r: Redis, key: varargs[string]) = - ## Watch the given keys to determine execution of the MULTI/EXEC block - r.sendCommand("WATCH", key) - raiseNoOK(r.readStatus(), r.pipeline.enabled) - -# Connection - -proc auth*(r: Redis, password: string) = - ## Authenticate to the server - r.sendCommand("AUTH", password) - raiseNoOK(r.readStatus(), r.pipeline.enabled) - -proc echoServ*(r: Redis, message: string): RedisString = - ## Echo the given string - r.sendCommand("ECHO", message) - return r.readBulkString() - -proc ping*(r: Redis): RedisStatus = - ## Ping the server - r.sendCommand("PING") - return r.readStatus() - -proc quit*(r: Redis) = - ## Close the connection - r.sendCommand("QUIT") - raiseNoOK(r.readStatus(), r.pipeline.enabled) - -proc select*(r: Redis, index: int): RedisStatus = - ## Change the selected database for the current connection - r.sendCommand("SELECT", $index) - return r.readStatus() - -# Server - -proc bgrewriteaof*(r: Redis) = - ## Asynchronously rewrite the append-only file - r.sendCommand("BGREWRITEAOF") - raiseNoOK(r.readStatus(), r.pipeline.enabled) - -proc bgsave*(r: Redis) = - ## Asynchronously save the dataset to disk - r.sendCommand("BGSAVE") - raiseNoOK(r.readStatus(), r.pipeline.enabled) - -proc configGet*(r: Redis, parameter: string): RedisList = - ## Get the value of a configuration parameter - r.sendCommand("CONFIG", "GET", parameter) - return r.readArray() - -proc configSet*(r: Redis, parameter: string, value: string) = - ## Set a configuration parameter to the given value - r.sendCommand("CONFIG", "SET", parameter, value) - raiseNoOK(r.readStatus(), r.pipeline.enabled) - -proc configResetStat*(r: Redis) = - ## Reset the stats returned by INFO - r.sendCommand("CONFIG", "RESETSTAT") - raiseNoOK(r.readStatus(), r.pipeline.enabled) - -proc dbsize*(r: Redis): RedisInteger = - ## Return the number of keys in the selected database - r.sendCommand("DBSIZE") - return r.readInteger() - -proc debugObject*(r: Redis, key: string): RedisStatus = - ## Get debugging information about a key - r.sendCommand("DEBUG", "OBJECT", key) - return r.readStatus() - -proc debugSegfault*(r: Redis) = - ## Make the server crash - r.sendCommand("DEBUG", "SEGFAULT") - -proc flushall*(r: Redis): RedisStatus = - ## Remove all keys from all databases - r.sendCommand("FLUSHALL") - raiseNoOK(r.readStatus(), r.pipeline.enabled) - -proc flushdb*(r: Redis): RedisStatus = - ## Remove all keys from the current database - r.sendCommand("FLUSHDB") - raiseNoOK(r.readStatus(), r.pipeline.enabled) - -proc info*(r: Redis): RedisString = - ## Get information and statistics about the server - r.sendCommand("INFO") - return r.readBulkString() - -proc lastsave*(r: Redis): RedisInteger = - ## Get the UNIX time stamp of the last successful save to disk - r.sendCommand("LASTSAVE") - return r.readInteger() - -discard """ -proc monitor*(r: Redis) = - ## Listen for all requests received by the server in real time - r.socket.send("MONITOR\c\L") - raiseNoOK(r.readStatus(), r.pipeline.enabled) -""" - -proc save*(r: Redis) = - ## Synchronously save the dataset to disk - r.sendCommand("SAVE") - raiseNoOK(r.readStatus(), r.pipeline.enabled) - -proc shutdown*(r: Redis) = - ## Synchronously save the dataset to disk and then shut down the server - r.sendCommand("SHUTDOWN") - var s = "".TaintedString - r.socket.readLine(s) - if s.string.len != 0: raise newException(RedisError, s.string) - -proc slaveof*(r: Redis, host: string, port: string) = - ## Make the server a slave of another instance, or promote it as master - r.sendCommand("SLAVEOF", host, port) - raiseNoOK(r.readStatus(), r.pipeline.enabled) - -iterator hPairs*(r: Redis, key: string): tuple[key, value: string] = - ## Iterator for keys and values in a hash. - var - contents = r.hGetAll(key) - k = "" - for i in items(contents): - if k == "": - k = i - else: - yield (k, i) - k = "" - -proc someTests(r: Redis, how: SendMode):seq[string] = - var list:seq[string] = @[] - - if how == pipelined: - r.startPipelining() - elif how == multiple: - r.multi() - - r.setk("nim:test", "Testing something.") - r.setk("nim:utf8", "こんにちは") - r.setk("nim:esc", "\\ths ągt\\") - r.setk("nim:int", "1") - list.add(r.get("nim:esc")) - list.add($(r.incr("nim:int"))) - list.add(r.get("nim:int")) - list.add(r.get("nim:utf8")) - list.add($(r.hSet("test1", "name", "A Test"))) - var res = r.hGetAll("test1") - for r in res: - list.add(r) - list.add(r.get("invalid_key")) - list.add($(r.lPush("mylist","itema"))) - list.add($(r.lPush("mylist","itemb"))) - r.lTrim("mylist",0,1) - var p = r.lRange("mylist", 0, -1) - - for i in items(p): - if not isNil(i): - list.add(i) - - list.add(r.debugObject("mylist")) - - r.configSet("timeout", "299") - var g = r.configGet("timeout") - for i in items(g): - list.add(i) - - list.add(r.echoServ("BLAH")) - - case how - of normal: - return list - of pipelined: - return r.flushPipeline() - of multiple: - return r.exec() - -proc assertListsIdentical(listA, listB: seq[string]) = - assert(listA.len == listB.len) - var i = 0 - for item in listA: - assert(item == listB[i]) - i = i + 1 - -when not defined(testing) and isMainModule: - when false: - var r = open() - - # Test with no pipelining - var listNormal = r.someTests(normal) - - # Test with pipelining enabled - var listPipelined = r.someTests(pipelined) - assertListsIdentical(listNormal, listPipelined) - - # Test with multi/exec() (automatic pipelining) - var listMulti = r.someTests(multiple) - assertListsIdentical(listNormal, listMulti) diff --git a/lib/pure/romans.nim b/lib/pure/romans.nim index 18c04ef58..aa047d1cc 100644 --- a/lib/pure/romans.nim +++ b/lib/pure/romans.nim @@ -9,6 +9,9 @@ ## Module for converting an integer to a Roman numeral. ## See http://en.wikipedia.org/wiki/Roman_numerals for reference. +## +## **Warning:** This module will be moved out of the stdlib and into a +## Nimble package, don't use it. const RomanNumeralDigits* = {'I', 'i', 'V', 'v', 'X', 'x', 'L', 'l', 'C', 'c', diff --git a/lib/pure/stats.nim b/lib/pure/stats.nim new file mode 100644 index 000000000..ec4cd182b --- /dev/null +++ b/lib/pure/stats.nim @@ -0,0 +1,348 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2015 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# +## Statistical analysis framework for performing +## basic statistical analysis of data. +## The data is analysed in a single pass, when a data value +## is pushed to the ``RunningStat`` or ``RunningRegress`` objects +## +## ``RunningStat`` calculates for a single data set +## - n (data count) +## - min (smallest value) +## - max (largest value) +## - sum +## - mean +## - variance +## - varianceS (sample var) +## - standardDeviation +## - standardDeviationS (sample stddev) +## - skewness (the third statistical moment) +## - kurtosis (the fourth statistical moment) +## +## ``RunningRegress`` calculates for two sets of data +## - n +## - slope +## - intercept +## - correlation +## +## Procs have been provided to calculate statistics on arrays and sequences. +## +## However, if more than a single statistical calculation is required, it is more +## efficient to push the data once to the RunningStat object, and +## call the numerous statistical procs for the RunningStat object. +## +## .. code-block:: Nim +## +## var rs: RunningStat +## rs.push(MySeqOfData) +## rs.mean() +## rs.variance() +## rs.skewness() +## rs.kurtosis() + +from math import FloatClass, sqrt, pow, round + +{.push debugger:off .} # the user does not want to trace a part + # of the standard library! +{.push checks:off, line_dir:off, stack_trace:off.} + +type + RunningStat* = object ## an accumulator for statistical data + n*: int ## number of pushed data + min*, max*, sum*: float ## self-explaining + mom1, mom2, mom3, mom4: float ## statistical moments, mom1 is mean + + + RunningRegress* = object ## an accumulator for regression calculations + n*: int ## number of pushed data + x_stats*: RunningStat ## stats for first set of data + y_stats*: RunningStat ## stats for second set of data + s_xy: float ## accumulated data for combined xy + +{.deprecated: [TFloatClass: FloatClass, TRunningStat: RunningStat].} + +# ----------- RunningStat -------------------------- +proc clear*(s: var RunningStat) = + ## reset `s` + s.n = 0 + s.min = toBiggestFloat(int.high) + s.max = 0.0 + s.sum = 0.0 + s.mom1 = 0.0 + s.mom2 = 0.0 + s.mom3 = 0.0 + s.mom4 = 0.0 + +proc push*(s: var RunningStat, x: float) = + ## pushes a value `x` for processing + if s.n == 0: s.min = x + inc(s.n) + # See Knuth TAOCP vol 2, 3rd edition, page 232 + if s.min > x: s.min = x + if s.max < x: s.max = x + s.sum += x + let n = toFloat(s.n) + let delta = x - s.mom1 + let delta_n = delta / toFloat(s.n) + let delta_n2 = delta_n * delta_n + let term1 = delta * delta_n * toFloat(s.n - 1) + s.mom4 += term1 * delta_n2 * (n*n - 3*n + 3) + + 6*delta_n2*s.mom2 - 4*delta_n*s.mom3 + s.mom3 += term1 * delta_n * (n - 2) - 3*delta_n*s.mom2 + s.mom2 += term1 + s.mom1 += delta_n + +proc push*(s: var RunningStat, x: int) = + ## pushes a value `x` for processing. + ## + ## `x` is simply converted to ``float`` + ## and the other push operation is called. + s.push(toFloat(x)) + +proc push*(s: var RunningStat, x: openarray[float|int]) = + ## pushes all values of `x` for processing. + ## + ## Int values of `x` are simply converted to ``float`` and + ## the other push operation is called. + for val in x: + s.push(val) + +proc mean*(s: RunningStat): float = + ## computes the current mean of `s` + result = s.mom1 + +proc variance*(s: RunningStat): float = + ## computes the current population variance of `s` + result = s.mom2 / toFloat(s.n) + +proc varianceS*(s: RunningStat): float = + ## computes the current sample variance of `s` + if s.n > 1: result = s.mom2 / toFloat(s.n - 1) + +proc standardDeviation*(s: RunningStat): float = + ## computes the current population standard deviation of `s` + result = sqrt(variance(s)) + +proc standardDeviationS*(s: RunningStat): float = + ## computes the current sample standard deviation of `s` + result = sqrt(varianceS(s)) + +proc skewness*(s: RunningStat): float = + ## computes the current population skewness of `s` + result = sqrt(toFloat(s.n)) * s.mom3 / pow(s.mom2, 1.5) + +proc skewnessS*(s: RunningStat): float = + ## computes the current sample skewness of `s` + let s2 = skewness(s) + result = sqrt(toFloat(s.n*(s.n-1)))*s2 / toFloat(s.n-2) + +proc kurtosis*(s: RunningStat): float = + ## computes the current population kurtosis of `s` + result = toFloat(s.n) * s.mom4 / (s.mom2 * s.mom2) - 3.0 + +proc kurtosisS*(s: RunningStat): float = + ## computes the current sample kurtosis of `s` + result = toFloat(s.n-1) / toFloat((s.n-2)*(s.n-3)) * + (toFloat(s.n+1)*kurtosis(s) + 6) + +proc `+`*(a, b: RunningStat): RunningStat = + ## combine two RunningStats. + ## + ## Useful if performing parallel analysis of data series + ## and need to re-combine parallel result sets + result.clear() + result.n = a.n + b.n + + let delta = b.mom1 - a.mom1 + let delta2 = delta*delta + let delta3 = delta*delta2 + let delta4 = delta2*delta2 + let n = toFloat(result.n) + + result.mom1 = (a.n.float*a.mom1 + b.n.float*b.mom1) / n + result.mom2 = a.mom2 + b.mom2 + delta2 * a.n.float * b.n.float / n + result.mom3 = a.mom3 + b.mom3 + + delta3 * a.n.float * b.n.float * (a.n.float - b.n.float)/(n*n); + result.mom3 += 3.0*delta * (a.n.float*b.mom2 - b.n.float*a.mom2) / n + result.mom4 = a.mom4 + b.mom4 + + delta4*a.n.float*b.n.float * toFloat(a.n*a.n - a.n*b.n + b.n*b.n) / + (n*n*n) + result.mom4 += 6.0*delta2 * (a.n.float*a.n.float*b.mom2 + b.n.float*b.n.float*a.mom2) / + (n*n) + + 4.0*delta*(a.n.float*b.mom3 - b.n.float*a.mom3) / n + result.max = max(a.max, b.max) + result.min = max(a.min, b.min) + +proc `+=`*(a: var RunningStat, b: RunningStat) {.inline.} = + ## add a second RunningStats `b` to `a` + a = a + b +# ---------------------- standalone array/seq stats --------------------- +proc mean*[T](x: openArray[T]): float = + ## computes the mean of `x` + var rs: RunningStat + rs.push(x) + result = rs.mean() + +proc variance*[T](x: openArray[T]): float = + ## computes the population variance of `x` + var rs: RunningStat + rs.push(x) + result = rs.variance() + +proc varianceS*[T](x: openArray[T]): float = + ## computes the sample variance of `x` + var rs: RunningStat + rs.push(x) + result = rs.varianceS() + +proc standardDeviation*[T](x: openArray[T]): float = + ## computes the population standardDeviation of `x` + var rs: RunningStat + rs.push(x) + result = rs.standardDeviation() + +proc standardDeviationS*[T](x: openArray[T]): float = + ## computes the sanple standardDeviation of `x` + var rs: RunningStat + rs.push(x) + result = rs.standardDeviationS() + +proc skewness*[T](x: openArray[T]): float = + ## computes the population skewness of `x` + var rs: RunningStat + rs.push(x) + result = rs.skewness() + +proc skewnessS*[T](x: openArray[T]): float = + ## computes the sample skewness of `x` + var rs: RunningStat + rs.push(x) + result = rs.skewnessS() + +proc kurtosis*[T](x: openArray[T]): float = + ## computes the population kurtosis of `x` + var rs: RunningStat + rs.push(x) + result = rs.kurtosis() + +proc kurtosisS*[T](x: openArray[T]): float = + ## computes the sample kurtosis of `x` + var rs: RunningStat + rs.push(x) + result = rs.kurtosisS() + +# ---------------------- Running Regression ----------------------------- + +proc clear*(r: var RunningRegress) = + ## reset `r` + r.x_stats.clear() + r.y_stats.clear() + r.s_xy = 0.0 + r.n = 0 + +proc push*(r: var RunningRegress, x, y: float) = + ## pushes two values `x` and `y` for processing + r.s_xy += (r.x_stats.mean() - x)*(r.y_stats.mean() - y)* + toFloat(r.n) / toFloat(r.n + 1) + r.x_stats.push(x) + r.y_stats.push(y) + inc(r.n) + +proc push*(r: var RunningRegress, x, y: int) {.inline.} = + ## pushes two values `x` and `y` for processing. + ## + ## `x` and `y` are converted to ``float`` + ## and the other push operation is called. + r.push(toFloat(x), toFloat(y)) + +proc push*(r: var RunningRegress, x, y: openarray[float|int]) = + ## pushes two sets of values `x` and `y` for processing. + assert(x.len == y.len) + for i in 0..<x.len: + r.push(x[i], y[i]) + +proc slope*(r: RunningRegress): float = + ## computes the current slope of `r` + let s_xx = r.x_stats.varianceS()*toFloat(r.n - 1) + result = r.s_xy / s_xx + +proc intercept*(r: RunningRegress): float = + ## computes the current intercept of `r` + result = r.y_stats.mean() - r.slope()*r.x_stats.mean() + +proc correlation*(r: RunningRegress): float = + ## computes the current correlation of the two data + ## sets pushed into `r` + let t = r.x_stats.standardDeviation() * r.y_stats.standardDeviation() + result = r.s_xy / ( toFloat(r.n) * t ) + +proc `+`*(a, b: RunningRegress): RunningRegress = + ## combine two `RunningRegress` objects. + ## + ## Useful if performing parallel analysis of data series + ## and need to re-combine parallel result sets + result.clear() + result.x_stats = a.x_stats + b.x_stats + result.y_stats = a.y_stats + b.y_stats + result.n = a.n + b.n + + let delta_x = b.x_stats.mean() - a.x_stats.mean() + let delta_y = b.y_stats.mean() - a.y_stats.mean() + result.s_xy = a.s_xy + b.s_xy + + toFloat(a.n*b.n)*delta_x*delta_y/toFloat(result.n) + +proc `+=`*(a: var RunningRegress, b: RunningRegress) = + ## add RunningRegress `b` to `a` + a = a + b + +{.pop.} +{.pop.} + +when isMainModule: + proc clean(x: float): float = + result = round(1.0e8*x).float * 1.0e-8 + + var rs: RunningStat + rs.push(@[1.0, 2.0, 1.0, 4.0, 1.0, 4.0, 1.0, 2.0]) + doAssert(rs.n == 8) + doAssert(clean(rs.mean) == 2.0) + doAssert(clean(rs.variance()) == 1.5) + doAssert(clean(rs.varianceS()) == 1.71428571) + doAssert(clean(rs.skewness()) == 0.81649658) + doAssert(clean(rs.skewnessS()) == 1.01835015) + doAssert(clean(rs.kurtosis()) == -1.0) + doAssert(clean(rs.kurtosisS()) == -0.7000000000000001) + + var rs1, rs2: RunningStat + rs1.push(@[1.0, 2.0, 1.0, 4.0]) + rs2.push(@[1.0, 4.0, 1.0, 2.0]) + let rs3 = rs1 + rs2 + doAssert(clean(rs3.mom2) == clean(rs.mom2)) + doAssert(clean(rs3.mom3) == clean(rs.mom3)) + doAssert(clean(rs3.mom4) == clean(rs.mom4)) + rs1 += rs2 + doAssert(clean(rs1.mom2) == clean(rs.mom2)) + doAssert(clean(rs1.mom3) == clean(rs.mom3)) + doAssert(clean(rs1.mom4) == clean(rs.mom4)) + rs1.clear() + rs1.push(@[1.0, 2.2, 1.4, 4.9]) + doAssert(rs1.sum == 9.5) + doAssert(rs1.mean() == 2.375) + + var rr: RunningRegress + rr.push(@[0.0,1.0,2.8,3.0,4.0], @[0.0,1.0,2.3,3.0,4.0]) + doAssert(rr.slope() == 0.9695585996955861) + doAssert(rr.intercept() == -0.03424657534246611) + doAssert(rr.correlation() == 0.9905100362239381) + var rr1, rr2: RunningRegress + rr1.push(@[0.0,1.0], @[0.0,1.0]) + rr2.push(@[2.8,3.0,4.0], @[2.3,3.0,4.0]) + let rr3 = rr1 + rr2 + doAssert(rr3.correlation() == rr.correlation()) + doAssert(clean(rr3.slope()) == clean(rr.slope())) + doAssert(clean(rr3.intercept()) == clean(rr.intercept())) diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index b61df6086..a446f85b4 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -1210,22 +1210,21 @@ proc unescape*(s: string, prefix = "\"", suffix = "\""): string {.noSideEffect, ## If `s` does not begin with ``prefix`` and end with ``suffix`` a ## ValueError exception will be raised. result = newStringOfCap(s.len) - var i = 0 + var i = prefix.len if not s.startsWith(prefix): raise newException(ValueError, "String does not start with a prefix of: " & prefix) - inc(i) while true: if i == s.len-suffix.len: break case s[i] of '\\': case s[i+1]: of 'x': - inc i + inc i, 2 var c: int - i += parseutils.parseHex(s, c, i) + i += parseutils.parseHex(s, c, i, maxLen=2) result.add(chr(c)) - inc(i, 2) + dec i, 2 of '\\': result.add('\\') of '\'': @@ -1281,7 +1280,7 @@ proc editDistance*(a, b: string): int {.noSideEffect, # another special case: if len1 == 1: - for j in s..len2-1: + for j in s..s+len2-1: if a[s] == b[j]: return len2 - 1 return len2 @@ -1344,8 +1343,8 @@ proc editDistance*(a, b: string): int {.noSideEffect, # floating point formating: - -proc c_sprintf(buf, frmt: cstring): cint {.header: "<stdio.h>", +when not defined(js): + proc c_sprintf(buf, frmt: cstring): cint {.header: "<stdio.h>", importc: "sprintf", varargs, noSideEffect.} type @@ -1370,29 +1369,44 @@ proc formatBiggestFloat*(f: BiggestFloat, format: FloatFormatMode = ffDefault, ## after the decimal point for Nim's ``biggestFloat`` type. ## ## If ``precision == 0``, it tries to format it nicely. - const floatFormatToChar: array[FloatFormatMode, char] = ['g', 'f', 'e'] - var - frmtstr {.noinit.}: array[0..5, char] - buf {.noinit.}: array[0..2500, char] - L: cint - frmtstr[0] = '%' - if precision > 0: - frmtstr[1] = '#' - frmtstr[2] = '.' - frmtstr[3] = '*' - frmtstr[4] = floatFormatToChar[format] - frmtstr[5] = '\0' - L = c_sprintf(buf, frmtstr, precision, f) + when defined(js): + var res: cstring + case format + of ffDefault: + {.emit: "`res` = `f`.toString();".} + of ffDecimal: + {.emit: "`res` = `f`.toFixed(`precision`);".} + of ffScientific: + {.emit: "`res` = `f`.toExponential(`precision`);".} + result = $res + for i in 0 ..< result.len: + # Depending on the locale either dot or comma is produced, + # but nothing else is possible: + if result[i] in {'.', ','}: result[i] = decimalsep else: - frmtstr[1] = floatFormatToChar[format] - frmtstr[2] = '\0' - L = c_sprintf(buf, frmtstr, f) - result = newString(L) - for i in 0 ..< L: - # Depending on the locale either dot or comma is produced, - # but nothing else is possible: - if buf[i] in {'.', ','}: result[i] = decimalsep - else: result[i] = buf[i] + const floatFormatToChar: array[FloatFormatMode, char] = ['g', 'f', 'e'] + var + frmtstr {.noinit.}: array[0..5, char] + buf {.noinit.}: array[0..2500, char] + L: cint + frmtstr[0] = '%' + if precision > 0: + frmtstr[1] = '#' + frmtstr[2] = '.' + frmtstr[3] = '*' + frmtstr[4] = floatFormatToChar[format] + frmtstr[5] = '\0' + L = c_sprintf(buf, frmtstr, precision, f) + else: + frmtstr[1] = floatFormatToChar[format] + frmtstr[2] = '\0' + L = c_sprintf(buf, frmtstr, f) + result = newString(L) + for i in 0 ..< L: + # Depending on the locale either dot or comma is produced, + # but nothing else is possible: + if buf[i] in {'.', ','}: result[i] = decimalsep + else: result[i] = buf[i] proc formatFloat*(f: float, format: FloatFormatMode = ffDefault, precision: range[0..32] = 16; decimalSep = '.'): string {. @@ -1706,3 +1720,4 @@ when isMainModule: doAssert isUpper("ABC") doAssert(not isUpper("AAcc")) doAssert(not isUpper("A#$")) + doAssert(unescape(r"\x013", "", "") == "\x013") diff --git a/lib/pure/times.nim b/lib/pure/times.nim index a478b9d65..03745d54e 100644 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -29,7 +29,7 @@ ## echo "epochTime() float value: ", epochTime() ## echo "getTime() float value: ", toSeconds(getTime()) ## echo "cpuTime() float value: ", cpuTime() -## echo "An hour from now : ", getLocalTime(getTime()) + initInterval(0,0,0,1) +## echo "An hour from now : ", getLocalTime(getTime()) + 1.hours ## echo "An hour from (UTC) now: ", getGmTime(getTime()) + initInterval(0,0,0,1) {.push debugger:off.} # the user does not want to trace a part @@ -171,11 +171,6 @@ type {.deprecated: [TMonth: Month, TWeekDay: WeekDay, TTime: Time, TTimeInterval: TimeInterval, TTimeInfo: TimeInfo].} -proc miliseconds*(t: TimeInterval): int {.deprecated.} = t.milliseconds - -proc `miliseconds=`*(t:var TimeInterval, milliseconds: int) {.deprecated.} = - t.milliseconds = milliseconds - proc getTime*(): Time {.tags: [TimeEffect], benign.} ## gets the current calendar time as a UNIX epoch value (number of seconds ## elapsed since 1970) with integer precission. Use epochTime for higher @@ -245,13 +240,59 @@ proc getStartMilsecs*(): int {.deprecated, tags: [TimeEffect], benign.} proc initInterval*(milliseconds, seconds, minutes, hours, days, months, years: int = 0): TimeInterval = ## creates a new ``TimeInterval``. - result.milliseconds = milliseconds - result.seconds = seconds - result.minutes = minutes - result.hours = hours - result.days = days - result.months = months - result.years = years + ## + ## You can also use the convenience procedures called ``milliseconds``, + ## ``seconds``, ``minutes``, ``hours``, ``days``, ``months``, and ``years``. + ## + ## Example: + ## + ## .. code-block:: nim + ## + ## let day = initInterval(hours=24) + ## let tomorrow = getTime() + day + ## echo(tomorrow) + var carryO = 0 + result.milliseconds = `mod`(milliseconds, 1000) + carryO = `div`(milliseconds, 1000) + result.seconds = `mod`(carryO + seconds, 60) + carryO = `div`(seconds, 60) + result.minutes = `mod`(carryO + minutes, 60) + carryO = `div`(minutes, 60) + result.hours = `mod`(carryO + hours, 24) + carryO = `div`(hours, 24) + result.days = carryO + days + carryO = 0 + result.months = `mod`(months, 12) + carryO = `div`(months, 12) + result.years = carryO + years + +proc `+`*(ti1, ti2: TimeInterval): TimeInterval = + ## Adds two ``TimeInterval`` objects together. + var carryO = 0 + result.milliseconds = `mod`(ti1.milliseconds + ti2.milliseconds, 1000) + carryO = `div`(ti1.milliseconds + ti2.milliseconds, 1000) + result.seconds = `mod`(carryO + ti1.seconds + ti2.seconds, 60) + carryO = `div`(ti1.seconds + ti2.seconds, 60) + result.minutes = `mod`(carryO + ti1.minutes + ti2.minutes, 60) + carryO = `div`(ti1.minutes + ti2.minutes, 60) + result.hours = `mod`(carryO + ti1.hours + ti2.hours, 24) + carryO = `div`(ti1.hours + ti2.hours, 24) + result.days = carryO + ti1.days + ti2.days + carryO = 0 + result.months = `mod`(ti1.months + ti2.months, 12) + carryO = `div`(ti1.months + ti2.months, 12) + result.years = carryO + ti1.years + ti2.years + +proc `-`*(ti1, ti2: TimeInterval): TimeInterval = + ## Subtracts TimeInterval ``ti1`` from ``ti2``. + result = ti1 + result.milliseconds -= ti2.milliseconds + result.seconds -= ti2.seconds + result.minutes -= ti2.minutes + result.hours -= ti2.hours + result.days -= ti2.days + result.months -= ti2.months + result.years -= ti2.years proc isLeapYear*(year: int): bool = ## returns true if ``year`` is a leap year @@ -288,13 +329,22 @@ proc toSeconds(a: TimeInfo, interval: TimeInterval): float = newinterv.months += interval.years * 12 var curMonth = anew.month - for mth in 1 .. newinterv.months: - result += float(getDaysInMonth(curMonth, anew.year) * 24 * 60 * 60) - if curMonth == mDec: - curMonth = mJan - anew.year.inc() - else: - curMonth.inc() + if newinterv.months < 0: # subtracting + for mth in countDown(-1 * newinterv.months, 1): + result -= float(getDaysInMonth(curMonth, anew.year) * 24 * 60 * 60) + if curMonth == mJan: + curMonth = mDec + anew.year.dec() + else: + curMonth.dec() + else: # adding + for mth in 1 .. newinterv.months: + result += float(getDaysInMonth(curMonth, anew.year) * 24 * 60 * 60) + if curMonth == mDec: + curMonth = mJan + anew.year.inc() + else: + curMonth.inc() result += float(newinterv.days * 24 * 60 * 60) result += float(newinterv.hours * 60 * 60) result += float(newinterv.minutes * 60) @@ -302,28 +352,39 @@ proc toSeconds(a: TimeInfo, interval: TimeInterval): float = result += newinterv.milliseconds / 1000 proc `+`*(a: TimeInfo, interval: TimeInterval): TimeInfo = - ## adds ``interval`` time. + ## adds ``interval`` time from TimeInfo ``a``. ## ## **Note:** This has been only briefly tested and it may not be ## very accurate. let t = toSeconds(timeInfoToTime(a)) let secs = toSeconds(a, interval) - #if a.tzname == "UTC": - # result = getGMTime(fromSeconds(t + secs)) - #else: result = getLocalTime(fromSeconds(t + secs)) proc `-`*(a: TimeInfo, interval: TimeInterval): TimeInfo = - ## subtracts ``interval`` time. + ## subtracts ``interval`` time from TimeInfo ``a``. ## ## **Note:** This has been only briefly tested, it is inaccurate especially ## when you subtract so much that you reach the Julian calendar. let t = toSeconds(timeInfoToTime(a)) - let secs = toSeconds(a, interval) - #if a.tzname == "UTC": - # result = getGMTime(fromSeconds(t - secs)) - #else: - result = getLocalTime(fromSeconds(t - secs)) + var intval: TimeInterval + intval.milliseconds = - interval.milliseconds + intval.seconds = - interval.seconds + intval.minutes = - interval.minutes + intval.hours = - interval.hours + intval.days = - interval.days + intval.months = - interval.months + intval.years = - interval.years + let secs = toSeconds(a, intval) + result = getLocalTime(fromSeconds(t + secs)) + +proc miliseconds*(t: TimeInterval): int {.deprecated.} = t.milliseconds + +proc `miliseconds=`*(t: var TimeInterval, milliseconds: int) {.deprecated.} = + ## An alias for a misspelled field in ``TimeInterval``. + ## + ## **Warning:** This should not be used! It will be removed in the next + ## version. + t.milliseconds = milliseconds when not defined(JS): proc epochTime*(): float {.rtl, extern: "nt$1", tags: [TimeEffect].} @@ -603,6 +664,69 @@ proc `$`*(m: Month): string = "November", "December"] return lookup[m] +proc milliseconds*(ms: int): TimeInterval {.inline.} = + ## TimeInterval of `ms` milliseconds + ## + ## Note: not all time functions have millisecond resolution + initInterval(`mod`(ms,1000), `div`(ms,1000)) + +proc seconds*(s: int): TimeInterval {.inline.} = + ## TimeInterval of `s` seconds + ## + ## ``echo getTime() + 5.second`` + initInterval(0,`mod`(s,60), `div`(s,60)) + +proc minutes*(m: int): TimeInterval {.inline.} = + ## TimeInterval of `m` minutes + ## + ## ``echo getTime() + 5.minutes`` + initInterval(0,0,`mod`(m,60), `div`(m,60)) + +proc hours*(h: int): TimeInterval {.inline.} = + ## TimeInterval of `h` hours + ## + ## ``echo getTime() + 2.hours`` + initInterval(0,0,0,`mod`(h,24),`div`(h,24)) + +proc days*(d: int): TimeInterval {.inline.} = + ## TimeInterval of `d` days + ## + ## ``echo getTime() + 2.days`` + initInterval(0,0,0,0,d) + +proc months*(m: int): TimeInterval {.inline.} = + ## TimeInterval of `m` months + ## + ## ``echo getTime() + 2.months`` + initInterval(0,0,0,0,0,`mod`(m,12),`div`(m,12)) + +proc years*(y: int): TimeInterval {.inline.} = + ## TimeInterval of `y` years + ## + ## ``echo getTime() + 2.years`` + initInterval(0,0,0,0,0,0,y) + +proc `+=`*(t: var Time, ti: TimeInterval) = + ## modifies `t` by adding the interval `ti` + t = timeInfoToTime(getLocalTime(t) + ti) + +proc `+`*(t: Time, ti: TimeInterval): Time = + ## adds the interval `ti` to Time `t` + ## by converting to localTime, adding the interval, and converting back + ## + ## ``echo getTime() + 1.day`` + result = timeInfoToTime(getLocalTime(t) + ti) + +proc `-=`*(t: var Time, ti: TimeInterval) = + ## modifies `t` by subtracting the interval `ti` + t = timeInfoToTime(getLocalTime(t) - ti) + +proc `-`*(t: Time, ti: TimeInterval): Time = + ## adds the interval `ti` to Time `t` + ## + ## ``echo getTime() - 1.day`` + result = timeInfoToTime(getLocalTime(t) - ti) + proc formatToken(info: TimeInfo, token: string, buf: var string) = ## Helper of the format proc to parse individual tokens. ## @@ -1192,112 +1316,10 @@ proc timeToTimeInterval*(t: Time): TimeInterval = # Milliseconds not available from Time when isMainModule: - # $ date --date='@2147483647' - # Tue 19 Jan 03:14:07 GMT 2038 - - var t = getGMTime(fromSeconds(2147483647)) - assert t.format("ddd dd MMM hh:mm:ss ZZZ yyyy") == "Tue 19 Jan 03:14:07 UTC 2038" - assert t.format("ddd ddMMMhh:mm:ssZZZyyyy") == "Tue 19Jan03:14:07UTC2038" - - assert t.format("d dd ddd dddd h hh H HH m mm M MM MMM MMMM s" & - " ss t tt y yy yyy yyyy yyyyy z zz zzz ZZZ") == - "19 19 Tue Tuesday 3 03 3 03 14 14 1 01 Jan January 7 07 A AM 8 38 038 2038 02038 0 00 00:00 UTC" - - assert t.format("yyyyMMddhhmmss") == "20380119031407" - - var t2 = getGMTime(fromSeconds(160070789)) # Mon 27 Jan 16:06:29 GMT 1975 - assert t2.format("d dd ddd dddd h hh H HH m mm M MM MMM MMMM s" & - " ss t tt y yy yyy yyyy yyyyy z zz zzz ZZZ") == - "27 27 Mon Monday 4 04 16 16 6 06 1 01 Jan January 29 29 P PM 5 75 975 1975 01975 0 00 00:00 UTC" - - when not defined(JS): - when sizeof(Time) == 8: - var t3 = getGMTime(fromSeconds(889067643645)) # Fri 7 Jun 19:20:45 BST 30143 - assert t3.format("d dd ddd dddd h hh H HH m mm M MM MMM MMMM s" & - " ss t tt y yy yyy yyyy yyyyy z zz zzz ZZZ") == - "7 07 Fri Friday 6 06 18 18 20 20 6 06 Jun June 45 45 P PM 3 43 143 0143 30143 0 00 00:00 UTC" - assert t3.format(":,[]()-/") == ":,[]()-/" - - var t4 = getGMTime(fromSeconds(876124714)) # Mon 6 Oct 08:58:34 BST 1997 - assert t4.format("M MM MMM MMMM") == "10 10 Oct October" - - # Interval tests - assert((t4 - initInterval(years = 2)).format("yyyy") == "1995") - assert((t4 - initInterval(years = 7, minutes = 34, seconds = 24)).format("yyyy mm ss") == "1990 24 10") - - var s = "Tuesday at 09:04am on Dec 15, 2015" - var f = "dddd at hh:mmtt on MMM d, yyyy" - assert($s.parse(f) == "Tue Dec 15 09:04:00 2015") - # ANSIC = "Mon Jan _2 15:04:05 2006" - s = "Thu Jan 12 15:04:05 2006" - f = "ddd MMM dd HH:mm:ss yyyy" - assert($s.parse(f) == "Thu Jan 12 15:04:05 2006") - # UnixDate = "Mon Jan _2 15:04:05 MST 2006" - s = "Thu Jan 12 15:04:05 MST 2006" - f = "ddd MMM dd HH:mm:ss ZZZ yyyy" - assert($s.parse(f) == "Thu Jan 12 15:04:05 2006") - # RubyDate = "Mon Jan 02 15:04:05 -0700 2006" - s = "Thu Jan 12 15:04:05 -07:00 2006" - f = "ddd MMM dd HH:mm:ss zzz yyyy" - assert($s.parse(f) == "Thu Jan 12 15:04:05 2006") - # RFC822 = "02 Jan 06 15:04 MST" - s = "12 Jan 16 15:04 MST" - f = "dd MMM yy HH:mm ZZZ" - assert($s.parse(f) == "Tue Jan 12 15:04:00 2016") - # RFC822Z = "02 Jan 06 15:04 -0700" # RFC822 with numeric zone - s = "12 Jan 16 15:04 -07:00" - f = "dd MMM yy HH:mm zzz" - assert($s.parse(f) == "Tue Jan 12 15:04:00 2016") - # RFC850 = "Monday, 02-Jan-06 15:04:05 MST" - s = "Monday, 12-Jan-06 15:04:05 MST" - f = "dddd, dd-MMM-yy HH:mm:ss ZZZ" - assert($s.parse(f) == "Thu Jan 12 15:04:05 2006") - # RFC1123 = "Mon, 02 Jan 2006 15:04:05 MST" - s = "Thu, 12 Jan 2006 15:04:05 MST" - f = "ddd, dd MMM yyyy HH:mm:ss ZZZ" - assert($s.parse(f) == "Thu Jan 12 15:04:05 2006") - # RFC1123Z = "Mon, 02 Jan 2006 15:04:05 -0700" # RFC1123 with numeric zone - s = "Thu, 12 Jan 2006 15:04:05 -07:00" - f = "ddd, dd MMM yyyy HH:mm:ss zzz" - assert($s.parse(f) == "Thu Jan 12 15:04:05 2006") - # RFC3339 = "2006-01-02T15:04:05Z07:00" - s = "2006-01-12T15:04:05Z-07:00" - f = "yyyy-MM-ddTHH:mm:ssZzzz" - assert($s.parse(f) == "Thu Jan 12 15:04:05 2006") - f = "yyyy-MM-dd'T'HH:mm:ss'Z'zzz" - assert($s.parse(f) == "Thu Jan 12 15:04:05 2006") - # RFC3339Nano = "2006-01-02T15:04:05.999999999Z07:00" - s = "2006-01-12T15:04:05.999999999Z-07:00" - f = "yyyy-MM-ddTHH:mm:ss.999999999Zzzz" - assert($s.parse(f) == "Thu Jan 12 15:04:05 2006") - # Kitchen = "3:04PM" - s = "3:04PM" - f = "h:mmtt" - assert "15:04:00" in $s.parse(f) - when not defined(testing): - echo "Kitchen: " & $s.parse(f) - var ti = timeToTimeInfo(getTime()) - echo "Todays date after decoding: ", ti - var tint = timeToTimeInterval(getTime()) - echo "Todays date after decoding to interval: ", tint - # checking dayOfWeek matches known days - assert getDayOfWeek(21, 9, 1900) == dFri - assert getDayOfWeek(1, 1, 1970) == dThu - assert getDayOfWeek(21, 9, 1970) == dMon - assert getDayOfWeek(1, 1, 2000) == dSat - assert getDayOfWeek(1, 1, 2021) == dFri - # Julian tests - assert getDayOfWeekJulian(21, 9, 1900) == dFri - assert getDayOfWeekJulian(21, 9, 1970) == dMon - assert getDayOfWeekJulian(1, 1, 2000) == dSat - assert getDayOfWeekJulian(1, 1, 2021) == dFri - - # toSeconds tests with GM and Local timezones - #var t4 = getGMTime(fromSeconds(876124714)) # Mon 6 Oct 08:58:34 BST 1997 - var t4L = getLocalTime(fromSeconds(876124714)) - assert toSeconds(timeInfoToTime(t4L)) == 876124714 # fromSeconds is effectively "localTime" - assert toSeconds(timeInfoToTime(t4L)) + t4L.timezone.float == toSeconds(timeInfoToTime(t4)) - + # this is testing non-exported function + var + t4 = getGMTime(fromSeconds(876124714)) # Mon 6 Oct 08:58:34 BST 1997 + t4L = getLocalTime(fromSeconds(876124714)) assert toSeconds(t4, initInterval(seconds=0)) == 0.0 assert toSeconds(t4L, initInterval(milliseconds=1)) == toSeconds(t4, initInterval(milliseconds=1)) assert toSeconds(t4L, initInterval(seconds=1)) == toSeconds(t4, initInterval(seconds=1)) @@ -1307,12 +1329,5 @@ when isMainModule: assert toSeconds(t4L, initInterval(months=1)) == toSeconds(t4, initInterval(months=1)) assert toSeconds(t4L, initInterval(years=1)) == toSeconds(t4, initInterval(years=1)) - # adding intervals - var - a1L = toSeconds(timeInfoToTime(t4L + initInterval(hours = 1))) + t4L.timezone.float - a1G = toSeconds(timeInfoToTime(t4)) + 60.0 * 60.0 - assert a1L == a1G - # subtracting intervals - a1L = toSeconds(timeInfoToTime(t4L - initInterval(hours = 1))) + t4L.timezone.float - a1G = toSeconds(timeInfoToTime(t4)) - (60.0 * 60.0) - assert a1L == a1G + # Further tests are in tests/stdlib/ttime.nim + # koch test c stdlib diff --git a/lib/pure/unicode.nim b/lib/pure/unicode.nim index b059a7315..45f52eb7f 100644 --- a/lib/pure/unicode.nim +++ b/lib/pure/unicode.nim @@ -114,6 +114,7 @@ proc validateUtf8*(s: string): int = if ord(s[i]) <=% 127: inc(i) elif ord(s[i]) shr 5 == 0b110: + if ord(s[i]) < 0xc2: return i # Catch overlong ascii representations. if i+1 < L and ord(s[i+1]) shr 6 == 0b10: inc(i, 2) else: return i elif ord(s[i]) shr 4 == 0b1110: diff --git a/lib/pure/xmlparser.nim b/lib/pure/xmlparser.nim index 56b122000..2a2c3e1dd 100644 --- a/lib/pure/xmlparser.nim +++ b/lib/pure/xmlparser.nim @@ -96,7 +96,7 @@ proc parse(x: var XmlParser, errors: var seq[string]): XmlNode = next(x) of xmlEntity: ## &entity; - errors.add(errorMsg(x, "unknown entity: " & x.entityName)) + result = newEntity(x.entityName) next(x) of xmlEof: discard @@ -143,17 +143,24 @@ proc loadXml*(path: string): XmlNode = result = loadXml(path, errors) if errors.len > 0: raiseInvalidXml(errors) -when not defined(testing) and isMainModule: - import os +when isMainModule: + when not defined(testing): + import os - var errors: seq[string] = @[] - var x = loadXml(paramStr(1), errors) - for e in items(errors): echo e + var errors: seq[string] = @[] + var x = loadXml(paramStr(1), errors) + for e in items(errors): echo e - var f: File - if open(f, "xmltest.txt", fmWrite): - f.write($x) - f.close() + var f: File + if open(f, "xmltest.txt", fmWrite): + f.write($x) + f.close() + else: + quit("cannot write test.txt") else: - quit("cannot write test.txt") + block: # correctly parse ../../tests/testdata/doc1.xml + let filePath = "tests/testdata/doc1.xml" + var errors: seq[string] = @[] + var xml = loadXml(filePath, errors) + assert(errors.len == 0, "The file tests/testdata/doc1.xml should be parsed without errors.") diff --git a/lib/stdlib.nimble b/lib/stdlib.nimble index 0805ead54..e8bb364f1 100644 --- a/lib/stdlib.nimble +++ b/lib/stdlib.nimble @@ -1,6 +1,6 @@ [Package] name = "stdlib" -version = "0.9.0" +version = "0.13.0" author = "Dominik Picheta" description = "Nim's standard library." license = "MIT" diff --git a/lib/system.nim b/lib/system.nim index c5dd58c7b..e884e784c 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -232,8 +232,8 @@ proc low*[T](x: T): T {.magic: "Low", noSideEffect.} ## ## .. code-block:: nim ## var arr = [1,2,3,4,5,6,7] - ## high(arr) #=> 0 - ## high(2) #=> -9223372036854775808 + ## low(arr) #=> 0 + ## low(2) #=> -9223372036854775808 type range*{.magic: "Range".}[T] ## Generic type to construct range types. @@ -840,7 +840,7 @@ proc `div` *(x, y: int32): int32 {.magic: "DivI", noSideEffect.} ## 1 div 2 == 0 ## 2 div 2 == 1 ## 3 div 2 == 1 - ## 7 div 5 == 2 + ## 7 div 5 == 1 when defined(nimnomagic64): proc `div` *(x, y: int64): int64 {.magic: "DivI", noSideEffect.} @@ -1808,10 +1808,10 @@ const NimMajor*: int = 0 ## is the major number of Nim's version. - NimMinor*: int = 12 + NimMinor*: int = 13 ## is the minor number of Nim's version. - NimPatch*: int = 1 + NimPatch*: int = 0 ## is the patch number of Nim's version. NimVersion*: string = $NimMajor & "." & $NimMinor & "." & $NimPatch @@ -2584,11 +2584,7 @@ when not defined(JS): #and not defined(nimscript): when hasAlloc: var - strDesc: TNimType - - strDesc.size = sizeof(string) - strDesc.kind = tyString - strDesc.flags = {ntfAcyclic} + strDesc = TNimType(size: sizeof(string), kind: tyString, flags: {ntfAcyclic}) when not defined(nimscript): include "system/ansi_c" diff --git a/lib/system/alloc.nim b/lib/system/alloc.nim index 3ebbc8c1e..b4462ed83 100644 --- a/lib/system/alloc.nim +++ b/lib/system/alloc.nim @@ -40,7 +40,7 @@ when defined(emscripten): MAP_PRIVATE = 2'i32 # Changes are private var MAP_ANONYMOUS {.importc: "MAP_ANONYMOUS", header: "<sys/mman.h>".}: cint - type + type PEmscriptenMMapBlock = ptr EmscriptenMMapBlock EmscriptenMMapBlock {.pure, inheritable.} = object realSize: int # size of previous chunk; for coalescing @@ -399,6 +399,9 @@ iterator allObjects(m: MemRegion): pointer {.inline.} = let c = cast[PBigChunk](c) yield addr(c.data) +proc iterToProc*(iter: typed, envType: typedesc; procName: untyped) {. + magic: "Plugin", compileTime.} + proc isCell(p: pointer): bool {.inline.} = result = cast[ptr FreeCell](p).zeroField >% 1 diff --git a/lib/system/cellsets.nim b/lib/system/cellsets.nim index bb5de6f42..776a2b7ec 100644 --- a/lib/system/cellsets.nim +++ b/lib/system/cellsets.nim @@ -201,6 +201,41 @@ iterator elements(t: CellSet): PCell {.inline.} = inc(i) r = r.next +when false: + type + CellSetIter = object + p: PPageDesc + i, w, j: int + + proc next(it: var CellSetIter): PCell = + while true: + while it.w != 0: # test all remaining bits for zero + if (it.w and 1) != 0: # the bit is set! + result = cast[PCell]((it.p.key shl PageShift) or + (it.i shl IntShift +% it.j) *% MemAlign) + + inc(it.j) + it.w = it.w shr 1 + return + else: + inc(it.j) + it.w = it.w shr 1 + # load next w: + if it.i >= high(it.p.bits): + it.i = 0 + it.j = 0 + it.p = it.p.next + if it.p == nil: return nil + else: + inc it.i + it.w = it.p.bits[i] + + proc init(it: var CellSetIter; t: CellSet): PCell = + it.p = t.head + it.i = -1 + it.w = 0 + result = it.next + iterator elementsExcept(t, s: CellSet): PCell {.inline.} = var r = t.head while r != nil: diff --git a/lib/system/dyncalls.nim b/lib/system/dyncalls.nim index 908aa551b..6dc8999d1 100644 --- a/lib/system/dyncalls.nim +++ b/lib/system/dyncalls.nim @@ -68,7 +68,10 @@ when defined(posix): proc nimLoadLibrary(path: string): LibHandle = result = dlopen(path, RTLD_NOW) - #c_fprintf(c_stdout, "%s\n", dlerror()) + when defined(nimDebugDlOpen): + let error = dlerror() + if error != nil: + c_fprintf(c_stdout, "%s\n", error) proc nimGetProcAddr(lib: LibHandle, name: cstring): ProcAddr = result = dlsym(lib, name) @@ -105,7 +108,12 @@ elif defined(windows) or defined(dos): proc nimGetProcAddr(lib: LibHandle, name: cstring): ProcAddr = result = getProcAddress(cast[THINSTANCE](lib), name) - if result == nil: procAddrError(name) + if result != nil: return + for i in countup(0, 50): + var decorated = "_" & $name & "@" & $(i * 4) + result = getProcAddress(cast[THINSTANCE](lib), cstring(decorated)) + if result != nil: return + procAddrError(name) else: {.error: "no implementation for dyncalls".} diff --git a/lib/system/gc.nim b/lib/system/gc.nim index 0c632aeb1..c25cf4606 100644 --- a/lib/system/gc.nim +++ b/lib/system/gc.nim @@ -558,7 +558,7 @@ proc growObj(old: pointer, newsize: int, gch: var GcHeap): pointer = # we split the old refcount in 2 parts. XXX This is still not entirely # correct if the pointer that receives growObj's result is on the stack. # A better fix would be to emit the location specific write barrier for - # 'growObj', but this is lost of more work and who knows what new problems + # 'growObj', but this is lots of more work and who knows what new problems # this would create. res.refcount = rcIncrement decRef(ol) diff --git a/lib/system/gc2.nim b/lib/system/gc2.nim index 4ca0d144f..e68a8586e 100644 --- a/lib/system/gc2.nim +++ b/lib/system/gc2.nim @@ -1,7 +1,7 @@ # # # Nim's Runtime Library -# (c) Copyright 2012 Andreas Rumpf +# (c) Copyright 2015 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -9,13 +9,15 @@ # Garbage Collector # -# The basic algorithm is *Deferrent Reference Counting* with cycle detection. -# This is achieved by combining a Deutsch-Bobrow garbage collector -# together with Christoper's partial mark-sweep garbage collector. -# -# Special care has been taken to avoid recursion as far as possible to avoid -# stack overflows when traversing deep datastructures. It is well-suited -# for soft real time applications (like games). +# The basic algorithm is *Deferred Reference Counting* with an incremental mark +# and sweep GC to free cycles. It is hard realtime in that if you play +# according to its rules, no deadline will ever be missed. + +# XXX Ensure by smart color masking that the object is not in the ZCT. + +when defined(nimCoroutines): + import arch + {.push profiler:off.} const @@ -29,82 +31,36 @@ const when withRealTime and not declared(getTicks): include "system/timers" when defined(memProfiler): - proc nimProfile(requestedSize: int) - -const - rcShift = 6 # the reference count is shifted so we can use - # the least significat bits for additinal flags: - - rcAlive = 0b00000 # object is reachable. - # color *black* in the original paper - - rcCycleCandidate = 0b00001 # possible root of a cycle. *purple* - - rcDecRefApplied = 0b00010 # the first dec-ref phase of the - # collector was already applied to this - # object. *gray* - - rcMaybeDead = 0b00011 # this object is a candidate for deletion - # during the collect cycles algorithm. - # *white*. - - rcReallyDead = 0b00100 # this is proved to be garbage - - rcRetiredBuffer = 0b00101 # this is a seq or string buffer that - # was replaced by a resize operation. - # see growObj for details + proc nimProfile(requestedSize: int) {.benign.} - rcColorMask = RefCount(0b00111) - - rcZct = 0b01000 # already added to ZCT - rcInCycleRoots = 0b10000 # already buffered as cycle candidate - rcHasStackRef = 0b100000 # the object had a stack ref in the last - # cycle collection - - rcMarkBit = rcHasStackRef # this is currently used for leak detection - # when traceGC is on - - rcBufferedAnywhere = rcZct or rcInCycleRoots +type + ObjectSpaceIter = object + state: range[-1..0] - rcIncrement = 1 shl rcShift # don't touch the color bits +iterToProc(allObjects, ptr ObjectSpaceIter, allObjectsAsProc) const - NewObjectsAreCycleRoots = true - # the alternative is to use the old strategy of adding cycle roots - # in incRef (in the compiler itself, this doesn't change much) - - IncRefRemovesCandidates = false - # this is safe only if we can reliably track the fact that the object - # has stack references. This could be easily done by adding another bit - # to the refcount field and setting it up in unmarkStackAndRegisters. - # The bit must also be set for new objects that are not rc1 and it must be - # examined in the decref loop in collectCycles. - # XXX: not implemented yet as tests didn't show any improvement from this - - MarkingSkipsAcyclicObjects = true - # Acyclic objects can be safely ignored in the mark and scan phases, - # because they cannot contribute to the internal count. - # XXX: if we generate specialized `markCyclic` and `markAcyclic` - # procs we can further optimize this as there won't be need for any - # checks in the code - - MinimumStackMarking = false - # Try to scan only the user stack and ignore the part of the stack - # belonging to the GC itself. see setStackTop for further info. - # XXX: still has problems in release mode in the compiler itself. - # investigate how it affects growObj - - CollectCyclesStats = false - + rcIncrement = 0b1000 # so that lowest 3 bits are not touched + rcBlackOrig = 0b000 + rcWhiteOrig = 0b001 + rcGrey = 0b010 # traditional color for incremental mark&sweep + rcUnused = 0b011 + ZctFlag = 0b100 # in ZCT + rcShift = 3 # shift by rcShift to get the reference counter + colorMask = 0b011 type WalkOp = enum - waPush + waMarkGlobal, # part of the backup mark&sweep + waMarkGrey, + waZctDecRef #, waDebug - Finalizer {.compilerproc.} = proc (self: pointer) {.nimcall.} + Phase {.pure.} = enum + None, Marking, Sweeping + Finalizer {.compilerproc.} = proc (self: pointer) {.nimcall, benign.} # A ref type can have a finalizer that is called before the object's # storage is freed. - GcStat {.final, pure.} = object + GcStat = object stackScans: int # number of performed stack scans (for statistics) cycleCollections: int # number of performed full collections maxThreshold: int # max threshold that has been set @@ -113,134 +69,78 @@ type cycleTableSize: int # max entries in cycle table maxPause: int64 # max measured GC pause in nanoseconds - GcHeap {.final, pure.} = object # this contains the zero count and - # non-zero count table + GcStack = object + prev: ptr GcStack + next: ptr GcStack + starts: pointer + pos: pointer + maxStackSize: int + + GcHeap = object # this contains the zero count and + # non-zero count table + black: int # either 0 or 1. + stack: ptr GcStack stackBottom: pointer - stackTop: pointer + phase: Phase cycleThreshold: int + when useCellIds: + idGenerator: int zct: CellSeq # the zero count table decStack: CellSeq # cells in the stack that are to decref again - cycleRoots: CellSeq - tempStack: CellSeq # temporary stack for recursion elimination - freeStack: CellSeq # objects ready to be freed + greyStack: CellSeq recGcLock: int # prevent recursion via finalizers; no thread lock - cycleRootsTrimIdx: int # Trimming is a light-weight collection of the - # cycle roots table that uses a cheap linear scan - # to find only possitively dead objects. - # One strategy is to perform it only for new objects - # allocated between the invocations of collectZCT. - # This index indicates the start of the range of - # such new objects within the table. when withRealTime: maxPause: Nanos # max allowed pause in nanoseconds; active if > 0 region: MemRegion # garbage collected region stat: GcStat -{.deprecated: [TWalkOp: WalkOp, TFinalizer: Finalizer, TGcStat: GcStat, - TGcHeap: GcHeap].} + additionalRoots: CellSeq # dummy roots for GC_ref/unref + spaceIter: ObjectSpaceIter + var - gch* {.rtlThreadVar.}: GcHeap + gch {.rtlThreadVar.}: GcHeap when not defined(useNimRtl): instantiateForRegion(gch.region) -template acquire(gch: GcHeap) = - when hasThreadSupport and hasSharedHeap: - AcquireSys(HeapLock) - -template release(gch: GcHeap) = - when hasThreadSupport and hasSharedHeap: - releaseSys(HeapLock) - -template setColor(c: PCell, color) = - c.refcount = (c.refcount and not rcColorMask) or color - -template color(c: PCell): expr = - c.refcount and rcColorMask - -template isBitDown(c: PCell, bit): expr = - (c.refcount and bit) == 0 - -template isBitUp(c: PCell, bit): expr = - (c.refcount and bit) != 0 - -template setBit(c: PCell, bit): expr = - c.refcount = c.refcount or bit - -template isDead(c: Pcell): expr = - c.isBitUp(rcReallyDead) # also covers rcRetiredBuffer - -template clearBit(c: PCell, bit): expr = - c.refcount = c.refcount and (not RefCount(bit)) - -when debugGC: - var gcCollectionIdx = 0 - - proc colorStr(c: PCell): cstring = - let color = c.color - case color - of rcAlive: return "alive" - of rcMaybeDead: return "maybedead" - of rcCycleCandidate: return "candidate" - of rcDecRefApplied: return "marked" - of rcRetiredBuffer: return "retired" - of rcReallyDead: return "dead" - else: return "unknown?" - - proc inCycleRootsStr(c: PCell): cstring = - if c.isBitUp(rcInCycleRoots): result = "cycleroot" - else: result = "" - - proc inZctStr(c: PCell): cstring = - if c.isBitUp(rcZct): result = "zct" - else: result = "" - - proc writeCell*(msg: CString, c: PCell, force = false) = - var kind = -1 - if c.typ != nil: kind = ord(c.typ.kind) - when trackAllocationSource: - c_fprintf(c_stdout, "[GC %d] %s: %p %d rc=%ld %s %s %s from %s(%ld)\n", - gcCollectionIdx, - msg, c, kind, c.refcount shr rcShift, - c.colorStr, c.inCycleRootsStr, c.inZctStr, - c.filename, c.line) - else: - c_fprintf(c_stdout, "[GC] %s: %p %d rc=%ld\n", - msg, c, kind, c.refcount shr rcShift) - -proc addZCT(zct: var CellSeq, c: PCell) {.noinline.} = - if c.isBitDown(rcZct): - c.setBit rcZct - zct.add c - -template setStackTop(gch) = - # This must be called immediately after we enter the GC code - # to minimize the size of the scanned stack. The stack consumed - # by the GC procs may amount to 200-400 bytes depending on the - # build settings and this contributes to false-positives - # in the conservative stack marking - when MinimumStackMarking: - var stackTop {.volatile.}: pointer - gch.stackTop = addr(stackTop) - -template addCycleRoot(cycleRoots: var CellSeq, c: PCell) = - if c.color != rcCycleCandidate: - c.setColor rcCycleCandidate - - # the object may be buffered already. for example, consider: - # decref; incref; decref - if c.isBitDown(rcInCycleRoots): - c.setBit rcInCycleRoots - cycleRoots.add c +proc initGC() = + when not defined(useNimRtl): + when traceGC: + for i in low(CellState)..high(CellState): init(states[i]) + gch.cycleThreshold = InitialCycleThreshold + gch.stat.stackScans = 0 + gch.stat.cycleCollections = 0 + gch.stat.maxThreshold = 0 + gch.stat.maxStackSize = 0 + gch.stat.maxStackCells = 0 + gch.stat.cycleTableSize = 0 + # init the rt + init(gch.zct) + init(gch.decStack) + init(gch.additionalRoots) + init(gch.greyStack) + +template gcAssert(cond: bool, msg: string) = + when defined(useGcAssert): + if not cond: + echo "[GCASSERT] ", msg + GC_disable() + writeStackTrace() + quit 1 + +proc addZCT(s: var CellSeq, c: PCell) {.noinline.} = + if (c.refcount and ZctFlag) == 0: + c.refcount = c.refcount or ZctFlag + add(s, c) proc cellToUsr(cell: PCell): pointer {.inline.} = # convert object (=pointer to refcount) to pointer to userdata result = cast[pointer](cast[ByteAddress](cell)+%ByteAddress(sizeof(Cell))) -proc usrToCell*(usr: pointer): PCell {.inline.} = +proc usrToCell(usr: pointer): PCell {.inline.} = # convert pointer to userdata to object (=pointer to refcount) result = cast[PCell](cast[ByteAddress](usr)-%ByteAddress(sizeof(Cell))) -proc canbeCycleRoot(c: PCell): bool {.inline.} = +proc canBeCycleRoot(c: PCell): bool {.inline.} = result = ntfAcyclic notin c.typ.flags proc extGetCellType(c: pointer): PNimType {.compilerproc.} = @@ -254,14 +154,40 @@ proc internRefcount(p: pointer): int {.exportc: "getRefcount".} = when BitsPerPage mod (sizeof(int)*8) != 0: {.error: "(BitsPerPage mod BitsPerUnit) should be zero!".} +template color(c): expr = c.refCount and colorMask +template setColor(c, col) = + c.refcount = c.refcount and not colorMask or col + +proc writeCell(msg: cstring, c: PCell) = + var kind = -1 + if c.typ != nil: kind = ord(c.typ.kind) + when leakDetector: + c_fprintf(c_stdout, "[GC] %s: %p %d rc=%ld from %s(%ld)\n", + msg, c, kind, c.refcount shr rcShift, c.filename, c.line) + else: + c_fprintf(c_stdout, "[GC] %s: %p %d rc=%ld; color=%ld\n", + msg, c, kind, c.refcount shr rcShift, c.color) + +template gcTrace(cell, state: expr): stmt {.immediate.} = + when traceGC: traceCell(cell, state) + # forward declarations: -proc collectCT(gch: var GcHeap) -proc isOnStack*(p: pointer): bool {.noinline.} -proc forAllChildren(cell: PCell, op: WalkOp) -proc doOperation(p: pointer, op: WalkOp) -proc forAllChildrenAux(dest: pointer, mt: PNimType, op: WalkOp) +proc collectCT(gch: var GcHeap) {.benign.} +proc isOnStack(p: pointer): bool {.noinline, benign.} +proc forAllChildren(cell: PCell, op: WalkOp) {.benign.} +proc doOperation(p: pointer, op: WalkOp) {.benign.} +proc forAllChildrenAux(dest: pointer, mt: PNimType, op: WalkOp) {.benign.} # we need the prototype here for debugging purposes +when hasThreadSupport and hasSharedHeap: + template `--`(x: expr): expr = atomicDec(x, rcIncrement) <% rcIncrement + template `++`(x: expr): stmt = discard atomicInc(x, rcIncrement) +else: + template `--`(x: expr): expr = + dec(x, rcIncrement) + x <% rcIncrement + template `++`(x: expr): stmt = inc(x, rcIncrement) + proc prepareDealloc(cell: PCell) = if cell.typ.finalizer != nil: # the finalizer could invoke something that @@ -273,246 +199,127 @@ proc prepareDealloc(cell: PCell) = (cast[Finalizer](cell.typ.finalizer))(cellToUsr(cell)) dec(gch.recGcLock) -when traceGC: - # traceGC is a special switch to enable extensive debugging - type - CellState = enum - csAllocated, csFreed - {.deprecated: [TCellState: CellState].} - var - states: array[CellState, CellSet] - - proc traceCell(c: PCell, state: CellState) = - case state - of csAllocated: - if c in states[csAllocated]: - writeCell("attempt to alloc an already allocated cell", c) - sysAssert(false, "traceCell 1") - excl(states[csFreed], c) - # writecell("allocated", c) - of csFreed: - if c in states[csFreed]: - writeCell("attempt to free a cell twice", c) - sysAssert(false, "traceCell 2") - if c notin states[csAllocated]: - writeCell("attempt to free not an allocated cell", c) - sysAssert(false, "traceCell 3") - excl(states[csAllocated], c) - # writecell("freed", c) - incl(states[state], c) - - proc computeCellWeight(c: PCell): int = - var x: CellSet - x.init - - let startLen = gch.tempStack.len - c.forAllChildren waPush - - while startLen != gch.tempStack.len: - dec gch.tempStack.len - var c = gch.tempStack.d[gch.tempStack.len] - if c in states[csFreed]: continue - inc result - if c notin x: - x.incl c - c.forAllChildren waPush - - template markChildrenRec(cell) = - let startLen = gch.tempStack.len - cell.forAllChildren waPush - let isMarked = cell.isBitUp(rcMarkBit) - while startLen != gch.tempStack.len: - dec gch.tempStack.len - var c = gch.tempStack.d[gch.tempStack.len] - if c in states[csFreed]: continue - if c.isBitDown(rcMarkBit): - c.setBit rcMarkBit - c.forAllChildren waPush - if c.isBitUp(rcMarkBit) and not isMarked: - writecell("cyclic cell", cell) - cprintf "Weight %d\n", cell.computeCellWeight - - proc writeLeakage(onlyRoots: bool) = - if onlyRoots: - for c in elements(states[csAllocated]): - if c notin states[csFreed]: - markChildrenRec(c) - var f = 0 - var a = 0 - for c in elements(states[csAllocated]): - inc a - if c in states[csFreed]: inc f - elif c.isBitDown(rcMarkBit): - writeCell("leak", c) - cprintf "Weight %d\n", c.computeCellWeight - cfprintf(cstdout, "Allocations: %ld; freed: %ld\n", a, f) - -template gcTrace(cell, state: expr): stmt {.immediate.} = - when logGC: writeCell($state, cell) - when traceGC: traceCell(cell, state) - -template WithHeapLock(blk: stmt): stmt = - when hasThreadSupport and hasSharedHeap: AcquireSys(HeapLock) - blk - when hasThreadSupport and hasSharedHeap: ReleaseSys(HeapLock) - proc rtlAddCycleRoot(c: PCell) {.rtl, inl.} = # we MUST access gch as a global here, because this crosses DLL boundaries! - WithHeapLock: addCycleRoot(gch.cycleRoots, c) + discard proc rtlAddZCT(c: PCell) {.rtl, inl.} = # we MUST access gch as a global here, because this crosses DLL boundaries! - WithHeapLock: addZCT(gch.zct, c) + addZCT(gch.zct, c) -type - CyclicMode = enum - Cyclic, - Acyclic, - MaybeCyclic - - ReleaseType = enum - AddToZTC - FreeImmediately - - HeapType = enum - LocalHeap - SharedHeap -{.deprecated: [TCyclicMode: CyclicMode, TReleaseType: ReleaseType, - THeapType: HeapType].} - -template `++` (rc: RefCount, heapType: HeapType): stmt = - when heapType == SharedHeap: - discard atomicInc(rc, rcIncrement) - else: - inc rc, rcIncrement - -template `--`(rc: RefCount): expr = - dec rc, rcIncrement - rc <% rcIncrement - -template `--` (rc: RefCount, heapType: HeapType): expr = - (when heapType == SharedHeap: atomicDec(rc, rcIncrement) <% rcIncrement else: --rc) - -template doDecRef(cc: PCell, - heapType = LocalHeap, - cycleFlag = MaybeCyclic): stmt = - var c = cc - sysAssert(isAllocatedPtr(gch.region, c), "decRef: interiorPtr") - # XXX: move this elesewhere - - sysAssert(c.refcount >=% rcIncrement, "decRef") - if c.refcount--(heapType): - # this is the last reference from the heap - # add to a zero-count-table that will be matched against stack pointers +proc decRef(c: PCell) {.inline.} = + gcAssert(isAllocatedPtr(gch.region, c), "decRef: interiorPtr") + gcAssert(c.refcount >=% rcIncrement, "decRef") + if --c.refcount: rtlAddZCT(c) - else: - when cycleFlag != Acyclic: - if cycleFlag == Cyclic or canBeCycleRoot(c): - # a cycle may have been broken - rtlAddCycleRoot(c) - -template doIncRef(cc: PCell, - heapType = LocalHeap, - cycleFlag = MaybeCyclic): stmt = - var c = cc - c.refcount++(heapType) - when cycleFlag != Acyclic: - when NewObjectsAreCycleRoots: - if canbeCycleRoot(c): - addCycleRoot(gch.cycleRoots, c) - elif IncRefRemovesCandidates: - c.setColor rcAlive - # XXX: this is not really atomic enough! - -proc nimGCref(p: pointer) {.compilerProc, inline.} = doIncRef(usrToCell(p)) -proc nimGCunref(p: pointer) {.compilerProc, inline.} = doDecRef(usrToCell(p)) + +proc incRef(c: PCell) {.inline.} = + gcAssert(isAllocatedPtr(gch.region, c), "incRef: interiorPtr") + c.refcount = c.refcount +% rcIncrement + +proc nimGCref(p: pointer) {.compilerProc.} = + let cell = usrToCell(p) + incRef(cell) + add(gch.additionalRoots, cell) + +proc nimGCunref(p: pointer) {.compilerProc.} = + let cell = usrToCell(p) + decRef(cell) + var L = gch.additionalRoots.len-1 + var i = L + let d = gch.additionalRoots.d + while i >= 0: + if d[i] == cell: + d[i] = d[L] + dec gch.additionalRoots.len + break + dec(i) + +template markGrey(x: PCell) = + if x.color == 1-gch.black and gch.phase == Phase.Marking: + x.setColor(rcGrey) + add(gch.greyStack, x) + +proc GC_addCycleRoot*[T](p: ref T) {.inline.} = + ## adds 'p' to the cycle candidate set for the cycle collector. It is + ## necessary if you used the 'acyclic' pragma for optimization + ## purposes and need to break cycles manually. + rtlAddCycleRoot(usrToCell(cast[pointer](p))) proc nimGCunrefNoCycle(p: pointer) {.compilerProc, inline.} = sysAssert(allocInv(gch.region), "begin nimGCunrefNoCycle") var c = usrToCell(p) - sysAssert(isAllocatedPtr(gch.region, c), "nimGCunrefNoCycle: isAllocatedPtr") - if c.refcount--(LocalHeap): + gcAssert(isAllocatedPtr(gch.region, c), "nimGCunrefNoCycle: isAllocatedPtr") + if --c.refcount: rtlAddZCT(c) sysAssert(allocInv(gch.region), "end nimGCunrefNoCycle 2") sysAssert(allocInv(gch.region), "end nimGCunrefNoCycle 5") -template doAsgnRef(dest: PPointer, src: pointer, - heapType = LocalHeap, cycleFlag = MaybeCyclic): stmt = - sysAssert(not isOnStack(dest), "asgnRef") - # BUGFIX: first incRef then decRef! - if src != nil: doIncRef(usrToCell(src), heapType, cycleFlag) - if dest[] != nil: doDecRef(usrToCell(dest[]), heapType, cycleFlag) - dest[] = src - proc asgnRef(dest: PPointer, src: pointer) {.compilerProc, inline.} = # the code generator calls this proc! - doAsgnRef(dest, src, LocalHeap, MaybeCyclic) + gcAssert(not isOnStack(dest), "asgnRef") + # BUGFIX: first incRef then decRef! + if src != nil: + let s = usrToCell(src) + incRef(s) + markGrey(s) + if dest[] != nil: decRef(usrToCell(dest[])) + dest[] = src proc asgnRefNoCycle(dest: PPointer, src: pointer) {.compilerProc, inline.} = # the code generator calls this proc if it is known at compile time that no # cycle is possible. - doAsgnRef(dest, src, LocalHeap, Acyclic) + gcAssert(not isOnStack(dest), "asgnRefNoCycle") + if src != nil: + var c = usrToCell(src) + ++c.refcount + markGrey(c) + if dest[] != nil: + var c = usrToCell(dest[]) + if --c.refcount: + rtlAddZCT(c) + dest[] = src proc unsureAsgnRef(dest: PPointer, src: pointer) {.compilerProc.} = # unsureAsgnRef updates the reference counters only if dest is not on the # stack. It is used by the code generator if it cannot decide wether a # reference is in the stack or not (this can happen for var parameters). if not isOnStack(dest): - if src != nil: doIncRef(usrToCell(src)) - # XXX we must detect a shared heap here - # better idea may be to just eliminate the need for unsureAsgnRef - # + if src != nil: + let s = usrToCell(src) + incRef(s) + markGrey(s) # XXX finally use assembler for the stack checking instead! # the test for '!= nil' is correct, but I got tired of the segfaults # resulting from the crappy stack checking: - if cast[int](dest[]) >=% PageSize: doDecRef(usrToCell(dest[])) + if cast[int](dest[]) >=% PageSize: decRef(usrToCell(dest[])) else: # can't be an interior pointer if it's a stack location! - sysAssert(interiorAllocatedPtr(gch.region, dest)==nil, - "stack loc AND interior pointer") + gcAssert(interiorAllocatedPtr(gch.region, dest) == nil, + "stack loc AND interior pointer") dest[] = src -when hasThreadSupport and hasSharedHeap: - # shared heap version of the above procs - proc asgnRefSh(dest: PPointer, src: pointer) {.compilerProc, inline.} = - doAsgnRef(dest, src, SharedHeap, MaybeCyclic) - - proc asgnRefNoCycleSh(dest: PPointer, src: pointer) {.compilerProc, inline.} = - doAsgnRef(dest, src, SharedHeap, Acyclic) +type + GlobalMarkerProc = proc () {.nimcall, benign.} +var + globalMarkersLen: int + globalMarkers: array[0.. 7_000, GlobalMarkerProc] -proc initGC() = - when not defined(useNimRtl): - when traceGC: - for i in low(CellState)..high(CellState): init(states[i]) - gch.cycleThreshold = InitialCycleThreshold - gch.stat.stackScans = 0 - gch.stat.cycleCollections = 0 - gch.stat.maxThreshold = 0 - gch.stat.maxStackSize = 0 - gch.stat.maxStackCells = 0 - gch.stat.cycleTableSize = 0 - # init the rt - init(gch.zct) - init(gch.tempStack) - init(gch.freeStack) - init(gch.cycleRoots) - init(gch.decStack) +proc nimRegisterGlobalMarker(markerProc: GlobalMarkerProc) {.compilerProc.} = + if globalMarkersLen <= high(globalMarkers): + globalMarkers[globalMarkersLen] = markerProc + inc globalMarkersLen + else: + echo "[GC] cannot register global variable; too many global variables" + quit 1 -proc forAllSlotsAux(dest: pointer, n: ptr TNimNode, op: WalkOp) = +proc forAllSlotsAux(dest: pointer, n: ptr TNimNode, op: WalkOp) {.benign.} = var d = cast[ByteAddress](dest) case n.kind of nkSlot: forAllChildrenAux(cast[pointer](d +% n.offset), n.typ, op) of nkList: for i in 0..n.len-1: - # inlined for speed - if n.sons[i].kind == nkSlot: - if n.sons[i].typ.kind in {tyRef, tyString, tySequence}: - doOperation(cast[PPointer](d +% n.sons[i].offset)[], op) - else: - forAllChildrenAux(cast[pointer](d +% n.sons[i].offset), - n.sons[i].typ, op) - else: - forAllSlotsAux(dest, n.sons[i], op) + forAllSlotsAux(dest, n.sons[i], op) of nkCase: var m = selectBranch(dest, n) if m != nil: forAllSlotsAux(dest, m, op) @@ -533,9 +340,10 @@ proc forAllChildrenAux(dest: pointer, mt: PNimType, op: WalkOp) = else: discard proc forAllChildren(cell: PCell, op: WalkOp) = - sysAssert(cell != nil, "forAllChildren: 1") - sysAssert(cell.typ != nil, "forAllChildren: 2") - sysAssert cell.typ.kind in {tyRef, tySequence, tyString}, "forAllChildren: 3" + gcAssert(cell != nil, "forAllChildren: 1") + gcAssert(isAllocatedPtr(gch.region, cell), "forAllChildren: 2") + gcAssert(cell.typ != nil, "forAllChildren: 3") + gcAssert cell.typ.kind in {tyRef, tySequence, tyString}, "forAllChildren: 4" let marker = cell.typ.marker if marker != nil: marker(cellToUsr(cell), op.int) @@ -547,10 +355,9 @@ proc forAllChildren(cell: PCell, op: WalkOp) = var d = cast[ByteAddress](cellToUsr(cell)) var s = cast[PGenericSeq](d) if s != nil: - let baseAddr = d +% GenericSeqSize for i in 0..s.len-1: - forAllChildrenAux(cast[pointer](baseAddr +% i *% cell.typ.base.size), - cell.typ.base, op) + forAllChildrenAux(cast[pointer](d +% i *% cell.typ.base.size +% + GenericSeqSize), cell.typ.base, op) else: discard proc addNewObjToZCT(res: PCell, gch: var GcHeap) {.inline.} = @@ -571,7 +378,7 @@ proc addNewObjToZCT(res: PCell, gch: var GcHeap) {.inline.} = template replaceZctEntry(i: expr) = c = d[i] if c.refcount >=% rcIncrement: - c.clearBit(rcZct) + c.refcount = c.refcount and not ZctFlag d[i] = res return if L > 8: @@ -592,408 +399,335 @@ proc addNewObjToZCT(res: PCell, gch: var GcHeap) {.inline.} = for i in countdown(L-1, max(0, L-8)): var c = d[i] if c.refcount >=% rcIncrement: - c.clearBit(rcZct) + c.refcount = c.refcount and not ZctFlag d[i] = res return add(gch.zct, res) -proc rawNewObj(typ: PNimType, size: int, gch: var GcHeap, rc1 = false): pointer = +{.push stackTrace: off, profiler:off.} +proc gcInvariant*() = + sysAssert(allocInv(gch.region), "injected") + when declared(markForDebug): + markForDebug(gch) +{.pop.} + +proc rawNewObj(typ: PNimType, size: int, gch: var GcHeap): pointer = # generates a new object and sets its reference counter to 0 - acquire(gch) sysAssert(allocInv(gch.region), "rawNewObj begin") - sysAssert(typ.kind in {tyRef, tyString, tySequence}, "newObj: 1") - + gcAssert(typ.kind in {tyRef, tyString, tySequence}, "newObj: 1") collectCT(gch) - sysAssert(allocInv(gch.region), "rawNewObj after collect") - var res = cast[PCell](rawAlloc(gch.region, size + sizeof(Cell))) - sysAssert(allocInv(gch.region), "rawNewObj after rawAlloc") - - sysAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "newObj: 2") - + gcAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "newObj: 2") + # now it is buffered in the ZCT res.typ = typ - - when trackAllocationSource and not hasThreadSupport: - if framePtr != nil and framePtr.prev != nil and framePtr.prev.prev != nil: - res.filename = framePtr.prev.prev.filename - res.line = framePtr.prev.prev.line - else: - res.filename = "nofile" - - if rc1: - res.refcount = rcIncrement # refcount is 1 - else: - # its refcount is zero, so add it to the ZCT: - res.refcount = rcZct - addNewObjToZCT(res, gch) - - if NewObjectsAreCycleRoots and canBeCycleRoot(res): - res.setBit(rcInCycleRoots) - res.setColor rcCycleCandidate - gch.cycleRoots.add res - + when leakDetector and not hasThreadSupport: + if framePtr != nil and framePtr.prev != nil: + res.filename = framePtr.prev.filename + res.line = framePtr.prev.line + # refcount is zero, color is black, but mark it to be in the ZCT + res.refcount = ZctFlag or gch.black sysAssert(isAllocatedPtr(gch.region, res), "newObj: 3") - + # its refcount is zero, so add it to the ZCT: + addNewObjToZCT(res, gch) when logGC: writeCell("new cell", res) gcTrace(res, csAllocated) - release(gch) + when useCellIds: + inc gch.idGenerator + res.id = gch.idGenerator result = cellToUsr(res) sysAssert(allocInv(gch.region), "rawNewObj end") {.pop.} -proc freeCell(gch: var GcHeap, c: PCell) = - # prepareDealloc(c) - gcTrace(c, csFreed) - - when reallyDealloc: rawDealloc(gch.region, c) - else: - sysAssert(c.typ != nil, "collectCycles") - zeroMem(c, sizeof(Cell)) - -template eraseAt(cells: var CellSeq, at: int): stmt = - cells.d[at] = cells.d[cells.len - 1] - dec cells.len - -template trimAt(roots: var CellSeq, at: int): stmt = - # This will remove a cycle root candidate during trimming. - # a candidate is removed either because it received a refup and - # it's no longer a candidate or because it received further refdowns - # and now it's dead for sure. - let c = roots.d[at] - c.clearBit(rcInCycleRoots) - roots.eraseAt(at) - if c.isBitUp(rcReallyDead) and c.refcount <% rcIncrement: - # This case covers both dead objects and retired buffers - # That's why we must also check the refcount (it may be - # kept possitive by stack references). - freeCell(gch, c) +proc newObjNoInit(typ: PNimType, size: int): pointer {.compilerRtl.} = + result = rawNewObj(typ, size, gch) + when defined(memProfiler): nimProfile(size) proc newObj(typ: PNimType, size: int): pointer {.compilerRtl.} = - setStackTop(gch) - result = rawNewObj(typ, size, gch, false) + result = rawNewObj(typ, size, gch) zeroMem(result, size) when defined(memProfiler): nimProfile(size) -proc newObjNoInit(typ: PNimType, size: int): pointer {.compilerRtl.} = - setStackTop(gch) - result = rawNewObj(typ, size, gch, false) - when defined(memProfiler): nimProfile(size) - proc newSeq(typ: PNimType, len: int): pointer {.compilerRtl.} = - setStackTop(gch) # `newObj` already uses locks, so no need for them here. let size = addInt(mulInt(len, typ.base.size), GenericSeqSize) result = newObj(typ, size) cast[PGenericSeq](result).len = len cast[PGenericSeq](result).reserved = len + when defined(memProfiler): nimProfile(size) proc newObjRC1(typ: PNimType, size: int): pointer {.compilerRtl.} = - setStackTop(gch) - result = rawNewObj(typ, size, gch, true) + # generates a new object and sets its reference counter to 1 + sysAssert(allocInv(gch.region), "newObjRC1 begin") + gcAssert(typ.kind in {tyRef, tyString, tySequence}, "newObj: 1") + collectCT(gch) + sysAssert(allocInv(gch.region), "newObjRC1 after collectCT") + + var res = cast[PCell](rawAlloc(gch.region, size + sizeof(Cell))) + sysAssert(allocInv(gch.region), "newObjRC1 after rawAlloc") + sysAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "newObj: 2") + # now it is buffered in the ZCT + res.typ = typ + when leakDetector and not hasThreadSupport: + if framePtr != nil and framePtr.prev != nil: + res.filename = framePtr.prev.filename + res.line = framePtr.prev.line + res.refcount = rcIncrement or gch.black # refcount is 1 + sysAssert(isAllocatedPtr(gch.region, res), "newObj: 3") + when logGC: writeCell("new cell", res) + gcTrace(res, csAllocated) + when useCellIds: + inc gch.idGenerator + res.id = gch.idGenerator + result = cellToUsr(res) + zeroMem(result, size) + sysAssert(allocInv(gch.region), "newObjRC1 end") when defined(memProfiler): nimProfile(size) proc newSeqRC1(typ: PNimType, len: int): pointer {.compilerRtl.} = - setStackTop(gch) let size = addInt(mulInt(len, typ.base.size), GenericSeqSize) result = newObjRC1(typ, size) cast[PGenericSeq](result).len = len cast[PGenericSeq](result).reserved = len + when defined(memProfiler): nimProfile(size) proc growObj(old: pointer, newsize: int, gch: var GcHeap): pointer = - acquire(gch) collectCT(gch) var ol = usrToCell(old) + gcAssert(isAllocatedPtr(gch.region, ol), "growObj: freed pointer?") + sysAssert(ol.typ != nil, "growObj: 1") - sysAssert(ol.typ.kind in {tyString, tySequence}, "growObj: 2") + gcAssert(ol.typ.kind in {tyString, tySequence}, "growObj: 2") sysAssert(allocInv(gch.region), "growObj begin") var res = cast[PCell](rawAlloc(gch.region, newsize + sizeof(Cell))) - var elemSize = if ol.typ.kind != tyString: ol.typ.base.size - else: 1 + var elemSize = 1 + if ol.typ.kind != tyString: elemSize = ol.typ.base.size - var oldsize = cast[PGenericSeq](old).len*elemSize + GenericSeqSize - - # XXX: This should happen outside - # call user-defined move code - # call user-defined default constructor + let oldsize = cast[PGenericSeq](old).len*elemSize + GenericSeqSize copyMem(res, ol, oldsize + sizeof(Cell)) - zeroMem(cast[pointer](cast[ByteAddress](res)+% oldsize +% sizeof(Cell)), + zeroMem(cast[pointer](cast[ByteAddress](res) +% oldsize +% sizeof(Cell)), newsize-oldsize) - sysAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "growObj: 3") - sysAssert(res.refcount shr rcShift <=% 1, "growObj: 4") - - when false: - if ol.isBitUp(rcZct): - var j = gch.zct.len-1 - var d = gch.zct.d - while j >= 0: - if d[j] == ol: - d[j] = res - break - dec(j) - - if ol.isBitUp(rcInCycleRoots): - for i in 0 .. <gch.cycleRoots.len: - if gch.cycleRoots.d[i] == ol: - eraseAt(gch.cycleRoots, i) - - freeCell(gch, ol) - - else: - # the new buffer inherits the GC state of the old one - if res.isBitUp(rcZct): gch.zct.add res - if res.isBitUp(rcInCycleRoots): gch.cycleRoots.add res - - # Pay attention to what's going on here! We're not releasing the old memory. - # This is because at this point there may be an interior pointer pointing - # into this buffer somewhere on the stack (due to `var` parameters now and - # and `let` and `var:var` stack locations in the future). - # We'll release the memory in the next GC cycle. If we release it here, - # we cannot guarantee that no memory will be corrupted when only safe - # language features are used. Accessing the memory after the seq/string - # has been invalidated may still result in logic errors in the user code. - # We may improve on that by protecting the page in debug builds or - # by providing a warning when we detect a stack pointer into it. - let bufferFlags = ol.refcount and rcBufferedAnywhere - if bufferFlags == 0: - # we need this in order to collect it safely later - ol.refcount = rcRetiredBuffer or rcZct - gch.zct.add ol - else: - ol.refcount = rcRetiredBuffer or bufferFlags - - when logGC: - writeCell("growObj old cell", ol) - writeCell("growObj new cell", res) - + # This can be wrong for intermediate temps that are nevertheless on the + # heap because of lambda lifting: + #gcAssert(res.refcount shr rcShift <=% 1, "growObj: 4") + when logGC: + writeCell("growObj old cell", ol) + writeCell("growObj new cell", res) + gcTrace(ol, csZctFreed) gcTrace(res, csAllocated) - release(gch) + when reallyDealloc: + sysAssert(allocInv(gch.region), "growObj before dealloc") + if ol.refcount shr rcShift <=% 1: + # free immediately to save space: + if (ol.refcount and ZctFlag) != 0: + var j = gch.zct.len-1 + var d = gch.zct.d + while j >= 0: + if d[j] == ol: + d[j] = res + break + dec(j) + rawDealloc(gch.region, ol) + else: + # we split the old refcount in 2 parts. XXX This is still not entirely + # correct if the pointer that receives growObj's result is on the stack. + # A better fix would be to emit the location specific write barrier for + # 'growObj', but this is lots of more work and who knows what new problems + # this would create. + res.refcount = rcIncrement or gch.black + decRef(ol) + else: + sysAssert(ol.typ != nil, "growObj: 5") + zeroMem(ol, sizeof(Cell)) + when useCellIds: + inc gch.idGenerator + res.id = gch.idGenerator result = cellToUsr(res) sysAssert(allocInv(gch.region), "growObj end") when defined(memProfiler): nimProfile(newsize-oldsize) proc growObj(old: pointer, newsize: int): pointer {.rtl.} = - setStackTop(gch) result = growObj(old, newsize, gch) {.push profiler:off.} -# ---------------- cycle collector ------------------------------------------- -proc doOperation(p: pointer, op: WalkOp) = - if p == nil: return - var c: PCell = usrToCell(p) - sysAssert(c != nil, "doOperation: 1") - gch.tempStack.add c +template takeStartTime(workPackageSize) {.dirty.} = + const workPackage = workPackageSize + var debugticker = 1000 + when withRealTime: + var steps = workPackage + var t0: Ticks + if gch.maxPause > 0: t0 = getticks() -proc nimGCvisit(d: pointer, op: int) {.compilerRtl.} = - doOperation(d, WalkOp(op)) +template takeTime {.dirty.} = + when withRealTime: dec steps + dec debugticker + +template checkTime {.dirty.} = + if debugticker <= 0: + echo "in loop" + debugticker = 1000 + when withRealTime: + if steps == 0: + steps = workPackage + if gch.maxPause > 0: + let duration = getticks() - t0 + # the GC's measuring is not accurate and needs some cleanup actions + # (stack unmarking), so subtract some short amount of time in + # order to miss deadlines less often: + if duration >= gch.maxPause - 50_000: + return false -type - RecursionType = enum - FromChildren, - FromRoot -{.deprecated: [TRecursionType: RecursionType].} +# ---------------- cycle collector ------------------------------------------- -proc collectZCT(gch: var GcHeap): bool +proc freeCyclicCell(gch: var GcHeap, c: PCell) = + gcAssert(isAllocatedPtr(gch.region, c), "freeCyclicCell: freed pointer?") -template pseudoRecursion(typ: RecursionType, body: stmt): stmt = - discard + var d = gch.decStack.d + for i in 0..gch.decStack.len-1: + gcAssert d[i] != c, "wtf man, freeing obviously alive stuff?!!" + + prepareDealloc(c) + gcTrace(c, csCycFreed) + when logGC: writeCell("cycle collector dealloc cell", c) + when reallyDealloc: + sysAssert(allocInv(gch.region), "free cyclic cell") + rawDealloc(gch.region, c) + else: + gcAssert(c.typ != nil, "freeCyclicCell") + zeroMem(c, sizeof(Cell)) -proc trimCycleRoots(gch: var GcHeap, startIdx = gch.cycleRootsTrimIdx) = - var i = startIdx - while i < gch.cycleRoots.len: - if gch.cycleRoots.d[i].color != rcCycleCandidate: - gch.cycleRoots.trimAt i - else: - inc i +proc sweep(gch: var GcHeap): bool = + takeStartTime(100) + echo "loop start" + let black = gch.black + while true: + let x = allObjectsAsProc(gch.region, addr gch.spaceIter) + if gch.spaceIter.state < 0: break + takeTime() + if isCell(x): + # cast to PCell is correct here: + var c = cast[PCell](x) + gcAssert c.color != rcGrey, "cell is still grey?" + if c.color != black: freeCyclicCell(gch, c) + # Since this is incremental, we MUST not set the object to 'white' here. + # We could set all the remaining objects to white after the 'sweep' + # completed but instead we flip the meaning of black/white to save one + # traversal over the heap! + checkTime() + # prepare for next iteration: + echo "loop end" + gch.spaceIter = ObjectSpaceIter() + result = true - gch.cycleRootsTrimIdx = gch.cycleRoots.len +proc markRoot(gch: var GcHeap, c: PCell) = + # since we start with 'black' cells, we need to mark them here too: + if c.color != rcGrey: + c.setColor(rcGrey) + add(gch.greyStack, c) -# we now use a much simpler and non-recursive algorithm for cycle removal -proc collectCycles(gch: var GcHeap) = - if gch.cycleRoots.len == 0: return - gch.stat.cycleTableSize = max(gch.stat.cycleTableSize, gch.cycleRoots.len) +proc markIncremental(gch: var GcHeap): bool = + var L = addr(gch.greyStack.len) + takeStartTime(100) + while L[] > 0: + var c = gch.greyStack.d[0] + sysAssert(isAllocatedPtr(gch.region, c), "markIncremental: isAllocatedPtr") + gch.greyStack.d[0] = gch.greyStack.d[L[] - 1] + dec(L[]) + takeTime() + if c.color == rcGrey: + c.setColor(gch.black) + forAllChildren(c, waMarkGrey) + checkTime() + gcAssert gch.greyStack.len == 0, "markIncremental: greystack not empty " + result = true - when CollectCyclesStats: - let l0 = gch.cycleRoots.len - let tStart = getTicks() +proc markGlobals(gch: var GcHeap) = + for i in 0 .. < globalMarkersLen: globalMarkers[i]() +proc markLocals(gch: var GcHeap) = + var d = gch.decStack.d + for i in 0 .. < gch.decStack.len: + sysAssert isAllocatedPtr(gch.region, d[i]), "markLocals" + markRoot(gch, d[i]) + +when logGC: var - decrefs = 0 - increfs = 0 - collected = 0 - maybedeads = 0 - - template ignoreObject(c: PCell): expr = - # This controls which objects will be ignored in the mark and scan stages - (when MarkingSkipsAcyclicObjects: not canbeCycleRoot(c) else: false) - # not canbeCycleRoot(c) - # false - # c.isBitUp(rcHasStackRef) - - template earlyMarkAliveRec(cell) = - let startLen = gch.tempStack.len - cell.setColor rcAlive - cell.forAllChildren waPush - - while startLen != gch.tempStack.len: - dec gch.tempStack.len - var c = gch.tempStack.d[gch.tempStack.len] - if c.color != rcAlive: - c.setColor rcAlive - c.forAllChildren waPush - - template earlyMarkAlive(stackRoots) = - # This marks all objects reachable from the stack as alive before any - # of the other stages is executed. Such objects cannot be garbage and - # they don't need to participate in the recursive decref/incref. - for i in 0 .. <stackRoots.len: - var c = stackRoots.d[i] - # c.setBit rcHasStackRef - earlyMarkAliveRec(c) - - earlyMarkAlive(gch.decStack) - - when CollectCyclesStats: - let tAfterEarlyMarkAlive = getTicks() - - template recursiveDecRef(cell) = - let startLen = gch.tempStack.len - cell.setColor rcDecRefApplied - cell.forAllChildren waPush - - while startLen != gch.tempStack.len: - dec gch.tempStack.len - var c = gch.tempStack.d[gch.tempStack.len] - if ignoreObject(c): continue - - sysAssert(c.refcount >=% rcIncrement, "recursive dec ref") - dec c.refcount, rcIncrement - inc decrefs - if c.color != rcDecRefApplied: - c.setColor rcDecRefApplied - c.forAllChildren waPush - - template markRoots(roots) = - var i = 0 - while i < roots.len: - if roots.d[i].color == rcCycleCandidate: - recursiveDecRef(roots.d[i]) - inc i - else: - roots.trimAt i - - markRoots(gch.cycleRoots) - - when CollectCyclesStats: - let tAfterMark = getTicks() - c_printf "COLLECT CYCLES %d: %d/%d\n", gcCollectionIdx, gch.cycleRoots.len, l0 - - template recursiveMarkAlive(cell) = - let startLen = gch.tempStack.len - cell.setColor rcAlive - cell.forAllChildren waPush - - while startLen != gch.tempStack.len: - dec gch.tempStack.len - var c = gch.tempStack.d[gch.tempStack.len] - if ignoreObject(c): continue - inc c.refcount, rcIncrement - inc increfs - - if c.color != rcAlive: - c.setColor rcAlive - c.forAllChildren waPush - - template scanRoots(roots) = - for i in 0 .. <roots.len: - let startLen = gch.tempStack.len - gch.tempStack.add roots.d[i] - - while startLen != gch.tempStack.len: - dec gch.tempStack.len - var c = gch.tempStack.d[gch.tempStack.len] - if ignoreObject(c): continue - if c.color == rcDecRefApplied: - if c.refcount >=% rcIncrement: - recursiveMarkAlive(c) - else: - # note that this is not necessarily the ultimate - # destiny of the object. we may still mark it alive - # later if we encounter another node from where it's - # reachable. - c.setColor rcMaybeDead - inc maybedeads - c.forAllChildren waPush - - scanRoots(gch.cycleRoots) - - when CollectCyclesStats: - let tAfterScan = getTicks() - - template collectDead(roots) = - for i in 0 .. <roots.len: - var c = roots.d[i] - c.clearBit(rcInCycleRoots) - - let startLen = gch.tempStack.len - gch.tempStack.add c - - while startLen != gch.tempStack.len: - dec gch.tempStack.len - var c = gch.tempStack.d[gch.tempStack.len] - when MarkingSkipsAcyclicObjects: - if not canbeCycleRoot(c): - # This is an acyclic object reachable from a dead cyclic object - # We must do a normal decref here that may add the acyclic object - # to the ZCT - doDecRef(c, LocalHeap, Cyclic) - continue - if c.color == rcMaybeDead and not c.isBitUp(rcInCycleRoots): - c.setColor(rcReallyDead) - inc collected - c.forAllChildren waPush - # we need to postpone the actual deallocation in order to allow - # the finalizers to run while the data structures are still intact - gch.freeStack.add c - prepareDealloc(c) - - for i in 0 .. <gch.freeStack.len: - freeCell(gch, gch.freeStack.d[i]) - - collectDead(gch.cycleRoots) - - when CollectCyclesStats: - let tFinal = getTicks() - cprintf "times:\n early mark alive: %d ms\n mark: %d ms\n scan: %d ms\n collect: %d ms\n decrefs: %d\n increfs: %d\n marked dead: %d\n collected: %d\n", - (tAfterEarlyMarkAlive - tStart) div 1_000_000, - (tAfterMark - tAfterEarlyMarkAlive) div 1_000_000, - (tAfterScan - tAfterMark) div 1_000_000, - (tFinal - tAfterScan) div 1_000_000, - decrefs, - increfs, - maybedeads, - collected - - deinit(gch.cycleRoots) - init(gch.cycleRoots) - - deinit(gch.freeStack) - init(gch.freeStack) - - when MarkingSkipsAcyclicObjects: - # Collect the acyclic objects that became unreachable due to collected - # cyclic objects. - discard collectZCT(gch) - # collectZCT may add new cycle candidates and we may decide to loop here - # if gch.cycleRoots.len > 0: repeat - -var gcDebugging* = false - -var seqdbg* : proc (s: PGenericSeq) {.cdecl.} + cycleCheckA: array[100, PCell] + cycleCheckALen = 0 + + proc alreadySeen(c: PCell): bool = + for i in 0 .. <cycleCheckALen: + if cycleCheckA[i] == c: return true + if cycleCheckALen == len(cycleCheckA): + gcAssert(false, "cycle detection overflow") + quit 1 + cycleCheckA[cycleCheckALen] = c + inc cycleCheckALen + + proc debugGraph(s: PCell) = + if alreadySeen(s): + writeCell("child cell (already seen) ", s) + else: + writeCell("cell {", s) + forAllChildren(s, waDebug) + c_fprintf(c_stdout, "}\n") + +proc doOperation(p: pointer, op: WalkOp) = + if p == nil: return + var c: PCell = usrToCell(p) + gcAssert(c != nil, "doOperation: 1") + # the 'case' should be faster than function pointers because of easy + # prediction: + case op + of waZctDecRef: + #if not isAllocatedPtr(gch.region, c): + # c_fprintf(c_stdout, "[GC] decref bug: %p", c) + gcAssert(isAllocatedPtr(gch.region, c), "decRef: waZctDecRef") + gcAssert(c.refcount >=% rcIncrement, "doOperation 2") + #c.refcount = c.refcount -% rcIncrement + when logGC: writeCell("decref (from doOperation)", c) + decRef(c) + #if c.refcount <% rcIncrement: addZCT(gch.zct, c) + of waMarkGlobal: + when hasThreadSupport: + # could point to a cell which we don't own and don't want to touch/trace + if isAllocatedPtr(gch.region, c): + markRoot(gch, c) + else: + markRoot(gch, c) + of waMarkGrey: + if c.color == 1-gch.black: + c.setColor(rcGrey) + add(gch.greyStack, c) + #of waDebug: debugGraph(c) + +proc nimGCvisit(d: pointer, op: int) {.compilerRtl.} = + doOperation(d, WalkOp(op)) + +proc collectZCT(gch: var GcHeap): bool {.benign.} + +proc collectCycles(gch: var GcHeap): bool = + # ensure the ZCT 'color' is not used: + while gch.zct.len > 0: discard collectZCT(gch) + case gch.phase + of Phase.None, Phase.Marking: + #if gch.phase == Phase.None: + gch.phase = Phase.Marking + markGlobals(gch) + markLocals(gch) + if markIncremental(gch): + gch.phase = Phase.Sweeping + of Phase.Sweeping: + gcAssert gch.greyStack.len == 0, "greystack not empty" + if sweep(gch): + gch.phase = Phase.None + # flip black/white meanings: + gch.black = 1 - gch.black + result = true proc gcMark(gch: var GcHeap, p: pointer) {.inline.} = # the addresses are not as cells on the stack, so turn them to cells: @@ -1005,235 +739,33 @@ proc gcMark(gch: var GcHeap, p: pointer) {.inline.} = var objStart = cast[PCell](interiorAllocatedPtr(gch.region, cell)) if objStart != nil: # mark the cell: - if objStart.color != rcReallyDead: - if gcDebugging: - # writeCell("marking ", objStart) - discard - else: - inc objStart.refcount, rcIncrement - gch.decStack.add objStart - else: - # With incremental clean-up, objects spend some time - # in various lists before being deallocated. - # We just found a reference on the stack to an object, - # which we have previously labeled as unreachable. - # This is either a bug in the GC or a pure accidental - # coincidence due to the conservative stack marking. - when debugGC: - # writeCell("marking dead object", objStart) - discard - when false: - if isAllocatedPtr(gch.region, cell): - sysAssert false, "allocated pointer but not interior?" - # mark the cell: - inc cell.refcount, rcIncrement - add(gch.decStack, cell) + objStart.refcount = objStart.refcount +% rcIncrement + add(gch.decStack, objStart) sysAssert(allocInv(gch.region), "gcMark end") -proc markThreadStacks(gch: var GcHeap) = - when hasThreadSupport and hasSharedHeap: - {.error: "not fully implemented".} - var it = threadList - while it != nil: - # mark registers: - for i in 0 .. high(it.registers): gcMark(gch, it.registers[i]) - var sp = cast[ByteAddress](it.stackBottom) - var max = cast[ByteAddress](it.stackTop) - # XXX stack direction? - # XXX unroll this loop: - while sp <=% max: - gcMark(gch, cast[PPointer](sp)[]) - sp = sp +% sizeof(pointer) - it = it.next - -# ----------------- stack management -------------------------------------- -# inspired from Smart Eiffel - -when defined(sparc): - const stackIncreases = false -elif defined(hppa) or defined(hp9000) or defined(hp9000s300) or - defined(hp9000s700) or defined(hp9000s800) or defined(hp9000s820): - const stackIncreases = true -else: - const stackIncreases = false - -when not defined(useNimRtl): - {.push stack_trace: off.} - proc setStackBottom(theStackBottom: pointer) = - #c_fprintf(c_stdout, "stack bottom: %p;\n", theStackBottom) - # the first init must be the one that defines the stack bottom: - if gch.stackBottom == nil: gch.stackBottom = theStackBottom - else: - var a = cast[ByteAddress](theStackBottom) # and not PageMask - PageSize*2 - var b = cast[ByteAddress](gch.stackBottom) - #c_fprintf(c_stdout, "old: %p new: %p;\n",gch.stackBottom,theStackBottom) - when stackIncreases: - gch.stackBottom = cast[pointer](min(a, b)) - else: - gch.stackBottom = cast[pointer](max(a, b)) - {.pop.} - -proc stackSize(): int {.noinline.} = - var stackTop {.volatile.}: pointer - result = abs(cast[int](addr(stackTop)) - cast[int](gch.stackBottom)) +include gc_common -var - jmpbufSize {.importc: "sizeof(jmp_buf)", nodecl.}: int - # a little hack to get the size of a JmpBuf in the generated C code - # in a platform independent way - -when defined(sparc): # For SPARC architecture. - proc isOnStack(p: pointer): bool = - var stackTop {.volatile.}: pointer - stackTop = addr(stackTop) - var b = cast[ByteAddress](gch.stackBottom) - var a = cast[ByteAddress](stackTop) - var x = cast[ByteAddress](p) - result = a <=% x and x <=% b - - proc markStackAndRegisters(gch: var GcHeap) {.noinline, cdecl.} = - when defined(sparcv9): - asm """"flushw \n" """ - else: - asm """"ta 0x3 ! ST_FLUSH_WINDOWS\n" """ - - var - max = gch.stackBottom - sp: PPointer - stackTop: array[0..1, pointer] - sp = addr(stackTop[0]) - # Addresses decrease as the stack grows. - while sp <= max: - gcMark(gch, sp[]) - sp = cast[PPointer](cast[ByteAddress](sp) +% sizeof(pointer)) - -elif defined(ELATE): - {.error: "stack marking code is to be written for this architecture".} - -elif stackIncreases: - # --------------------------------------------------------------------------- - # Generic code for architectures where addresses increase as the stack grows. - # --------------------------------------------------------------------------- - proc isOnStack(p: pointer): bool = - var stackTop {.volatile.}: pointer - stackTop = addr(stackTop) - var a = cast[ByteAddress](gch.stackBottom) - var b = cast[ByteAddress](stackTop) - var x = cast[ByteAddress](p) - result = a <=% x and x <=% b - - proc markStackAndRegisters(gch: var GcHeap) {.noinline, cdecl.} = - var registers: C_JmpBuf - if c_setjmp(registers) == 0'i32: # To fill the C stack with registers. - var max = cast[ByteAddress](gch.stackBottom) - var sp = cast[ByteAddress](addr(registers)) +% jmpbufSize -% sizeof(pointer) - # sp will traverse the JMP_BUF as well (jmp_buf size is added, - # otherwise sp would be below the registers structure). - while sp >=% max: - gcMark(gch, cast[PPointer](sp)[]) - sp = sp -% sizeof(pointer) - -else: - # --------------------------------------------------------------------------- - # Generic code for architectures where addresses decrease as the stack grows. - # --------------------------------------------------------------------------- - proc isOnStack(p: pointer): bool = - var stackTop {.volatile.}: pointer - stackTop = addr(stackTop) - var b = cast[ByteAddress](gch.stackBottom) - var a = cast[ByteAddress](stackTop) - var x = cast[ByteAddress](p) - result = a <=% x and x <=% b - - proc markStackAndRegisters(gch: var GcHeap) {.noinline, cdecl.} = - # We use a jmp_buf buffer that is in the C stack. - # Used to traverse the stack and registers assuming - # that 'setjmp' will save registers in the C stack. - type PStackSlice = ptr array [0..7, pointer] - var registers: C_JmpBuf - if c_setjmp(registers) == 0'i32: # To fill the C stack with registers. - when MinimumStackMarking: - # mark the registers - var jmpbufPtr = cast[ByteAddress](addr(registers)) - var jmpbufEnd = jmpbufPtr +% jmpbufSize - - while jmpbufPtr <=% jmpbufEnd: - gcMark(gch, cast[PPointer](jmpbufPtr)[]) - jmpbufPtr = jmpbufPtr +% sizeof(pointer) - - var sp = cast[ByteAddress](gch.stackTop) - else: - var sp = cast[ByteAddress](addr(registers)) - # mark the user stack - var max = cast[ByteAddress](gch.stackBottom) - # loop unrolled: - while sp <% max - 8*sizeof(pointer): - gcMark(gch, cast[PStackSlice](sp)[0]) - gcMark(gch, cast[PStackSlice](sp)[1]) - gcMark(gch, cast[PStackSlice](sp)[2]) - gcMark(gch, cast[PStackSlice](sp)[3]) - gcMark(gch, cast[PStackSlice](sp)[4]) - gcMark(gch, cast[PStackSlice](sp)[5]) - gcMark(gch, cast[PStackSlice](sp)[6]) - gcMark(gch, cast[PStackSlice](sp)[7]) - sp = sp +% sizeof(pointer)*8 - # last few entries: - while sp <=% max: - gcMark(gch, cast[PPointer](sp)[]) - sp = sp +% sizeof(pointer) - -# ---------------------------------------------------------------------------- -# end of non-portable code -# ---------------------------------------------------------------------------- - -proc releaseCell(gch: var GcHeap, cell: PCell) = - if cell.color != rcReallyDead: - prepareDealloc(cell) - cell.setColor rcReallyDead - - let l1 = gch.tempStack.len - cell.forAllChildren waPush - let l2 = gch.tempStack.len - for i in l1 .. <l2: - var cc = gch.tempStack.d[i] - if cc.refcount--(LocalHeap): - releaseCell(gch, cc) - else: - if canbeCycleRoot(cc): - addCycleRoot(gch.cycleRoots, cc) - - gch.tempStack.len = l1 - - if cell.isBitDown(rcBufferedAnywhere): - freeCell(gch, cell) - # else: - # This object is either buffered in the cycleRoots list and we'll leave - # it there to be collected in the next collectCycles or it's pending in - # the ZCT: - # (e.g. we are now cleaning the 15th object, but this one is 18th in the - # list. Note that this can happen only if we reached this point by the - # recursion). - # We can ignore it now as the ZCT cleaner will reach it soon. +proc markStackAndRegisters(gch: var GcHeap) {.noinline, cdecl.} = + forEachStackSlot(gch, gcMark) proc collectZCT(gch: var GcHeap): bool = - const workPackage = 100 + # Note: Freeing may add child objects to the ZCT! So essentially we do + # deep freeing, which is bad for incremental operation. In order to + # avoid a deep stack, we move objects to keep the ZCT small. + # This is performance critical! var L = addr(gch.zct.len) - - when withRealtime: - var steps = workPackage - var t0: Ticks - if gch.maxPause > 0: t0 = getticks() + takeStartTime(100) while L[] > 0: var c = gch.zct.d[0] - sysAssert c.isBitUp(rcZct), "collectZCT: rcZct missing!" - sysAssert(isAllocatedPtr(gch.region, c), "collectZCT: isAllocatedPtr") - + sysAssert(isAllocatedPtr(gch.region, c), "CollectZCT: isAllocatedPtr") # remove from ZCT: - c.clearBit(rcZct) + gcAssert((c.refcount and ZctFlag) == ZctFlag, "collectZCT") + + c.refcount = c.refcount and not ZctFlag gch.zct.d[0] = gch.zct.d[L[] - 1] dec(L[]) - when withRealtime: dec steps + takeTime() if c.refcount <% rcIncrement: # It may have a RC > 0, if it is in the hardware stack or # it has not been removed yet from the ZCT. This is because @@ -1241,92 +773,78 @@ proc collectZCT(gch: var GcHeap): bool = # as this might be too slow. # In any case, it should be removed from the ZCT. But not # freed. **KEEP THIS IN MIND WHEN MAKING THIS INCREMENTAL!** - if c.color == rcRetiredBuffer: - if c.isBitDown(rcInCycleRoots): - freeCell(gch, c) + when logGC: writeCell("zct dealloc cell", c) + gcTrace(c, csZctFreed) + # We are about to free the object, call the finalizer BEFORE its + # children are deleted as well, because otherwise the finalizer may + # access invalid memory. This is done by prepareDealloc(): + prepareDealloc(c) + forAllChildren(c, waZctDecRef) + when reallyDealloc: + sysAssert(allocInv(gch.region), "collectZCT: rawDealloc") + rawDealloc(gch.region, c) else: - # if c.color == rcReallyDead: writeCell("ReallyDead in ZCT?", c) - releaseCell(gch, c) - when withRealtime: - if steps == 0: - steps = workPackage - if gch.maxPause > 0: - let duration = getticks() - t0 - # the GC's measuring is not accurate and needs some cleanup actions - # (stack unmarking), so subtract some short amount of time in to - # order to miss deadlines less often: - if duration >= gch.maxPause - 50_000: - return false + sysAssert(c.typ != nil, "collectZCT 2") + zeroMem(c, sizeof(Cell)) + checkTime() result = true - gch.trimCycleRoots - #deInit(gch.zct) - #init(gch.zct) proc unmarkStackAndRegisters(gch: var GcHeap) = var d = gch.decStack.d - for i in 0 .. <gch.decStack.len: + for i in 0..gch.decStack.len-1: sysAssert isAllocatedPtr(gch.region, d[i]), "unmarkStackAndRegisters" - # XXX: just call doDecRef? - var c = d[i] - sysAssert c.typ != nil, "unmarkStackAndRegisters 2" - - if c.color == rcRetiredBuffer: - continue - - # XXX no need for an atomic dec here: - if c.refcount--(LocalHeap): - # the object survived only because of a stack reference - # it still doesn't have heap references - addZCT(gch.zct, c) - - if canbeCycleRoot(c): - # any cyclic object reachable from the stack can be turned into - # a leak if it's orphaned through the stack reference - # that's because the write-barrier won't be executed for stack - # locations - addCycleRoot(gch.cycleRoots, c) - + decRef(d[i]) gch.decStack.len = 0 proc collectCTBody(gch: var GcHeap) = - when withRealtime: + when withRealTime: let t0 = getticks() - when debugGC: inc gcCollectionIdx sysAssert(allocInv(gch.region), "collectCT: begin") - gch.stat.maxStackSize = max(gch.stat.maxStackSize, stackSize()) + when not defined(nimCoroutines): + gch.stat.maxStackSize = max(gch.stat.maxStackSize, stackSize()) sysAssert(gch.decStack.len == 0, "collectCT") prepareForInteriorPointerChecking(gch.region) markStackAndRegisters(gch) - markThreadStacks(gch) gch.stat.maxStackCells = max(gch.stat.maxStackCells, gch.decStack.len) inc(gch.stat.stackScans) if collectZCT(gch): when cycleGC: if getOccupiedMem(gch.region) >= gch.cycleThreshold or alwaysCycleGC: - collectCycles(gch) - sysAssert gch.zct.len == 0, "zct is not null after collect cycles" - inc(gch.stat.cycleCollections) - gch.cycleThreshold = max(InitialCycleThreshold, getOccupiedMem() * - CycleIncrease) - gch.stat.maxThreshold = max(gch.stat.maxThreshold, gch.cycleThreshold) + if collectCycles(gch): + inc(gch.stat.cycleCollections) + gch.cycleThreshold = max(InitialCycleThreshold, getOccupiedMem() * + CycleIncrease) + gch.stat.maxThreshold = max(gch.stat.maxThreshold, gch.cycleThreshold) unmarkStackAndRegisters(gch) sysAssert(allocInv(gch.region), "collectCT: end") - when withRealtime: + when withRealTime: let duration = getticks() - t0 gch.stat.maxPause = max(gch.stat.maxPause, duration) when defined(reportMissedDeadlines): if gch.maxPause > 0 and duration > gch.maxPause: c_fprintf(c_stdout, "[GC] missed deadline: %ld\n", duration) +when defined(nimCoroutines): + proc currentStackSizes(): int = + for stack in items(gch.stack): + result = result + stackSize(stack.starts, stack.pos) + proc collectCT(gch: var GcHeap) = - if (gch.zct.len >= ZctThreshold or (cycleGC and + # stackMarkCosts prevents some pathological behaviour: Stack marking + # becomes more expensive with large stacks and large stacks mean that + # cells with RC=0 are more likely to be kept alive by the stack. + when defined(nimCoroutines): + let stackMarkCosts = max(currentStackSizes() div (16*sizeof(int)), ZctThreshold) + else: + let stackMarkCosts = max(stackSize() div (16*sizeof(int)), ZctThreshold) + if (gch.zct.len >= stackMarkCosts or (cycleGC and getOccupiedMem(gch.region)>=gch.cycleThreshold) or alwaysGC) and gch.recGcLock == 0: collectCTBody(gch) -when withRealtime: +when withRealTime: proc toNano(x: int): Nanos {.inline.} = result = x * 1000 @@ -1334,13 +852,11 @@ when withRealtime: gch.maxPause = MaxPauseInUs.toNano proc GC_step(gch: var GcHeap, us: int, strongAdvice: bool) = - acquire(gch) gch.maxPause = us.toNano if (gch.zct.len >= ZctThreshold or (cycleGC and getOccupiedMem(gch.region)>=gch.cycleThreshold) or alwaysGC) or strongAdvice: collectCTBody(gch) - release(gch) proc GC_step*(us: int, strongAdvice = false) = GC_step(gch, us, strongAdvice) @@ -1358,11 +874,7 @@ when not defined(useNimRtl): dec(gch.recGcLock) proc GC_setStrategy(strategy: GC_Strategy) = - case strategy - of gcThroughput: discard - of gcResponsiveness: discard - of gcOptimizeSpace: discard - of gcOptimizeTime: discard + discard proc GC_enableMarkAndSweep() = gch.cycleThreshold = InitialCycleThreshold @@ -1372,13 +884,10 @@ when not defined(useNimRtl): # set to the max value to suppress the cycle detector proc GC_fullCollect() = - setStackTop(gch) - acquire(gch) var oldThreshold = gch.cycleThreshold gch.cycleThreshold = 0 # forces cycle collection collectCT(gch) gch.cycleThreshold = oldThreshold - release(gch) proc GC_getStatistics(): string = GC_disable() @@ -1390,9 +899,13 @@ when not defined(useNimRtl): "[GC] max threshold: " & $gch.stat.maxThreshold & "\n" & "[GC] zct capacity: " & $gch.zct.cap & "\n" & "[GC] max cycle table size: " & $gch.stat.cycleTableSize & "\n" & - "[GC] max stack size: " & $gch.stat.maxStackSize & "\n" & "[GC] max pause time [ms]: " & $(gch.stat.maxPause div 1000_000) - when traceGC: writeLeakage(true) + when defined(nimCoroutines): + result = result & "[GC] number of stacks: " & $gch.stack.len & "\n" + for stack in items(gch.stack): + result = result & "[GC] stack " & stack.starts.repr & "[GC] max stack size " & $stack.maxStackSize & "\n" + else: + result = result & "[GC] max stack size: " & $gch.stat.maxStackSize & "\n" GC_enable() {.pop.} diff --git a/lib/system/jssys.nim b/lib/system/jssys.nim index 5bcddc5e6..5bac54772 100644 --- a/lib/system/jssys.nim +++ b/lib/system/jssys.nim @@ -33,6 +33,9 @@ type lineNumber {.importc.}: int message {.importc.}: cstring stack {.importc.}: cstring + + JSRef = ref RootObj # Fake type. + {.deprecated: [TSafePoint: SafePoint, TCallFrame: CallFrame].} var @@ -282,61 +285,6 @@ proc eqStrings(a, b: string): bool {.asmNoStackFrame, compilerProc.} = return true; """ -type - Document {.importc.} = object of RootObj - write: proc (text: cstring) {.nimcall.} - writeln: proc (text: cstring) {.nimcall.} - createAttribute: proc (identifier: cstring): ref Node {.nimcall.} - createElement: proc (identifier: cstring): ref Node {.nimcall.} - createTextNode: proc (identifier: cstring): ref Node {.nimcall.} - getElementById: proc (id: cstring): ref Node {.nimcall.} - getElementsByName: proc (name: cstring): seq[ref Node] {.nimcall.} - getElementsByTagName: proc (name: cstring): seq[ref Node] {.nimcall.} - - NodeType* = enum - ElementNode = 1, - AttributeNode, - TextNode, - CDATANode, - EntityRefNode, - EntityNode, - ProcessingInstructionNode, - CommentNode, - DocumentNode, - DocumentTypeNode, - DocumentFragmentNode, - NotationNode - Node* {.importc.} = object of RootObj - attributes*: seq[ref Node] - childNodes*: seq[ref Node] - data*: cstring - firstChild*: ref Node - lastChild*: ref Node - nextSibling*: ref Node - nodeName*: cstring - nodeType*: NodeType - nodeValue*: cstring - parentNode*: ref Node - previousSibling*: ref Node - appendChild*: proc (child: ref Node) {.nimcall.} - appendData*: proc (data: cstring) {.nimcall.} - cloneNode*: proc (copyContent: bool) {.nimcall.} - deleteData*: proc (start, len: int) {.nimcall.} - getAttribute*: proc (attr: cstring): cstring {.nimcall.} - getAttributeNode*: proc (attr: cstring): ref Node {.nimcall.} - getElementsByTagName*: proc (): seq[ref Node] {.nimcall.} - hasChildNodes*: proc (): bool {.nimcall.} - insertBefore*: proc (newNode, before: ref Node) {.nimcall.} - insertData*: proc (position: int, data: cstring) {.nimcall.} - removeAttribute*: proc (attr: cstring) {.nimcall.} - removeAttributeNode*: proc (attr: ref Node) {.nimcall.} - removeChild*: proc (child: ref Node) {.nimcall.} - replaceChild*: proc (newNode, oldNode: ref Node) {.nimcall.} - replaceData*: proc (start, len: int, text: cstring) {.nimcall.} - setAttribute*: proc (name, value: cstring) {.nimcall.} - setAttributeNode*: proc (attr: ref Node) {.nimcall.} -{.deprecated: [TNode: Node, TNodeType: NodeType, TDocument: Document].} - when defined(kwin): proc rawEcho {.compilerproc, asmNoStackFrame.} = asm """ @@ -360,28 +308,28 @@ elif defined(nodejs): """ else: - var - document {.importc, nodecl.}: ref Document - proc ewriteln(x: cstring) = - var node = document.getElementsByTagName("body")[0] - if node != nil: - node.appendChild(document.createTextNode(x)) - node.appendChild(document.createElement("br")) - else: + var node : JSRef + {.emit: "`node` = document.getElementsByTagName('body')[0];".} + if node.isNil: raise newException(ValueError, "<body> element does not exist yet!") + {.emit: """ + `node`.appendChild(document.createTextNode(`x`)); + `node`.appendChild(document.createElement("br")); + """.} proc rawEcho {.compilerproc.} = - var node = document.getElementsByTagName("body")[0] - if node == nil: + var node : JSRef + {.emit: "`node` = document.getElementsByTagName('body')[0];".} + if node.isNil: raise newException(IOError, "<body> element does not exist yet!") - asm """ - for (var i = 0; i < arguments.length; ++i) { - var x = `toJSStr`(arguments[i]); - `node`.appendChild(document.createTextNode(x)) - } - """ - node.appendChild(document.createElement("br")) + {.emit: """ + for (var i = 0; i < arguments.length; ++i) { + var x = `toJSStr`(arguments[i]); + `node`.appendChild(document.createTextNode(x)); + } + `node`.appendChild(document.createElement("br")); + """.} # Arithmetic: proc addInt(a, b: int): int {.asmNoStackFrame, compilerproc.} = @@ -532,8 +480,6 @@ proc nimMax(a, b: int): int {.compilerproc.} = return if a >= b: a else: b type NimString = string # hack for hti.nim include "system/hti" -type JSRef = ref RootObj # Fake type. - proc isFatPointer(ti: PNimType): bool = # This has to be consistent with the code generator! return ti.base.kind notin {tyObject, @@ -587,15 +533,20 @@ proc nimCopy(dest, src: JSRef, ti: PNimType): JSRef = nimCopyAux(result, src, ti.node) of tySequence, tyArrayConstr, tyOpenArray, tyArray: asm """ - if (`dest` === null || `dest` === undefined) { - `dest` = new Array(`src`.length); + if (`src` === null) { + `result` = null; } else { - `dest`.length = `src`.length; - } - `result` = `dest`; - for (var i = 0; i < `src`.length; ++i) { - `result`[i] = nimCopy(`result`[i], `src`[i], `ti`.base); + if (`dest` === null || `dest` === undefined) { + `dest` = new Array(`src`.length); + } + else { + `dest`.length = `src`.length; + } + `result` = `dest`; + for (var i = 0; i < `src`.length; ++i) { + `result`[i] = nimCopy(`result`[i], `src`[i], `ti`.base); + } } """ of tyString: diff --git a/lib/system/nimscript.nim b/lib/system/nimscript.nim index aaba11324..772d25343 100644 --- a/lib/system/nimscript.nim +++ b/lib/system/nimscript.nim @@ -242,7 +242,7 @@ template task*(name: untyped; description: string; body: untyped): untyped = ## .. code-block:: nim ## task build, "default build is via the C backend": ## setCommand "c" - proc `name Task`() = body + proc `name Task`*() = body let cmd = getCommand() if cmd.len == 0 or cmd ==? "help": diff --git a/lib/system/profiler.nim b/lib/system/profiler.nim index 4f600417e..ae8ff4e19 100644 --- a/lib/system/profiler.nim +++ b/lib/system/profiler.nim @@ -50,10 +50,15 @@ proc captureStackTrace(f: PFrame, st: var StackTrace) = inc(i) b = b.prev +var + profilingRequestedHook*: proc (): bool {.nimcall, benign.} + ## set this variable to provide a procedure that implements a profiler in + ## user space. See the `nimprof` module for a reference implementation. + when defined(memProfiler): type MemProfilerHook* = proc (st: StackTrace, requestedSize: int) {.nimcall, benign.} - {.deprecated: [TMemProfilerHook: MemProfilerHook].} + var profilerHook*: MemProfilerHook ## set this variable to provide a procedure that implements a profiler in @@ -65,17 +70,13 @@ when defined(memProfiler): hook(st, requestedSize) proc nimProfile(requestedSize: int) = - if not isNil(profilerHook): + if not isNil(profilingRequestedHook) and profilingRequestedHook(): callProfilerHook(profilerHook, requestedSize) else: - const - SamplingInterval = 50_000 - # set this to change the default sampling interval var profilerHook*: ProfilerHook ## set this variable to provide a procedure that implements a profiler in ## user space. See the `nimprof` module for a reference implementation. - gTicker {.threadvar.}: int proc callProfilerHook(hook: ProfilerHook) {.noinline.} = # 'noinline' so that 'nimProfile' does not perform the stack allocation @@ -86,16 +87,7 @@ else: proc nimProfile() = ## This is invoked by the compiler in every loop and on every proc entry! - if gTicker == 0: - gTicker = -1 - if not isNil(profilerHook): - # disable recursive calls: XXX should use try..finally, - # but that's too expensive! - let oldHook = profilerHook - profilerHook = nil - callProfilerHook(oldHook) - profilerHook = oldHook - gTicker = SamplingInterval - dec gTicker + if not isNil(profilingRequestedHook) and profilingRequestedHook(): + callProfilerHook(profilerHook) {.pop.} diff --git a/lib/system/repr.nim b/lib/system/repr.nim index 1f81a0813..986994203 100644 --- a/lib/system/repr.nim +++ b/lib/system/repr.nim @@ -259,8 +259,10 @@ when not defined(useNimRtl): of tyInt16: add result, $int(cast[ptr int16](p)[]) of tyInt32: add result, $int(cast[ptr int32](p)[]) of tyInt64: add result, $(cast[ptr int64](p)[]) - of tyUInt8: add result, $ze(cast[ptr int8](p)[]) - of tyUInt16: add result, $ze(cast[ptr int16](p)[]) + of tyUInt8: add result, $(cast[ptr uint8](p)[]) + of tyUInt16: add result, $(cast[ptr uint16](p)[]) + of tyUInt32: add result, $(cast[ptr uint32](p)[]) + of tyUInt64: add result, $(cast[ptr uint64](p)[]) of tyFloat: add result, $(cast[ptr float](p)[]) of tyFloat32: add result, $(cast[ptr float32](p)[]) diff --git a/lib/system/sysstr.nim b/lib/system/sysstr.nim index 326c601bd..e2137e8f4 100644 --- a/lib/system/sysstr.nim +++ b/lib/system/sysstr.nim @@ -229,7 +229,7 @@ proc setLengthSeq(seq: PGenericSeq, elemSize, newLen: int): PGenericSeq {. # we need to decref here, otherwise the GC leaks! when not defined(boehmGC) and not defined(nogc) and not defined(gcMarkAndSweep) and not defined(gogc): - when compileOption("gc", "v2"): + when false: # compileOption("gc", "v2"): for i in newLen..result.len-1: let len0 = gch.tempStack.len forAllChildrenAux(cast[pointer](cast[ByteAddress](result) +% diff --git a/lib/wrappers/linenoise/clinenoise.c b/lib/wrappers/linenoise/clinenoise.c index b4ae32472..a03e6f379 100644 --- a/lib/wrappers/linenoise/clinenoise.c +++ b/lib/wrappers/linenoise/clinenoise.c @@ -139,7 +139,7 @@ struct linenoiseState { int ofd; /* Terminal stdout file descriptor. */ char *buf; /* Edited line buffer. */ size_t buflen; /* Edited line buffer size. */ - const char *prompt; /* Prompt to display. */ + char *prompt; /* Prompt to display. */ size_t plen; /* Prompt length. */ size_t pos; /* Current cursor position. */ size_t oldpos; /* Previous refresh cursor position. */ @@ -172,7 +172,7 @@ enum KEY_ACTION{ }; static void linenoiseAtExit(void); -int linenoiseHistoryAdd(const char *line); +int linenoiseHistoryAdd(char *line); static void refreshLine(struct linenoiseState *l); /* Debugging macro. */ @@ -413,14 +413,14 @@ void linenoiseSetCompletionCallback(linenoiseCompletionCallback *fn) { * in order to add completion options given the input string when the * user typed <tab>. See the example.c source code for a very easy to * understand example. */ -void linenoiseAddCompletion(linenoiseCompletions *lc, const char *str) { +void linenoiseAddCompletion(linenoiseCompletions *lc, char *str) { size_t len = strlen(str); char *copy, **cvec; - copy = malloc(len+1); + copy = (char*)malloc(len+1); if (copy == NULL) return; memcpy(copy,str,len+1); - cvec = realloc(lc->cvec,sizeof(char*)*(lc->len+1)); + cvec = (char**)realloc(lc->cvec,sizeof(char*)*(lc->len+1)); if (cvec == NULL) { free(copy); return; @@ -445,12 +445,12 @@ static void abInit(struct abuf *ab) { ab->len = 0; } -static void abAppend(struct abuf *ab, const char *s, int len) { - char *new = realloc(ab->b,ab->len+len); +static void abAppend(struct abuf *ab, char *s, int len) { + char *neww = (char*)realloc(ab->b,ab->len+len); - if (new == NULL) return; - memcpy(new+ab->len,s,len); - ab->b = new; + if (neww == NULL) return; + memcpy(neww+ab->len,s,len); + ab->b = neww; ab->len += len; } @@ -723,7 +723,7 @@ void linenoiseEditDeletePrevWord(struct linenoiseState *l) { * when ctrl+d is typed. * * The function returns the length of the current buffer. */ -static int linenoiseEdit(int stdin_fd, int stdout_fd, char *buf, size_t buflen, const char *prompt) +static int linenoiseEdit(int stdin_fd, int stdout_fd, char *buf, size_t buflen, char *prompt) { struct linenoiseState l; @@ -929,7 +929,7 @@ void linenoisePrintKeyCodes(void) { /* This function calls the line editing function linenoiseEdit() using * the STDIN file descriptor set in raw mode. */ -static int linenoiseRaw(char *buf, size_t buflen, const char *prompt) { +static int linenoiseRaw(char *buf, size_t buflen, char *prompt) { int count; if (buflen == 0) { @@ -959,7 +959,7 @@ static int linenoiseRaw(char *buf, size_t buflen, const char *prompt) { * for a blacklist of stupid terminals, and later either calls the line * editing function or uses dummy fgets() so that you will be able to type * something even in the most desperate of the conditions. */ -char *linenoise(const char *prompt) { +char *linenoise(char *prompt) { char buf[LINENOISE_MAX_LINE]; int count; @@ -1009,14 +1009,14 @@ static void linenoiseAtExit(void) { * histories, but will work well for a few hundred of entries. * * Using a circular buffer is smarter, but a bit more complex to handle. */ -int linenoiseHistoryAdd(const char *line) { +int linenoiseHistoryAdd(char *line) { char *linecopy; if (history_max_len == 0) return 0; /* Initialization on first call. */ if (history == NULL) { - history = malloc(sizeof(char*)*history_max_len); + history = (char**)malloc(sizeof(char*)*history_max_len); if (history == NULL) return 0; memset(history,0,(sizeof(char*)*history_max_len)); } @@ -1043,14 +1043,14 @@ int linenoiseHistoryAdd(const char *line) { * just the latest 'len' elements if the new history length value is smaller * than the amount of items already inside the history. */ int linenoiseHistorySetMaxLen(int len) { - char **new; + char **neww; if (len < 1) return 0; if (history) { int tocopy = history_len; - new = malloc(sizeof(char*)*len); - if (new == NULL) return 0; + neww = (char**)malloc(sizeof(char*)*len); + if (neww == NULL) return 0; /* If we can't copy everything, free the elements we'll not use. */ if (len < tocopy) { @@ -1059,10 +1059,10 @@ int linenoiseHistorySetMaxLen(int len) { for (j = 0; j < tocopy-len; j++) free(history[j]); tocopy = len; } - memset(new,0,sizeof(char*)*len); - memcpy(new,history+(history_len-tocopy), sizeof(char*)*tocopy); + memset(neww,0,sizeof(char*)*len); + memcpy(neww,history+(history_len-tocopy), sizeof(char*)*tocopy); free(history); - history = new; + history = neww; } history_max_len = len; if (history_len > history_max_len) @@ -1072,7 +1072,7 @@ int linenoiseHistorySetMaxLen(int len) { /* Save the history in the specified file. On success 0 is returned * otherwise -1 is returned. */ -int linenoiseHistorySave(const char *filename) { +int linenoiseHistorySave(char *filename) { FILE *fp = fopen(filename,"w"); int j; @@ -1088,7 +1088,7 @@ int linenoiseHistorySave(const char *filename) { * * If the file exists and the operation succeeded 0 is returned, otherwise * on error -1 is returned. */ -int linenoiseHistoryLoad(const char *filename) { +int linenoiseHistoryLoad(char *filename) { FILE *fp = fopen(filename,"r"); char buf[LINENOISE_MAX_LINE]; diff --git a/lib/wrappers/linenoise/clinenoise.h b/lib/wrappers/linenoise/clinenoise.h index fbb01cfaa..a15845f86 100644 --- a/lib/wrappers/linenoise/clinenoise.h +++ b/lib/wrappers/linenoise/clinenoise.h @@ -39,30 +39,22 @@ #ifndef __LINENOISE_H #define __LINENOISE_H -#ifdef __cplusplus -extern "C" { -#endif - typedef struct linenoiseCompletions { size_t len; char **cvec; } linenoiseCompletions; -typedef void(linenoiseCompletionCallback)(const char *, linenoiseCompletions *); +typedef void(linenoiseCompletionCallback)(char *, linenoiseCompletions *); void linenoiseSetCompletionCallback(linenoiseCompletionCallback *); -void linenoiseAddCompletion(linenoiseCompletions *, const char *); +void linenoiseAddCompletion(linenoiseCompletions *, char *); -char *linenoise(const char *prompt); -int linenoiseHistoryAdd(const char *line); +char *linenoise(char *prompt); +int linenoiseHistoryAdd(char *line); int linenoiseHistorySetMaxLen(int len); -int linenoiseHistorySave(const char *filename); -int linenoiseHistoryLoad(const char *filename); +int linenoiseHistorySave(char *filename); +int linenoiseHistoryLoad(char *filename); void linenoiseClearScreen(void); void linenoiseSetMultiLine(int ml); void linenoisePrintKeyCodes(void); -#ifdef __cplusplus -} -#endif - #endif /* __LINENOISE_H */ diff --git a/lib/wrappers/mysql.nim b/lib/wrappers/mysql.nim index 8253e53a5..af504864d 100644 --- a/lib/wrappers/mysql.nim +++ b/lib/wrappers/mysql.nim @@ -418,6 +418,7 @@ type decimals*: cuint # Number of decimals in field charsetnr*: cuint # Character set ftype*: Enum_field_types # Type of field. See mysql_com.h for types + extension*: pointer FIELD* = St_mysql_field PFIELD* = ptr FIELD diff --git a/lib/wrappers/odbcsql.nim b/lib/wrappers/odbcsql.nim index 43ad80f76..1b2544ec0 100644 --- a/lib/wrappers/odbcsql.nim +++ b/lib/wrappers/odbcsql.nim @@ -641,11 +641,42 @@ const ODBC_CONFIG_SYS_DSN* = 5 ODBC_REMOVE_SYS_DSN* = 6 + SQL_ACTIVE_CONNECTIONS* = 0 # SQLGetInfo + SQL_DATA_SOURCE_NAME* = 2 + SQL_DATA_SOURCE_READ_ONLY* = 25 + SQL_DATABASE_NAME* = 2 + SQL_DBMS_NAME* = 17 + SQL_DBMS_VERSION* = 18 + SQL_DRIVER_HDBC* = 3 + SQL_DRIVER_HENV* = 4 + SQL_DRIVER_HSTMT* = 5 + SQL_DRIVER_NAME* = 6 + SQL_DRIVER_VER* = 7 + SQL_FETCH_DIRECTION* = 8 + SQL_ODBC_VER* = 10 + SQL_DRIVER_ODBC_VER* = 77 + SQL_SERVER_NAME* = 13 + SQL_ACTIVE_ENVIRONMENTS* = 116 + SQL_ACTIVE_STATEMENTS* = 1 + SQL_SQL_CONFORMANCE* = 118 + SQL_DATETIME_LITERALS* = 119 + SQL_ASYNC_MODE* = 10021 + SQL_BATCH_ROW_COUNT* = 120 + SQL_BATCH_SUPPORT* = 121 + SQL_CATALOG_LOCATION* = 114 + #SQL_CATALOG_NAME* = 10003 + SQL_CATALOG_NAME_SEPARATOR* = 41 + SQL_CATALOG_TERM* = 42 + SQL_CATALOG_USAGE* = 92 + #SQL_COLLATION_SEQ* = 10004 + SQL_COLUMN_ALIAS* = 87 + #SQL_USER_NAME* = 47 + proc SQLAllocHandle*(HandleType: TSqlSmallInt, InputHandle: SqlHandle, OutputHandlePtr: var SqlHandle): TSqlSmallInt{. dynlib: odbclib, importc.} proc SQLSetEnvAttr*(EnvironmentHandle: SqlHEnv, Attribute: TSqlInteger, - Value: SqlPointer, StringLength: TSqlInteger): TSqlSmallInt{. + Value: TSqlInteger, StringLength: TSqlInteger): TSqlSmallInt{. dynlib: odbclib, importc.} proc SQLGetEnvAttr*(EnvironmentHandle: SqlHEnv, Attribute: TSqlInteger, Value: SqlPointer, BufferLength: TSqlInteger, @@ -807,5 +838,10 @@ proc SQLStatistics*(hstmt: SqlHStmt, CatalogName: PSQLCHAR, NameLength3: TSqlSmallInt, Unique: SqlUSmallInt, Reserved: SqlUSmallInt): TSqlSmallInt {. dynlib: odbclib, importc.} +proc SQLErr*(henv: SqlHEnv, hdbc: SqlHDBC, hstmt: SqlHStmt, + szSqlState, pfNativeError, szErrorMsg: PSQLCHAR, + cbErrorMsgMax: TSqlSmallInt, + pcbErrorMsg: PSQLINTEGER): TSqlSmallInt {. + dynlib: odbclib, importc: "SQLError".} {.pop.} diff --git a/lib/wrappers/sqlite3.nim b/lib/wrappers/sqlite3.nim index c5019960c..e7fd2bc36 100644 --- a/lib/wrappers/sqlite3.nim +++ b/lib/wrappers/sqlite3.nim @@ -239,6 +239,8 @@ proc column_count*(pStmt: Pstmt): int32{.cdecl, dynlib: Lib, importc: "sqlite3_column_count".} proc column_name*(para1: Pstmt, para2: int32): cstring{.cdecl, dynlib: Lib, importc: "sqlite3_column_name".} +proc column_table_name*(para1: Pstmt; para2: int32): cstring{.cdecl, dynlib: Lib, + importc: "sqlite3_column_table_name".} proc column_name16*(para1: Pstmt, para2: int32): pointer{.cdecl, dynlib: Lib, importc: "sqlite3_column_name16".} proc column_decltype*(para1: Pstmt, i: int32): cstring{.cdecl, dynlib: Lib, diff --git a/readme.md b/readme.md index 4c996ebae..2fee6855d 100644 --- a/readme.md +++ b/readme.md @@ -1,6 +1,14 @@ # Nim Compiler -[](https://gitter.im/nim-lang/Nim?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) +[](https://webchat.freenode.net/?channels=nim) +[](http://forum.nim-lang.org) +[](http://stackoverflow.com/questions/tagged/nim?sort=newest&pageSize=15) +[](https://twitter.com/nim_lang) + +[](https://travis-ci.org/nim-lang/Nim) + +[](https://gratipay.com/nim/) +[](https://www.bountysource.com/teams/nim) This repo contains the Nim compiler, Nim's stdlib, tools and diff --git a/tests/async/tlambda.nim b/tests/async/tlambda.nim new file mode 100644 index 000000000..e0ff1f483 --- /dev/null +++ b/tests/async/tlambda.nim @@ -0,0 +1,55 @@ + +# bug 2007 + +import asyncdispatch, asyncnet, logging, json, uri, strutils, future + +type + Builder = ref object + client: Client + build: Build + + ProgressCB* = proc (message: string): Future[void] {.closure, gcsafe.} + + Build* = ref object + onProgress*: ProgressCB + + Client = ref ClientObj + ClientObj = object + onMessage: proc (client: Client, msg: JsonNode): Future[void] + +proc newClient*(name: string, + onMessage: (Client, JsonNode) -> Future[void]): Client = + new result + result.onMessage = onMessage + +proc newBuild*(onProgress: ProgressCB): Build = + new result + result.onProgress = onProgress + +proc start(build: Build, repo, hash: string) {.async.} = + let path = repo.parseUri().path.toLower() + +proc onProgress(builder: Builder, message: string) {.async.} = + debug($message) + +proc onMessage(builder: Builder, message: JsonNode) {.async.} = + debug("onMessage") + +proc newBuilder(): Builder = + var cres: Builder + new cres + + cres.client = newClient("builder", (client, msg) => (onMessage(cres, msg))) + cres.build = newBuild( + proc (msg: string): Future[void] {.closure, gcsafe.} = onProgress(cres, msg)) + return cres + +proc main() = + # Set up logging. + var console = newConsoleLogger(fmtStr = verboseFmtStr) + addHandler(console) + + var builder = newBuilder() + + +main() diff --git a/tests/async/tnimcall_to_closure.nim b/tests/async/tnimcall_to_closure.nim new file mode 100644 index 000000000..748b67cb1 --- /dev/null +++ b/tests/async/tnimcall_to_closure.nim @@ -0,0 +1,17 @@ + +import asyncdispatch + +proc defaultOnProgressChanged() = discard + +proc ask(x: proc()) = x() + +proc retrFile*(onProgressChanged: proc() {.nimcall.}): Future[void] = + var retFuture = newFuture[void]("retrFile") + iterator retrFileIter(): FutureBase {.closure.} = + ask(onProgressChanged) + complete(retFuture) + + var nameIterVar = retrFileIter + return retFuture + +discard retrFile(defaultOnProgressChanged) diff --git a/tests/ccgbugs/tgeneric_closure.nim b/tests/ccgbugs/tgeneric_closure.nim new file mode 100644 index 000000000..f9d5e7910 --- /dev/null +++ b/tests/ccgbugs/tgeneric_closure.nim @@ -0,0 +1,28 @@ + + +# bug 2659 + +type + GenProcType[T,U] = proc(x:T, y:var U) + IntProcType = proc(x:int, y:var int) + +proc mult(x:int, y:var int) = + y = 2 * x + +when isMainModule: + + var input = 1 + var output = 0 + + var someIntProc:IntProcType = mult + var someGenProc:GenProcType[int,int] = mult + + mult(input, output) + echo output + + someIntProc(input, output) + echo output + + # Uncommenting causes an error in the C compiler. + someGenProc(input, output) + echo output diff --git a/tests/closure/tclosure0.nim b/tests/closure/tclosure0.nim new file mode 100644 index 000000000..9952268d5 --- /dev/null +++ b/tests/closure/tclosure0.nim @@ -0,0 +1,87 @@ +discard """ + output: '''foo88 +23 24foo 88 +18 +18 +99 +99 +99 +99 99 +99 99 +12 99 99 +12 99 99''' +""" + +when true: + # test simple closure within dummy 'main': + proc dummy = + proc main2(param: int) = + var fooB = 23 + proc outer(outerParam: string) = + var outerVar = 88 + echo outerParam, outerVar + proc inner() = + block Test: + echo fooB, " ", param, outerParam, " ", outerVar + inner() + outer("foo") + main2(24) + + dummy() + +when true: + proc outer2(x:int) : proc(y:int):int = # curry-ed application + return proc(y:int):int = x*y + + var fn = outer2(6) # the closure + echo fn(3) # it works + + var rawP = fn.rawProc() + var rawE = fn.rawEnv() + + # A type to cast the function pointer into a nimcall + type + TimesClosure = proc(a: int, x: pointer): int {.nimcall.} + + # Call the function with its closure + echo cast[TimesClosure](rawP)(3, rawE) + +when true: + proc outer = + var x, y: int = 99 + proc innerA = echo x + proc innerB = + echo y + innerA() + + innerA() + innerB() + + outer() + +when true: + proc indirectDep = + var x, y: int = 99 + proc innerA = echo x, " ", y + proc innerB = + innerA() + + innerA() + innerB() + + indirectDep() + +when true: + proc needlessIndirection = + var x, y: int = 99 + proc indirection = + var z = 12 + proc innerA = echo z, " ", x, " ", y + proc innerB = + innerA() + + innerA() + innerB() + indirection() + + needlessIndirection() diff --git a/tests/closure/tclosure2.nim b/tests/closure/tclosure2.nim index d331388cf..9c5ee1426 100644 --- a/tests/closure/tclosure2.nim +++ b/tests/closure/tclosure2.nim @@ -87,7 +87,7 @@ when true: proc py() {.closure.} = echo "py" - const + let mapping = { "abc": px, "xyz": py diff --git a/tests/closure/tclosure3.nim b/tests/closure/tclosure3.nim index 8f6f4a70f..d5ffb5ab2 100644 --- a/tests/closure/tclosure3.nim +++ b/tests/closure/tclosure3.nim @@ -8,8 +8,9 @@ proc main = for iterations in 0..50_000: var s: seq[proc(): string {.closure.}] = @[] for i in 0 .. n-1: - let ii = i - s.add(proc(): string = return $(ii*ii)) + (proc () = + let ii = i + s.add(proc(): string = return $(ii*ii)))() for i in 0 .. n-1: let val = s[i]() if val != $(i*i): echo "bug ", val diff --git a/tests/closure/tclosurebug2.nim b/tests/closure/tclosurebug2.nim index 581b735bf..f131406a3 100644 --- a/tests/closure/tclosurebug2.nim +++ b/tests/closure/tclosurebug2.nim @@ -19,11 +19,11 @@ proc mustRehash(length, counter: int): bool {.inline.} = assert(length > counter) result = (length * 2 < counter * 3) or (length - counter < 4) -proc nextTry(h, maxHash: THash): THash {.inline.} = +proc nextTry(h, maxHash: Hash): Hash {.inline.} = result = ((5 * h) + 1) and maxHash template rawGetImpl() {.dirty.} = - var h: THash = hash(key) and high(t.data) # start with real hash value + var h: Hash = hash(key) and high(t.data) # start with real hash value while t.data[h].slot != seEmpty: if t.data[h].key == key and t.data[h].slot == seFilled: return h @@ -31,7 +31,7 @@ template rawGetImpl() {.dirty.} = result = -1 template rawInsertImpl() {.dirty.} = - var h: THash = hash(key) and high(data) + var h: Hash = hash(key) and high(data) while data[h].slot == seFilled: h = nextTry(h, high(data)) data[h].key = key diff --git a/tests/closure/tclosureinference3304.nim b/tests/closure/tclosureinference3304.nim new file mode 100644 index 000000000..db4aa1d04 --- /dev/null +++ b/tests/closure/tclosureinference3304.nim @@ -0,0 +1,15 @@ +discard """ + output: '''@[1, 2, 5]''' +""" + +import future, sequtils + +type + List[T] = ref object + val: T + +proc foo[T](l: List[T]): seq[int] = + @[1,2,3,5].filter(x => x != l.val) + +when isMainModule: + echo(foo(List[int](val: 3))) diff --git a/tests/closure/tcodegenerr1923.nim b/tests/closure/tcodegenerr1923.nim new file mode 100644 index 000000000..ee131ae15 --- /dev/null +++ b/tests/closure/tcodegenerr1923.nim @@ -0,0 +1,9 @@ +type + Foo[M] = proc() : M + +proc bar[M](f : Foo[M]) = + discard f() + +proc baz() : int = 42 + +bar(baz) \ No newline at end of file diff --git a/tests/closure/texplicit_dummy_closure.nim b/tests/closure/texplicit_dummy_closure.nim new file mode 100644 index 000000000..ec608b31a --- /dev/null +++ b/tests/closure/texplicit_dummy_closure.nim @@ -0,0 +1,22 @@ + +# This is a regression of the new lambda lifting; detected by Aporia +import asyncio, sockets +import os + +type + Window = object + oneInstSock*: PAsyncSocket + IODispatcher*: PDispatcher + +var + win: Window + +proc initSocket() = + win.oneInstSock = asyncSocket() + #win.oneInstSock.handleAccept = + proc test(s: PAsyncSocket) = + var client: PAsyncSocket + proc dummy(c: PAsyncSocket) {.closure.} = + discard + client.handleRead = dummy + test(win.oneInstSock) diff --git a/tests/closure/tfutclosure2138.nim b/tests/closure/tfutclosure2138.nim new file mode 100644 index 000000000..e18834074 --- /dev/null +++ b/tests/closure/tfutclosure2138.nim @@ -0,0 +1,10 @@ +import future, sequtils + +proc any[T](list: varargs[T], pred: (T) -> bool): bool = + for item in list: + if pred(item): + result = true + break + +proc contains(s: string, words: varargs[string]): bool = + any(words, (word) => s.contains(word)) \ No newline at end of file diff --git a/tests/closure/tinvalidclosure.nim b/tests/closure/tinvalidclosure.nim index c9136a736..d3f38cde5 100644 --- a/tests/closure/tinvalidclosure.nim +++ b/tests/closure/tinvalidclosure.nim @@ -1,9 +1,9 @@ discard """ line: 12 - errormsg: "type mismatch: got (proc (x: int){.closure, gcsafe, locks: 0.})" + errormsg: "type mismatch: got (proc (x: int){.gcsafe, locks: 0.})" """ -proc ugh[T](x: T) {.closure.} = +proc ugh[T](x: T) {.nimcall.} = echo "ugha" diff --git a/tests/closure/tissue1502def.nim b/tests/closure/tissue1502def.nim new file mode 100644 index 000000000..0aa6b16e3 --- /dev/null +++ b/tests/closure/tissue1502def.nim @@ -0,0 +1,6 @@ +import sequtils +let xs: seq[tuple[key: string, val: seq[string]]] = @[("foo", @["bar"])] + +let maps = xs.map( + proc(x: auto): tuple[typ: string, maps: seq[string]] = + (x.key, x.val.map(proc(x: string): string = x))) \ No newline at end of file diff --git a/tests/closure/tissue1642.nim b/tests/closure/tissue1642.nim index e3028c88e..5b921fc05 100644 --- a/tests/closure/tissue1642.nim +++ b/tests/closure/tissue1642.nim @@ -1,7 +1,3 @@ -discard """ - file: "tissue1642.nim" - disabled: true -""" block: - var i = 0 - proc p() = inc(i) + var i = 0 + proc p() = inc(i) \ No newline at end of file diff --git a/tests/closure/tissue1846.nim b/tests/closure/tissue1846.nim new file mode 100644 index 000000000..3fbef169d --- /dev/null +++ b/tests/closure/tissue1846.nim @@ -0,0 +1,16 @@ +type + TBinOp*[T] = proc (x,y: T): bool + + THeap*[T] = object + cmp*: TBinOp[T] + +proc less*[T](x,y: T): bool = + x < y + +proc initHeap*[T](cmp: TBinOp[T]): THeap[T] = + result.cmp = cmp + +when isMainModule: + var h = initHeap[int](less[int]) + + echo h.cmp(2,3) \ No newline at end of file diff --git a/tests/closure/tissue1911.nim b/tests/closure/tissue1911.nim new file mode 100644 index 000000000..311d99134 --- /dev/null +++ b/tests/closure/tissue1911.nim @@ -0,0 +1,7 @@ +proc foo(x: int) : auto = + + proc helper() : int = x + proc bar() : int = helper() + proc baz() : int = helper() + + return (bar, baz) \ No newline at end of file diff --git a/tests/closure/tissue600.nim b/tests/closure/tissue600.nim new file mode 100644 index 000000000..eacc7a123 --- /dev/null +++ b/tests/closure/tissue600.nim @@ -0,0 +1,4 @@ +for i in 1..1: + var reported = false + proc report() = + reported = true \ No newline at end of file diff --git a/tests/closure/tjester.nim b/tests/closure/tjester.nim index 3bd10120a..84e0fcb71 100644 --- a/tests/closure/tjester.nim +++ b/tests/closure/tjester.nim @@ -7,7 +7,7 @@ type data: T callback: proc () {.closure.} -proc cbOuter(response: string) {.closure, discardable.} = +proc cbOuter(response: string) {.discardable.} = iterator cbIter(): Future[int] {.closure.} = for i in 0..7: proc foo(): int = diff --git a/tests/closure/tmacrobust1512.nim b/tests/closure/tmacrobust1512.nim new file mode 100644 index 000000000..95681e750 --- /dev/null +++ b/tests/closure/tmacrobust1512.nim @@ -0,0 +1,137 @@ +import macros, strutils + +# https://github.com/nim-lang/Nim/issues/1512 + +proc macrobust0 (raw_input: string) = + var output = "" + proc p1 (a:string) = + output.add (a) + + proc p2 (a:string) = p1 (a) + proc p3 (a:string) = p2 (a) + proc p4 (a:string) = p3 (a) + proc p5 (a:string) = p4 (a) + proc p6 (a:string) = p5 (a) + proc p7 (a:string) = p6 (a) + proc p8 (a:string) = p7 (a) + proc p9 (a:string) = p8 (a) + proc p10 (a:string) = p9 (a) + proc p11 (a:string) = p10 (a) + proc p12 (a:string) = p11 (a) + proc p13 (a:string) = p12 (a) + proc p14 (a:string) = p13 (a) + proc p15 (a:string) = p14 (a) + proc p16 (a:string) = p15 (a) + proc p17 (a:string) = p16 (a) + proc p18 (a:string) = p17 (a) + proc p19 (a:string) = p18 (a) + proc p20 (a:string) = p19 (a) + + let input = $raw_input + + for a in input.split (): + p20 (a) + p19 (a) + + + p18 (a) + p17 (a) + p16 (a) + p15 (a) + p14 (a) + p13 (a) + p12 (a) + p11 (a) + p10 (a) + p9 (a) + p8 (a) + p7 (a) + p6 (a) + p5 (a) + p4 (a) + p3 (a) + p2 (a) + p1 (a) + + + echo output + +macro macrobust (raw_input: expr) : stmt = + + var output = "" + proc p1 (a:string) = + output.add (a) + + proc p2 (a:string) = p1 (a) + proc p3 (a:string) = p2 (a) + proc p4 (a:string) = p3 (a) + proc p5 (a:string) = p4 (a) + proc p6 (a:string) = p5 (a) + proc p7 (a:string) = p6 (a) + proc p8 (a:string) = p7 (a) + proc p9 (a:string) = p8 (a) + proc p10 (a:string) = p9 (a) + proc p11 (a:string) = p10 (a) + proc p12 (a:string) = p11 (a) + proc p13 (a:string) = p12 (a) + proc p14 (a:string) = p13 (a) + proc p15 (a:string) = p14 (a) + proc p16 (a:string) = p15 (a) + proc p17 (a:string) = p16 (a) + proc p18 (a:string) = p17 (a) + proc p19 (a:string) = p18 (a) + proc p20 (a:string) = p19 (a) + + let input = $raw_input + + for a in input.split (): + p20 (a) + p19 (a) + + p18 (a) + p17 (a) + p16 (a) + p15 (a) + p14 (a) + p13 (a) + p12 (a) + p11 (a) + p10 (a) + p9 (a) + p8 (a) + p7 (a) + p6 (a) + p5 (a) + p4 (a) + p3 (a) + p2 (a) + + echo output + discard result + +macrobust """ + fdsasadfsdfa sadfsdafsdaf + dsfsdafdsfadsfa fsdaasdfasdf + fsdafsadfsad asdfasdfasdf + fdsasdfasdfa sadfsadfsadf + sadfasdfsdaf sadfsdafsdaf dsfasdaf + sadfsdafsadf fdsasdafsadf fdsasadfsdaf + sdfasadfsdafdfsa sadfsadfsdaf + sdafsdaffsda sdfasadfsadf + fsdasdafsdfa sdfasdfafsda + sdfasdafsadf sdfasdafsdaf sdfasdafsdaf +""" + + +macrobust0 """ + fdsasadfsdfa sadfsdafsdaf + dsfsdafdsfadsfa fsdaasdfasdf + fsdafsadfsad asdfasdfasdf + fdsasdfasdfa sadfsadfsadf + sadfasdfsdaf sadfsdafsdaf dsfasdaf + sadfsdafsadf fdsasdafsadf fdsasadfsdaf + sdfasadfsdafdfsa sadfsadfsdaf + sdafsdaffsda sdfasadfsadf + fsdasdafsdfa sdfasdfafsda + sdfasdafsadf sdfasdafsdaf sdfasdafsdaf +""" \ No newline at end of file diff --git a/tests/closure/tnestedclosure.nim b/tests/closure/tnestedclosure.nim index 67e196f66..0628a6977 100644 --- a/tests/closure/tnestedclosure.nim +++ b/tests/closure/tnestedclosure.nim @@ -21,13 +21,13 @@ proc main(param: int) = # test simple closure within dummy 'main': proc dummy = proc main2(param: int) = - var foo = 23 + var fooB = 23 proc outer(outerParam: string) = var outerVar = 88 echo outerParam, outerVar proc inner() = block Test: - echo foo, " ", param, outerParam, " ", outerVar + echo fooB, " ", param, outerParam, " ", outerVar inner() outer("foo") main2(24) diff --git a/tests/closure/tnoclosure.nim b/tests/closure/tnoclosure.nim new file mode 100644 index 000000000..25cce0040 --- /dev/null +++ b/tests/closure/tnoclosure.nim @@ -0,0 +1,25 @@ +discard """ + output: '''@[1] +@[1, 1] +@[1, 2, 1] +@[1, 3, 3, 1] +@[1, 4, 6, 4, 1] +@[1, 5, 10, 10, 5, 1] +@[1, 6, 15, 20, 15, 6, 1] +@[1, 7, 21, 35, 35, 21, 7, 1] +@[1, 8, 28, 56, 70, 56, 28, 8, 1] +@[1, 9, 36, 84, 126, 126, 84, 36, 9, 1]''' +""" + +import sequtils + +proc pascal(n: int) = + var row = @[1] + for r in 1..n: + echo row + row = zip(row & @[0], @[0] & row).mapIt(it[0] + it[1]) + +pascal(10) + +# bug #3499 last snippet fixed +# bug 705 last snippet fixed diff --git a/tests/concepts/tmanual.nim b/tests/concepts/tmanual.nim index 7cf08af06..43290a6ad 100644 --- a/tests/concepts/tmanual.nim +++ b/tests/concepts/tmanual.nim @@ -23,7 +23,7 @@ template reject(e: expr) = type Container[T] = concept c c.len is Ordinal - items(c) is iterator + items(c) is T for value in c: type(value) is T diff --git a/tests/destructor/tdestructor3.nim b/tests/destructor/tdestructor3.nim index 0968f1fd7..d0c53c7bd 100644 --- a/tests/destructor/tdestructor3.nim +++ b/tests/destructor/tdestructor3.nim @@ -19,10 +19,11 @@ proc `=`(lhs: var T, rhs: T) = proc `=destroy`(v: var T) = echo "destroy" -block: +proc usedToBeBlock = var v1 : T var v2 : T = v1 +usedToBeBlock() # bug #1632 diff --git a/tests/gc/gcbench.nim b/tests/gc/gcbench.nim index 72337911d..782daf793 100644 --- a/tests/gc/gcbench.nim +++ b/tests/gc/gcbench.nim @@ -143,7 +143,7 @@ proc main() = # Create long-lived array, filling half of it echo(" Creating a long-lived array of " & $kArraySize & " doubles") newSeq(myarray, kArraySize) - for i in 0..kArraySize div 2 -1: + for i in 0..kArraySize div 2 - 1: myarray[i] = 1.0 / toFloat(i) PrintDiagnostics() diff --git a/tests/generics/tgeneric3.nim b/tests/generics/tgeneric3.nim index d4dac9385..0dbd5b03c 100644 --- a/tests/generics/tgeneric3.nim +++ b/tests/generics/tgeneric3.nim @@ -69,7 +69,7 @@ proc cmp[T:int8|int16|int32|int64|int] (a,b: T): T {.inline.} = template binSearchImpl *(docmp: expr) {.immediate.} = var bFound = false result = 0 - var H = haystack.len -1 + var H = haystack.len - 1 while result <= H : var I {.inject.} = (result + H) shr 1 var SW = docmp @@ -90,7 +90,7 @@ proc DeleteItem[T,D] (n: PNode[T,D], x: int): PNode[T,D] {.inline.} = return n dec(n.count) if n.count > 0 : - for i in countup(x, n.count -1) : n.slots[i] = n.slots[i + 1] + for i in countup(x, n.count - 1) : n.slots[i] = n.slots[i + 1] n.slots[n.count] = nil case n.count of cLen1 : setLen(n.slots, cLen1) @@ -121,7 +121,7 @@ proc internalDelete[T,D] (ANode: PNode[T,D], key: T, Avalue: var D): PNode[T,D] if x == 0 : n = n.left else : - x = (-x) -1 + x = (-x) - 1 if x < n.count : n = n.slots[x].node else : @@ -132,10 +132,10 @@ proc internalDelete[T,D] (ANode: PNode[T,D], key: T, Avalue: var D): PNode[T,D] Avalue = n.slots[x].value var n2 = DeleteItem(n, x) dec(h) - while (n2 != n) and (h >=0) : + while (n2 != n) and (h >= 0) : n = n2 var w = addr Path[h] - x = w.Xi -1 + x = w.Xi - 1 if x >= 0 : if (n == nil) and isClean(w.Nd, x) : n = w.Nd @@ -160,7 +160,7 @@ proc internalFind[T,D] (n: PNode[T,D], key: T): ref TItem[T,D] {.inline.} = if x == 0 : wn = wn.left else : - x = (-x) -1 + x = (-x) - 1 if x < wn.count : wn = wn.slots[x].node else : @@ -199,7 +199,7 @@ proc traceTree[T,D](root: PNode[T,D]) = if n.left != nil: traceln(space) write stdout, "left: " - doTrace(n.left, level +1) + doTrace(n.left, level+1) for i, el in n.slots : if el != nil and not isClean(el): traceln(space) @@ -208,7 +208,7 @@ proc traceTree[T,D](root: PNode[T,D]) = write stdout, "error " else: traceEl(el) - if el.node != nil: doTrace(el.node, level +1) + if el.node != nil: doTrace(el.node, level+1) else : write stdout, " empty " elif i < n.count : traceln(space) @@ -217,7 +217,7 @@ proc traceTree[T,D](root: PNode[T,D]) = when T is string : if el.key != nil: write stdout, el.key else : write stdout, el.key - if el.node != nil: doTrace(el.node, level +1) + if el.node != nil: doTrace(el.node, level+1) else : write stdout, " empty " writeLine stdout,"" @@ -245,25 +245,25 @@ proc SplitPage[T,D](n, left: PNode[T,D], xi: int, Akey:var T, Avalue:var D): PNo result.slots.newSeq(cLenCenter) result.count = cCenter if x == cCenter: - for i in 0..cCenter -1: shallowCopy(it1[i], left.slots[i]) - for i in 0..cCenter -1: shallowCopy(result.slots[i], left.slots[cCenter + i]) + for i in 0..cCenter-1: shallowCopy(it1[i], left.slots[i]) + for i in 0..cCenter-1: shallowCopy(result.slots[i], left.slots[cCenter + i]) result.left = n else : if x < cCenter : for i in 0..x-1: shallowCopy(it1[i], left.slots[i]) it1[x] = setItem(Akey, Avalue, n) - for i in x+1 .. cCenter -1: shallowCopy(it1[i], left.slots[i-1]) - var w = left.slots[cCenter -1] + for i in x+1 .. cCenter-1: shallowCopy(it1[i], left.slots[i-1]) + var w = left.slots[cCenter-1] Akey = w.key Avalue = w.value result.left = w.node - for i in 0..cCenter -1: shallowCopy(result.slots[i], left.slots[cCenter + i]) + for i in 0..cCenter-1: shallowCopy(result.slots[i], left.slots[cCenter + i]) else : - for i in 0..cCenter -1: shallowCopy(it1[i], left.slots[i]) + for i in 0..cCenter-1: shallowCopy(it1[i], left.slots[i]) x = x - (cCenter + 1) for i in 0..x-1: shallowCopy(result.slots[i], left.slots[cCenter + i + 1]) result.slots[x] = setItem(Akey, Avalue, n) - for i in x+1 .. cCenter -1: shallowCopy(result.slots[i], left.slots[cCenter + i]) + for i in x+1 .. cCenter-1: shallowCopy(result.slots[i], left.slots[cCenter + i]) var w = left.slots[cCenter] Akey = w.key Avalue = w.value @@ -290,7 +290,7 @@ proc internalPut[T,D](ANode: ref TNode[T,D], Akey: T, Avalue: D, Oldvalue: var D if x == 0 : n = n.left else : - x = (-x) -1 + x = (-x)-1 if x < n.count : n = n.slots[x].node else : diff --git a/tests/generics/tgenerictmpl.nim b/tests/generics/tgenerictmpl.nim index a749e6570..c71ce4e2e 100644 --- a/tests/generics/tgenerictmpl.nim +++ b/tests/generics/tgenerictmpl.nim @@ -1,12 +1,21 @@ +discard """ + output: '''0 +123''' +""" -template tmp[T](x: var seq[T]) = - #var yz: T # XXX doesn't work yet - x = @[1, 2, 3] +# bug #3498 + +template defaultOf[T](t: T): expr = (var d: T; d) + +echo defaultOf(1) #<- excpected 0 -macro tmp2[T](x: var seq[T]): stmt = - nil +# assignment using template + +template tassign[T](x: var seq[T]) = + x = @[1, 2, 3] var y: seq[int] -tmp(y) -tmp(y) -echo y.repr +tassign(y) #<- x is expected = @[1, 2, 3] +tassign(y) + +echo y[0], y[1], y[2] diff --git a/tests/generics/tspecialized_procvar.nim b/tests/generics/tspecialized_procvar.nim new file mode 100644 index 000000000..4bdc94a66 --- /dev/null +++ b/tests/generics/tspecialized_procvar.nim @@ -0,0 +1,17 @@ +discard """ + output: '''concrete 88''' +""" + +# Another regression triggered by changed closure computations: + +proc foo[T](x: proc(): T) = + echo "generic ", x() + +proc foo(x: proc(): int) = + echo "concrete ", x() + +# note the following 'proc' is not .closure! +foo(proc (): auto {.nimcall.} = 88) + +# bug #3499 last snippet fixed +# bug 705 last snippet fixed diff --git a/tests/iter/tclosureiters.nim b/tests/iter/tclosureiters.nim new file mode 100644 index 000000000..0eb624a8c --- /dev/null +++ b/tests/iter/tclosureiters.nim @@ -0,0 +1,73 @@ +discard """ + output: '''0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +5 5 +7 7 +9 9 +0 +0 +0 +0 +1 +2''' +""" + +when true: + proc main() = + let + lo=0 + hi=10 + + iterator itA(): int = + for x in lo..hi: + yield x + + for x in itA(): + echo x + + var y: int + + iterator itB(): int = + while y <= hi: + yield y + inc y + + y = 5 + for x in itB(): + echo x, " ", y + inc y + + main() + + +iterator infinite(): int {.closure.} = + var i = 0 + while true: + yield i + inc i + +iterator take[T](it: iterator (): T, numToTake: int): T {.closure.} = + var i = 0 + for x in it(): + if i >= numToTake: + break + yield x + inc i + +# gives wrong reasult (3 times 0) +for x in infinite.take(3): + echo x + +# does what we want +let inf = infinite +for x in inf.take(3): + echo x diff --git a/tests/iter/timplicit_auto.nim b/tests/iter/timplicit_auto.nim index ccb279fe0..d5cb95eb8 100644 --- a/tests/iter/timplicit_auto.nim +++ b/tests/iter/timplicit_auto.nim @@ -9,7 +9,7 @@ proc univ(x, y: int): State = Tree var w, h = 30 -iterator fields(a = (0,0), b = (h-1,w-1)) = +iterator fields(a = (0,0), b = (h-1,w-1)): auto = for y in max(a[0], 0) .. min(b[0], h-1): for x in max(a[1], 0) .. min(b[1], w-1): yield (y,x) diff --git a/tests/iter/titer10.nim b/tests/iter/titer10.nim new file mode 100644 index 000000000..6a6afc780 --- /dev/null +++ b/tests/iter/titer10.nim @@ -0,0 +1,51 @@ +discard """ + output: '''3 +2 +5 +1 +@[@[0, 0], @[0, 1]] +@[@[0, 0], @[0, 1]] +@[@[2, 2], @[2, 3]] +@[@[2, 2], @[2, 3]]''' +""" + +when true: + # bug #2604 + + import algorithm + + iterator byDistance*[int]( ints: openArray[int], base: int ): int = + var sortable = @ints + + sortable.sort do (a, b: int) -> int: + result = cmp( abs(base - a), abs(base - b) ) + + for val in sortable: + yield val + + when isMainModule: + proc main = + for val in byDistance([2, 3, 5, 1], 3): + echo val + main() + +when true: + # bug #1527 + + import sequtils + + let thread = @[@[0, 0], + @[0, 1], + @[2, 2], + @[2, 3]] + + iterator threadUniqs(seq1: seq[seq[int]]): seq[seq[int]] = + for i in 0 .. <seq1.len: + block: + let i = i + yield seq1.filter do (x: seq[int]) -> bool: x[0] == seq1[i][0] + proc main2 = + for uniqs in thread.threadUniqs: + echo uniqs + + main2() diff --git a/tests/iter/titer7.nim b/tests/iter/titer7.nim index d0337b7bd..c2bd9b9cb 100644 --- a/tests/iter/titer7.nim +++ b/tests/iter/titer7.nim @@ -14,11 +14,7 @@ discard """ 49 64 81 ---- squares of evens, only -4 -16 -36 -64''' +''' """ iterator `/`[T](sequence: seq[T], @@ -40,10 +36,10 @@ iterator `/>>`[I,O](sequence: seq[I], if (filtermap.f(element)): yield filtermap.m(element) -proc isEven(x:int): bool {.closure.} = result = +proc isEven(x:int): bool = (x and 1) == 0 -proc square(x:int): int {.closure.} = result = +proc square(x:int): int = x * x let list = @[1,2,3,4,5,6,7,8,9] @@ -52,6 +48,6 @@ echo ("--- evens") for item in list / isEven : echo(item) echo ("--- squares") for item in list >> square : echo(item) -echo ("--- squares of evens, only") +#echo ("--- squares of evens, only") # next line doesn't compile. Generic types are not inferred -for item in list />> (isEven, square) : echo(item) +#for item in list />> (isEven, square) : echo(item) diff --git a/tests/iter/titerable.nim b/tests/iter/titerable.nim index 3ec79f68d..fc1a8f934 100644 --- a/tests/iter/titerable.nim +++ b/tests/iter/titerable.nim @@ -6,8 +6,11 @@ discard """ 8 12 ''' + disabled: "true" """ +# Will eventually fix it... + iterator map[T, U](s: iterator:T{.inline.}, f: proc(x: T): U): U = for e in s: yield f(e) diff --git a/tests/iter/tkeep_state_between_yield.nim b/tests/iter/tkeep_state_between_yield.nim new file mode 100644 index 000000000..f4f0ee363 --- /dev/null +++ b/tests/iter/tkeep_state_between_yield.nim @@ -0,0 +1,36 @@ +discard """ + output: '''@[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 18, 20, 21, 24, 27, 30, 36, 40, 42] +1002''' +""" + +import strutils + +proc slice[T](iter: iterator(): T {.closure.}, sl: auto): seq[T] = + var res: seq[int64] = @[] + var i = 0 + for n in iter(): + if i > sl.b: + break + if i >= sl.a: + res.add(n) + inc i + res + +iterator harshad(): int64 {.closure.} = + for n in 1 .. < int64.high: + var sum = 0 + for ch in string($n): + sum += parseInt("" & ch) + if n mod sum == 0: + yield n + +echo harshad.slice 0 .. <20 + +for n in harshad(): + if n > 1000: + echo n + break + + +# bug #3499 last snippet fixed +# bug 705 last snippet fixed diff --git a/tests/iter/tnested_closure_iter.nim b/tests/iter/tnested_closure_iter.nim new file mode 100644 index 000000000..ec2253cf1 --- /dev/null +++ b/tests/iter/tnested_closure_iter.nim @@ -0,0 +1,16 @@ +discard """ + output: '''0 +1 +2''' +""" +# bug #1725 +iterator factory(): int {.closure.} = + iterator bar(): int {.closure.} = + yield 0 + yield 1 + yield 2 + + for x in bar(): yield x + +for x in factory(): + echo x diff --git a/tests/iter/tpermutations.nim b/tests/iter/tpermutations.nim new file mode 100644 index 000000000..a3b383323 --- /dev/null +++ b/tests/iter/tpermutations.nim @@ -0,0 +1,58 @@ + +import sequtils, future + +iterator permutations*[T](ys: openarray[T]): tuple[perm: seq[T], sign: int] = + var + d = 1 + c = newSeq[int](ys.len) + xs = newSeq[T](ys.len) + sign = 1 + + for i, y in ys: xs[i] = y + yield (xs, sign) + + block outter: + while true: + while d > 1: + dec d + c[d] = 0 + while c[d] >= d: + inc d + if d >= ys.len: break outter + + let i = if (d and 1) == 1: c[d] else: 0 + swap xs[i], xs[d] + sign *= -1 + yield (xs, sign) + inc c[d] + +proc det(a: seq[seq[float]]): float = + let n = toSeq 0..a.high + for sigma, sign in n.permutations: + result += sign.float * n.map((i: int) => a[i][sigma[i]]).foldl(a * b) + +proc perm(a: seq[seq[float]]): float = + let n = toSeq 0..a.high + for sigma, sign in n.permutations: + result += n.map((i: int) => a[i][sigma[i]]).foldl(a * b) + +for a in [ + @[ @[1.0, 2.0] + , @[3.0, 4.0] + ], + @[ @[ 1.0, 2, 3, 4] + , @[ 4.0, 5, 6, 7] + , @[ 7.0, 8, 9, 10] + , @[10.0, 11, 12, 13] + ], + @[ @[ 0.0, 1, 2, 3, 4] + , @[ 5.0, 6, 7, 8, 9] + , @[10.0, 11, 12, 13, 14] + , @[15.0, 16, 17, 18, 19] + , @[20.0, 21, 22, 23, 24] + ] ]: + echo a + echo "perm: ", a.perm, " det: ", a.det + +# bug #3499 last snippet fixed +# bug 705 last snippet fixed diff --git a/tests/iter/twrap_walkdir.nim b/tests/iter/twrap_walkdir.nim new file mode 100644 index 000000000..4ac487d8e --- /dev/null +++ b/tests/iter/twrap_walkdir.nim @@ -0,0 +1,16 @@ + + + +import os + +# bug #3636 + +proc fooIt(foo: string): iterator(): (string) = + iterator temp(): (string) = + for f in walkDirRec(foo): # No problem with walkFiles + yield f + return temp + +let it = fooIt(".") +for x in it(): + echo x diff --git a/tests/js/tvarargs.nim b/tests/js/tvarargs.nim new file mode 100644 index 000000000..e2366d7a9 --- /dev/null +++ b/tests/js/tvarargs.nim @@ -0,0 +1,12 @@ + +# bug #3584 + +type + ConsoleObj {.importc.} = object of RootObj + log*: proc() {.nimcall varargs.} + Console = ref ConsoleObj + +var console* {.importc nodecl.}: Console + +when isMainModule: + console.log "Hello, world" diff --git a/tests/macros/tnodecompare.nim b/tests/macros/tnodecompare.nim index ef25ae370..3870c7559 100644 --- a/tests/macros/tnodecompare.nim +++ b/tests/macros/tnodecompare.nim @@ -1,12 +1,12 @@ discard """ -output: '''1 -0 -1 -0 -1 -0 -1 -0''' +output: '''true +false +true +false +true +false +true +false''' """ import macros diff --git a/tests/macros/tsametype.nim b/tests/macros/tsametype.nim index 6baa34751..34296015f 100644 --- a/tests/macros/tsametype.nim +++ b/tests/macros/tsametype.nim @@ -1,14 +1,14 @@ discard """ -output: '''1 -0 -1 -0 -1 -0 -1 -0 -1 -0''' +output: '''true +false +true +false +true +false +true +false +true +false''' """ import macros diff --git a/tests/manyloc/keineschweine/dependencies/genpacket/genpacket_enet.nim b/tests/manyloc/keineschweine/dependencies/genpacket/genpacket_enet.nim index 7cfd67c49..142b190ab 100644 --- a/tests/manyloc/keineschweine/dependencies/genpacket/genpacket_enet.nim +++ b/tests/manyloc/keineschweine/dependencies/genpacket/genpacket_enet.nim @@ -131,10 +131,10 @@ macro defPacket*(typeNameN: expr, typeFields: expr): stmt {.immediate.} = emptyNode())), emptyNode(), emptyNode(), - newNimNode(nnkStmtList).und(#[6] + newNimNode(nnkStmtList).und(# [6] newNimNode(nnkAsgn).und( ^"result", ## result = - newNimNode(nnkCall).und(#[6][0][1] + newNimNode(nnkCall).und(# [6][0][1] ^"format", ## format emptyNode())))) ## "[TypeName $1 $2]" formatStr = "["& $typeName.ident @@ -277,7 +277,7 @@ when isMainModule: s.flush defPacket(Y, tuple[z: int8]) - proc `$`(z: Y): string = result = "Y("& $z.z &")" + proc `$`(z: Y): string = result = "Y(" & $z.z & ")" defPacket(TestPkt, tuple[x: seq[Y]]) var test = newTestPkt() test.x.add([newY(5), newY(4), newY(3), newY(2), newY(1)]) diff --git a/tests/manyloc/keineschweine/lib/client_helpers.nim b/tests/manyloc/keineschweine/lib/client_helpers.nim index f2833fe14..5f819a7d1 100644 --- a/tests/manyloc/keineschweine/lib/client_helpers.nim +++ b/tests/manyloc/keineschweine/lib/client_helpers.nim @@ -66,7 +66,7 @@ proc handlePackets*(server: PServer; buf: PBuffer) = proc updateFileProgress*() = let progress = currentFileTransfer.pos / currentFileTransfer.fullLen downloadProgress.bg.setSize(vec2f(progress * 100, 20)) - downloadProgress.setString($currentFileTransfer.pos &'/'& $currentFileTransfer.fullLen) + downloadProgress.setString($currentFileTransfer.pos & '/' & $currentFileTransfer.fullLen) ## HFileTransfer proc handleFilePartRecv*(serv: PServer; buffer: PBuffer) {.procvar.} = diff --git a/tests/manyloc/keineschweine/lib/sg_assets.nim b/tests/manyloc/keineschweine/lib/sg_assets.nim index 3b9781649..801c3456b 100644 --- a/tests/manyloc/keineschweine/lib/sg_assets.nim +++ b/tests/manyloc/keineschweine/lib/sg_assets.nim @@ -573,13 +573,13 @@ proc importItem(data: PJsonNode; errors: var seq[string]): PItemRecord = elif data[2]["bullet"].kind == JObject: result.bullet = importBullet(data[2]["bullet"], errors) else: - errors.add "UNKNOWN BULLET TYPE for item "& result.name + errors.add "UNKNOWN BULLET TYPE for item " & result.name of "ammo": result.kind = Ammo of "utility": nil else: - errors.add "Invalid item type \""& data[1].str &"\" for item "& result.name + errors.add "Invalid item type \""&data[1].str&"\" for item "&result.name proc importBullet(data: PJsonNode; errors: var seq[string]): PBulletRecord = new(result) diff --git a/tests/manyloc/nake/nakefile.nim b/tests/manyloc/nake/nakefile.nim index de6a1af40..e91b86986 100644 --- a/tests/manyloc/nake/nakefile.nim +++ b/tests/manyloc/nake/nakefile.nim @@ -146,7 +146,7 @@ task "download", "download game assets": task "zip-lib", "zip up the libs dir": var z: TZipArchive - if not z.open("libs-"& getDateStr() &".zip", fmReadWrite): + if not z.open("libs-" & getDateStr() & ".zip", fmReadWrite): quit "Could not open zip" for file in walkDirRec("libs", {pcFile, pcDir}): echo "adding file ", file diff --git a/tests/metatype/ttypeclasses.nim b/tests/metatype/ttypeclasses.nim index db9db7713..720527088 100644 --- a/tests/metatype/ttypeclasses.nim +++ b/tests/metatype/ttypeclasses.nim @@ -17,12 +17,12 @@ type TFoo[T] = object val: T - T1 = expr - T2 = expr + T1 = auto + T2 = auto Numeric = int|float -proc takesExpr(x, y) = +proc takesExpr(x, y: auto) = echo x, y proc same(x, y: T1) = @@ -31,7 +31,7 @@ proc same(x, y: T1) = proc takesFoo(x, y: TFoo) = echo x.val, y.val -proc takes2Types(x,y: T1, z: T2) = +proc takes2Types[T1, T2](x,y: T1, z: T2) = echo x, y, z takesExpr(1, 2) diff --git a/tests/metatype/ttypedesc3.nim b/tests/metatype/ttypedesc3.nim index 3d40b25b2..9f19bd6e3 100644 --- a/tests/metatype/ttypedesc3.nim +++ b/tests/metatype/ttypedesc3.nim @@ -6,7 +6,7 @@ type proc pr(T: typedesc[Base]) = echo "proc " & T.name method me(T: typedesc[Base]) = echo "method " & T.name -iterator it(T: typedesc[Base]) = yield "yield " & T.name +iterator it(T: typedesc[Base]): auto = yield "yield " & T.name Base.pr Child.pr diff --git a/tests/method/tmultim8.nim b/tests/method/tmultim8.nim new file mode 100644 index 000000000..0d067b668 --- /dev/null +++ b/tests/method/tmultim8.nim @@ -0,0 +1,19 @@ + +# bug #3550 + +type + BaseClass = ref object of RootObj + Class1 = ref object of BaseClass + Class2 = ref object of BaseClass + +method test(obj: Class1, obj2: BaseClass) = + discard + +method test(obj: Class2, obj2: BaseClass) = + discard + +var obj1 = Class1() +var obj2 = Class2() + +obj1.test(obj2) +obj2.test(obj1) diff --git a/tests/misc/tfsmonitor.nim b/tests/misc/tfsmonitor.nim new file mode 100644 index 000000000..27e1a2e32 --- /dev/null +++ b/tests/misc/tfsmonitor.nim @@ -0,0 +1,12 @@ +# +# fsmonitor test +# + +import unittest +import fsmonitor + +suite "fsmonitor": + test "should not raise OSError, bug# 3611": + let m = newMonitor() + m.add("foo", {MonitorCloseWrite, MonitorCloseNoWrite}) + diff --git a/tests/openarray/tptrarrayderef.nim b/tests/openarray/tptrarrayderef.nim new file mode 100644 index 000000000..1e73be108 --- /dev/null +++ b/tests/openarray/tptrarrayderef.nim @@ -0,0 +1,54 @@ +discard """ + file: "tptrarrayderef.nim" + output: "OK" +""" + +var + arr = [1,2,3] + arrp = addr(arr) + sss = @[4,5,6,7] + sssp = addr(sss) + ra = new(array[3, int]) + raa = [11,12,13] + +#bug #3586 +proc mutate[T](arr:openarray[T], brr: openArray[T]) = + for i in 0..arr.len-1: + doAssert(arr[i] == brr[i]) + +mutate(arr, arr) + +#bug #2240 +proc f(a: openarray[int], b: openArray[int]) = + for i in 0..a.len-1: + doAssert(a[i] == b[i]) + +var a = [7,8,9] +var p = addr a +f(p[], a) +f(sssp[], sss) + +ra[0] = 11 +ra[1] = 12 +ra[2] = 13 +f(ra[], raa) + +#bug #2240b +proc fillBuffer(buf: var openarray[char]) = + for i in 0..buf.len-1: + buf[i] = chr(i) + +proc fillSeqBuffer(b: ref seq[char]) = + fillBuffer(b[]) + +proc getFilledBuffer(sz: int): ref seq[char] = + let s : ref seq[char] = new(seq[char]) + s[] = newSeq[char](sz) + fillBuffer(s[]) + return s + +let aa = getFilledBuffer(3) +for i in 0..aa[].len-1: + doAssert(aa[i] == chr(i)) + +echo "OK" \ No newline at end of file diff --git a/tests/parser/tmultiline_comments.nim b/tests/parser/tmultiline_comments.nim new file mode 100644 index 000000000..7a3bb5304 --- /dev/null +++ b/tests/parser/tmultiline_comments.nim @@ -0,0 +1,64 @@ +discard """ + output: '''3''' +""" + +proc main* = + ##[Mutltie akdlsf comment with #[nesting]. + Yay, that is so cool. + ]## + echo "foo bar" + for s in ["one", "two", #["three",]# "four"]: + echo s + +var foo #[ Test the new inline comments ]#: int = 3 +##[ A +novel documentation comment +#[Nesting works to some extend] +##[ Nested doc comment! ]## +]# +]## +echo $foo + + #[Comment here. + Multiple lines + are not a problem.]# + + #[ #[ Multiline comment in already + commented out code. ]# + proc p[T](x: T) = discard + ]# + +proc bar = + ##[Long documentation comment + here. + ]## + + +proc write(a: auto, x: varargs[string, `$`]) = + stdout.write ($a) + for o in x: + stdout.write(o) + +proc writeln(a: auto, x: varargs[string, `$`]) = + write a, x + stdout.write "\n" + +proc write() = write(stdout) +proc writeln() = + stdout.write "\n" + +#[ #[ Multiline comment in already + commented out code. ]# +proc p[T](x: T) = discard +]# + +var hello = #[(x in bar)^^ "Hello" # greetings +]#"Hello" +proc maino = + write hello, " Test Me " + writeln() + write 3 + block: + write() + write " times more" + #[ test ]# writeln " Again" diff --git a/tests/stdlib/tpegs.nim b/tests/stdlib/tpegs.nim index 3fe964d82..ec839e288 100644 --- a/tests/stdlib/tpegs.nim +++ b/tests/stdlib/tpegs.nim @@ -764,7 +764,7 @@ proc match*(s: string, pattern: TPeg, matches: var openarray[string], ## returned. var c: TCaptures c.origStart = start - result = rawMatch(s, pattern, start, c) == len(s) -start + result = rawMatch(s, pattern, start, c) == len(s)-start if result: for i in 0..c.ml-1: matches[i] = substr(s, c.matches[i][0], c.matches[i][1]) diff --git a/tests/stdlib/tstrutil.nim b/tests/stdlib/tstrutil.nim index b15bf0e68..b97f2b1e9 100644 --- a/tests/stdlib/tstrutil.nim +++ b/tests/stdlib/tstrutil.nim @@ -78,6 +78,7 @@ assert(editDistance("prefix__hallo_suffix", "prefix__ha_suffix") == 3) assert(editDistance("prefix__hallo_suffix", "prefix") == 14) assert(editDistance("prefix__hallo_suffix", "suffix") == 14) assert(editDistance("prefix__hallo_suffix", "prefix__hao_suffix") == 2) +assert(editDistance("main", "malign") == 2) assert "/1/2/3".rfind('/') == 4 assert "/1/2/3".rfind('/', 1) == 0 diff --git a/tests/stdlib/ttime.nim b/tests/stdlib/ttime.nim index 859f0abdd..ac37196fb 100644 --- a/tests/stdlib/ttime.nim +++ b/tests/stdlib/ttime.nim @@ -1,6 +1,149 @@ # test the new time module +discard """ + file: "ttime.nim" +""" import - times + times, strutils -write(stdout, $getTime()) +# $ date --date='@2147483647' +# Tue 19 Jan 03:14:07 GMT 2038 + +var t = getGMTime(fromSeconds(2147483647)) +doAssert t.format("ddd dd MMM hh:mm:ss ZZZ yyyy") == "Tue 19 Jan 03:14:07 UTC 2038" +doAssert t.format("ddd ddMMMhh:mm:ssZZZyyyy") == "Tue 19Jan03:14:07UTC2038" + +doAssert t.format("d dd ddd dddd h hh H HH m mm M MM MMM MMMM s" & + " ss t tt y yy yyy yyyy yyyyy z zz zzz ZZZ") == + "19 19 Tue Tuesday 3 03 3 03 14 14 1 01 Jan January 7 07 A AM 8 38 038 2038 02038 0 00 00:00 UTC" + +doAssert t.format("yyyyMMddhhmmss") == "20380119031407" + +var t2 = getGMTime(fromSeconds(160070789)) # Mon 27 Jan 16:06:29 GMT 1975 +doAssert t2.format("d dd ddd dddd h hh H HH m mm M MM MMM MMMM s" & + " ss t tt y yy yyy yyyy yyyyy z zz zzz ZZZ") == + "27 27 Mon Monday 4 04 16 16 6 06 1 01 Jan January 29 29 P PM 5 75 975 1975 01975 0 00 00:00 UTC" + +when not defined(JS): + when sizeof(Time) == 8: + var t3 = getGMTime(fromSeconds(889067643645)) # Fri 7 Jun 19:20:45 BST 30143 + doAssert t3.format("d dd ddd dddd h hh H HH m mm M MM MMM MMMM s" & + " ss t tt y yy yyy yyyy yyyyy z zz zzz ZZZ") == + "7 07 Fri Friday 6 06 18 18 20 20 6 06 Jun June 45 45 P PM 3 43 143 0143 30143 0 00 00:00 UTC" + doAssert t3.format(":,[]()-/") == ":,[]()-/" + +var t4 = getGMTime(fromSeconds(876124714)) # Mon 6 Oct 08:58:34 BST 1997 +doAssert t4.format("M MM MMM MMMM") == "10 10 Oct October" + +# Interval tests +doAssert((t4 - initInterval(years = 2)).format("yyyy") == "1995") +doAssert((t4 - initInterval(years = 7, minutes = 34, seconds = 24)).format("yyyy mm ss") == "1990 24 10") + +var s = "Tuesday at 09:04am on Dec 15, 2015" +var f = "dddd at hh:mmtt on MMM d, yyyy" +doAssert($s.parse(f) == "Tue Dec 15 09:04:00 2015") +# ANSIC = "Mon Jan _2 15:04:05 2006" +s = "Thu Jan 12 15:04:05 2006" +f = "ddd MMM dd HH:mm:ss yyyy" +doAssert($s.parse(f) == "Thu Jan 12 15:04:05 2006") +# UnixDate = "Mon Jan _2 15:04:05 MST 2006" +s = "Thu Jan 12 15:04:05 MST 2006" +f = "ddd MMM dd HH:mm:ss ZZZ yyyy" +doAssert($s.parse(f) == "Thu Jan 12 15:04:05 2006") +# RubyDate = "Mon Jan 02 15:04:05 -0700 2006" +s = "Thu Jan 12 15:04:05 -07:00 2006" +f = "ddd MMM dd HH:mm:ss zzz yyyy" +doAssert($s.parse(f) == "Thu Jan 12 15:04:05 2006") +# RFC822 = "02 Jan 06 15:04 MST" +s = "12 Jan 16 15:04 MST" +f = "dd MMM yy HH:mm ZZZ" +doAssert($s.parse(f) == "Tue Jan 12 15:04:00 2016") +# RFC822Z = "02 Jan 06 15:04 -0700" # RFC822 with numeric zone +s = "12 Jan 16 15:04 -07:00" +f = "dd MMM yy HH:mm zzz" +doAssert($s.parse(f) == "Tue Jan 12 15:04:00 2016") +# RFC850 = "Monday, 02-Jan-06 15:04:05 MST" +s = "Monday, 12-Jan-06 15:04:05 MST" +f = "dddd, dd-MMM-yy HH:mm:ss ZZZ" +doAssert($s.parse(f) == "Thu Jan 12 15:04:05 2006") +# RFC1123 = "Mon, 02 Jan 2006 15:04:05 MST" +s = "Thu, 12 Jan 2006 15:04:05 MST" +f = "ddd, dd MMM yyyy HH:mm:ss ZZZ" +doAssert($s.parse(f) == "Thu Jan 12 15:04:05 2006") +# RFC1123Z = "Mon, 02 Jan 2006 15:04:05 -0700" # RFC1123 with numeric zone +s = "Thu, 12 Jan 2006 15:04:05 -07:00" +f = "ddd, dd MMM yyyy HH:mm:ss zzz" +doAssert($s.parse(f) == "Thu Jan 12 15:04:05 2006") +# RFC3339 = "2006-01-02T15:04:05Z07:00" +s = "2006-01-12T15:04:05Z-07:00" +f = "yyyy-MM-ddTHH:mm:ssZzzz" +doAssert($s.parse(f) == "Thu Jan 12 15:04:05 2006") +f = "yyyy-MM-dd'T'HH:mm:ss'Z'zzz" +doAssert($s.parse(f) == "Thu Jan 12 15:04:05 2006") +# RFC3339Nano = "2006-01-02T15:04:05.999999999Z07:00" +s = "2006-01-12T15:04:05.999999999Z-07:00" +f = "yyyy-MM-ddTHH:mm:ss.999999999Zzzz" +doAssert($s.parse(f) == "Thu Jan 12 15:04:05 2006") +# Kitchen = "3:04PM" +s = "3:04PM" +f = "h:mmtt" +doAssert "15:04:00" in $s.parse(f) +#when not defined(testing): +# echo "Kitchen: " & $s.parse(f) +# var ti = timeToTimeInfo(getTime()) +# echo "Todays date after decoding: ", ti +# var tint = timeToTimeInterval(getTime()) +# echo "Todays date after decoding to interval: ", tint + +# checking dayOfWeek matches known days +doAssert getDayOfWeek(21, 9, 1900) == dFri +doAssert getDayOfWeek(1, 1, 1970) == dThu +doAssert getDayOfWeek(21, 9, 1970) == dMon +doAssert getDayOfWeek(1, 1, 2000) == dSat +doAssert getDayOfWeek(1, 1, 2021) == dFri +# Julian tests +doAssert getDayOfWeekJulian(21, 9, 1900) == dFri +doAssert getDayOfWeekJulian(21, 9, 1970) == dMon +doAssert getDayOfWeekJulian(1, 1, 2000) == dSat +doAssert getDayOfWeekJulian(1, 1, 2021) == dFri + +# toSeconds tests with GM and Local timezones +#var t4 = getGMTime(fromSeconds(876124714)) # Mon 6 Oct 08:58:34 BST 1997 +var t4L = getLocalTime(fromSeconds(876124714)) +doAssert toSeconds(timeInfoToTime(t4L)) == 876124714 # fromSeconds is effectively "localTime" +doAssert toSeconds(timeInfoToTime(t4L)) + t4L.timezone.float == toSeconds(timeInfoToTime(t4)) + +# adding intervals +var + a1L = toSeconds(timeInfoToTime(t4L + initInterval(hours = 1))) + t4L.timezone.float + a1G = toSeconds(timeInfoToTime(t4)) + 60.0 * 60.0 +doAssert a1L == a1G + +# subtracting intervals +a1L = toSeconds(timeInfoToTime(t4L - initInterval(hours = 1))) + t4L.timezone.float +a1G = toSeconds(timeInfoToTime(t4)) - (60.0 * 60.0) +doAssert a1L == a1G + +# add/subtract TimeIntervals and Time/TimeInfo +doAssert getTime() - 1.seconds == getTime() - 3.seconds + 2.seconds +doAssert getTime() + 65.seconds == getTime() + 1.minutes + 5.seconds +doAssert getTime() + 60.minutes == getTime() + 1.hours +doAssert getTime() + 24.hours == getTime() + 1.days +doAssert getTime() + 13.months == getTime() + 1.years + 1.months +var + ti1 = getTime() + 1.years +ti1 -= 1.years +doAssert ti1 == getTime() +ti1 += 1.days +doAssert ti1 == getTime() + 1.days + +# overflow of TimeIntervals on initalisation +doAssert initInterval(milliseconds = 25000) == initInterval(seconds = 25) +doAssert initInterval(seconds = 65) == initInterval(seconds = 5, minutes = 1) +doAssert initInterval(hours = 25) == initInterval(hours = 1, days = 1) +doAssert initInterval(months = 13) == initInterval(months = 1, years = 1) + +# Bug with adding a day to a Time +let day = 24.hours +let tomorrow = getTime() + day +doAssert tomorrow - getTime() == 60*60*24 \ No newline at end of file diff --git a/tests/testament/categories.nim b/tests/testament/categories.nim index 762c92792..73d72289c 100644 --- a/tests/testament/categories.nim +++ b/tests/testament/categories.nim @@ -223,6 +223,9 @@ proc jsTests(r: var TResults, cat: Category, options: string) = "varres/tvartup"]: test "tests/" & testfile & ".nim" + for testfile in ["pure/strutils"]: + test "lib/" & testfile & ".nim" + # ------------------------- manyloc ------------------------------------------- #proc runSpecialTests(r: var TResults, options: string) = # for t in ["lib/packages/docutils/highlite"]: diff --git a/tests/testdata/doc1.xml b/tests/testdata/doc1.xml index 4e77481aa..07cbceeb7 100644 --- a/tests/testdata/doc1.xml +++ b/tests/testdata/doc1.xml @@ -5,6 +5,7 @@ <test2> bla ah absy hsh hsh + &woohoo; sjj </test2> <test><teh>bla</teh></test> diff --git a/tests/types/tillegaltyperecursion.nim b/tests/types/tillegaltyperecursion.nim index bace2dfc8..52fbd622f 100644 --- a/tests/types/tillegaltyperecursion.nim +++ b/tests/types/tillegaltyperecursion.nim @@ -10,14 +10,14 @@ import strutils import os type - TMessageReceivedEventArgs = object of TEventArgs + TMessageReceivedEventArgs = object of EventArgs Nick*: string Message*: string TIRC = object - EventEmitter: TEventEmitter - MessageReceivedHandler*: TEventHandler - Socket: TSocket - Thread: TThread[TIRC] + EventEmitter: EventEmitter + MessageReceivedHandler*: EventHandler + Socket: Socket + Thread: Thread[TIRC] proc initIRC*(): TIRC = result.Socket = socket() @@ -49,8 +49,8 @@ proc handleData(irc: TIRC) {.thread.} = return proc Connect*(irc: var TIRC, nick: string, host: string, port: int = 6667) = - connect(irc.Socket ,host ,TPort(port),TDomain.AF_INET) - send(irc.Socket,"USER " & nick & " " & nick & " " & nick & " " & nick &"\r\L") + connect(irc.Socket, host, TPort(port), TDomain.AF_INET) + send(irc.Socket,"USER " & nick & " " & nick & " " & nick & " " & nick & "\r\L") send(irc.Socket,"NICK " & nick & "\r\L") var thread: TThread[TIRC] createThread(thread, handleData, irc) diff --git a/tests/types/tisop.nim b/tests/types/tisop.nim index d0b7c32b2..ad5928016 100644 --- a/tests/types/tisop.nim +++ b/tests/types/tisop.nim @@ -32,7 +32,7 @@ static: assert(f.y.type.name == "string") when compiles(f.z): {.error: "Foo should not have a `z` field".} -proc p(a, b) = +proc p(a, b: auto) = when a.type is int: static: assert false diff --git a/tests/types/tisopr.nim b/tests/types/tisopr.nim index b9acfa5fb..14999ebee 100644 --- a/tests/types/tisopr.nim +++ b/tests/types/tisopr.nim @@ -23,22 +23,23 @@ template yes(e: expr): stmt = template no(e: expr): stmt = static: assert(not e) -var s = @[1, 2, 3] +when false: + var s = @[1, 2, 3] -yes s.items is iterator -no s.items is proc + yes s.items is iterator + no s.items is proc -yes s.items is iterator: int -no s.items is iterator: float + yes s.items is iterator: int + no s.items is iterator: float -yes s.items is iterator: TNumber -no s.items is iterator: object + yes s.items is iterator: TNumber + no s.items is iterator: object -type - Iter[T] = iterator: T + type + Iter[T] = iterator: T -yes s.items is Iter[TNumber] -no s.items is Iter[float] + yes s.items is Iter[TNumber] + no s.items is Iter[float] type Foo[N: static[int], T] = object diff --git a/tests/untestable/tpostgres.nim b/tests/untestable/tpostgres.nim index 5f29da091..dcbdaad39 100644 --- a/tests/untestable/tpostgres.nim +++ b/tests/untestable/tpostgres.nim @@ -1,4 +1,5 @@ -import db_postgres +import db_postgres, strutils + let db = open("localhost", "dom", "", "test") db.exec(sql"DROP TABLE IF EXISTS myTable") @@ -9,5 +10,73 @@ let name = "Dom" db.exec(sql"INSERT INTO myTable (id, name) VALUES (0, ?)", name) doAssert db.getValue(sql"SELECT name FROM myTable") == name +# Check issue #3513 +doAssert db.getValue(sql"SELECT name FROM myTable") == name + + +# issue #3560 +proc addToDb(conn: DbConn, fileId: int, fileName: string): int64 = + result = conn.insertId(sql("INSERT into files (id, filename) VALUES (?, ?)"), fileId, fileName) + +db.exec(sql"DROP TABLE IF EXISTS files") +db.exec(sql"DROP TABLE IF EXISTS fileobjects") +db.exec(sql("""CREATE TABLE FILEOBJECTS( + ID SERIAL PRIMARY KEY, + FILE_SIZE INT, + MD5 CHAR(32) NOT NULL UNIQUE + );""")) + +db.exec(sql("""CREATE TABLE FILES( + ID SERIAL PRIMARY KEY, + OBJECT_ID INT, + FILENAME TEXT NOT NULL, + URI TEXT, + SCHEME CHAR(10), + PUBLIC BOOLEAN DEFAULT FALSE, + CONSTRAINT fk1_fileobjs FOREIGN KEY (object_id) + REFERENCES fileobjects (id) MATCH SIMPLE + ON DELETE CASCADE + );""")) + +let f1 = db.addToDb(1, "hello.tmp") +doAssert f1 == 1 +let f2 = db.addToDb(2, "hello2.tmp") +doAssert f2 == 2 + +# PreparedStmt vs. normal query +try: + echo db.getValue(sql("select * from files where id = $1"), 1) + doAssert false, "Exception expected" +except DbError: + let msg = getCurrentExceptionMsg().normalize + doAssert "expects" in msg + doAssert "?" in msg + doAssert "parameter substitution" in msg + +doAssert db.getValue(sql("select filename from files where id = ?"), 1) == "hello.tmp" + +var first = prepare(db, "one", sql"select filename from files where id = $1", 1) +doAssert db.getValue(first, 1) == "hello.tmp" + +try: + var second = prepare(db, "two", sql"select filename from files where id = ?", 1) + doAssert false, "Exception expected" +except: + let msg = getCurrentExceptionMsg().normalize + doAssert "expects" in msg + doAssert "$1" in msg + doAssert "parameter substitution" in msg + +# issue #3569 +db.exec(SqlQuery("DROP TABLE IF EXISTS tags")) +db.exec(SqlQuery("CREATE TABLE tags(id serial UNIQUE, name varchar(255))")) + +for i in 1..10: + var name = "t" & $i + echo(name) + discard db.getRow( + SqlQuery("INSERT INTO tags(name) VALUES(\'$1\') RETURNING id" % [name])) + +echo("All tests succeeded!") -db.close() \ No newline at end of file +db.close() diff --git a/tests/vm/texcl.nim b/tests/vm/texcl.nim new file mode 100644 index 000000000..4ccfd6bfa --- /dev/null +++ b/tests/vm/texcl.nim @@ -0,0 +1,27 @@ +discard """ + output: '''false''' +""" + +import macros + +type + nlOptions = enum + nloNone + nloDebug + +var nlOpts {.compileTime.} = {nloDebug} + +proc initOpts(): set[nlOptions] = + result.incl nloDebug + result.incl nloNone + result.excl nloDebug + +const cOpts = initOpts() + +macro nlo(): stmt = + nlOpts.incl(nloNone) + nlOpts.excl(nloDebug) + result = newEmptyNode() + +nlo() +echo nloDebug in cOpts \ No newline at end of file diff --git a/tests/vm/ttouintconv.nim b/tests/vm/ttouintconv.nim new file mode 100644 index 000000000..cd25ffb00 --- /dev/null +++ b/tests/vm/ttouintconv.nim @@ -0,0 +1,77 @@ +import macros + +discard """ +msg: ''' +8 9 17 +239 255 +61439 65534 65535 +4026531839 4294967294 +17293822569102704639 +18446744073709551614 +18446744073709551615 +127 +32767 +2147483647 +9223372036854775807 +0 +128 +4294967287''' +""" + +#bug #2514 + +macro foo(): stmt = + var x = 8'u8 + var y = 9'u16 + var z = 17'u32 + + echo x," ", y," ", z + + var a = 0xEF'u8 + var aa = 0xFF'u8 + echo a, " ", aa + + var b = 0xEFFF'u16 + var bb = 0xFFFE'u16 + var bbb = 0xFFFF'u16 + echo b, " ", bb, " ", bbb + + var c = 0xEFFFFFFF'u32 + var cc = 0xFFFFFFFE'u32 + echo c, " ", cc + + var d = 0xEFFFFFFFFFFFFFFF'u64 + echo d + + var f = 0xFFFFFFFFFFFFFFFE'u64 + echo f + + var g = 0xFFFFFFFFFFFFFFFF'u64 + echo g + + var xx = 0x7F'u8 and 0xFF + echo xx + + var yy = 0x7FFF'u16 + echo yy + + var zz = 0x7FFFFFFF'u32 + echo zz + +macro foo2(): stmt = + var xx = 0x7FFFFFFFFFFFFFFF + echo xx + + var yy = 0 + echo yy + + var zz = 0x80'u8 + echo zz + + var ww = -9 + var vv = ww.uint + var kk = vv.uint32 + echo kk + +foo() +foo2() diff --git a/todo.txt b/todo.txt index 62ffdbd8a..6c1c602f5 100644 --- a/todo.txt +++ b/todo.txt @@ -1,24 +1,24 @@ essential for 1.0 ================= -- document special cased varargs[untyped] and varargs[typed] -- The remaining bugs of the lambda lifting pass that is responsible to enable - closures and closure iterators need to be fixed. +- introduce newSeqOfCap(10) +- annotation support for getType() +- overloading of `()` needs to be in .experimental +- find a solution for the x.f[T](y) gotcha - ``concept`` needs to be refined, a nice name for the feature is not enough. - Destructors need to be refined. - make '--implicitStatic:on' the default; then we can also clean up the 'static[T]' mess in the compiler! -- add "all threads are blocked" detection to 'spawn' - Deprecate ``immediate`` for templates and macros - document NimMain and check whether it works for threading -- remove echo $foo gotcha - ``not`` or ``~`` for the effects system Not critical for 1.0 ==================== +- add "all threads are blocked" detection to 'spawn' - figure out why C++ bootstrapping is so much slower - The bitwise 'not' operator cold be renamed to 'bnot' to prevent 'not 4 == 5' from compiling. -> requires 'mixin' annotation for procs! diff --git a/tools/nimgrep.nim b/tools/nimgrep.nim index 221181f66..e93168847 100644 --- a/tools/nimgrep.nim +++ b/tools/nimgrep.nim @@ -11,7 +11,7 @@ import os, strutils, parseopt, pegs, re, terminal const - Version = "0.9" + Version = "1.0" Usage = "nimgrep - Nim Grep Utility Version " & Version & """ (c) 2012 Andreas Rumpf @@ -56,6 +56,7 @@ var proc ask(msg: string): string = stdout.write(msg) + stdout.flushFile() result = stdin.readLine() proc confirm: TConfirmEnum = @@ -108,18 +109,21 @@ proc highlight(s, match, repl: string, t: tuple[first, last: int], writeColored(match) for i in t.last+1 .. y: stdout.write(s[i]) stdout.write("\n") + stdout.flushFile() if showRepl: stdout.write(spaces(alignment-1), "-> ") for i in x .. t.first-1: stdout.write(s[i]) writeColored(repl) for i in t.last+1 .. y: stdout.write(s[i]) stdout.write("\n") + stdout.flushFile() proc processFile(filename: string) = var filenameShown = false template beforeHighlight = if not filenameShown and optVerbose notin options: stdout.writeLine(filename) + stdout.flushFile() filenameShown = true var buffer: string @@ -128,7 +132,9 @@ proc processFile(filename: string) = except IOError: echo "cannot open file: ", filename return - if optVerbose in options: stdout.writeLine(filename) + if optVerbose in options: + stdout.writeLine(filename) + stdout.flushFile() var pegp: TPeg var rep: Regex var result: string @@ -254,10 +260,12 @@ proc walker(dir: string) = proc writeHelp() = stdout.write(Usage) + stdout.flushFile() quit(0) proc writeVersion() = stdout.write(Version & "\n") + stdout.flushFile() quit(0) proc checkOptions(subset: TOptions, a, b: string) = @@ -291,7 +299,7 @@ for kind, key, val in getopt(): of "word", "w": incl(options, optWord) of "ignorecase", "i": incl(options, optIgnoreCase) of "ignorestyle", "y": incl(options, optIgnoreStyle) - of "ext": extensions = val.split('|') + of "ext": extensions.add val.split('|') of "nocolor": useWriteStyled = false of "verbose": incl(options, optVerbose) of "help", "h": writeHelp() diff --git a/tools/website.tmpl b/tools/website.tmpl index 7541fae4c..d2fcb0afd 100644 --- a/tools/website.tmpl +++ b/tools/website.tmpl @@ -49,26 +49,7 @@ # if currentTab == "index": <div id="slideshow"> <!-- slides --> - <div id="slide0" class="active zeoslide"> - <a href="news.html#Z2015-10-16-first-nim-conference"> - <img src="assets/zeo/banner.jpg" alt="First Nim workshop in Ukraine!"/> - </a> - </div> - <div id="slide1" class="codeslide1"> - <h2>Why should I be excited?</h2> - <span class="desc">Nim is the only language that leverages automated proof technology to perform a <i>disjoint check</i> for your parallel code. Working on disjoint data means no locking is required and yet data races are impossible:</span><br><br> -<pre> -<span class="kwd">parallel</span>: -<span class="tab"> </span><span class="kwd">var</span> i = <span class="val">0</span> -<span class="tab"> </span><span class="kwd">while</span> i <= a.high: -<span class="tab"> <span class="tab"> </span></span></span><span class="kwd">spawn</span> f(a[i]) -<span class="tab"> <span class="tab"> </span></span></span><span class="kwd">spawn</span> f(a[i+<span class="val">1</span>]) -<span class="tab"> <span class="tab"> </span></span></span><span class="cmt"># ERROR: cannot prove a[i] is disjoint from a[i+1]</span> -<span class="tab"> <span class="tab"> </span></span></span><span class="cmt"># BUT: replace 'i += 1' with 'i += 2' and the code compiles!</span> -<span class="tab end"> <span class="tab end"> </span></span>i += <span class="val">1</span> -</pre> - </div> - <div id="slide2" class="codeslide2"> + <div id="slide0" class="active codeslide2"> <div> <h2>Nim is simple..</h2> <pre> @@ -102,7 +83,7 @@ p.greet() <span class="cmt"># or greet(p)</span> </pre> </div> </div> - <div id="slide3" class="codeslide3"> + <div id="slide1" class="codeslide3"> <div> <h2>C FFI is easy in Nim..</h2> <pre> @@ -138,8 +119,6 @@ runForever() <div id="slideshow-nav"> <div id="slideControl0" onclick="slideshow_click(0)" class="active"></div> <div id="slideControl1" onclick="slideshow_click(1)"></div> - <div id="slideControl2" onclick="slideshow_click(2)"></div> - <div id="slideControl3" onclick="slideshow_click(3)"></div> </div> # end <aside id="sidebar"> @@ -181,7 +160,7 @@ runForever() <a href="documentation.html">Stable Documentation</a> <a href="learn.html">Learning Resources</a> <!-- <a href="">Development Documentation</a> --> - <a href="https://github.com/Araq/Nimrod">Issues & Requests</a> + <a href="https://github.com/nim-lang/nim">Issues & Requests</a> </div> <div> <h4>Community</h4> @@ -193,7 +172,7 @@ runForever() <div id="foot-legal"> <h4>Written in Nim - Powered by <a href="https://github.com/dom96/jester">Jester</a></h4> Web Design by <a href="http://reign-studios.net/philipwitte/">Philip Witte</a> & <a href="http://picheta.me/">Dominik Picheta</a><br> - Copyright © 2015 - <a href="http://nim-lang.org/blog/">Andreas Rumpf</a> & <a href="https://github.com/Araq/Nimrod/graphs/contributors">Contributors</a> + Copyright © 2015 - <a href="http://nim-lang.org/blog/">Andreas Rumpf</a> & <a href="https://github.com/nim-lang/nim/graphs/contributors">Contributors</a> </div> </div> </footer> diff --git a/web/documentation.txt b/web/documentation.txt index 65aba0660..ec33d0827 100644 --- a/web/documentation.txt +++ b/web/documentation.txt @@ -3,6 +3,16 @@ Nim's Documentation .. container:: standout + Search Options + -------------- + + .. container:: internals + + `Documentation Index <docs/theindex.html>`_ - The generated + index. **Index + (Ctrl+F) == Joy** + +.. container:: standout + Standards & Guides ------------------ @@ -58,10 +68,3 @@ Nim's Documentation - | `Internal Documentation <docs/intern.html>`_ | The internal documentation describes how the compiler is implemented. Read this if you want to hack the compiler. - - -Search Options --------------- - -`Documentation Index <docs/theindex.html>`_ - The generated -index. **Index + (Ctrl+F) == Joy** diff --git a/web/download.txt b/web/download.txt index 6ffb0775f..fdb9b768a 100644 --- a/web/download.txt +++ b/web/download.txt @@ -13,8 +13,8 @@ Binaries -------- Unfortunately for now we only provide builds for Windows. -* 32 bit: `nim-0.12.0_x32.exe <download/nim-0.12.0_x32.exe>`_ -* 64 bit: `nim-0.12.0_x64.exe <download/nim-0.12.0_x64.exe>`_ +* 32 bit: `nim-0.13.0_x32.exe <download/nim-0.13.0_x32.exe>`_ +* 64 bit: `nim-0.13.0_x64.exe <download/nim-0.13.0_x64.exe>`_ Installation based on generated C code @@ -26,8 +26,8 @@ like systems. Binary packages may be provided later. Download one of these: -* `nim-0.12.0.zip (28 MB) <download/nim-0.12.0.zip>`_ -* `nim-0.12.0.tar.xz (2.6MB) <download/nim-0.12.0.tar.xz>`_ +* `nim-0.13.0.zip (28 MB) <download/nim-0.13.0.zip>`_ +* `nim-0.13.0.tar.xz (2.6MB) <download/nim-0.13.0.tar.xz>`_ Extract the file and follow these instructions: diff --git a/web/news.txt b/web/news.txt index 5e6e7894e..aa772a415 100644 --- a/web/news.txt +++ b/web/news.txt @@ -2,6 +2,199 @@ News ==== + +2016-01-18 Version 0.13.0 released +================================== + +Once again we are proud to announce the latest release of the Nim compiler +and related tools. This release comes just 3 months after the last +release! + +A new version of Nimble which depends on this release, has also been +released. See `this <http://forum.nim-lang.org/t/1912>`_ forum thread for +more information about the Nimble release. + +This release of Nim includes over 116 bug fixes, many of which are related +to closures. The lambda lifting algorithm in the compiler has been completely +rewritten, and some changes have been made to the semantics of closures in +Nim as a result. These changes may affect backwards compatibility and are all +described in the section below. + +With this release, we are one step closer to Nim version 1.0. +The 1.0 release will be a big milestone for Nim, because after that version +is released there will be no more breaking changes made to the language +or the standard library. + +That being said, the next release will likely be Nim 0.14. It will focus on +improvements to the GC and concurrency. We will in particular be looking at +ways to add multi-core support to async await. Standard library improvements +are also on our roadmap but may not make it for Nim 0.14. + +As always you can download the latest version of Nim from the +`download <download.html>`_ page. + +Happy coding! + +Changes affecting backwards compatibility +----------------------------------------- + +- ``macros.newLit`` for ``bool`` now produces false/true symbols which + actually work with the bool datatype. +- When compiling to JS: ``Node``, ``NodeType`` and ``Document`` are no longer + defined. Use the types defined in ``dom.nim`` instead. +- The check ``x is iterator`` (used for instance in concepts) was always a + weird special case (you could not use ``x is proc``) and was removed from + the language. +- Top level routines cannot have the calling convention ``closure`` + anymore. +- The ``redis`` module has been moved out of the standard library. It can + now be installed via Nimble and is located here: + https://github.com/nim-lang/redis +- ``math.RunningStat`` and its associated procs have been moved from + the ``math`` module to a new ``stats`` module. + + +Syntax changes +~~~~~~~~~~~~~~ + +The parser now considers leading whitespace in front of operators +to determine if an operator is used in prefix or infix position. +This means that finally ``echo $foo`` is parsed as people expect, +which is as ``echo($foo)``. It used to be parsed as ``(echo) $ (foo)``. + +``echo $ foo`` continues to be parsed as ``(echo) $ (foo)``. + +This also means that ``-1`` is always parsed as prefix operator so +code like ``0..kArraySize div 2 -1`` needs to be changed to +``0..kArraySize div 2 - 1``. + +This release also adds multi-line comments to Nim. The syntax for them is: +``#[ comment here ]#``. For more details read the section of +the `manual <manual#multiline-comments>`_. + +Iterator changes +~~~~~~~~~~~~~~~~ + +Implicit return type inference for iterators has been removed from the language. The following used to work: + +.. code-block:: nim + iterator it = + yield 7 + +This was a strange special case and has been removed. Now you need to write it like so which is consistent with procs: + +.. code-block:: nim + iterator it: auto = + yield 7 + + +Closure changes +~~~~~~~~~~~~~~~ + +The semantics of closures changed: Capturing variables that are in loops do not produce a new environment. Nim closures behave like JavaScript closures now. + +The following used to work as the environment creation used to be attached to the loop body: + +.. code-block:: nim + + proc outer = + var s: seq[proc(): int {.closure.}] = @[] + for i in 0 ..< 30: + let ii = i + s.add(proc(): string = return ii*ii)) + +This behaviour has changed in 0.13.0 and now needs to be written as: + +.. code-block:: nim + + proc outer = + var s: seq[proc(): int {.closure.}] = @[] + for i in 0 ..< 30: + (proc () = + let ii = i + s.add(proc(): int = return ii*ii))() + +The reason is that environment creations are now only performed once +per proc call. This change is subtle and unfortunate, but: + +1. Affects almost no code out there. +2. Is easier to implement and we are at a point in Nim's development process where simple+stable wins over perfect-in-theory+unstable-in-practice. +3. Implies programmers are more in control of where memory is allocated which is benefitical for a systems programming language. + +Bugfixes +-------- + +The list below has been generated based on the commits in Nim's git +repository. As such it lists only the issues which have been closed +via a commit, for a full list see +`this link on Github <https://github.com/nim-lang/Nim/issues?utf8=%E2%9C%93&q=is%3Aissue+closed%3A%222015-10-27+..+2016-01-19%22+>`_. + +- Fixed "Generic arguments cannot be used in templates (raising undeclared identifier)" + (`#3498 <https://github.com/nim-lang/Nim/issues/3498>`_) +- Fixed "multimethods: Error: internal error: cgmeth.genConv" + (`#3550 <https://github.com/nim-lang/Nim/issues/3550>`_) +- Fixed "multimethods: Error: internal error: cgmeth.genConv" + (`#3550 <https://github.com/nim-lang/Nim/issues/3550>`_) +- Fixed "nimscript - SIGSEGV in except block" + (`#3546 <https://github.com/nim-lang/Nim/issues/3546>`_) +- Fixed "Bool literals in macros do not work." + (`#3541 <https://github.com/nim-lang/Nim/issues/3541>`_) +- Fixed "Docs: nativesocket.html - 404" + (`#3582 <https://github.com/nim-lang/Nim/issues/3582>`_) +- Fixed ""not nil" return types never trigger an error or warning" + (`#2285 <https://github.com/nim-lang/Nim/issues/2285>`_) +- Fixed "No warning or error is raised even if not nil is specified " + (`#3222 <https://github.com/nim-lang/Nim/issues/3222>`_) +- Fixed "Incorrect fsmonitor add() filter logic" + (`#3611 <https://github.com/nim-lang/Nim/issues/3611>`_) +- Fixed ""nimble install nimsuggest" failed" + (`#3622 <https://github.com/nim-lang/Nim/issues/3622>`_) +- Fixed "compile time `excl ` cause SIGSEGV" + (`#3639 <https://github.com/nim-lang/Nim/issues/3639>`_) +- Fixed "Unable to echo unsigned ints at compile-time" + (`#2514 <https://github.com/nim-lang/Nim/issues/2514>`_) +- Fixed "Nested closure iterator produces internal error" + (`#1725 <https://github.com/nim-lang/Nim/issues/1725>`_) +- Fixed "C Error on walkDirRec closure" + (`#3636 <https://github.com/nim-lang/Nim/issues/3636>`_) +- Fixed "Error in generated c code" + (`#3201 <https://github.com/nim-lang/Nim/issues/3201>`_) +- Fixed "C Compile-time error with generic proc type." + (`#2659 <https://github.com/nim-lang/Nim/issues/2659>`_) +- Fixed "ICE dereferencing array pointer" + (`#2240 <https://github.com/nim-lang/Nim/issues/2240>`_) +- Fixed "Lambda lifting crash" + (`#2007 <https://github.com/nim-lang/Nim/issues/2007>`_) +- Fixed "Can't reference outer variables from a closure in an iterator" + (`#2604 <https://github.com/nim-lang/Nim/issues/2604>`_) +- Fixed "M&S collector breaks with nested for loops." + (`#603 <https://github.com/nim-lang/Nim/issues/603>`_) +- Fixed "Regression: bad C codegen" + (`#3723 <https://github.com/nim-lang/Nim/issues/3723>`_) +- Fixed "JS backend - handle bool type in case statement" + (`#3722 <https://github.com/nim-lang/Nim/issues/3722>`_) +- Fixed "linenoise compilation with cpp" + (`#3720 <https://github.com/nim-lang/Nim/issues/3720>`_) +- Fixed "(???,???) duplicate case label" + (`#3665 <https://github.com/nim-lang/Nim/issues/3665>`_) +- Fixed "linenoise compilation with cpp" + (`#3720 <https://github.com/nim-lang/Nim/issues/3720>`_) +- Fixed "Update list of backward incompatibilities for Nim 0.12.0 in the main site" + (`#3689 <https://github.com/nim-lang/Nim/issues/3689>`_) +- Fixed "Can't compile nimble with latest devel - codegen bug" + (`#3730 <https://github.com/nim-lang/Nim/issues/3730>`_) + + +2016-01-18 Andreas Rumpf's talk at OSCON Amsterdam +================================================== + +In case you have missed it, here is Andreas' Nim: An Overview talk at +OSCON Amsterdam. + +.. raw:: html + + <iframe width="560" height="315" src="https://www.youtube.com/embed/4rJEBs_Nnaw" frameborder="0" allowfullscreen></iframe> + 2015-10-27 Version 0.12.0 released ================================== @@ -30,6 +223,11 @@ changes are also documented in this forum Changes affecting backwards compatibility ----------------------------------------- +- The regular expression modules, ``re`` and ``nre`` now depend on version + 8.36 of PCRE. If you have an older version you may see a message similar + to ``could not import: pcre_free_study`` output when you start your + program. See `this issue <https://github.com/docopt/docopt.nim/issues/13>`_ + for more information. - ``tables.[]``, ``strtabs.[]``, ``critbits.[]`` **now raise** the ``KeyError`` **exception when the key does not exist**! Use the new ``getOrDefault`` instead to get the old behaviour. Compile all your @@ -120,6 +318,34 @@ Changes affecting backwards compatibility is renamed to ``apply``. - The template ``mapIt`` now doesn't require the result's type parameter. Also the inplace ``mapIt`` is renamed to ``apply``. +- The compiler is now stricter with what is allowed as a case object + discriminator. The following code used to compile but was not supported + completely and so now fails: + +.. code-block:: nim + type + DataType* {.pure.} = enum + Char = 1, + Int8 = 2, + Int16 = 3, + Int32 = 4, + Int64 = 5, + Float32 = 6, + Float64 = 7 + + DataSeq* = object + case kind* : DataType + of DataType.Char: charSeq* : seq[char] + of DataType.Int8: int8Seq* : seq[int8] + of DataType.Int16: int16Seq* : seq[int16] + of DataType.Int32: int32Seq* : seq[int32] + of DataType.Int64: int64Seq* : seq[int64] + of DataType.Float32: float32Seq* : seq[float32] + of DataType.Float64: float64Seq* : seq[float64] + + length* : int + + Library Additions ----------------- diff --git a/web/question.txt b/web/question.txt index 2c3191b9b..4e7c15a10 100644 --- a/web/question.txt +++ b/web/question.txt @@ -23,27 +23,27 @@ General FAQ shared memory heap is also provided for the increased efficiency that results from that model. -.. - .. container:: standout - Why should I use Nim? - --------------------- +.. .. container:: standout - It's a conservative language in a sense that we stick to features that have - proven themselves for larger scale programming. But it's revolutionary by - the features which have been laid on top. +.. Why should I use Nim? +.. --------------------- - One of Nim's goals is to increase developer productivity without sacrificing - the produced software's stability. The way that this is done is by providing +.. It's a conservative language in a sense that we stick to features that have +.. proven themselves for larger scale programming. But it's revolutionary by +.. the features which have been laid on top. - Depending on your use case. +.. One of Nim's goals is to increase developer productivity without sacrificing +.. the produced software's stability. The way that this is done is by providing - Nim is one of the few programming languages in the world which allows you to +.. Depending on your use case. +.. Nim is one of the few programming languages in the world which allows you to - The language inventor describes it as the ultimate programming language - with features which make it perfect for just about any problem. + +.. The language inventor describes it as the ultimate programming language +.. with features which make it perfect for just about any problem. .. container:: standout diff --git a/web/support.txt b/web/support.txt index c0ffafcbc..9a526605e 100644 --- a/web/support.txt +++ b/web/support.txt @@ -32,8 +32,9 @@ Commercial support includes: All interested parties should email ``support@nim-lang.org``. The bid for contracting work is a commercial offer provided by: -| **Andreas Rumpf** -| St.-Quentin-Ring 47 -| 67663 Kaiserslautern +| **METATEXX GmbH** +| Spicher Str. 30 +| 53859 Niederkassel | GERMANY -| EU VAT-IN: DE297783450 +| EU VAT-IN: DE287088604 +| http://metatexx.de/index.php?index=12 diff --git a/web/ticker.txt b/web/ticker.txt index 106f8f008..f4a3cac2a 100644 --- a/web/ticker.txt +++ b/web/ticker.txt @@ -1,3 +1,13 @@ +<a class="news" href="news.html#Z2016-01-18-version-0-13-0-released"> + <h4>January 18, 2016</h4> + <p>Nim version 0.13.0 has been released!</p> +</a> + +<a class="news" href="news.html#Z2016-01-18-andreas-rumpf-s-talk-at-oscon-amsterdam"> + <h4>January 18, 2016</h4> + <p>Andreas Rumpf's talk at OSCON Amsterdam</p> +</a> + <a class="news" href="news.html#Z2015-10-27-version-0-12-0-released"> <h4>October 27, 2015</h4> <p>Nim version 0.12.0 has been released!</p> @@ -13,9 +23,4 @@ <p>Nim version 0.11.2 has been released!</p> </a> -<a class="news" href="news.html#Z2014-12-29-version-0-10-2-released"> - <h4>Dec 29, 2014</h4> - <p>Nim version 0.10.2 has been released!</p> -</a> - <a href="news.html" class="blue">See All News...</a> diff --git a/web/website.ini b/web/website.ini index 9d7aab664..46564d19f 100644 --- a/web/website.ini +++ b/web/website.ini @@ -39,6 +39,7 @@ srcdoc2: "impure/re;pure/typetraits" srcdoc2: "pure/concurrency/threadpool.nim;pure/concurrency/cpuinfo.nim" srcdoc: "system/threads.nim;system/channels.nim;js/dom" srcdoc2: "pure/os;pure/strutils;pure/math;pure/matchers;pure/algorithm" +srcdoc2: "pure/stats;impure/nre;windows/winlean" srcdoc2: "pure/complex;pure/times;pure/osproc;pure/pegs;pure/dynlib" srcdoc2: "pure/parseopt;pure/parseopt2;pure/hashes;pure/strtabs;pure/lexbase" srcdoc2: "pure/parsecfg;pure/parsexml;pure/parsecsv;pure/parsesql" @@ -56,14 +57,14 @@ srcdoc2: "pure/memfiles;pure/subexes;pure/collections/critbits" srcdoc2: "deprecated/pure/asyncio;deprecated/pure/actors;core/locks;pure/oids;pure/endians;pure/uri" srcdoc2: "pure/nimprof;pure/unittest;packages/docutils/highlite" srcdoc2: "packages/docutils/rst;packages/docutils/rstast" -srcdoc2: "packages/docutils/rstgen;pure/logging;pure/asyncdispatch;pure/asyncnet" -srcdoc2: "deprecated/pure/rawsockets;pure/asynchttpserver;pure/net;pure/selectors;pure/future" +srcdoc2: "packages/docutils/rstgen;pure/logging;pure/options;pure/asyncdispatch;pure/asyncnet" +srcdoc2: "pure/nativesockets;pure/asynchttpserver;pure/net;pure/selectors;pure/future" srcdoc2: "deprecated/pure/ftpclient" srcdoc2: "pure/asyncfile;pure/asyncftpclient" srcdoc2: "pure/md5;pure/rationals" srcdoc2: "posix/posix" -srcdoc2: "pure/fenv" -srcdoc2: "pure/basic2d;pure/basic3d" +srcdoc2: "pure/fenv;pure/securehash" +srcdoc2: "pure/basic2d;pure/basic3d;pure/mersenne;pure/coro" ; Note: everything under 'webdoc' doesn't get listed in the index, so wrappers ; should live here @@ -76,5 +77,3 @@ webdoc: "wrappers/libuv;wrappers/joyent_http_parser" webdoc: "posix/posix;wrappers/odbcsql" webdoc: "wrappers/libsvm.nim" - - |