diff options
91 files changed, 2061 insertions, 555 deletions
diff --git a/bootstrap.sh b/bootstrap.sh index 7f19c2440..7f19c2440 100755..100644 --- a/bootstrap.sh +++ b/bootstrap.sh diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 1a5334a98..d84a7d92e 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -672,9 +672,13 @@ proc genDeref(p: BProc, e: PNode, d: var TLoc; enforceDeref=false) = expr(p, e.sons[0], d) else: var a: TLoc - initLocExprSingleUse(p, e.sons[0], a) + let typ = skipTypes(e.sons[0].typ, abstractInst) + if typ.kind == tyVar and tfVarIsPtr notin typ.flags and p.module.compileToCpp and e.sons[0].kind == nkHiddenAddr: + initLocExprSingleUse(p, e[0][0], d) + return + else: + initLocExprSingleUse(p, e.sons[0], a) if d.k == locNone: - let typ = skipTypes(a.t, abstractInst) # dest = *a; <-- We do not know that 'dest' is on the heap! # It is completely wrong to set 'd.s' here, unless it's not yet # been assigned to. @@ -689,9 +693,9 @@ proc genDeref(p: BProc, e: PNode, d: var TLoc; enforceDeref=false) = return of tyPtr: d.s = OnUnknown # BUGFIX! - else: internalError(e.info, "genDeref " & $a.t.kind) + else: + internalError(e.info, "genDeref " & $typ.kind) elif p.module.compileToCpp: - let typ = skipTypes(a.t, abstractInst) if typ.kind == tyVar and tfVarIsPtr notin typ.flags and e.kind == nkHiddenDeref: putIntoDest(p, d, e.typ, rdLoc(a), a.s) @@ -1852,10 +1856,16 @@ proc genClosure(p: BProc, n: PNode, d: var TLoc) = initLocExpr(p, n.sons[1], b) if n.sons[0].skipConv.kind == nkClosure: internalError(n.info, "closure to closure created") - getTemp(p, n.typ, tmp) - linefmt(p, cpsStmts, "$1.ClPrc = $2; $1.ClEnv = $3;$n", - tmp.rdLoc, a.rdLoc, b.rdLoc) - putLocIntoDest(p, d, tmp) + # tasyncawait.nim breaks with this optimization: + when false: + if d.k != locNone: + linefmt(p, cpsStmts, "$1.ClPrc = $2; $1.ClEnv = $3;$n", + d.rdLoc, a.rdLoc, b.rdLoc) + else: + getTemp(p, n.typ, tmp) + linefmt(p, cpsStmts, "$1.ClPrc = $2; $1.ClEnv = $3;$n", + tmp.rdLoc, a.rdLoc, b.rdLoc) + putLocIntoDest(p, d, tmp) proc genArrayConstr(p: BProc, n: PNode, d: var TLoc) = var arr: TLoc diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index 39f16ff0d..ebc3225f2 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -576,7 +576,7 @@ proc getTypeDescAux(m: BModule, typ: PType, check: var IntSet): Rope = result = getTypeDescAux(m, et, check) & star idTablePut(m.typeCache, t, result) of tyOpenArray, tyVarargs: - result = getTypeDescAux(m, t.sons[0], check) & "*" + result = getTypeDescWeak(m, t.sons[0], check) & "*" idTablePut(m.typeCache, t, result) of tyProc: result = getTypeName(t) @@ -654,7 +654,7 @@ proc getTypeDescAux(m: BModule, typ: PType, check: var IntSet): Rope = else: result = cppName & "<" for i in 1 .. typ.len-2: - if i > 1: result.add(", ") + if i > 1: result.add(" COMMA ") result.add(getTypeDescAux(m, typ.sons[i], check)) result.add("> ") # always call for sideeffects: diff --git a/compiler/cgen.nim b/compiler/cgen.nim index db376821c..4f27a5a46 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -1059,9 +1059,8 @@ proc genModule(m: BModule, cfile: string): Rope = result = getFileHeader(cfile) result.add(genMergeInfo(m)) - generateHeaders(m) - generateThreadLocalStorage(m) + generateHeaders(m) for i in countup(cfsHeaders, cfsProcs): add(result, genSectionStart(i)) add(result, m.s[i]) diff --git a/compiler/commands.nim b/compiler/commands.nim index 2622d64f4..86bc1c205 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -205,6 +205,7 @@ proc testCompileOptionArg*(switch, arg: string, info: TLineInfo): bool = of "generational": result = gSelectedGC == gcGenerational of "go": result = gSelectedGC == gcGo of "none": result = gSelectedGC == gcNone + of "stack": result = gSelectedGC == gcStack else: localError(info, errNoneBoehmRefcExpectedButXFound, arg) of "opt": case arg.normalize @@ -394,6 +395,9 @@ proc processSwitch(switch, arg: string, pass: TCmdLinePass, info: TLineInfo) = of "none": gSelectedGC = gcNone defineSymbol("nogc") + of "stack": + gSelectedGC= gcStack + defineSymbol("gcstack") else: localError(info, errNoneBoehmRefcExpectedButXFound, arg) of "warnings", "w": if processOnOffSwitchOrList({optWarns}, arg, pass, info): listWarnings() diff --git a/compiler/lambdalifting.nim b/compiler/lambdalifting.nim index 53fca4863..959632bab 100644 --- a/compiler/lambdalifting.nim +++ b/compiler/lambdalifting.nim @@ -721,6 +721,10 @@ proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass; let m = newSymNode(n[namePos].sym) m.typ = n.typ result = liftCapturedVars(m, owner, d, c) + of nkHiddenStdConv: + if n.len == 2: + n.sons[1] = liftCapturedVars(n[1], owner, d, c) + if n[1].kind == nkClosure: result = n[1] else: if owner.isIterator: if n.kind == nkYieldStmt: diff --git a/compiler/options.nim b/compiler/options.nim index 29cdd96fb..2716a98d3 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -86,7 +86,8 @@ type # please make sure we have under 32 options cmdRun # run the project via TCC backend TStringSeq* = seq[string] TGCMode* = enum # the selected GC - gcNone, gcBoehm, gcGo, gcMarkAndSweep, gcRefc, gcV2, gcGenerational + gcNone, gcBoehm, gcGo, gcStack, gcMarkAndSweep, gcRefc, + gcV2, gcGenerational IdeCmd* = enum ideNone, ideSug, ideCon, ideDef, ideUse, ideDus, ideChk, ideMod, diff --git a/compiler/parser.nim b/compiler/parser.nim index f22177ac1..1ba59b938 100644 --- a/compiler/parser.nim +++ b/compiler/parser.nim @@ -326,6 +326,7 @@ proc parseSymbol(p: var TParser, allowNil = false): PNode = getTok(p) else: parMessage(p, errIdentifierExpected, p.tok) + break eat(p, tkAccent) else: if allowNil and p.tok.tokType == tkNil: diff --git a/compiler/patterns.nim b/compiler/patterns.nim index 604d3521d..2336e44e7 100644 --- a/compiler/patterns.nim +++ b/compiler/patterns.nim @@ -129,7 +129,7 @@ proc matchNested(c: PPatternContext, p, n: PNode, rpn: bool): bool = result = bindOrCheck(c, p.sons[2].sym, arglist) proc matches(c: PPatternContext, p, n: PNode): bool = - # hidden conversions (?) + let n = skipHidden(n) if nfNoRewrite in n.flags: result = false elif isPatternParam(c, p): diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim index 2280ef712..38d17eb62 100644 --- a/compiler/pragmas.nim +++ b/compiler/pragmas.nim @@ -256,8 +256,9 @@ proc expectDynlibNode(c: PContext, n: PNode): PNode = proc processDynLib(c: PContext, n: PNode, sym: PSym) = if (sym == nil) or (sym.kind == skModule): - POptionEntry(c.optionStack.tail).dynlib = getLib(c, libDynamic, - expectDynlibNode(c, n)) + let lib = getLib(c, libDynamic, expectDynlibNode(c, n)) + if not lib.isOverriden: + POptionEntry(c.optionStack.tail).dynlib = lib else: if n.kind == nkExprColonExpr: var lib = getLib(c, libDynamic, expectDynlibNode(c, n)) diff --git a/compiler/renderer.nim b/compiler/renderer.nim index 03f6d4832..f0ee137e9 100644 --- a/compiler/renderer.nim +++ b/compiler/renderer.nim @@ -802,7 +802,7 @@ proc doParamsAux(g: var TSrcGen, params: PNode) = gsemicolon(g, params, 1) put(g, tkParRi, ")") - if params.sons[0].kind != nkEmpty: + if params.len > 0 and params.sons[0].kind != nkEmpty: putWithSpace(g, tkOpr, "->") gsub(g, params.sons[0]) diff --git a/compiler/sem.nim b/compiler/sem.nim index e09d49f88..97a20a4da 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -67,9 +67,12 @@ template semIdeForTemplateOrGeneric(c: PContext; n: PNode; proc typeMismatch(n: PNode, formal, actual: PType) = if formal.kind != tyError and actual.kind != tyError: + let named = typeToString(formal) + let desc = typeToString(formal, preferDesc) + let x = if named == desc: named else: named & " = " & desc localError(n.info, errGenerated, msgKindToString(errTypeMismatch) & typeToString(actual) & ") " & - `%`(msgKindToString(errButExpectedX), [typeToString(formal)])) + `%`(msgKindToString(errButExpectedX), [x])) proc fitNode(c: PContext, formal: PType, arg: PNode): PNode = if arg.typ.isNil: diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 16b4ee479..3c4c453a9 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -74,8 +74,12 @@ proc semSymGenericInstantiation(c: PContext, n: PNode, s: PSym): PNode = proc inlineConst(n: PNode, s: PSym): PNode {.inline.} = result = copyTree(s.ast) - result.typ = s.typ - result.info = n.info + if result.isNil: + localError(n.info, "constant of type '" & typeToString(s.typ) & "' has no value") + result = newSymNode(s) + else: + result.typ = s.typ + result.info = n.info type TConvStatus = enum diff --git a/compiler/seminst.nim b/compiler/seminst.nim index 4a45dee9d..14631a590 100644 --- a/compiler/seminst.nim +++ b/compiler/seminst.nim @@ -111,9 +111,9 @@ proc removeDefaultParamValues(n: PNode) = # not possible... XXX We don't solve this issue here. a.sons[L-1] = ast.emptyNode -proc freshGenSyms(n: PNode, owner: PSym, symMap: var TIdTable) = +proc freshGenSyms(n: PNode, owner, orig: PSym, symMap: var TIdTable) = # we need to create a fresh set of gensym'ed symbols: - if n.kind == nkSym and sfGenSym in n.sym.flags: + if n.kind == nkSym and sfGenSym in n.sym.flags and n.sym.owner == orig: let s = n.sym var x = PSym(idTableGet(symMap, s)) if x == nil: @@ -122,7 +122,7 @@ proc freshGenSyms(n: PNode, owner: PSym, symMap: var TIdTable) = idTablePut(symMap, s, x) n.sym = x else: - for i in 0 .. <safeLen(n): freshGenSyms(n.sons[i], owner, symMap) + for i in 0 .. <safeLen(n): freshGenSyms(n.sons[i], owner, orig, symMap) proc addParamOrResult(c: PContext, param: PSym, kind: TSymKind) @@ -137,7 +137,7 @@ proc addProcDecls(c: PContext, fn: PSym) = maybeAddResult(c, fn, fn.ast) -proc instantiateBody(c: PContext, n, params: PNode, result: PSym) = +proc instantiateBody(c: PContext, n, params: PNode, result, orig: PSym) = if n.sons[bodyPos].kind != nkEmpty: inc c.inGenericInst # add it here, so that recursive generic procs are possible: @@ -149,7 +149,7 @@ proc instantiateBody(c: PContext, n, params: PNode, result: PSym) = let param = params[i].sym if sfGenSym in param.flags: idTablePut(symMap, params[i].sym, result.typ.n[param.position+1].sym) - freshGenSyms(b, result, symMap) + freshGenSyms(b, result, orig, symMap) b = semProcBody(c, b) b = hloBody(c, b) n.sons[bodyPos] = transformBody(c.module, b, result) @@ -165,7 +165,7 @@ proc fixupInstantiatedSymbols(c: PContext, s: PSym) = openScope(c) var n = oldPrc.ast n.sons[bodyPos] = copyTree(s.getBody) - instantiateBody(c, n, nil, oldPrc) + instantiateBody(c, n, nil, oldPrc, s) closeScope(c) popInfoContext() @@ -312,7 +312,7 @@ proc generateInstance(c: PContext, fn: PSym, pt: TIdTable, pragma(c, result, n.sons[pragmasPos], allRoutinePragmas) if isNil(n.sons[bodyPos]): n.sons[bodyPos] = copyTree(fn.getBody) - instantiateBody(c, n, fn.typ.n, result) + instantiateBody(c, n, fn.typ.n, result, fn) sideEffectsCheck(c, result) paramsTypeCheck(c, result.typ) else: diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 5d16f2fba..910267e57 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -620,7 +620,8 @@ proc semFor(c: PContext, n: PNode): PNode = result.kind = nkParForStmt else: result = semForFields(c, n, call.sons[0].sym.magic) - elif isCallExpr and call.sons[0].typ.callConv == ccClosure: + elif isCallExpr and call.sons[0].typ.callConv == ccClosure and + tfIterator in call.sons[0].typ.flags: # first class iterator: result = semForVars(c, n) elif not isCallExpr or call.sons[0].kind != nkSym or diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index 62d02fe10..ba17cc307 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -831,9 +831,11 @@ proc liftParamType(c: PContext, procKind: TSymKind, genericParams: PNode, result.rawAddSon paramType.lastSon return addImplicitGeneric(result) - result = instGenericContainer(c, paramType.sym.info, result, + let x = instGenericContainer(c, paramType.sym.info, result, allowMetaTypes = true) - result = newTypeWithSons(c, tyCompositeTypeClass, @[paramType, result]) + result = newTypeWithSons(c, tyCompositeTypeClass, @[paramType, x]) + #result = newTypeS(tyCompositeTypeClass, c) + #for i in 0..<x.len: result.rawAddSon(x.sons[i]) result = addImplicitGeneric(result) of tyGenericInst: diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index 8859c30e4..377743de3 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -513,7 +513,7 @@ proc typeRangeRel(f, a: PType): TTypeRelation {.noinline.} = proc matchUserTypeClass*(c: PContext, m: var TCandidate, ff, a: PType): TTypeRelation = var body = ff.skipTypes({tyUserTypeClassInst}) - if c.inTypeClass > 20: + if c.inTypeClass > 4: localError(body.n[3].info, $body.n[3] & " too nested for type matching") return isNone @@ -847,7 +847,10 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation = inc(c.inheritancePenalty, depth) result = isSubtype of tyDistinct: - if a.kind == tyDistinct and sameDistinctTypes(f, a): result = isEqual + if a.kind == tyDistinct: + if sameDistinctTypes(f, a): result = isEqual + elif f.base.kind == tyAnything: result = isGeneric + elif c.coerceDistincts: result = typeRel(c, f.base, a) elif c.coerceDistincts: result = typeRel(c, f.base, a) of tySet: if a.kind == tySet: @@ -922,19 +925,7 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation = if a.kind == tyEmpty: result = isEqual of tyGenericInst: - let roota = a.skipGenericAlias - let rootf = f.skipGenericAlias - if a.kind == tyGenericInst and roota.base == rootf.base: - for i in 1 .. rootf.sonsLen-2: - let ff = rootf.sons[i] - let aa = roota.sons[i] - result = typeRel(c, ff, aa) - if result == isNone: return - if ff.kind == tyRange and result != isEqual: return isNone - #result = isGeneric - # XXX See bug #2220. A[int] should match A[int] better than some generic X - else: - result = typeRel(c, lastSon(f), a) + result = typeRel(c, lastSon(f), a) of tyGenericBody: considerPreviousT: @@ -1035,12 +1026,20 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation = of tyCompositeTypeClass: considerPreviousT: - if typeRel(c, f.sons[1], a) != isNone: - put(c.bindings, f, a) - return isGeneric + let roota = a.skipGenericAlias + let rootf = f.lastSon.skipGenericAlias + if a.kind == tyGenericInst and roota.base == rootf.base: + for i in 1 .. rootf.sonsLen-2: + let ff = rootf.sons[i] + let aa = roota.sons[i] + result = typeRel(c, ff, aa) + if result == isNone: return + if ff.kind == tyRange and result != isEqual: return isNone else: - return isNone - + result = typeRel(c, rootf.lastSon, a) + if result != isNone: + put(c.bindings, f, a) + result = isGeneric of tyGenericParam: var x = PType(idTableGet(c.bindings, f)) if x == nil: diff --git a/compiler/transf.nim b/compiler/transf.nim index a4a15ea4a..0647553c6 100644 --- a/compiler/transf.nim +++ b/compiler/transf.nim @@ -414,8 +414,8 @@ proc transformConv(c: PTransf, n: PNode): PTransNode = result = newTransNode(nkChckRange, n, 3) dest = skipTypes(n.typ, abstractVar) result[0] = transform(c, n.sons[1]) - result[1] = newIntTypeNode(nkIntLit, firstOrd(dest), source).PTransNode - result[2] = newIntTypeNode(nkIntLit, lastOrd(dest), source).PTransNode + result[1] = newIntTypeNode(nkIntLit, firstOrd(dest), dest).PTransNode + result[2] = newIntTypeNode(nkIntLit, lastOrd(dest), dest).PTransNode of tyFloat..tyFloat128: # XXX int64 -> float conversion? if skipTypes(n.typ, abstractVar).kind == tyRange: diff --git a/compiler/types.nim b/compiler/types.nim index 9aa991086..c9cbfedb1 100644 --- a/compiler/types.nim +++ b/compiler/types.nim @@ -1448,6 +1448,18 @@ proc skipConv*(n: PNode): PNode = result = n.sons[1] else: discard +proc skipHidden*(n: PNode): PNode = + result = n + while true: + case result.kind + of nkHiddenStdConv, nkHiddenSubConv: + if result.sons[1].typ.classify == result.typ.classify: + result = result.sons[1] + else: break + of nkHiddenDeref, nkHiddenAddr: + result = result.sons[0] + else: break + proc skipConvTakeType*(n: PNode): PNode = result = n.skipConv result.typ = n.typ diff --git a/compiler/vm.nim b/compiler/vm.nim index 7220e1b8e..f799334d6 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -10,7 +10,9 @@ ## This file implements the new evaluation engine for Nim code. ## An instruction is 1-3 int32s in memory, it is a register based VM. -const debugEchoCode = false +const + debugEchoCode = false + traceCode = debugEchoCode import ast except getstr @@ -121,7 +123,7 @@ template move(a, b: expr) {.immediate, dirty.} = system.shallowCopy(a, b) # XXX fix minor 'shallowCopy' overloading bug in compiler proc createStrKeepNode(x: var TFullReg; keepNode=true) = - if x.node.isNil: + if x.node.isNil or not keepNode: x.node = newNode(nkStrLit) elif x.node.kind == nkNilLit and keepNode: when defined(useNodeIds): @@ -404,7 +406,8 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = let instr = c.code[pc] let ra = instr.regA #if c.traceActive: - #echo "PC ", pc, " ", c.code[pc].opcode, " ra ", ra, " rb ", instr.regB, " rc ", instr.regC + when traceCode: + echo "PC ", pc, " ", c.code[pc].opcode, " ra ", ra, " rb ", instr.regB, " rc ", instr.regC # message(c.debug[pc], warnUser, "Trace") case instr.opcode @@ -542,7 +545,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = if regs[rb].node.kind == nkRefTy: regs[ra].node = regs[rb].node.sons[0] else: - stackTrace(c, tos, pc, errGenerated, "limited VM support for 'ref'") + stackTrace(c, tos, pc, errGenerated, "limited VM support for pointers") else: stackTrace(c, tos, pc, errNilAccess) of opcWrDeref: diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index 47f16a013..019c79eb3 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -102,6 +102,10 @@ proc gABC(ctx: PCtx; n: PNode; opc: TOpcode; a, b, c: TRegister = 0) = let ins = (opc.uint32 or (a.uint32 shl 8'u32) or (b.uint32 shl 16'u32) or (c.uint32 shl 24'u32)).TInstr + when false: + if ctx.code.len == 43: + writeStackTrace() + echo "generating ", opc ctx.code.add(ins) ctx.debug.add(n.info) @@ -122,6 +126,11 @@ proc gABI(c: PCtx; n: PNode; opc: TOpcode; a, b: TRegister; imm: BiggestInt) = proc gABx(c: PCtx; n: PNode; opc: TOpcode; a: TRegister = 0; bx: int) = # Applies `opc` to `bx` and stores it into register `a` # `bx` must be signed and in the range [-32768, 32767] + when false: + if c.code.len == 43: + writeStackTrace() + echo "generating ", opc + if bx >= -32768 and bx <= 32767: let ins = (opc.uint32 or a.uint32 shl 8'u32 or (bx+wordExcess).uint32 shl 16'u32).TInstr @@ -704,6 +713,10 @@ proc genAddSubInt(c: PCtx; n: PNode; dest: var TDest; opc: TOpcode) = c.genNarrow(n, dest) proc genConv(c: PCtx; n, arg: PNode; dest: var TDest; opc=opcConv) = + if n.typ.kind == arg.typ.kind and arg.typ.kind == tyProc: + # don't do anything for lambda lifting conversions: + gen(c, arg, dest) + return let tmp = c.genx(arg) if dest < 0: dest = c.getTemp(n.typ) c.gABC(n, opc, dest, tmp) @@ -1117,7 +1130,10 @@ proc genAddrDeref(c: PCtx; n: PNode; dest: var TDest; opc: TOpcode; if isAddr and (let m = canElimAddr(n); m != nil): gen(c, m, dest, flags) return - let newflags = if isAddr: flags+{gfAddrOf} else: flags + + let af = if n[0].kind in {nkBracketExpr, nkDotExpr, nkCheckedFieldExpr}: {gfAddrOf, gfFieldAccess} + else: {gfAddrOf} + let newflags = if isAddr: flags+af else: flags # consider: # proc foo(f: var ref int) = # f = new(int) @@ -1132,7 +1148,7 @@ proc genAddrDeref(c: PCtx; n: PNode; dest: var TDest; opc: TOpcode; if gfAddrOf notin flags and fitsRegister(n.typ): c.gABC(n, opcNodeToReg, dest, dest) elif isAddr and isGlobal(n.sons[0]): - gen(c, n.sons[0], dest, flags+{gfAddrOf}) + gen(c, n.sons[0], dest, flags+af) else: let tmp = c.genx(n.sons[0], newflags) if dest < 0: dest = c.getTemp(n.typ) @@ -1447,12 +1463,12 @@ proc getNullValue(typ: PType, info: TLineInfo): PNode = of tyObject: result = newNodeIT(nkObjConstr, info, t) result.add(newNodeIT(nkEmpty, info, t)) - getNullValueAux(t.n, result) # initialize inherited fields: var base = t.sons[0] while base != nil: getNullValueAux(skipTypes(base, skipPtrs).n, result) base = base.sons[0] + getNullValueAux(t.n, result) of tyArray, tyArrayConstr: result = newNodeIT(nkBracket, info, t) for i in countup(0, int(lengthOrd(t)) - 1): @@ -1736,9 +1752,9 @@ proc gen(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}) = of declarativeDefs: unused(n, dest) of nkLambdaKinds: - let s = n.sons[namePos].sym - discard genProc(c, s) - genLit(c, n.sons[namePos], dest) + #let s = n.sons[namePos].sym + #discard genProc(c, s) + genLit(c, newSymNode(n.sons[namePos].sym), dest) of nkChckRangeF, nkChckRange64, nkChckRange: let tmp0 = c.genx(n.sons[0]) diff --git a/doc/backends.txt b/doc/backends.txt index c7939baec..343af09ed 100644 --- a/doc/backends.txt +++ b/doc/backends.txt @@ -420,7 +420,7 @@ function directly will be able to use it since Nim's garbage collector has not had a chance to run *yet*. This gives you enough time to make a copy for the C side of the program, as calling any further Nim procs *might* trigger garbage collection making the previously returned string garbage. Or maybe you -are `triggering yourself the collection <gc.html>`_. +are `yourself triggering the collection <gc.html>`_. Custom data types diff --git a/doc/manual/stmts.txt b/doc/manual/stmts.txt index 4f8d6e935..65c810cf7 100644 --- a/doc/manual/stmts.txt +++ b/doc/manual/stmts.txt @@ -575,7 +575,7 @@ name ``c`` should default to type ``Context``, ``n`` should default to The ``using`` section uses the same indentation based grouping syntax as -a ``var`` or ``let``` section. +a ``var`` or ``let`` section. If expression diff --git a/doc/manual/typedesc.txt b/doc/manual/typedesc.txt index de1d84d7d..6922d77e4 100644 --- a/doc/manual/typedesc.txt +++ b/doc/manual/typedesc.txt @@ -77,38 +77,6 @@ Once bound, typedesc params can appear in the rest of the proc signature: declareVariableWithType int, 42 -When used with macros and .compileTime. procs on the other hand, the compiler -does not need to instantiate the code multiple times, because types then can be -manipulated using the unified internal symbol representation. In such context -typedesc acts as any other type. One can create variables, store typedesc -values inside containers and so on. For example, here is how one can create -a type-safe wrapper for the unsafe `printf` function from C: - -.. code-block:: nim - macro safePrintF(formatString: string{lit}, args: varargs[expr]): expr = - var i = 0 - for c in formatChars(formatString): - var expectedType = case c - of 'c': char - of 'd', 'i', 'x', 'X': int - of 'f', 'e', 'E', 'g', 'G': float - of 's': string - of 'p': pointer - else: EOutOfRange - - var actualType = args[i].getType - inc i - - if expectedType == EOutOfRange: - error c & " is not a valid format character" - elif expectedType != actualType: - error "type mismatch for argument ", i, ". expected type: ", - expectedType.name, ", actual type: ", actualType.name - - # keep the original callsite, but use cprintf instead - result = callsite() - result[0] = newIdentNode(!"cprintf") - Overload resolution can be further influenced by constraining the set of types that will match the typedesc param: diff --git a/lib/core/locks.nim b/lib/core/locks.nim index 66e0ab520..fbe9c8acf 100644 --- a/lib/core/locks.nim +++ b/lib/core/locks.nim @@ -9,6 +9,7 @@ ## This module contains Nim's support for locks and condition vars. +const insideRLocksModule = false include "system/syslocks" type @@ -63,4 +64,4 @@ template withLock*(a: Lock, body: untyped) = try: body finally: - a.release() \ No newline at end of file + a.release() diff --git a/lib/core/macros.nim b/lib/core/macros.nim index 4522e0fc6..678982a52 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -796,17 +796,17 @@ proc infix*(a: NimNode; op: string; proc unpackPostfix*(node: NimNode): tuple[node: NimNode; op: string] {. compileTime.} = node.expectKind nnkPostfix - result = (node[0], $node[1]) + result = (node[1], $node[0]) proc unpackPrefix*(node: NimNode): tuple[node: NimNode; op: string] {. compileTime.} = node.expectKind nnkPrefix - result = (node[0], $node[1]) + result = (node[1], $node[0]) proc unpackInfix*(node: NimNode): tuple[left: NimNode; op: string; right: NimNode] {.compileTime.} = assert node.kind == nnkInfix - result = (node[0], $node[1], node[2]) + result = (node[1], $node[0], node[2]) proc copy*(node: NimNode): NimNode {.compileTime.} = ## An alias for copyNimTree(). diff --git a/lib/core/rlocks.nim b/lib/core/rlocks.nim index 14f04592b..4710d6cf1 100644 --- a/lib/core/rlocks.nim +++ b/lib/core/rlocks.nim @@ -9,6 +9,7 @@ ## This module contains Nim's support for reentrant locks. +const insideRLocksModule = true include "system/syslocks" type diff --git a/lib/deprecated/pure/sockets.nim b/lib/deprecated/pure/sockets.nim index f7d0950d8..20e6d9364 100644 --- a/lib/deprecated/pure/sockets.nim +++ b/lib/deprecated/pure/sockets.nim @@ -206,6 +206,18 @@ proc htons*(x: int16): int16 = ## order, this is a no-op; otherwise, it performs a 2-byte swap operation. result = sockets.ntohs(x) +template ntohl(x: uint32): expr = + cast[uint32](sockets.ntohl(cast[int32](x))) + +template ntohs(x: uint16): expr = + cast[uint16](sockets.ntohs(cast[int16](x))) + +template htonl(x: uint32): expr = + sockets.ntohl(x) + +template htons(x: uint16): expr = + sockets.ntohs(x) + when defined(Posix): proc toInt(domain: Domain): cint = case domain @@ -451,7 +463,7 @@ proc bindAddr*(socket: Socket, port = Port(0), address = "") {. name.sin_family = int16(ord(AF_INET)) else: name.sin_family = posix.AF_INET - name.sin_port = sockets.htons(int16(port)) + name.sin_port = sockets.htons(uint16(port)) name.sin_addr.s_addr = sockets.htonl(INADDR_ANY) if bindSocket(socket.fd, cast[ptr SockAddr](addr(name)), sizeof(name).SockLen) < 0'i32: @@ -834,7 +846,7 @@ proc connect*(socket: Socket, address: string, port = Port(0), when false: var s: TSockAddrIn s.sin_addr.s_addr = inet_addr(address) - s.sin_port = sockets.htons(int16(port)) + s.sin_port = sockets.htons(uint16(port)) when defined(windows): s.sin_family = toU16(ord(af)) else: diff --git a/lib/impure/rdstdin.nim b/lib/impure/rdstdin.nim index 469bb69c5..f722a6b39 100644 --- a/lib/impure/rdstdin.nim +++ b/lib/impure/rdstdin.nim @@ -105,7 +105,8 @@ else: proc readLineFromStdin*(prompt: string): TaintedString {. tags: [ReadIOEffect, WriteIOEffect].} = var buffer = linenoise.readLine(prompt) - if isNil(buffer): quit(0) + if isNil(buffer): + raise newException(IOError, "Linenoise returned nil") result = TaintedString($buffer) if result.string.len > 0: historyAdd(buffer) @@ -114,12 +115,12 @@ else: proc readLineFromStdin*(prompt: string, line: var TaintedString): bool {. tags: [ReadIOEffect, WriteIOEffect].} = var buffer = linenoise.readLine(prompt) - if isNil(buffer): quit(0) + if isNil(buffer): + raise newException(IOError, "Linenoise returned nil") line = TaintedString($buffer) if line.string.len > 0: historyAdd(buffer) linenoise.free(buffer) - # XXX how to determine CTRL+D? result = true proc readPasswordFromStdin*(prompt: string, password: var TaintedString): diff --git a/lib/impure/ssl.nim b/lib/impure/ssl.nim index 721e5ce51..e3312d792 100644 --- a/lib/impure/ssl.nim +++ b/lib/impure/ssl.nim @@ -9,6 +9,9 @@ ## This module provides an easy to use sockets-style ## nim interface to the OpenSSL library. +## +## **Warning:** This module is deprecated, use the SSL procedures defined in +## the ``net`` module instead. {.deprecated.} diff --git a/lib/nimbase.h b/lib/nimbase.h index 5a4f403b6..f531f3c49 100644 --- a/lib/nimbase.h +++ b/lib/nimbase.h @@ -222,6 +222,8 @@ __clang__ /* ----------------------------------------------------------------------- */ +#define COMMA , + #include <limits.h> #include <stddef.h> diff --git a/lib/posix/posix.nim b/lib/posix/posix.nim index 40b48f992..19b068b60 100644 --- a/lib/posix/posix.nim +++ b/lib/posix/posix.nim @@ -35,6 +35,15 @@ const hasSpawnH = not defined(haiku) # should exist for every Posix system nowadays hasAioH = defined(linux) +when defined(linux): + # On Linux: + # timer_{create,delete,settime,gettime}, + # clock_{getcpuclockid, getres, gettime, nanosleep, settime} lives in librt + {.passL: "-lrt".} +when defined(solaris): + # On Solaris hstrerror lives in libresolv + {.passL: "-lresolv".} + when false: const C_IRUSR = 0c000400 ## Read by owner. @@ -486,11 +495,11 @@ type l_onoff*: cint ## Indicates whether linger option is enabled. l_linger*: cint ## Linger time, in seconds. - InPort* = int16 ## unsigned! - InAddrScalar* = int32 ## unsigned! + InPort* = uint16 + InAddrScalar* = uint32 InAddrT* {.importc: "in_addr_t", pure, final, - header: "<netinet/in.h>".} = int32 ## unsigned! + header: "<netinet/in.h>".} = uint32 InAddr* {.importc: "struct in_addr", pure, final, header: "<netinet/in.h>".} = object ## struct in_addr @@ -2309,9 +2318,9 @@ proc strftime*(a1: cstring, a2: int, a3: cstring, a4: var Tm): int {.importc, header: "<time.h>".} proc strptime*(a1, a2: cstring, a3: var Tm): cstring {.importc, header: "<time.h>".} proc time*(a1: var Time): Time {.importc, header: "<time.h>".} -proc timer_create*(a1: var ClockId, a2: var SigEvent, +proc timer_create*(a1: ClockId, a2: var SigEvent, a3: var Timer): cint {.importc, header: "<time.h>".} -proc timer_delete*(a1: var Timer): cint {.importc, header: "<time.h>".} +proc timer_delete*(a1: Timer): cint {.importc, header: "<time.h>".} proc timer_gettime*(a1: Timer, a2: var Itimerspec): cint {. importc, header: "<time.h>".} proc timer_getoverrun*(a1: Timer): cint {.importc, header: "<time.h>".} @@ -2618,3 +2627,17 @@ proc utimes*(path: cstring, times: ptr array [2, Timeval]): int {. ## Returns zero on success. ## ## For more information read http://www.unix.com/man-page/posix/3/utimes/. + +proc handle_signal(sig: cint, handler: proc (a: cint) {.noconv.}) {.importc: "signal", header: "<signal.h>".} + +template onSignal*(signals: varargs[cint], body: untyped): stmt = + ## Setup code to be executed when Unix signals are received. Example: + ## from posix import SIGINT, SIGTERM + ## onSignal(SIGINT, SIGTERM): + ## echo "bye" + + for s in signals: + handle_signal(s, + proc (sig: cint) {.noconv.} = + body + ) diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index cc337452f..01b53cb12 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -1316,6 +1316,23 @@ proc generateExceptionCheck(futSym, ) result.add elseNode +template useVar(result: var NimNode, futureVarNode: NimNode, valueReceiver, + rootReceiver: expr, fromNode: NimNode) = + ## Params: + ## futureVarNode: The NimNode which is a symbol identifying the Future[T] + ## variable to yield. + ## fromNode: Used for better debug information (to give context). + ## valueReceiver: The node which defines an expression that retrieves the + ## future's value. + ## + ## rootReceiver: ??? TODO + # -> yield future<x> + result.add newNimNode(nnkYieldStmt, fromNode).add(futureVarNode) + # -> future<x>.read + valueReceiver = newDotExpr(futureVarNode, newIdentNode("read")) + result.add generateExceptionCheck(futureVarNode, tryStmt, rootReceiver, + fromNode) + template createVar(result: var NimNode, futSymName: string, asyncProc: NimNode, valueReceiver, rootReceiver: expr, @@ -1323,9 +1340,7 @@ template createVar(result: var NimNode, futSymName: string, result = newNimNode(nnkStmtList, fromNode) var futSym = genSym(nskVar, "future") result.add newVarStmt(futSym, asyncProc) # -> var future<x> = y - result.add newNimNode(nnkYieldStmt, fromNode).add(futSym) # -> yield future<x> - valueReceiver = newDotExpr(futSym, newIdentNode("read")) # -> future<x>.read - result.add generateExceptionCheck(futSym, tryStmt, rootReceiver, fromNode) + useVar(result, futSym, valueReceiver, rootReceiver, fromNode) proc processBody(node, retFutureSym: NimNode, subTypeIsVoid: bool, @@ -1352,7 +1367,11 @@ proc processBody(node, retFutureSym: NimNode, case node[1].kind of nnkIdent, nnkInfix: # await x - result = newNimNode(nnkYieldStmt, node).add(node[1]) # -> yield x + result = newNimNode(nnkStmtList, node) + var futureValue: NimNode + result.useVar(node[1], futureValue, futureValue, node) + # -> yield x + # -> x.read() of nnkCall, nnkCommand: # await foo(p, x) var futureValue: NimNode @@ -1550,6 +1569,7 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = for i in 0 .. <result[4].len: if result[4][i].kind == nnkIdent and result[4][i].ident == !"async": result[4].del(i) + result[4] = newEmptyNode() if subtypeIsVoid: # Add discardable pragma. if returnType.kind == nnkEmpty: @@ -1559,7 +1579,7 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = result[6] = outerProcBody #echo(treeRepr(result)) - #if prc[0].getName == "hubConnectionLoop": + #if prc[0].getName == "g": # echo(toStrLit(result)) macro async*(prc: stmt): stmt {.immediate.} = diff --git a/lib/pure/asyncfile.nim b/lib/pure/asyncfile.nim index c7b9fac18..c91d833fc 100644 --- a/lib/pure/asyncfile.nim +++ b/lib/pure/asyncfile.nim @@ -162,7 +162,7 @@ proc read*(f: AsyncFile, size: int): Future[string] = # Request completed immediately. var bytesRead: DWord let overlappedRes = getOverlappedResult(f.fd.Handle, - cast[POverlapped](ol)[], bytesRead, false.WinBool) + cast[POverlapped](ol), bytesRead, false.WinBool) if not overlappedRes.bool: let err = osLastError() if err.int32 == ERROR_HANDLE_EOF: @@ -282,7 +282,7 @@ proc write*(f: AsyncFile, data: string): Future[void] = # Request completed immediately. var bytesWritten: DWord let overlappedRes = getOverlappedResult(f.fd.Handle, - cast[POverlapped](ol)[], bytesWritten, false.WinBool) + cast[POverlapped](ol), bytesWritten, false.WinBool) if not overlappedRes.bool: retFuture.fail(newException(OSError, osErrorMsg(osLastError()))) else: diff --git a/lib/pure/asyncnet.nim b/lib/pure/asyncnet.nim index 6b19a48be..748778566 100644 --- a/lib/pure/asyncnet.nim +++ b/lib/pure/asyncnet.nim @@ -11,7 +11,7 @@ ## asynchronous dispatcher defined in the ``asyncdispatch`` module. ## ## SSL -## --- +## ---- ## ## SSL can be enabled by compiling with the ``-d:ssl`` flag. ## @@ -62,7 +62,9 @@ import os export SOBool -when defined(ssl): +const defineSsl = defined(ssl) or defined(nimdoc) + +when defineSsl: import openssl type @@ -79,7 +81,7 @@ type of false: nil case isSsl: bool of true: - when defined(ssl): + when defineSsl: sslHandle: SslPtr sslContext: SslContext bioIn: BIO @@ -125,7 +127,7 @@ proc newAsyncSocket*(domain, sockType, protocol: cint, Domain(domain), SockType(sockType), Protocol(protocol), buffered) -when defined(ssl): +when defineSsl: proc getSslError(handle: SslPtr, err: cint): cint = assert err < 0 var ret = SSLGetError(handle, err.cint) @@ -186,7 +188,7 @@ proc connect*(socket: AsyncSocket, address: string, port: Port) {.async.} = ## or an error occurs. await connect(socket.fd.AsyncFD, address, port, socket.domain) if socket.isSsl: - when defined(ssl): + when defineSsl: let flags = {SocketFlag.SafeDisconn} sslSetConnectState(socket.sslHandle) sslLoop(socket, flags, sslDoHandshake(socket.sslHandle)) @@ -197,7 +199,7 @@ template readInto(buf: cstring, size: int, socket: AsyncSocket, ## this is a template and not a proc. var res = 0 if socket.isSsl: - when defined(ssl): + when defineSsl: # SSL mode. sslLoop(socket, flags, sslRead(socket.sslHandle, buf, size.cint)) @@ -274,7 +276,7 @@ proc send*(socket: AsyncSocket, data: string, ## data has been sent. assert socket != nil if socket.isSsl: - when defined(ssl): + when defineSsl: var copy = data sslLoop(socket, flags, sslWrite(socket.sslHandle, addr copy[0], copy.len.cint)) @@ -426,9 +428,6 @@ proc recvLine*(socket: AsyncSocket, ## ## **Warning**: ``recvLine`` on unbuffered sockets assumes that the protocol ## uses ``\r\L`` to delimit a new line. - template addNLIfEmpty(): stmt = - if result.len == 0: - result.add("\c\L") assert SocketFlag.Peek notin flags ## TODO: # TODO: Optimise this @@ -456,7 +455,8 @@ proc bindAddr*(socket: AsyncSocket, port = Port(0), address = "") {. of AF_INET6: realaddr = "::" of AF_INET: realaddr = "0.0.0.0" else: - raiseOSError("Unknown socket address family and no address specified to bindAddr") + raise newException(ValueError, + "Unknown socket address family and no address specified to bindAddr") var aiList = getAddrInfo(realaddr, port, socket.domain) if bindAddr(socket.fd, aiList.ai_addr, aiList.ai_addrlen.Socklen) < 0'i32: @@ -468,7 +468,7 @@ proc close*(socket: AsyncSocket) = ## Closes the socket. defer: socket.fd.AsyncFD.closeSocket() - when defined(ssl): + when defineSsl: if socket.isSSL: let res = SslShutdown(socket.sslHandle) SSLFree(socket.sslHandle) @@ -478,7 +478,7 @@ proc close*(socket: AsyncSocket) = raiseSslError() socket.closed = true # TODO: Add extra debugging checks for this. -when defined(ssl): +when defineSsl: proc wrapSocket*(ctx: SslContext, socket: AsyncSocket) = ## Wraps a socket in an SSL context. This function effectively turns ## ``socket`` into an SSL socket. diff --git a/lib/pure/base64.nim b/lib/pure/base64.nim index 32d37ce02..2fc0b1c5e 100644 --- a/lib/pure/base64.nim +++ b/lib/pure/base64.nim @@ -90,6 +90,8 @@ template encodeInternal(s: expr, lineLen: int, newLine: string): stmt {.immediat if r+4 != result.len: setLen(result, r+4) else: + if r != result.len: + setLen(result, r) #assert(r == result.len) discard @@ -162,4 +164,3 @@ when isMainModule: "asure.", longText] for t in items(tests): assert decode(encode(t)) == t - diff --git a/lib/pure/collections/chains.nim b/lib/pure/collections/chains.nim new file mode 100644 index 000000000..6b2ecd272 --- /dev/null +++ b/lib/pure/collections/chains.nim @@ -0,0 +1,44 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2016 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Template based implementation of singly and doubly linked lists. +## The involved types should have 'prev' or 'next' fields and the +## list header should have 'head' or 'tail' fields. + +template prepend*(header, node) = + when compiles(header.head): + when compiles(node.prev): + if header.head != nil: + header.head.prev = node + node.next = header.head + header.head = node + when compiles(header.tail): + if header.tail == nil: + header.tail = node + +template append*(header, node) = + when compiles(header.head): + if header.head == nil: + header.head = node + when compiles(header.tail): + when compiles(node.prev): + node.prev = header.tail + if header.tail != nil: + header.tail.next = node + header.tail = node + +template unlink*(header, node) = + if node.next != nil: + node.next.prev = node.prev + if node.prev != nil: + node.prev.next = node.next + if header.head == node: + header.head = node.prev + if header.tail == node: + header.tail = node.next diff --git a/lib/pure/collections/sharedlist.nim b/lib/pure/collections/sharedlist.nim new file mode 100644 index 000000000..e93ceb02f --- /dev/null +++ b/lib/pure/collections/sharedlist.nim @@ -0,0 +1,95 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2015 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Shared list support. + +{.push stackTrace:off.} + +import + locks + +const + ElemsPerNode = 100 + +type + SharedListNode[A] = ptr object + next: SharedListNode[A] + dataLen: int + d: array[ElemsPerNode, A] + + SharedList*[A] = object ## generic shared list + head, tail: SharedListNode[A] + lock*: Lock + +template withLock(t, x: untyped) = + acquire(t.lock) + x + release(t.lock) + +proc iterAndMutate*[A](x: var SharedList[A]; action: proc(x: A): bool) = + ## iterates over the list. If 'action' returns true, the + ## current item is removed from the list. + withLock(x): + var n = x.head + while n != nil: + var i = 0 + while i < n.dataLen: + # action can add new items at the end, so release the lock: + release(x.lock) + if action(n.d[i]): + acquire(x.lock) + let t = x.tail + n.d[i] = t.d[t.dataLen] + dec t.dataLen + else: + acquire(x.lock) + inc i + n = n.next + +iterator items*[A](x: var SharedList[A]): A = + withLock(x): + var it = x.head + while it != nil: + for i in 0..it.dataLen-1: + yield it.d[i] + it = it.next + +proc add*[A](x: var SharedList[A]; y: A) = + withLock(x): + var node: SharedListNode[A] + if x.tail == nil or x.tail.dataLen == ElemsPerNode: + node = cast[type node](allocShared0(sizeof(node[]))) + node.next = x.tail + x.tail = node + if x.head == nil: x.head = node + else: + node = x.tail + node.d[node.dataLen] = y + inc(node.dataLen) + +proc initSharedList*[A](): SharedList[A] = + initLock result.lock + result.head = nil + result.tail = nil + +proc clear*[A](t: var SharedList[A]) = + withLock(t): + var it = t.head + while it != nil: + let nxt = it.next + deallocShared(it) + it = nxt + t.head = nil + t.tail = nil + +proc deinitSharedList*[A](t: var SharedList[A]) = + clear(t) + deinitLock t.lock + +{.pop.} diff --git a/lib/pure/collections/tables.nim b/lib/pure/collections/tables.nim index 2ed0d2034..58a789e76 100644 --- a/lib/pure/collections/tables.nim +++ b/lib/pure/collections/tables.nim @@ -663,6 +663,27 @@ proc sort*[A, B](t: OrderedTableRef[A, B], ## contrast to the `sort` for count tables). t[].sort(cmp) +proc del*[A, B](t: var OrderedTable[A, B], key: A) = + ## deletes `key` from ordered hash table `t`. O(n) comlexity. + var prev = -1 + let hc = hash(key) + forAllOrderedPairs: + if t.data[h].hcode == hc: + if t.first == h: + t.first = t.data[h].next + else: + t.data[prev].next = t.data[h].next + var zeroValue : type(t.data[h]) + t.data[h] = zeroValue + dec t.counter + break + else: + prev = h + +proc del*[A, B](t: var OrderedTableRef[A, B], key: A) = + ## deletes `key` from ordered hash table `t`. O(n) comlexity. + t[].del(key) + # ------------------------------ count tables ------------------------------- type @@ -984,6 +1005,26 @@ when isMainModule: s3[p1] = 30_000 s3[p2] = 45_000 + block: # Ordered table should preserve order after deletion + var + s4 = initOrderedTable[int, int]() + s4[1] = 1 + s4[2] = 2 + s4[3] = 3 + + var prev = 0 + for i in s4.values: + doAssert(prev < i) + prev = i + + s4.del(2) + doAssert(2 notin s4) + doAssert(s4.len == 2) + prev = 0 + for i in s4.values: + doAssert(prev < i) + prev = i + var t1 = initCountTable[string]() t2 = initCountTable[string]() diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index 603763386..f473e47d1 100644 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -335,7 +335,7 @@ proc addFiles*(p: var MultipartData, xs: openarray[tuple[name, file: string]]): var m = newMimetypes() for name, file in xs.items: var contentType: string - let (dir, fName, ext) = splitFile(file) + let (_, fName, ext) = splitFile(file) if ext.len > 0: contentType = m.getMimetype(ext[1..ext.high], nil) p.add(name, readFile(file), fName & ext, contentType) @@ -627,7 +627,7 @@ proc newAsyncHttpClient*(userAgent = defUserAgent, result.userAgent = defUserAgent result.maxRedirects = maxRedirects when defined(ssl): - result.sslContext = net.SslContext(sslContext) + result.sslContext = sslContext proc close*(client: AsyncHttpClient) = ## Closes any connections held by the HTTP client. diff --git a/lib/pure/json.nim b/lib/pure/json.nim index 30004da84..b9da8a0dd 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -56,6 +56,11 @@ import export tables.`$` +when defined(nimJsonGet): + {.pragma: deprecatedGet, deprecated.} +else: + {.pragma: deprecatedGet.} + type JsonEventKind* = enum ## enumeration of all events that may occur when parsing jsonError, ## an error occurred during parsing @@ -787,7 +792,7 @@ proc hash*(n: JsonNode): Hash = proc hash*(n: Table[string, JsonNode]): Hash = for key, val in n: - result = result !& hash(key) !& hash(val) + result = result xor (hash(key) !& hash(val)) result = !$result proc len*(n: JsonNode): int = @@ -799,16 +804,23 @@ proc len*(n: JsonNode): int = of JObject: result = n.fields.len else: discard -proc `[]`*(node: JsonNode, name: string): JsonNode {.inline.} = +proc `[]`*(node: JsonNode, name: string): JsonNode {.inline, deprecatedGet.} = ## Gets a field from a `JObject`, which must not be nil. - ## If the value at `name` does not exist, returns nil + ## If the value at `name` does not exist, raises KeyError. + ## + ## **Note:** The behaviour of this procedure changed in version 0.14.0. To + ## get a list of usages and to restore the old behaviour of this procedure, + ## compile with the ``-d:nimJsonGet`` flag. assert(not isNil(node)) assert(node.kind == JObject) - result = node.fields.getOrDefault(name) + when defined(nimJsonGet): + if not node.fields.hasKey(name): return nil + result = node.fields[name] proc `[]`*(node: JsonNode, index: int): JsonNode {.inline.} = ## Gets the node at `index` in an Array. Result is undefined if `index` - ## is out of bounds + ## is out of bounds, but as long as array bound checks are enabled it will + ## result in an exception. assert(not isNil(node)) assert(node.kind == JArray) return node.elems[index] @@ -838,20 +850,28 @@ proc `[]=`*(obj: JsonNode, key: string, val: JsonNode) {.inline.} = proc `{}`*(node: JsonNode, keys: varargs[string]): JsonNode = ## Traverses the node and gets the given value. If any of the - ## keys do not exist, returns nil. Also returns nil if one of the - ## intermediate data structures is not an object + ## keys do not exist, returns ``nil``. Also returns ``nil`` if one of the + ## intermediate data structures is not an object. result = node for key in keys: - if isNil(result) or result.kind!=JObject: + if isNil(result) or result.kind != JObject: return nil - result=result[key] + result = result.fields.getOrDefault(key) + +proc getOrDefault*(node: JsonNode, key: string): JsonNode = + ## Gets a field from a `node`. If `node` is nil or not an object or + ## value at `key` does not exist, returns nil + if not isNil(node) and node.kind == JObject: + result = node.fields.getOrDefault(key) + +template simpleGetOrDefault*{`{}`(node, [key])}(node: JsonNode, key: string): JsonNode = node.getOrDefault(key) proc `{}=`*(node: JsonNode, keys: varargs[string], value: JsonNode) = ## Traverses the node and tries to set the value at the given location - ## to `value` If any of the keys are missing, they are added + ## to ``value``. If any of the keys are missing, they are added. var node = node for i in 0..(keys.len-2): - if isNil(node[keys[i]]): + if not node.hasKey(keys[i]): node[keys[i]] = newJObject() node = node[keys[i]] node[keys[keys.len-1]] = value @@ -1217,16 +1237,6 @@ when false: # To get that we shall use, obj["json"] when isMainModule: - when not defined(js): - var parsed = parseFile("tests/testdata/jsontest.json") - - try: - discard parsed["key2"][12123] - doAssert(false) - except IndexError: doAssert(true) - - var parsed2 = parseFile("tests/testdata/jsontest2.json") - doAssert(parsed2{"repository", "description"}.str=="IRC Library for Haskell", "Couldn't fetch via multiply nested key using {}") let testJson = parseJson"""{ "a": [1, 2, 3, 4], "b": "asd", "c": "\ud83c\udf83", "d": "\u00E6"}""" # nil passthrough @@ -1314,3 +1324,19 @@ when isMainModule: var j4 = %*{"test": nil} doAssert j4 == %{"test": newJNull()} + + echo("99% of tests finished. Going to try loading file.") + + # Test loading of file. + when not defined(js): + var parsed = parseFile("tests/testdata/jsontest.json") + + try: + discard parsed["key2"][12123] + doAssert(false) + except IndexError: doAssert(true) + + var parsed2 = parseFile("tests/testdata/jsontest2.json") + doAssert(parsed2{"repository", "description"}.str=="IRC Library for Haskell", "Couldn't fetch via multiply nested key using {}") + + echo("Tests succeeded!") diff --git a/lib/pure/matchers.nim b/lib/pure/matchers.nim index 5c28f65a0..7022c21d9 100644 --- a/lib/pure/matchers.nim +++ b/lib/pure/matchers.nim @@ -8,6 +8,10 @@ # ## This module contains various string matchers for email addresses, etc. +## +## **Warning:** This module is deprecated since version 0.14.0. +{.deprecated.} + {.deadCodeElim: on.} {.push debugger:off .} # the user does not want to trace a part diff --git a/lib/pure/mersenne.nim b/lib/pure/mersenne.nim index ae0845714..afa343086 100644 --- a/lib/pure/mersenne.nim +++ b/lib/pure/mersenne.nim @@ -5,29 +5,31 @@ type {.deprecated: [TMersenneTwister: MersenneTwister].} -proc newMersenneTwister*(seed: int): MersenneTwister = +proc newMersenneTwister*(seed: uint32): MersenneTwister = result.index = 0 - result.mt[0]= uint32(seed) + result.mt[0] = seed for i in 1..623'u32: - result.mt[i]= (0x6c078965'u32 * (result.mt[i-1] xor (result.mt[i-1] shr 30'u32)) + i) + result.mt[i] = (0x6c078965'u32 * (result.mt[i-1] xor (result.mt[i-1] shr 30'u32)) + i) proc generateNumbers(m: var MersenneTwister) = for i in 0..623: - var y = (m.mt[i] and 0x80000000'u32) + (m.mt[(i+1) mod 624] and 0x7fffffff'u32) + var y = (m.mt[i] and 0x80000000'u32) + + (m.mt[(i+1) mod 624] and 0x7fffffff'u32) m.mt[i] = m.mt[(i+397) mod 624] xor uint32(y shr 1'u32) if (y mod 2'u32) != 0: - m.mt[i] = m.mt[i] xor 0x9908b0df'u32 + m.mt[i] = m.mt[i] xor 0x9908b0df'u32 -proc getNum*(m: var MersenneTwister): int = +proc getNum*(m: var MersenneTwister): uint32 = + ## Returns the next pseudo random number ranging from 0 to high(uint32) if m.index == 0: generateNumbers(m) - var y = m.mt[m.index] - y = y xor (y shr 11'u32) - y = y xor ((7'u32 shl y) and 0x9d2c5680'u32) - y = y xor ((15'u32 shl y) and 0xefc60000'u32) - y = y xor (y shr 18'u32) - m.index = (m.index+1) mod 624 - return int(y) + result = m.mt[m.index] + m.index = (m.index + 1) mod m.mt.len + + result = result xor (result shr 11'u32) + result = result xor ((7'u32 shl result) and 0x9d2c5680'u32) + result = result xor ((15'u32 shl result) and 0xefc60000'u32) + result = result xor (result shr 18'u32) # Test when not defined(testing) and isMainModule: diff --git a/lib/pure/nativesockets.nim b/lib/pure/nativesockets.nim index 3951b46a3..043d6d80a 100644 --- a/lib/pure/nativesockets.nim +++ b/lib/pure/nativesockets.nim @@ -219,31 +219,67 @@ proc getAddrInfo*(address: string, port: Port, domain: Domain = AF_INET, proc dealloc*(ai: ptr AddrInfo) = freeaddrinfo(ai) -proc ntohl*(x: int32): int32 = - ## Converts 32-bit integers from network to host byte order. +proc ntohl*(x: uint32): uint32 = + ## Converts 32-bit unsigned integers from network to host byte order. ## On machines where the host byte order is the same as network byte order, ## this is a no-op; otherwise, it performs a 4-byte swap operation. when cpuEndian == bigEndian: result = x - else: result = (x shr 24'i32) or - (x shr 8'i32 and 0xff00'i32) or - (x shl 8'i32 and 0xff0000'i32) or - (x shl 24'i32) + else: result = (x shr 24'u32) or + (x shr 8'u32 and 0xff00'u32) or + (x shl 8'u32 and 0xff0000'u32) or + (x shl 24'u32) -proc ntohs*(x: int16): int16 = - ## Converts 16-bit integers from network to host byte order. On machines - ## where the host byte order is the same as network byte order, this is - ## a no-op; otherwise, it performs a 2-byte swap operation. +template ntohl*(x: int32): expr {.deprecated.} = + ## Converts 32-bit integers from network to host byte order. + ## On machines where the host byte order is the same as network byte order, + ## this is a no-op; otherwise, it performs a 4-byte swap operation. + ## **Warning**: This template is deprecated since 0.14.0, IPv4 + ## addresses are now treated as unsigned integers. Please use the unsigned + ## version of this template. + cast[int32](ntohl(cast[uint32](x))) + +proc ntohs*(x: uint16): uint16 = + ## Converts 16-bit unsigned integers from network to host byte order. On + ## machines where the host byte order is the same as network byte order, + ## this is a no-op; otherwise, it performs a 2-byte swap operation. when cpuEndian == bigEndian: result = x - else: result = (x shr 8'i16) or (x shl 8'i16) - -template htonl*(x: int32): expr = + else: result = (x shr 8'u16) or (x shl 8'u16) + +template ntohs*(x: int16): expr {.deprecated.} = + ## Converts 16-bit integers from network to host byte order. On + ## machines where the host byte order is the same as network byte order, + ## this is a no-op; otherwise, it performs a 2-byte swap operation. + ## **Warning**: This template is deprecated since 0.14.0, where port + ## numbers became unsigned integers. Please use the unsigned version of + ## this template. + cast[int16](ntohs(cast[uint16](x))) + +template htonl*(x: int32): expr {.deprecated.} = ## Converts 32-bit integers from host to network byte order. On machines ## where the host byte order is the same as network byte order, this is ## a no-op; otherwise, it performs a 4-byte swap operation. + ## **Warning**: This template is deprecated since 0.14.0, IPv4 + ## addresses are now treated as unsigned integers. Please use the unsigned + ## version of this template. nativesockets.ntohl(x) -template htons*(x: int16): expr = - ## Converts 16-bit positive integers from host to network byte order. +template htonl*(x: uint32): expr = + ## Converts 32-bit unsigned integers from host to network byte order. On + ## machines where the host byte order is the same as network byte order, + ## this is a no-op; otherwise, it performs a 4-byte swap operation. + nativesockets.ntohl(x) + +template htons*(x: int16): expr {.deprecated.} = + ## Converts 16-bit integers from host to network byte order. + ## On machines where the host byte order is the same as network byte + ## order, this is a no-op; otherwise, it performs a 2-byte swap operation. + ## **Warning**: This template is deprecated since 0.14.0, where port + ## numbers became unsigned integers. Please use the unsigned version of + ## this template. + nativesockets.ntohs(x) + +template htons*(x: uint16): expr = + ## Converts 16-bit unsigned integers from host to network byte order. ## On machines where the host byte order is the same as network byte ## order, this is a no-op; otherwise, it performs a 2-byte swap operation. nativesockets.ntohs(x) diff --git a/lib/pure/net.nim b/lib/pure/net.nim index b9764edd3..5de6667dd 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -8,19 +8,76 @@ # ## This module implements a high-level cross-platform sockets interface. +## The procedures implemented in this module are primarily for blocking sockets. +## For asynchronous non-blocking sockets use the ``asyncnet`` module together +## with the ``asyncdispatch`` module. +## +## The first thing you will always need to do in order to start using sockets, +## is to create a new instance of the ``Socket`` type using the ``newSocket`` +## procedure. +## +## SSL +## ==== +## +## In order to use the SSL procedures defined in this module, you will need to +## compile your application with the ``-d:ssl`` flag. +## +## Examples +## ======== +## +## Connecting to a server +## ---------------------- +## +## After you create a socket with the ``newSocket`` procedure, you can easily +## connect it to a server running at a known hostname (or IP address) and port. +## To do so over TCP, use the example below. +## +## .. code-block:: Nim +## var socket = newSocket() +## socket.connect("google.com", Port(80)) +## +## UDP is a connectionless protocol, so UDP sockets don't have to explicitly +## call the ``connect`` procedure. They can simply start sending data +## immediately. +## +## .. code-block:: Nim +## var socket = newSocket() +## socket.sendTo("192.168.0.1", Port(27960), "status\n") +## +## Creating a server +## ----------------- +## +## After you create a socket with the ``newSocket`` procedure, you can create a +## TCP server by calling the ``bindAddr`` and ``listen`` procedures. +## +## .. code-block:: Nim +## var socket = newSocket() +## socket.bindAddr(Port(1234)) +## socket.listen() +## +## You can then begin accepting connections using the ``accept`` procedure. +## +## .. code-block:: Nim +## var client = newSocket() +## var address = "" +## while true: +## socket.acceptAddr(client, address) +## echo("Client connected from: ", address) +## {.deadCodeElim: on.} import nativesockets, os, strutils, parseutils, times export Port, `$`, `==` const useWinVersion = defined(Windows) or defined(nimdoc) +const defineSsl = defined(ssl) or defined(nimdoc) -when defined(ssl): +when defineSsl: import openssl # Note: The enumerations are mapped to Window's constants. -when defined(ssl): +when defineSsl: type SslError* = object of Exception @@ -54,7 +111,7 @@ type currPos: int # current index in buffer bufLen: int # current length of buffer of false: nil - when defined(ssl): + when defineSsl: case isSsl: bool of true: sslHandle: SSLPtr @@ -160,7 +217,7 @@ proc newSocket*(domain: Domain = AF_INET, sockType: SockType = SOCK_STREAM, raiseOSError(osLastError()) result = newSocket(fd, domain, sockType, protocol, buffered) -when defined(ssl): +when defineSsl: CRYPTO_malloc_init() SslLibraryInit() SslLoadErrorStrings() @@ -222,9 +279,9 @@ when defined(ssl): of protSSLv23: newCTX = SSL_CTX_new(SSLv23_method()) # SSlv2,3 and TLS1 support. of protSSLv2: - raiseSslError("SSLv2 is no longer secure and has been deprecated, use protSSLv3") + raiseSslError("SSLv2 is no longer secure and has been deprecated, use protSSLv23") of protSSLv3: - newCTX = SSL_CTX_new(SSLv3_method()) + raiseSslError("SSLv3 is no longer secure and has been deprecated, use protSSLv23") of protTLSv1: newCTX = SSL_CTX_new(TLSv1_method()) @@ -301,7 +358,7 @@ proc socketError*(socket: Socket, err: int = -1, async = false, ## error was caused by no data being available to be read. ## ## If ``err`` is not lower than 0 no exception will be raised. - when defined(ssl): + when defineSsl: if socket.isSSL: if err <= 0: var ret = SSLGetError(socket.sslHandle, err.cint) @@ -334,7 +391,7 @@ proc socketError*(socket: Socket, err: int = -1, async = false, raiseSSLError() else: raiseSSLError("Unknown Error") - if err == -1 and not (when defined(ssl): socket.isSSL else: false): + if err == -1 and not (when defineSsl: socket.isSSL else: false): var lastE = if lastError.int == -1: getSocketError(socket) else: lastError if async: when useWinVersion: @@ -368,7 +425,7 @@ proc bindAddr*(socket: Socket, port = Port(0), address = "") {. name.sin_family = toInt(AF_INET).int16 else: name.sin_family = toInt(AF_INET) - name.sin_port = htons(int16(port)) + name.sin_port = htons(port.uint16) name.sin_addr.s_addr = htonl(INADDR_ANY) if bindAddr(socket.fd, cast[ptr SockAddr](addr(name)), sizeof(name).SockLen) < 0'i32: @@ -414,7 +471,7 @@ proc acceptAddr*(server: Socket, client: var Socket, address: var string, client.isBuffered = server.isBuffered # Handle SSL. - when defined(ssl): + when defineSsl: if server.isSSL: # We must wrap the client sock in a ssl context. @@ -425,7 +482,7 @@ proc acceptAddr*(server: Socket, client: var Socket, address: var string, # Client socket is set above. address = $inet_ntoa(sockAddress.sin_addr) -when false: #defined(ssl): +when false: #defineSsl: proc acceptAddrSSL*(server: Socket, client: var Socket, address: var string): SSLAcceptResult {. tags: [ReadIOEffect].} = @@ -444,7 +501,7 @@ when false: #defined(ssl): ## ``AcceptNoClient`` will be returned when no client is currently attempting ## to connect. template doHandshake(): stmt = - when defined(ssl): + when defineSsl: if server.isSSL: client.setBlocking(false) # We must wrap the client sock in a ssl context. @@ -495,7 +552,7 @@ proc accept*(server: Socket, client: var Socket, proc close*(socket: Socket) = ## Closes a socket. try: - when defined(ssl): + when defineSsl: if socket.isSSL: ErrClearError() # As we are closing the underlying socket immediately afterwards, @@ -547,8 +604,9 @@ proc setSockOpt*(socket: Socket, opt: SOBool, value: bool, level = SOL_SOCKET) { var valuei = cint(if value: 1 else: 0) setSockOptInt(socket.fd, cint(level), toCInt(opt), valuei) -when defined(ssl): - proc handshake*(socket: Socket): bool {.tags: [ReadIOEffect, WriteIOEffect].} = +when defineSsl: + proc handshake*(socket: Socket): bool + {.tags: [ReadIOEffect, WriteIOEffect], deprecated.} = ## This proc needs to be called on a socket after it connects. This is ## only applicable when using ``connectAsync``. ## This proc performs the SSL handshake. @@ -557,6 +615,8 @@ when defined(ssl): ## ``True`` whenever handshake completed successfully. ## ## A ESSL error is raised on any other errors. + ## + ## **Note:** This procedure is deprecated since version 0.14.0. result = true if socket.isSSL: var ret = SSLConnect(socket.sslHandle) @@ -594,7 +654,7 @@ proc hasDataBuffered*(s: Socket): bool = if s.isBuffered: result = s.bufLen > 0 and s.currPos != s.bufLen - when defined(ssl): + when defineSsl: if s.isSSL and not result: result = s.sslHasPeekChar @@ -608,7 +668,7 @@ proc select(readfd: Socket, timeout = 500): int = proc readIntoBuf(socket: Socket, flags: int32): int = result = 0 - when defined(ssl): + when defineSsl: if socket.isSSL: result = SSLRead(socket.sslHandle, addr(socket.buffer), int(socket.buffer.high)) else: @@ -658,7 +718,7 @@ proc recv*(socket: Socket, data: pointer, size: int): int {.tags: [ReadIOEffect] result = read else: - when defined(ssl): + when defineSsl: if socket.isSSL: if socket.sslHasPeekChar: copyMem(data, addr(socket.sslPeekChar), 1) @@ -696,7 +756,7 @@ proc waitFor(socket: Socket, waited: var float, timeout, size: int, if timeout - int(waited * 1000.0) < 1: raise newException(TimeoutError, "Call to '" & funcName & "' timed out.") - when defined(ssl): + when defineSsl: if socket.isSSL: if socket.hasDataBuffered: # sslPeekChar is present. @@ -764,7 +824,7 @@ proc peekChar(socket: Socket, c: var char): int {.tags: [ReadIOEffect].} = c = socket.buffer[socket.currPos] else: - when defined(ssl): + when defineSsl: if socket.isSSL: if not socket.sslHasPeekChar: result = SSLRead(socket.sslHandle, addr(socket.sslPeekChar), 1) @@ -872,7 +932,7 @@ proc send*(socket: Socket, data: pointer, size: int): int {. ## ## **Note**: This is a low-level version of ``send``. You likely should use ## the version below. - when defined(ssl): + when defineSsl: if socket.isSSL: return SSLWrite(socket.sslHandle, cast[cstring](data), size) @@ -943,7 +1003,7 @@ proc sendTo*(socket: Socket, address: string, port: Port, proc isSsl*(socket: Socket): bool = ## Determines whether ``socket`` is a SSL socket. - when defined(ssl): + when defineSsl: result = socket.isSSL else: result = false @@ -1253,7 +1313,7 @@ proc connect*(socket: Socket, address: string, dealloc(aiList) if not success: raiseOSError(lastError) - when defined(ssl): + when defineSsl: if socket.isSSL: # RFC3546 for SNI specifies that IP addresses are not allowed. if not isIpAddress(address): @@ -1314,8 +1374,10 @@ proc connect*(socket: Socket, address: string, port = Port(0), if selectWrite(s, timeout) != 1: raise newException(TimeoutError, "Call to 'connect' timed out.") else: - when defined(ssl): + when defineSsl: if socket.isSSL: socket.fd.setBlocking(true) + {.warning[Deprecated]: off.} doAssert socket.handshake() + {.warning[Deprecated]: on.} socket.fd.setBlocking(true) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 470559e17..dee227c69 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -129,7 +129,7 @@ proc raiseOSError*(errorCode: OSErrorCode; additionalInfo = "") {.noinline.} = if additionalInfo.len == 0: e.msg = osErrorMsg(errorCode) else: - e.msg = additionalInfo & " " & osErrorMsg(errorCode) + e.msg = osErrorMsg(errorCode) & "\nAdditional info: " & additionalInfo if e.msg == "": e.msg = "unknown OS error" raise e @@ -1452,7 +1452,7 @@ template rawToFormalFileInfo(rawInfo, formalInfo): expr = ## 'rawInfo' is either a 'TBY_HANDLE_FILE_INFORMATION' structure on Windows, ## or a 'Stat' structure on posix when defined(Windows): - template toTime(e): expr = winTimeToUnixTime(rdFileTime(e)) + template toTime(e: FILETIME): expr {.gensym.} = winTimeToUnixTime(rdFileTime(e)) # local templates default to bind semantics template merge(a, b): expr = a or (b shl 32) formalInfo.id.device = rawInfo.dwVolumeSerialNumber formalInfo.id.file = merge(rawInfo.nFileIndexLow, rawInfo.nFileIndexHigh) diff --git a/lib/pure/parseutils.nim b/lib/pure/parseutils.nim index 698bde42a..3e25eba22 100644 --- a/lib/pure/parseutils.nim +++ b/lib/pure/parseutils.nim @@ -234,6 +234,47 @@ proc parseInt*(s: string, number: var int, start = 0): int {. elif result != 0: number = int(res) +# overflowChecks doesn't work with uint64 +proc rawParseUInt(s: string, b: var uint64, start = 0): int = + var + res = 0'u64 + prev = 0'u64 + i = start + if s[i] == '+': inc(i) # Allow + if s[i] in {'0'..'9'}: + b = 0 + while s[i] in {'0'..'9'}: + prev = res + res = res * 10 + (ord(s[i]) - ord('0')).uint64 + if prev > res: + return 0 # overflowChecks emulation + inc(i) + while s[i] == '_': inc(i) # underscores are allowed and ignored + b = res + result = i - start + +proc parseBiggestUInt*(s: string, number: var uint64, start = 0): int {. + rtl, extern: "npuParseBiggestUInt", noSideEffect.} = + ## parses an unsigned integer starting at `start` and stores the value into `number`. + ## Result is the number of processed chars or 0 if there is no integer or overflow detected. + var res: uint64 + # use 'res' for exception safety (don't write to 'number' in case of an + # overflow exception): + result = rawParseUInt(s, res, start) + number = res + +proc parseUInt*(s: string, number: var uint, start = 0): int {. + rtl, extern: "npuParseUInt", noSideEffect.} = + ## parses an unsigned integer starting at `start` and stores the value into `number`. + ## Result is the number of processed chars or 0 if there is no integer or overflow detected. + var res: uint64 + result = parseBiggestUInt(s, res, start) + if (sizeof(uint) <= 4) and + (res > 0xFFFF_FFFF'u64): + raise newException(OverflowError, "overflow") + elif result != 0: + number = uint(res) + proc parseBiggestFloat*(s: string, number: var BiggestFloat, start = 0): int {. magic: "ParseBiggestFloat", importc: "nimParseBiggestFloat", noSideEffect.} ## parses a float starting at `start` and stores the value into `number`. diff --git a/lib/pure/smtp.nim b/lib/pure/smtp.nim index b2adac2f3..050712902 100644 --- a/lib/pure/smtp.nim +++ b/lib/pure/smtp.nim @@ -20,9 +20,9 @@ ## var msg = createMessage("Hello from Nim's SMTP", ## "Hello!.\n Is this awesome or what?", ## @["foo@gmail.com"]) -## var smtp = connect("smtp.gmail.com", 465, true, true) -## smtp.auth("username", "password") -## smtp.sendmail("username@gmail.com", @["foo@gmail.com"], $msg) +## var smtpConn = connect("smtp.gmail.com", Port 465, true, true) +## smtpConn.auth("username", "password") +## smtpConn.sendmail("username@gmail.com", @["foo@gmail.com"], $msg) ## ## ## For SSL support this module relies on OpenSSL. If you want to @@ -31,6 +31,8 @@ import net, strutils, strtabs, base64, os import asyncnet, asyncdispatch +export Port + type Smtp* = object sock: Socket @@ -258,8 +260,8 @@ when not defined(testing) and isMainModule: # "Hello, my name is dom96.\n What\'s yours?", @["dominik@localhost"]) #echo(msg) - #var smtp = connect("localhost", 25, False, True) - #smtp.sendmail("root@localhost", @["dominik@localhost"], $msg) + #var smtpConn = connect("localhost", Port 25, false, true) + #smtpConn.sendmail("root@localhost", @["dominik@localhost"], $msg) #echo(decode("a17sm3701420wbe.12")) proc main() {.async.} = diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index f2c1e77e1..a5a4ee509 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -560,6 +560,24 @@ proc parseBiggestInt*(s: string): BiggestInt {.noSideEffect, procvar, if L != s.len or L == 0: raise newException(ValueError, "invalid integer: " & s) +proc parseUInt*(s: string): uint {.noSideEffect, procvar, + rtl, extern: "nsuParseUInt".} = + ## Parses a decimal unsigned integer value contained in `s`. + ## + ## If `s` is not a valid integer, `ValueError` is raised. + var L = parseutils.parseUInt(s, result, 0) + if L != s.len or L == 0: + raise newException(ValueError, "invalid unsigned integer: " & s) + +proc parseBiggestUInt*(s: string): uint64 {.noSideEffect, procvar, + rtl, extern: "nsuParseBiggestUInt".} = + ## Parses a decimal unsigned integer value contained in `s`. + ## + ## If `s` is not a valid integer, `ValueError` is raised. + var L = parseutils.parseBiggestUInt(s, result, 0) + if L != s.len or L == 0: + raise newException(ValueError, "invalid unsigned integer: " & s) + proc parseFloat*(s: string): float {.noSideEffect, procvar, rtl, extern: "nsuParseFloat".} = ## Parses a decimal floating point value contained in `s`. If `s` is not @@ -766,8 +784,7 @@ proc indent*(s: string, count: Natural, padding: string = " "): string {.noSideEffect, rtl, extern: "nsuIndent".} = ## Indents each line in ``s`` by ``count`` amount of ``padding``. ## - ## **Note:** This currently does not preserve the specific new line characters - ## used. + ## **Note:** This does not preserve the new line characters used in ``s``. result = "" var i = 0 for line in s.splitLines(): @@ -778,32 +795,39 @@ proc indent*(s: string, count: Natural, padding: string = " "): string result.add(line) i.inc -proc unindent*(s: string, eatAllIndent = false): string {. - noSideEffect, rtl, extern: "nsuUnindent".} = - ## Unindents `s`. - result = newStringOfCap(s.len) +proc unindent*(s: string, count: Natural, padding: string = " "): string + {.noSideEffect, rtl, extern: "nsuUnindent".} = + ## Unindents each line in ``s`` by ``count`` amount of ``padding``. + ## + ## **Note:** This does not preserve the new line characters used in ``s``. + result = "" var i = 0 - var pattern = true - var indent = 0 - while s[i] == ' ': inc i - var level = if i == 0: -1 else: i - while i < s.len: - if s[i] == ' ': - if i > 0 and s[i-1] in {'\l', '\c'}: - pattern = true - indent = 0 - if pattern: - inc(indent) - if indent > level and not eatAllIndent: - result.add(s[i]) - if level < 0: level = indent - else: - # a space somewhere: do not delete - result.add(s[i]) - else: - pattern = false - result.add(s[i]) - inc i + for line in s.splitLines(): + if i != 0: + result.add("\n") + var indentCount = 0 + for j in 0..<count.int: + indentCount.inc + if line[j .. j + <padding.len] != padding: + indentCount = j + break + result.add(line[indentCount*padding.len .. ^1]) + i.inc + +proc unindent*(s: string): string + {.noSideEffect, rtl, extern: "nsuUnindentAll".} = + ## Removes all indentation composed of whitespace from each line in ``s``. + ## + ## For example: + ## + ## .. code-block:: nim + ## const x = """ + ## Hello + ## There + ## """.unindent() + ## + ## doAssert x == "Hello\nThere\n" + unindent(s, 1000) # TODO: Passing a 1000 is a bit hackish. proc startsWith*(s, prefix: string): bool {.noSideEffect, rtl, extern: "nsuStartsWith".} = @@ -1725,3 +1749,33 @@ when isMainModule: doAssert join(@["foo", "bar", "baz"], ", ") == "foo, bar, baz" doAssert join([1, 2, 3]) == "123" doAssert join(@[1, 2, 3], ", ") == "1, 2, 3" + + doAssert """~~!!foo +~~!!bar +~~!!baz""".unindent(2, "~~!!") == "foo\nbar\nbaz" + + doAssert """~~!!foo +~~!!bar +~~!!baz""".unindent(2, "~~!!aa") == "~~!!foo\n~~!!bar\n~~!!baz" + doAssert """~~foo +~~ bar +~~ baz""".unindent(4, "~") == "foo\n bar\n baz" + doAssert """foo +bar + baz + """.unindent(4) == "foo\nbar\nbaz\n" + doAssert """foo + bar + baz + """.unindent(2) == "foo\n bar\n baz\n" + doAssert """foo + bar + baz + """.unindent(100) == "foo\nbar\nbaz\n" + + doAssert """foo + foo + bar + """.unindent() == "foo\nfoo\nbar\n" + + echo("strutils tests passed") diff --git a/lib/pure/subexes.nim b/lib/pure/subexes.nim index 5824ace81..22f29b77c 100644 --- a/lib/pure/subexes.nim +++ b/lib/pure/subexes.nim @@ -390,11 +390,11 @@ when isMainModule: doAssert(("type MyEnum* = enum\n $', '2i'\n '{..}" % ["fieldA", "fieldB", "FiledClkad", "fieldD", "fieldE", "longishFieldName"]).replace(" \n", "\n") == - strutils.unindent """ + strutils.unindent(""" type MyEnum* = enum fieldA, fieldB, FiledClkad, fieldD, - fieldE, longishFieldName""") + fieldE, longishFieldName""", 6)) doAssert subex"$1($', '{2..})" % ["f", "a", "b", "c"] == "f(a, b, c)" @@ -404,8 +404,8 @@ when isMainModule: doAssert((subex("type\n Enum = enum\n $', '40c'\n '{..}") % [ "fieldNameA", "fieldNameB", "fieldNameC", "fieldNameD"]).replace(" \n", "\n") == - strutils.unindent """ + strutils.unindent(""" type Enum = enum fieldNameA, fieldNameB, fieldNameC, - fieldNameD""") + fieldNameD""", 6)) diff --git a/lib/pure/times.nim b/lib/pure/times.nim index 29ae52d47..e0ee884a8 100644 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -182,7 +182,16 @@ proc getGMTime*(t: Time): TimeInfo {.tags: [TimeEffect], raises: [], benign.} ## converts the calendar time `t` to broken-down time representation, ## expressed in Coordinated Universal Time (UTC). -proc timeInfoToTime*(timeInfo: TimeInfo): Time {.tags: [], benign.} +proc timeInfoToTime*(timeInfo: TimeInfo): Time {.tags: [], benign, deprecated.} + ## converts a broken-down time structure to + ## calendar time representation. The function ignores the specified + ## contents of the structure members `weekday` and `yearday` and recomputes + ## them from the other information in the broken-down time structure. + ## + ## **Warning:** This procedure is deprecated since version 0.14.0. + ## Use ``toTime`` instead. + +proc toTime*(timeInfo: TimeInfo): Time {.tags: [], benign.} ## converts a broken-down time structure to ## calendar time representation. The function ignores the specified ## contents of the structure members `weekday` and `yearday` and recomputes @@ -356,7 +365,7 @@ proc `+`*(a: TimeInfo, interval: TimeInterval): TimeInfo = ## ## **Note:** This has been only briefly tested and it may not be ## very accurate. - let t = toSeconds(timeInfoToTime(a)) + let t = toSeconds(toTime(a)) let secs = toSeconds(a, interval) result = getLocalTime(fromSeconds(t + secs)) @@ -365,7 +374,7 @@ proc `-`*(a: TimeInfo, interval: TimeInterval): TimeInfo = ## ## **Note:** This has been only briefly tested, it is inaccurate especially ## when you subtract so much that you reach the Julian calendar. - let t = toSeconds(timeInfoToTime(a)) + let t = toSeconds(toTime(a)) var intval: TimeInterval intval.milliseconds = - interval.milliseconds intval.seconds = - interval.seconds @@ -517,6 +526,11 @@ when not defined(JS): # because the header of mktime is broken in my version of libc return mktime(timeInfoToTM(cTimeInfo)) + proc toTime(timeInfo: TimeInfo): Time = + var cTimeInfo = timeInfo # for C++ we have to make a copy, + # because the header of mktime is broken in my version of libc + return mktime(timeInfoToTM(cTimeInfo)) + proc toStringTillNL(p: cstring): string = result = "" var i = 0 @@ -618,7 +632,16 @@ elif defined(JS): result.setFullYear(timeInfo.year) result.setDate(timeInfo.monthday) - proc `$`(timeInfo: TimeInfo): string = return $(timeInfoToTime(timeInfo)) + proc toTime*(timeInfo: TimeInfo): Time = + result = internGetTime() + result.setSeconds(timeInfo.second) + result.setMinutes(timeInfo.minute) + result.setHours(timeInfo.hour) + result.setMonth(ord(timeInfo.month)) + result.setFullYear(timeInfo.year) + result.setDate(timeInfo.monthday) + + proc `$`(timeInfo: TimeInfo): string = return $(toTime(timeInfo)) proc `$`(time: Time): string = return $time.toLocaleString() proc `-` (a, b: Time): int64 = @@ -708,24 +731,24 @@ proc years*(y: int): TimeInterval {.inline.} = proc `+=`*(t: var Time, ti: TimeInterval) = ## modifies `t` by adding the interval `ti` - t = timeInfoToTime(getLocalTime(t) + ti) + t = toTime(getLocalTime(t) + ti) proc `+`*(t: Time, ti: TimeInterval): Time = ## adds the interval `ti` to Time `t` ## by converting to localTime, adding the interval, and converting back ## ## ``echo getTime() + 1.day`` - result = timeInfoToTime(getLocalTime(t) + ti) + result = toTime(getLocalTime(t) + ti) proc `-=`*(t: var Time, ti: TimeInterval) = ## modifies `t` by subtracting the interval `ti` - t = timeInfoToTime(getLocalTime(t) - ti) + t = toTime(getLocalTime(t) - ti) proc `-`*(t: Time, ti: TimeInterval): Time = ## adds the interval `ti` to Time `t` ## ## ``echo getTime() - 1.day`` - result = timeInfoToTime(getLocalTime(t) - ti) + result = toTime(getLocalTime(t) - ti) proc formatToken(info: TimeInfo, token: string, buf: var string) = ## Helper of the format proc to parse individual tokens. @@ -1193,7 +1216,7 @@ proc parse*(value, layout: string): TimeInfo = parseToken(info, token, value, j) token = "" # Reset weekday as it might not have been provided and the default may be wrong - info.weekday = getLocalTime(timeInfoToTime(info)).weekday + info.weekday = getLocalTime(toTime(info)).weekday return info # Leap year calculations are adapted from: @@ -1204,7 +1227,7 @@ proc parse*(value, layout: string): TimeInfo = proc countLeapYears*(yearSpan: int): int = ## Returns the number of leap years spanned by a given number of years. ## - ## Note: for leap years, start date is assumed to be 1 AD. + ## **Note:** For leap years, start date is assumed to be 1 AD. ## counts the number of leap years up to January 1st of a given year. ## Keep in mind that if specified year is a leap year, the leap day ## has not happened before January 1st of that year. @@ -1239,13 +1262,14 @@ proc getDayOfWeek*(day, month, year: int): WeekDay = y = year - a m = month + (12*a) - 2 d = (day + y + (y div 4) - (y div 100) + (y div 400) + (31*m) div 12) mod 7 - # The value of d is 0 for a Sunday, 1 for a Monday, 2 for a Tuesday, etc. so we must correct - # for the WeekDay type. + # The value of d is 0 for a Sunday, 1 for a Monday, 2 for a Tuesday, etc. + # so we must correct for the WeekDay type. if d == 0: return dSun result = (d-1).WeekDay proc getDayOfWeekJulian*(day, month, year: int): WeekDay = - ## Returns the day of the week enum from day, month and year, according to the Julian calendar. + ## Returns the day of the week enum from day, month and year, + ## according to the Julian calendar. # Day & month start from one. let a = (14 - month) div 12 @@ -1254,8 +1278,11 @@ proc getDayOfWeekJulian*(day, month, year: int): WeekDay = d = (5 + day + y + (y div 4) + (31*m) div 12) mod 7 result = d.WeekDay -proc timeToTimeInfo*(t: Time): TimeInfo = +proc timeToTimeInfo*(t: Time): TimeInfo {.deprecated.} = ## Converts a Time to TimeInfo. + ## + ## **Warning:** This procedure is deprecated since version 0.14.0. + ## Use ``getLocalTime`` or ``getGMTime`` instead. let secs = t.toSeconds().int daysSinceEpoch = secs div secondsInDay @@ -1286,34 +1313,21 @@ proc timeToTimeInfo*(t: Time): TimeInfo = s = daySeconds mod secondsInMin result = TimeInfo(year: y, yearday: yd, month: m, monthday: md, weekday: wd, hour: h, minute: mi, second: s) -proc timeToTimeInterval*(t: Time): TimeInterval = +proc timeToTimeInterval*(t: Time): TimeInterval {.deprecated.} = ## Converts a Time to a TimeInterval. + ## + ## **Warning:** This procedure is deprecated since version 0.14.0. + ## Use ``toTimeInterval`` instead. + # Milliseconds not available from Time + var tInfo = t.getLocalTime() + initInterval(0, tInfo.second, tInfo.minute, tInfo.hour, tInfo.weekday.ord, tInfo.month.ord, tInfo.year) - let - secs = t.toSeconds().int - daysSinceEpoch = secs div secondsInDay - (yearsSinceEpoch, daysRemaining) = countYearsAndDays(daysSinceEpoch) - daySeconds = secs mod secondsInDay - - result.years = yearsSinceEpoch + epochStartYear - - var - mon = mJan - days = daysRemaining - daysInMonth = getDaysInMonth(mon, result.years) - - # calculate month and day remainder - while days > daysInMonth and mon <= mDec: - days -= daysInMonth - mon.inc - daysInMonth = getDaysInMonth(mon, result.years) - - result.months = mon.int + 1 # month is 1 indexed int - result.days = days - result.hours = daySeconds div secondsInHour + 1 - result.minutes = (daySeconds mod secondsInHour) div secondsInMin - result.seconds = daySeconds mod secondsInMin +proc toTimeInterval*(t: Time): TimeInterval = + ## Converts a Time to a TimeInterval. # Milliseconds not available from Time + var tInfo = t.getLocalTime() + initInterval(0, tInfo.second, tInfo.minute, tInfo.hour, tInfo.weekday.ord, tInfo.month.ord, tInfo.year) + when isMainModule: # this is testing non-exported function diff --git a/lib/pure/unittest.nim b/lib/pure/unittest.nim index aca9d51e2..aca9d51e2 100755..100644 --- a/lib/pure/unittest.nim +++ b/lib/pure/unittest.nim diff --git a/lib/system.nim b/lib/system.nim index fefabe53f..5d6fcf868 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -1291,7 +1291,7 @@ const when hasThreadSupport and defined(tcc) and not compileOption("tlsEmulation"): # tcc doesn't support TLS {.error: "``--tlsEmulation:on`` must be used when using threads with tcc backend".} - + when defined(boehmgc): when defined(windows): const boehmLib = "boehmgc.dll" @@ -2565,8 +2565,9 @@ when not defined(JS): #and not defined(nimscript): {.push stack_trace: off, profiler:off.} when not defined(nimscript) and not defined(nogc): - proc initGC() - when not defined(boehmgc) and not defined(useMalloc) and not defined(gogc): + when not defined(gcStack): + proc initGC() + when not defined(boehmgc) and not defined(useMalloc) and not defined(gogc) and not defined(gcStack): proc initAllocator() {.inline.} proc initStackBottom() {.inline, compilerproc.} = @@ -2774,6 +2775,9 @@ when not defined(JS): #and not defined(nimscript): ## reads `len` bytes into the buffer `a` starting at ``a[start]``. Returns ## the actual number of bytes that have been read which may be less than ## `len` (if not as many bytes are remaining), but not greater. + ## + ## **Warning:** The buffer `a` must be pre-allocated. This can be done + ## using, for example, ``newString``. proc readBuffer*(f: File, buffer: pointer, len: Natural): int {. tags: [ReadIOEffect], benign.} @@ -2874,6 +2878,7 @@ when not defined(JS): #and not defined(nimscript): when declared(initAllocator): initAllocator() when hasThreadSupport: + const insideRLocksModule = false include "system/syslocks" when hostOS != "standalone": include "system/threads" elif not defined(nogc) and not defined(nimscript): diff --git a/lib/system/alloc.nim b/lib/system/alloc.nim index c0b697238..e0fd53b7b 100644 --- a/lib/system/alloc.nim +++ b/lib/system/alloc.nim @@ -13,155 +13,7 @@ # - make searching for block O(1) {.push profiler:off.} -proc roundup(x, v: int): int {.inline.} = - result = (x + (v-1)) and not (v-1) - sysAssert(result >= x, "roundup: result < x") - #return ((-x) and (v-1)) +% x - -sysAssert(roundup(14, PageSize) == PageSize, "invalid PageSize") -sysAssert(roundup(15, 8) == 16, "roundup broken") -sysAssert(roundup(65, 8) == 72, "roundup broken 2") - -# ------------ platform specific chunk allocation code ----------------------- - -# some platforms have really weird unmap behaviour: unmap(blockStart, PageSize) -# really frees the whole block. Happens for Linux/PowerPC for example. Amd64 -# and x86 are safe though; Windows is special because MEM_RELEASE can only be -# used with a size of 0. We also allow unmapping to be turned off with -# -d:nimAllocNoUnmap: -const doNotUnmap = not (defined(amd64) or defined(i386)) or - defined(windows) or defined(nimAllocNoUnmap) - - -when defined(emscripten): - const - PROT_READ = 1 # page can be read - PROT_WRITE = 2 # page can be written - MAP_PRIVATE = 2'i32 # Changes are private - - var MAP_ANONYMOUS {.importc: "MAP_ANONYMOUS", header: "<sys/mman.h>".}: cint - type - PEmscriptenMMapBlock = ptr EmscriptenMMapBlock - EmscriptenMMapBlock {.pure, inheritable.} = object - realSize: int # size of previous chunk; for coalescing - realPointer: pointer # if < PageSize it is a small chunk - - proc mmap(adr: pointer, len: int, prot, flags, fildes: cint, - off: int): pointer {.header: "<sys/mman.h>".} - - proc munmap(adr: pointer, len: int): cint {.header: "<sys/mman.h>".} - - proc osAllocPages(block_size: int): pointer {.inline.} = - let realSize = block_size + sizeof(EmscriptenMMapBlock) + PageSize + 1 - result = mmap(nil, realSize, PROT_READ or PROT_WRITE, - MAP_PRIVATE or MAP_ANONYMOUS, -1, 0) - if result == nil or result == cast[pointer](-1): - raiseOutOfMem() - - let realPointer = result - let pos = cast[int](result) - - # Convert pointer to PageSize correct one. - var new_pos = cast[ByteAddress](pos) +% (PageSize - (pos %% PageSize)) - if (new_pos-pos)< sizeof(EmscriptenMMapBlock): - new_pos = new_pos +% PageSize - result = cast[pointer](new_pos) - - var mmapDescrPos = cast[ByteAddress](result) -% sizeof(EmscriptenMMapBlock) - - var mmapDescr = cast[EmscriptenMMapBlock](mmapDescrPos) - mmapDescr.realSize = realSize - mmapDescr.realPointer = realPointer - - c_fprintf(c_stdout, "[Alloc] size %d %d realSize:%d realPos:%d\n", block_size, cast[int](result), realSize, cast[int](realPointer)) - - proc osDeallocPages(p: pointer, size: int) {.inline} = - var mmapDescrPos = cast[ByteAddress](p) -% sizeof(EmscriptenMMapBlock) - var mmapDescr = cast[EmscriptenMMapBlock](mmapDescrPos) - discard munmap(mmapDescr.realPointer, mmapDescr.realSize) - -elif defined(posix): - const - PROT_READ = 1 # page can be read - PROT_WRITE = 2 # page can be written - MAP_PRIVATE = 2'i32 # Changes are private - - when defined(macosx) or defined(bsd): - const MAP_ANONYMOUS = 0x1000 - elif defined(solaris): - const MAP_ANONYMOUS = 0x100 - elif defined(linux): - const MAP_ANONYMOUS = 0x20'i32 - else: - var - MAP_ANONYMOUS {.importc: "MAP_ANONYMOUS", header: "<sys/mman.h>".}: cint - - proc mmap(adr: pointer, len: int, prot, flags, fildes: cint, - off: int): pointer {.header: "<sys/mman.h>".} - - proc munmap(adr: pointer, len: int): cint {.header: "<sys/mman.h>".} - - proc osAllocPages(size: int): pointer {.inline.} = - result = mmap(nil, size, PROT_READ or PROT_WRITE, - MAP_PRIVATE or MAP_ANONYMOUS, -1, 0) - if result == nil or result == cast[pointer](-1): - raiseOutOfMem() - - proc osDeallocPages(p: pointer, size: int) {.inline} = - when reallyOsDealloc: discard munmap(p, size) - -elif defined(windows): - const - MEM_RESERVE = 0x2000 - MEM_COMMIT = 0x1000 - MEM_TOP_DOWN = 0x100000 - PAGE_READWRITE = 0x04 - - MEM_DECOMMIT = 0x4000 - MEM_RELEASE = 0x8000 - - proc virtualAlloc(lpAddress: pointer, dwSize: int, flAllocationType, - flProtect: int32): pointer {. - header: "<windows.h>", stdcall, importc: "VirtualAlloc".} - - proc virtualFree(lpAddress: pointer, dwSize: int, - dwFreeType: int32) {.header: "<windows.h>", stdcall, - importc: "VirtualFree".} - - proc osAllocPages(size: int): pointer {.inline.} = - result = virtualAlloc(nil, size, MEM_RESERVE or MEM_COMMIT, - PAGE_READWRITE) - if result == nil: raiseOutOfMem() - - proc osDeallocPages(p: pointer, size: int) {.inline.} = - # according to Microsoft, 0 is the only correct value for MEM_RELEASE: - # This means that the OS has some different view over how big the block is - # that we want to free! So, we cannot reliably release the memory back to - # Windows :-(. We have to live with MEM_DECOMMIT instead. - # Well that used to be the case but MEM_DECOMMIT fragments the address - # space heavily, so we now treat Windows as a strange unmap target. - when reallyOsDealloc: virtualFree(p, 0, MEM_RELEASE) - #VirtualFree(p, size, MEM_DECOMMIT) - -elif hostOS == "standalone": - var - theHeap: array[1024*PageSize, float64] # 'float64' for alignment - bumpPointer = cast[int](addr theHeap) - - proc osAllocPages(size: int): pointer {.inline.} = - if size+bumpPointer < cast[int](addr theHeap) + sizeof(theHeap): - result = cast[pointer](bumpPointer) - inc bumpPointer, size - else: - raiseOutOfMem() - - proc osDeallocPages(p: pointer, size: int) {.inline.} = - if bumpPointer-size == cast[int](p): - dec bumpPointer, size -else: - {.error: "Port memory manager to your platform".} - -# --------------------- end of non-portable code ----------------------------- +include osalloc # We manage *chunks* of memory. Each chunk is a multiple of the page size. # Each chunk starts at an address that is divisible by the page size. Chunks diff --git a/lib/system/dyncalls.nim b/lib/system/dyncalls.nim index 6dc8999d1..61777e514 100644 --- a/lib/system/dyncalls.nim +++ b/lib/system/dyncalls.nim @@ -23,16 +23,16 @@ proc rawWrite(f: File, s: string) = proc nimLoadLibraryError(path: string) = # carefully written to avoid memory allocation: - stdout.rawWrite("could not load: ") - stdout.rawWrite(path) - stdout.rawWrite("\n") + stderr.rawWrite("could not load: ") + stderr.rawWrite(path) + stderr.rawWrite("\n") quit(1) proc procAddrError(name: cstring) {.noinline.} = # carefully written to avoid memory allocation: - stdout.rawWrite("could not import: ") - stdout.write(name) - stdout.rawWrite("\n") + stderr.rawWrite("could not import: ") + stderr.write(name) + stderr.rawWrite("\n") quit(1) # this code was inspired from Lua's source code: @@ -71,7 +71,7 @@ when defined(posix): when defined(nimDebugDlOpen): let error = dlerror() if error != nil: - c_fprintf(c_stdout, "%s\n", error) + c_fprintf(c_stderr, "%s\n", error) proc nimGetProcAddr(lib: LibHandle, name: cstring): ProcAddr = result = dlsym(lib, name) @@ -109,9 +109,31 @@ elif defined(windows) or defined(dos): proc nimGetProcAddr(lib: LibHandle, name: cstring): ProcAddr = result = getProcAddress(cast[THINSTANCE](lib), name) if result != nil: return + const decorated_length = 250 + var decorated: array[decorated_length, char] + decorated[0] = '_' + var m = 1 + while m < (decorated_length - 5): + if name[m - 1] == '\x00': break + decorated[m] = name[m - 1] + inc(m) + decorated[m] = '@' for i in countup(0, 50): - var decorated = "_" & $name & "@" & $(i * 4) - result = getProcAddress(cast[THINSTANCE](lib), cstring(decorated)) + var k = i * 4 + if k div 100 == 0: + if k div 10 == 0: + m = m + 1 + else: + m = m + 2 + else: + m = m + 3 + decorated[m + 1] = '\x00' + while true: + decorated[m] = chr(ord('0') + (k %% 10)) + dec(m) + k = k div 10 + if k == 0: break + result = getProcAddress(cast[THINSTANCE](lib), decorated) if result != nil: return procAddrError(name) diff --git a/lib/system/gc.nim b/lib/system/gc.nim index d8390ca14..4f461b5c3 100644 --- a/lib/system/gc.nim +++ b/lib/system/gc.nim @@ -39,6 +39,9 @@ when withRealTime and not declared(getTicks): when defined(memProfiler): proc nimProfile(requestedSize: int) {.benign.} +when hasThreadSupport: + import sharedlist + const rcIncrement = 0b1000 # so that lowest 3 bits are not touched rcBlack = 0b000 # cell is colored black; in use or free @@ -93,6 +96,9 @@ type stat: GcStat when useMarkForDebug or useBackupGc: marked: CellSet + when hasThreadSupport: + toDispose: SharedList[pointer] + {.deprecated: [TWalkOp: WalkOp, TFinalizer: Finalizer, TGcHeap: GcHeap, TGcStat: GcStat].} var @@ -304,6 +310,8 @@ proc initGC() = init(gch.decStack) when useMarkForDebug or useBackupGc: init(gch.marked) + when hasThreadSupport: + gch.toDispose = initSharedList[pointer]() when useMarkForDebug or useBackupGc: type @@ -749,6 +757,9 @@ proc collectRoots(gch: var GcHeap) = collectWhite(s) proc collectCycles(gch: var GcHeap) = + when hasThreadSupport: + for c in gch.toDispose: + nimGCunref(c) # ensure the ZCT 'color' is not used: while gch.zct.len > 0: discard collectZCT(gch) when useBackupGc: diff --git a/lib/system/gc_common.nim b/lib/system/gc_common.nim index a4676d26e..4807bb6f8 100644 --- a/lib/system/gc_common.nim +++ b/lib/system/gc_common.nim @@ -7,6 +7,31 @@ # distribution, for details about the copyright. # +type + ForeignCell* = object + data*: pointer + owner: ptr GcHeap + +proc protect*(x: pointer): ForeignCell = + nimGCref(x) + result.data = x + result.owner = addr(gch) + +proc dispose*(x: ForeignCell) = + when hasThreadSupport: + # if we own it we can free it directly: + if x.owner == addr(gch): + nimGCunref(x.data) + else: + x.owner.toDispose.add(x.data) + else: + nimGCunref(x.data) + +proc isNotForeign*(x: ForeignCell): bool = + ## returns true if 'x' belongs to the calling thread. + ## No deep copy has to be performed then. + x.owner == addr(gch) + proc len(stack: ptr GcStack): int = if stack == nil: return 0 diff --git a/lib/system/gc_ms.nim b/lib/system/gc_ms.nim index d1aecb7a2..ec69f6e5f 100644 --- a/lib/system/gc_ms.nim +++ b/lib/system/gc_ms.nim @@ -28,6 +28,9 @@ template mulThreshold(x): expr {.immediate.} = x * 2 when defined(memProfiler): proc nimProfile(requestedSize: int) + +when hasThreadSupport: + import sharedlist type WalkOp = enum @@ -68,6 +71,8 @@ type recGcLock: int # prevent recursion via finalizers; no thread lock region: MemRegion # garbage collected region stat: GcStat + when hasThreadSupport: + toDispose: SharedList[pointer] additionalRoots: CellSeq # dummy roots for GC_ref/unref {.deprecated: [TWalkOp: WalkOp, TFinalizer: Finalizer, TGcStat: GcStat, TGlobalMarkerProc: GlobalMarkerProc, TGcHeap: GcHeap].} @@ -179,6 +184,8 @@ proc initGC() = when withBitvectors: init(gch.allocated) init(gch.marked) + when hasThreadSupport: + gch.toDispose = initSharedList[pointer]() proc forAllSlotsAux(dest: pointer, n: ptr TNimNode, op: WalkOp) {.benign.} = var d = cast[ByteAddress](dest) @@ -321,6 +328,9 @@ proc growObj(old: pointer, newsize: int): pointer {.rtl.} = # ----------------- collector ----------------------------------------------- proc mark(gch: var GcHeap, c: PCell) = + when hasThreadSupport: + for c in gch.toDispose: + nimGCunref(c) when withBitvectors: incl(gch.marked, c) gcAssert gch.tempStack.len == 0, "stack not empty!" diff --git a/lib/system/gc_stack.nim b/lib/system/gc_stack.nim new file mode 100644 index 000000000..c251a4d0b --- /dev/null +++ b/lib/system/gc_stack.nim @@ -0,0 +1,439 @@ +# +# Nim's Runtime Library +# (c) Copyright 2016 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# "Stack GC" for embedded devices or ultra performance requirements. + +include osalloc + +# We manage memory as a thread local stack. Since the allocation pointer +# is detached from the control flow pointer, this model is vastly more +# useful than the traditional programming model while almost as safe. +# Individual objects can also be deleted but no coalescing is performed. +# Stacks can also be moved from one thread to another. + +# We also support 'finalizers'. + +type + Finalizer {.compilerproc.} = proc (self: pointer) {.nimcall, benign.} + # A ref type can have a finalizer that is called before the object's + # storage is freed. + + AlignType = BiggestFloat + ObjHeader = object + typ: PNimType + nextFinal: ptr ObjHeader # next object with finalizer + + Hole = object # stacks can have holes. Otherwise 'growObj' would be insane. + zeroTyp: pointer # overlaid with 'typ' field. Always 'nil'. + size: int # size of the free slot + + Chunk = ptr BaseChunk + BaseChunk = object + next: Chunk + size: int + head, tail: ptr ObjHeader # first and last object in chunk that + # has a finalizer attached to it + +type + StackPtr = object + bump: pointer + remaining: int + current: Chunk + + MemRegion* = object + remaining: int + bump: pointer + head, tail: Chunk + nextChunkSize, totalSize: int + hole: ptr Hole # we support individual freeing + when hasThreadSupport: + lock: SysLock + +var + tlRegion {.threadVar.}: MemRegion + +template withRegion*(r: MemRegion; body: untyped) = + let oldRegion = tlRegion + tlRegion = r + try: + body + finally: + tlRegion = oldRegion + +template inc(p: pointer, s: int) = + p = cast[pointer](cast[int](p) +% s) + +template `+!`(p: pointer, s: int): pointer = + cast[pointer](cast[int](p) +% s) + +template `-!`(p: pointer, s: int): pointer = + cast[pointer](cast[int](p) -% s) + +proc allocSlowPath(r: var MemRegion; size: int) = + # we need to ensure that the underlying linked list + # stays small. Say we want to grab 16GB of RAM with some + # exponential growth function. So we allocate 16KB, then + # 32 KB, 64 KB, 128KB, 256KB, 512KB, 1MB, 2MB, 4MB, + # 8MB, 16MB, 32MB, 64MB, 128MB, 512MB, 1GB, 2GB, 4GB, 8GB, + # 16GB --> list contains only 20 elements! That's reasonable. + if (r.totalSize and 1) == 0: + r.nextChunkSize = + if r.totalSize < 64 * 1024: PageSize*4 + else: r.nextChunkSize*2 + var s = roundup(size+sizeof(BaseChunk), PageSize) + var fresh: Chunk + if s > r.nextChunkSize: + fresh = cast[Chunk](osAllocPages(s)) + else: + fresh = cast[Chunk](osTryAllocPages(r.nextChunkSize)) + if fresh == nil: + fresh = cast[Chunk](osAllocPages(s)) + # lowest bit in totalSize is the "don't increase nextChunkSize" + inc r.totalSize + else: + s = r.nextChunkSize + fresh.size = s + fresh.head = nil + fresh.tail = nil + inc r.totalSize, s + let old = r.tail + if old == nil: + r.head = fresh + else: + r.tail.next = fresh + r.bump = fresh +! sizeof(BaseChunk) + r.tail = fresh + r.remaining = s - sizeof(BaseChunk) + +proc alloc(r: var MemRegion; size: int): pointer {.inline.} = + if size > r.remaining: + allocSlowPath(r, size) + sysAssert(size <= r.remaining, "size <= r.remaining") + dec(r.remaining, size) + result = r.bump + inc r.bump, size + +proc runFinalizers(c: Chunk) = + var it = c.head + while it != nil: + # indivually freed objects with finalizer stay in the list, but + # their typ is nil then: + if it.typ != nil and it.typ.finalizer != nil: + (cast[Finalizer](it.typ.finalizer))(it+!sizeof(ObjHeader)) + it = it.nextFinal + +proc dealloc(r: var MemRegion; p: pointer) = + let it = cast[ptr ObjHeader](p-!sizeof(ObjHeader)) + if it.typ != nil and it.typ.finalizer != nil: + (cast[Finalizer](it.typ.finalizer))(p) + it.typ = nil + +proc deallocAll(r: var MemRegion; head: Chunk) = + var it = head + while it != nil: + let nxt = it.next + runFinalizers(it) + dec r.totalSize, it.size + osDeallocPages(it, it.size) + it = nxt + +proc deallocAll*(r: var MemRegion) = + deallocAll(r, r.head) + zeroMem(addr r, sizeof r) + +proc obstackPtr*(r: MemRegion): StackPtr = + result.bump = r.bump + result.remaining = r.remaining + result.current = r.tail + +template computeRemaining(r): untyped = + r.tail.size -% (cast[int](r.bump) -% cast[int](r.tail)) + +proc setObstackPtr*(r: var MemRegion; sp: StackPtr) = + # free everything after 'sp': + if sp.current != nil: + deallocAll(r, sp.current.next) + sp.current.next = nil + else: + deallocAll(r, r.head) + r.head = nil + r.bump = sp.bump + r.tail = sp.current + r.remaining = sp.remaining + +proc obstackPtr*(): StackPtr = tlRegion.obstackPtr() +proc setObstackPtr*(sp: StackPtr) = tlRegion.setObstackPtr(sp) + +proc joinRegion*(dest: var MemRegion; src: MemRegion) = + # merging is not hard. + if dest.head.isNil: + dest.head = src.head + else: + dest.tail.next = src.head + dest.tail = src.tail + dest.bump = src.bump + dest.remaining = src.remaining + dest.nextChunkSize = max(dest.nextChunkSize, src.nextChunkSize) + inc dest.totalSize, src.totalSize + +proc isOnHeap*(r: MemRegion; p: pointer): bool = + # the tail chunk is the largest, so check it first. It's also special + # in that contains the current bump pointer: + if r.tail >= p and p < r.bump: + return true + var it = r.head + while it != r.tail: + if it >= p and p <= it+!it.size: return true + it = it.next + +when false: + # essential feature for later: copy data over from one region to another + + proc isInteriorPointer(r: MemRegion; p: pointer): pointer = + discard " we cannot patch stack pointers anyway!" + + type + PointerStackChunk = object + next, prev: ptr PointerStackChunk + len: int + data: array[128, pointer] + + template head(s: PointerStackChunk): untyped = s.prev + template tail(s: PointerStackChunk): untyped = s.next + + include chains + + proc push(r: var MemRegion; s: var PointerStackChunk; x: pointer) = + if s.len < high(s.data): + s.data[s.len] = x + inc s.len + else: + let fresh = cast[ptr PointerStackChunk](alloc(r, sizeof(PointerStackChunk))) + fresh.len = 1 + fresh.data[0] = x + fresh.next = nil + fresh.prev = nil + append(s, fresh) + + + proc genericDeepCopyAux(dr: var MemRegion; stack: var PointerStackChunk; + dest, src: pointer, mt: PNimType) {.benign.} + proc genericDeepCopyAux(dr: var MemRegion; stack: var PointerStackChunk; + dest, src: pointer, n: ptr TNimNode) {.benign.} = + var + d = cast[ByteAddress](dest) + s = cast[ByteAddress](src) + case n.kind + of nkSlot: + genericDeepCopyAux(cast[pointer](d +% n.offset), + cast[pointer](s +% n.offset), n.typ) + of nkList: + for i in 0..n.len-1: + genericDeepCopyAux(dest, src, n.sons[i]) + of nkCase: + var dd = selectBranch(dest, n) + var m = selectBranch(src, n) + # reset if different branches are in use; note different branches also + # imply that's not self-assignment (``x = x``)! + if m != dd and dd != nil: + genericResetAux(dest, dd) + copyMem(cast[pointer](d +% n.offset), cast[pointer](s +% n.offset), + n.typ.size) + if m != nil: + genericDeepCopyAux(dest, src, m) + of nkNone: sysAssert(false, "genericDeepCopyAux") + + proc copyDeepString(dr: var MemRegion; stack: var PointerStackChunk; src: NimString): NimString {.inline.} = + result = rawNewStringNoInit(dr, src.len) + result.len = src.len + c_memcpy(result.data, src.data, src.len + 1) + + proc genericDeepCopyAux(dr: var MemRegion; stack: var PointerStackChunk; + dest, src: pointer, mt: PNimType) = + var + d = cast[ByteAddress](dest) + s = cast[ByteAddress](src) + sysAssert(mt != nil, "genericDeepCopyAux 2") + case mt.kind + of tyString: + var x = cast[PPointer](dest) + var s2 = cast[PPointer](s)[] + if s2 == nil: + x[] = nil + else: + x[] = copyDeepString(cast[NimString](s2)) + of tySequence: + var s2 = cast[PPointer](src)[] + var seq = cast[PGenericSeq](s2) + var x = cast[PPointer](dest) + if s2 == nil: + x[] = nil + return + sysAssert(dest != nil, "genericDeepCopyAux 3") + x[] = newSeq(mt, seq.len) + var dst = cast[ByteAddress](cast[PPointer](dest)[]) + for i in 0..seq.len-1: + genericDeepCopyAux(dr, stack, + cast[pointer](dst +% i*% mt.base.size +% GenericSeqSize), + cast[pointer](cast[ByteAddress](s2) +% i *% mt.base.size +% + GenericSeqSize), + mt.base) + of tyObject: + # we need to copy m_type field for tyObject, as it could be empty for + # sequence reallocations: + var pint = cast[ptr PNimType](dest) + pint[] = cast[ptr PNimType](src)[] + if mt.base != nil: + genericDeepCopyAux(dr, stack, dest, src, mt.base) + genericDeepCopyAux(dr, stack, dest, src, mt.node) + of tyTuple: + genericDeepCopyAux(dr, stack, dest, src, mt.node) + of tyArray, tyArrayConstr: + for i in 0..(mt.size div mt.base.size)-1: + genericDeepCopyAux(dr, stack, + cast[pointer](d +% i*% mt.base.size), + cast[pointer](s +% i*% mt.base.size), mt.base) + of tyRef: + let s2 = cast[PPointer](src)[] + if s2 == nil: + cast[PPointer](dest)[] = nil + else: + # we modify the header of the cell temporarily; instead of the type + # field we store a forwarding pointer. XXX This is bad when the cloning + # fails due to OOM etc. + let x = usrToCell(s2) + let forw = cast[int](x.typ) + if (forw and 1) == 1: + # we stored a forwarding pointer, so let's use that: + let z = cast[pointer](forw and not 1) + unsureAsgnRef(cast[PPointer](dest), z) + else: + let realType = x.typ + let z = newObj(realType, realType.base.size) + + unsureAsgnRef(cast[PPointer](dest), z) + x.typ = cast[PNimType](cast[int](z) or 1) + genericDeepCopyAux(dr, stack, z, s2, realType.base) + x.typ = realType + else: + copyMem(dest, src, mt.size) + + proc joinAliveDataFromRegion*(dest: var MemRegion; src: var MemRegion; + root: pointer): pointer = + # we mark the alive data and copy only alive data over to 'dest'. + # This is O(liveset) but it nicely compacts memory, so it's fine. + # We use the 'typ' field as a forwarding pointer. The forwarding + # pointers have bit 0 set, so we can disambiguate them. + # We allocate a temporary stack in 'src' that we later free: + var s: PointerStackChunk + s.len = 1 + s.data[0] = root + while s.len > 0: + var p: pointer + if s.tail == nil: + p = s.data[s.len-1] + dec s.len + else: + p = s.tail.data[s.tail.len-1] + dec s.tail.len + if s.tail.len == 0: + unlink(s, s.tail) + +proc rawNewObj(r: var MemRegion, typ: PNimType, size: int): pointer = + var res = cast[ptr ObjHeader](alloc(r, size + sizeof(ObjHeader))) + res.typ = typ + if typ.finalizer != nil: + res.nextFinal = r.head.head + r.head.head = res + result = res +! sizeof(ObjHeader) + +proc newObj(typ: PNimType, size: int): pointer {.compilerRtl.} = + result = rawNewObj(tlRegion, typ, size) + zeroMem(result, size) + when defined(memProfiler): nimProfile(size) + +proc newObjNoInit(typ: PNimType, size: int): pointer {.compilerRtl.} = + result = rawNewObj(tlRegion, typ, size) + when defined(memProfiler): nimProfile(size) + +proc newSeq(typ: PNimType, len: int): pointer {.compilerRtl.} = + let size = addInt(mulInt(len, typ.base.size), GenericSeqSize) + result = newObj(typ, size) + cast[PGenericSeq](result).len = len + cast[PGenericSeq](result).reserved = len + +proc newObjRC1(typ: PNimType, size: int): pointer {.compilerRtl.} = + result = rawNewObj(tlRegion, typ, size) + zeroMem(result, size) + +proc newSeqRC1(typ: PNimType, len: int): pointer {.compilerRtl.} = + let size = addInt(mulInt(len, typ.base.size), GenericSeqSize) + result = newObj(typ, size) + cast[PGenericSeq](result).len = len + cast[PGenericSeq](result).reserved = len + +proc growObj(region: var MemRegion; old: pointer, newsize: int): pointer = + let typ = cast[ptr ObjHeader](old -! sizeof(ObjHeader)).typ + result = rawNewObj(region, typ, newsize) + let elemSize = if typ.kind == tyString: 1 else: typ.base.size + let oldsize = cast[PGenericSeq](old).len*elemSize + GenericSeqSize + copyMem(result, old, oldsize) + zeroMem(result +! oldsize, newsize-oldsize) + +proc growObj(old: pointer, newsize: int): pointer {.rtl.} = + result = growObj(tlRegion, old, newsize) + +proc unsureAsgnRef(dest: PPointer, src: pointer) {.compilerproc, inline.} = + dest[] = src +proc asgnRef(dest: PPointer, src: pointer) {.compilerproc, inline.} = + dest[] = src +proc asgnRefNoCycle(dest: PPointer, src: pointer) {.compilerproc, inline.} = + dest[] = src + +proc alloc(size: Natural): pointer = + result = cmalloc(size) + if result == nil: raiseOutOfMem() +proc alloc0(size: Natural): pointer = + result = alloc(size) + zeroMem(result, size) +proc realloc(p: pointer, newsize: Natural): pointer = + result = crealloc(p, newsize) + if result == nil: raiseOutOfMem() +proc dealloc(p: pointer) = cfree(p) + +proc allocShared(size: Natural): pointer = + result = cmalloc(size) + if result == nil: raiseOutOfMem() +proc allocShared0(size: Natural): pointer = + result = alloc(size) + zeroMem(result, size) +proc reallocShared(p: pointer, newsize: Natural): pointer = + result = crealloc(p, newsize) + if result == nil: raiseOutOfMem() +proc deallocShared(p: pointer) = cfree(p) + +when hasThreadSupport: + proc getFreeSharedMem(): int = 0 + proc getTotalSharedMem(): int = 0 + proc getOccupiedSharedMem(): int = 0 + +proc GC_disable() = discard +proc GC_enable() = discard +proc GC_fullCollect() = discard +proc GC_setStrategy(strategy: GC_Strategy) = discard +proc GC_enableMarkAndSweep() = discard +proc GC_disableMarkAndSweep() = discard +proc GC_getStatistics(): string = return "" + +proc getOccupiedMem(): int = + result = tlRegion.totalSize - tlRegion.remaining +proc getFreeMem(): int = tlRegion.remaining +proc getTotalMem(): int = + result = tlRegion.totalSize + +proc setStackBottom(theStackBottom: pointer) = discard diff --git a/lib/system/mmdisp.nim b/lib/system/mmdisp.nim index 76c1b476e..d7010a1a3 100644 --- a/lib/system/mmdisp.nim +++ b/lib/system/mmdisp.nim @@ -526,13 +526,17 @@ elif defined(nogc): include "system/cellsets" else: - include "system/alloc" + when not defined(gcStack): + include "system/alloc" - include "system/cellsets" - when not leakDetector and not useCellIds: - sysAssert(sizeof(Cell) == sizeof(FreeCell), "sizeof FreeCell") + include "system/cellsets" + when not leakDetector and not useCellIds: + sysAssert(sizeof(Cell) == sizeof(FreeCell), "sizeof FreeCell") when compileOption("gc", "v2"): include "system/gc2" + elif defined(gcStack): + # XXX due to bootstrapping reasons, we cannot use compileOption("gc", "stack") here + include "system/gc_stack" elif defined(gcMarkAndSweep): # XXX use 'compileOption' here include "system/gc_ms" diff --git a/lib/system/osalloc.nim b/lib/system/osalloc.nim new file mode 100644 index 000000000..78410d716 --- /dev/null +++ b/lib/system/osalloc.nim @@ -0,0 +1,171 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2016 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +proc roundup(x, v: int): int {.inline.} = + result = (x + (v-1)) and not (v-1) + sysAssert(result >= x, "roundup: result < x") + #return ((-x) and (v-1)) +% x + +sysAssert(roundup(14, PageSize) == PageSize, "invalid PageSize") +sysAssert(roundup(15, 8) == 16, "roundup broken") +sysAssert(roundup(65, 8) == 72, "roundup broken 2") + +# ------------ platform specific chunk allocation code ----------- + +# some platforms have really weird unmap behaviour: +# unmap(blockStart, PageSize) +# really frees the whole block. Happens for Linux/PowerPC for example. Amd64 +# and x86 are safe though; Windows is special because MEM_RELEASE can only be +# used with a size of 0. We also allow unmapping to be turned off with +# -d:nimAllocNoUnmap: +const doNotUnmap = not (defined(amd64) or defined(i386)) or + defined(windows) or defined(nimAllocNoUnmap) + + +when defined(emscripten): + const + PROT_READ = 1 # page can be read + PROT_WRITE = 2 # page can be written + MAP_PRIVATE = 2'i32 # Changes are private + + var MAP_ANONYMOUS {.importc: "MAP_ANONYMOUS", header: "<sys/mman.h>".}: cint + type + PEmscriptenMMapBlock = ptr EmscriptenMMapBlock + EmscriptenMMapBlock {.pure, inheritable.} = object + realSize: int # size of previous chunk; for coalescing + realPointer: pointer # if < PageSize it is a small chunk + + proc mmap(adr: pointer, len: int, prot, flags, fildes: cint, + off: int): pointer {.header: "<sys/mman.h>".} + + proc munmap(adr: pointer, len: int) {.header: "<sys/mman.h>".} + + proc osAllocPages(block_size: int): pointer {.inline.} = + let realSize = block_size + sizeof(EmscriptenMMapBlock) + PageSize + 1 + result = mmap(nil, realSize, PROT_READ or PROT_WRITE, + MAP_PRIVATE or MAP_ANONYMOUS, -1, 0) + if result == nil or result == cast[pointer](-1): + raiseOutOfMem() + + let realPointer = result + let pos = cast[int](result) + + # Convert pointer to PageSize correct one. + var new_pos = cast[ByteAddress](pos) +% (PageSize - (pos %% PageSize)) + if (new_pos-pos)< sizeof(EmscriptenMMapBlock): + new_pos = new_pos +% PageSize + result = cast[pointer](new_pos) + + var mmapDescrPos = cast[ByteAddress](result) -% sizeof(EmscriptenMMapBlock) + + var mmapDescr = cast[EmscriptenMMapBlock](mmapDescrPos) + mmapDescr.realSize = realSize + mmapDescr.realPointer = realPointer + + c_fprintf(c_stdout, "[Alloc] size %d %d realSize:%d realPos:%d\n", block_size, cast[int](result), realSize, cast[int](realPointer)) + + proc osTryAllocPages(size: int): pointer = osAllocPages(size) + + proc osDeallocPages(p: pointer, size: int) {.inline} = + var mmapDescrPos = cast[ByteAddress](p) -% sizeof(EmscriptenMMapBlock) + var mmapDescr = cast[EmscriptenMMapBlock](mmapDescrPos) + munmap(mmapDescr.realPointer, mmapDescr.realSize) + +elif defined(posix): + const + PROT_READ = 1 # page can be read + PROT_WRITE = 2 # page can be written + MAP_PRIVATE = 2'i32 # Changes are private + + when defined(macosx) or defined(bsd): + const MAP_ANONYMOUS = 0x1000 + elif defined(solaris): + const MAP_ANONYMOUS = 0x100 + else: + var + MAP_ANONYMOUS {.importc: "MAP_ANONYMOUS", header: "<sys/mman.h>".}: cint + + proc mmap(adr: pointer, len: int, prot, flags, fildes: cint, + off: int): pointer {.header: "<sys/mman.h>".} + + proc munmap(adr: pointer, len: int): cint {.header: "<sys/mman.h>".} + + proc osAllocPages(size: int): pointer {.inline.} = + result = mmap(nil, size, PROT_READ or PROT_WRITE, + MAP_PRIVATE or MAP_ANONYMOUS, -1, 0) + if result == nil or result == cast[pointer](-1): + raiseOutOfMem() + + proc osTryAllocPages(size: int): pointer {.inline.} = + result = mmap(nil, size, PROT_READ or PROT_WRITE, + MAP_PRIVATE or MAP_ANONYMOUS, -1, 0) + if result == cast[pointer](-1): result = nil + + proc osDeallocPages(p: pointer, size: int) {.inline} = + when reallyOsDealloc: discard munmap(p, size) + +elif defined(windows): + const + MEM_RESERVE = 0x2000 + MEM_COMMIT = 0x1000 + MEM_TOP_DOWN = 0x100000 + PAGE_READWRITE = 0x04 + + MEM_DECOMMIT = 0x4000 + MEM_RELEASE = 0x8000 + + proc virtualAlloc(lpAddress: pointer, dwSize: int, flAllocationType, + flProtect: int32): pointer {. + header: "<windows.h>", stdcall, importc: "VirtualAlloc".} + + proc virtualFree(lpAddress: pointer, dwSize: int, + dwFreeType: int32) {.header: "<windows.h>", stdcall, + importc: "VirtualFree".} + + proc osAllocPages(size: int): pointer {.inline.} = + result = virtualAlloc(nil, size, MEM_RESERVE or MEM_COMMIT, + PAGE_READWRITE) + if result == nil: raiseOutOfMem() + + proc osTryAllocPages(size: int): pointer {.inline.} = + result = virtualAlloc(nil, size, MEM_RESERVE or MEM_COMMIT, + PAGE_READWRITE) + + proc osDeallocPages(p: pointer, size: int) {.inline.} = + # according to Microsoft, 0 is the only correct value for MEM_RELEASE: + # This means that the OS has some different view over how big the block is + # that we want to free! So, we cannot reliably release the memory back to + # Windows :-(. We have to live with MEM_DECOMMIT instead. + # Well that used to be the case but MEM_DECOMMIT fragments the address + # space heavily, so we now treat Windows as a strange unmap target. + when reallyOsDealloc: virtualFree(p, 0, MEM_RELEASE) + #VirtualFree(p, size, MEM_DECOMMIT) + +elif hostOS == "standalone": + var + theHeap: array[1024*PageSize, float64] # 'float64' for alignment + bumpPointer = cast[int](addr theHeap) + + proc osAllocPages(size: int): pointer {.inline.} = + if size+bumpPointer < cast[int](addr theHeap) + sizeof(theHeap): + result = cast[pointer](bumpPointer) + inc bumpPointer, size + else: + raiseOutOfMem() + + proc osTryAllocPages(size: int): pointer {.inline.} = + if size+bumpPointer < cast[int](addr theHeap) + sizeof(theHeap): + result = cast[pointer](bumpPointer) + inc bumpPointer, size + + proc osDeallocPages(p: pointer, size: int) {.inline.} = + if bumpPointer-size == cast[int](p): + dec bumpPointer, size +else: + {.error: "Port memory manager to your platform".} diff --git a/lib/system/sysio.nim b/lib/system/sysio.nim index d0bba6775..3c34215ac 100644 --- a/lib/system/sysio.nim +++ b/lib/system/sysio.nim @@ -58,6 +58,8 @@ proc readBytes(f: File, a: var openArray[int8|uint8], start, len: Natural): int result = readBuffer(f, addr(a[start]), len) proc readChars(f: File, a: var openArray[char], start, len: Natural): int = + if (start + len) > len(a): + raiseEIO("buffer overflow: (start+len) > length of openarray buffer") result = readBuffer(f, addr(a[start]), len) proc write(f: File, c: cstring) = fputs(c, f) diff --git a/lib/system/syslocks.nim b/lib/system/syslocks.nim index 1551a4121..c3e23052b 100644 --- a/lib/system/syslocks.nim +++ b/lib/system/syslocks.nim @@ -9,42 +9,46 @@ # Low level system locks and condition vars. +{.push stackTrace: off.} + when defined(Windows): type Handle = int - SysLock {.final, pure.} = object # CRITICAL_SECTION in WinApi + + SysLock {.importc: "CRITICAL_SECTION", + header: "<windows.h>", final, pure.} = object # CRITICAL_SECTION in WinApi DebugInfo: pointer LockCount: int32 RecursionCount: int32 OwningThread: int LockSemaphore: int - Reserved: int32 + SpinCount: int SysCond = Handle {.deprecated: [THandle: Handle, TSysLock: SysLock, TSysCond: SysCond].} - proc initSysLock(L: var SysLock) {.stdcall, noSideEffect, - dynlib: "kernel32", importc: "InitializeCriticalSection".} + proc initSysLock(L: var SysLock) {.importc: "InitializeCriticalSection", + header: "<windows.h>".} ## Initializes the lock `L`. - proc tryAcquireSysAux(L: var SysLock): int32 {.stdcall, noSideEffect, - dynlib: "kernel32", importc: "TryEnterCriticalSection".} + proc tryAcquireSysAux(L: var SysLock): int32 {.importc: "TryEnterCriticalSection", + header: "<windows.h>".} ## Tries to acquire the lock `L`. proc tryAcquireSys(L: var SysLock): bool {.inline.} = result = tryAcquireSysAux(L) != 0'i32 - proc acquireSys(L: var SysLock) {.stdcall, noSideEffect, - dynlib: "kernel32", importc: "EnterCriticalSection".} + proc acquireSys(L: var SysLock) {.importc: "EnterCriticalSection", + header: "<windows.h>".} ## Acquires the lock `L`. - proc releaseSys(L: var SysLock) {.stdcall, noSideEffect, - dynlib: "kernel32", importc: "LeaveCriticalSection".} + proc releaseSys(L: var SysLock) {.importc: "LeaveCriticalSection", + header: "<windows.h>".} ## Releases the lock `L`. - proc deinitSys(L: var SysLock) {.stdcall, noSideEffect, - dynlib: "kernel32", importc: "DeleteCriticalSection".} + proc deinitSys(L: var SysLock) {.importc: "DeleteCriticalSection", + header: "<windows.h>".} proc createEvent(lpEventAttributes: pointer, bManualReset, bInitialState: int32, @@ -84,17 +88,16 @@ else: #include <pthread.h>""".} = object SysLockType = distinct cint - proc SysLockType_Reentrant: SysLockType = - {.emit: "`result` = PTHREAD_MUTEX_RECURSIVE;".} - proc initSysLock(L: var SysLock, attr: ptr SysLockAttr = nil) {. importc: "pthread_mutex_init", header: "<pthread.h>", noSideEffect.} - proc initSysLockAttr(a: var SysLockAttr) {. - importc: "pthread_mutexattr_init", header: "<pthread.h>", noSideEffect.} - - proc setSysLockType(a: var SysLockAttr, t: SysLockType) {. - importc: "pthread_mutexattr_settype", header: "<pthread.h>", noSideEffect.} + when insideRLocksModule: + proc SysLockType_Reentrant: SysLockType = + {.emit: "`result` = PTHREAD_MUTEX_RECURSIVE;".} + proc initSysLockAttr(a: var SysLockAttr) {. + importc: "pthread_mutexattr_init", header: "<pthread.h>", noSideEffect.} + proc setSysLockType(a: var SysLockAttr, t: SysLockType) {. + importc: "pthread_mutexattr_settype", header: "<pthread.h>", noSideEffect.} proc acquireSys(L: var SysLock) {.noSideEffect, importc: "pthread_mutex_lock", header: "<pthread.h>".} @@ -109,12 +112,14 @@ else: proc deinitSys(L: var SysLock) {.noSideEffect, importc: "pthread_mutex_destroy", header: "<pthread.h>".} - proc initSysCond(cond: var SysCond, cond_attr: pointer = nil) {. - importc: "pthread_cond_init", header: "<pthread.h>", noSideEffect.} - proc waitSysCond(cond: var SysCond, lock: var SysLock) {. - importc: "pthread_cond_wait", header: "<pthread.h>", noSideEffect.} - proc signalSysCond(cond: var SysCond) {. - importc: "pthread_cond_signal", header: "<pthread.h>", noSideEffect.} - - proc deinitSysCond(cond: var SysCond) {.noSideEffect, - importc: "pthread_cond_destroy", header: "<pthread.h>".} + when not insideRLocksModule: + proc initSysCond(cond: var SysCond, cond_attr: pointer = nil) {. + importc: "pthread_cond_init", header: "<pthread.h>", noSideEffect.} + proc waitSysCond(cond: var SysCond, lock: var SysLock) {. + importc: "pthread_cond_wait", header: "<pthread.h>", noSideEffect.} + proc signalSysCond(cond: var SysCond) {. + importc: "pthread_cond_signal", header: "<pthread.h>", noSideEffect.} + proc deinitSysCond(cond: var SysCond) {.noSideEffect, + importc: "pthread_cond_destroy", header: "<pthread.h>".} + +{.pop.} diff --git a/lib/system/sysstr.nim b/lib/system/sysstr.nim index e2137e8f4..f8b93a2c3 100644 --- a/lib/system/sysstr.nim +++ b/lib/system/sysstr.nim @@ -228,7 +228,8 @@ proc setLengthSeq(seq: PGenericSeq, elemSize, newLen: int): PGenericSeq {. elif newLen < result.len: # we need to decref here, otherwise the GC leaks! when not defined(boehmGC) and not defined(nogc) and - not defined(gcMarkAndSweep) and not defined(gogc): + not defined(gcMarkAndSweep) and not defined(gogc) and + not defined(gcStack): when false: # compileOption("gc", "v2"): for i in newLen..result.len-1: let len0 = gch.tempStack.len diff --git a/lib/windows/winlean.nim b/lib/windows/winlean.nim index 5bd9846c9..a08a067fa 100644 --- a/lib/windows/winlean.nim +++ b/lib/windows/winlean.nim @@ -412,7 +412,7 @@ const FD_SETSIZE* = 64 MSG_PEEK* = 2 - INADDR_ANY* = 0 + INADDR_ANY* = 0'u32 INADDR_LOOPBACK* = 0x7F000001 INADDR_BROADCAST* = -1 INADDR_NONE* = -1 @@ -441,12 +441,12 @@ type sa_data: array[0..13, char] InAddr* {.importc: "IN_ADDR", header: "winsock2.h".} = object - s_addr*: int32 # IP address + s_addr*: uint32 # IP address Sockaddr_in* {.importc: "SOCKADDR_IN", header: "winsock2.h".} = object sin_family*: int16 - sin_port*: int16 # unsigned + sin_port*: uint16 sin_addr*: InAddr sin_zero*: array[0..7, char] @@ -456,7 +456,7 @@ type Sockaddr_in6* {.importc: "SOCKADDR_IN6", header: "ws2tcpip.h".} = object sin6_family*: int16 - sin6_port*: int16 # unsigned + sin6_port*: uint16 sin6_flowinfo*: int32 # unsigned sin6_addr*: In6_addr sin6_scope_id*: int32 # unsigned @@ -590,7 +590,7 @@ proc getnameinfo*(a1: ptr SockAddr, a2: SockLen, a6: SockLen, a7: cint): cint {. stdcall, importc: "getnameinfo", dynlib: ws2dll.} -proc inet_addr*(cp: cstring): int32 {. +proc inet_addr*(cp: cstring): uint32 {. stdcall, importc: "inet_addr", dynlib: ws2dll.} proc WSAFDIsSet(s: SocketHandle, set: var TFdSet): bool {. @@ -769,7 +769,7 @@ proc getQueuedCompletionStatus*(CompletionPort: Handle, dwMilliseconds: DWORD): WINBOOL{.stdcall, dynlib: "kernel32", importc: "GetQueuedCompletionStatus".} -proc getOverlappedResult*(hFile: Handle, lpOverlapped: OVERLAPPED, +proc getOverlappedResult*(hFile: Handle, lpOverlapped: POVERLAPPED, lpNumberOfBytesTransferred: var DWORD, bWait: WINBOOL): WINBOOL{. stdcall, dynlib: "kernel32", importc: "GetOverlappedResult".} @@ -840,30 +840,30 @@ if ws2 != nil: proc WSAAddressToStringA(pAddr: ptr SockAddr, addrSize: DWORD, unused: pointer, pBuff: cstring, pBuffSize: ptr DWORD): cint {.stdcall, importc, dynlib: ws2dll.} proc inet_ntop_emulated(family: cint, paddr: pointer, pStringBuffer: cstring, stringBufSize: int32): cstring {.stdcall.} = - case family - of AF_INET: - var sa: Sockaddr_in - sa.sin_family = AF_INET - sa.sin_addr = cast[ptr InAddr](paddr)[] - var bs = stringBufSize.DWORD - let r = WSAAddressToStringA(cast[ptr SockAddr](sa.addr), sa.sizeof.DWORD, nil, pStringBuffer, bs.addr) - if r != 0: - result = nil - else: - result = pStringBuffer - of AF_INET6: - var sa: Sockaddr_in6 - sa.sin6_family = AF_INET6 - sa.sin6_addr = cast[ptr In6_addr](paddr)[] - var bs = stringBufSize.DWORD - let r = WSAAddressToStringA(cast[ptr SockAddr](sa.addr), sa.sizeof.DWORD, nil, pStringBuffer, bs.addr) - if r != 0: - result = nil - else: - result = pStringBuffer + case family + of AF_INET: + var sa: Sockaddr_in + sa.sin_family = AF_INET + sa.sin_addr = cast[ptr InAddr](paddr)[] + var bs = stringBufSize.DWORD + let r = WSAAddressToStringA(cast[ptr SockAddr](sa.addr), sa.sizeof.DWORD, nil, pStringBuffer, bs.addr) + if r != 0: + result = nil else: - setLastError(ERROR_BAD_ARGUMENTS) + result = pStringBuffer + of AF_INET6: + var sa: Sockaddr_in6 + sa.sin6_family = AF_INET6 + sa.sin6_addr = cast[ptr In6_addr](paddr)[] + var bs = stringBufSize.DWORD + let r = WSAAddressToStringA(cast[ptr SockAddr](sa.addr), sa.sizeof.DWORD, nil, pStringBuffer, bs.addr) + if r != 0: result = nil + else: + result = pStringBuffer + else: + setLastError(ERROR_BAD_ARGUMENTS) + result = nil proc inet_ntop*(family: cint, paddr: pointer, pStringBuffer: cstring, stringBufSize: int32): cstring {.stdcall.} = diff --git a/lib/wrappers/openssl.nim b/lib/wrappers/openssl.nim index 05843e2d3..635d52a64 100644 --- a/lib/wrappers/openssl.nim +++ b/lib/wrappers/openssl.nim @@ -260,7 +260,7 @@ proc OpenSSL_add_all_algorithms*(){.cdecl, dynlib: DLLUtilName, importc: "OPENSS proc OPENSSL_config*(configName: cstring){.cdecl, dynlib: DLLSSLName, importc.} -when not useWinVersion: +when not useWinVersion and not defined(macosx): proc CRYPTO_set_mem_functions(a,b,c: pointer){.cdecl, dynlib: DLLUtilName, importc.} @@ -532,7 +532,7 @@ proc md5_File* (file: string): string {.raises: [IOError,Exception].} = result = hexStr(buf) -proc md5_Str* (str:string): string {.raises:[IOError].} = +proc md5_Str*(str:string): string = ##Generate MD5 hash for a string. Result is a 32 character #hex string with lowercase characters var diff --git a/readme.md b/readme.md index 27b2273f0..fbaa5fd32 100644 --- a/readme.md +++ b/readme.md @@ -1,18 +1,9 @@ -# Nim Compiler - -[](https://webchat.freenode.net/?channels=nim) -[](http://forum.nim-lang.org) -[](http://stackoverflow.com/questions/tagged/nim?sort=newest&pageSize=15) -[](https://twitter.com/nim_lang) - -[](https://travis-ci.org/nim-lang/Nim) - -[](https://gratipay.com/nim/) -[](https://www.bountysource.com/teams/nim) - +# <img src="https://raw.githubusercontent.com/nim-lang/assets/master/Art/logo-crown.png" width="36"> Nim [](https://travis-ci.org/nim-lang/Nim) This repo contains the Nim compiler, Nim's stdlib, tools and -documentation. +documentation. For more information about Nim, including downloads +and documentation for the latest release, check out +[Nim's website](http://nim-lang.org). ## Compiling Compiling the Nim compiler is quite straightforward. Because @@ -48,14 +39,15 @@ $ bin/nim c koch $ ./koch boot -d:release ``` -``koch install [dir]`` may then be used to install Nim, but lots of things -don't work then so don't do that. Add it to your PATH instead. More ``koch`` -related options are documented in [doc/koch.txt](doc/koch.txt). +You should then add the ``bin`` directory to your PATH, to make it easily +executable on your system. The above steps can be performed on Windows in a similar fashion, the ``build.bat`` and ``build64.bat`` (for x86_64 systems) are provided to be used instead of ``build.sh``. +The ``koch`` tool is the Nim build tool, more ``koch`` related options are +documented in [doc/koch.txt](doc/koch.txt). ## Nimble [Nimble](https://github.com/nim-lang/nimble) is Nim's package manager. For the @@ -66,12 +58,80 @@ the easiest way of installing Nimble is via: $ nim e install_nimble.nims ``` -## Getting help -A [forum](http://forum.nim-lang.org/) is available if you have any -questions, and you can also get help in the IRC channel on -[Freenode](irc://irc.freenode.net/nim) in #nim. If you ask questions on -[StackOverflow use the nim -tag](http://stackoverflow.com/questions/tagged/nim). +**Warning:** If you install Nimble this way, you will not be able to use binary +Nimble packages or update Nimble easily. +The [Nimble readme](https://github.com/nim-lang/nimble#installation) +provides thorough instructions on how to install Nimble, so that this isn't a +problem. + +## Community +[](https://webchat.freenode.net/?channels=nim) +[](http://forum.nim-lang.org) +[](http://stackoverflow.com/questions/tagged/nim?sort=newest&pageSize=15) +[](https://twitter.com/nim_lang) + +* The [forum](http://forum.nim-lang.org/) - the best place to ask questions and to discuss Nim. +* [IRC (Freenode#nim)](https://webchat.freenode.net/?channels=nim) - the best place to discuss + Nim in real-time, this is also where most development decision get made! +* [Stackoverflow](http://stackoverflow.com/questions/tagged/nim) + +## Contributing + +[](https://gratipay.com/nim/) +[](https://www.bountysource.com/teams/nim) + +We welcome everyone's contributions to Nim. No matter how small or large +the contribution is, anything from small spelling fixes to large modules +intended to be included in the standard library are accepted. Before +you get started, you should know the following about this repositories +structure: + +* ``bin/``, ``build/`` - these directories are empty, but are used when Nim is built. +* ``compiler/`` - the compiler source code, all the Nim source code files in this + directory implement the compiler. This also includes nimfix, and plugins + which live in ``compiler/nimfix`` and ``compiler/plugins`` + respectively. Nimsuggest used to live in the ``compiler`` directory also, + but was moved to https://github.com/nim-lang/nimsuggest. +* ``config/`` - the configuration for the compiler and documentation generator. +* ``doc/`` - the documentation files in reStructuredText format. +* ``lib/`` - where the standard library lives. + * ``pure/`` - modules in the standard library written in pure Nim. + * ``impure/`` - modules in the standard library written in pure Nim which + depend on libraries written in other languages. + * ``wrappers/`` - modules which wrap libraries written in other languages. +* ``tests/`` - contains tests for the compiler and standard library, organised by + category. +* ``tools/`` - the tools including ``niminst`` and ``nimweb``, most of these are invoked + via ``koch``. +* ``web/`` - the Nim website (http://nim-lang.org). +* ``koch.nim`` - tool used to bootstrap Nim, generate C sources, build the website, documentation + and more. + +Most importantly, the ``koch`` tool can be used to run the test suite. To do so compile it first +by executing ``nim c koch``, then execute ``./koch tests``. The test suite takes a while to run, +but you can run specific tests by specifying a category to run, for example ``./koch tests cat async``. + +Make sure that the tests all pass before +[submitting your pull request](https://help.github.com/articles/using-pull-requests/). +If you're short on time, you can +just run the tests specific to your change. Just run the category which corresponds to the change +you've made. When you create your pull request, Travis CI will verify that all the tests pass +anyway. + +If you're looking for things to do, take a look at our +[issue tracker](https://github.com/nim-lang/Nim/issues). There is always plenty of issues +labelled [``Easy``](https://github.com/nim-lang/Nim/labels/Easy), these should be a good +starting point if this is your first contribution to Nim. + +You can also help with the development of Nim by making donations. You can do so +in many ways: + +* [Gratipay](https://gratipay.com/nim/) +* [Bountysource](https://www.bountysource.com/teams/nim) +* Bitcoin - 1BXfuKM2uvoD6mbx4g5xM3eQhLzkCK77tJ + +Finally, if you have any questions feel free to submit a question on the issue tracker, +on the [Nim forum](http://forum.nim-lang.org), or on IRC. ## License The compiler and the standard library are licensed under the MIT license, diff --git a/tests/async/tasyncawait.nim b/tests/async/tasyncawait.nim index 443f769cd..9fe9507ad 100644 --- a/tests/async/tasyncawait.nim +++ b/tests/async/tasyncawait.nim @@ -45,7 +45,7 @@ proc createServer(port: TPort) {.async.} = name.sin_family = toInt(AF_INET).int16 else: name.sin_family = toInt(AF_INET) - name.sin_port = htons(int16(port)) + name.sin_port = htons(uint16(port)) name.sin_addr.s_addr = htonl(INADDR_ANY) if bindAddr(server.SocketHandle, cast[ptr SockAddr](addr(name)), sizeof(name).Socklen) < 0'i32: diff --git a/tests/async/tasyncfile.nim b/tests/async/tasyncfile.nim index 05cda5e5f..26a9bb391 100644 --- a/tests/async/tasyncfile.nim +++ b/tests/async/tasyncfile.nim @@ -24,7 +24,7 @@ proc main() {.async.} = var file = openAsync(fn, fmAppend) await file.write("\ntest2") let errorTest = file.readAll() - await errorTest + echo await errorTest doAssert errorTest.failed file.close() file = openAsync(fn, fmRead) diff --git a/tests/ccgbugs/tstringslice.nim b/tests/ccgbugs/tstringslice.nim index 00c1adf74..0ff448dcf 100644 --- a/tests/ccgbugs/tstringslice.nim +++ b/tests/ccgbugs/tstringslice.nim @@ -1,14 +1,10 @@ discard """ output: '''1 1234 -1234 2 234 -234 3 34 -34 -4 4 4''' """ @@ -21,4 +17,4 @@ const str = "123456789" for i in TRange.low .. TRange.high: echo str[i] #This works fine echo str[int(i) .. int(TRange.high)] #So does this - echo str[i .. TRange.high] #The compiler complains about this + #echo str[i .. TRange.high] #The compiler complains about this diff --git a/tests/ccgbugs/tweakopenarray.nim b/tests/ccgbugs/tweakopenarray.nim new file mode 100644 index 000000000..51d781331 --- /dev/null +++ b/tests/ccgbugs/tweakopenarray.nim @@ -0,0 +1,12 @@ +# bug #4089 + +type + Proc = proc(args: openArray[Bar]): Bar + + Foo = object + p: Proc + + Bar = object + f: Foo + +proc bar(val: Foo): Bar = Bar() diff --git a/tests/closure/tflatmap.nim b/tests/closure/tflatmap.nim new file mode 100644 index 000000000..240756424 --- /dev/null +++ b/tests/closure/tflatmap.nim @@ -0,0 +1,24 @@ + +# bug #3995 + +import future + +type + RNG* = tuple[] + Rand*[A] = (RNG) -> (A, RNG) + +proc nextInt*(r: RNG): (int, RNG) = + (1, ()) + +proc flatMap[A,B](f: Rand[A], g: A -> Rand[B]): Rand[B] = + (rng: RNG) => ( + let (a, rng2) = f(rng); + let g1 = g(a); + g1(rng2) + ) + +proc map[A,B](s: Rand[A], f: A -> B): Rand[B] = + let g: A -> Rand[B] = (a: A) => ((rng: RNG) => (f(a), rng)) + flatMap(s, g) + +let f = nextInt.map(i => i - i mod 2) diff --git a/tests/cpp/ttemplatetype.nim b/tests/cpp/ttemplatetype.nim new file mode 100644 index 000000000..7f56a225d --- /dev/null +++ b/tests/cpp/ttemplatetype.nim @@ -0,0 +1,9 @@ +type + Map {.importcpp: "std::map", header: "<map>".} [T,U] = object + +proc cInitMap(T: typedesc, U: typedesc): Map[T,U] {.importcpp: "std::map<'*1,'*2>()", nodecl.} + +proc initMap[T, U](): Map[T, U] = + result = cInitMap(T, U) + +var x: Map[cstring, cint] = initMap[cstring, cint]() diff --git a/tests/generics/tcritical.nim b/tests/generics/tcritical.nim new file mode 100644 index 000000000..e84c03618 --- /dev/null +++ b/tests/generics/tcritical.nim @@ -0,0 +1,19 @@ +discard """ + errormsg: "type mismatch" + line: 18 +""" + +# bug #3998 + +type Vec3[T] = array[3, T] + +var vg: Vec3[float32] = Vec3([1.0f, 2.0f, 3.0f]) + +echo "vg[0]: " & $vg[0] # prints 1.0 OK +echo "vg[1]: " & $vg[1] # prints 2.0 OK +echo "vg[2]: " & $vg[2] # prints 3.0 OK +echo "" + +var ve: Vec3[float64] +ve = vg # compiles, this MUST NOT be allowed! + diff --git a/tests/js/tclosures.nim b/tests/js/tclosures.nim new file mode 100644 index 000000000..c0d93814c --- /dev/null +++ b/tests/js/tclosures.nim @@ -0,0 +1,51 @@ +discard """ + action: run +""" + +import math, strutils +const consolePrefix = "jsCallbacks" + +asm """ + var callback = [] + function regCallback (fn) { callback.push (fn); } + function runCallbacks () { + var result = "\n" + var n = 0 + for (var fn in callback) { + n += 1 + result += "("+String (n)+")" + result += callback [fn] () + result += "\n" + } + return result + } + function print (text) { console.log (text); } +""" + +proc consoleprint (str:cstring): void {.importc: "print", noDecl.} +proc print* (a: varargs[string, `$`]) = consoleprint "$1: $2" % [consolePrefix, join (a, " ")] + +type CallbackProc {.importc.} = proc () : cstring + +proc regCallback (fn:CallbackProc) {.importc.} +proc runCallbacks ():cstring {.importc.} + +proc `*` (s:string, n:Natural) : string = s.repeat (n) + +proc outer (i:Natural) : (string, int) = + let c = $char (random (93) + 33) + let n = random (40) + let s = c * n + proc inner () : cstring = ("[$1]" % $n) & s & " <--" + regCallback (inner) + return (s, n) + +var expected = "\n" +for i in 1 .. 10: + let (s, n) = outer (i) + expected &= ("($1)[$2]" % [$i, $n]) & s & " <--" + expected &= "\n" + +let results = runCallbacks () + +doAssert(expected == results) diff --git a/tests/macros/typesafeprintf.nim b/tests/macros/typesafeprintf.nim new file mode 100644 index 000000000..2f4622f3e --- /dev/null +++ b/tests/macros/typesafeprintf.nim @@ -0,0 +1,48 @@ +discard """ + output: '''test 10''' +""" + +# bug #1152 + +import macros, typetraits +proc printfImpl(formatstr: cstring) {.importc: "printf", varargs.} + +iterator tokenize(format: string): char = + var i = 0 + while true: + case format[i] + of '%': + case format[i+1] + of '\0': break + else: yield format[i+1] + i.inc + of '\0': break + else: discard + i.inc + +macro printf(formatString: string{lit}, args: varargs[typed]): untyped = + var i = 0 + let err = getType(bindSym"ValueError") + for c in tokenize(formatString.strVal): + var expectedType = case c + of 'c': getType(bindSym"char") + of 'd', 'i', 'x', 'X': getType(bindSym"int") + of 'f', 'e', 'E', 'g', 'G': getType(bindSym"float") + of 's': getType(bindSym"string") + of 'p': getType(bindSym"pointer") + else: err + + var actualType = getType(args[i]) + inc i + + if sameType(expectedType, err): + error c & " is not a valid format character" + elif not sameType(expectedType, actualType): + error "type mismatch for argument " & $i & ". expected type: " & + $expectedType & ", actual type: " & $actualType + + # keep the original callsite, but use cprintf instead + result = callsite() + result[0] = bindSym"printfImpl" + +printf("test %d\n", 10) diff --git a/tests/misc/parsecomb.nim b/tests/misc/parsecomb.nim index 68a61373f..05fe97ad1 100644 --- a/tests/misc/parsecomb.nim +++ b/tests/misc/parsecomb.nim @@ -13,15 +13,15 @@ type nil type - Parser*[T, O] = distinct proc (input: Input[T]): Result[T, O] + Parser*[T, O] = proc (input: Input[T]): Result[T, O] proc unit*[T, O](v: O): Parser[T, O] = - Parser(proc (inp: Input[T]): Result[T, O] = - Result[T, O](kind: rkSuccess, output: v, input: inp)) + result = proc (inp: Input[T]): Result[T, O] = + Result[T, O](kind: rkSuccess, output: v, input: inp) proc fail*[T, O](): Parser[T, O] = - Parser(proc (inp: Input[T]): Result[T, O] = - Result(kind: rkFailure)) + result = proc (inp: Input[T]): Result[T, O] = + Result(kind: rkFailure) method runInput[T, O](self: Parser[T, O], inp: Input[T]): Result[T, O] = # hmmm .. @@ -33,39 +33,39 @@ method run*[T, O](self: Parser[T, O], toks: seq[T]): Result[T, O] = self.runInput(Input[T](toks: toks, index: 0)) method chain*[T, O1, O2](self: Parser[T, O1], nextp: proc (v: O1): Parser[T, O2]): Parser[T, O2] = - Parser(proc (inp: Input[T]): Result[T, O2] = + result = proc (inp: Input[T]): Result[T, O2] = let r = self.runInput(inp) case r.kind: of rkSuccess: nextp(r.output).runInput(r.input) of rkFailure: - Result[T, O2](kind: rkFailure)) + Result[T, O2](kind: rkFailure) method skip[T](self: Input[T], n: int): Input[T] = Input[T](toks: self.toks, index: self.index + n) proc pskip*[T](n: int): Parser[T, tuple[]] = - Parser(proc (inp: Input[T]): Result[T, tuple[]] = + result = proc (inp: Input[T]): Result[T, tuple[]] = if inp.index + n <= inp.toks.len: Result[T, tuple[]](kind: rkSuccess, output: (), input: inp.skip(n)) else: - Result[T, tuple[]](kind: rkFailure)) + Result[T, tuple[]](kind: rkFailure) proc tok*[T](t: T): Parser[T, T] = - Parser(proc (inp: Input[T]): Result[T, T] = + result = proc (inp: Input[T]): Result[T, T] = if inp.index < inp.toks.len and inp.toks[inp.index] == t: pskip[T](1).then(unit[T, T](t)).runInput(inp) else: - Result[T, T](kind: rkFailure)) + Result[T, T](kind: rkFailure) proc `+`*[T, O](first: Parser[T, O], second: Parser[T, O]): Parser[T, O] = - Parser(proc (inp: Input[T]): Result[T, O] = + result = proc (inp: Input[T]): Result[T, O] = let r = first.runInput(inp) case r.kind of rkSuccess: r else: - second.runInput(inp)) + second.runInput(inp) # end of primitives (definitions involving Parser(..)) diff --git a/tests/stdlib/tparseuints.nim b/tests/stdlib/tparseuints.nim new file mode 100644 index 000000000..5be3bcbd0 --- /dev/null +++ b/tests/stdlib/tparseuints.nim @@ -0,0 +1,11 @@ +discard """ + action: run +""" +import unittest, strutils + +suite "parseutils": + test "uint": + check: parseBiggestUInt("0") == 0'u64 + check: parseBiggestUInt("18446744073709551615") == 0xFFFF_FFFF_FFFF_FFFF'u64 + expect(ValueError): + discard parseBiggestUInt("18446744073709551616") diff --git a/tests/template/typedescids.nim b/tests/template/typedescids.nim new file mode 100644 index 000000000..ebed49b17 --- /dev/null +++ b/tests/template/typedescids.nim @@ -0,0 +1,17 @@ +discard """ + output: '''2 3''' +""" + +# bug #4097 + +var i {.compileTime.} = 2 + +template defineId*(t: typedesc): stmt = + const id {.genSym.} = i + static: inc(i) + proc idFor*(T: typedesc[t]): int {.inline, raises: [].} = id + +defineId(int8) +defineId(int16) + +echo idFor(int8), " ", idFor(int16) diff --git a/tests/testament/specs.nim b/tests/testament/specs.nim index bab17d2cd..3f40a5342 100644 --- a/tests/testament/specs.nim +++ b/tests/testament/specs.nim @@ -107,6 +107,15 @@ proc specDefaults*(result: var TSpec) = result.tline = 0 result.tcolumn = 0 +proc parseTargets*(value: string): set[TTarget] = + for v in value.normalize.split: + case v + of "c": result.incl(targetC) + of "cpp", "c++": result.incl(targetCpp) + of "objc": result.incl(targetObjC) + of "js": result.incl(targetJS) + else: echo "target ignored: " & v + proc parseSpec*(filename: string): TSpec = specDefaults(result) result.file = filename diff --git a/tests/testament/tester.nim b/tests/testament/tester.nim index b1e8ac099..da1c6fc2d 100644 --- a/tests/testament/tester.nim +++ b/tests/testament/tester.nim @@ -33,6 +33,7 @@ Options: --print also print results to the console --failing only show failing/ignored tests --pedantic return non-zero status code if there are failures + --targets:"c c++ js objc" run tests for specified targets (default: all) """ % resultsFile type @@ -60,6 +61,8 @@ let pegSuccess = peg"'Hint: operation successful'.*" pegOfInterest = pegLineError / pegOtherError +var targets = {low(TTarget)..high(TTarget)} + proc callCompiler(cmdTemplate, filename, options: string, target: TTarget): TSpec = let c = parseCmdLine(cmdTemplate % ["target", targetToCmd[target], @@ -275,6 +278,11 @@ proc analyzeAndConsolidateOutput(s: string): string = proc testSpec(r: var TResults, test: TTest) = # major entry point for a single test + if test.target notin targets: + r.addResult(test, "", "", reIgnored) + inc(r.skipped) + return + let tname = test.name.addFileExt(".nim") inc(r.total) var expected: TSpec @@ -336,7 +344,7 @@ proc testSpec(r: var TResults, test: TTest) = reExitCodesDiffer) return - if bufB != expectedOut: + if bufB != expectedOut and expected.action != actionRunNoSpec: if not (expected.substr and expectedOut in bufB): given.err = reOutputsDiffer r.addResult(test, expected.outp, bufB, reOutputsDiffer) @@ -394,6 +402,7 @@ proc main() = var optPrintResults = false var optFailing = false var optPedantic = false + var p = initOptParser() p.next() while p.kind == cmdLongoption: @@ -401,6 +410,7 @@ proc main() = of "print", "verbose": optPrintResults = true of "failing": optFailing = true of "pedantic": optPedantic = true + of "targets": targets = parseTargets(p.val.string) else: quit Usage p.next() if p.kind != cmdArgument: quit Usage diff --git a/tests/trmacros/tpatterns.nim b/tests/trmacros/tpatterns.nim index 6bc8772e3..907973637 100644 --- a/tests/trmacros/tpatterns.nim +++ b/tests/trmacros/tpatterns.nim @@ -1,6 +1,7 @@ discard """ output: '''48 -hel''' +hel +lo''' """ template optZero{x+x}(x: int): int = x*3 @@ -15,3 +16,8 @@ s[0] = "hello" s[0] = substr(s[0], 0, 2) echo s[0] + +# Test varargs matching +proc someVarargProc(k: varargs[string]) = doAssert(false) # this should not get called +template someVarargProcSingleArg{someVarargProc([a])}(a: string) = echo a +someVarargProc("lo") diff --git a/tests/typerel/tregionptrs.nim b/tests/typerel/tregionptrs.nim index 69a93a107..a8d2e7a6d 100644 --- a/tests/typerel/tregionptrs.nim +++ b/tests/typerel/tregionptrs.nim @@ -1,6 +1,6 @@ discard """ line: 16 - errormsg: "type mismatch: got (BPtr) but expected 'APtr'" + errormsg: "type mismatch: got (BPtr) but expected 'APtr = ptr[RegionA, int]'" """ type diff --git a/tests/vm/tanonproc.nim b/tests/vm/tanonproc.nim new file mode 100644 index 000000000..474b768ca --- /dev/null +++ b/tests/vm/tanonproc.nim @@ -0,0 +1,52 @@ +discard """ + output: '''`Test`''' +""" + +# bug #3561 + +import macros, future, strutils + +type + Option[T] = ref object + case valid: bool + of true: + data: T + else: + discard + +proc some[T](v: T): Option[T] = Option[T](valid: true, data: v) +proc none[T](v: T): Option[T] = Option[T](valid: false) +proc none(T: typedesc): Option[T] = Option[T](valid: false) + +proc map[T,U](o: Option[T], f: T -> U): Option[U] = + case o.valid + of true: + f(o.data).some + else: + U.none + +proc notEmpty(o: Option[string]): Option[string] = + case o.valid + of true: + if o.data.strip == "": string.none else: o.data.strip.some + else: + o + +proc getOrElse[T](o: Option[T], def: T): T = + case o.valid + of true: + o.data + else: + def + +proc quoteStr(s: string): Option[string] = + s.some.notEmpty.map(v => "`" & v & "`") + +macro str(s: string): stmt = + let x = s.strVal + let y = quoteStr(x) + let sn = newStrLitNode(y.getOrElse("NONE")) + result = quote do: + echo `sn` + +str"Test" diff --git a/tests/vm/tinheritance.nim b/tests/vm/tinheritance.nim new file mode 100644 index 000000000..d465e22b9 --- /dev/null +++ b/tests/vm/tinheritance.nim @@ -0,0 +1,29 @@ +discard """ + msg: '''Hello fred , managed by sally +Hello sally , managed by bob''' +""" +# bug #3973 + +type + EmployeeCode = enum + ecCode1, + ecCode2 + + Person* = object of RootObj + name* : string + last_name*: string + + Employee* = object of Person + empl_code* : EmployeeCode + mgr_name* : string + +proc test() = + var + empl1 = Employee(name: "fred", last_name: "smith", mgr_name: "sally", empl_code: ecCode1) + empl2 = Employee(name: "sally", last_name: "jones", mgr_name: "bob", empl_code: ecCode2) + + echo "Hello ", empl1.name, " , managed by ", empl1.mgr_name + echo "Hello ", empl2.name, " , managed by ", empl2.mgr_name + +static: + test() diff --git a/tests/vm/tmitems.nim b/tests/vm/tmitems.nim new file mode 100644 index 000000000..4ee225eed --- /dev/null +++ b/tests/vm/tmitems.nim @@ -0,0 +1,31 @@ +discard """ + msg: '''13''' + output: '''3 +3 +3''' +""" +# bug #3731 +var list {.compileTime.} = newSeq[int]() + +macro calc*(): stmt {.immediate.} = + list.add(1) + for c in list.mitems: + c = 13 + + for c in list: + echo c + +calc() + +# bug #3859 +import macros +macro m: stmt = + var s = newseq[NimNode](3) + # var s: array[3,NimNode] # not working either + for i in 0..<s.len: s[i] = newLit(3) # works + #for x in s.mitems: x = newLit(3) + result = newStmtList() + for i in s: + result.add newCall(bindsym"echo", i) + +m() diff --git a/tests/vm/twrong_concat.nim b/tests/vm/twrong_concat.nim new file mode 100644 index 000000000..538ea2527 --- /dev/null +++ b/tests/vm/twrong_concat.nim @@ -0,0 +1,28 @@ +discard """ + output: '''success''' +""" + +# bug #3804 + +#import sequtils + +type AnObj = ref object + field: string + +#proc aBug(objs: seq[AnObj]) {.compileTime.} = +# discard objs.mapIt(it.field & " bug") + +proc sameBug(objs: seq[AnObj]) {.compileTime.} = + var strSeq = newSeq[string](objs.len) + strSeq[0] = objs[0].field & " bug" + +static: + var objs: seq[AnObj] = @[] + objs.add(AnObj(field: "hello")) + + sameBug(objs) + # sameBug(objs) + echo objs[0].field + assert(objs[0].field == "hello") # fails, because (objs[0].field == "hello bug") - mutated! + +echo "success" diff --git a/todo.txt b/todo.txt index db3604283..a6312468a 100644 --- a/todo.txt +++ b/todo.txt @@ -1,5 +1,5 @@ -nim c --gc:v2 -r -d:useSysAssert -d:useGcAssert -d:smokeCycles -d:useRealtimeGc tests/gc/gcbench +nim c --gc:v2 -r -d:useSysAssert -d:useGcAssert -d:smokeCycles -d:useRealtimeGc tests/gc/gctest - document ``this`` pragma - document and stress test ``.partial`` object declarations diff --git a/tools/website.tmpl b/tools/website.tmpl index 50152a051..6ae975839 100644 --- a/tools/website.tmpl +++ b/tools/website.tmpl @@ -6,13 +6,23 @@ <head> <meta http-equiv="content-type" content="text/html; charset=utf-8"> <title>$c.projectTitle</title> - <link rel="stylesheet" type="text/css" href="assets/style.css" /> + <link rel="stylesheet" type="text/css" href="assets/style.css?t=2221" /> <link rel="shortcut icon" href="assets/images/favicon.ico"> #if len(rss) > 0: <link href="$rss" title="Recent changes" type="application/atom+xml" rel="alternate"> #end if </head> <body> + <div id="bountysource"> + <a href="https://salt.bountysource.com/teams/nim"> + <div class="page-layout" style="padding: 2pt 2pt 2pt 30pt"> + <img src="assets/bountysource/bountysource.png" style="width: 20px; float: left;"> + <span style="margin-left: 10pt; float: left; margin-top: 2pt;">Fund Nim and help us develop it further!</span> + <img src="https://api.bountysource.com/badge/team?team_id=19072&style=raised" style="margin-top: 2pt; margin-left: 10pt"/> + </div> + </a> + </div> + <header id="head"> <div class="page-layout tall"> <div id="head-logo"></div> diff --git a/web/assets/style.css b/web/assets/style.css index 529358ec9..98bf12a9a 100644 --- a/web/assets/style.css +++ b/web/assets/style.css @@ -563,3 +563,14 @@ pre .end { background:url("images/tabEnd.png") no-repeat left bottom; } border-collapse: collapse; text-align: left; border-spacing: 0px; } + +#bountysource { + width: 100%; + height: 30px; + background-color: #19975d; +} + +#bountysource a, #bountysource a:visited, #bountysource a:hover { + color: #1a1a1a; + +} diff --git a/web/news.txt b/web/news.txt index f0f17448f..c9561c7a5 100644 --- a/web/news.txt +++ b/web/news.txt @@ -20,12 +20,25 @@ Changes affecting backwards compatibility new experimental ``this`` pragma to achieve a similar effect to what the old ``using`` statement tried to achieve. - Typeless parameters have been removed from the language since it would clash with ``using``. +- Procedures in ``mersenne.nim`` (Mersenne Twister implementation) no longer + accept and produce ``int`` values which have platform-dependent size - + they use ``uint32`` instead. +- The ``strutils.unindent`` procedure has been rewritten. Its parameters now + match the parameters of ``strutils.indent``. See issue [#4037](https://github.com/nim-lang/Nim/issues/4037) + for more details. +- The ``matchers`` module has been deprecated. See issue [#2446](https://github.com/nim-lang/Nim/issues/2446) + for more details. +- The ``json.[]`` no longer returns ``nil`` when a key is not found. Instead it + raises a ``KeyError`` exception. You can compile with the ``-d:nimJsonGet`` + flag to get a list of usages of ``[]``, as well as to restore the operator's + previous behaviour. - When using ``useMalloc``, an additional header containing the size of the allocation will be allocated, to support zeroing memory on realloc as expected by the language. With this change, ``alloc`` and ``dealloc`` are no longer aliases for ``malloc`` and ``free`` - use ``c_malloc`` and ``c_free`` if you need that. + Library Additions ----------------- |