diff options
author | Araq <rumpf_a@web.de> | 2018-02-09 16:38:53 +0100 |
---|---|---|
committer | Araq <rumpf_a@web.de> | 2018-02-09 16:38:53 +0100 |
commit | b565c0234ca18d6a77ad28db481fbbd09548f3be (patch) | |
tree | 51bef12ac422b4556631a7214292c8b5933ce83c | |
parent | 6dc8bedb984c8a57379aeb120b1ab0fe6a2ba07d (diff) | |
parent | 40e3b5798a66095ded8f5ba0f45ba211f07d531d (diff) | |
download | Nim-b565c0234ca18d6a77ad28db481fbbd09548f3be.tar.gz |
Merge branch 'devel' of github.com:nim-lang/Nim into devel
64 files changed, 852 insertions, 2035 deletions
diff --git a/changelog.md b/changelog.md index 5c552b486..15ee5fcfe 100644 --- a/changelog.md +++ b/changelog.md @@ -12,6 +12,15 @@ `getBool`, `getFloat`, `getBiggestInt`. Also `getInt` procedure was added. - `reExtended` is no longer default for the `re` constructor in the `re` module. +- `newAsyncSocket` taking an `AsyncFD` now runs `setBlocking(false)` on the + fd. +- The `ReadyKey` type in the selectors module now contains an ``errorCode`` + field to help distinguish between ``Event.Error`` events. +- Implemented an `accept` proc that works on a `SocketHandle` in + ``nativesockets``. +- Implemented ``getIoHandler`` proc in the ``asyncdispatch`` module that allows + you to retrieve the underlying IO Completion Port or ``Selector[AsyncData]`` + object in the specified dispatcher. - The overloading rules changed slightly so that constrained generics are preferred over unconstrained generics. (Bug #6526) - It is now possible to forward declare object types so that mutually @@ -226,3 +235,17 @@ styledEcho "Red on Green.", resetStyle - ``\n`` is now only the single line feed character like in most other programming languages. The new platform specific newline escape sequence is written as ``\p``. This change only affects the Windows platform. +- Type inference for generic type parameters involving numeric types is now symetric. See + [Generic type inference for numeric types](https://nim-lang.org/docs/manual.html#generics-generic-type-inference-fornumeric-types) + for more information. +- The ``deprecated`` pragma now supports a user-definable warning message for procs. + +```nim + +proc bar {.deprecated: "use foo instead".} = + return + +bar() +``` + +- The ``securehash`` module is now deprecated. Instead import ``std / sha1``. diff --git a/compiler/ast.nim b/compiler/ast.nim index 5c70bda18..0639ebf8e 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -10,7 +10,7 @@ # abstract syntax tree + symbol table import - msgs, hashes, nversion, options, strutils, securehash, ropes, idents, + msgs, hashes, nversion, options, strutils, std / sha1, ropes, idents, intsets, idgen type diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index 8a60143fb..3a001ebf7 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -970,7 +970,8 @@ proc genTypeInfoAuxBase(m: BModule; typ, origType: PType; addf(m.s[cfsTypeInit3], "$1.flags = $2;$n", [name, rope(flags)]) discard cgsym(m, "TNimType") if isDefined("nimTypeNames"): - var typename = typeToString(origType, preferName) + var typename = typeToString(if origType.typeInst != nil: origType.typeInst + else: origType, preferName) if typename == "ref object" and origType.skipTypes(skipPtrs).sym != nil: typename = "anon ref object from " & $origType.skipTypes(skipPtrs).sym.info addf(m.s[cfsTypeInit3], "$1.name = $2;$n", diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 630426cfd..83c8501a4 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -11,7 +11,7 @@ import ast, astalgo, hashes, trees, platform, magicsys, extccomp, options, intsets, - nversion, nimsets, msgs, securehash, bitsets, idents, types, + nversion, nimsets, msgs, std / sha1, bitsets, idents, types, ccgutils, os, ropes, math, passes, rodread, wordrecg, treetab, cgmeth, condsyms, rodutils, renderer, idgen, cgendata, ccgmerge, semfold, aliases, lowerings, semparallel, tables, sets, ndi @@ -908,7 +908,6 @@ proc addIntTypes(result: var Rope) {.inline.} = platform.CPU[targetCPU].intSize.rope]) proc getCopyright(cfile: Cfile): Rope = - const copyrightYear = "2017" if optCompileOnly in gGlobalOptions: result = ("/* Generated by Nim Compiler v$1 */$N" & "/* (c) " & copyrightYear & " Andreas Rumpf */$N" & diff --git a/compiler/commands.nim b/compiler/commands.nim index 45c0f586c..aa7d13703 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -54,7 +54,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; const HelpMessage = "Nim Compiler Version $1 [$2: $3]\n" & - "Copyright (c) 2006-2017 by Andreas Rumpf\n" + "Copyright (c) 2006-" & copyrightYear & " by Andreas Rumpf\n" const Usage = slurp"../doc/basicopt.txt".replace("//", "") diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index 62990593d..8b5a3bf3d 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -14,7 +14,7 @@ import ropes, os, strutils, osproc, platform, condsyms, options, msgs, - securehash, streams + std / sha1, streams #from debuginfo import writeDebugInfo diff --git a/compiler/gorgeimpl.nim b/compiler/gorgeimpl.nim index 2c51752cd..80302b4b5 100644 --- a/compiler/gorgeimpl.nim +++ b/compiler/gorgeimpl.nim @@ -9,7 +9,7 @@ ## Module that implements ``gorge`` for the compiler. -import msgs, securehash, os, osproc, streams, strutils, options +import msgs, std / sha1, os, osproc, streams, strutils, options proc readOutput(p: Process): (string, int) = result[0] = "" diff --git a/compiler/importer.nim b/compiler/importer.nim index 46d675b27..3d7f62464 100644 --- a/compiler/importer.nim +++ b/compiler/importer.nim @@ -11,7 +11,7 @@ import intsets, strutils, os, ast, astalgo, msgs, options, idents, rodread, lookups, - semdata, passes, renderer, modulepaths + semdata, passes, renderer, modulepaths, sigmatch proc evalImport*(c: PContext, n: PNode): PNode proc evalFrom*(c: PContext, n: PNode): PNode @@ -149,7 +149,7 @@ proc myImportModule(c: PContext, n: PNode): PSym = localError(n.info, errGenerated, "A module cannot import itself") if sfDeprecated in result.flags: message(n.info, warnDeprecated, result.name.s) - #suggestSym(n.info, result, false) + suggestSym(n.info, result, c.graph.usageSym, false) proc impMod(c: PContext; it: PNode) = let m = myImportModule(c, it) diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index d58008516..dc74fa933 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -31,7 +31,7 @@ implements the required case distinction. import ast, astalgo, strutils, hashes, trees, platform, magicsys, extccomp, options, - nversion, nimsets, msgs, securehash, bitsets, idents, types, os, + nversion, nimsets, msgs, std / sha1, bitsets, idents, types, os, times, ropes, math, passes, ccgutils, wordrecg, renderer, rodread, rodutils, intsets, cgmeth, lowerings @@ -2284,11 +2284,17 @@ proc gen(p: PProc, n: PNode, r: var TCompRes) = r.kind = resExpr of nkFloatLit..nkFloat64Lit: let f = n.floatVal - if f != f: r.res = rope"NaN" - elif f == 0.0: r.res = rope"0.0" - elif f == 0.5 * f: - if f > 0.0: r.res = rope"Infinity" - else: r.res = rope"-Infinity" + case classify(f) + of fcNaN: + r.res = rope"NaN" + of fcNegZero: + r.res = rope"-0.0" + of fcZero: + r.res = rope"0.0" + of fcInf: + r.res = rope"Infinity" + of fcNegInf: + r.res = rope"-Infinity" else: r.res = rope(f.toStrMaxPrecision) r.kind = resExpr of nkCallKinds: @@ -2390,7 +2396,7 @@ proc genHeader(target: TTarget): Rope = if target == targetJS: result = ( "/* Generated by the Nim Compiler v$1 */$n" & - "/* (c) 2017 Andreas Rumpf */$n$n" & + "/* (c) " & copyrightYear & " Andreas Rumpf */$n$n" & "var framePtr = null;$n" & "var excHandler = 0;$n" & "var lastJSError = null;$n" & @@ -2406,7 +2412,7 @@ proc genHeader(target: TTarget): Rope = else: result = ("<?php$n" & "/* Generated by the Nim Compiler v$1 */$n" & - "/* (c) 2017 Andreas Rumpf */$n$n" & + "/* (c) " & copyrightYear & " Andreas Rumpf */$n$n" & "$$framePtr = null;$n" & "$$excHandler = 0;$n" & "$$lastJSError = null;$n") % @@ -2464,7 +2470,7 @@ proc genClass(obj: PType; content: Rope; ext: string) = else: nil let result = ("<?php$n" & "/* Generated by the Nim Compiler v$# */$n" & - "/* (c) 2017 Andreas Rumpf */$n$n" & + "/* (c) " & copyrightYear & " Andreas Rumpf */$n$n" & "require_once \"nimsystem.php\";$n" & "class $#$# {$n$#$n}$n") % [rope(VersionAsString), cls, extends, content] diff --git a/compiler/modules.nim b/compiler/modules.nim index 4763ac79b..cba152e21 100644 --- a/compiler/modules.nim +++ b/compiler/modules.nim @@ -10,7 +10,7 @@ ## Implements the module handling, including the caching of modules. import - ast, astalgo, magicsys, securehash, rodread, msgs, cgendata, sigmatch, options, + ast, astalgo, magicsys, std / sha1, rodread, msgs, cgendata, sigmatch, options, idents, os, lexer, idgen, passes, syntaxes, llstream, modulegraphs when false: diff --git a/compiler/msgs.nim b/compiler/msgs.nim index 4e6226122..954883b01 100644 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -26,7 +26,7 @@ type errAtPopWithoutPush, errEmptyAsm, errInvalidIndentation, errExceptionExpected, errExceptionAlreadyHandled, errYieldNotAllowedHere, errYieldNotAllowedInTryStmt, - errInvalidNumberOfYieldExpr, errCannotReturnExpr, + errInvalidNumberOfYieldExpr, errCannotReturnExpr, errNoReturnWithReturnTypeNotAllowed, errAttemptToRedefine, errStmtInvalidAfterReturn, errStmtExpected, errInvalidLabel, errInvalidCmdLineOption, errCmdLineArgExpected, errCmdLineNoArgExpected, @@ -370,7 +370,7 @@ const errXhasSideEffects: "\'$1\' can have side effects", errIteratorExpected: "iterator within for loop context expected", errLetNeedsInit: "'let' symbol requires an initialization", - errThreadvarCannotInit: "a thread var cannot be initialized explicitly", + errThreadvarCannotInit: "a thread var cannot be initialized explicitly; this would only run for the main thread", errWrongSymbolX: "usage of \'$1\' is a user-defined error", errIllegalCaptureX: "illegal capture '$1'", errXCannotBeClosure: "'$1' cannot have 'closure' calling convention", @@ -1025,6 +1025,9 @@ proc liMessage(info: TLineInfo, msg: TMsgKind, arg: string, handleError(msg, eh, s) proc fatal*(info: TLineInfo, msg: TMsgKind, arg = "") = + # this fixes bug #7080 so that it is at least obvious 'fatal' + # was executed. + errorOutputs = {eStdOut, eStdErr} liMessage(info, msg, arg, doAbort) proc globalError*(info: TLineInfo, msg: TMsgKind, arg = "") = diff --git a/compiler/options.nim b/compiler/options.nim index 2f136a8f2..65f5a6245 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -18,6 +18,7 @@ const newScopeForIf* = true useCaas* = not defined(noCaas) noTimeMachine* = defined(avoidTimeMachine) and defined(macosx) + copyrightYear* = "2018" type # please make sure we have under 32 options # (improves code efficiency a lot!) diff --git a/compiler/parser.nim b/compiler/parser.nim index b1cfd7609..4974abcc3 100644 --- a/compiler/parser.nim +++ b/compiler/parser.nim @@ -914,7 +914,7 @@ proc parseIdentColonEquals(p: var TParser, flags: TDeclaredIdentFlags): PNode = optInd(p, result) addSon(result, parseTypeDesc(p)) else: - addSon(result, ast.emptyNode) + addSon(result, newNodeP(nkEmpty, p)) if p.tok.tokType != tkEquals and withBothOptional notin flags: parMessage(p, errColonOrEqualsExpected, p.tok) if p.tok.tokType == tkEquals: @@ -922,7 +922,7 @@ proc parseIdentColonEquals(p: var TParser, flags: TDeclaredIdentFlags): PNode = optInd(p, result) addSon(result, parseExpr(p)) else: - addSon(result, ast.emptyNode) + addSon(result, newNodeP(nkEmpty, p)) proc parseTuple(p: var TParser, indentAllowed = false): PNode = #| inlTupleDecl = 'tuple' diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim index 810c4c416..d52d8b614 100644 --- a/compiler/pragmas.nim +++ b/compiler/pragmas.nim @@ -658,7 +658,7 @@ proc pragmaGuard(c: PContext; it: PNode; kind: TSymKind): PSym = proc semCustomPragma(c: PContext, n: PNode): PNode = assert(n.kind in nkPragmaCallKinds + {nkIdent}) - + if n.kind == nkIdent: result = newTree(nkCall, n) elif n.kind == nkExprColonExpr: @@ -667,14 +667,15 @@ proc semCustomPragma(c: PContext, n: PNode): PNode = else: result = n - result = c.semOverloadedCall(c, result, n, {skTemplate}, {}) - if sfCustomPragma notin result[0].sym.flags: + let r = c.semOverloadedCall(c, result, n, {skTemplate}, {}) + if r.isNil or sfCustomPragma notin r[0].sym.flags: invalidPragma(n) - - if n.kind == nkIdent: - result = result[0] - elif n.kind == nkExprColonExpr: - result.kind = n.kind # pragma(arg) -> pragma: arg + else: + result = r + if n.kind == nkIdent: + result = result[0] + elif n.kind == nkExprColonExpr: + result.kind = n.kind # pragma(arg) -> pragma: arg proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int, validPragmas: TSpecialWords): bool = @@ -809,7 +810,10 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int, of wExplain: sym.flags.incl sfExplain of wDeprecated: - if it.kind in nkPragmaCallKinds: deprecatedStmt(c, it) + if sym != nil and sym.kind in routineKinds: + if it.kind in nkPragmaCallKinds: discard getStrLitNode(c, it) + incl(sym.flags, sfDeprecated) + elif it.kind in nkPragmaCallKinds: deprecatedStmt(c, it) elif sym != nil: incl(sym.flags, sfDeprecated) else: incl(c.module.flags, sfDeprecated) of wVarargs: @@ -1016,7 +1020,7 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int, else: sym.flags.incl sfUsed of wLiftLocals: discard else: invalidPragma(it) - else: + else: n.sons[i] = semCustomPragma(c, it) diff --git a/compiler/rodread.nim b/compiler/rodread.nim index dfa8fc52b..6e952606e 100644 --- a/compiler/rodread.nim +++ b/compiler/rodread.nim @@ -90,7 +90,7 @@ import os, options, strutils, nversion, ast, astalgo, msgs, platform, condsyms, - ropes, idents, securehash, idgen, types, rodutils, memfiles, tables + ropes, idents, std / sha1, idgen, types, rodutils, memfiles, tables type TReasonForRecompile* = enum ## all the reasons that can trigger recompilation diff --git a/compiler/rodutils.nim b/compiler/rodutils.nim index 0456e9349..0d9c7112f 100644 --- a/compiler/rodutils.nim +++ b/compiler/rodutils.nim @@ -8,18 +8,22 @@ # ## Serialization utilities for the compiler. -import strutils +import strutils, math proc c_snprintf(s: cstring; n:uint; frmt: cstring): cint {.importc: "snprintf", header: "<stdio.h>", nodecl, varargs.} proc toStrMaxPrecision*(f: BiggestFloat, literalPostfix = ""): string = - if f != f: + case classify(f) + of fcNaN: result = "NAN" - elif f == 0.0: + of fcNegZero: + result = "-0.0" & literalPostfix + of fcZero: result = "0.0" & literalPostfix - elif f == 0.5 * f: - if f > 0.0: result = "INF" - else: result = "-INF" + of fcInf: + result = "INF" + of fcNegInf: + result = "-INF" else: when defined(nimNoArrayToCstringConversion): result = newString(81) diff --git a/compiler/rodwrite.nim b/compiler/rodwrite.nim index 9aed33ec9..b9f33236d 100644 --- a/compiler/rodwrite.nim +++ b/compiler/rodwrite.nim @@ -13,7 +13,7 @@ import intsets, os, options, strutils, nversion, ast, astalgo, msgs, platform, - condsyms, ropes, idents, securehash, rodread, passes, idgen, + condsyms, ropes, idents, std / sha1, rodread, passes, idgen, rodutils, modulepaths from modulegraphs import ModuleGraph diff --git a/compiler/sem.nim b/compiler/sem.nim index 0e97a66b2..39b40b2e6 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -215,12 +215,17 @@ proc semIdentVis(c: PContext, kind: TSymKind, n: PNode, proc semIdentWithPragma(c: PContext, kind: TSymKind, n: PNode, allowed: TSymFlags): PSym -proc typeAllowedCheck(info: TLineInfo; typ: PType; kind: TSymKind) = - let t = typeAllowed(typ, kind) +proc typeAllowedCheck(info: TLineInfo; typ: PType; kind: TSymKind; + flags: TTypeAllowedFlags = {}) = + let t = typeAllowed(typ, kind, flags) if t != nil: - if t == typ: localError(info, "invalid type: '" & typeToString(typ) & "'") - else: localError(info, "invalid type: '" & typeToString(t) & - "' in this context: '" & typeToString(typ) & "'") + if t == typ: + localError(info, "invalid type: '" & typeToString(typ) & + "' for " & substr($kind, 2).toLowerAscii) + else: + localError(info, "invalid type: '" & typeToString(t) & + "' in this context: '" & typeToString(typ) & + "' for " & substr($kind, 2).toLowerAscii) proc paramsTypeCheck(c: PContext, typ: PType) {.inline.} = typeAllowedCheck(typ.n.info, typ, skProc) diff --git a/compiler/semcall.nim b/compiler/semcall.nim index c580f8fd5..01f291017 100644 --- a/compiler/semcall.nim +++ b/compiler/semcall.nim @@ -106,6 +106,7 @@ proc pickBestCandidate(c: PContext, headSymbol: PNode, errors.safeAdd(CandidateError( sym: sym, unmatchedVarParam: int z.mutabilityProblem, + firstMismatch: z.firstMismatch, diagnostics: z.diagnostics)) else: # Symbol table has been modified. Restart and pre-calculate all syms @@ -154,7 +155,20 @@ proc presentFailedCandidates(c: PContext, n: PNode, errors: CandidateErrors): else: add(candidates, err.sym.getProcHeader(prefer)) add(candidates, "\n") - if err.unmatchedVarParam != 0 and err.unmatchedVarParam < n.len: + if err.firstMismatch != 0 and n.len > 2: + add(candidates, "first type mismatch at position: " & $err.firstMismatch & + "\nrequired type: ") + if err.firstMismatch < err.sym.typ.len: + candidates.add typeToString(err.sym.typ.sons[err.firstMismatch]) + else: + candidates.add "none" + if err.firstMismatch < n.len: + candidates.add "\nbut expression '" + candidates.add renderTree(n[err.firstMismatch]) + candidates.add "' is of type: " + candidates.add typeToString(n[err.firstMismatch].typ) + candidates.add "\n" + elif err.unmatchedVarParam != 0 and err.unmatchedVarParam < n.len: add(candidates, "for a 'var' type a variable needs to be passed, but '" & renderTree(n[err.unmatchedVarParam]) & "' is immutable\n") for diag in err.diagnostics: @@ -189,7 +203,7 @@ proc bracketNotFoundError(c: PContext; n: PNode) = while symx != nil: if symx.kind in routineKinds: errors.add(CandidateError(sym: symx, - unmatchedVarParam: 0, + unmatchedVarParam: 0, firstMismatch: 0, diagnostics: nil)) symx = nextOverloadIter(o, c, headSymbol) if errors.len == 0: @@ -263,9 +277,9 @@ proc resolveOverloads(c: PContext, n, orig: PNode, if overloadsState == csEmpty and result.state == csEmpty: if nfDotField in n.flags and nfExplicitCall notin n.flags: - localError(n.info, errUndeclaredField, considerQuotedIdent(f).s) + localError(n.info, errUndeclaredField, considerQuotedIdent(f, n).s) else: - localError(n.info, errUndeclaredRoutine, considerQuotedIdent(f).s) + localError(n.info, errUndeclaredRoutine, considerQuotedIdent(f, n).s) return elif result.state != csMatch: if nfExprCall in n.flags: diff --git a/compiler/semfold.nim b/compiler/semfold.nim index 55cdc334c..59c55010f 100644 --- a/compiler/semfold.nim +++ b/compiler/semfold.nim @@ -225,6 +225,7 @@ proc evalOp(m: TMagic, n, a, b, c: PNode): PNode = of mDivF64: if getFloat(b) == 0.0: if getFloat(a) == 0.0: result = newFloatNodeT(NaN, n) + elif getFloat(b).classify == fcNegZero: result = newFloatNodeT(-Inf, n) else: result = newFloatNodeT(Inf, n) else: result = newFloatNodeT(getFloat(a) / getFloat(b), n) diff --git a/compiler/semobjconstr.nim b/compiler/semobjconstr.nim index 46e6ef25d..63a4eb99a 100644 --- a/compiler/semobjconstr.nim +++ b/compiler/semobjconstr.nim @@ -260,6 +260,10 @@ proc semObjConstr(c: PContext, n: PNode, flags: TExprFlags): PNode = result = newNodeIT(nkObjConstr, n.info, t) for child in n: result.add child + if t == nil: + localError(n.info, errGenerated, "object constructor needs an object type") + return + t = skipTypes(t, {tyGenericInst, tyAlias, tySink}) if t.kind == tyRef: t = skipTypes(t.sons[0], {tyGenericInst, tyAlias, tySink}) if t.kind != tyObject: diff --git a/compiler/sempass2.nim b/compiler/sempass2.nim index d427750e4..25525d412 100644 --- a/compiler/sempass2.nim +++ b/compiler/sempass2.nim @@ -593,17 +593,14 @@ proc trackOperand(tracked: PEffects, n: PNode, paramType: PType) = notNilCheck(tracked, n, paramType) proc breaksBlock(n: PNode): bool = - case n.kind - of nkStmtList, nkStmtListExpr: - for c in n: - if breaksBlock(c): return true - of nkBreakStmt, nkReturnStmt, nkRaiseStmt: - return true - of nkCallKinds: - if n.sons[0].kind == nkSym and sfNoReturn in n.sons[0].sym.flags: - return true - else: - discard + # sematic check doesn't allow statements after raise, break, return or + # call to noreturn proc, so it is safe to check just the last statements + var it = n + while it.kind in {nkStmtList, nkStmtListExpr} and it.len > 0: + it = it.lastSon + + result = it.kind in {nkBreakStmt, nkReturnStmt, nkRaiseStmt} or + it.kind in nkCallKinds and it[0].kind == nkSym and sfNoReturn in it[0].sym.flags proc trackCase(tracked: PEffects, n: PNode) = track(tracked, n.sons[0]) @@ -747,7 +744,7 @@ proc track(tracked: PEffects, n: PNode) = # may not look like an assignment, but it is: let arg = n.sons[1] initVarViaNew(tracked, arg) - if {tfNeedsInit} * arg.typ.lastSon.flags != {}: + if arg.typ.len != 0 and {tfNeedsInit} * arg.typ.lastSon.flags != {}: if a.sym.magic == mNewSeq and n[2].kind in {nkCharLit..nkUInt64Lit} and n[2].intVal == 0: # var s: seq[notnil]; newSeq(s, 0) is a special case! diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 287f81b76..f4614272c 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -150,9 +150,12 @@ proc discardCheck(c: PContext, result: PNode) = while n.kind in skipForDiscardable: n = n.lastSon var s = "expression '" & $n & "' is of type '" & result.typ.typeToString & "' and has to be discarded" + if result.info.line != n.info.line or + result.info.fileIndex != n.info.fileIndex: + s.add "; start of expression here: " & $result.info if result.typ.kind == tyProc: s.add "; for a function call use ()" - localError(n.info, s) + localError(n.info, s) proc semIf(c: PContext, n: PNode): PNode = result = n @@ -531,7 +534,7 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode = # this can only happen for errornous var statements: if typ == nil: continue - typeAllowedCheck(a.info, typ, symkind) + typeAllowedCheck(a.info, typ, symkind, if c.matchedConcept != nil: {taConcept} else: {}) liftTypeBoundOps(c, typ, a.info) var tup = skipTypes(typ, {tyGenericInst, tyAlias, tySink}) if a.kind == nkVarTuple: diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index fffe92b2f..090f30f16 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -24,7 +24,7 @@ type CandidateError* = object sym*: PSym - unmatchedVarParam*: int + unmatchedVarParam*, firstMismatch*: int diagnostics*: seq[string] CandidateErrors* = seq[CandidateError] @@ -67,7 +67,9 @@ type # or when the explain pragma is used. may be # triggered with an idetools command in the # future. - inheritancePenalty: int # to prefer closest father object type + inheritancePenalty: int # to prefer closest father object type + firstMismatch*: int # position of the first type mismatch for + # better error messages TTypeRelFlag* = enum trDontBind @@ -913,6 +915,25 @@ proc isCovariantPtr(c: var TCandidate, f, a: PType): bool = else: return false +proc maxNumericType(prev, candidate: PType): PType = + let c = candidate.skipTypes({tyRange}) + template greater(s) = + if c.kind in s: result = c + case prev.kind + of tyInt: greater({tyInt64}) + of tyInt8: greater({tyInt, tyInt16, tyInt32, tyInt64}) + of tyInt16: greater({tyInt, tyInt32, tyInt64}) + of tyInt32: greater({tyInt64}) + + of tyUInt: greater({tyUInt64}) + of tyUInt8: greater({tyUInt, tyUInt16, tyUInt32, tyUInt64}) + of tyUInt16: greater({tyUInt, tyUInt32, tyUInt64}) + of tyUInt32: greater({tyUInt64}) + + of tyFloat32: greater({tyFloat64, tyFloat128}) + of tyFloat64: greater({tyFloat128}) + else: discard + proc typeRelImpl(c: var TCandidate, f, aOrig: PType, flags: TTypeRelFlags = {}): TTypeRelation = # typeRel can be used to establish various relationships between types: @@ -1602,9 +1623,16 @@ proc typeRelImpl(c: var TCandidate, f, aOrig: PType, elif x.kind == tyGenericParam: result = isGeneric else: - result = typeRel(c, x, a) # check if it fits - if result > isGeneric: result = isGeneric - + # Special type binding rule for numeric types. + # See section "Generic type inference for numeric types" of the + # manual for further details: + let rebinding = maxNumericType(x.skipTypes({tyRange}), a) + if rebinding != nil: + put(c, f, rebinding) + result = isGeneric + else: + result = typeRel(c, x, a) # check if it fits + if result > isGeneric: result = isGeneric of tyStatic: let prev = PType(idTableGet(c.bindings, f)) if prev == nil: @@ -1833,8 +1861,11 @@ proc paramTypesMatchAux(m: var TCandidate, f, a: PType, var r = typeRel(m, f, a) + # This special typing rule for macros and templates is not documented + # anywhere and breaks symmetry. It's hard to get rid of though, my + # custom seqs example fails to compile without this: if r != isNone and m.calleeSym != nil and - m.calleeSym.kind in {skMacro, skTemplate}: + m.calleeSym.kind in {skMacro, skTemplate}: # XXX: duplicating this is ugly, but we cannot (!) move this # directly into typeRel using return-like templates incMatches(m, r) @@ -2220,6 +2251,7 @@ proc matchesAux(c: PContext, n, nOrig: PNode, n.sons[a], nOrig.sons[a]) if arg == nil: m.state = csNoMatch + m.firstMismatch = f return if m.baseTypeMatch: #assert(container == nil) @@ -2274,6 +2306,7 @@ proc matches*(c: PContext, n, nOrig: PNode, m: var TCandidate) = else: # no default value m.state = csNoMatch + m.firstMismatch = f break else: # use default value: diff --git a/compiler/suggest.nim b/compiler/suggest.nim index 28ce3c591..af31495aa 100644 --- a/compiler/suggest.nim +++ b/compiler/suggest.nim @@ -33,6 +33,7 @@ # included from sigmatch.nim import algorithm, prefixmatches +from wordrecg import wDeprecated when defined(nimsuggest): import passes, tables # importer @@ -479,12 +480,23 @@ proc suggestSym*(info: TLineInfo; s: PSym; usageSym: var PSym; isDecl=true) {.in isDecl: suggestResult(symToSuggest(s, isLocal=false, ideOutline, info, 100, PrefixMatch.None, false, 0)) +proc warnAboutDeprecated(info: TLineInfo; s: PSym) = + if s.kind in routineKinds: + let n = s.ast[pragmasPos] + if n.kind != nkEmpty: + for it in n: + if whichPragma(it) == wDeprecated and it.safeLen == 2 and + it[1].kind in {nkStrLit..nkTripleStrLit}: + message(info, warnDeprecated, it[1].strVal & "; " & s.name.s) + return + message(info, warnDeprecated, s.name.s) + proc markUsed(info: TLineInfo; s: PSym; usageSym: var PSym) = incl(s.flags, sfUsed) if s.kind == skEnumField and s.owner != nil: incl(s.owner.flags, sfUsed) if {sfDeprecated, sfError} * s.flags != {}: - if sfDeprecated in s.flags: message(info, warnDeprecated, s.name.s) + if sfDeprecated in s.flags: warnAboutDeprecated(info, s) if sfError in s.flags: localError(info, errWrongSymbolX, s.name.s) when defined(nimsuggest): suggestSym(info, s, usageSym, false) diff --git a/compiler/transf.nim b/compiler/transf.nim index 14ff58c90..94899c5d4 100644 --- a/compiler/transf.nim +++ b/compiler/transf.nim @@ -716,7 +716,7 @@ proc transformExceptBranch(c: PTransf, n: PNode): PTransNode = result = transformSons(c, n) if n[0].isInfixAs(): let excTypeNode = n[0][1] - let actions = newTransNode(nkStmtList, n[1].info, 2) + let actions = newTransNode(nkStmtListExpr, n[1], 2) # Generating `let exc = (excType)(getCurrentException())` # -> getCurrentException() let excCall = PTransNode(callCodegenProc("getCurrentException", ast.emptyNode)) diff --git a/compiler/trees.nim b/compiler/trees.nim index 577ea75ee..f69108942 100644 --- a/compiler/trees.nim +++ b/compiler/trees.nim @@ -118,7 +118,7 @@ proc isRange*(n: PNode): bool {.inline.} = result = true proc whichPragma*(n: PNode): TSpecialWord = - let key = if n.kind == nkExprColonExpr: n.sons[0] else: n + let key = if n.kind in nkPragmaCallKinds and n.len > 0: n.sons[0] else: n if key.kind == nkIdent: result = whichKeyword(key.ident) proc unnestStmts(n, result: PNode) = diff --git a/compiler/types.nim b/compiler/types.nim index cbbfa8631..452e95dfd 100644 --- a/compiler/types.nim +++ b/compiler/types.nim @@ -1058,11 +1058,12 @@ proc commonSuperclass*(a, b: PType): PType = y = y.sons[0] type - TTypeAllowedFlag = enum + TTypeAllowedFlag* = enum taField, - taHeap + taHeap, + taConcept - TTypeAllowedFlags = set[TTypeAllowedFlag] + TTypeAllowedFlags* = set[TTypeAllowedFlag] proc typeAllowedAux(marker: var IntSet, typ: PType, kind: TSymKind, flags: TTypeAllowedFlags = {}): PType @@ -1130,7 +1131,10 @@ proc typeAllowedAux(marker: var IntSet, typ: PType, kind: TSymKind, of tyVoid: if taField notin flags: result = t of tyTypeClasses: - if not (tfGenericTypeParam in t.flags or taField notin flags): result = t + if tfGenericTypeParam in t.flags or taConcept in flags: #or taField notin flags: + discard + elif kind notin {skParam, skResult}: + result = t of tyGenericBody, tyGenericParam, tyGenericInvocation, tyNone, tyForward, tyFromExpr: result = t @@ -1178,11 +1182,11 @@ proc typeAllowedAux(marker: var IntSet, typ: PType, kind: TSymKind, result = nil of tyUnused, tyOptAsRef: internalError("typeAllowedAux") -proc typeAllowed*(t: PType, kind: TSymKind): PType = +proc typeAllowed*(t: PType, kind: TSymKind; flags: TTypeAllowedFlags = {}): PType = # returns 'nil' on success and otherwise the part of the type that is # wrong! var marker = initIntSet() - result = typeAllowedAux(marker, t, kind, {}) + result = typeAllowedAux(marker, t, kind, flags) proc align(address, alignment: BiggestInt): BiggestInt = result = (address + (alignment - 1)) and not (alignment - 1) diff --git a/doc/lib.rst b/doc/lib.rst index 755c11899..0932b25e4 100644 --- a/doc/lib.rst +++ b/doc/lib.rst @@ -384,7 +384,7 @@ Cryptography and Hashing * `base64 <base64.html>`_ This module implements a base64 encoder and decoder. -* `securehash <securehash.html>`_ +* `sha1 <sha1.html>`_ This module implements a sha1 encoder and decoder. diff --git a/doc/manual/generics.txt b/doc/manual/generics.txt index 1ba62bb5c..30055f035 100644 --- a/doc/manual/generics.txt +++ b/doc/manual/generics.txt @@ -63,6 +63,8 @@ The following example shows a generic binary tree can be modelled: for str in preorder(root): stdout.writeLine(str) +The ``T`` is called a `generic type parameter`:idx:. + Is operator ----------- @@ -314,7 +316,7 @@ The concept types can be parametric just like the regular generic types: .. code-block:: nim ### matrixalgo.nim - + import typetraits type @@ -360,7 +362,7 @@ The concept types can be parametric just like the regular generic types: template Rows*(M: type Matrix): expr = M.M template Cols*(M: type Matrix): expr = M.N template ValueType*(M: type Matrix): typedesc = M.T - + ------------- ### usage.nim @@ -582,60 +584,61 @@ the concept body: proc log(format: static[string], varargs[distinct StringRef]) -VTable types ------------- +.. + VTable types + ------------ -Concepts allow Nim to define a great number of algorithms, using only -static polymorphism and without erasing any type information or sacrificing -any execution speed. But when polymorphic collections of objects are required, -the user must use one of the provided type erasure techniques - either common -base types or VTable types. + Concepts allow Nim to define a great number of algorithms, using only + static polymorphism and without erasing any type information or sacrificing + any execution speed. But when polymorphic collections of objects are required, + the user must use one of the provided type erasure techniques - either common + base types or VTable types. -VTable types are represented as "fat pointers" storing a reference to an -object together with a reference to a table of procs implementing a set of -required operations (the so called vtable). + VTable types are represented as "fat pointers" storing a reference to an + object together with a reference to a table of procs implementing a set of + required operations (the so called vtable). -In contrast to other programming languages, the vtable in Nim is stored -externally to the object, allowing you to create multiple different vtable -views for the same object. Thus, the polymorphism in Nim is unbounded - -any type can implement an unlimited number of protocols or interfaces not -originally envisioned by the type's author. + In contrast to other programming languages, the vtable in Nim is stored + externally to the object, allowing you to create multiple different vtable + views for the same object. Thus, the polymorphism in Nim is unbounded - + any type can implement an unlimited number of protocols or interfaces not + originally envisioned by the type's author. -Any concept type can be turned into a VTable type by using the ``vtref`` -or the ``vtptr`` compiler magics. Under the hood, these magics generate -a converter type class, which converts the regular instances of the matching -types to the corresponding VTable type. + Any concept type can be turned into a VTable type by using the ``vtref`` + or the ``vtptr`` compiler magics. Under the hood, these magics generate + a converter type class, which converts the regular instances of the matching + types to the corresponding VTable type. -.. code-block:: nim - type - IntEnumerable = vtref Enumerable[int] + .. code-block:: nim + type + IntEnumerable = vtref Enumerable[int] - MyObject = object - enumerables: seq[IntEnumerable] - streams: seq[OutputStream.vtref] + MyObject = object + enumerables: seq[IntEnumerable] + streams: seq[OutputStream.vtref] - proc addEnumerable(o: var MyObject, e: IntEnumerable) = - o.enumerables.add e + proc addEnumerable(o: var MyObject, e: IntEnumerable) = + o.enumerables.add e - proc addStream(o: var MyObject, e: OutputStream.vtref) = - o.streams.add e + proc addStream(o: var MyObject, e: OutputStream.vtref) = + o.streams.add e -The procs that will be included in the vtable are derived from the concept -body and include all proc calls for which all param types were specified as -concrete types. All such calls should include exactly one param of the type -matched against the concept (not necessarily in the first position), which -will be considered the value bound to the vtable. + The procs that will be included in the vtable are derived from the concept + body and include all proc calls for which all param types were specified as + concrete types. All such calls should include exactly one param of the type + matched against the concept (not necessarily in the first position), which + will be considered the value bound to the vtable. -Overloads will be created for all captured procs, accepting the vtable type -in the position of the captured underlying object. + Overloads will be created for all captured procs, accepting the vtable type + in the position of the captured underlying object. -Under these rules, it's possible to obtain a vtable type for a concept with -unbound type parameters or one instantiated with metatypes (type classes), -but it will include a smaller number of captured procs. A completely empty -vtable will be reported as an error. + Under these rules, it's possible to obtain a vtable type for a concept with + unbound type parameters or one instantiated with metatypes (type classes), + but it will include a smaller number of captured procs. A completely empty + vtable will be reported as an error. -The ``vtref`` magic produces types which can be bound to ``ref`` types and -the ``vtptr`` magic produced types bound to ``ptr`` types. + The ``vtref`` magic produces types which can be bound to ``ref`` types and + the ``vtptr`` magic produced types bound to ``ptr`` types. Symbol lookup in generics @@ -709,3 +712,30 @@ definition): But a ``bind`` is rarely useful because symbol binding from the definition scope is the default. + + +Generic type inference for numeric types +---------------------------------------- + +A `numeric`:idx: type is any signed, unsigned integer type, floating point +type or a subrange thereof. Let ``maxNumericType(T1, T2)`` be the "greater" +type of ``T1`` and ``T2``, that is the type that uses more bits. For +example ``maxNumericType(int32, int64) == int64``. ``maxNumericType`` is only +defined for numeric types of the same class (signed, unsigned, floating point). +``maxNumericType`` strips away subranges, +``maxNumericType(subrangeof(int16), int8)`` produces ``int16`` not its +subrange. The definition ``maxNumericType`` is extended to take a variable +number of arguments in the obvious way; +``maxNumericType(x, y, z) == maxNumericType(maxNumericType(x, y), z)``. + +A generic type parameter ``T`` that is bound to multiple numeric types ``N1``, +``N2``, ``N3``, ... during type checking is inferred to +be ``maxNumericType(N1, N2, N3, ...)``. This special type inference rule ensures +that the builtin arithmetic operators can be written in an intuitive way: + +.. code-block:: nim + proc `@`[T: int|int16|int32](x, y: T): T + + 4'i32 @ 6'i64 # inferred to be of type ``int64`` + + 4'i64 @ 6'i32 # inferred to be of type ``int64`` diff --git a/doc/manual/types.txt b/doc/manual/types.txt index 1477995dd..8a90dee05 100644 --- a/doc/manual/types.txt +++ b/doc/manual/types.txt @@ -181,11 +181,11 @@ Nim exceptions: `FloatInvalidOpError`:idx:, `FloatDivByZeroError`:idx:, and `FloatInexactError`:idx:. These exceptions inherit from the `FloatingPointError`:idx: base class. -Nim provides the pragmas `NaNChecks`:idx: and `InfChecks`:idx: to control +Nim provides the pragmas `nanChecks`:idx: and `infChecks`:idx: to control whether the IEEE exceptions are ignored or trap a Nim exception: .. code-block:: nim - {.NanChecks: on, InfChecks: on.} + {.nanChecks: on, infChecks: on.} var a = 1.0 var b = 0.0 echo b / b # raises FloatInvalidOpError @@ -195,7 +195,7 @@ In the current implementation ``FloatDivByZeroError`` and ``FloatInexactError`` are never raised. ``FloatOverflowError`` is raised instead of ``FloatDivByZeroError``. There is also a `floatChecks`:idx: pragma that is a short-cut for the -combination of ``NaNChecks`` and ``InfChecks`` pragmas. ``floatChecks`` are +combination of ``nanChecks`` and ``infChecks`` pragmas. ``floatChecks`` are turned off as default. The only operations that are affected by the ``floatChecks`` pragma are diff --git a/doc/sets_fragment.txt b/doc/sets_fragment.txt index 435807e1a..5c341b7c8 100644 --- a/doc/sets_fragment.txt +++ b/doc/sets_fragment.txt @@ -1,9 +1,11 @@ The set type models the mathematical notion of a set. The set's basetype can only be an ordinal type of a certain size, namely: + * ``int8``-``int16`` * ``uint8``/``byte``-``uint16`` * ``char`` * ``enum`` + or equivalent. The reason is that sets are implemented as high performance bit vectors. Attempting to declare a set with a larger type will result in an error: diff --git a/doc/tut1.rst b/doc/tut1.rst index f935e7935..cbfef183e 100644 --- a/doc/tut1.rst +++ b/doc/tut1.rst @@ -1032,7 +1032,7 @@ procs for these conversions. Type Conversion --------------- -Conversion between basic types is performed by using the +Conversion between numerical types is performed by using the type as a function: .. code-block:: nim diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index 42ffa236c..598b0195b 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -265,9 +265,15 @@ when defined(windows) or defined(nimdoc): setGlobalDispatcher(newDispatcher()) result = gDisp + proc getIoHandler*(disp: PDispatcher): Handle = + ## Returns the underlying IO Completion Port handle (Windows) or selector + ## (Unix) for the specified dispatcher. + return disp.ioPort + proc register*(fd: AsyncFD) = ## Registers ``fd`` with the dispatcher. let p = getGlobalDispatcher() + if createIoCompletionPort(fd.Handle, p.ioPort, cast[CompletionKey](fd), 1) == 0: raiseOSError(osLastError()) @@ -757,6 +763,9 @@ when defined(windows) or defined(nimdoc): ## Unregisters ``fd``. getGlobalDispatcher().handles.excl(fd) + proc contains*(disp: PDispatcher, fd: AsyncFD): bool = + return fd in disp.handles + {.push stackTrace:off.} proc waitableCallback(param: pointer, timerOrWaitFired: WINBOOL): void {.stdcall.} = @@ -977,7 +986,7 @@ when defined(windows) or defined(nimdoc): proc newAsyncEvent*(): AsyncEvent = ## Creates a new thread-safe ``AsyncEvent`` object. ## - ## New ``AsyncEvent`` object is not automatically registered with # TODO: Why? -- DP + ## New ``AsyncEvent`` object is not automatically registered with ## dispatcher like ``AsyncSocket``. var sa = SECURITY_ATTRIBUTES( nLength: sizeof(SECURITY_ATTRIBUTES).cint, @@ -1095,6 +1104,9 @@ else: setGlobalDispatcher(newDispatcher()) result = gDisp + proc getIoHandler*(disp: PDispatcher): Selector[AsyncData] = + return disp.selector + proc register*(fd: AsyncFD) = let p = getGlobalDispatcher() var data = newAsyncData() @@ -1110,6 +1122,9 @@ else: proc unregister*(ev: AsyncEvent) = getGlobalDispatcher().selector.unregister(SelectEvent(ev)) + + proc contains*(disp: PDispatcher, fd: AsyncFd): bool = + return fd.SocketHandle in disp.selector proc addRead*(fd: AsyncFD, cb: Callback) = let p = getGlobalDispatcher() diff --git a/lib/pure/asyncfile.nim b/lib/pure/asyncfile.nim index 9f4da16a3..6ce9e8f75 100644 --- a/lib/pure/asyncfile.nim +++ b/lib/pure/asyncfile.nim @@ -85,7 +85,7 @@ proc newAsyncFile*(fd: AsyncFd): AsyncFile = ## Creates `AsyncFile` with a previously opened file descriptor `fd`. new result result.fd = fd - register(result.fd) + register(fd) proc openAsync*(filename: string, mode = fmRead): AsyncFile = ## Opens a file specified by the path in ``filename`` using @@ -97,16 +97,16 @@ proc openAsync*(filename: string, mode = fmRead): AsyncFile = when useWinUnicode: let fd = createFileW(newWideCString(filename), desiredAccess, FILE_SHARE_READ, - nil, creationDisposition, flags, 0).AsyncFd + nil, creationDisposition, flags, 0) else: let fd = createFileA(filename, desiredAccess, FILE_SHARE_READ, - nil, creationDisposition, flags, 0).AsyncFd + nil, creationDisposition, flags, 0) - if fd.Handle == INVALID_HANDLE_VALUE: + if fd == INVALID_HANDLE_VALUE: raiseOSError(osLastError()) - result = newAsyncFile(fd) + result = newAsyncFile(fd.AsyncFd) if mode == fmAppend: result.offset = getFileSize(result) @@ -115,11 +115,11 @@ proc openAsync*(filename: string, mode = fmRead): AsyncFile = let flags = getPosixFlags(mode) # RW (Owner), RW (Group), R (Other) let perm = S_IRUSR or S_IWUSR or S_IRGRP or S_IWGRP or S_IROTH - let fd = open(filename, flags, perm).AsyncFD - if fd.cint == -1: + let fd = open(filename, flags, perm) + if fd == -1: raiseOSError(osLastError()) - result = newAsyncFile(fd) + result = newAsyncFile(fd.AsyncFd) proc readBuffer*(f: AsyncFile, buf: pointer, size: int): Future[int] = ## Read ``size`` bytes from the specified file asynchronously starting at diff --git a/lib/pure/asyncfutures.nim b/lib/pure/asyncfutures.nim index 11461d994..6df6527d5 100644 --- a/lib/pure/asyncfutures.nim +++ b/lib/pure/asyncfutures.nim @@ -342,6 +342,7 @@ proc asyncCheck*[T](future: Future[T]) = ## finished with an error. ## ## This should be used instead of ``discard`` to discard void futures. + assert(not future.isNil, "Future is nil") future.callback = proc () = if future.failed: diff --git a/lib/pure/asyncnet.nim b/lib/pure/asyncnet.nim index 93399bb40..bdbf47004 100644 --- a/lib/pure/asyncnet.nim +++ b/lib/pure/asyncnet.nim @@ -140,9 +140,16 @@ proc newAsyncSocket*(fd: AsyncFD, domain: Domain = AF_INET, sockType: SockType = SOCK_STREAM, protocol: Protocol = IPPROTO_TCP, buffered = true): AsyncSocket = ## Creates a new ``AsyncSocket`` based on the supplied params. + ## + ## The supplied ``fd``'s non-blocking state will be enabled implicitly. + ## + ## **Note**: This procedure will **NOT** register ``fd`` with the global + ## async dispatcher. You need to do this manually. If you have used + ## ``newAsyncNativeSocket`` to create ``fd`` then it's already registered. assert fd != osInvalidSocket.AsyncFD new(result) result.fd = fd.SocketHandle + fd.SocketHandle.setBlocking(false) result.isBuffered = buffered result.domain = domain result.sockType = sockType diff --git a/lib/pure/ioselects/ioselectors_epoll.nim b/lib/pure/ioselects/ioselectors_epoll.nim index 8827f239f..98b8a2b2b 100644 --- a/lib/pure/ioselects/ioselectors_epoll.nim +++ b/lib/pure/ioselects/ioselectors_epoll.nim @@ -141,7 +141,7 @@ template checkFd(s, f) = if f >= s.maxFD: raiseIOSelectorsError("Maximum number of descriptors is exhausted!") -proc registerHandle*[T](s: Selector[T], fd: SocketHandle, +proc registerHandle*[T](s: Selector[T], fd: int | SocketHandle, events: set[Event], data: T) = let fdi = int(fd) s.checkFd(fdi) @@ -156,7 +156,7 @@ proc registerHandle*[T](s: Selector[T], fd: SocketHandle, raiseIOSelectorsError(osLastError()) inc(s.count) -proc updateHandle*[T](s: Selector[T], fd: SocketHandle, events: set[Event]) = +proc updateHandle*[T](s: Selector[T], fd: int | SocketHandle, events: set[Event]) = let maskEvents = {Event.Timer, Event.Signal, Event.Process, Event.Vnode, Event.User, Event.Oneshot, Event.Error} let fdi = int(fd) @@ -392,9 +392,19 @@ proc selectInto*[T](s: Selector[T], timeout: int, let pevents = resTable[i].events var pkey = addr(s.fds[fdi]) doAssert(pkey.ident != 0) - var rkey = ReadyKey(fd: int(fdi), events: {}) + var rkey = ReadyKey(fd: fdi, events: {}) if (pevents and EPOLLERR) != 0 or (pevents and EPOLLHUP) != 0: + if (pevents and EPOLLHUP) != 0: + rkey.errorCode = ECONNRESET.OSErrorCode + else: + # Try reading SO_ERROR from fd. + var error: cint + var size = sizeof(error).SockLen + if getsockopt(fdi.SocketHandle, SOL_SOCKET, SO_ERROR, addr(error), + addr(size)) == 0'i32: + rkey.errorCode = error.OSErrorCode + rkey.events.incl(Event.Error) if (pevents and EPOLLOUT) != 0: rkey.events.incl(Event.Write) @@ -482,7 +492,7 @@ template isEmpty*[T](s: Selector[T]): bool = (s.count == 0) proc contains*[T](s: Selector[T], fd: SocketHandle|int): bool {.inline.} = - return s.fds[fd].ident != 0 + return s.fds[fd.int].ident != 0 proc getData*[T](s: Selector[T], fd: SocketHandle|int): var T = let fdi = int(fd) @@ -516,3 +526,6 @@ template withData*[T](s: Selector[T], fd: SocketHandle|int, value, body1, body1 else: body2 + +proc getFd*[T](s: Selector[T]): int = + return s.epollFd.int \ No newline at end of file diff --git a/lib/pure/ioselects/ioselectors_kqueue.nim b/lib/pure/ioselects/ioselectors_kqueue.nim index af5aa15df..10e23c072 100644 --- a/lib/pure/ioselects/ioselectors_kqueue.nim +++ b/lib/pure/ioselects/ioselectors_kqueue.nim @@ -217,7 +217,7 @@ else: raiseIOSelectorsError(osLastError()) s.changes.setLen(0) -proc registerHandle*[T](s: Selector[T], fd: SocketHandle, +proc registerHandle*[T](s: Selector[T], fd: int | SocketHandle, events: set[Event], data: T) = let fdi = int(fd) s.checkFd(fdi) @@ -235,7 +235,7 @@ proc registerHandle*[T](s: Selector[T], fd: SocketHandle, when not declared(CACHE_EVENTS): flushKQueue(s) -proc updateHandle*[T](s: Selector[T], fd: SocketHandle, +proc updateHandle*[T](s: Selector[T], fd: int | SocketHandle, events: set[Event]) = let maskEvents = {Event.Timer, Event.Signal, Event.Process, Event.Vnode, Event.User, Event.Oneshot, Event.Error} @@ -503,6 +503,7 @@ proc selectInto*[T](s: Selector[T], timeout: int, if (kevent.flags and EV_ERROR) != 0: rkey.events = {Event.Error} + rkey.errorCode = kevent.data.OSErrorCode case kevent.filter: of EVFILT_READ: @@ -569,6 +570,13 @@ proc selectInto*[T](s: Selector[T], timeout: int, doAssert(true, "Unsupported kqueue filter in the queue!") if (kevent.flags and EV_EOF) != 0: + if kevent.fflags != 0: + rkey.errorCode = kevent.fflags.OSErrorCode + else: + # This assumes we are dealing with sockets. + # TODO: For future-proofing it might be a good idea to give the + # user access to the raw `kevent`. + rkey.errorCode = ECONNRESET.OSErrorCode rkey.events.incl(Event.Error) results[k] = rkey @@ -585,7 +593,7 @@ template isEmpty*[T](s: Selector[T]): bool = (s.count == 0) proc contains*[T](s: Selector[T], fd: SocketHandle|int): bool {.inline.} = - return s.fds[fd].ident != 0 + return s.fds[fd.int].ident != 0 proc getData*[T](s: Selector[T], fd: SocketHandle|int): var T = let fdi = int(fd) @@ -619,3 +627,7 @@ template withData*[T](s: Selector[T], fd: SocketHandle|int, value, body1, body1 else: body2 + + +proc getFd*[T](s: Selector[T]): int = + return s.kqFD.int \ No newline at end of file diff --git a/lib/pure/ioselects/ioselectors_poll.nim b/lib/pure/ioselects/ioselectors_poll.nim index cc06aa592..66d52b352 100644 --- a/lib/pure/ioselects/ioselectors_poll.nim +++ b/lib/pure/ioselects/ioselectors_poll.nim @@ -141,7 +141,7 @@ template checkFd(s, f) = if f >= s.maxFD: raiseIOSelectorsError("Maximum number of descriptors is exhausted!") -proc registerHandle*[T](s: Selector[T], fd: SocketHandle, +proc registerHandle*[T](s: Selector[T], fd: int | SocketHandle, events: set[Event], data: T) = var fdi = int(fd) s.checkFd(fdi) @@ -149,7 +149,7 @@ proc registerHandle*[T](s: Selector[T], fd: SocketHandle, setKey(s, fdi, events, 0, data) if events != {}: s.pollAdd(fdi.cint, events) -proc updateHandle*[T](s: Selector[T], fd: SocketHandle, +proc updateHandle*[T](s: Selector[T], fd: int | SocketHandle, events: set[Event]) = let maskEvents = {Event.Timer, Event.Signal, Event.Process, Event.Vnode, Event.User, Event.Oneshot, Event.Error} @@ -280,7 +280,7 @@ template isEmpty*[T](s: Selector[T]): bool = (s.count == 0) proc contains*[T](s: Selector[T], fd: SocketHandle|int): bool {.inline.} = - return s.fds[fd].ident != 0 + return s.fds[fd.int].ident != 0 proc getData*[T](s: Selector[T], fd: SocketHandle|int): var T = let fdi = int(fd) @@ -314,3 +314,7 @@ template withData*[T](s: Selector[T], fd: SocketHandle|int, value, body1, body1 else: body2 + + +proc getFd*[T](s: Selector[T]): int = + return -1 \ No newline at end of file diff --git a/lib/pure/ioselects/ioselectors_select.nim b/lib/pure/ioselects/ioselectors_select.nim index c787f0070..7ed250307 100644 --- a/lib/pure/ioselects/ioselectors_select.nim +++ b/lib/pure/ioselects/ioselectors_select.nim @@ -229,7 +229,7 @@ proc delKey[T](s: Selector[T], fd: SocketHandle) = doAssert(i < FD_SETSIZE, "Descriptor [" & $int(fd) & "] is not registered in the queue!") -proc registerHandle*[T](s: Selector[T], fd: SocketHandle, +proc registerHandle*[T](s: Selector[T], fd: int | SocketHandle, events: set[Event], data: T) = when not defined(windows): let fdi = int(fd) @@ -255,7 +255,7 @@ proc registerEvent*[T](s: Selector[T], ev: SelectEvent, data: T) = IOFD_SET(ev.rsock, addr s.rSet) inc(s.count) -proc updateHandle*[T](s: Selector[T], fd: SocketHandle, +proc updateHandle*[T](s: Selector[T], fd: int | SocketHandle, events: set[Event]) = let maskEvents = {Event.Timer, Event.Signal, Event.Process, Event.Vnode, Event.User, Event.Oneshot, Event.Error} @@ -453,3 +453,6 @@ template withData*[T](s: Selector[T], fd: SocketHandle|int, value, else: body2 + +proc getFd*[T](s: Selector[T]): int = + return -1 \ No newline at end of file diff --git a/lib/pure/nativesockets.nim b/lib/pure/nativesockets.nim index 6c8701843..790ad627d 100644 --- a/lib/pure/nativesockets.nim +++ b/lib/pure/nativesockets.nim @@ -187,12 +187,12 @@ proc toSockType*(protocol: Protocol): SockType = proc newNativeSocket*(domain: Domain = AF_INET, sockType: SockType = SOCK_STREAM, protocol: Protocol = IPPROTO_TCP): SocketHandle = - ## Creates a new socket; returns `InvalidSocket` if an error occurs. + ## Creates a new socket; returns `osInvalidSocket` if an error occurs. socket(toInt(domain), toInt(sockType), toInt(protocol)) proc newNativeSocket*(domain: cint, sockType: cint, protocol: cint): SocketHandle = - ## Creates a new socket; returns `InvalidSocket` if an error occurs. + ## Creates a new socket; returns `osInvalidSocket` if an error occurs. ## ## Use this overload if one of the enums specified above does ## not contain what you need. @@ -666,6 +666,19 @@ proc selectWrite*(writefds: var seq[SocketHandle], pruneSocketSet(writefds, (wr)) +proc accept*(fd: SocketHandle): (SocketHandle, string) = + ## Accepts a new client connection. + ## + ## Returns (osInvalidSocket, "") if an error occurred. + var sockAddress: Sockaddr_in + var addrLen = sizeof(sockAddress).SockLen + var sock = accept(fd, cast[ptr SockAddr](addr(sockAddress)), + addr(addrLen)) + if sock == osInvalidSocket: + return (osInvalidSocket, "") + else: + return (sock, $inet_ntoa(sockAddress.sin_addr)) + when defined(Windows): var wsa: WSAData if wsaStartup(0x0101'i16, addr wsa) != 0: raiseOSError(osLastError()) diff --git a/lib/pure/net.nim b/lib/pure/net.nim index f348b7c51..e63f7ad55 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -753,10 +753,8 @@ proc acceptAddr*(server: Socket, client: var Socket, address: var string, ## flag is specified then this error will not be raised and instead ## accept will be called again. assert(client != nil) - var sockAddress: Sockaddr_in - var addrLen = sizeof(sockAddress).SockLen - var sock = accept(server.fd, cast[ptr SockAddr](addr(sockAddress)), - addr(addrLen)) + let ret = accept(server.fd) + let sock = ret[0] if sock == osInvalidSocket: let err = osLastError() @@ -764,6 +762,7 @@ proc acceptAddr*(server: Socket, client: var Socket, address: var string, acceptAddr(server, client, address, flags) raiseOSError(err) else: + address = ret[1] client.fd = sock client.isBuffered = server.isBuffered @@ -776,9 +775,6 @@ proc acceptAddr*(server: Socket, client: var Socket, address: var string, let ret = SSLAccept(client.sslHandle) socketError(client, ret, false) - # Client socket is set above. - address = $inet_ntoa(sockAddress.sin_addr) - when false: #defineSsl: proc acceptAddrSSL*(server: Socket, client: var Socket, address: var string): SSLAcceptResult {. diff --git a/lib/pure/os.nim b/lib/pure/os.nim index a5db4ed22..f8936f549 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -139,10 +139,15 @@ proc findExe*(exe: string, followSymlinks: bool = true; ## is added the `ExeExts <#ExeExts>`_ file extensions if it has none. ## If the system supports symlinks it also resolves them until it ## meets the actual file. This behavior can be disabled if desired. - for ext in extensions: - result = addFileExt(exe, ext) - if existsFile(result): return - var path = string(getEnv("PATH")) + template checkCurrentDir() = + for ext in extensions: + result = addFileExt(exe, ext) + if existsFile(result): return + when defined(posix): + if '/' in exe: checkCurrentDir() + else: + checkCurrentDir() + let path = string(getEnv("PATH")) for candidate in split(path, PathSep): when defined(windows): var x = (if candidate[0] == '"' and candidate[^1] == '"': @@ -824,7 +829,7 @@ iterator walkDir*(dir: string; relative=false): tuple[kind: PathComponent, path: iterator walkDirRec*(dir: string, yieldFilter = {pcFile}, followFilter = {pcDir}): string {.tags: [ReadDirEffect].} = - ## Recursively walks over the directory `dir` and yields for each file + ## Recursively walks over the directory `dir` and yields for each file ## or directory in `dir`. ## The full path for each file or directory is returned. ## **Warning**: diff --git a/lib/pure/securehash.nim b/lib/pure/securehash.nim index 57c1f3631..c6cde599a 100644 --- a/lib/pure/securehash.nim +++ b/lib/pure/securehash.nim @@ -1,195 +1,6 @@ -# -# -# The Nim Compiler -# (c) Copyright 2015 Nim Contributors -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# -import strutils -const Sha1DigestSize = 20 +## This module is a deprecated alias for the ``sha1`` module. +{.deprecated.} -type - Sha1Digest = array[0 .. Sha1DigestSize-1, uint8] - SecureHash* = distinct Sha1Digest - -# Copyright (c) 2011, Micael Hildenborg -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of Micael Hildenborg nor the -# names of its contributors may be used to endorse or promote products -# derived from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY Micael Hildenborg ''AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL Micael Hildenborg BE LIABLE FOR ANY -# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# Ported to Nim by Erik O'Leary - -type - Sha1State* = array[0 .. 5-1, uint32] - Sha1Buffer = array[0 .. 80-1, uint32] - -template clearBuffer(w: Sha1Buffer, len = 16) = - zeroMem(addr(w), len * sizeof(uint32)) - -proc init*(result: var Sha1State) = - result[0] = 0x67452301'u32 - result[1] = 0xefcdab89'u32 - result[2] = 0x98badcfe'u32 - result[3] = 0x10325476'u32 - result[4] = 0xc3d2e1f0'u32 - -proc innerHash(state: var Sha1State, w: var Sha1Buffer) = - var - a = state[0] - b = state[1] - c = state[2] - d = state[3] - e = state[4] - - var round = 0 - - template rot(value, bits: uint32): uint32 = - (value shl bits) or (value shr (32 - bits)) - - template sha1(fun, val: uint32) = - let t = rot(a, 5) + fun + e + val + w[round] - e = d - d = c - c = rot(b, 30) - b = a - a = t - - template process(body: untyped) = - w[round] = rot(w[round - 3] xor w[round - 8] xor w[round - 14] xor w[round - 16], 1) - body - inc(round) - - template wrap(dest, value: untyped) = - let v = dest + value - dest = v - - while round < 16: - sha1((b and c) or (not b and d), 0x5a827999'u32) - inc(round) - - while round < 20: - process: - sha1((b and c) or (not b and d), 0x5a827999'u32) - - while round < 40: - process: - sha1(b xor c xor d, 0x6ed9eba1'u32) - - while round < 60: - process: - sha1((b and c) or (b and d) or (c and d), 0x8f1bbcdc'u32) - - while round < 80: - process: - sha1(b xor c xor d, 0xca62c1d6'u32) - - wrap state[0], a - wrap state[1], b - wrap state[2], c - wrap state[3], d - wrap state[4], e - -proc sha1(src: cstring; len: int): Sha1Digest = - #Initialize state - var state: Sha1State - init(state) - - #Create w buffer - var w: Sha1Buffer - - #Loop through all complete 64byte blocks. - let byteLen = len - let endOfFullBlocks = byteLen - 64 - var endCurrentBlock = 0 - var currentBlock = 0 - - while currentBlock <= endOfFullBlocks: - endCurrentBlock = currentBlock + 64 - - var i = 0 - while currentBlock < endCurrentBlock: - w[i] = uint32(src[currentBlock+3]) or - uint32(src[currentBlock+2]) shl 8'u32 or - uint32(src[currentBlock+1]) shl 16'u32 or - uint32(src[currentBlock]) shl 24'u32 - currentBlock += 4 - inc(i) - - innerHash(state, w) - - #Handle last and not full 64 byte block if existing - endCurrentBlock = byteLen - currentBlock - clearBuffer(w) - var lastBlockBytes = 0 - - while lastBlockBytes < endCurrentBlock: - - var value = uint32(src[lastBlockBytes + currentBlock]) shl - ((3'u32 - uint32(lastBlockBytes and 3)) shl 3) - - w[lastBlockBytes shr 2] = w[lastBlockBytes shr 2] or value - inc(lastBlockBytes) - - w[lastBlockBytes shr 2] = w[lastBlockBytes shr 2] or ( - 0x80'u32 shl ((3'u32 - uint32(lastBlockBytes and 3)) shl 3) - ) - - if endCurrentBlock >= 56: - innerHash(state, w) - clearBuffer(w) - - w[15] = uint32(byteLen) shl 3 - innerHash(state, w) - - # Store hash in result pointer, and make sure we get in in the correct order - # on both endian models. - for i in 0 .. Sha1DigestSize-1: - result[i] = uint8((int(state[i shr 2]) shr ((3-(i and 3)) * 8)) and 255) - -proc sha1(src: string): Sha1Digest = - ## Calculate SHA1 from input string - sha1(src, src.len) - -proc secureHash*(str: string): SecureHash = SecureHash(sha1(str)) -proc secureHashFile*(filename: string): SecureHash = secureHash(readFile(filename)) -proc `$`*(self: SecureHash): string = - result = "" - for v in Sha1Digest(self): - result.add(toHex(int(v), 2)) - -proc parseSecureHash*(hash: string): SecureHash = - for i in 0 ..< Sha1DigestSize: - Sha1Digest(result)[i] = uint8(parseHexInt(hash[i*2] & hash[i*2 + 1])) - -proc `==`*(a, b: SecureHash): bool = - # Not a constant-time comparison, but that's acceptable in this context - Sha1Digest(a) == Sha1Digest(b) - - -when isMainModule: - let hash1 = secureHash("a93tgj0p34jagp9[agjp98ajrhp9aej]") - doAssert hash1 == hash1 - doAssert parseSecureHash($hash1) == hash1 +include "../std/sha1" diff --git a/lib/pure/selectors.nim b/lib/pure/selectors.nim index 518cc4bd5..ea90972fe 100644 --- a/lib/pure/selectors.nim +++ b/lib/pure/selectors.nim @@ -54,9 +54,9 @@ when defined(nimdoc): Timer, ## Timer descriptor is completed Signal, ## Signal is raised Process, ## Process is finished - Vnode, ## BSD specific file change happens + Vnode, ## BSD specific file change User, ## User event is raised - Error, ## Error happens while waiting, for descriptor + Error, ## Error occurred while waiting for descriptor VnodeWrite, ## NOTE_WRITE (BSD specific, write to file occurred) VnodeDelete, ## NOTE_DELETE (BSD specific, unlink of file occurred) VnodeExtend, ## NOTE_EXTEND (BSD specific, file extended) @@ -69,6 +69,8 @@ when defined(nimdoc): ## An object which holds result for descriptor fd* : int ## file/socket descriptor events*: set[Event] ## set of events + errorCode*: OSErrorCode ## additional error code information for + ## Error events SelectEvent* = object ## An object which holds user defined event @@ -79,13 +81,14 @@ when defined(nimdoc): proc close*[T](s: Selector[T]) = ## Closes the selector. - proc registerHandle*[T](s: Selector[T], fd: SocketHandle, events: set[Event], - data: T) = + proc registerHandle*[T](s: Selector[T], fd: int | SocketHandle, + events: set[Event], data: T) = ## Registers file/socket descriptor ``fd`` to selector ``s`` ## with events set in ``events``. The ``data`` is application-defined ## data, which will be passed when an event is triggered. - proc updateHandle*[T](s: Selector[T], fd: SocketHandle, events: set[Event]) = + proc updateHandle*[T](s: Selector[T], fd: int | SocketHandle, + events: set[Event]) = ## Update file/socket descriptor ``fd``, registered in selector ## ``s`` with new events set ``event``. @@ -221,11 +224,15 @@ when defined(nimdoc): proc contains*[T](s: Selector[T], fd: SocketHandle|int): bool {.inline.} = ## Determines whether selector contains a file descriptor. + proc getFd*[T](s: Selector[T]): int = + ## Retrieves the underlying selector's file descriptor. + ## + ## For *poll* and *select* selectors ``-1`` is returned. + else: when hasThreadSupport: import locks - type SharedArray[T] = UncheckedArray[T] @@ -234,7 +241,6 @@ else: proc deallocSharedArray[T](sa: ptr SharedArray[T]) = deallocShared(cast[pointer](sa)) - type Event* {.pure.} = enum Read, Write, Timer, Signal, Process, Vnode, User, Error, Oneshot, @@ -247,6 +253,7 @@ else: ReadyKey* = object fd* : int events*: set[Event] + errorCode*: OSErrorCode SelectorKey[T] = object ident: int @@ -264,7 +271,7 @@ else: msg.add("Internal Error\n") var err = newException(IOSelectorsException, msg) raise err - + proc setNonBlocking(fd: cint) {.inline.} = setBlocking(fd.SocketHandle, false) diff --git a/lib/pure/uri.nim b/lib/pure/uri.nim index a651530c3..d2d11253a 100644 --- a/lib/pure/uri.nim +++ b/lib/pure/uri.nim @@ -178,9 +178,8 @@ proc parseUri*(uri: string, result: var Uri) = i.inc(2) # Skip // var authority = "" i.inc parseUntil(uri, authority, {'/', '?', '#'}, i) - if authority == "": - raise newException(ValueError, "Expected authority got nothing.") - parseAuthority(authority, result) + if authority.len > 0: + parseAuthority(authority, result) else: result.opaque = true @@ -465,6 +464,15 @@ when isMainModule: doAssert test.hostname == "github.com" doAssert test.port == "dom96" doAssert test.path == "/packages" + + block: + let str = "file:///foo/bar/baz.txt" + let test = parseUri(str) + doAssert test.scheme == "file" + doAssert test.username == "" + doAssert test.hostname == "" + doAssert test.port == "" + doAssert test.path == "/foo/bar/baz.txt" # Remove dot segments tests block: diff --git a/lib/std/sha1.nim b/lib/std/sha1.nim new file mode 100644 index 000000000..b18095ff6 --- /dev/null +++ b/lib/std/sha1.nim @@ -0,0 +1,195 @@ +# +# +# The Nim Compiler +# (c) Copyright 2015 Nim Contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +import strutils + +const Sha1DigestSize = 20 + +type + Sha1Digest = array[0 .. Sha1DigestSize-1, uint8] + SecureHash* = distinct Sha1Digest + +# Copyright (c) 2011, Micael Hildenborg +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Micael Hildenborg nor the +# names of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY Micael Hildenborg ''AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL Micael Hildenborg BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Ported to Nim by Erik O'Leary + +type + Sha1State* = array[0 .. 5-1, uint32] + Sha1Buffer = array[0 .. 80-1, uint32] + +template clearBuffer(w: Sha1Buffer, len = 16) = + zeroMem(addr(w), len * sizeof(uint32)) + +proc init*(result: var Sha1State) = + result[0] = 0x67452301'u32 + result[1] = 0xefcdab89'u32 + result[2] = 0x98badcfe'u32 + result[3] = 0x10325476'u32 + result[4] = 0xc3d2e1f0'u32 + +proc innerHash(state: var Sha1State, w: var Sha1Buffer) = + var + a = state[0] + b = state[1] + c = state[2] + d = state[3] + e = state[4] + + var round = 0 + + template rot(value, bits: uint32): uint32 = + (value shl bits) or (value shr (32u32 - bits)) + + template sha1(fun, val: uint32) = + let t = rot(a, 5) + fun + e + val + w[round] + e = d + d = c + c = rot(b, 30) + b = a + a = t + + template process(body: untyped) = + w[round] = rot(w[round - 3] xor w[round - 8] xor w[round - 14] xor w[round - 16], 1) + body + inc(round) + + template wrap(dest, value: untyped) = + let v = dest + value + dest = v + + while round < 16: + sha1((b and c) or (not b and d), 0x5a827999'u32) + inc(round) + + while round < 20: + process: + sha1((b and c) or (not b and d), 0x5a827999'u32) + + while round < 40: + process: + sha1(b xor c xor d, 0x6ed9eba1'u32) + + while round < 60: + process: + sha1((b and c) or (b and d) or (c and d), 0x8f1bbcdc'u32) + + while round < 80: + process: + sha1(b xor c xor d, 0xca62c1d6'u32) + + wrap state[0], a + wrap state[1], b + wrap state[2], c + wrap state[3], d + wrap state[4], e + +proc sha1(src: cstring; len: int): Sha1Digest = + #Initialize state + var state: Sha1State + init(state) + + #Create w buffer + var w: Sha1Buffer + + #Loop through all complete 64byte blocks. + let byteLen = len + let endOfFullBlocks = byteLen - 64 + var endCurrentBlock = 0 + var currentBlock = 0 + + while currentBlock <= endOfFullBlocks: + endCurrentBlock = currentBlock + 64 + + var i = 0 + while currentBlock < endCurrentBlock: + w[i] = uint32(src[currentBlock+3]) or + uint32(src[currentBlock+2]) shl 8'u32 or + uint32(src[currentBlock+1]) shl 16'u32 or + uint32(src[currentBlock]) shl 24'u32 + currentBlock += 4 + inc(i) + + innerHash(state, w) + + #Handle last and not full 64 byte block if existing + endCurrentBlock = byteLen - currentBlock + clearBuffer(w) + var lastBlockBytes = 0 + + while lastBlockBytes < endCurrentBlock: + + var value = uint32(src[lastBlockBytes + currentBlock]) shl + ((3'u32 - uint32(lastBlockBytes and 3)) shl 3) + + w[lastBlockBytes shr 2] = w[lastBlockBytes shr 2] or value + inc(lastBlockBytes) + + w[lastBlockBytes shr 2] = w[lastBlockBytes shr 2] or ( + 0x80'u32 shl ((3'u32 - uint32(lastBlockBytes and 3)) shl 3) + ) + + if endCurrentBlock >= 56: + innerHash(state, w) + clearBuffer(w) + + w[15] = uint32(byteLen) shl 3 + innerHash(state, w) + + # Store hash in result pointer, and make sure we get in in the correct order + # on both endian models. + for i in 0 .. Sha1DigestSize-1: + result[i] = uint8((int(state[i shr 2]) shr ((3-(i and 3)) * 8)) and 255) + +proc sha1(src: string): Sha1Digest = + ## Calculate SHA1 from input string + sha1(src, src.len) + +proc secureHash*(str: string): SecureHash = SecureHash(sha1(str)) +proc secureHashFile*(filename: string): SecureHash = secureHash(readFile(filename)) +proc `$`*(self: SecureHash): string = + result = "" + for v in Sha1Digest(self): + result.add(toHex(int(v), 2)) + +proc parseSecureHash*(hash: string): SecureHash = + for i in 0 ..< Sha1DigestSize: + Sha1Digest(result)[i] = uint8(parseHexInt(hash[i*2] & hash[i*2 + 1])) + +proc `==`*(a, b: SecureHash): bool = + # Not a constant-time comparison, but that's acceptable in this context + Sha1Digest(a) == Sha1Digest(b) + + +when isMainModule: + let hash1 = secureHash("a93tgj0p34jagp9[agjp98ajrhp9aej]") + doAssert hash1 == hash1 + doAssert parseSecureHash($hash1) == hash1 diff --git a/lib/system.nim b/lib/system.nim index 4e071e802..a0330be76 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -1955,13 +1955,13 @@ const ## that you cannot compare a floating point value to this value ## and expect a reasonable result - use the `classify` procedure ## in the module ``math`` for checking for NaN. - NimMajor*: int = 0 + NimMajor* {.intdefine.}: int = 0 ## is the major number of Nim's version. - NimMinor*: int = 17 + NimMinor* {.intdefine.}: int = 17 ## is the minor number of Nim's version. - NimPatch*: int = 3 + NimPatch* {.intdefine.}: int = 3 ## is the patch number of Nim's version. NimVersion*: string = $NimMajor & "." & $NimMinor & "." & $NimPatch @@ -3025,9 +3025,9 @@ when not defined(JS): #and not defined(nimscript): proc endOfFile*(f: File): bool {.tags: [], benign.} ## Returns true iff `f` is at the end. - proc readChar*(f: File): char {.tags: [ReadIOEffect], deprecated.} - ## Reads a single character from the stream `f`. **Deprecated** since - ## version 0.16.2. Use some variant of ``readBuffer`` instead. + proc readChar*(f: File): char {.tags: [ReadIOEffect].} + ## Reads a single character from the stream `f`. Should not be used in + ## performance sensitive code. proc flushFile*(f: File) {.tags: [WriteIOEffect].} ## Flushes `f`'s buffer. @@ -3769,7 +3769,6 @@ proc failedAssertImpl*(msg: string) {.raises: [], tags: [].} = # by ``assert``. type Hide = proc (msg: string) {.noinline, raises: [], noSideEffect, tags: [].} - {.deprecated: [THide: Hide].} Hide(raiseAssert)(msg) template assert*(cond: bool, msg = "") = diff --git a/lib/system/alloc.nim b/lib/system/alloc.nim index f1f3cdb6c..33e7b3898 100644 --- a/lib/system/alloc.nim +++ b/lib/system/alloc.nim @@ -104,7 +104,7 @@ type slBitmap: array[RealFli, uint32] matrix: array[RealFli, array[MaxSli, PBigChunk]] llmem: PLLChunk - currMem, maxMem, freeMem: int # memory sizes (allocated from OS) + currMem, maxMem, freeMem, occ: int # memory sizes (allocated from OS) lastSize: int # needed for the case that OS gives us pages linearly chunkStarts: IntSet root, deleted, last, freeAvlNodes: PAvlNode @@ -421,7 +421,7 @@ const nimMaxHeap {.intdefine.} = 0 proc requestOsChunks(a: var MemRegion, size: int): PBigChunk = when not defined(emscripten): if not a.blockChunkSizeIncrease: - let usedMem = a.currMem # - a.freeMem + let usedMem = a.occ #a.currMem # - a.freeMem when nimMaxHeap != 0: if usedMem > nimMaxHeap * 1024 * 1024: raiseOutOfMem() @@ -567,7 +567,6 @@ proc splitChunk(a: var MemRegion, c: PBigChunk, size: int) = addChunkToMatrix(a, rest) proc getBigChunk(a: var MemRegion, size: int): PBigChunk = - # use first fit for now: sysAssert(size > 0, "getBigChunk 2") var size = size # roundup(size, PageSize) var fl, sl: int @@ -627,6 +626,85 @@ else: c = c.next result = true +when false: + var + rsizes: array[50_000, int] + rsizesLen: int + + proc trackSize(size: int) = + rsizes[rsizesLen] = size + inc rsizesLen + + proc untrackSize(size: int) = + for i in 0 .. rsizesLen-1: + if rsizes[i] == size: + rsizes[i] = rsizes[rsizesLen-1] + dec rsizesLen + return + c_fprintf(stdout, "%ld\n", size) + sysAssert(false, "untracked size!") +else: + template trackSize(x) = discard + template untrackSize(x) = discard + +when false: + # not yet used by the GCs + proc rawTryAlloc(a: var MemRegion; requestedSize: int): pointer = + sysAssert(allocInv(a), "rawAlloc: begin") + sysAssert(roundup(65, 8) == 72, "rawAlloc: roundup broken") + sysAssert(requestedSize >= sizeof(FreeCell), "rawAlloc: requested size too small") + var size = roundup(requestedSize, MemAlign) + inc a.occ, size + trackSize(size) + sysAssert(size >= requestedSize, "insufficient allocated size!") + #c_fprintf(stdout, "alloc; size: %ld; %ld\n", requestedSize, size) + if size <= SmallChunkSize-smallChunkOverhead(): + # allocate a small block: for small chunks, we use only its next pointer + var s = size div MemAlign + var c = a.freeSmallChunks[s] + if c == nil: + result = nil + else: + sysAssert c.size == size, "rawAlloc 6" + if c.freeList == nil: + sysAssert(c.acc + smallChunkOverhead() + size <= SmallChunkSize, + "rawAlloc 7") + result = cast[pointer](cast[ByteAddress](addr(c.data)) +% c.acc) + inc(c.acc, size) + else: + result = c.freeList + sysAssert(c.freeList.zeroField == 0, "rawAlloc 8") + c.freeList = c.freeList.next + dec(c.free, size) + sysAssert((cast[ByteAddress](result) and (MemAlign-1)) == 0, "rawAlloc 9") + if c.free < size: + listRemove(a.freeSmallChunks[s], c) + sysAssert(allocInv(a), "rawAlloc: end listRemove test") + sysAssert(((cast[ByteAddress](result) and PageMask) - smallChunkOverhead()) %% + size == 0, "rawAlloc 21") + sysAssert(allocInv(a), "rawAlloc: end small size") + else: + inc size, bigChunkOverhead() + var fl, sl: int + mappingSearch(size, fl, sl) + sysAssert((size and PageMask) == 0, "getBigChunk: unaligned chunk") + let c = findSuitableBlock(a, fl, sl) + if c != nil: + removeChunkFromMatrix2(a, c, fl, sl) + if c.size >= size + PageSize: + splitChunk(a, c, size) + # set 'used' to to true: + c.prevSize = 1 + incl(a, a.chunkStarts, pageIndex(c)) + dec(a.freeMem, size) + result = addr(c.data) + sysAssert((cast[ByteAddress](c) and (MemAlign-1)) == 0, "rawAlloc 13") + sysAssert((cast[ByteAddress](c) and PageMask) == 0, "rawAlloc: Not aligned on a page boundary") + if a.root == nil: a.root = getBottom(a) + add(a, a.root, cast[ByteAddress](result), cast[ByteAddress](result)+%size) + else: + result = nil + proc rawAlloc(a: var MemRegion, requestedSize: int): pointer = sysAssert(allocInv(a), "rawAlloc: begin") sysAssert(roundup(65, 8) == 72, "rawAlloc: roundup broken") @@ -676,6 +754,8 @@ proc rawAlloc(a: var MemRegion, requestedSize: int): pointer = sysAssert(((cast[ByteAddress](result) and PageMask) - smallChunkOverhead()) %% size == 0, "rawAlloc 21") sysAssert(allocInv(a), "rawAlloc: end small size") + inc a.occ, size + trackSize(c.size) else: size = requestedSize + bigChunkOverhead() # roundup(requestedSize+bigChunkOverhead(), PageSize) # allocate a large block @@ -687,6 +767,8 @@ proc rawAlloc(a: var MemRegion, requestedSize: int): pointer = sysAssert((cast[ByteAddress](c) and PageMask) == 0, "rawAlloc: Not aligned on a page boundary") if a.root == nil: a.root = getBottom(a) add(a, a.root, cast[ByteAddress](result), cast[ByteAddress](result)+%size) + inc a.occ, c.size + trackSize(c.size) sysAssert(isAccessible(a, result), "rawAlloc 14") sysAssert(allocInv(a), "rawAlloc: end") when logAlloc: cprintf("var pointer_%p = alloc(%ld)\n", result, requestedSize) @@ -703,6 +785,9 @@ proc rawDealloc(a: var MemRegion, p: pointer) = # `p` is within a small chunk: var c = cast[PSmallChunk](c) var s = c.size + dec a.occ, s + untrackSize(s) + sysAssert a.occ >= 0, "rawDealloc: negative occupied memory (case A)" sysAssert(((cast[ByteAddress](p) and PageMask) - smallChunkOverhead()) %% s == 0, "rawDealloc 3") var f = cast[ptr FreeCell](p) @@ -733,6 +818,9 @@ proc rawDealloc(a: var MemRegion, p: pointer) = when overwriteFree: c_memset(p, -1'i32, c.size -% bigChunkOverhead()) # free big chunk var c = cast[PBigChunk](c) + dec a.occ, c.size + untrackSize(c.size) + sysAssert a.occ >= 0, "rawDealloc: negative occupied memory (case B)" a.deleted = getBottom(a) del(a, a.root, cast[int](addr(c.data))) freeBigChunk(a, c) @@ -851,7 +939,8 @@ proc deallocOsPages(a: var MemRegion) = proc getFreeMem(a: MemRegion): int {.inline.} = result = a.freeMem proc getTotalMem(a: MemRegion): int {.inline.} = result = a.currMem proc getOccupiedMem(a: MemRegion): int {.inline.} = - result = a.currMem - a.freeMem + result = a.occ + # a.currMem - a.freeMem # ---------------------- thread memory region ------------------------------- @@ -893,7 +982,7 @@ template instantiateForRegion(allocator: untyped) = #sysAssert(result == countFreeMem()) proc getTotalMem(): int = return allocator.currMem - proc getOccupiedMem(): int = return getTotalMem() - getFreeMem() + proc getOccupiedMem(): int = return allocator.occ #getTotalMem() - getFreeMem() proc getMaxMem*(): int = return getMaxMem(allocator) # -------------------- shared heap region ---------------------------------- @@ -944,7 +1033,8 @@ template instantiateForRegion(allocator: untyped) = sharedMemStatsShared(sharedHeap.currMem) proc getOccupiedSharedMem(): int = - sharedMemStatsShared(sharedHeap.currMem - sharedHeap.freeMem) + sharedMemStatsShared(sharedHeap.occ) + #sharedMemStatsShared(sharedHeap.currMem - sharedHeap.freeMem) {.pop.} {.pop.} diff --git a/lib/system/gc_ms.nim b/lib/system/gc_ms.nim index 6f28601d0..75f9c6749 100644 --- a/lib/system/gc_ms.nim +++ b/lib/system/gc_ms.nim @@ -125,7 +125,7 @@ when BitsPerPage mod (sizeof(int)*8) != 0: {.error: "(BitsPerPage mod BitsPerUnit) should be zero!".} # forward declarations: -proc collectCT(gch: var GcHeap) {.benign.} +proc collectCT(gch: var GcHeap; size: int) {.benign.} proc forAllChildren(cell: PCell, op: WalkOp) {.benign.} proc doOperation(p: pointer, op: WalkOp) {.benign.} proc forAllChildrenAux(dest: pointer, mt: PNimType, op: WalkOp) {.benign.} @@ -277,7 +277,7 @@ proc rawNewObj(typ: PNimType, size: int, gch: var GcHeap): pointer = incTypeSize typ, size acquire(gch) gcAssert(typ.kind in {tyRef, tyOptAsRef, tyString, tySequence}, "newObj: 1") - collectCT(gch) + collectCT(gch, size + sizeof(Cell)) var res = cast[PCell](rawAlloc(gch.region, size + sizeof(Cell))) gcAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "newObj: 2") # now it is buffered in the ZCT @@ -332,7 +332,7 @@ proc newSeqRC1(typ: PNimType, len: int): pointer {.compilerRtl.} = proc growObj(old: pointer, newsize: int, gch: var GcHeap): pointer = acquire(gch) - collectCT(gch) + collectCT(gch, newsize + sizeof(Cell)) var ol = usrToCell(old) sysAssert(ol.typ != nil, "growObj: 1") gcAssert(ol.typ.kind in {tyString, tySequence}, "growObj: 2") @@ -494,8 +494,9 @@ proc collectCTBody(gch: var GcHeap) = gch.stat.maxThreshold = max(gch.stat.maxThreshold, gch.cycleThreshold) sysAssert(allocInv(gch.region), "collectCT: end") -proc collectCT(gch: var GcHeap) = - if getOccupiedMem(gch.region) >= gch.cycleThreshold and gch.recGcLock == 0: +proc collectCT(gch: var GcHeap; size: int) = + if (getOccupiedMem(gch.region) >= gch.cycleThreshold or + size > getFreeMem(gch.region)) and gch.recGcLock == 0: collectCTBody(gch) when not defined(useNimRtl): @@ -530,7 +531,7 @@ when not defined(useNimRtl): acquire(gch) var oldThreshold = gch.cycleThreshold gch.cycleThreshold = 0 # forces cycle collection - collectCT(gch) + collectCT(gch, 0) gch.cycleThreshold = oldThreshold release(gch) diff --git a/lib/system/sysio.nim b/lib/system/sysio.nim index f638b299c..285bf1adc 100644 --- a/lib/system/sysio.nim +++ b/lib/system/sysio.nim @@ -47,10 +47,22 @@ when not declared(c_fwrite): # C routine that is used here: proc c_fread(buf: pointer, size, n: csize, f: File): csize {. importc: "fread", header: "<stdio.h>", tags: [ReadIOEffect].} -proc c_fseek(f: File, offset: clong, whence: cint): cint {. - importc: "fseek", header: "<stdio.h>", tags: [].} -proc c_ftell(f: File): clong {. - importc: "ftell", header: "<stdio.h>", tags: [].} +when defined(windows): + when not defined(amd64): + proc c_fseek(f: File, offset: int64, whence: cint): cint {. + importc: "fseek", header: "<stdio.h>", tags: [].} + proc c_ftell(f: File): int64 {. + importc: "ftell", header: "<stdio.h>", tags: [].} + else: + proc c_fseek(f: File, offset: int64, whence: cint): cint {. + importc: "_fseeki64", header: "<stdio.h>", tags: [].} + proc c_ftell(f: File): int64 {. + importc: "_ftelli64", header: "<stdio.h>", tags: [].} +else: + proc c_fseek(f: File, offset: int64, whence: cint): cint {. + importc: "fseeko", header: "<stdio.h>", tags: [].} + proc c_ftell(f: File): int64 {. + importc: "ftello", header: "<stdio.h>", tags: [].} proc c_ferror(f: File): cint {. importc: "ferror", header: "<stdio.h>", tags: [].} proc c_setvbuf(f: File, buf: pointer, mode: cint, size: csize): cint {. @@ -210,12 +222,12 @@ proc readAllBuffer(file: File): string = result.add(buffer) break -proc rawFileSize(file: File): int = +proc rawFileSize(file: File): int64 = # this does not raise an error opposed to `getFileSize` var oldPos = c_ftell(file) discard c_fseek(file, 0, 2) # seek the end of the file result = c_ftell(file) - discard c_fseek(file, clong(oldPos), 0) + discard c_fseek(file, oldPos, 0) proc endOfFile(f: File): bool = var c = c_fgetc(f) @@ -223,7 +235,7 @@ proc endOfFile(f: File): bool = return c < 0'i32 #result = c_feof(f) != 0 -proc readAllFile(file: File, len: int): string = +proc readAllFile(file: File, len: int64): string = # We acquire the filesize beforehand and hope it doesn't change. # Speeds things up. result = newString(len) @@ -363,7 +375,7 @@ proc open(f: var File, filehandle: FileHandle, mode: FileMode): bool = result = f != nil proc setFilePos(f: File, pos: int64, relativeTo: FileSeekPos = fspSet) = - if c_fseek(f, clong(pos), cint(relativeTo)) != 0: + if c_fseek(f, pos, cint(relativeTo)) != 0: raiseEIO("cannot set file position") proc getFilePos(f: File): int64 = diff --git a/lib/upcoming/asyncdispatch.nim b/lib/upcoming/asyncdispatch.nim deleted file mode 100644 index 4e3b06173..000000000 --- a/lib/upcoming/asyncdispatch.nim +++ /dev/null @@ -1,1630 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Dominik Picheta -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -include "system/inclrtl" - -import os, tables, strutils, times, heapqueue, lists, options, asyncstreams -import asyncfutures except callSoon - -import nativesockets, net, deques - -export Port, SocketFlag -export asyncfutures, asyncstreams - -#{.injectStmt: newGcInvariant().} - -## AsyncDispatch -## ************* -## -## This module implements asynchronous IO. This includes a dispatcher, -## a ``Future`` type implementation, and an ``async`` macro which allows -## asynchronous code to be written in a synchronous style with the ``await`` -## keyword. -## -## The dispatcher acts as a kind of event loop. You must call ``poll`` on it -## (or a function which does so for you such as ``waitFor`` or ``runForever``) -## in order to poll for any outstanding events. The underlying implementation -## is based on epoll on Linux, IO Completion Ports on Windows and select on -## other operating systems. -## -## The ``poll`` function will not, on its own, return any events. Instead -## an appropriate ``Future`` object will be completed. A ``Future`` is a -## type which holds a value which is not yet available, but which *may* be -## available in the future. You can check whether a future is finished -## by using the ``finished`` function. When a future is finished it means that -## either the value that it holds is now available or it holds an error instead. -## The latter situation occurs when the operation to complete a future fails -## with an exception. You can distinguish between the two situations with the -## ``failed`` function. -## -## Future objects can also store a callback procedure which will be called -## automatically once the future completes. -## -## Futures therefore can be thought of as an implementation of the proactor -## pattern. In this -## pattern you make a request for an action, and once that action is fulfilled -## a future is completed with the result of that action. Requests can be -## made by calling the appropriate functions. For example: calling the ``recv`` -## function will create a request for some data to be read from a socket. The -## future which the ``recv`` function returns will then complete once the -## requested amount of data is read **or** an exception occurs. -## -## Code to read some data from a socket may look something like this: -## -## .. code-block::nim -## var future = socket.recv(100) -## future.callback = -## proc () = -## echo(future.read) -## -## All asynchronous functions returning a ``Future`` will not block. They -## will not however return immediately. An asynchronous function will have -## code which will be executed before an asynchronous request is made, in most -## cases this code sets up the request. -## -## In the above example, the ``recv`` function will return a brand new -## ``Future`` instance once the request for data to be read from the socket -## is made. This ``Future`` instance will complete once the requested amount -## of data is read, in this case it is 100 bytes. The second line sets a -## callback on this future which will be called once the future completes. -## All the callback does is write the data stored in the future to ``stdout``. -## The ``read`` function is used for this and it checks whether the future -## completes with an error for you (if it did it will simply raise the -## error), if there is no error however it returns the value of the future. -## -## Asynchronous procedures -## ----------------------- -## -## Asynchronous procedures remove the pain of working with callbacks. They do -## this by allowing you to write asynchronous code the same way as you would -## write synchronous code. -## -## An asynchronous procedure is marked using the ``{.async.}`` pragma. -## When marking a procedure with the ``{.async.}`` pragma it must have a -## ``Future[T]`` return type or no return type at all. If you do not specify -## a return type then ``Future[void]`` is assumed. -## -## Inside asynchronous procedures ``await`` can be used to call any -## procedures which return a -## ``Future``; this includes asynchronous procedures. When a procedure is -## "awaited", the asynchronous procedure it is awaited in will -## suspend its execution -## until the awaited procedure's Future completes. At which point the -## asynchronous procedure will resume its execution. During the period -## when an asynchronous procedure is suspended other asynchronous procedures -## will be run by the dispatcher. -## -## The ``await`` call may be used in many contexts. It can be used on the right -## hand side of a variable declaration: ``var data = await socket.recv(100)``, -## in which case the variable will be set to the value of the future -## automatically. It can be used to await a ``Future`` object, and it can -## be used to await a procedure returning a ``Future[void]``: -## ``await socket.send("foobar")``. -## -## Discarding futures -## ------------------ -## -## Futures should **never** be discarded. This is because they may contain -## errors. If you do not care for the result of a Future then you should -## use the ``asyncCheck`` procedure instead of the ``discard`` keyword. -## -## Examples -## -------- -## -## For examples take a look at the documentation for the modules implementing -## asynchronous IO. A good place to start is the -## `asyncnet module <asyncnet.html>`_. -## -## Limitations/Bugs -## ---------------- -## -## * The effect system (``raises: []``) does not work with async procedures. -## * Can't await in a ``except`` body -## * Forward declarations for async procs are broken, -## link includes workaround: https://github.com/nim-lang/Nim/issues/3182. -## * FutureVar[T] needs to be completed manually. - -# TODO: Check if yielded future is nil and throw a more meaningful exception - -type - PDispatcherBase = ref object of RootRef - timers: HeapQueue[tuple[finishAt: float, fut: Future[void]]] - callbacks: Deque[proc ()] - -proc processTimers(p: PDispatcherBase) {.inline.} = - #Process just part if timers at a step - var count = p.timers.len - let t = epochTime() - while count > 0 and t >= p.timers[0].finishAt: - p.timers.pop().fut.complete() - dec count - -proc processPendingCallbacks(p: PDispatcherBase) = - while p.callbacks.len > 0: - var cb = p.callbacks.popFirst() - cb() - -proc adjustedTimeout(p: PDispatcherBase, timeout: int): int {.inline.} = - # If dispatcher has active timers this proc returns the timeout - # of the nearest timer. Returns `timeout` otherwise. - result = timeout - if p.timers.len > 0: - let timerTimeout = p.timers[0].finishAt - let curTime = epochTime() - if timeout == -1 or (curTime + (timeout / 1000)) > timerTimeout: - result = int((timerTimeout - curTime) * 1000) - if result < 0: result = 0 - -proc callSoon(cbproc: proc ()) {.gcsafe.} - -proc initCallSoonProc = - if asyncfutures.getCallSoonProc().isNil: - asyncfutures.setCallSoonProc(callSoon) - -when defined(windows) or defined(nimdoc): - import winlean, sets, hashes - type - CompletionKey = ULONG_PTR - - CompletionData* = object - fd*: AsyncFD # TODO: Rename this. - cb*: proc (fd: AsyncFD, bytesTransferred: Dword, - errcode: OSErrorCode) {.closure,gcsafe.} - cell*: ForeignCell # we need this `cell` to protect our `cb` environment, - # when using RegisterWaitForSingleObject, because - # waiting is done in different thread. - - PDispatcher* = ref object of PDispatcherBase - ioPort: Handle - handles: HashSet[AsyncFD] - - CustomOverlapped = object of OVERLAPPED - data*: CompletionData - - PCustomOverlapped* = ref CustomOverlapped - - AsyncFD* = distinct int - - PostCallbackData = object - ioPort: Handle - handleFd: AsyncFD - waitFd: Handle - ovl: PCustomOverlapped - PostCallbackDataPtr = ptr PostCallbackData - - AsyncEventImpl = object - hEvent: Handle - hWaiter: Handle - pcd: PostCallbackDataPtr - AsyncEvent* = ptr AsyncEventImpl - - Callback = proc (fd: AsyncFD): bool {.closure,gcsafe.} - {.deprecated: [TCompletionKey: CompletionKey, TAsyncFD: AsyncFD, - TCustomOverlapped: CustomOverlapped, TCompletionData: CompletionData].} - - proc hash(x: AsyncFD): Hash {.borrow.} - proc `==`*(x: AsyncFD, y: AsyncFD): bool {.borrow.} - - proc newDispatcher*(): PDispatcher = - ## Creates a new Dispatcher instance. - new result - result.ioPort = createIoCompletionPort(INVALID_HANDLE_VALUE, 0, 0, 1) - result.handles = initSet[AsyncFD]() - result.timers.newHeapQueue() - result.callbacks = initDeque[proc ()](64) - - var gDisp{.threadvar.}: PDispatcher ## Global dispatcher - - proc setGlobalDispatcher*(disp: PDispatcher) = - if not gDisp.isNil: - assert gDisp.callbacks.len == 0 - gDisp = disp - initCallSoonProc() - - proc getGlobalDispatcher*(): PDispatcher = - if gDisp.isNil: - setGlobalDispatcher(newDispatcher()) - result = gDisp - - proc register*(fd: AsyncFD) = - ## Registers ``fd`` with the dispatcher. - let p = getGlobalDispatcher() - if createIoCompletionPort(fd.Handle, p.ioPort, - cast[CompletionKey](fd), 1) == 0: - raiseOSError(osLastError()) - p.handles.incl(fd) - - proc verifyPresence(fd: AsyncFD) = - ## Ensures that file descriptor has been registered with the dispatcher. - let p = getGlobalDispatcher() - if fd notin p.handles: - raise newException(ValueError, - "Operation performed on a socket which has not been registered with" & - " the dispatcher yet.") - - proc hasPendingOperations*(): bool = - ## Returns `true` if the global dispatcher has pending operations. - let p = getGlobalDispatcher() - p.handles.len != 0 or p.timers.len != 0 or p.callbacks.len != 0 - - proc poll*(timeout = 500) = - ## Waits for completion events and processes them. Raises ``ValueError`` - ## if there are no pending operations. - let p = getGlobalDispatcher() - if p.handles.len == 0 and p.timers.len == 0 and p.callbacks.len == 0: - raise newException(ValueError, - "No handles or timers registered in dispatcher.") - - let at = p.adjustedTimeout(timeout) - var llTimeout = - if at == -1: winlean.INFINITE - else: at.int32 - - if p.handles.len != 0: - var lpNumberOfBytesTransferred: Dword - var lpCompletionKey: ULONG_PTR - var customOverlapped: PCustomOverlapped - let res = getQueuedCompletionStatus(p.ioPort, - addr lpNumberOfBytesTransferred, addr lpCompletionKey, - cast[ptr POVERLAPPED](addr customOverlapped), llTimeout).bool - - # http://stackoverflow.com/a/12277264/492186 - # TODO: http://www.serverframework.com/handling-multiple-pending-socket-read-and-write-operations.html - if res: - # This is useful for ensuring the reliability of the overlapped struct. - assert customOverlapped.data.fd == lpCompletionKey.AsyncFD - - customOverlapped.data.cb(customOverlapped.data.fd, - lpNumberOfBytesTransferred, OSErrorCode(-1)) - - # If cell.data != nil, then system.protect(rawEnv(cb)) was called, - # so we need to dispose our `cb` environment, because it is not needed - # anymore. - if customOverlapped.data.cell.data != nil: - system.dispose(customOverlapped.data.cell) - - GC_unref(customOverlapped) - else: - let errCode = osLastError() - if customOverlapped != nil: - assert customOverlapped.data.fd == lpCompletionKey.AsyncFD - customOverlapped.data.cb(customOverlapped.data.fd, - lpNumberOfBytesTransferred, errCode) - if customOverlapped.data.cell.data != nil: - system.dispose(customOverlapped.data.cell) - GC_unref(customOverlapped) - else: - if errCode.int32 == WAIT_TIMEOUT: - # Timed out - discard - else: raiseOSError(errCode) - - # Timer processing. - processTimers(p) - # Callback queue processing - processPendingCallbacks(p) - - var acceptEx*: WSAPROC_ACCEPTEX - var connectEx*: WSAPROC_CONNECTEX - var getAcceptExSockAddrs*: WSAPROC_GETACCEPTEXSOCKADDRS - - proc initPointer(s: SocketHandle, fun: var pointer, guid: var GUID): bool = - # Ref: https://github.com/powdahound/twisted/blob/master/twisted/internet/iocpreactor/iocpsupport/winsock_pointers.c - var bytesRet: Dword - fun = nil - result = WSAIoctl(s, SIO_GET_EXTENSION_FUNCTION_POINTER, addr guid, - sizeof(GUID).Dword, addr fun, sizeof(pointer).Dword, - addr bytesRet, nil, nil) == 0 - - proc initAll() = - let dummySock = newNativeSocket() - if dummySock == INVALID_SOCKET: - raiseOSError(osLastError()) - var fun: pointer = nil - if not initPointer(dummySock, fun, WSAID_CONNECTEX): - raiseOSError(osLastError()) - connectEx = cast[WSAPROC_CONNECTEX](fun) - if not initPointer(dummySock, fun, WSAID_ACCEPTEX): - raiseOSError(osLastError()) - acceptEx = cast[WSAPROC_ACCEPTEX](fun) - if not initPointer(dummySock, fun, WSAID_GETACCEPTEXSOCKADDRS): - raiseOSError(osLastError()) - getAcceptExSockAddrs = cast[WSAPROC_GETACCEPTEXSOCKADDRS](fun) - close(dummySock) - - proc recv*(socket: AsyncFD, size: int, - flags = {SocketFlag.SafeDisconn}): Future[string] = - ## Reads **up to** ``size`` bytes from ``socket``. Returned future will - ## complete once all the data requested is read, a part of the data has been - ## read, or the socket has disconnected in which case the future will - ## complete with a value of ``""``. - ## - ## **Warning**: The ``Peek`` socket flag is not supported on Windows. - - - # Things to note: - # * When WSARecv completes immediately then ``bytesReceived`` is very - # unreliable. - # * Still need to implement message-oriented socket disconnection, - # '\0' in the message currently signifies a socket disconnect. Who - # knows what will happen when someone sends that to our socket. - verifyPresence(socket) - assert SocketFlag.Peek notin flags, "Peek not supported on Windows." - - var retFuture = newFuture[string]("recv") - var dataBuf: TWSABuf - dataBuf.buf = cast[cstring](alloc0(size)) - dataBuf.len = size.ULONG - - var bytesReceived: Dword - var flagsio = flags.toOSFlags().Dword - var ol = PCustomOverlapped() - GC_ref(ol) - ol.data = CompletionData(fd: socket, cb: - proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = - if not retFuture.finished: - if errcode == OSErrorCode(-1): - if bytesCount == 0 and dataBuf.buf[0] == '\0': - retFuture.complete("") - else: - var data = newString(bytesCount) - assert bytesCount <= size - copyMem(addr data[0], addr dataBuf.buf[0], bytesCount) - retFuture.complete($data) - else: - if flags.isDisconnectionError(errcode): - retFuture.complete("") - else: - retFuture.fail(newException(OSError, osErrorMsg(errcode))) - if dataBuf.buf != nil: - dealloc dataBuf.buf - dataBuf.buf = nil - ) - - let ret = WSARecv(socket.SocketHandle, addr dataBuf, 1, addr bytesReceived, - addr flagsio, cast[POVERLAPPED](ol), nil) - if ret == -1: - let err = osLastError() - if err.int32 != ERROR_IO_PENDING: - if dataBuf.buf != nil: - dealloc dataBuf.buf - dataBuf.buf = nil - GC_unref(ol) - if flags.isDisconnectionError(err): - retFuture.complete("") - else: - retFuture.fail(newException(OSError, osErrorMsg(err))) - elif ret == 0: - # Request completed immediately. - if bytesReceived != 0: - var data = newString(bytesReceived) - assert bytesReceived <= size - copyMem(addr data[0], addr dataBuf.buf[0], bytesReceived) - retFuture.complete($data) - else: - if hasOverlappedIoCompleted(cast[POVERLAPPED](ol)): - retFuture.complete("") - return retFuture - - proc recvInto*(socket: AsyncFD, buf: pointer, size: int, - flags = {SocketFlag.SafeDisconn}): Future[int] = - ## Reads **up to** ``size`` bytes from ``socket`` into ``buf``, which must - ## at least be of that size. Returned future will complete once all the - ## data requested is read, a part of the data has been read, or the socket - ## has disconnected in which case the future will complete with a value of - ## ``0``. - ## - ## **Warning**: The ``Peek`` socket flag is not supported on Windows. - - - # Things to note: - # * When WSARecv completes immediately then ``bytesReceived`` is very - # unreliable. - # * Still need to implement message-oriented socket disconnection, - # '\0' in the message currently signifies a socket disconnect. Who - # knows what will happen when someone sends that to our socket. - verifyPresence(socket) - assert SocketFlag.Peek notin flags, "Peek not supported on Windows." - - var retFuture = newFuture[int]("recvInto") - - #buf[] = '\0' - var dataBuf: TWSABuf - dataBuf.buf = cast[cstring](buf) - dataBuf.len = size.ULONG - - var bytesReceived: Dword - var flagsio = flags.toOSFlags().Dword - var ol = PCustomOverlapped() - GC_ref(ol) - ol.data = CompletionData(fd: socket, cb: - proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = - if not retFuture.finished: - if errcode == OSErrorCode(-1): - retFuture.complete(bytesCount) - else: - if flags.isDisconnectionError(errcode): - retFuture.complete(0) - else: - retFuture.fail(newException(OSError, osErrorMsg(errcode))) - if dataBuf.buf != nil: - dataBuf.buf = nil - ) - - let ret = WSARecv(socket.SocketHandle, addr dataBuf, 1, addr bytesReceived, - addr flagsio, cast[POVERLAPPED](ol), nil) - if ret == -1: - let err = osLastError() - if err.int32 != ERROR_IO_PENDING: - if dataBuf.buf != nil: - dataBuf.buf = nil - GC_unref(ol) - if flags.isDisconnectionError(err): - retFuture.complete(0) - else: - retFuture.fail(newException(OSError, osErrorMsg(err))) - elif ret == 0: - # Request completed immediately. - if bytesReceived != 0: - assert bytesReceived <= size - retFuture.complete(bytesReceived) - else: - if hasOverlappedIoCompleted(cast[POVERLAPPED](ol)): - retFuture.complete(bytesReceived) - return retFuture - - proc send*(socket: AsyncFD, buf: pointer, size: int, - flags = {SocketFlag.SafeDisconn}): Future[void] = - ## Sends ``size`` bytes from ``buf`` to ``socket``. The returned future - ## will complete once all data has been sent. - ## **WARNING**: Use it with caution. If ``buf`` refers to GC'ed object, - ## you must use GC_ref/GC_unref calls to avoid early freeing of the buffer. - verifyPresence(socket) - var retFuture = newFuture[void]("send") - - var dataBuf: TWSABuf - dataBuf.buf = cast[cstring](buf) - dataBuf.len = size.ULONG - - var bytesReceived, lowFlags: Dword - var ol = PCustomOverlapped() - GC_ref(ol) - ol.data = CompletionData(fd: socket, cb: - proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = - if not retFuture.finished: - if errcode == OSErrorCode(-1): - retFuture.complete() - else: - if flags.isDisconnectionError(errcode): - retFuture.complete() - else: - retFuture.fail(newException(OSError, osErrorMsg(errcode))) - ) - - let ret = WSASend(socket.SocketHandle, addr dataBuf, 1, addr bytesReceived, - lowFlags, cast[POVERLAPPED](ol), nil) - if ret == -1: - let err = osLastError() - if err.int32 != ERROR_IO_PENDING: - GC_unref(ol) - if flags.isDisconnectionError(err): - retFuture.complete() - else: - retFuture.fail(newException(OSError, osErrorMsg(err))) - else: - retFuture.complete() - # We don't deallocate ``ol`` here because even though this completed - # immediately poll will still be notified about its completion and it will - # free ``ol``. - return retFuture - - proc send*(socket: AsyncFD, data: string, - flags = {SocketFlag.SafeDisconn}): Future[void] = - ## Sends ``data`` to ``socket``. The returned future will complete once all - ## data has been sent. - verifyPresence(socket) - var retFuture = newFuture[void]("send") - - var dataBuf: TWSABuf - dataBuf.buf = data - GC_ref(data) # we need to protect data until send operation is completed - # or failed. - dataBuf.len = data.len.ULONG - - var bytesReceived, lowFlags: Dword - var ol = PCustomOverlapped() - GC_ref(ol) - ol.data = CompletionData(fd: socket, cb: - proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = - GC_unref(data) # if operation completed `data` must be released. - if not retFuture.finished: - if errcode == OSErrorCode(-1): - retFuture.complete() - else: - if flags.isDisconnectionError(errcode): - retFuture.complete() - else: - retFuture.fail(newException(OSError, osErrorMsg(errcode))) - ) - - let ret = WSASend(socket.SocketHandle, addr dataBuf, 1, addr bytesReceived, - lowFlags, cast[POVERLAPPED](ol), nil) - if ret == -1: - let err = osLastError() - if err.int32 != ERROR_IO_PENDING: - GC_unref(ol) - GC_unref(data) # if operation failed `data` must be released, because - # completion routine will not be called. - if flags.isDisconnectionError(err): - retFuture.complete() - else: - retFuture.fail(newException(OSError, osErrorMsg(err))) - else: - retFuture.complete() - # We don't deallocate ``ol`` here because even though this completed - # immediately poll will still be notified about its completion and it will - # free ``ol``. - return retFuture - - proc sendTo*(socket: AsyncFD, data: pointer, size: int, saddr: ptr SockAddr, - saddrLen: Socklen, - flags = {SocketFlag.SafeDisconn}): Future[void] = - ## Sends ``data`` to specified destination ``saddr``, using - ## socket ``socket``. The returned future will complete once all data - ## has been sent. - verifyPresence(socket) - var retFuture = newFuture[void]("sendTo") - var dataBuf: TWSABuf - dataBuf.buf = cast[cstring](data) - dataBuf.len = size.ULONG - var bytesSent = 0.Dword - var lowFlags = 0.Dword - - # we will preserve address in our stack - var staddr: array[128, char] # SOCKADDR_STORAGE size is 128 bytes - var stalen: cint = cint(saddrLen) - zeroMem(addr(staddr[0]), 128) - copyMem(addr(staddr[0]), saddr, saddrLen) - - var ol = PCustomOverlapped() - GC_ref(ol) - ol.data = CompletionData(fd: socket, cb: - proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = - if not retFuture.finished: - if errcode == OSErrorCode(-1): - retFuture.complete() - else: - retFuture.fail(newException(OSError, osErrorMsg(errcode))) - ) - - let ret = WSASendTo(socket.SocketHandle, addr dataBuf, 1, addr bytesSent, - lowFlags, cast[ptr SockAddr](addr(staddr[0])), - stalen, cast[POVERLAPPED](ol), nil) - if ret == -1: - let err = osLastError() - if err.int32 != ERROR_IO_PENDING: - GC_unref(ol) - retFuture.fail(newException(OSError, osErrorMsg(err))) - else: - retFuture.complete() - # We don't deallocate ``ol`` here because even though this completed - # immediately poll will still be notified about its completion and it will - # free ``ol``. - return retFuture - - proc recvFromInto*(socket: AsyncFD, data: pointer, size: int, - saddr: ptr SockAddr, saddrLen: ptr SockLen, - flags = {SocketFlag.SafeDisconn}): Future[int] = - ## Receives a datagram data from ``socket`` into ``buf``, which must - ## be at least of size ``size``, address of datagram's sender will be - ## stored into ``saddr`` and ``saddrLen``. Returned future will complete - ## once one datagram has been received, and will return size of packet - ## received. - verifyPresence(socket) - var retFuture = newFuture[int]("recvFromInto") - - var dataBuf = TWSABuf(buf: cast[cstring](data), len: size.ULONG) - - var bytesReceived = 0.Dword - var lowFlags = 0.Dword - - var ol = PCustomOverlapped() - GC_ref(ol) - ol.data = CompletionData(fd: socket, cb: - proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = - if not retFuture.finished: - if errcode == OSErrorCode(-1): - assert bytesCount <= size - retFuture.complete(bytesCount) - else: - # datagram sockets don't have disconnection, - # so we can just raise an exception - retFuture.fail(newException(OSError, osErrorMsg(errcode))) - ) - - let res = WSARecvFrom(socket.SocketHandle, addr dataBuf, 1, - addr bytesReceived, addr lowFlags, - saddr, cast[ptr cint](saddrLen), - cast[POVERLAPPED](ol), nil) - if res == -1: - let err = osLastError() - if err.int32 != ERROR_IO_PENDING: - GC_unref(ol) - retFuture.fail(newException(OSError, osErrorMsg(err))) - else: - # Request completed immediately. - if bytesReceived != 0: - assert bytesReceived <= size - retFuture.complete(bytesReceived) - else: - if hasOverlappedIoCompleted(cast[POVERLAPPED](ol)): - retFuture.complete(bytesReceived) - return retFuture - - proc acceptAddr*(socket: AsyncFD, flags = {SocketFlag.SafeDisconn}): - Future[tuple[address: string, client: AsyncFD]] = - ## Accepts a new connection. Returns a future containing the client socket - ## corresponding to that connection and the remote address of the client. - ## The future will complete when the connection is successfully accepted. - ## - ## The resulting client socket is automatically registered to the - ## dispatcher. - ## - ## The ``accept`` call may result in an error if the connecting socket - ## disconnects during the duration of the ``accept``. If the ``SafeDisconn`` - ## flag is specified then this error will not be raised and instead - ## accept will be called again. - verifyPresence(socket) - var retFuture = newFuture[tuple[address: string, client: AsyncFD]]("acceptAddr") - - var clientSock = newNativeSocket() - if clientSock == osInvalidSocket: raiseOSError(osLastError()) - - const lpOutputLen = 1024 - var lpOutputBuf = newString(lpOutputLen) - var dwBytesReceived: Dword - let dwReceiveDataLength = 0.Dword # We don't want any data to be read. - let dwLocalAddressLength = Dword(sizeof(Sockaddr_in6) + 16) - let dwRemoteAddressLength = Dword(sizeof(Sockaddr_in6) + 16) - - template failAccept(errcode) = - if flags.isDisconnectionError(errcode): - var newAcceptFut = acceptAddr(socket, flags) - newAcceptFut.callback = - proc () = - if newAcceptFut.failed: - retFuture.fail(newAcceptFut.readError) - else: - retFuture.complete(newAcceptFut.read) - else: - retFuture.fail(newException(OSError, osErrorMsg(errcode))) - - template completeAccept() {.dirty.} = - var listenSock = socket - let setoptRet = setsockopt(clientSock, SOL_SOCKET, - SO_UPDATE_ACCEPT_CONTEXT, addr listenSock, - sizeof(listenSock).SockLen) - if setoptRet != 0: - let errcode = osLastError() - discard clientSock.closeSocket() - failAccept(errcode) - else: - var localSockaddr, remoteSockaddr: ptr SockAddr - var localLen, remoteLen: int32 - getAcceptExSockaddrs(addr lpOutputBuf[0], dwReceiveDataLength, - dwLocalAddressLength, dwRemoteAddressLength, - addr localSockaddr, addr localLen, - addr remoteSockaddr, addr remoteLen) - try: - let address = getAddrString(remoteSockAddr) - register(clientSock.AsyncFD) - retFuture.complete((address: address, client: clientSock.AsyncFD)) - except: - # getAddrString may raise - clientSock.close() - retFuture.fail(getCurrentException()) - - var ol = PCustomOverlapped() - GC_ref(ol) - ol.data = CompletionData(fd: socket, cb: - proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = - if not retFuture.finished: - if errcode == OSErrorCode(-1): - completeAccept() - else: - failAccept(errcode) - ) - - # http://msdn.microsoft.com/en-us/library/windows/desktop/ms737524%28v=vs.85%29.aspx - let ret = acceptEx(socket.SocketHandle, clientSock, addr lpOutputBuf[0], - dwReceiveDataLength, - dwLocalAddressLength, - dwRemoteAddressLength, - addr dwBytesReceived, cast[POVERLAPPED](ol)) - - if not ret: - let err = osLastError() - if err.int32 != ERROR_IO_PENDING: - failAccept(err) - GC_unref(ol) - else: - completeAccept() - # We don't deallocate ``ol`` here because even though this completed - # immediately poll will still be notified about its completion and it will - # free ``ol``. - - return retFuture - - proc closeSocket*(socket: AsyncFD) = - ## Closes a socket and ensures that it is unregistered. - socket.SocketHandle.close() - getGlobalDispatcher().handles.excl(socket) - - proc unregister*(fd: AsyncFD) = - ## Unregisters ``fd``. - getGlobalDispatcher().handles.excl(fd) - - {.push stackTrace:off.} - proc waitableCallback(param: pointer, - timerOrWaitFired: WINBOOL): void {.stdcall.} = - var p = cast[PostCallbackDataPtr](param) - discard postQueuedCompletionStatus(p.ioPort, timerOrWaitFired.Dword, - ULONG_PTR(p.handleFd), - cast[pointer](p.ovl)) - {.pop.} - - proc registerWaitableEvent(fd: AsyncFD, cb: Callback; mask: Dword) = - let p = getGlobalDispatcher() - var flags = (WT_EXECUTEINWAITTHREAD or WT_EXECUTEONLYONCE).Dword - var hEvent = wsaCreateEvent() - if hEvent == 0: - raiseOSError(osLastError()) - var pcd = cast[PostCallbackDataPtr](allocShared0(sizeof(PostCallbackData))) - pcd.ioPort = p.ioPort - pcd.handleFd = fd - var ol = PCustomOverlapped() - GC_ref(ol) - - ol.data = CompletionData(fd: fd, cb: - proc(fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = - # we excluding our `fd` because cb(fd) can register own handler - # for this `fd` - p.handles.excl(fd) - # unregisterWait() is called before callback, because appropriate - # winsockets function can re-enable event. - # https://msdn.microsoft.com/en-us/library/windows/desktop/ms741576(v=vs.85).aspx - if unregisterWait(pcd.waitFd) == 0: - let err = osLastError() - if err.int32 != ERROR_IO_PENDING: - deallocShared(cast[pointer](pcd)) - discard wsaCloseEvent(hEvent) - raiseOSError(err) - if cb(fd): - # callback returned `true`, so we free all allocated resources - deallocShared(cast[pointer](pcd)) - if not wsaCloseEvent(hEvent): - raiseOSError(osLastError()) - # pcd.ovl will be unrefed in poll(). - else: - # callback returned `false` we need to continue - if p.handles.contains(fd): - # new callback was already registered with `fd`, so we free all - # allocated resources. This happens because in callback `cb` - # addRead/addWrite was called with same `fd`. - deallocShared(cast[pointer](pcd)) - if not wsaCloseEvent(hEvent): - raiseOSError(osLastError()) - else: - # we need to include `fd` again - p.handles.incl(fd) - # and register WaitForSingleObject again - if not registerWaitForSingleObject(addr(pcd.waitFd), hEvent, - cast[WAITORTIMERCALLBACK](waitableCallback), - cast[pointer](pcd), INFINITE, flags): - # pcd.ovl will be unrefed in poll() - let err = osLastError() - deallocShared(cast[pointer](pcd)) - discard wsaCloseEvent(hEvent) - raiseOSError(err) - else: - # we incref `pcd.ovl` and `protect` callback one more time, - # because it will be unrefed and disposed in `poll()` after - # callback finishes. - GC_ref(pcd.ovl) - pcd.ovl.data.cell = system.protect(rawEnv(pcd.ovl.data.cb)) - ) - # We need to protect our callback environment value, so GC will not free it - # accidentally. - ol.data.cell = system.protect(rawEnv(ol.data.cb)) - - # This is main part of `hacky way` is using WSAEventSelect, so `hEvent` - # will be signaled when appropriate `mask` events will be triggered. - if wsaEventSelect(fd.SocketHandle, hEvent, mask) != 0: - let err = osLastError() - GC_unref(ol) - deallocShared(cast[pointer](pcd)) - discard wsaCloseEvent(hEvent) - raiseOSError(err) - - pcd.ovl = ol - if not registerWaitForSingleObject(addr(pcd.waitFd), hEvent, - cast[WAITORTIMERCALLBACK](waitableCallback), - cast[pointer](pcd), INFINITE, flags): - let err = osLastError() - GC_unref(ol) - deallocShared(cast[pointer](pcd)) - discard wsaCloseEvent(hEvent) - raiseOSError(err) - p.handles.incl(fd) - - proc addRead*(fd: AsyncFD, cb: Callback) = - ## Start watching the file descriptor for read availability and then call - ## the callback ``cb``. - ## - ## This is not ``pure`` mechanism for Windows Completion Ports (IOCP), - ## so if you can avoid it, please do it. Use `addRead` only if really - ## need it (main usecase is adaptation of `unix like` libraries to be - ## asynchronous on Windows). - ## If you use this function, you dont need to use asyncdispatch.recv() - ## or asyncdispatch.accept(), because they are using IOCP, please use - ## nativesockets.recv() and nativesockets.accept() instead. - ## - ## Be sure your callback ``cb`` returns ``true``, if you want to remove - ## watch of `read` notifications, and ``false``, if you want to continue - ## receiving notifies. - registerWaitableEvent(fd, cb, FD_READ or FD_ACCEPT or FD_OOB or FD_CLOSE) - - proc addWrite*(fd: AsyncFD, cb: Callback) = - ## Start watching the file descriptor for write availability and then call - ## the callback ``cb``. - ## - ## This is not ``pure`` mechanism for Windows Completion Ports (IOCP), - ## so if you can avoid it, please do it. Use `addWrite` only if really - ## need it (main usecase is adaptation of `unix like` libraries to be - ## asynchronous on Windows). - ## If you use this function, you dont need to use asyncdispatch.send() - ## or asyncdispatch.connect(), because they are using IOCP, please use - ## nativesockets.send() and nativesockets.connect() instead. - ## - ## Be sure your callback ``cb`` returns ``true``, if you want to remove - ## watch of `write` notifications, and ``false``, if you want to continue - ## receiving notifies. - registerWaitableEvent(fd, cb, FD_WRITE or FD_CONNECT or FD_CLOSE) - - template registerWaitableHandle(p, hEvent, flags, pcd, timeout, - handleCallback) = - let handleFD = AsyncFD(hEvent) - pcd.ioPort = p.ioPort - pcd.handleFd = handleFD - var ol = PCustomOverlapped() - GC_ref(ol) - ol.data.fd = handleFD - ol.data.cb = handleCallback - # We need to protect our callback environment value, so GC will not free it - # accidentally. - ol.data.cell = system.protect(rawEnv(ol.data.cb)) - - pcd.ovl = ol - if not registerWaitForSingleObject(addr(pcd.waitFd), hEvent, - cast[WAITORTIMERCALLBACK](waitableCallback), - cast[pointer](pcd), timeout.Dword, flags): - let err = osLastError() - GC_unref(ol) - deallocShared(cast[pointer](pcd)) - discard closeHandle(hEvent) - raiseOSError(err) - p.handles.incl(handleFD) - - template closeWaitable(handle: untyped) = - let waitFd = pcd.waitFd - deallocShared(cast[pointer](pcd)) - p.handles.excl(fd) - if unregisterWait(waitFd) == 0: - let err = osLastError() - if err.int32 != ERROR_IO_PENDING: - discard closeHandle(handle) - raiseOSError(err) - if closeHandle(handle) == 0: - raiseOSError(osLastError()) - - proc addTimer*(timeout: int, oneshot: bool, cb: Callback) = - ## Registers callback ``cb`` to be called when timer expired. - ## ``timeout`` - timeout value in milliseconds. - ## ``oneshot`` - `true`, to generate only one timeout event, `false`, to - ## generate timeout events periodically. - - doAssert(timeout > 0) - let p = getGlobalDispatcher() - - var hEvent = createEvent(nil, 1, 0, nil) - if hEvent == INVALID_HANDLE_VALUE: - raiseOSError(osLastError()) - - var pcd = cast[PostCallbackDataPtr](allocShared0(sizeof(PostCallbackData))) - var flags = WT_EXECUTEINWAITTHREAD.Dword - if oneshot: flags = flags or WT_EXECUTEONLYONCE - - proc timercb(fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = - let res = cb(fd) - if res or oneshot: - closeWaitable(hEvent) - else: - # if callback returned `false`, then it wants to be called again, so - # we need to ref and protect `pcd.ovl` again, because it will be - # unrefed and disposed in `poll()`. - GC_ref(pcd.ovl) - pcd.ovl.data.cell = system.protect(rawEnv(pcd.ovl.data.cb)) - - registerWaitableHandle(p, hEvent, flags, pcd, timeout, timercb) - - proc addProcess*(pid: int, cb: Callback) = - ## Registers callback ``cb`` to be called when process with pid ``pid`` - ## exited. - let p = getGlobalDispatcher() - let procFlags = SYNCHRONIZE - var hProcess = openProcess(procFlags, 0, pid.Dword) - if hProcess == INVALID_HANDLE_VALUE: - raiseOSError(osLastError()) - - var pcd = cast[PostCallbackDataPtr](allocShared0(sizeof(PostCallbackData))) - var flags = WT_EXECUTEINWAITTHREAD.Dword - - proc proccb(fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = - closeWaitable(hProcess) - discard cb(fd) - - registerWaitableHandle(p, hProcess, flags, pcd, INFINITE, proccb) - - proc newAsyncEvent*(): AsyncEvent = - ## Creates new ``AsyncEvent`` object. - ## New ``AsyncEvent`` object is not automatically registered with - ## dispatcher like ``AsyncSocket``. - var sa = SECURITY_ATTRIBUTES( - nLength: sizeof(SECURITY_ATTRIBUTES).cint, - bInheritHandle: 1 - ) - var event = createEvent(addr(sa), 0'i32, 0'i32, nil) - if event == INVALID_HANDLE_VALUE: - raiseOSError(osLastError()) - result = cast[AsyncEvent](allocShared0(sizeof(AsyncEventImpl))) - result.hEvent = event - - proc setEvent*(ev: AsyncEvent) = - ## Set event ``ev`` to signaled state. - if setEvent(ev.hEvent) == 0: - raiseOSError(osLastError()) - - proc unregister*(ev: AsyncEvent) = - ## Unregisters event ``ev``. - doAssert(ev.hWaiter != 0, "Event is not registered in the queue!") - let p = getGlobalDispatcher() - p.handles.excl(AsyncFD(ev.hEvent)) - if unregisterWait(ev.hWaiter) == 0: - let err = osLastError() - if err.int32 != ERROR_IO_PENDING: - raiseOSError(err) - ev.hWaiter = 0 - - proc close*(ev: AsyncEvent) = - ## Closes event ``ev``. - let res = closeHandle(ev.hEvent) - deallocShared(cast[pointer](ev)) - if res == 0: - raiseOSError(osLastError()) - - proc addEvent*(ev: AsyncEvent, cb: Callback) = - ## Registers callback ``cb`` to be called when ``ev`` will be signaled - doAssert(ev.hWaiter == 0, "Event is already registered in the queue!") - - let p = getGlobalDispatcher() - let hEvent = ev.hEvent - - var pcd = cast[PostCallbackDataPtr](allocShared0(sizeof(PostCallbackData))) - var flags = WT_EXECUTEINWAITTHREAD.Dword - - proc eventcb(fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = - if ev.hWaiter != 0: - if cb(fd): - # we need this check to avoid exception, if `unregister(event)` was - # called in callback. - deallocShared(cast[pointer](pcd)) - if ev.hWaiter != 0: - unregister(ev) - else: - # if callback returned `false`, then it wants to be called again, so - # we need to ref and protect `pcd.ovl` again, because it will be - # unrefed and disposed in `poll()`. - GC_ref(pcd.ovl) - pcd.ovl.data.cell = system.protect(rawEnv(pcd.ovl.data.cb)) - else: - # if ev.hWaiter == 0, then event was unregistered before `poll()` call. - deallocShared(cast[pointer](pcd)) - - registerWaitableHandle(p, hEvent, flags, pcd, INFINITE, eventcb) - ev.hWaiter = pcd.waitFd - - initAll() -else: - import ioselectors - from posix import EINTR, EAGAIN, EINPROGRESS, EWOULDBLOCK, MSG_PEEK, - MSG_NOSIGNAL - const - InitCallbackListSize = 4 # initial size of callbacks sequence, - # associated with file/socket descriptor. - InitDelayedCallbackListSize = 64 # initial size of delayed callbacks - # queue. - type - AsyncFD* = distinct cint - Callback = proc (fd: AsyncFD): bool {.closure,gcsafe.} - - AsyncData = object - readList: seq[Callback] - writeList: seq[Callback] - - AsyncEvent* = distinct SelectEvent - - PDispatcher* = ref object of PDispatcherBase - selector: Selector[AsyncData] - {.deprecated: [TAsyncFD: AsyncFD, TCallback: Callback].} - - proc `==`*(x, y: AsyncFD): bool {.borrow.} - proc `==`*(x, y: AsyncEvent): bool {.borrow.} - - template newAsyncData(): AsyncData = - AsyncData( - readList: newSeqOfCap[Callback](InitCallbackListSize), - writeList: newSeqOfCap[Callback](InitCallbackListSize) - ) - - proc newDispatcher*(): PDispatcher = - new result - result.selector = newSelector[AsyncData]() - result.timers.newHeapQueue() - result.callbacks = initDeque[proc ()](InitDelayedCallbackListSize) - - var gDisp{.threadvar.}: PDispatcher ## Global dispatcher - - proc setGlobalDispatcher*(disp: PDispatcher) = - if not gDisp.isNil: - assert gDisp.callbacks.len == 0 - gDisp = disp - initCallSoonProc() - - proc getGlobalDispatcher*(): PDispatcher = - if gDisp.isNil: - setGlobalDispatcher(newDispatcher()) - result = gDisp - - proc register*(fd: AsyncFD) = - let p = getGlobalDispatcher() - var data = newAsyncData() - p.selector.registerHandle(fd.SocketHandle, {}, data) - - proc closeSocket*(sock: AsyncFD) = - let disp = getGlobalDispatcher() - disp.selector.unregister(sock.SocketHandle) - sock.SocketHandle.close() - - proc unregister*(fd: AsyncFD) = - getGlobalDispatcher().selector.unregister(fd.SocketHandle) - - proc unregister*(ev: AsyncEvent) = - getGlobalDispatcher().selector.unregister(SelectEvent(ev)) - - proc addRead*(fd: AsyncFD, cb: Callback) = - let p = getGlobalDispatcher() - var newEvents = {Event.Read} - withData(p.selector, fd.SocketHandle, adata) do: - adata.readList.add(cb) - newEvents.incl(Event.Read) - if len(adata.writeList) != 0: newEvents.incl(Event.Write) - do: - raise newException(ValueError, "File descriptor not registered.") - p.selector.updateHandle(fd.SocketHandle, newEvents) - - proc addWrite*(fd: AsyncFD, cb: Callback) = - let p = getGlobalDispatcher() - var newEvents = {Event.Write} - withData(p.selector, fd.SocketHandle, adata) do: - adata.writeList.add(cb) - newEvents.incl(Event.Write) - if len(adata.readList) != 0: newEvents.incl(Event.Read) - do: - raise newException(ValueError, "File descriptor not registered.") - p.selector.updateHandle(fd.SocketHandle, newEvents) - - proc hasPendingOperations*(): bool = - let p = getGlobalDispatcher() - not p.selector.isEmpty() or p.timers.len != 0 or p.callbacks.len != 0 - - template processBasicCallbacks(ident, rwlist: untyped) = - # Process pending descriptor's and AsyncEvent callbacks. - # Invoke every callback stored in `rwlist`, until first one - # returned `false`, which means callback wants to stay - # alive. In such case all remaining callbacks will be added - # to `rwlist` again, in the order they have been inserted. - # - # `rwlist` associated with file descriptor MUST BE emptied before - # dispatching callback (See https://github.com/nim-lang/Nim/issues/5128), - # or it can be possible to fall into endless cycle. - var curList: seq[Callback] - - withData(p.selector, ident, adata) do: - shallowCopy(curList, adata.rwlist) - adata.rwlist = newSeqOfCap[Callback](InitCallbackListSize) - - let newLength = max(len(curList), InitCallbackListSize) - var newList = newSeqOfCap[Callback](newLength) - - for cb in curList: - if len(newList) > 0: - newList.add(cb) - else: - if not cb(fd.AsyncFD): - newList.add(cb) - - withData(p.selector, ident, adata) do: - # descriptor still present in queue. - adata.rwlist = newList & adata.rwlist - rLength = len(adata.readList) - wLength = len(adata.writeList) - do: - # descriptor was unregistered in callback via `unregister()`. - rLength = -1 - wLength = -1 - - template processCustomCallbacks(ident: untyped) = - # Process pending custom event callbacks. Custom events are - # {Event.Timer, Event.Signal, Event.Process, Event.Vnode}. - # There can be only one callback registered with one descriptor, - # so there no need to iterate over list. - var curList: seq[Callback] - - withData(p.selector, ident, adata) do: - shallowCopy(curList, adata.readList) - adata.readList = newSeqOfCap[Callback](InitCallbackListSize) - - let newLength = len(curList) - var newList = newSeqOfCap[Callback](newLength) - - var cb = curList[0] - if not cb(fd.AsyncFD): - newList.add(cb) - - withData(p.selector, ident, adata) do: - # descriptor still present in queue. - adata.readList = newList & adata.readList - if len(adata.readList) == 0: - # if no callbacks registered with descriptor, unregister it. - p.selector.unregister(fd) - do: - # descriptor was unregistered in callback via `unregister()`. - discard - - proc poll*(timeout = 500) = - var keys: array[64, ReadyKey] - - let p = getGlobalDispatcher() - when ioselSupportedPlatform: - let customSet = {Event.Timer, Event.Signal, Event.Process, - Event.Vnode} - - if p.selector.isEmpty() and p.timers.len == 0 and p.callbacks.len == 0: - raise newException(ValueError, - "No handles or timers registered in dispatcher.") - - if not p.selector.isEmpty(): - var count = p.selector.selectInto(p.adjustedTimeout(timeout), keys) - var i = 0 - while i < count: - var custom = false - let fd = keys[i].fd - let events = keys[i].events - var rLength = 0 # len(data.readList) after callback - var wLength = 0 # len(data.writeList) after callback - - if Event.Read in events or events == {Event.Error}: - processBasicCallbacks(fd, readList) - - if Event.Write in events or events == {Event.Error}: - processBasicCallbacks(fd, writeList) - - if Event.User in events or events == {Event.Error}: - processBasicCallbacks(fd, readList) - custom = true - if rLength == 0: - p.selector.unregister(fd) - - when ioselSupportedPlatform: - if (customSet * events) != {}: - custom = true - processCustomCallbacks(fd) - - # because state `data` can be modified in callback we need to update - # descriptor events with currently registered callbacks. - if not custom: - var newEvents: set[Event] = {} - if rLength != -1 and wLength != -1: - if rLength > 0: incl(newEvents, Event.Read) - if wLength > 0: incl(newEvents, Event.Write) - p.selector.updateHandle(SocketHandle(fd), newEvents) - inc(i) - - # Timer processing. - processTimers(p) - # Callback queue processing - processPendingCallbacks(p) - - proc recv*(socket: AsyncFD, size: int, - flags = {SocketFlag.SafeDisconn}): Future[string] = - var retFuture = newFuture[string]("recv") - - var readBuffer = newString(size) - - proc cb(sock: AsyncFD): bool = - result = true - let res = recv(sock.SocketHandle, addr readBuffer[0], size.cint, - flags.toOSFlags()) - if res < 0: - let lastError = osLastError() - if lastError.int32 notin {EINTR, EWOULDBLOCK, EAGAIN}: - if flags.isDisconnectionError(lastError): - retFuture.complete("") - else: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) - else: - result = false # We still want this callback to be called. - elif res == 0: - # Disconnected - retFuture.complete("") - else: - readBuffer.setLen(res) - retFuture.complete(readBuffer) - # TODO: The following causes a massive slowdown. - #if not cb(socket): - addRead(socket, cb) - return retFuture - - proc recvInto*(socket: AsyncFD, buf: pointer, size: int, - flags = {SocketFlag.SafeDisconn}): Future[int] = - var retFuture = newFuture[int]("recvInto") - - proc cb(sock: AsyncFD): bool = - result = true - let res = recv(sock.SocketHandle, buf, size.cint, - flags.toOSFlags()) - if res < 0: - let lastError = osLastError() - if lastError.int32 notin {EINTR, EWOULDBLOCK, EAGAIN}: - if flags.isDisconnectionError(lastError): - retFuture.complete(0) - else: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) - else: - result = false # We still want this callback to be called. - else: - retFuture.complete(res) - # TODO: The following causes a massive slowdown. - #if not cb(socket): - addRead(socket, cb) - return retFuture - - proc send*(socket: AsyncFD, buf: pointer, size: int, - flags = {SocketFlag.SafeDisconn}): Future[void] = - var retFuture = newFuture[void]("send") - - var written = 0 - - proc cb(sock: AsyncFD): bool = - result = true - let netSize = size-written - var d = cast[cstring](buf) - let res = send(sock.SocketHandle, addr d[written], netSize.cint, - MSG_NOSIGNAL) - if res < 0: - let lastError = osLastError() - if lastError.int32 notin {EINTR, EWOULDBLOCK, EAGAIN}: - if flags.isDisconnectionError(lastError): - retFuture.complete() - else: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) - else: - result = false # We still want this callback to be called. - else: - written.inc(res) - if res != netSize: - result = false # We still have data to send. - else: - retFuture.complete() - # TODO: The following causes crashes. - #if not cb(socket): - addWrite(socket, cb) - return retFuture - - proc send*(socket: AsyncFD, data: string, - flags = {SocketFlag.SafeDisconn}): Future[void] = - var retFuture = newFuture[void]("send") - - var written = 0 - - proc cb(sock: AsyncFD): bool = - result = true - let netSize = data.len-written - var d = data.cstring - let res = send(sock.SocketHandle, addr d[written], netSize.cint, - MSG_NOSIGNAL) - if res < 0: - let lastError = osLastError() - if lastError.int32 notin {EINTR, EWOULDBLOCK, EAGAIN}: - if flags.isDisconnectionError(lastError): - retFuture.complete() - else: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) - else: - result = false # We still want this callback to be called. - else: - written.inc(res) - if res != netSize: - result = false # We still have data to send. - else: - retFuture.complete() - # TODO: The following causes crashes. - #if not cb(socket): - addWrite(socket, cb) - return retFuture - - proc sendTo*(socket: AsyncFD, data: pointer, size: int, saddr: ptr SockAddr, - saddrLen: SockLen, - flags = {SocketFlag.SafeDisconn}): Future[void] = - ## Sends ``data`` of size ``size`` in bytes to specified destination - ## (``saddr`` of size ``saddrLen`` in bytes, using socket ``socket``. - ## The returned future will complete once all data has been sent. - var retFuture = newFuture[void]("sendTo") - - # we will preserve address in our stack - var staddr: array[128, char] # SOCKADDR_STORAGE size is 128 bytes - var stalen = saddrLen - zeroMem(addr(staddr[0]), 128) - copyMem(addr(staddr[0]), saddr, saddrLen) - - proc cb(sock: AsyncFD): bool = - result = true - let res = sendto(sock.SocketHandle, data, size, MSG_NOSIGNAL, - cast[ptr SockAddr](addr(staddr[0])), stalen) - if res < 0: - let lastError = osLastError() - if lastError.int32 notin {EINTR, EWOULDBLOCK, EAGAIN}: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) - else: - result = false # We still want this callback to be called. - else: - retFuture.complete() - - addWrite(socket, cb) - return retFuture - - proc recvFromInto*(socket: AsyncFD, data: pointer, size: int, - saddr: ptr SockAddr, saddrLen: ptr SockLen, - flags = {SocketFlag.SafeDisconn}): Future[int] = - ## Receives a datagram data from ``socket`` into ``data``, which must - ## be at least of size ``size`` in bytes, address of datagram's sender - ## will be stored into ``saddr`` and ``saddrLen``. Returned future will - ## complete once one datagram has been received, and will return size - ## of packet received. - var retFuture = newFuture[int]("recvFromInto") - proc cb(sock: AsyncFD): bool = - result = true - let res = recvfrom(sock.SocketHandle, data, size.cint, flags.toOSFlags(), - saddr, saddrLen) - if res < 0: - let lastError = osLastError() - if lastError.int32 notin {EINTR, EWOULDBLOCK, EAGAIN}: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) - else: - result = false - else: - retFuture.complete(res) - addRead(socket, cb) - return retFuture - - proc acceptAddr*(socket: AsyncFD, flags = {SocketFlag.SafeDisconn}): - Future[tuple[address: string, client: AsyncFD]] = - var retFuture = newFuture[tuple[address: string, - client: AsyncFD]]("acceptAddr") - proc cb(sock: AsyncFD): bool = - result = true - var sockAddress: Sockaddr_storage - var addrLen = sizeof(sockAddress).Socklen - var client = accept(sock.SocketHandle, - cast[ptr SockAddr](addr(sockAddress)), addr(addrLen)) - if client == osInvalidSocket: - let lastError = osLastError() - assert lastError.int32 notin {EWOULDBLOCK, EAGAIN} - if lastError.int32 == EINTR: - return false - else: - if flags.isDisconnectionError(lastError): - return false - else: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) - else: - try: - let address = getAddrString(cast[ptr SockAddr](addr sockAddress)) - register(client.AsyncFD) - retFuture.complete((address, client.AsyncFD)) - except: - # getAddrString may raise - client.close() - retFuture.fail(getCurrentException()) - addRead(socket, cb) - return retFuture - - when ioselSupportedPlatform: - - proc addTimer*(timeout: int, oneshot: bool, cb: Callback) = - ## Start watching for timeout expiration, and then call the - ## callback ``cb``. - ## ``timeout`` - time in milliseconds, - ## ``oneshot`` - if ``true`` only one event will be dispatched, - ## if ``false`` continuous events every ``timeout`` milliseconds. - let p = getGlobalDispatcher() - var data = newAsyncData() - data.readList.add(cb) - p.selector.registerTimer(timeout, oneshot, data) - - proc addSignal*(signal: int, cb: Callback) = - ## Start watching signal ``signal``, and when signal appears, call the - ## callback ``cb``. - let p = getGlobalDispatcher() - var data = newAsyncData() - data.readList.add(cb) - p.selector.registerSignal(signal, data) - - proc addProcess*(pid: int, cb: Callback) = - ## Start watching for process exit with pid ``pid``, and then call - ## the callback ``cb``. - let p = getGlobalDispatcher() - var data = newAsyncData() - data.readList.add(cb) - p.selector.registerProcess(pid, data) - - proc newAsyncEvent*(): AsyncEvent = - ## Creates new ``AsyncEvent``. - result = AsyncEvent(newSelectEvent()) - - proc setEvent*(ev: AsyncEvent) = - ## Sets new ``AsyncEvent`` to signaled state. - setEvent(SelectEvent(ev)) - - proc close*(ev: AsyncEvent) = - ## Closes ``AsyncEvent`` - close(SelectEvent(ev)) - - proc addEvent*(ev: AsyncEvent, cb: Callback) = - ## Start watching for event ``ev``, and call callback ``cb``, when - ## ev will be set to signaled state. - let p = getGlobalDispatcher() - var data = newAsyncData() - data.readList.add(cb) - p.selector.registerEvent(SelectEvent(ev), data) - -# Common procedures between current and upcoming asyncdispatch -include includes.asynccommon - -proc sleepAsync*(ms: int): Future[void] = - ## Suspends the execution of the current async procedure for the next - ## ``ms`` milliseconds. - var retFuture = newFuture[void]("sleepAsync") - let p = getGlobalDispatcher() - p.timers.push((epochTime() + (ms / 1000), retFuture)) - return retFuture - -proc withTimeout*[T](fut: Future[T], timeout: int): Future[bool] = - ## Returns a future which will complete once ``fut`` completes or after - ## ``timeout`` milliseconds has elapsed. - ## - ## If ``fut`` completes first the returned future will hold true, - ## otherwise, if ``timeout`` milliseconds has elapsed first, the returned - ## future will hold false. - - var retFuture = newFuture[bool]("asyncdispatch.`withTimeout`") - var timeoutFuture = sleepAsync(timeout) - fut.callback = - proc () = - if not retFuture.finished: retFuture.complete(true) - timeoutFuture.callback = - proc () = - if not retFuture.finished: retFuture.complete(false) - return retFuture - -proc accept*(socket: AsyncFD, - flags = {SocketFlag.SafeDisconn}): Future[AsyncFD] = - ## Accepts a new connection. Returns a future containing the client socket - ## corresponding to that connection. - ## The future will complete when the connection is successfully accepted. - var retFut = newFuture[AsyncFD]("accept") - var fut = acceptAddr(socket, flags) - fut.callback = - proc (future: Future[tuple[address: string, client: AsyncFD]]) = - assert future.finished - if future.failed: - retFut.fail(future.error) - else: - retFut.complete(future.read.client) - return retFut - -# -- Await Macro -include asyncmacro - -proc readAll*(future: FutureStream[string]): Future[string] {.async.} = - ## Returns a future that will complete when all the string data from the - ## specified future stream is retrieved. - result = "" - while true: - let (hasValue, value) = await future.read() - if hasValue: - result.add(value) - else: - break - -proc recvLine*(socket: AsyncFD): Future[string] {.async.} = - ## Reads a line of data from ``socket``. Returned future will complete once - ## a full line is read or an error occurs. - ## - ## If a full line is read ``\r\L`` is not - ## added to ``line``, however if solely ``\r\L`` is read then ``line`` - ## will be set to it. - ## - ## If the socket is disconnected, ``line`` will be set to ``""``. - ## - ## If the socket is disconnected in the middle of a line (before ``\r\L`` - ## is read) then line will be set to ``""``. - ## The partial line **will be lost**. - ## - ## **Warning**: This assumes that lines are delimited by ``\r\L``. - ## - ## **Note**: This procedure is mostly used for testing. You likely want to - ## use ``asyncnet.recvLine`` instead. - - template addNLIfEmpty(): typed = - if result.len == 0: - result.add("\c\L") - - result = "" - var c = "" - while true: - c = await recv(socket, 1) - if c.len == 0: - return "" - if c == "\r": - c = await recv(socket, 1) - assert c == "\l" - addNLIfEmpty() - return - elif c == "\L": - addNLIfEmpty() - return - add(result, c) - -proc callSoon(cbproc: proc ()) = - ## Schedule `cbproc` to be called as soon as possible. - ## The callback is called when control returns to the event loop. - getGlobalDispatcher().callbacks.addLast(cbproc) - -proc runForever*() = - ## Begins a never ending global dispatcher poll loop. - while true: - poll() - -proc waitFor*[T](fut: Future[T]): T = - ## **Blocks** the current thread until the specified future completes. - while not fut.finished: - poll() - - fut.read diff --git a/readme.md b/readme.md index 1d4ef4f5b..ebcf5f802 100644 --- a/readme.md +++ b/readme.md @@ -95,6 +95,7 @@ This project exists thanks to all the people who contribute. [Read on to find ou [![Backers on Open Collective](https://opencollective.com/nim/backers/badge.svg)](#backers) [![Sponsors on Open Collective](https://opencollective.com/nim/sponsors/badge.svg)](#sponsors) [![Setup a bounty via Bountysource][badge-nim-bountysource]][nim-bountysource] [![Donate Bitcoins][badge-nim-bitcoin]][nim-bitcoin] +[![Open Source Helpers](https://www.codetriage.com/nim-lang/nim/badges/users.svg)](https://www.codetriage.com/nim-lang/nim) We welcome all contributions to Nim regardless of how small or large they are. Everything from spelling fixes to new modules to be included in the diff --git a/tests/ccgbugs/t7079.nim b/tests/ccgbugs/t7079.nim new file mode 100644 index 000000000..bfa1b77a6 --- /dev/null +++ b/tests/ccgbugs/t7079.nim @@ -0,0 +1,9 @@ +discard """ + action: run + targets: '''c js''' +""" + +import math +let x = -0.0 +doAssert classify(x) == fcNegZero +doAssert classify(1 / -0.0) == fcNegInf \ No newline at end of file diff --git a/tests/concepts/t3330.nim b/tests/concepts/t3330.nim index bf8bffc7d..6c9883618 100644 --- a/tests/concepts/t3330.nim +++ b/tests/concepts/t3330.nim @@ -1,25 +1,48 @@ discard """ errormsg: "type mismatch: got (Bar[system.int])" nimout: ''' -t3330.nim(40, 4) Error: type mismatch: got (Bar[system.int]) +t3330.nim(63, 4) Error: type mismatch: got (Bar[system.int]) but expected one of: proc test(foo: Foo[int]) -t3330.nim(25, 8) Hint: Non-matching candidates for add(k, string, T) +t3330.nim(48, 8) Hint: Non-matching candidates for add(k, string, T) proc add(x: var string; y: string) +first type mismatch at position: 1 +required type: var string +but expression 'k' is of type: Alias proc add(x: var string; y: char) +first type mismatch at position: 1 +required type: var string +but expression 'k' is of type: Alias proc add(result: var string; x: int64) +first type mismatch at position: 1 +required type: var string +but expression 'k' is of type: Alias proc add(result: var string; x: float) +first type mismatch at position: 1 +required type: var string +but expression 'k' is of type: Alias proc add(x: var string; y: cstring) +first type mismatch at position: 1 +required type: var string +but expression 'k' is of type: Alias proc add[T](x: var seq[T]; y: openArray[T]) +first type mismatch at position: 1 +required type: var seq[T] +but expression 'k' is of type: Alias proc add[T](x: var seq[T]; y: T) +first type mismatch at position: 1 +required type: var seq[T] +but expression 'k' is of type: Alias -t3330.nim(25, 8) template/generic instantiation from here -t3330.nim(32, 6) Foo: 'bar.value' cannot be assigned to -t3330.nim(25, 8) template/generic instantiation from here -t3330.nim(33, 6) Foo: 'bar.x' cannot be assigned to -''' +t3330.nim(48, 8) template/generic instantiation from here +t3330.nim(55, 6) Foo: 'bar.value' cannot be assigned to +t3330.nim(48, 8) template/generic instantiation from here +t3330.nim(56, 6) Foo: 'bar.x' cannot be assigned to + +expression: test(bar)''' """ + type Foo[T] = concept k add(k, string, T) diff --git a/tests/discard/tneedsdiscard.nim b/tests/discard/tneedsdiscard.nim index 509e8233b..8d59e7fec 100644 --- a/tests/discard/tneedsdiscard.nim +++ b/tests/discard/tneedsdiscard.nim @@ -1,6 +1,6 @@ discard """ line: 10 - errormsg: '''expression 'open(f, "arg.txt", fmRead, -1)' is of type 'bool' and has to be discarded''' + errormsg: '''expression 'open(f, "arg.txt", fmRead, -1)' is of type 'bool' and has to be discarded; start of expression here: tneedsdiscard.nim(7, 2)''' """ proc p = diff --git a/tests/errmsgs/tdetailed_position.nim b/tests/errmsgs/tdetailed_position.nim new file mode 100644 index 000000000..d6db94c3e --- /dev/null +++ b/tests/errmsgs/tdetailed_position.nim @@ -0,0 +1,22 @@ + +discard """ +cmd: "nim check $file" +errormsg: "type mismatch: got (int literal(1), int literal(2), int literal(3))" +nimout: ''' +but expected one of: +proc main(a, b, c: string) +first type mismatch at position: 1 +required type: string +but expression '1' is of type: int literal(1) + +expression: main(1, 2, 3) +''' +""" + +const + myconst = "abcdefghijklmnopqrstuvwxyz" + +proc main(a, b, c: string) {.deprecated: "use foo " & "instead " & myconst.} = + return + +main(1, 2, 3) diff --git a/tests/errmsgs/twrongcolon.nim b/tests/errmsgs/twrongcolon.nim new file mode 100644 index 000000000..6f5cc3e5d --- /dev/null +++ b/tests/errmsgs/twrongcolon.nim @@ -0,0 +1,11 @@ +discard """ +errormsg: "in expression '(" +nimout: ''' +Error: in expression '( + 890)': identifier expected, but found '' +''' + +line: 11 +""" + +var n: int : 890 diff --git a/tests/exception/texcas.nim b/tests/exception/texcas.nim index 4b4ebe448..fee45af3f 100644 --- a/tests/exception/texcas.nim +++ b/tests/exception/texcas.nim @@ -21,5 +21,13 @@ proc test2() = testTemplate(Exception) doAssert(not declared(foobar)) + +proc testTryAsExpr(i: int) = + let x = try: i + except ValueError as ex: + echo(ex.msg) + -1 + test[Exception]() -test2() \ No newline at end of file +test2() +testTryAsExpr(5) diff --git a/tests/generics/tspecial_numeric_inference.nim b/tests/generics/tspecial_numeric_inference.nim new file mode 100644 index 000000000..d93544ba4 --- /dev/null +++ b/tests/generics/tspecial_numeric_inference.nim @@ -0,0 +1,12 @@ +discard """ + output: '''int64 +int64''' +""" + +import typetraits + +proc `@`[T: SomeInteger](x, y: T): T = x + +echo(type(5'i64 @ 6'i32)) + +echo(type(5'i32 @ 6'i64)) diff --git a/tests/system/io.nim b/tests/system/io.nim index b0ccfda9f..3d4df806b 100644 --- a/tests/system/io.nim +++ b/tests/system/io.nim @@ -1,5 +1,5 @@ import - unittest, osproc, streams, os + unittest, osproc, streams, os, strformat const STRING_DATA = "Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet." const TEST_FILE = "tests/testdata/string.txt" @@ -23,3 +23,26 @@ suite "io": test "file": check: readFile(TEST_FILE) == STRING_DATA + + +proc verifyFileSize(sz: int64) = + # issue 7121, large file size (2-4GB and >4Gb) + const fn = "tmpfile112358" + let size_in_mb = sz div 1_000_000 + + when defined(windows): + discard execProcess(&"fsutil file createnew {fn} {sz}" ) + else: + discard execProcess(&"dd if=/dev/zero of={fn} bs=1000000 count={size_in_mb}") + + doAssert os.getFileSize(fn) == sz # Verify OS filesize by string + + var f = open(fn) + doAssert f.getFileSize() == sz # Verify file handle filesize + f.close() + + os.removeFile(fn) + +#disable tests for automatic testers +#for s in [50_000_000'i64, 3_000_000_000, 5_000_000_000]: +# verifyFileSize(s) diff --git a/tools/niminst/niminst.nim b/tools/niminst/niminst.nim index 9c15326b0..c2816a0ef 100644 --- a/tools/niminst/niminst.nim +++ b/tools/niminst/niminst.nim @@ -15,7 +15,7 @@ when haveZipLib: import os, osproc, strutils, parseopt, parsecfg, strtabs, streams, debcreation, - securehash + std / sha1 const maxOS = 20 # max number of OSes diff --git a/web/website.ini b/web/website.ini index 273c3223d..7edaaa712 100644 --- a/web/website.ini +++ b/web/website.ini @@ -65,7 +65,7 @@ srcdoc2: "pure/asyncfile;pure/asyncftpclient;pure/lenientops" srcdoc2: "pure/md5;pure/rationals" srcdoc2: "posix/posix;pure/distros;pure/oswalkdir" srcdoc2: "pure/collections/heapqueue" -srcdoc2: "pure/fenv;pure/securehash;impure/rdstdin;pure/strformat" +srcdoc2: "pure/fenv;std/sha1;impure/rdstdin;pure/strformat" srcdoc2: "pure/segfaults" srcdoc2: "pure/basic2d;pure/basic3d;pure/mersenne;pure/coro;pure/httpcore" srcdoc2: "pure/bitops;pure/nimtracker;pure/punycode;pure/volatile;js/asyncjs" |