diff options
author | Jacek Sieka <arnetheduck@gmail.com> | 2016-08-25 22:59:51 +0800 |
---|---|---|
committer | Jacek Sieka <arnetheduck@gmail.com> | 2016-08-25 22:59:51 +0800 |
commit | db2f96daba9c04db2f24cb783c79fb37799cd9ea (patch) | |
tree | 567beb43c7e4549abfcae1ea66e5232d7525e001 /lib | |
parent | 3116744c86f37ac4e4e5fec3d6d1635304ed717f (diff) | |
parent | 84a09d2f5b0866491e55fef0fef541e8cc548852 (diff) | |
download | Nim-db2f96daba9c04db2f24cb783c79fb37799cd9ea.tar.gz |
Merge remote-tracking branch 'origin/devel' into initallocator-fix
Diffstat (limited to 'lib')
109 files changed, 11466 insertions, 2177 deletions
diff --git a/lib/core/locks.nim b/lib/core/locks.nim index 66e0ab520..fbe9c8acf 100644 --- a/lib/core/locks.nim +++ b/lib/core/locks.nim @@ -9,6 +9,7 @@ ## This module contains Nim's support for locks and condition vars. +const insideRLocksModule = false include "system/syslocks" type @@ -63,4 +64,4 @@ template withLock*(a: Lock, body: untyped) = try: body finally: - a.release() \ No newline at end of file + a.release() diff --git a/lib/core/macros.nim b/lib/core/macros.nim index 4522e0fc6..4296cb0ae 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -12,7 +12,7 @@ include "system/inclrtl" ## This module contains the interface to the compiler's abstract syntax ## tree (`AST`:idx:). Macros operate on this tree. -## .. include:: ../doc/astspec.txt +## .. include:: ../../doc/astspec.txt type NimNodeKind* = enum @@ -197,6 +197,18 @@ proc typeKind*(n: NimNode): NimTypeKind {.magic: "NGetType", noSideEffect.} ## Returns the type kind of the node 'n' that should represent a type, that ## means the node should have been obtained via `getType`. +proc getTypeInst*(n: NimNode): NimNode {.magic: "NGetType", noSideEffect.} + ## Like getType except it includes generic parameters for a specific instance + +proc getTypeInst*(n: typedesc): NimNode {.magic: "NGetType", noSideEffect.} + ## Like getType except it includes generic parameters for a specific instance + +proc getTypeImpl*(n: NimNode): NimNode {.magic: "NGetType", noSideEffect.} + ## Like getType except it includes generic parameters for the implementation + +proc getTypeImpl*(n: typedesc): NimNode {.magic: "NGetType", noSideEffect.} + ## Like getType except it includes generic parameters for the implementation + proc strVal*(n: NimNode): string {.magic: "NStrVal", noSideEffect.} proc `intVal=`*(n: NimNode, val: BiggestInt) {.magic: "NSetIntVal", noSideEffect.} @@ -496,7 +508,7 @@ proc lispRepr*(n: NimNode): string {.compileTime, benign.} = add(result, ")") -macro dumpTree*(s: stmt): stmt {.immediate.} = echo s.treeRepr +macro dumpTree*(s: untyped): untyped = echo s.treeRepr ## Accepts a block of nim code and prints the parsed abstract syntax ## tree using the `toTree` function. Printing is done *at compile time*. ## @@ -504,17 +516,17 @@ macro dumpTree*(s: stmt): stmt {.immediate.} = echo s.treeRepr ## tree and to discover what kind of nodes must be created to represent ## a certain expression/statement. -macro dumpLisp*(s: stmt): stmt {.immediate.} = echo s.lispRepr +macro dumpLisp*(s: untyped): untyped = echo s.lispRepr ## Accepts a block of nim code and prints the parsed abstract syntax ## tree using the `toLisp` function. Printing is done *at compile time*. ## ## See `dumpTree`. -macro dumpTreeImm*(s: stmt): stmt {.immediate, deprecated.} = echo s.treeRepr - ## The ``immediate`` version of `dumpTree`. +macro dumpTreeImm*(s: untyped): untyped {.deprecated.} = echo s.treeRepr + ## Deprecated. -macro dumpLispImm*(s: stmt): stmt {.immediate, deprecated.} = echo s.lispRepr - ## The ``immediate`` version of `dumpLisp`. +macro dumpLispImm*(s: untyped): untyped {.deprecated.} = echo s.lispRepr + ## Deprecated. proc newEmptyNode*(): NimNode {.compileTime, noSideEffect.} = @@ -680,8 +692,16 @@ proc `pragma=`*(someProc: NimNode; val: NimNode){.compileTime.}= assert val.kind in {nnkEmpty, nnkPragma} someProc[4] = val +proc addPragma*(someProc, pragma: NimNode) {.compileTime.} = + ## Adds pragma to routine definition + someProc.expectRoutine + var pragmaNode = someProc.pragma + if pragmaNode.isNil or pragmaNode.kind == nnkEmpty: + pragmaNode = newNimNode(nnkPragma) + someProc.pragma = pragmaNode + pragmaNode.add(pragma) -template badNodeKind(k; f): stmt{.immediate.} = +template badNodeKind(k, f) = assert false, "Invalid node kind " & $k & " for macros.`" & $f & "`" proc body*(someProc: NimNode): NimNode {.compileTime.} = @@ -738,20 +758,19 @@ iterator children*(n: NimNode): NimNode {.inline.} = for i in 0 ..< n.len: yield n[i] -template findChild*(n: NimNode; cond: expr): NimNode {. - immediate, dirty.} = +template findChild*(n: NimNode; cond: untyped): NimNode {.dirty.} = ## Find the first child node matching condition (or nil). ## ## .. code-block:: nim ## var res = findChild(n, it.kind == nnkPostfix and ## it.basename.ident == !"foo") block: - var result: NimNode + var res: NimNode for it in n.children: if cond: - result = it + res = it break - result + res proc insert*(a: NimNode; pos: int; b: NimNode) {.compileTime.} = ## Insert node B into A at pos @@ -796,17 +815,17 @@ proc infix*(a: NimNode; op: string; proc unpackPostfix*(node: NimNode): tuple[node: NimNode; op: string] {. compileTime.} = node.expectKind nnkPostfix - result = (node[0], $node[1]) + result = (node[1], $node[0]) proc unpackPrefix*(node: NimNode): tuple[node: NimNode; op: string] {. compileTime.} = node.expectKind nnkPrefix - result = (node[0], $node[1]) + result = (node[1], $node[0]) proc unpackInfix*(node: NimNode): tuple[left: NimNode; op: string; right: NimNode] {.compileTime.} = assert node.kind == nnkInfix - result = (node[0], $node[1], node[2]) + result = (node[1], $node[0], node[2]) proc copy*(node: NimNode): NimNode {.compileTime.} = ## An alias for copyNimTree(). @@ -818,6 +837,8 @@ proc cmpIgnoreStyle(a, b: cstring): int {.noSideEffect.} = else: result = c var i = 0 var j = 0 + # first char is case sensitive + if a[0] != b[0]: return 1 while true: while a[i] == '_': inc(i) while b[j] == '_': inc(j) # BUGFIX: typo @@ -828,9 +849,23 @@ proc cmpIgnoreStyle(a, b: cstring): int {.noSideEffect.} = inc(i) inc(j) -proc eqIdent* (a, b: string): bool = cmpIgnoreStyle(a, b) == 0 +proc eqIdent*(a, b: string): bool = cmpIgnoreStyle(a, b) == 0 ## Check if two idents are identical. +proc eqIdent*(node: NimNode; s: string): bool {.compileTime.} = + ## Check if node is some identifier node (``nnkIdent``, ``nnkSym``, etc.) + ## is the same as ``s``. Note that this is the preferred way to check! Most + ## other ways like ``node.ident`` are much more error-prone, unfortunately. + case node.kind + of nnkIdent: + result = node.ident == !s + of nnkSym: + result = eqIdent($node.symbol, s) + of nnkOpenSymChoice, nnkClosedSymChoice: + result = eqIdent($node[0], s) + else: + result = false + proc hasArgOfName* (params: NimNode; name: string): bool {.compiletime.}= ## Search nnkFormalParams for an argument. assert params.kind == nnkFormalParams @@ -856,7 +891,7 @@ proc boolVal*(n: NimNode): bool {.compileTime, noSideEffect.} = else: n == bindSym"true" # hacky solution for now when not defined(booting): - template emit*(e: static[string]): stmt = + template emit*(e: static[string]): stmt {.deprecated.} = ## accepts a single string argument and treats it as nim code ## that should be inserted verbatim in the program ## Example: @@ -864,6 +899,7 @@ when not defined(booting): ## .. code-block:: nim ## emit("echo " & '"' & "hello world".toUpper & '"') ## + ## Deprecated since version 0.15 since it's so rarely useful. macro payload: stmt {.gensym.} = result = parseStmt(e) payload() diff --git a/lib/core/rlocks.nim b/lib/core/rlocks.nim index 14f04592b..4710d6cf1 100644 --- a/lib/core/rlocks.nim +++ b/lib/core/rlocks.nim @@ -9,6 +9,7 @@ ## This module contains Nim's support for reentrant locks. +const insideRLocksModule = true include "system/syslocks" type diff --git a/lib/deprecated/pure/ftpclient.nim b/lib/deprecated/pure/ftpclient.nim index 1188c0795..ed2f14450 100644 --- a/lib/deprecated/pure/ftpclient.nim +++ b/lib/deprecated/pure/ftpclient.nim @@ -129,10 +129,10 @@ proc ftpClient*(address: string, port = Port(21), result.csock = socket() if result.csock == invalidSocket: raiseOSError(osLastError()) -template blockingOperation(sock: Socket, body: stmt) {.immediate.} = +template blockingOperation(sock: Socket, body: untyped) = body -template blockingOperation(sock: asyncio.AsyncSocket, body: stmt) {.immediate.} = +template blockingOperation(sock: asyncio.AsyncSocket, body: untyped) = sock.setBlocking(true) body sock.setBlocking(false) diff --git a/lib/deprecated/pure/sockets.nim b/lib/deprecated/pure/sockets.nim index 20e6d9364..34c2e7a7d 100644 --- a/lib/deprecated/pure/sockets.nim +++ b/lib/deprecated/pure/sockets.nim @@ -711,8 +711,13 @@ proc getHostByAddr*(ip: string): Hostent {.tags: [ReadIOEffect].} = cint(sockets.AF_INET)) if s == nil: raiseOSError(osLastError()) else: - var s = posix.gethostbyaddr(addr(myaddr), sizeof(myaddr).Socklen, - cint(posix.AF_INET)) + var s = + when defined(android4): + posix.gethostbyaddr(cast[cstring](addr(myaddr)), sizeof(myaddr).cint, + cint(posix.AF_INET)) + else: + posix.gethostbyaddr(addr(myaddr), sizeof(myaddr).Socklen, + cint(posix.AF_INET)) if s == nil: raiseOSError(osLastError(), $hstrerror(h_errno)) diff --git a/lib/impure/db_odbc.nim b/lib/impure/db_odbc.nim index 3a14e6304..d6343acc7 100644 --- a/lib/impure/db_odbc.nim +++ b/lib/impure/db_odbc.nim @@ -210,7 +210,7 @@ proc dbFormat(formatstr: SqlQuery, args: varargs[string]): string {. add(result, c) proc prepareFetch(db: var DbConn, query: SqlQuery, - args: varargs[string, `$`]) : TSqlSmallInt {. + args: varargs[string, `$`]): TSqlSmallInt {. tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} = # Prepare a statement, execute it and fetch the data to the driver # ready for retrieval of the data @@ -222,9 +222,8 @@ proc prepareFetch(db: var DbConn, query: SqlQuery, var q = dbFormat(query, args) db.sqlCheck(SQLPrepare(db.stmt, q.PSQLCHAR, q.len.TSqlSmallInt)) db.sqlCheck(SQLExecute(db.stmt)) - var retcode = SQLFetch(db.stmt) - db.sqlCheck(retcode) - result=retcode + result = SQLFetch(db.stmt) + db.sqlCheck(result) proc prepareFetchDirect(db: var DbConn, query: SqlQuery, args: varargs[string, `$`]) {. @@ -250,8 +249,8 @@ proc tryExec*(db: var DbConn, query: SqlQuery, args: varargs[string, `$`]): bool var rCnt = -1 res = SQLRowCount(db.stmt, rCnt) - if res != SQL_SUCCESS: dbError(db) properFreeResult(SQL_HANDLE_STMT, db.stmt) + if res != SQL_SUCCESS: dbError(db) except: discard return res == SQL_SUCCESS @@ -286,10 +285,10 @@ iterator fastRows*(db: var DbConn, query: SqlQuery, sz: TSqlSmallInt = 0 cCnt: TSqlSmallInt = 0.TSqlSmallInt res: TSqlSmallInt = 0.TSqlSmallInt - tempcCnt:TSqlSmallInt # temporary cCnt,Fix the field values to be null when the release schema is compiled. - # tempcCnt,A field to store the number of temporary variables, for unknown reasons, + tempcCnt: TSqlSmallInt # temporary cCnt,Fix the field values to be null when the release schema is compiled. + # tempcCnt,A field to store the number of temporary variables, for unknown reasons, # after performing a sqlgetdata function and circulating variables cCnt value will be changed to 0, - # so the values of the temporary variable to store the cCnt. + # so the values of the temporary variable to store the cCnt. # After every cycle and specified to cCnt. To ensure the traversal of all fields. res = db.prepareFetch(query, args) if res == SQL_NO_DATA: @@ -308,8 +307,8 @@ iterator fastRows*(db: var DbConn, query: SqlQuery, cCnt = tempcCnt yield rowRes res = SQLFetch(db.stmt) - db.sqlCheck(res) properFreeResult(SQL_HANDLE_STMT, db.stmt) + db.sqlCheck(res) iterator instantRows*(db: var DbConn, query: SqlQuery, args: varargs[string, `$`]): InstantRow @@ -317,14 +316,14 @@ iterator instantRows*(db: var DbConn, query: SqlQuery, ## Same as fastRows but returns a handle that can be used to get column text ## on demand using []. Returned handle is valid only within the interator body. var - rowRes: Row + rowRes: Row = @[] sz: TSqlSmallInt = 0 cCnt: TSqlSmallInt = 0.TSqlSmallInt res: TSqlSmallInt = 0.TSqlSmallInt - tempcCnt:TSqlSmallInt # temporary cCnt,Fix the field values to be null when the release schema is compiled. - # tempcCnt,A field to store the number of temporary variables, for unknown reasons, + tempcCnt: TSqlSmallInt # temporary cCnt,Fix the field values to be null when the release schema is compiled. + # tempcCnt,A field to store the number of temporary variables, for unknown reasons, # after performing a sqlgetdata function and circulating variables cCnt value will be changed to 0, - # so the values of the temporary variable to store the cCnt. + # so the values of the temporary variable to store the cCnt. # After every cycle and specified to cCnt. To ensure the traversal of all fields. res = db.prepareFetch(query, args) if res == SQL_NO_DATA: @@ -343,8 +342,8 @@ iterator instantRows*(db: var DbConn, query: SqlQuery, cCnt = tempcCnt yield (row: rowRes, len: cCnt.int) res = SQLFetch(db.stmt) - db.sqlCheck(res) properFreeResult(SQL_HANDLE_STMT, db.stmt) + db.sqlCheck(res) proc `[]`*(row: InstantRow, col: int): string {.inline.} = ## Returns text for given column of the row @@ -364,10 +363,10 @@ proc getRow*(db: var DbConn, query: SqlQuery, sz: TSqlSmallInt = 0.TSqlSmallInt cCnt: TSqlSmallInt = 0.TSqlSmallInt res: TSqlSmallInt = 0.TSqlSmallInt - tempcCnt:TSqlSmallInt # temporary cCnt,Fix the field values to be null when the release schema is compiled. - ## tempcCnt,A field to store the number of temporary variables, for unknown reasons, + tempcCnt: TSqlSmallInt # temporary cCnt,Fix the field values to be null when the release schema is compiled. + ## tempcCnt,A field to store the number of temporary variables, for unknown reasons, ## after performing a sqlgetdata function and circulating variables cCnt value will be changed to 0, - ## so the values of the temporary variable to store the cCnt. + ## so the values of the temporary variable to store the cCnt. ## After every cycle and specified to cCnt. To ensure the traversal of all fields. res = db.prepareFetch(query, args) if res == SQL_NO_DATA: @@ -385,8 +384,8 @@ proc getRow*(db: var DbConn, query: SqlQuery, cCnt = tempcCnt res = SQLFetch(db.stmt) result = rowRes - db.sqlCheck(res) properFreeResult(SQL_HANDLE_STMT, db.stmt) + db.sqlCheck(res) proc getAllRows*(db: var DbConn, query: SqlQuery, args: varargs[string, `$`]): seq[Row] {. @@ -398,10 +397,10 @@ proc getAllRows*(db: var DbConn, query: SqlQuery, sz: TSqlSmallInt = 0 cCnt: TSqlSmallInt = 0.TSqlSmallInt res: TSqlSmallInt = 0.TSqlSmallInt - tempcCnt:TSqlSmallInt # temporary cCnt,Fix the field values to be null when the release schema is compiled. - ## tempcCnt,A field to store the number of temporary variables, for unknown reasons, + tempcCnt: TSqlSmallInt # temporary cCnt,Fix the field values to be null when the release schema is compiled. + ## tempcCnt,A field to store the number of temporary variables, for unknown reasons, ## after performing a sqlgetdata function and circulating variables cCnt value will be changed to 0, - ## so the values of the temporary variable to store the cCnt. + ## so the values of the temporary variable to store the cCnt. ## After every cycle and specified to cCnt. To ensure the traversal of all fields. res = db.prepareFetch(query, args) if res == SQL_NO_DATA: @@ -421,8 +420,8 @@ proc getAllRows*(db: var DbConn, query: SqlQuery, rows.add(rowRes) res = SQLFetch(db.stmt) result = rows - db.sqlCheck(res) properFreeResult(SQL_HANDLE_STMT, db.stmt) + db.sqlCheck(res) iterator rows*(db: var DbConn, query: SqlQuery, args: varargs[string, `$`]): Row {. @@ -544,4 +543,4 @@ proc setEncoding*(connection: DbConn, encoding: string): bool {. ## Sets the encoding of a database connection, returns true for ## success, false for failure. ##result = set_character_set(connection, encoding) == 0 - dbError("setEncoding() is currently not implemented by the db_odbc module") \ No newline at end of file + dbError("setEncoding() is currently not implemented by the db_odbc module") diff --git a/lib/impure/nre.nim b/lib/impure/nre.nim index c8f690461..557bb0549 100644 --- a/lib/impure/nre.nim +++ b/lib/impure/nre.nim @@ -15,6 +15,8 @@ from math import ceil import options from unicode import runeLenAt +export options + ## What is NRE? ## ============ @@ -24,46 +26,34 @@ from unicode import runeLenAt ## Licencing ## --------- ## -## PCRE has some additional terms that you must comply with if you use this module.:: +## PCRE has `some additional terms`_ that you must agree to in order to use +## this module. +## +## .. _`some additional terms`: http://pcre.sourceforge.net/license.txt +## +## Example +## ------- +## +## .. code-block:: nim +## +## import nre ## -## > Copyright (c) 1997-2001 University of Cambridge -## > -## > Permission is granted to anyone to use this software for any purpose on any -## > computer system, and to redistribute it freely, subject to the following -## > restrictions: -## > -## > 1. This software is distributed in the hope that it will be useful, -## > but WITHOUT ANY WARRANTY; without even the implied warranty of -## > MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -## > -## > 2. The origin of this software must not be misrepresented, either by -## > explicit claim or by omission. In practice, this means that if you use -## > PCRE in software that you distribute to others, commercially or -## > otherwise, you must put a sentence like this -## > -## > Regular expression support is provided by the PCRE library package, -## > which is open source software, written by Philip Hazel, and copyright -## > by the University of Cambridge, England. -## > -## > somewhere reasonably visible in your documentation and in any relevant -## > files or online help data or similar. A reference to the ftp site for -## > the source, that is, to -## > -## > ftp://ftp.csx.cam.ac.uk/pub/software/programming/pcre/ -## > -## > should also be given in the documentation. However, this condition is not -## > intended to apply to whole chains of software. If package A includes PCRE, -## > it must acknowledge it, but if package B is software that includes package -## > A, the condition is not imposed on package B (unless it uses PCRE -## > independently). -## > -## > 3. Altered versions must be plainly marked as such, and must not be -## > misrepresented as being the original software. -## > -## > 4. If PCRE is embedded in any software that is released under the GNU -## > General Purpose Licence (GPL), or Lesser General Purpose Licence (LGPL), -## > then the terms of that licence shall supersede any condition above with -## > which it is incompatible. +## let vowels = re"[aeoui]" +## +## for match in "moigagoo".findIter(vowels): +## echo match.matchBounds +## # (a: 1, b: 1) +## # (a: 2, b: 2) +## # (a: 4, b: 4) +## # (a: 6, b: 6) +## # (a: 7, b: 7) +## +## let firstVowel = "foo".find(vowels) +## let hasVowel = firstVowel.isSome() +## if hasVowel: +## let matchBounds = firstVowel.get().captureBounds[-1] +## echo "first vowel @", matchBounds.get().a +## # first vowel @1 # Type definitions {{{ @@ -125,11 +115,11 @@ type ## - ``(*NO_STUDY)`` - turn off studying; study is enabled by default ## ## For more details on the leading option groups, see the `Option - ## Setting <http://man7.org/linux/man-pages/man3/pcresyntax.3.html#OPTION_SETTING>`__ + ## Setting <http://man7.org/linux/man-pages/man3/pcresyntax.3.html#OPTION_SETTING>`_ ## and the `Newline - ## Convention <http://man7.org/linux/man-pages/man3/pcresyntax.3.html#NEWLINE_CONVENTION>`__ + ## Convention <http://man7.org/linux/man-pages/man3/pcresyntax.3.html#NEWLINE_CONVENTION>`_ ## sections of the `PCRE syntax - ## manual <http://man7.org/linux/man-pages/man3/pcresyntax.3.html>`__. + ## manual <http://man7.org/linux/man-pages/man3/pcresyntax.3.html>`_. pattern*: string ## not nil pcreObj: ptr pcre.Pcre ## not nil pcreExtra: ptr pcre.ExtraData ## nil @@ -284,7 +274,7 @@ proc `[]`*(pattern: Captures, name: string): string = let pattern = RegexMatch(pattern) return pattern.captures[pattern.pattern.captureNameToId.fget(name)] -template toTableImpl(cond: bool): stmt {.immediate, dirty.} = +template toTableImpl(cond: untyped) {.dirty.} = for key in RegexMatch(pattern).pattern.captureNameId.keys: let nextVal = pattern[key] if cond: @@ -301,7 +291,7 @@ proc toTable*(pattern: CaptureBounds, default = none(Slice[int])): result = initTable[string, Option[Slice[int]]]() toTableImpl(nextVal.isNone) -template itemsImpl(cond: bool): stmt {.immediate, dirty.} = +template itemsImpl(cond: untyped) {.dirty.} = for i in 0 .. <RegexMatch(pattern).pattern.captureCount: let nextVal = pattern[i] # done in this roundabout way to avoid multiple yields (potential code @@ -493,17 +483,17 @@ proc matchImpl(str: string, pattern: Regex, start, endpos: int, flags: int): Opt raise RegexInternalError(msg : "Unknown internal error: " & $execRet) proc match*(str: string, pattern: Regex, start = 0, endpos = int.high): Option[RegexMatch] = - ## Like ```find(...)`` <#proc-find>`__, but anchored to the start of the + ## Like ```find(...)`` <#proc-find>`_, but anchored to the start of the ## string. This means that ``"foo".match(re"f") == true``, but ## ``"foo".match(re"o") == false``. return str.matchImpl(pattern, start, endpos, pcre.ANCHORED) iterator findIter*(str: string, pattern: Regex, start = 0, endpos = int.high): RegexMatch = - ## Works the same as ```find(...)`` <#proc-find>`__, but finds every + ## Works the same as ```find(...)`` <#proc-find>`_, but finds every ## non-overlapping match. ``"2222".find(re"22")`` is ``"22", "22"``, not ## ``"22", "22", "22"``. ## - ## Arguments are the same as ```find(...)`` <#proc-find>`__ + ## Arguments are the same as ```find(...)`` <#proc-find>`_ ## ## Variants: ## @@ -566,6 +556,16 @@ proc findAll*(str: string, pattern: Regex, start = 0, endpos = int.high): seq[st for match in str.findIter(pattern, start, endpos): result.add(match.match) +proc contains*(str: string, pattern: Regex, start = 0, endpos = int.high): bool = + ## Determine if the string contains the given pattern between the end and + ## start positions: + ## - "abc".contains(re"bc") == true + ## - "abc".contains(re"cd") == false + ## - "abc".contains(re"a", start = 1) == false + ## + ## Same as ``isSome(str.find(pattern, start, endpos))``. + return isSome(str.find(pattern, start, endpos)) + proc split*(str: string, pattern: Regex, maxSplit = -1, start = 0): seq[string] = ## Splits the string with the given regex. This works according to the ## rules that Perl and Javascript use: @@ -581,7 +581,7 @@ proc split*(str: string, pattern: Regex, maxSplit = -1, start = 0): seq[string] ## strings in the output seq. ## ``"1.2.3".split(re"\.", maxsplit = 2) == @["1", "2.3"]`` ## - ## ``start`` behaves the same as in ```find(...)`` <#proc-find>`__. + ## ``start`` behaves the same as in ```find(...)`` <#proc-find>`_. result = @[] var lastIdx = start var splits = 0 @@ -625,7 +625,7 @@ proc split*(str: string, pattern: Regex, maxSplit = -1, start = 0): seq[string] result.add(str.substr(bounds.b + 1, str.high)) template replaceImpl(str: string, pattern: Regex, - replacement: expr): stmt {.immediate, dirty.} = + replacement: untyped) {.dirty.} = # XXX seems very similar to split, maybe I can reduce code duplication # somehow? result = "" diff --git a/lib/impure/rdstdin.nim b/lib/impure/rdstdin.nim index 469bb69c5..f722a6b39 100644 --- a/lib/impure/rdstdin.nim +++ b/lib/impure/rdstdin.nim @@ -105,7 +105,8 @@ else: proc readLineFromStdin*(prompt: string): TaintedString {. tags: [ReadIOEffect, WriteIOEffect].} = var buffer = linenoise.readLine(prompt) - if isNil(buffer): quit(0) + if isNil(buffer): + raise newException(IOError, "Linenoise returned nil") result = TaintedString($buffer) if result.string.len > 0: historyAdd(buffer) @@ -114,12 +115,12 @@ else: proc readLineFromStdin*(prompt: string, line: var TaintedString): bool {. tags: [ReadIOEffect, WriteIOEffect].} = var buffer = linenoise.readLine(prompt) - if isNil(buffer): quit(0) + if isNil(buffer): + raise newException(IOError, "Linenoise returned nil") line = TaintedString($buffer) if line.string.len > 0: historyAdd(buffer) linenoise.free(buffer) - # XXX how to determine CTRL+D? result = true proc readPasswordFromStdin*(prompt: string, password: var TaintedString): diff --git a/lib/impure/re.nim b/lib/impure/re.nim index 60bb6c77f..bf397550a 100644 --- a/lib/impure/re.nim +++ b/lib/impure/re.nim @@ -7,8 +7,11 @@ # distribution, for details about the copyright. # -## Regular expression support for Nim. Deprecated. Consider using the ``nre`` -## or ``pegs`` modules instead. +## Regular expression support for Nim. This module still has some +## obscure bugs and limitations, +## consider using the ``nre`` or ``pegs`` modules instead. +## We had to de-deprecate this module since too much code relies on it +## and many people prefer its API over ``nre``'s. ## ## **Note:** The 're' proc defaults to the **extended regular expression ## syntax** which lets you use whitespace freely to make your regexes readable. @@ -22,14 +25,12 @@ ## though. ## PRCE's licence follows: ## -## .. include:: ../doc/regexprs.txt +## .. include:: ../../doc/regexprs.txt ## import pcre, strutils, rtarrays -{.deprecated.} - const MaxSubpatterns* = 20 ## defines the maximum number of subpatterns that can be captured. @@ -78,7 +79,7 @@ proc finalizeRegEx(x: Regex) = if not isNil(x.e): pcre.free_substring(cast[cstring](x.e)) -proc re*(s: string, flags = {reExtended, reStudy}): Regex {.deprecated.} = +proc re*(s: string, flags = {reExtended, reStudy}): Regex = ## Constructor of regular expressions. Note that Nim's ## extended raw string literals support this syntax ``re"[abc]"`` as ## a short form for ``re(r"[abc]")``. diff --git a/lib/impure/ssl.nim b/lib/impure/ssl.nim index 721e5ce51..e3312d792 100644 --- a/lib/impure/ssl.nim +++ b/lib/impure/ssl.nim @@ -9,6 +9,9 @@ ## This module provides an easy to use sockets-style ## nim interface to the OpenSSL library. +## +## **Warning:** This module is deprecated, use the SSL procedures defined in +## the ``net`` module instead. {.deprecated.} diff --git a/lib/js/dom.nim b/lib/js/dom.nim index 11df959d7..5104712e8 100644 --- a/lib/js/dom.nim +++ b/lib/js/dom.nim @@ -402,7 +402,9 @@ proc routeEvent*(w: Window, event: Event) proc scrollBy*(w: Window, x, y: int) proc scrollTo*(w: Window, x, y: int) proc setInterval*(w: Window, code: cstring, pause: int): ref TInterval +proc setInterval*(w: Window, function: proc (), pause: int): ref TInterval proc setTimeout*(w: Window, code: cstring, pause: int): ref TTimeOut +proc setTimeout*(w: Window, function: proc (), pause: int): ref TInterval proc stop*(w: Window) # Node "methods" @@ -481,6 +483,9 @@ proc getAttribute*(s: Style, attr: cstring, caseSensitive=false): cstring proc removeAttribute*(s: Style, attr: cstring, caseSensitive=false) proc setAttribute*(s: Style, attr, value: cstring, caseSensitive=false) +# Event "methods" +proc preventDefault*(ev: Event) + {.pop.} var diff --git a/lib/nimbase.h b/lib/nimbase.h index 5a4f403b6..37f8f5c3d 100644 --- a/lib/nimbase.h +++ b/lib/nimbase.h @@ -44,7 +44,7 @@ __clang__ #if defined(_MSC_VER) # pragma warning(disable: 4005 4100 4101 4189 4191 4200 4244 4293 4296 4309) # pragma warning(disable: 4310 4365 4456 4477 4514 4574 4611 4668 4702 4706) -# pragma warning(disable: 4710 4711 4774 4800 4820 4996) +# pragma warning(disable: 4710 4711 4774 4800 4820 4996 4090) #endif /* ------------------------------------------------------------------------- */ @@ -102,7 +102,7 @@ __clang__ defined __ICL || \ defined __DMC__ || \ defined __BORLANDC__ ) -# define NIM_THREADVAR __declspec(thread) +# define NIM_THREADVAR __declspec(thread) /* note that ICC (linux) and Clang are covered by __GNUC__ */ #elif defined __GNUC__ || \ defined __SUNPRO_C || \ @@ -222,6 +222,8 @@ __clang__ /* ----------------------------------------------------------------------- */ +#define COMMA , + #include <limits.h> #include <stddef.h> @@ -345,9 +347,6 @@ static N_INLINE(NI32, float32ToInt32)(float x) { #define float64ToInt64(x) ((NI64) (x)) -#define zeroMem(a, size) memset(a, 0, size) -#define equalMem(a, b, size) (memcmp(a, b, size) == 0) - #define STRING_LITERAL(name, str, length) \ static const struct { \ TGenericSeq Sup; \ @@ -450,8 +449,8 @@ static inline void GCGuard (void *ptr) { asm volatile ("" :: "X" (ptr)); } /* Test to see if Nim and the C compiler agree on the size of a pointer. On disagreement, your C compiler will say something like: - "error: 'assert_numbits' declared as an array with a negative size" */ -typedef int assert_numbits[sizeof(NI) == sizeof(void*) && NIM_INTBITS == sizeof(NI)*8 ? 1 : -1]; + "error: 'Nim_and_C_compiler_disagree_on_target_architecture' declared as an array with a negative size" */ +typedef int Nim_and_C_compiler_disagree_on_target_architecture[sizeof(NI) == sizeof(void*) && NIM_INTBITS == sizeof(NI)*8 ? 1 : -1]; #endif #ifdef __cplusplus diff --git a/lib/packages/docutils/highlite.nim b/lib/packages/docutils/highlite.nim index 1bc0af1b6..9de25f82b 100644 --- a/lib/packages/docutils/highlite.nim +++ b/lib/packages/docutils/highlite.nim @@ -31,13 +31,14 @@ type state: TokenClass SourceLanguage* = enum - langNone, langNim, langNimrod, langCpp, langCsharp, langC, langJava + langNone, langNim, langNimrod, langCpp, langCsharp, langC, langJava, + langYaml {.deprecated: [TSourceLanguage: SourceLanguage, TTokenClass: TokenClass, TGeneralTokenizer: GeneralTokenizer].} const sourceLanguageToStr*: array[SourceLanguage, string] = ["none", - "Nim", "Nimrod", "C++", "C#", "C", "Java"] + "Nim", "Nimrod", "C++", "C#", "C", "Java", "Yaml"] tokenClassToStr*: array[TokenClass, string] = ["Eof", "None", "Whitespace", "DecNumber", "BinNumber", "HexNumber", "OctNumber", "FloatNumber", "Identifier", "Keyword", "StringLit", "LongStringLit", "CharLit", @@ -578,6 +579,309 @@ proc javaNextToken(g: var GeneralTokenizer) = "try", "void", "volatile", "while"] clikeNextToken(g, keywords, {}) +proc yamlPlainStrLit(g: var GeneralTokenizer, pos: var int) = + g.kind = gtStringLit + while g.buf[pos] notin {'\0', '\x09'..'\x0D', ',', ']', '}'}: + if g.buf[pos] == ':' and + g.buf[pos + 1] in {'\0', '\x09'..'\x0D', ' '}: + break + inc(pos) + +proc yamlPossibleNumber(g: var GeneralTokenizer, pos: var int) = + g.kind = gtNone + if g.buf[pos] == '-': inc(pos) + if g.buf[pos] == '0': inc(pos) + elif g.buf[pos] in '1'..'9': + inc(pos) + while g.buf[pos] in {'0'..'9'}: inc(pos) + else: yamlPlainStrLit(g, pos) + if g.kind == gtNone: + if g.buf[pos] in {'\0', '\x09'..'\x0D', ' ', ',', ']', '}'}: + g.kind = gtDecNumber + elif g.buf[pos] == '.': + inc(pos) + if g.buf[pos] notin {'0'..'9'}: yamlPlainStrLit(g, pos) + else: + while g.buf[pos] in {'0'..'9'}: inc(pos) + if g.buf[pos] in {'\0', '\x09'..'\x0D', ' ', ',', ']', '}'}: + g.kind = gtFloatNumber + if g.kind == gtNone: + if g.buf[pos] in {'e', 'E'}: + inc(pos) + if g.buf[pos] in {'-', '+'}: inc(pos) + if g.buf[pos] notin {'0'..'9'}: yamlPlainStrLit(g, pos) + else: + while g.buf[pos] in {'0'..'9'}: inc(pos) + if g.buf[pos] in {'\0', '\x09'..'\x0D', ' ', ',', ']', '}'}: + g.kind = gtFloatNumber + else: yamlPlainStrLit(g, pos) + else: yamlPlainStrLit(g, pos) + while g.buf[pos] notin {'\0', ',', ']', '}', '\x0A', '\x0D'}: + inc(pos) + if g.buf[pos] notin {'\x09'..'\x0D', ' ', ',', ']', '}'}: + yamlPlainStrLit(g, pos) + break + # theoretically, we would need to parse indentation (like with block scalars) + # because of possible multiline flow scalars that start with number-like + # content, but that is far too troublesome. I think it is fine that the + # highlighter is sloppy here. + +proc yamlNextToken(g: var GeneralTokenizer) = + const + hexChars = {'0'..'9', 'A'..'F', 'a'..'f'} + var pos = g.pos + g.start = g.pos + if g.state == gtStringLit: + g.kind = gtStringLit + while true: + case g.buf[pos] + of '\\': + if pos != g.pos: break + g.kind = gtEscapeSequence + inc(pos) + case g.buf[pos] + of 'x': + inc(pos) + for i in 1..2: + {.unroll.} + if g.buf[pos] in hexChars: inc(pos) + break + of 'u': + inc(pos) + for i in 1..4: + {.unroll.} + if g.buf[pos] in hexChars: inc(pos) + break + of 'U': + inc(pos) + for i in 1..8: + {.unroll.} + if g.buf[pos] in hexChars: inc(pos) + break + else: inc(pos) + break + of '\0': + g.state = gtOther + break + of '\"': + inc(pos) + g.state = gtOther + break + else: inc(pos) + elif g.state == gtCharLit: + # abusing gtCharLit as single-quoted string lit + g.kind = gtStringLit + inc(pos) # skip the starting ' + while true: + case g.buf[pos] + of '\'': + inc(pos) + if g.buf[pos] == '\'': + inc(pos) + g.kind = gtEscapeSequence + else: g.state = gtOther + break + else: inc(pos) + elif g.state == gtCommand: + # gtCommand means 'block scalar header' + case g.buf[pos] + of ' ', '\t': + g.kind = gtWhitespace + while g.buf[pos] in {' ', '\t'}: inc(pos) + of '#': + g.kind = gtComment + while g.buf[pos] notin {'\0', '\x0A', '\x0D'}: inc(pos) + of '\x0A', '\x0D': discard + else: + # illegal here. just don't parse a block scalar + g.kind = gtNone + g.state = gtOther + if g.buf[pos] in {'\x0A', '\x0D'} and g.state == gtCommand: + g.state = gtLongStringLit + elif g.state == gtLongStringLit: + # beware, this is the only token where we actually have to parse + # indentation. + + g.kind = gtLongStringLit + # first, we have to find the parent indentation of the block scalar, so that + # we know when to stop + assert g.buf[pos] in {'\x0A', '\x0D'} + var lookbehind = pos - 1 + var headerStart = -1 + while lookbehind >= 0 and g.buf[lookbehind] notin {'\x0A', '\x0D'}: + if headerStart == -1 and g.buf[lookbehind] in {'|', '>'}: + headerStart = lookbehind + dec(lookbehind) + assert headerStart != -1 + var indentation = 1 + while g.buf[lookbehind + indentation] == ' ': inc(indentation) + if g.buf[lookbehind + indentation] in {'|', '>'}: + # when the header is alone in a line, this line does not show the parent's + # indentation, so we must go further. search the first previous line with + # non-whitespace content. + while lookbehind >= 0 and g.buf[lookbehind] in {'\x0A', '\x0D'}: + dec(lookbehind) + while lookbehind >= 0 and + g.buf[lookbehind] in {' ', '\t'}: dec(lookbehind) + # now, find the beginning of the line... + while lookbehind >= 0 and g.buf[lookbehind] notin {'\x0A', '\x0D'}: + dec(lookbehind) + # ... and its indentation + indentation = 1 + while g.buf[lookbehind + indentation] == ' ': inc(indentation) + if lookbehind == -1: indentation = 0 # top level + elif g.buf[lookbehind + 1] == '-' and g.buf[lookbehind + 2] == '-' and + g.buf[lookbehind + 3] == '-' and + g.buf[lookbehind + 4] in {'\x09'..'\x0D', ' '}: + # this is a document start, therefore, we are at top level + indentation = 0 + # because lookbehind was at newline char when calculating indentation, we're + # off by one. fix that. top level's parent will have indentation of -1. + let parentIndentation = indentation - 1 + + # find first content + while g.buf[pos] in {' ', '\x0A', '\x0D'}: + if g.buf[pos] == ' ': inc(indentation) + else: indentation = 0 + inc(pos) + var minIndentation = indentation + + # for stupid edge cases, we must check whether an explicit indentation depth + # is given at the header. + while g.buf[headerStart] in {'>', '|', '+', '-'}: inc(headerStart) + if g.buf[headerStart] in {'0'..'9'}: + minIndentation = min(minIndentation, ord(g.buf[headerStart]) - ord('0')) + + # process content lines + while indentation > parentIndentation and g.buf[pos] != '\0': + if (indentation < minIndentation and g.buf[pos] == '#') or + (indentation == 0 and g.buf[pos] == '.' and g.buf[pos + 1] == '.' and + g.buf[pos + 2] == '.' and + g.buf[pos + 3] in {'\0', '\x09'..'\x0D', ' '}): + # comment after end of block scalar, or end of document + break + minIndentation = min(indentation, minIndentation) + while g.buf[pos] notin {'\0', '\x0A', '\x0D'}: inc(pos) + while g.buf[pos] in {' ', '\x0A', '\x0D'}: + if g.buf[pos] == ' ': inc(indentation) + else: indentation = 0 + inc(pos) + + g.state = gtOther + elif g.state == gtOther: + # gtOther means 'inside YAML document' + case g.buf[pos] + of ' ', '\x09'..'\x0D': + g.kind = gtWhitespace + while g.buf[pos] in {' ', '\x09'..'\x0D'}: inc(pos) + of '#': + g.kind = gtComment + inc(pos) + while g.buf[pos] notin {'\0', '\x0A', '\x0D'}: inc(pos) + of '-': + inc(pos) + if g.buf[pos] in {'\0', ' ', '\x09'..'\x0D'}: + g.kind = gtPunctuation + elif g.buf[pos] == '-' and + (pos == 1 or g.buf[pos - 2] in {'\x0A', '\x0D'}): # start of line + inc(pos) + if g.buf[pos] == '-' and g.buf[pos + 1] in {'\0', '\x09'..'\x0D', ' '}: + inc(pos) + g.kind = gtKeyword + else: yamlPossibleNumber(g, pos) + else: yamlPossibleNumber(g, pos) + of '.': + if pos == 0 or g.buf[pos - 1] in {'\x0A', '\x0D'}: + inc(pos) + for i in 1..2: + {.unroll.} + if g.buf[pos] != '.': break + inc(pos) + if pos == g.start + 3: + g.kind = gtKeyword + g.state = gtNone + else: yamlPlainStrLit(g, pos) + else: yamlPlainStrLit(g, pos) + of '?': + inc(pos) + if g.buf[pos] in {'\0', ' ', '\x09'..'\x0D'}: + g.kind = gtPunctuation + else: yamlPlainStrLit(g, pos) + of ':': + inc(pos) + if g.buf[pos] in {'\0', '\x09'..'\x0D', ' ', '\'', '\"'} or + (pos > 0 and g.buf[pos - 2] in {'}', ']', '\"', '\''}): + g.kind = gtPunctuation + else: yamlPlainStrLit(g, pos) + of '[', ']', '{', '}', ',': + inc(pos) + g.kind = gtPunctuation + of '\"': + inc(pos) + g.state = gtStringLit + g.kind = gtStringLit + of '\'': + g.state = gtCharLit + g.kind = gtNone + of '!': + g.kind = gtTagStart + inc(pos) + if g.buf[pos] == '<': + # literal tag (e.g. `!<tag:yaml.org,2002:str>`) + while g.buf[pos] notin {'\0', '>', '\x09'..'\x0D', ' '}: inc(pos) + if g.buf[pos] == '>': inc(pos) + else: + while g.buf[pos] in {'A'..'Z', 'a'..'z', '0'..'9', '-'}: inc(pos) + case g.buf[pos] + of '!': + # prefixed tag (e.g. `!!str`) + inc(pos) + while g.buf[pos] notin + {'\0', '\x09'..'\x0D', ' ', ',', '[', ']', '{', '}'}: inc(pos) + of '\0', '\x09'..'\x0D', ' ': discard + else: + # local tag (e.g. `!nim:system:int`) + while g.buf[pos] notin {'\0', '\x09'..'\x0D', ' '}: inc(pos) + of '&': + g.kind = gtLabel + while g.buf[pos] notin {'\0', '\x09'..'\x0D', ' '}: inc(pos) + of '*': + g.kind = gtReference + while g.buf[pos] notin {'\0', '\x09'..'\x0D', ' '}: inc(pos) + of '|', '>': + # this can lead to incorrect tokenization when | or > appear inside flow + # content. checking whether we're inside flow content is not + # chomsky type-3, so we won't do that here. + g.kind = gtCommand + g.state = gtCommand + inc(pos) + while g.buf[pos] in {'0'..'9', '+', '-'}: inc(pos) + of '0'..'9': yamlPossibleNumber(g, pos) + of '\0': g.kind = gtEOF + else: yamlPlainStrLit(g, pos) + else: + # outside document + case g.buf[pos] + of '%': + if pos == 0 or g.buf[pos - 1] in {'\x0A', '\x0D'}: + g.kind = gtDirective + while g.buf[pos] notin {'\0', '\x0A', '\x0D'}: inc(pos) + else: + g.state = gtOther + yamlPlainStrLit(g, pos) + of ' ', '\x09'..'\x0D': + g.kind = gtWhitespace + while g.buf[pos] in {' ', '\x09'..'\x0D'}: inc(pos) + of '#': + g.kind = gtComment + while g.buf[pos] notin {'\0', '\x0A', '\x0D'}: inc(pos) + of '\0': g.kind = gtEOF + else: + g.kind = gtNone + g.state = gtOther + g.length = pos - g.pos + g.pos = pos + proc getNextToken*(g: var GeneralTokenizer, lang: SourceLanguage) = case lang of langNone: assert false @@ -586,6 +890,7 @@ proc getNextToken*(g: var GeneralTokenizer, lang: SourceLanguage) = of langCsharp: csharpNextToken(g) of langC: cNextToken(g) of langJava: javaNextToken(g) + of langYaml: yamlNextToken(g) when isMainModule: var keywords: seq[string] diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index e1d5f902e..53699166f 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -49,7 +49,7 @@ type TMsgKind: MsgKind].} const - messages: array [MsgKind, string] = [ + messages: array[MsgKind, string] = [ meCannotOpenFile: "cannot open '$1'", meExpected: "'$1' expected", meGridTableNotImplemented: "grid table is not implemented", @@ -323,6 +323,11 @@ proc newSharedState(options: RstParseOptions, result.msgHandler = if not isNil(msgHandler): msgHandler else: defaultMsgHandler result.findFile = if not isNil(findFile): findFile else: defaultFindFile +proc findRelativeFile(p: RstParser; filename: string): string = + result = p.filename.splitFile.dir / filename + if not existsFile(result): + result = p.s.findFile(filename) + proc rstMessage(p: RstParser, msgKind: MsgKind, arg: string) = p.s.msgHandler(p.filename, p.line + p.tok[p.idx].line, p.col + p.tok[p.idx].col, msgKind, arg) @@ -1500,7 +1505,7 @@ proc dirInclude(p: var RstParser): PRstNode = result = nil var n = parseDirective(p, {hasArg, argIsFile, hasOptions}, nil) var filename = strip(addNodes(n.sons[0])) - var path = p.s.findFile(filename) + var path = p.findRelativeFile(filename) if path == "": rstMessage(p, meCannotOpenFile, filename) else: @@ -1511,7 +1516,7 @@ proc dirInclude(p: var RstParser): PRstNode = else: var q: RstParser initParser(q, p.s) - q.filename = filename + q.filename = path q.col += getTokens(readFile(path), false, q.tok) # workaround a GCC bug; more like the interior pointer bug? #if find(q.tok[high(q.tok)].symbol, "\0\x01\x02") > 0: @@ -1538,7 +1543,7 @@ proc dirCodeBlock(p: var RstParser, nimExtension = false): PRstNode = result = parseDirective(p, {hasArg, hasOptions}, parseLiteralBlock) var filename = strip(getFieldValue(result, "file")) if filename != "": - var path = p.s.findFile(filename) + var path = p.findRelativeFile(filename) if path == "": rstMessage(p, meCannotOpenFile, filename) var n = newRstNode(rnLiteralBlock) add(n, newRstNode(rnLeaf, readFile(path))) @@ -1590,7 +1595,7 @@ proc dirRawAux(p: var RstParser, result: var PRstNode, kind: RstNodeKind, contentParser: SectionParser) = var filename = getFieldValue(result, "file") if filename.len > 0: - var path = p.s.findFile(filename) + var path = p.findRelativeFile(filename) if path.len == 0: rstMessage(p, meCannotOpenFile, filename) else: diff --git a/lib/posix/posix.nim b/lib/posix/posix.nim index 4c7b817cb..bd69b2328 100644 --- a/lib/posix/posix.nim +++ b/lib/posix/posix.nim @@ -35,6 +35,15 @@ const hasSpawnH = not defined(haiku) # should exist for every Posix system nowadays hasAioH = defined(linux) +when defined(linux): + # On Linux: + # timer_{create,delete,settime,gettime}, + # clock_{getcpuclockid, getres, gettime, nanosleep, settime} lives in librt + {.passL: "-lrt".} +when defined(solaris): + # On Solaris hstrerror lives in libresolv + {.passL: "-lresolv".} + when false: const C_IRUSR = 0c000400 ## Read by owner. @@ -101,7 +110,7 @@ type ## (not POSIX) when defined(linux) or defined(bsd): d_off*: Off ## Not an offset. Value that ``telldir()`` would return. - d_name*: array [0..255, char] ## Name of entry. + d_name*: array[0..255, char] ## Name of entry. Tflock* {.importc: "struct flock", final, pure, header: "<fcntl.h>".} = object ## flock type @@ -233,7 +242,7 @@ type ## network to which this node is attached, if any. release*, ## Current release level of this implementation. version*, ## Current version level of this release. - machine*: array [0..255, char] ## Name of the hardware type on which the + machine*: array[0..255, char] ## Name of the hardware type on which the ## system is running. Sem* {.importc: "sem_t", header: "<semaphore.h>", final, pure.} = object @@ -439,6 +448,14 @@ when hasSpawnH: Tposix_spawn_file_actions* {.importc: "posix_spawn_file_actions_t", header: "<spawn.h>", final, pure.} = object +when defined(linux): + # from sys/un.h + const Sockaddr_un_path_length* = 108 +else: + # according to http://pubs.opengroup.org/onlinepubs/009604499/basedefs/sys/un.h.html + # this is >=92 + const Sockaddr_un_path_length* = 92 + type Socklen* {.importc: "socklen_t", header: "<sys/socket.h>".} = cuint TSa_Family* {.importc: "sa_family_t", header: "<sys/socket.h>".} = cint @@ -446,7 +463,12 @@ type SockAddr* {.importc: "struct sockaddr", header: "<sys/socket.h>", pure, final.} = object ## struct sockaddr sa_family*: TSa_Family ## Address family. - sa_data*: array [0..255, char] ## Socket address (variable-length data). + sa_data*: array[0..255, char] ## Socket address (variable-length data). + + Sockaddr_un* {.importc: "struct sockaddr_un", header: "<sys/un.h>", + pure, final.} = object ## struct sockaddr_un + sun_family*: TSa_Family ## Address family. + sun_path*: array[0..Sockaddr_un_path_length-1, char] ## Socket path Sockaddr_storage* {.importc: "struct sockaddr_storage", header: "<sys/socket.h>", @@ -504,7 +526,7 @@ type In6Addr* {.importc: "struct in6_addr", pure, final, header: "<netinet/in.h>".} = object ## struct in6_addr - s6_addr*: array [0..15, char] + s6_addr*: array[0..15, char] Sockaddr_in6* {.importc: "struct sockaddr_in6", pure, final, header: "<netinet/in.h>".} = object ## struct sockaddr_in6 @@ -1604,6 +1626,16 @@ else: var MAP_POPULATE*: cint = 0 +when defined(linux) or defined(nimdoc): + when defined(alpha) or defined(mips) or defined(parisc) or + defined(sparc) or defined(nimdoc): + const SO_REUSEPORT* = cint(0x0200) + ## Multiple binding: load balancing on incoming TCP connections + ## or UDP packets. (Requires Linux kernel > 3.9) + else: + const SO_REUSEPORT* = cint(15) +else: + var SO_REUSEPORT* {.importc, header: "<sys/socket.h>".}: cint when defined(macosx): # We can't use the NOSIGNAL flag in the ``send`` function, it has no effect @@ -1612,6 +1644,10 @@ when defined(macosx): MSG_NOSIGNAL* = 0'i32 var SO_NOSIGPIPE* {.importc, header: "<sys/socket.h>".}: cint +elif defined(solaris): + # Solaris dont have MSG_NOSIGNAL + const + MSG_NOSIGNAL* = 0'i32 else: var MSG_NOSIGNAL* {.importc, header: "<sys/socket.h>".}: cint @@ -2309,9 +2345,9 @@ proc strftime*(a1: cstring, a2: int, a3: cstring, a4: var Tm): int {.importc, header: "<time.h>".} proc strptime*(a1, a2: cstring, a3: var Tm): cstring {.importc, header: "<time.h>".} proc time*(a1: var Time): Time {.importc, header: "<time.h>".} -proc timer_create*(a1: var ClockId, a2: var SigEvent, +proc timer_create*(a1: ClockId, a2: var SigEvent, a3: var Timer): cint {.importc, header: "<time.h>".} -proc timer_delete*(a1: var Timer): cint {.importc, header: "<time.h>".} +proc timer_delete*(a1: Timer): cint {.importc, header: "<time.h>".} proc timer_gettime*(a1: Timer, a2: var Itimerspec): cint {. importc, header: "<time.h>".} proc timer_getoverrun*(a1: Timer): cint {.importc, header: "<time.h>".} @@ -2357,8 +2393,14 @@ proc sigrelse*(a1: cint): cint {.importc, header: "<signal.h>".} proc sigset*(a1: int, a2: proc (x: cint) {.noconv.}) {. importc, header: "<signal.h>".} proc sigsuspend*(a1: var Sigset): cint {.importc, header: "<signal.h>".} -proc sigtimedwait*(a1: var Sigset, a2: var SigInfo, + +when defined(android): + proc sigtimedwait*(a1: var Sigset, a2: var SigInfo, + a3: var Timespec, sigsetsize: csize = sizeof(culong)*2): cint {.importc: "__rt_sigtimedwait", header:"<signal.h>".} +else: + proc sigtimedwait*(a1: var Sigset, a2: var SigInfo, a3: var Timespec): cint {.importc, header: "<signal.h>".} + proc sigwait*(a1: var Sigset, a2: var cint): cint {. importc, header: "<signal.h>".} proc sigwaitinfo*(a1: var Sigset, a2: var SigInfo): cint {. @@ -2574,8 +2616,12 @@ proc gai_strerror*(a1: cint): cstring {.importc:"(char *)$1", header: "<netdb.h> proc getaddrinfo*(a1, a2: cstring, a3: ptr AddrInfo, a4: var ptr AddrInfo): cint {.importc, header: "<netdb.h>".} -proc gethostbyaddr*(a1: pointer, a2: Socklen, a3: cint): ptr Hostent {. - importc, header: "<netdb.h>".} +when not defined(android4): + proc gethostbyaddr*(a1: pointer, a2: Socklen, a3: cint): ptr Hostent {. + importc, header: "<netdb.h>".} +else: + proc gethostbyaddr*(a1: cstring, a2: cint, a3: cint): ptr Hostent {. + importc, header: "<netdb.h>".} proc gethostbyname*(a1: cstring): ptr Hostent {.importc, header: "<netdb.h>".} proc gethostent*(): ptr Hostent {.importc, header: "<netdb.h>".} @@ -2607,7 +2653,7 @@ proc poll*(a1: ptr TPollfd, a2: Tnfds, a3: int): cint {. proc realpath*(name, resolved: cstring): cstring {. importc: "realpath", header: "<stdlib.h>".} -proc utimes*(path: cstring, times: ptr array [2, Timeval]): int {. +proc utimes*(path: cstring, times: ptr array[2, Timeval]): int {. importc: "utimes", header: "<sys/time.h>".} ## Sets file access and modification times. ## @@ -2618,3 +2664,17 @@ proc utimes*(path: cstring, times: ptr array [2, Timeval]): int {. ## Returns zero on success. ## ## For more information read http://www.unix.com/man-page/posix/3/utimes/. + +proc handle_signal(sig: cint, handler: proc (a: cint) {.noconv.}) {.importc: "signal", header: "<signal.h>".} + +template onSignal*(signals: varargs[cint], body: untyped) = + ## Setup code to be executed when Unix signals are received. Example: + ## from posix import SIGINT, SIGTERM + ## onSignal(SIGINT, SIGTERM): + ## echo "bye" + + for s in signals: + handle_signal(s, + proc (sig: cint) {.noconv.} = + body + ) diff --git a/lib/pure/algorithm.nim b/lib/pure/algorithm.nim index c0acae138..b83daf245 100644 --- a/lib/pure/algorithm.nim +++ b/lib/pure/algorithm.nim @@ -114,7 +114,7 @@ proc lowerBound*[T](a: openArray[T], key: T, cmp: proc(x,y: T): int {.closure.}) proc lowerBound*[T](a: openArray[T], key: T): int = lowerBound(a, key, cmp[T]) proc merge[T](a, b: var openArray[T], lo, m, hi: int, cmp: proc (x, y: T): int {.closure.}, order: SortOrder) = - template `<-` (a, b: expr) = + template `<-` (a, b) = when false: a = b elif onlySafeCode: @@ -206,7 +206,7 @@ proc sorted*[T](a: openArray[T], cmp: proc(x, y: T): int {.closure.}, result[i] = a[i] sort(result, cmp, order) -template sortedByIt*(seq1, op: expr): expr = +template sortedByIt*(seq1, op: untyped): untyped = ## Convenience template around the ``sorted`` proc to reduce typing. ## ## The template injects the ``it`` variable which you can use directly in an @@ -231,7 +231,7 @@ template sortedByIt*(seq1, op: expr): expr = ## ## echo people.sortedByIt((it.age, it.name)) ## - var result {.gensym.} = sorted(seq1, proc(x, y: type(seq1[0])): int = + var result = sorted(seq1, proc(x, y: type(seq1[0])): int = var it {.inject.} = x let a = op it = y diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index cc337452f..79bc1b96d 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -9,9 +9,9 @@ include "system/inclrtl" -import os, oids, tables, strutils, macros, times +import os, oids, tables, strutils, macros, times, heapqueue -import nativesockets, net +import nativesockets, net, queues export Port, SocketFlag @@ -155,6 +155,9 @@ type when not defined(release): var currentID = 0 + +proc callSoon*(cbproc: proc ()) {.gcsafe.} + proc newFuture*[T](fromProc: string = "unspecified"): Future[T] = ## Creates a new future. ## @@ -257,7 +260,7 @@ proc `callback=`*(future: FutureBase, cb: proc () {.closure,gcsafe.}) = ## passes ``future`` as a param to the callback. future.cb = cb if future.finished: - future.cb() + callSoon(future.cb) proc `callback=`*[T](future: Future[T], cb: proc (future: Future[T]) {.closure,gcsafe.}) = @@ -352,28 +355,88 @@ proc `or`*[T, Y](fut1: Future[T], fut2: Future[Y]): Future[void] = fut2.callback = cb return retFuture +proc all*[T](futs: varargs[Future[T]]): auto = + ## Returns a future which will complete once + ## all futures in ``futs`` complete. + ## + ## If the awaited futures are not ``Future[void]``, the returned future + ## will hold the values of all awaited futures in a sequence. + ## + ## If the awaited futures *are* ``Future[void]``, + ## this proc returns ``Future[void]``. + + when T is void: + var + retFuture = newFuture[void]("asyncdispatch.all") + completedFutures = 0 + + let totalFutures = len(futs) + + for fut in futs: + fut.callback = proc(f: Future[T]) = + inc(completedFutures) + + if completedFutures == totalFutures: + retFuture.complete() + + return retFuture + + else: + var + retFuture = newFuture[seq[T]]("asyncdispatch.all") + retValues = newSeq[T](len(futs)) + completedFutures = 0 + + for i, fut in futs: + proc setCallback(i: int) = + fut.callback = proc(f: Future[T]) = + retValues[i] = f.read() + inc(completedFutures) + + if completedFutures == len(retValues): + retFuture.complete(retValues) + + setCallback(i) + + return retFuture + type PDispatcherBase = ref object of RootRef - timers: seq[tuple[finishAt: float, fut: Future[void]]] - -proc processTimers(p: PDispatcherBase) = - var oldTimers = p.timers - p.timers = @[] - for t in oldTimers: - if epochTime() >= t.finishAt: - t.fut.complete() - else: - p.timers.add(t) + timers: HeapQueue[tuple[finishAt: float, fut: Future[void]]] + callbacks: Queue[proc ()] + +proc processTimers(p: PDispatcherBase) {.inline.} = + while p.timers.len > 0 and epochTime() >= p.timers[0].finishAt: + p.timers.pop().fut.complete() + +proc processPendingCallbacks(p: PDispatcherBase) = + while p.callbacks.len > 0: + var cb = p.callbacks.dequeue() + cb() + +proc adjustedTimeout(p: PDispatcherBase, timeout: int): int {.inline.} = + # If dispatcher has active timers this proc returns the timeout + # of the nearest timer. Returns `timeout` otherwise. + result = timeout + if p.timers.len > 0: + let timerTimeout = p.timers[0].finishAt + let curTime = epochTime() + if timeout == -1 or (curTime + (timeout / 1000)) > timerTimeout: + result = int((timerTimeout - curTime) * 1000) + if result < 0: result = 0 when defined(windows) or defined(nimdoc): import winlean, sets, hashes type - CompletionKey = Dword + CompletionKey = ULONG_PTR CompletionData* = object fd*: AsyncFD # TODO: Rename this. cb*: proc (fd: AsyncFD, bytesTransferred: Dword, errcode: OSErrorCode) {.closure,gcsafe.} + cell*: ForeignCell # we need this `cell` to protect our `cb` environment, + # when using RegisterWaitForSingleObject, because + # waiting is done in different thread. PDispatcher* = ref object of PDispatcherBase ioPort: Handle @@ -385,6 +448,15 @@ when defined(windows) or defined(nimdoc): PCustomOverlapped* = ref CustomOverlapped AsyncFD* = distinct int + + PostCallbackData = object + ioPort: Handle + handleFd: AsyncFD + waitFd: Handle + ovl: PCustomOverlapped + PostCallbackDataPtr = ptr PostCallbackData + + Callback = proc (fd: AsyncFD): bool {.closure,gcsafe.} {.deprecated: [TCompletionKey: CompletionKey, TAsyncFD: AsyncFD, TCustomOverlapped: CustomOverlapped, TCompletionData: CompletionData].} @@ -396,7 +468,8 @@ when defined(windows) or defined(nimdoc): new result result.ioPort = createIoCompletionPort(INVALID_HANDLE_VALUE, 0, 0, 1) result.handles = initSet[AsyncFD]() - result.timers = @[] + result.timers.newHeapQueue() + result.callbacks = initQueue[proc ()](64) var gDisp{.threadvar.}: PDispatcher ## Global dispatcher proc getGlobalDispatcher*(): PDispatcher = @@ -423,15 +496,17 @@ when defined(windows) or defined(nimdoc): proc poll*(timeout = 500) = ## Waits for completion events and processes them. let p = getGlobalDispatcher() - if p.handles.len == 0 and p.timers.len == 0: + if p.handles.len == 0 and p.timers.len == 0 and p.callbacks.len == 0: raise newException(ValueError, "No handles or timers registered in dispatcher.") - let llTimeout = - if timeout == -1: winlean.INFINITE - else: timeout.int32 + let at = p.adjustedTimeout(timeout) + var llTimeout = + if at == -1: winlean.INFINITE + else: at.int32 + var lpNumberOfBytesTransferred: Dword - var lpCompletionKey: ULONG + var lpCompletionKey: ULONG_PTR var customOverlapped: PCustomOverlapped let res = getQueuedCompletionStatus(p.ioPort, addr lpNumberOfBytesTransferred, addr lpCompletionKey, @@ -445,6 +520,13 @@ when defined(windows) or defined(nimdoc): customOverlapped.data.cb(customOverlapped.data.fd, lpNumberOfBytesTransferred, OSErrorCode(-1)) + + # If cell.data != nil, then system.protect(rawEnv(cb)) was called, + # so we need to dispose our `cb` environment, because it is not needed + # anymore. + if customOverlapped.data.cell.data != nil: + system.dispose(customOverlapped.data.cell) + GC_unref(customOverlapped) else: let errCode = osLastError() @@ -452,6 +534,8 @@ when defined(windows) or defined(nimdoc): assert customOverlapped.data.fd == lpCompletionKey.AsyncFD customOverlapped.data.cb(customOverlapped.data.fd, lpNumberOfBytesTransferred, errCode) + if customOverlapped.data.cell.data != nil: + system.dispose(customOverlapped.data.cell) GC_unref(customOverlapped) else: if errCode.int32 == WAIT_TIMEOUT: @@ -461,6 +545,8 @@ when defined(windows) or defined(nimdoc): # Timer processing. processTimers(p) + # Callback queue processing + processPendingCallbacks(p) var connectExPtr: pointer = nil var acceptExPtr: pointer = nil @@ -651,34 +737,16 @@ when defined(windows) or defined(nimdoc): retFuture.complete("") else: retFuture.fail(newException(OSError, osErrorMsg(err))) - elif ret == 0 and bytesReceived == 0 and dataBuf.buf[0] == '\0': - # We have to ensure that the buffer is empty because WSARecv will tell - # us immediately when it was disconnected, even when there is still - # data in the buffer. - # We want to give the user as much data as we can. So we only return - # the empty string (which signals a disconnection) when there is - # nothing left to read. - retFuture.complete("") - # TODO: "For message-oriented sockets, where a zero byte message is often - # allowable, a failure with an error code of WSAEDISCON is used to - # indicate graceful closure." - # ~ http://msdn.microsoft.com/en-us/library/ms741688%28v=vs.85%29.aspx - else: - # Request to read completed immediately. - # From my tests bytesReceived isn't reliable. - let realSize = - if bytesReceived == 0: - size - else: - bytesReceived - var data = newString(realSize) - assert realSize <= size - copyMem(addr data[0], addr dataBuf.buf[0], realSize) - #dealloc dataBuf.buf - retFuture.complete($data) - # We don't deallocate ``ol`` here because even though this completed - # immediately poll will still be notified about its completion and it will - # free ``ol``. + elif ret == 0: + # Request completed immediately. + if bytesReceived != 0: + var data = newString(bytesReceived) + assert bytesReceived <= size + copyMem(addr data[0], addr dataBuf.buf[0], bytesReceived) + retFuture.complete($data) + else: + if hasOverlappedIoCompleted(cast[POVERLAPPED](ol)): + retFuture.complete("") return retFuture proc recvInto*(socket: AsyncFD, buf: cstring, size: int, @@ -741,31 +809,14 @@ when defined(windows) or defined(nimdoc): retFuture.complete(0) else: retFuture.fail(newException(OSError, osErrorMsg(err))) - elif ret == 0 and bytesReceived == 0 and dataBuf.buf[0] == '\0': - # We have to ensure that the buffer is empty because WSARecv will tell - # us immediately when it was disconnected, even when there is still - # data in the buffer. - # We want to give the user as much data as we can. So we only return - # the empty string (which signals a disconnection) when there is - # nothing left to read. - retFuture.complete(0) - # TODO: "For message-oriented sockets, where a zero byte message is often - # allowable, a failure with an error code of WSAEDISCON is used to - # indicate graceful closure." - # ~ http://msdn.microsoft.com/en-us/library/ms741688%28v=vs.85%29.aspx - else: - # Request to read completed immediately. - # From my tests bytesReceived isn't reliable. - let realSize = - if bytesReceived == 0: - size - else: - bytesReceived - assert realSize <= size - retFuture.complete(realSize) - # We don't deallocate ``ol`` here because even though this completed - # immediately poll will still be notified about its completion and it will - # free ``ol``. + elif ret == 0: + # Request completed immediately. + if bytesReceived != 0: + assert bytesReceived <= size + retFuture.complete(bytesReceived) + else: + if hasOverlappedIoCompleted(cast[POVERLAPPED](ol)): + retFuture.complete(bytesReceived) return retFuture proc send*(socket: AsyncFD, data: string, @@ -811,6 +862,101 @@ when defined(windows) or defined(nimdoc): # free ``ol``. return retFuture + proc sendTo*(socket: AsyncFD, data: pointer, size: int, saddr: ptr SockAddr, + saddrLen: Socklen, + flags = {SocketFlag.SafeDisconn}): Future[void] = + ## Sends ``data`` to specified destination ``saddr``, using + ## socket ``socket``. The returned future will complete once all data + ## has been sent. + verifyPresence(socket) + var retFuture = newFuture[void]("sendTo") + var dataBuf: TWSABuf + dataBuf.buf = cast[cstring](data) + dataBuf.len = size.ULONG + var bytesSent = 0.Dword + var lowFlags = 0.Dword + + # we will preserve address in our stack + var staddr: array[128, char] # SOCKADDR_STORAGE size is 128 bytes + var stalen: cint = cint(saddrLen) + zeroMem(addr(staddr[0]), 128) + copyMem(addr(staddr[0]), saddr, saddrLen) + + var ol = PCustomOverlapped() + GC_ref(ol) + ol.data = CompletionData(fd: socket, cb: + proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + if not retFuture.finished: + if errcode == OSErrorCode(-1): + retFuture.complete() + else: + retFuture.fail(newException(OSError, osErrorMsg(errcode))) + ) + + let ret = WSASendTo(socket.SocketHandle, addr dataBuf, 1, addr bytesSent, + lowFlags, cast[ptr SockAddr](addr(staddr[0])), + stalen, cast[POVERLAPPED](ol), nil) + if ret == -1: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + GC_unref(ol) + retFuture.fail(newException(OSError, osErrorMsg(err))) + else: + retFuture.complete() + # We don't deallocate ``ol`` here because even though this completed + # immediately poll will still be notified about its completion and it will + # free ``ol``. + return retFuture + + proc recvFromInto*(socket: AsyncFD, data: pointer, size: int, + saddr: ptr SockAddr, saddrLen: ptr SockLen, + flags = {SocketFlag.SafeDisconn}): Future[int] = + ## Receives a datagram data from ``socket`` into ``buf``, which must + ## be at least of size ``size``, address of datagram's sender will be + ## stored into ``saddr`` and ``saddrLen``. Returned future will complete + ## once one datagram has been received, and will return size of packet + ## received. + verifyPresence(socket) + var retFuture = newFuture[int]("recvFromInto") + + var dataBuf = TWSABuf(buf: cast[cstring](data), len: size.ULONG) + + var bytesReceived = 0.Dword + var lowFlags = 0.Dword + + var ol = PCustomOverlapped() + GC_ref(ol) + ol.data = CompletionData(fd: socket, cb: + proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + if not retFuture.finished: + if errcode == OSErrorCode(-1): + assert bytesCount <= size + retFuture.complete(bytesCount) + else: + # datagram sockets don't have disconnection, + # so we can just raise an exception + retFuture.fail(newException(OSError, osErrorMsg(errcode))) + ) + + let res = WSARecvFrom(socket.SocketHandle, addr dataBuf, 1, + addr bytesReceived, addr lowFlags, + saddr, cast[ptr cint](saddrLen), + cast[POVERLAPPED](ol), nil) + if res == -1: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + GC_unref(ol) + retFuture.fail(newException(OSError, osErrorMsg(err))) + else: + # Request completed immediately. + if bytesReceived != 0: + assert bytesReceived <= size + retFuture.complete(bytesReceived) + else: + if hasOverlappedIoCompleted(cast[POVERLAPPED](ol)): + retFuture.complete(bytesReceived) + return retFuture + proc acceptAddr*(socket: AsyncFD, flags = {SocketFlag.SafeDisconn}): Future[tuple[address: string, client: AsyncFD]] = ## Accepts a new connection. Returns a future containing the client socket @@ -837,7 +983,7 @@ when defined(windows) or defined(nimdoc): let dwLocalAddressLength = Dword(sizeof (Sockaddr_in) + 16) let dwRemoteAddressLength = Dword(sizeof(Sockaddr_in) + 16) - template completeAccept(): stmt {.immediate, dirty.} = + template completeAccept() {.dirty.} = var listenSock = socket let setoptRet = setsockopt(clientSock, SOL_SOCKET, SO_UPDATE_ACCEPT_CONTEXT, addr listenSock, @@ -857,7 +1003,7 @@ when defined(windows) or defined(nimdoc): client: clientSock.AsyncFD) ) - template failAccept(errcode): stmt = + template failAccept(errcode) = if flags.isDisconnectionError(errcode): var newAcceptFut = acceptAddr(socket, flags) newAcceptFut.callback = @@ -923,6 +1069,126 @@ when defined(windows) or defined(nimdoc): ## Unregisters ``fd``. getGlobalDispatcher().handles.excl(fd) + {.push stackTrace:off.} + proc waitableCallback(param: pointer, + timerOrWaitFired: WINBOOL): void {.stdcall.} = + var p = cast[PostCallbackDataPtr](param) + discard postQueuedCompletionStatus(p.ioPort, timerOrWaitFired.Dword, + ULONG_PTR(p.handleFd), + cast[pointer](p.ovl)) + {.pop.} + + template registerWaitableEvent(mask) = + let p = getGlobalDispatcher() + var flags = (WT_EXECUTEINWAITTHREAD or WT_EXECUTEONLYONCE).Dword + var hEvent = wsaCreateEvent() + if hEvent == 0: + raiseOSError(osLastError()) + var pcd = cast[PostCallbackDataPtr](allocShared0(sizeof(PostCallbackData))) + pcd.ioPort = p.ioPort + pcd.handleFd = fd + var ol = PCustomOverlapped() + GC_ref(ol) + + ol.data = CompletionData(fd: fd, cb: + proc(fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + # we excluding our `fd` because cb(fd) can register own handler + # for this `fd` + p.handles.excl(fd) + # unregisterWait() is called before callback, because appropriate + # winsockets function can re-enable event. + # https://msdn.microsoft.com/en-us/library/windows/desktop/ms741576(v=vs.85).aspx + if unregisterWait(pcd.waitFd) == 0: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + raiseOSError(osLastError()) + if cb(fd): + # callback returned `true`, so we free all allocated resources + deallocShared(cast[pointer](pcd)) + if not wsaCloseEvent(hEvent): + raiseOSError(osLastError()) + # pcd.ovl will be unrefed in poll(). + else: + # callback returned `false` we need to continue + if p.handles.contains(fd): + # new callback was already registered with `fd`, so we free all + # allocated resources. This happens because in callback `cb` + # addRead/addWrite was called with same `fd`. + deallocShared(cast[pointer](pcd)) + if not wsaCloseEvent(hEvent): + raiseOSError(osLastError()) + else: + # we need to include `fd` again + p.handles.incl(fd) + # and register WaitForSingleObject again + if not registerWaitForSingleObject(addr(pcd.waitFd), hEvent, + cast[WAITORTIMERCALLBACK](waitableCallback), + cast[pointer](pcd), INFINITE, flags): + # pcd.ovl will be unrefed in poll() + discard wsaCloseEvent(hEvent) + deallocShared(cast[pointer](pcd)) + raiseOSError(osLastError()) + else: + # we ref pcd.ovl one more time, because it will be unrefed in + # poll() + GC_ref(pcd.ovl) + ) + # We need to protect our callback environment value, so GC will not free it + # accidentally. + ol.data.cell = system.protect(rawEnv(ol.data.cb)) + + # This is main part of `hacky way` is using WSAEventSelect, so `hEvent` + # will be signaled when appropriate `mask` events will be triggered. + if wsaEventSelect(fd.SocketHandle, hEvent, mask) != 0: + GC_unref(ol) + deallocShared(cast[pointer](pcd)) + discard wsaCloseEvent(hEvent) + raiseOSError(osLastError()) + + pcd.ovl = ol + if not registerWaitForSingleObject(addr(pcd.waitFd), hEvent, + cast[WAITORTIMERCALLBACK](waitableCallback), + cast[pointer](pcd), INFINITE, flags): + GC_unref(ol) + deallocShared(cast[pointer](pcd)) + discard wsaCloseEvent(hEvent) + raiseOSError(osLastError()) + p.handles.incl(fd) + + proc addRead*(fd: AsyncFD, cb: Callback) = + ## Start watching the file descriptor for read availability and then call + ## the callback ``cb``. + ## + ## This is not ``pure`` mechanism for Windows Completion Ports (IOCP), + ## so if you can avoid it, please do it. Use `addRead` only if really + ## need it (main usecase is adaptation of `unix like` libraries to be + ## asynchronous on Windows). + ## If you use this function, you dont need to use asyncdispatch.recv() + ## or asyncdispatch.accept(), because they are using IOCP, please use + ## nativesockets.recv() and nativesockets.accept() instead. + ## + ## Be sure your callback ``cb`` returns ``true``, if you want to remove + ## watch of `read` notifications, and ``false``, if you want to continue + ## receiving notifies. + registerWaitableEvent(FD_READ or FD_ACCEPT or FD_OOB or FD_CLOSE) + + proc addWrite*(fd: AsyncFD, cb: Callback) = + ## Start watching the file descriptor for write availability and then call + ## the callback ``cb``. + ## + ## This is not ``pure`` mechanism for Windows Completion Ports (IOCP), + ## so if you can avoid it, please do it. Use `addWrite` only if really + ## need it (main usecase is adaptation of `unix like` libraries to be + ## asynchronous on Windows). + ## If you use this function, you dont need to use asyncdispatch.send() + ## or asyncdispatch.connect(), because they are using IOCP, please use + ## nativesockets.send() and nativesockets.connect() instead. + ## + ## Be sure your callback ``cb`` returns ``true``, if you want to remove + ## watch of `write` notifications, and ``false``, if you want to continue + ## receiving notifies. + registerWaitableEvent(FD_WRITE or FD_CONNECT or FD_CLOSE) + initAll() else: import selectors @@ -956,7 +1222,8 @@ else: proc newDispatcher*(): PDispatcher = new result result.selector = newSelector() - result.timers = @[] + result.timers.newHeapQueue() + result.callbacks = initQueue[proc ()](64) var gDisp{.threadvar.}: PDispatcher ## Global dispatcher proc getGlobalDispatcher*(): PDispatcher = @@ -1014,7 +1281,7 @@ else: proc poll*(timeout = 500) = let p = getGlobalDispatcher() - for info in p.selector.select(timeout): + for info in p.selector.select(p.adjustedTimeout(timeout)): let data = PData(info.key.data) assert data.fd == info.key.fd.AsyncFD #echo("In poll ", data.fd.cint) @@ -1052,7 +1319,10 @@ else: # (e.g. socket disconnected). discard + # Timer processing. processTimers(p) + # Callback queue processing + processPendingCallbacks(p) proc connect*(socket: AsyncFD, address: string, port: Port, domain = AF_INET): Future[void] = @@ -1184,6 +1454,60 @@ else: addWrite(socket, cb) return retFuture + proc sendTo*(socket: AsyncFD, data: pointer, size: int, saddr: ptr SockAddr, + saddrLen: SockLen, + flags = {SocketFlag.SafeDisconn}): Future[void] = + ## Sends ``data`` of size ``size`` in bytes to specified destination + ## (``saddr`` of size ``saddrLen`` in bytes, using socket ``socket``. + ## The returned future will complete once all data has been sent. + var retFuture = newFuture[void]("sendTo") + + # we will preserve address in our stack + var staddr: array[128, char] # SOCKADDR_STORAGE size is 128 bytes + var stalen = saddrLen + zeroMem(addr(staddr[0]), 128) + copyMem(addr(staddr[0]), saddr, saddrLen) + + proc cb(sock: AsyncFD): bool = + result = true + let res = sendto(sock.SocketHandle, data, size, MSG_NOSIGNAL, + cast[ptr SockAddr](addr(staddr[0])), stalen) + if res < 0: + let lastError = osLastError() + if lastError.int32 notin {EINTR, EWOULDBLOCK, EAGAIN}: + retFuture.fail(newException(OSError, osErrorMsg(lastError))) + else: + result = false # We still want this callback to be called. + else: + retFuture.complete() + + addWrite(socket, cb) + return retFuture + + proc recvFromInto*(socket: AsyncFD, data: pointer, size: int, + saddr: ptr SockAddr, saddrLen: ptr SockLen, + flags = {SocketFlag.SafeDisconn}): Future[int] = + ## Receives a datagram data from ``socket`` into ``data``, which must + ## be at least of size ``size`` in bytes, address of datagram's sender + ## will be stored into ``saddr`` and ``saddrLen``. Returned future will + ## complete once one datagram has been received, and will return size + ## of packet received. + var retFuture = newFuture[int]("recvFromInto") + proc cb(sock: AsyncFD): bool = + result = true + let res = recvfrom(sock.SocketHandle, data, size.cint, flags.toOSFlags(), + saddr, saddrLen) + if res < 0: + let lastError = osLastError() + if lastError.int32 notin {EINTR, EWOULDBLOCK, EAGAIN}: + retFuture.fail(newException(OSError, osErrorMsg(lastError))) + else: + result = false + else: + retFuture.complete(res) + addRead(socket, cb) + return retFuture + proc acceptAddr*(socket: AsyncFD, flags = {SocketFlag.SafeDisconn}): Future[tuple[address: string, client: AsyncFD]] = var retFuture = newFuture[tuple[address: string, @@ -1215,7 +1539,25 @@ proc sleepAsync*(ms: int): Future[void] = ## ``ms`` milliseconds. var retFuture = newFuture[void]("sleepAsync") let p = getGlobalDispatcher() - p.timers.add((epochTime() + (ms / 1000), retFuture)) + p.timers.push((epochTime() + (ms / 1000), retFuture)) + return retFuture + +proc withTimeout*[T](fut: Future[T], timeout: int): Future[bool] = + ## Returns a future which will complete once ``fut`` completes or after + ## ``timeout`` milliseconds has elapsed. + ## + ## If ``fut`` completes first the returned future will hold true, + ## otherwise, if ``timeout`` milliseconds has elapsed first, the returned + ## future will hold false. + + var retFuture = newFuture[bool]("asyncdispatch.`withTimeout`") + var timeoutFuture = sleepAsync(timeout) + fut.callback = + proc () = + if not retFuture.finished: retFuture.complete(true) + timeoutFuture.callback = + proc () = + if not retFuture.finished: retFuture.complete(false) return retFuture proc accept*(socket: AsyncFD, @@ -1248,7 +1590,7 @@ proc skipStmtList(node: NimNode): NimNode {.compileTime.} = result = node[0] template createCb(retFutureSym, iteratorNameSym, - name: expr): stmt {.immediate.} = + name: untyped) = var nameIterVar = iteratorNameSym #{.push stackTrace: off.} proc cb {.closure,gcsafe.} = @@ -1316,6 +1658,23 @@ proc generateExceptionCheck(futSym, ) result.add elseNode +template useVar(result: var NimNode, futureVarNode: NimNode, valueReceiver, + rootReceiver: expr, fromNode: NimNode) = + ## Params: + ## futureVarNode: The NimNode which is a symbol identifying the Future[T] + ## variable to yield. + ## fromNode: Used for better debug information (to give context). + ## valueReceiver: The node which defines an expression that retrieves the + ## future's value. + ## + ## rootReceiver: ??? TODO + # -> yield future<x> + result.add newNimNode(nnkYieldStmt, fromNode).add(futureVarNode) + # -> future<x>.read + valueReceiver = newDotExpr(futureVarNode, newIdentNode("read")) + result.add generateExceptionCheck(futureVarNode, tryStmt, rootReceiver, + fromNode) + template createVar(result: var NimNode, futSymName: string, asyncProc: NimNode, valueReceiver, rootReceiver: expr, @@ -1323,9 +1682,7 @@ template createVar(result: var NimNode, futSymName: string, result = newNimNode(nnkStmtList, fromNode) var futSym = genSym(nskVar, "future") result.add newVarStmt(futSym, asyncProc) # -> var future<x> = y - result.add newNimNode(nnkYieldStmt, fromNode).add(futSym) # -> yield future<x> - valueReceiver = newDotExpr(futSym, newIdentNode("read")) # -> future<x>.read - result.add generateExceptionCheck(futSym, tryStmt, rootReceiver, fromNode) + useVar(result, futSym, valueReceiver, rootReceiver, fromNode) proc processBody(node, retFutureSym: NimNode, subTypeIsVoid: bool, @@ -1342,19 +1699,23 @@ proc processBody(node, retFutureSym: NimNode, else: result.add newCall(newIdentNode("complete"), retFutureSym) else: - result.add newCall(newIdentNode("complete"), retFutureSym, - node[0].processBody(retFutureSym, subTypeIsVoid, tryStmt)) + let x = node[0].processBody(retFutureSym, subTypeIsVoid, tryStmt) + if x.kind == nnkYieldStmt: result.add x + else: + result.add newCall(newIdentNode("complete"), retFutureSym, x) result.add newNimNode(nnkReturnStmt, node).add(newNilLit()) return # Don't process the children of this return stmt of nnkCommand, nnkCall: if node[0].kind == nnkIdent and node[0].ident == !"await": case node[1].kind - of nnkIdent, nnkInfix: + of nnkIdent, nnkInfix, nnkDotExpr: # await x + # await x or y result = newNimNode(nnkYieldStmt, node).add(node[1]) # -> yield x of nnkCall, nnkCommand: # await foo(p, x) + # await foo p, x var futureValue: NimNode result.createVar("future" & $node[1][0].toStrLit, node[1], futureValue, futureValue, node) @@ -1511,38 +1872,40 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = # -> complete(retFuture, result) var iteratorNameSym = genSym(nskIterator, $prc[0].getName & "Iter") var procBody = prc[6].processBody(retFutureSym, subtypeIsVoid, nil) - if not subtypeIsVoid: - procBody.insert(0, newNimNode(nnkPragma).add(newIdentNode("push"), - newNimNode(nnkExprColonExpr).add(newNimNode(nnkBracketExpr).add( - newIdentNode("warning"), newIdentNode("resultshadowed")), - newIdentNode("off")))) # -> {.push warning[resultshadowed]: off.} - - procBody.insert(1, newNimNode(nnkVarSection, prc[6]).add( - newIdentDefs(newIdentNode("result"), baseType))) # -> var result: T - - procBody.insert(2, newNimNode(nnkPragma).add( - newIdentNode("pop"))) # -> {.pop.}) - - procBody.add( - newCall(newIdentNode("complete"), - retFutureSym, newIdentNode("result"))) # -> complete(retFuture, result) - else: - # -> complete(retFuture) - procBody.add(newCall(newIdentNode("complete"), retFutureSym)) + # don't do anything with forward bodies (empty) + if procBody.kind != nnkEmpty: + if not subtypeIsVoid: + procBody.insert(0, newNimNode(nnkPragma).add(newIdentNode("push"), + newNimNode(nnkExprColonExpr).add(newNimNode(nnkBracketExpr).add( + newIdentNode("warning"), newIdentNode("resultshadowed")), + newIdentNode("off")))) # -> {.push warning[resultshadowed]: off.} + + procBody.insert(1, newNimNode(nnkVarSection, prc[6]).add( + newIdentDefs(newIdentNode("result"), baseType))) # -> var result: T + + procBody.insert(2, newNimNode(nnkPragma).add( + newIdentNode("pop"))) # -> {.pop.}) + + procBody.add( + newCall(newIdentNode("complete"), + retFutureSym, newIdentNode("result"))) # -> complete(retFuture, result) + else: + # -> complete(retFuture) + procBody.add(newCall(newIdentNode("complete"), retFutureSym)) - var closureIterator = newProc(iteratorNameSym, [newIdentNode("FutureBase")], - procBody, nnkIteratorDef) - closureIterator[4] = newNimNode(nnkPragma, prc[6]).add(newIdentNode("closure")) - outerProcBody.add(closureIterator) + var closureIterator = newProc(iteratorNameSym, [newIdentNode("FutureBase")], + procBody, nnkIteratorDef) + closureIterator[4] = newNimNode(nnkPragma, prc[6]).add(newIdentNode("closure")) + outerProcBody.add(closureIterator) - # -> createCb(retFuture) - #var cbName = newIdentNode("cb") - var procCb = newCall(bindSym"createCb", retFutureSym, iteratorNameSym, - newStrLitNode(prc[0].getName)) - outerProcBody.add procCb + # -> createCb(retFuture) + #var cbName = newIdentNode("cb") + var procCb = getAst createCb(retFutureSym, iteratorNameSym, + newStrLitNode(prc[0].getName)) + outerProcBody.add procCb - # -> return retFuture - outerProcBody.add newNimNode(nnkReturnStmt, prc[6][prc[6].len-1]).add(retFutureSym) + # -> return retFuture + outerProcBody.add newNimNode(nnkReturnStmt, prc[6][prc[6].len-1]).add(retFutureSym) result = prc @@ -1550,19 +1913,19 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = for i in 0 .. <result[4].len: if result[4][i].kind == nnkIdent and result[4][i].ident == !"async": result[4].del(i) + result[4] = newEmptyNode() if subtypeIsVoid: # Add discardable pragma. if returnType.kind == nnkEmpty: # Add Future[void] result[3][0] = parseExpr("Future[void]") - - result[6] = outerProcBody - + if procBody.kind != nnkEmpty: + result[6] = outerProcBody #echo(treeRepr(result)) - #if prc[0].getName == "hubConnectionLoop": + #if prc[0].getName == "testInfix": # echo(toStrLit(result)) -macro async*(prc: stmt): stmt {.immediate.} = +macro async*(prc: untyped): untyped = ## Macro which processes async procedures into the appropriate ## iterators and yield statements. if prc.kind == nnkStmtList: @@ -1571,6 +1934,8 @@ macro async*(prc: stmt): stmt {.immediate.} = result.add asyncSingleProc(oneProc) else: result = asyncSingleProc(prc) + when defined(nimDumpAsync): + echo repr result proc recvLine*(socket: AsyncFD): Future[string] {.async.} = ## Reads a line of data from ``socket``. Returned future will complete once @@ -1611,6 +1976,11 @@ proc recvLine*(socket: AsyncFD): Future[string] {.async.} = return add(result, c) +proc callSoon*(cbproc: proc ()) = + ## Schedule `cbproc` to be called as soon as possible. + ## The callback is called when control returns to the event loop. + getGlobalDispatcher().callbacks.enqueue(cbproc) + proc runForever*() = ## Begins a never ending global dispatcher poll loop. while true: diff --git a/lib/pure/asyncfile.nim b/lib/pure/asyncfile.nim index c7b9fac18..5df606ea8 100644 --- a/lib/pure/asyncfile.nim +++ b/lib/pure/asyncfile.nim @@ -162,7 +162,7 @@ proc read*(f: AsyncFile, size: int): Future[string] = # Request completed immediately. var bytesRead: DWord let overlappedRes = getOverlappedResult(f.fd.Handle, - cast[POverlapped](ol)[], bytesRead, false.WinBool) + cast[POverlapped](ol), bytesRead, false.WinBool) if not overlappedRes.bool: let err = osLastError() if err.int32 == ERROR_HANDLE_EOF: @@ -282,7 +282,7 @@ proc write*(f: AsyncFile, data: string): Future[void] = # Request completed immediately. var bytesWritten: DWord let overlappedRes = getOverlappedResult(f.fd.Handle, - cast[POverlapped](ol)[], bytesWritten, false.WinBool) + cast[POverlapped](ol), bytesWritten, false.WinBool) if not overlappedRes.bool: retFuture.fail(newException(OSError, osErrorMsg(osLastError()))) else: @@ -316,6 +316,7 @@ proc write*(f: AsyncFile, data: string): Future[void] = proc close*(f: AsyncFile) = ## Closes the file specified. + unregister(f.fd) when defined(windows) or defined(nimdoc): if not closeHandle(f.fd.Handle).bool: raiseOSError(osLastError()) diff --git a/lib/pure/asynchttpserver.nim b/lib/pure/asynchttpserver.nim index 590b52c1a..6a7326e83 100644 --- a/lib/pure/asynchttpserver.nim +++ b/lib/pure/asynchttpserver.nim @@ -25,12 +25,21 @@ ## ## waitFor server.serve(Port(8080), cb) -import strtabs, asyncnet, asyncdispatch, parseutils, uri, strutils +import tables, asyncnet, asyncdispatch, parseutils, uri, strutils +import httpcore + +export httpcore except parseHeader + +# TODO: If it turns out that the decisions that asynchttpserver makes +# explicitly, about whether to close the client sockets or upgrade them are +# wrong, then add a return value which determines what to do for the callback. +# Also, maybe move `client` out of `Request` object and into the args for +# the proc. type Request* = object client*: AsyncSocket # TODO: Separate this into a Response object? reqMethod*: string - headers*: StringTableRef + headers*: HttpHeaders protocol*: tuple[orig: string, major, minor: int] url*: Uri hostname*: string ## The hostname of the client that made the request. @@ -39,83 +48,29 @@ type AsyncHttpServer* = ref object socket: AsyncSocket reuseAddr: bool - - HttpCode* = enum - Http100 = "100 Continue", - Http101 = "101 Switching Protocols", - Http200 = "200 OK", - Http201 = "201 Created", - Http202 = "202 Accepted", - Http204 = "204 No Content", - Http205 = "205 Reset Content", - Http206 = "206 Partial Content", - Http300 = "300 Multiple Choices", - Http301 = "301 Moved Permanently", - Http302 = "302 Found", - Http303 = "303 See Other", - Http304 = "304 Not Modified", - Http305 = "305 Use Proxy", - Http307 = "307 Temporary Redirect", - Http400 = "400 Bad Request", - Http401 = "401 Unauthorized", - Http403 = "403 Forbidden", - Http404 = "404 Not Found", - Http405 = "405 Method Not Allowed", - Http406 = "406 Not Acceptable", - Http407 = "407 Proxy Authentication Required", - Http408 = "408 Request Timeout", - Http409 = "409 Conflict", - Http410 = "410 Gone", - Http411 = "411 Length Required", - Http412 = "412 Precondition Failed", - Http413 = "413 Request Entity Too Large", - Http414 = "414 Request-URI Too Long", - Http415 = "415 Unsupported Media Type", - Http416 = "416 Requested Range Not Satisfiable", - Http417 = "417 Expectation Failed", - Http418 = "418 I'm a teapot", - Http500 = "500 Internal Server Error", - Http501 = "501 Not Implemented", - Http502 = "502 Bad Gateway", - Http503 = "503 Service Unavailable", - Http504 = "504 Gateway Timeout", - Http505 = "505 HTTP Version Not Supported" - - HttpVersion* = enum - HttpVer11, - HttpVer10 + reusePort: bool {.deprecated: [TRequest: Request, PAsyncHttpServer: AsyncHttpServer, THttpCode: HttpCode, THttpVersion: HttpVersion].} -proc `==`*(protocol: tuple[orig: string, major, minor: int], - ver: HttpVersion): bool = - let major = - case ver - of HttpVer11, HttpVer10: 1 - let minor = - case ver - of HttpVer11: 1 - of HttpVer10: 0 - result = protocol.major == major and protocol.minor == minor - -proc newAsyncHttpServer*(reuseAddr = true): AsyncHttpServer = +proc newAsyncHttpServer*(reuseAddr = true, reusePort = false): AsyncHttpServer = ## Creates a new ``AsyncHttpServer`` instance. new result result.reuseAddr = reuseAddr + result.reusePort = reusePort -proc addHeaders(msg: var string, headers: StringTableRef) = +proc addHeaders(msg: var string, headers: HttpHeaders) = for k, v in headers: msg.add(k & ": " & v & "\c\L") -proc sendHeaders*(req: Request, headers: StringTableRef): Future[void] = +proc sendHeaders*(req: Request, headers: HttpHeaders): Future[void] = ## Sends the specified headers to the requesting client. var msg = "" addHeaders(msg, headers) return req.client.send(msg) proc respond*(req: Request, code: HttpCode, content: string, - headers: StringTableRef = nil): Future[void] = + headers: HttpHeaders = nil): Future[void] = ## Responds to the request with the specified ``HttpCode``, headers and ## content. ## @@ -128,16 +83,6 @@ proc respond*(req: Request, code: HttpCode, content: string, msg.add(content) result = req.client.send(msg) -proc parseHeader(line: string): tuple[key, value: string] = - var i = 0 - i = line.parseUntil(result.key, ':') - inc(i) # skip : - if i < len(line): - i += line.skipWhiteSpace(i) - i += line.parseUntil(result.value, {'\c', '\L'}, i) - else: - result.value = "" - proc parseProtocol(protocol: string): tuple[orig: string, major, minor: int] = var i = protocol.skipIgnoreCase("HTTP/") if i != 5: @@ -156,7 +101,7 @@ proc processClient(client: AsyncSocket, address: string, Future[void] {.closure, gcsafe.}) {.async.} = var request: Request request.url = initUri() - request.headers = newStringTable(modeCaseInsensitive) + request.headers = newHttpHeaders() var lineFut = newFutureVar[string]("asynchttpserver.processClient") lineFut.mget() = newStringOfCap(80) var key, value = "" @@ -165,7 +110,7 @@ proc processClient(client: AsyncSocket, address: string, # GET /path HTTP/1.1 # Header: val # \n - request.headers.clear(modeCaseInsensitive) + request.headers.clear() request.body = "" request.hostname.shallowCopy(address) assert client != nil @@ -208,29 +153,34 @@ proc processClient(client: AsyncSocket, address: string, if lineFut.mget == "\c\L": break let (key, value) = parseHeader(lineFut.mget) request.headers[key] = value + # Ensure the client isn't trying to DoS us. + if request.headers.len > headerLimit: + await client.sendStatus("400 Bad Request") + request.client.close() + return if request.reqMethod == "post": # Check for Expect header if request.headers.hasKey("Expect"): - if request.headers.getOrDefault("Expect").toLower == "100-continue": + if "100-continue" in request.headers["Expect"]: await client.sendStatus("100 Continue") else: await client.sendStatus("417 Expectation Failed") - # Read the body - # - Check for Content-length header - if request.headers.hasKey("Content-Length"): - var contentLength = 0 - if parseInt(request.headers.getOrDefault("Content-Length"), - contentLength) == 0: - await request.respond(Http400, "Bad Request. Invalid Content-Length.") - continue - else: - request.body = await client.recv(contentLength) - assert request.body.len == contentLength - else: - await request.respond(Http400, "Bad Request. No Content-Length.") + # Read the body + # - Check for Content-length header + if request.headers.hasKey("Content-Length"): + var contentLength = 0 + if parseInt(request.headers["Content-Length"], + contentLength) == 0: + await request.respond(Http400, "Bad Request. Invalid Content-Length.") continue + else: + request.body = await client.recv(contentLength) + assert request.body.len == contentLength + elif request.reqMethod == "post": + await request.respond(Http400, "Bad Request. No Content-Length.") + continue case request.reqMethod of "get", "post", "head", "put", "delete", "trace", "options", @@ -240,6 +190,9 @@ proc processClient(client: AsyncSocket, address: string, await request.respond(Http400, "Invalid request method. Got: " & request.reqMethod) + if "upgrade" in request.headers.getOrDefault("connection"): + return + # Persistent connections if (request.protocol == HttpVer11 and request.headers.getOrDefault("connection").normalize != "close") or @@ -264,6 +217,8 @@ proc serve*(server: AsyncHttpServer, port: Port, server.socket = newAsyncSocket() if server.reuseAddr: server.socket.setSockOpt(OptReuseAddr, true) + if server.reusePort: + server.socket.setSockOpt(OptReusePort, true) server.socket.bindAddr(port, address) server.socket.listen() @@ -287,7 +242,7 @@ when not defined(testing) and isMainModule: #echo(req.headers) let headers = {"Date": "Tue, 29 Apr 2014 23:40:08 GMT", "Content-type": "text/plain; charset=utf-8"} - await req.respond(Http200, "Hello World", headers.newStringTable()) + await req.respond(Http200, "Hello World", headers.newHttpHeaders()) asyncCheck server.serve(Port(5555), cb) runForever() diff --git a/lib/pure/asyncnet.nim b/lib/pure/asyncnet.nim index 6b19a48be..a1988f4a6 100644 --- a/lib/pure/asyncnet.nim +++ b/lib/pure/asyncnet.nim @@ -11,7 +11,7 @@ ## asynchronous dispatcher defined in the ``asyncdispatch`` module. ## ## SSL -## --- +## ---- ## ## SSL can be enabled by compiling with the ``-d:ssl`` flag. ## @@ -62,7 +62,9 @@ import os export SOBool -when defined(ssl): +const defineSsl = defined(ssl) or defined(nimdoc) + +when defineSsl: import openssl type @@ -79,7 +81,7 @@ type of false: nil case isSsl: bool of true: - when defined(ssl): + when defineSsl: sslHandle: SslPtr sslContext: SslContext bioIn: BIO @@ -125,7 +127,7 @@ proc newAsyncSocket*(domain, sockType, protocol: cint, Domain(domain), SockType(sockType), Protocol(protocol), buffered) -when defined(ssl): +when defineSsl: proc getSslError(handle: SslPtr, err: cint): cint = assert err < 0 var ret = SSLGetError(handle, err.cint) @@ -186,7 +188,7 @@ proc connect*(socket: AsyncSocket, address: string, port: Port) {.async.} = ## or an error occurs. await connect(socket.fd.AsyncFD, address, port, socket.domain) if socket.isSsl: - when defined(ssl): + when defineSsl: let flags = {SocketFlag.SafeDisconn} sslSetConnectState(socket.sslHandle) sslLoop(socket, flags, sslDoHandshake(socket.sslHandle)) @@ -197,7 +199,7 @@ template readInto(buf: cstring, size: int, socket: AsyncSocket, ## this is a template and not a proc. var res = 0 if socket.isSsl: - when defined(ssl): + when defineSsl: # SSL mode. sslLoop(socket, flags, sslRead(socket.sslHandle, buf, size.cint)) @@ -274,7 +276,7 @@ proc send*(socket: AsyncSocket, data: string, ## data has been sent. assert socket != nil if socket.isSsl: - when defined(ssl): + when defineSsl: var copy = data sslLoop(socket, flags, sslWrite(socket.sslHandle, addr copy[0], copy.len.cint)) @@ -426,9 +428,6 @@ proc recvLine*(socket: AsyncSocket, ## ## **Warning**: ``recvLine`` on unbuffered sockets assumes that the protocol ## uses ``\r\L`` to delimit a new line. - template addNLIfEmpty(): stmt = - if result.len == 0: - result.add("\c\L") assert SocketFlag.Peek notin flags ## TODO: # TODO: Optimise this @@ -456,7 +455,8 @@ proc bindAddr*(socket: AsyncSocket, port = Port(0), address = "") {. of AF_INET6: realaddr = "::" of AF_INET: realaddr = "0.0.0.0" else: - raiseOSError("Unknown socket address family and no address specified to bindAddr") + raise newException(ValueError, + "Unknown socket address family and no address specified to bindAddr") var aiList = getAddrInfo(realaddr, port, socket.domain) if bindAddr(socket.fd, aiList.ai_addr, aiList.ai_addrlen.Socklen) < 0'i32: @@ -468,7 +468,7 @@ proc close*(socket: AsyncSocket) = ## Closes the socket. defer: socket.fd.AsyncFD.closeSocket() - when defined(ssl): + when defineSsl: if socket.isSSL: let res = SslShutdown(socket.sslHandle) SSLFree(socket.sslHandle) @@ -478,7 +478,7 @@ proc close*(socket: AsyncSocket) = raiseSslError() socket.closed = true # TODO: Add extra debugging checks for this. -when defined(ssl): +when defineSsl: proc wrapSocket*(ctx: SslContext, socket: AsyncSocket) = ## Wraps a socket in an SSL context. This function effectively turns ## ``socket`` into an SSL socket. @@ -487,7 +487,7 @@ when defined(ssl): ## prone to security vulnerabilities. socket.isSsl = true socket.sslContext = ctx - socket.sslHandle = SSLNew(SSLCTX(socket.sslContext)) + socket.sslHandle = SSLNew(socket.sslContext.context) if socket.sslHandle == nil: raiseSslError() diff --git a/lib/pure/base64.nim b/lib/pure/base64.nim index 32d37ce02..2fc0b1c5e 100644 --- a/lib/pure/base64.nim +++ b/lib/pure/base64.nim @@ -90,6 +90,8 @@ template encodeInternal(s: expr, lineLen: int, newLine: string): stmt {.immediat if r+4 != result.len: setLen(result, r+4) else: + if r != result.len: + setLen(result, r) #assert(r == result.len) discard @@ -162,4 +164,3 @@ when isMainModule: "asure.", longText] for t in items(tests): assert decode(encode(t)) == t - diff --git a/lib/pure/collections/LockFreeHash.nim b/lib/pure/collections/LockFreeHash.nim index a3ead81e3..954d62491 100644 --- a/lib/pure/collections/LockFreeHash.nim +++ b/lib/pure/collections/LockFreeHash.nim @@ -52,7 +52,7 @@ when sizeof(int) == 4: # 32bit {.deprecated: [TRaw: Raw].} elif sizeof(int) == 8: # 64bit type - Raw = range[0..4611686018427387903] + Raw = range[0'i64..4611686018427387903'i64] ## The range of uint values that can be stored directly in a value slot ## when on a 64 bit platform {.deprecated: [TRaw: Raw].} @@ -74,7 +74,7 @@ type copyDone: int next: PConcTable[K,V] data: EntryArr -{.deprecated: [TEntry: Entry, TEntryArr: EntryArr.} +{.deprecated: [TEntry: Entry, TEntryArr: EntryArr].} proc setVal[K,V](table: var PConcTable[K,V], key: int, val: int, expVal: int, match: bool): int @@ -244,9 +244,9 @@ proc copySlot[K,V](idx: int, oldTbl: var PConcTable[K,V], newTbl: var PConcTable echo("oldVal is Tomb!!!, should not happen") if pop(oldVal) != 0: result = setVal(newTbl, pop(oldKey), pop(oldVal), 0, true) == 0 - if result: + #if result: #echo("Copied a Slot! idx= " & $idx & " key= " & $oldKey & " val= " & $oldVal) - else: + #else: #echo("copy slot failed") # Our copy is done so we disable the old slot while not ok: diff --git a/lib/pure/collections/chains.nim b/lib/pure/collections/chains.nim new file mode 100644 index 000000000..6b2ecd272 --- /dev/null +++ b/lib/pure/collections/chains.nim @@ -0,0 +1,44 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2016 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Template based implementation of singly and doubly linked lists. +## The involved types should have 'prev' or 'next' fields and the +## list header should have 'head' or 'tail' fields. + +template prepend*(header, node) = + when compiles(header.head): + when compiles(node.prev): + if header.head != nil: + header.head.prev = node + node.next = header.head + header.head = node + when compiles(header.tail): + if header.tail == nil: + header.tail = node + +template append*(header, node) = + when compiles(header.head): + if header.head == nil: + header.head = node + when compiles(header.tail): + when compiles(node.prev): + node.prev = header.tail + if header.tail != nil: + header.tail.next = node + header.tail = node + +template unlink*(header, node) = + if node.next != nil: + node.next.prev = node.prev + if node.prev != nil: + node.prev.next = node.next + if header.head == node: + header.head = node.prev + if header.tail == node: + header.tail = node.next diff --git a/lib/pure/collections/heapqueue.nim b/lib/pure/collections/heapqueue.nim new file mode 100644 index 000000000..149a1c9fc --- /dev/null +++ b/lib/pure/collections/heapqueue.nim @@ -0,0 +1,107 @@ +##[ Heap queue algorithm (a.k.a. priority queue). Ported from Python heapq. + +Heaps are arrays for which a[k] <= a[2*k+1] and a[k] <= a[2*k+2] for +all k, counting elements from 0. For the sake of comparison, +non-existing elements are considered to be infinite. The interesting +property of a heap is that a[0] is always its smallest element. + +]## + +type HeapQueue*[T] = distinct seq[T] + +proc newHeapQueue*[T](): HeapQueue[T] {.inline.} = HeapQueue[T](newSeq[T]()) +proc newHeapQueue*[T](h: var HeapQueue[T]) {.inline.} = h = HeapQueue[T](newSeq[T]()) + +proc len*[T](h: HeapQueue[T]): int {.inline.} = seq[T](h).len +proc `[]`*[T](h: HeapQueue[T], i: int): T {.inline.} = seq[T](h)[i] +proc `[]=`[T](h: var HeapQueue[T], i: int, v: T) {.inline.} = seq[T](h)[i] = v +proc add[T](h: var HeapQueue[T], v: T) {.inline.} = seq[T](h).add(v) + +proc heapCmp[T](x, y: T): bool {.inline.} = + return (x < y) + +# 'heap' is a heap at all indices >= startpos, except possibly for pos. pos +# is the index of a leaf with a possibly out-of-order value. Restore the +# heap invariant. +proc siftdown[T](heap: var HeapQueue[T], startpos, p: int) = + var pos = p + var newitem = heap[pos] + # Follow the path to the root, moving parents down until finding a place + # newitem fits. + while pos > startpos: + let parentpos = (pos - 1) shr 1 + let parent = heap[parentpos] + if heapCmp(newitem, parent): + heap[pos] = parent + pos = parentpos + else: + break + heap[pos] = newitem + +proc siftup[T](heap: var HeapQueue[T], p: int) = + let endpos = len(heap) + var pos = p + let startpos = pos + let newitem = heap[pos] + # Bubble up the smaller child until hitting a leaf. + var childpos = 2*pos + 1 # leftmost child position + while childpos < endpos: + # Set childpos to index of smaller child. + let rightpos = childpos + 1 + if rightpos < endpos and not heapCmp(heap[childpos], heap[rightpos]): + childpos = rightpos + # Move the smaller child up. + heap[pos] = heap[childpos] + pos = childpos + childpos = 2*pos + 1 + # The leaf at pos is empty now. Put newitem there, and bubble it up + # to its final resting place (by sifting its parents down). + heap[pos] = newitem + siftdown(heap, startpos, pos) + +proc push*[T](heap: var HeapQueue[T], item: T) = + ## Push item onto heap, maintaining the heap invariant. + (seq[T](heap)).add(item) + siftdown(heap, 0, len(heap)-1) + +proc pop*[T](heap: var HeapQueue[T]): T = + ## Pop the smallest item off the heap, maintaining the heap invariant. + let lastelt = seq[T](heap).pop() + if heap.len > 0: + result = heap[0] + heap[0] = lastelt + siftup(heap, 0) + else: + result = lastelt + +proc replace*[T](heap: var HeapQueue[T], item: T): T = + ## Pop and return the current smallest value, and add the new item. + ## This is more efficient than pop() followed by push(), and can be + ## more appropriate when using a fixed-size heap. Note that the value + ## returned may be larger than item! That constrains reasonable uses of + ## this routine unless written as part of a conditional replacement: + + ## if item > heap[0]: + ## item = replace(heap, item) + result = heap[0] + heap[0] = item + siftup(heap, 0) + +proc pushpop*[T](heap: var HeapQueue[T], item: T): T = + ## Fast version of a push followed by a pop. + if heap.len > 0 and heapCmp(heap[0], item): + swap(item, heap[0]) + siftup(heap, 0) + return item + +when isMainModule: + # Simple sanity test + var heap = newHeapQueue[int]() + let data = [1, 3, 5, 7, 9, 2, 4, 6, 8, 0] + for item in data: + push(heap, item) + doAssert(heap[0] == 0) + var sort = newSeq[int]() + while heap.len > 0: + sort.add(pop(heap)) + doAssert(sort == @[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) diff --git a/lib/pure/collections/queues.nim b/lib/pure/collections/queues.nim index b9bf33bff..399e4d413 100644 --- a/lib/pure/collections/queues.nim +++ b/lib/pure/collections/queues.nim @@ -8,56 +8,142 @@ # ## Implementation of a `queue`:idx:. The underlying implementation uses a ``seq``. +## +## None of the procs that get an individual value from the queue can be used +## on an empty queue. +## If compiled with `boundChecks` option, those procs will raise an `IndexError` +## on such access. This should not be relied upon, as `-d:release` will +## disable those checks and may return garbage or crash the program. +## +## As such, a check to see if the queue is empty is needed before any +## access, unless your program logic guarantees it indirectly. +## +## .. code-block:: Nim +## proc foo(a, b: Positive) = # assume random positive values for `a` and `b` +## var q = initQueue[int]() # initializes the object +## for i in 1 ..< a: q.add i # populates the queue +## +## if b < q.len: # checking before indexed access +## echo "The element at index position ", b, " is ", q[b] +## +## # The following two lines don't need any checking on access due to the +## # logic of the program, but that would not be the case if `a` could be 0. +## assert q.front == 1 +## assert q.back == a +## +## while q.len > 0: # checking if the queue is empty +## echo q.pop() +## ## Note: For inter thread communication use ## a `Channel <channels.html>`_ instead. import math type - Queue*[T] = object ## a queue + Queue*[T] = object ## A queue. data: seq[T] rd, wr, count, mask: int {.deprecated: [TQueue: Queue].} -proc initQueue*[T](initialSize=4): Queue[T] = - ## creates a new queue. `initialSize` needs to be a power of 2. +proc initQueue*[T](initialSize: int = 4): Queue[T] = + ## Create a new queue. + ## Optionally, the initial capacity can be reserved via `initialSize` as a + ## performance optimization. The length of a newly created queue will still + ## be 0. + ## + ## `initialSize` needs to be a power of two. If you need to accept runtime + ## values for this you could use the ``nextPowerOfTwo`` proc from the + ## `math <math.html>`_ module. assert isPowerOfTwo(initialSize) result.mask = initialSize-1 newSeq(result.data, initialSize) -proc len*[T](q: Queue[T]): int = - ## returns the number of elements of `q`. +proc len*[T](q: Queue[T]): int {.inline.}= + ## Return the number of elements of `q`. result = q.count +template emptyCheck(q) = + # Bounds check for the regular queue access. + when compileOption("boundChecks"): + if unlikely(q.count < 1): + raise newException(IndexError, "Empty queue.") + +template xBoundsCheck(q, i) = + # Bounds check for the array like accesses. + when compileOption("boundChecks"): # d:release should disable this. + if unlikely(i >= q.count): # x < q.low is taken care by the Natural parameter + raise newException(IndexError, + "Out of bounds: " & $i & " > " & $(q.count - 1)) + +proc front*[T](q: Queue[T]): T {.inline.}= + ## Return the oldest element of `q`. Equivalent to `q.pop()` but does not + ## remove it from the queue. + emptyCheck(q) + result = q.data[q.rd] + +proc back*[T](q: Queue[T]): T {.inline.} = + ## Return the newest element of `q` but does not remove it from the queue. + emptyCheck(q) + result = q.data[q.wr - 1 and q.mask] + +proc `[]`*[T](q: Queue[T], i: Natural) : T {.inline.} = + ## Access the i-th element of `q` by order of insertion. + ## q[0] is the oldest (the next one q.pop() will extract), + ## q[^1] is the newest (last one added to the queue). + xBoundsCheck(q, i) + return q.data[q.rd + i and q.mask] + +proc `[]`*[T](q: var Queue[T], i: Natural): var T {.inline.} = + ## Access the i-th element of `q` and returns a mutable + ## reference to it. + xBoundsCheck(q, i) + return q.data[q.rd + i and q.mask] + +proc `[]=`* [T] (q: var Queue[T], i: Natural, val : T) {.inline.} = + ## Change the i-th element of `q`. + xBoundsCheck(q, i) + q.data[q.rd + i and q.mask] = val + iterator items*[T](q: Queue[T]): T = - ## yields every element of `q`. + ## Yield every element of `q`. var i = q.rd - var c = q.count - while c > 0: - dec c + for c in 0 ..< q.count: yield q.data[i] i = (i + 1) and q.mask iterator mitems*[T](q: var Queue[T]): var T = - ## yields every element of `q`. + ## Yield every element of `q`. var i = q.rd - var c = q.count - while c > 0: - dec c + for c in 0 ..< q.count: yield q.data[i] i = (i + 1) and q.mask +iterator pairs*[T](q: Queue[T]): tuple[key: int, val: T] = + ## Yield every (position, value) of `q`. + var i = q.rd + for c in 0 ..< q.count: + yield (c, q.data[i]) + i = (i + 1) and q.mask + +proc contains*[T](q: Queue[T], item: T): bool {.inline.} = + ## Return true if `item` is in `q` or false if not found. Usually used + ## via the ``in`` operator. It is the equivalent of ``q.find(item) >= 0``. + ## + ## .. code-block:: Nim + ## if x in q: + ## assert q.contains x + for e in q: + if e == item: return true + return false + proc add*[T](q: var Queue[T], item: T) = - ## adds an `item` to the end of the queue `q`. + ## Add an `item` to the end of the queue `q`. var cap = q.mask+1 - if q.count >= cap: - var n: seq[T] - newSeq(n, cap*2) - var i = 0 - for x in items(q): + if unlikely(q.count >= cap): + var n = newSeq[T](cap*2) + for i, x in q: # don't use copyMem because the GC and because it's slower. shallowCopy(n[i], x) - inc i shallowCopy(q.data, n) q.mask = cap*2 - 1 q.wr = q.count @@ -66,37 +152,104 @@ proc add*[T](q: var Queue[T], item: T) = q.data[q.wr] = item q.wr = (q.wr + 1) and q.mask -proc enqueue*[T](q: var Queue[T], item: T) = - ## alias for the ``add`` operation. - add(q, item) - -proc dequeue*[T](q: var Queue[T]): T = - ## removes and returns the first element of the queue `q`. - assert q.count > 0 +proc default[T](t: typedesc[T]): T {.inline.} = discard +proc pop*[T](q: var Queue[T]): T {.inline, discardable.} = + ## Remove and returns the first (oldest) element of the queue `q`. + emptyCheck(q) dec q.count result = q.data[q.rd] + q.data[q.rd] = default(type(result)) q.rd = (q.rd + 1) and q.mask +proc enqueue*[T](q: var Queue[T], item: T) = + ## Alias for the ``add`` operation. + q.add(item) + +proc dequeue*[T](q: var Queue[T]): T = + ## Alias for the ``pop`` operation. + q.pop() + proc `$`*[T](q: Queue[T]): string = - ## turns a queue into its string representation. + ## Turn a queue into its string representation. result = "[" - for x in items(q): + for x in items(q): # Don't remove the items here for reasons that don't fit in this margin. if result.len > 1: result.add(", ") result.add($x) result.add("]") when isMainModule: - var q = initQueue[int]() + var q = initQueue[int](1) q.add(123) q.add(9) - q.add(4) - var first = q.dequeue + q.enqueue(4) + var first = q.dequeue() q.add(56) q.add(6) - var second = q.dequeue + var second = q.pop() q.add(789) assert first == 123 assert second == 9 assert($q == "[4, 56, 6, 789]") + assert q[0] == q.front and q.front == 4 + assert q[^1] == q.back and q.back == 789 + q[0] = 42 + q[^1] = 7 + + assert 6 in q and 789 notin q + assert q.find(6) >= 0 + assert q.find(789) < 0 + + for i in -2 .. 10: + if i in q: + assert q.contains(i) and q.find(i) >= 0 + else: + assert(not q.contains(i) and q.find(i) < 0) + + when compileOption("boundChecks"): + try: + echo q[99] + assert false + except IndexError: + discard + + try: + assert q.len == 4 + for i in 0 ..< 5: q.pop() + assert false + except IndexError: + discard + + # grabs some types of resize error. + q = initQueue[int]() + for i in 1 .. 4: q.add i + q.pop() + q.pop() + for i in 5 .. 8: q.add i + assert $q == "[3, 4, 5, 6, 7, 8]" + + # Similar to proc from the documentation example + proc foo(a, b: Positive) = # assume random positive values for `a` and `b`. + var q = initQueue[int]() + assert q.len == 0 + for i in 1 .. a: q.add i + + if b < q.len: # checking before indexed access. + assert q[b] == b + 1 + + # The following two lines don't need any checking on access due to the logic + # of the program, but that would not be the case if `a` could be 0. + assert q.front == 1 + assert q.back == a + + while q.len > 0: # checking if the queue is empty + assert q.pop() > 0 + + #foo(0,0) + foo(8,5) + foo(10,9) + foo(1,1) + foo(2,1) + foo(1,5) + foo(3,2) diff --git a/lib/pure/collections/rtarrays.nim b/lib/pure/collections/rtarrays.nim index 9d8085643..2abe9d1f8 100644 --- a/lib/pure/collections/rtarrays.nim +++ b/lib/pure/collections/rtarrays.nim @@ -18,7 +18,7 @@ type RtArray*[T] = object ## L: Natural spart: seq[T] - apart: array [ArrayPartSize, T] + apart: array[ArrayPartSize, T] UncheckedArray* {.unchecked.}[T] = array[0..100_000_000, T] template usesSeqPart(x): expr = x.L > ArrayPartSize diff --git a/lib/pure/collections/sequtils.nim b/lib/pure/collections/sequtils.nim index 0e3824a81..f458b7636 100644 --- a/lib/pure/collections/sequtils.nim +++ b/lib/pure/collections/sequtils.nim @@ -20,6 +20,8 @@ ## **Note**: This interface will change as soon as the compiler supports ## closures and proper coroutines. +include "system/inclrtl" + when not defined(nimhygiene): {.pragma: dirty.} @@ -355,7 +357,7 @@ proc insert*[T](dest: var seq[T], src: openArray[T], pos=0) = inc(j) -template filterIt*(seq1, pred: expr): expr = +template filterIt*(seq1, pred: untyped): untyped = ## Returns a new sequence with all the items that fulfilled the predicate. ## ## Unlike the `proc` version, the predicate needs to be an expression using @@ -369,12 +371,12 @@ template filterIt*(seq1, pred: expr): expr = ## notAcceptable = filterIt(temperatures, it > 50 or it < -10) ## assert acceptable == @[-2.0, 24.5, 44.31] ## assert notAcceptable == @[-272.15, 99.9, -113.44] - var result {.gensym.} = newSeq[type(seq1[0])]() + var result = newSeq[type(seq1[0])]() for it {.inject.} in items(seq1): if pred: result.add(it) result -template keepItIf*(varSeq: seq, pred: expr) = +template keepItIf*(varSeq: seq, pred: untyped) = ## Convenience template around the ``keepIf`` proc to reduce typing. ## ## Unlike the `proc` version, the predicate needs to be an expression using @@ -409,7 +411,7 @@ proc all*[T](seq1: seq[T], pred: proc(item: T): bool {.closure.}): bool = return false return true -template allIt*(seq1, pred: expr): bool {.immediate.} = +template allIt*(seq1, pred: untyped): bool = ## Checks if every item fulfills the predicate. ## ## Example: @@ -418,7 +420,7 @@ template allIt*(seq1, pred: expr): bool {.immediate.} = ## let numbers = @[1, 4, 5, 8, 9, 7, 4] ## assert allIt(numbers, it < 10) == true ## assert allIt(numbers, it < 9) == false - var result {.gensym.} = true + var result = true for it {.inject.} in items(seq1): if not pred: result = false @@ -440,7 +442,7 @@ proc any*[T](seq1: seq[T], pred: proc(item: T): bool {.closure.}): bool = return true return false -template anyIt*(seq1, pred: expr): bool {.immediate.} = +template anyIt*(seq1, pred: untyped): bool = ## Checks if some item fulfills the predicate. ## ## Example: @@ -449,14 +451,14 @@ template anyIt*(seq1, pred: expr): bool {.immediate.} = ## let numbers = @[1, 4, 5, 8, 9, 7, 4] ## assert anyIt(numbers, it > 8) == true ## assert anyIt(numbers, it > 9) == false - var result {.gensym.} = false + var result = false for it {.inject.} in items(seq1): if pred: result = true break result -template toSeq*(iter: expr): expr {.immediate.} = +template toSeq*(iter: untyped): untyped {.oldimmediate.} = ## Transforms any iterator into a sequence. ## ## Example: @@ -482,7 +484,7 @@ template toSeq*(iter: expr): expr {.immediate.} = result.add(x) result -template foldl*(sequence, operation: expr): expr = +template foldl*(sequence, operation: untyped): untyped = ## Template to fold a sequence from left to right, returning the accumulation. ## ## The sequence is required to have at least a single element. Debug versions @@ -510,7 +512,7 @@ template foldl*(sequence, operation: expr): expr = ## assert concatenation == "nimiscool" let s = sequence assert s.len > 0, "Can't fold empty sequences" - var result {.gensym.}: type(s[0]) + var result: type(s[0]) result = s[0] for i in 1..<s.len: let @@ -519,7 +521,7 @@ template foldl*(sequence, operation: expr): expr = result = operation result -template foldl*(sequence, operation: expr, first): expr = +template foldl*(sequence, operation, first): untyped = ## Template to fold a sequence from left to right, returning the accumulation. ## ## This version of ``foldl`` gets a starting parameter. This makes it possible @@ -535,7 +537,7 @@ template foldl*(sequence, operation: expr, first): expr = ## numbers = @[0, 8, 1, 5] ## digits = foldl(numbers, a & (chr(b + ord('0'))), "") ## assert digits == "0815" - var result {.gensym.}: type(first) + var result: type(first) result = first for x in items(sequence): let @@ -544,7 +546,7 @@ template foldl*(sequence, operation: expr, first): expr = result = operation result -template foldr*(sequence, operation: expr): expr = +template foldr*(sequence, operation: untyped): untyped = ## Template to fold a sequence from right to left, returning the accumulation. ## ## The sequence is required to have at least a single element. Debug versions @@ -572,7 +574,7 @@ template foldr*(sequence, operation: expr): expr = ## assert concatenation == "nimiscool" let s = sequence assert s.len > 0, "Can't fold empty sequences" - var result {.gensym.}: type(s[0]) + var result: type(s[0]) result = sequence[s.len - 1] for i in countdown(s.len - 2, 0): let @@ -581,7 +583,7 @@ template foldr*(sequence, operation: expr): expr = result = operation result -template mapIt*(seq1, typ, op: expr): expr {.deprecated.}= +template mapIt*(seq1, typ, op: untyped): untyped = ## Convenience template around the ``map`` proc to reduce typing. ## ## The template injects the ``it`` variable which you can use directly in an @@ -596,13 +598,13 @@ template mapIt*(seq1, typ, op: expr): expr {.deprecated.}= ## assert strings == @["4", "8", "12", "16"] ## **Deprecated since version 0.12.0:** Use the ``mapIt(seq1, op)`` ## template instead. - var result {.gensym.}: seq[typ] = @[] + var result: seq[typ] = @[] for it {.inject.} in items(seq1): result.add(op) result -template mapIt*(seq1, op: expr): expr = +template mapIt*(seq1, op: untyped): untyped = ## Convenience template around the ``map`` proc to reduce typing. ## ## The template injects the ``it`` variable which you can use directly in an @@ -631,7 +633,7 @@ template mapIt*(seq1, op: expr): expr = result.add(op) result -template applyIt*(varSeq, op: expr) = +template applyIt*(varSeq, op: untyped) = ## Convenience template around the mutable ``apply`` proc to reduce typing. ## ## The template injects the ``it`` variable which you can use directly in an @@ -648,7 +650,7 @@ template applyIt*(varSeq, op: expr) = -template newSeqWith*(len: int, init: expr): expr = +template newSeqWith*(len: int, init: untyped): untyped = ## creates a new sequence, calling `init` to initialize each value. Example: ## ## .. code-block:: @@ -657,10 +659,10 @@ template newSeqWith*(len: int, init: expr): expr = ## seq2D[1][0] = true ## seq2D[0][1] = true ## - ## import math + ## import random ## var seqRand = newSeqWith(20, random(10)) ## echo seqRand - var result {.gensym.} = newSeq[type(init)](len) + var result = newSeq[type(init)](len) for i in 0 .. <len: result[i] = init result diff --git a/lib/pure/collections/sets.nim b/lib/pure/collections/sets.nim index 20f73ded3..552e41ef7 100644 --- a/lib/pure/collections/sets.nim +++ b/lib/pure/collections/sets.nim @@ -58,7 +58,7 @@ proc isValid*[A](s: HashSet[A]): bool = ## initialized. Example: ## ## .. code-block :: - ## proc savePreferences(options: Set[string]) = + ## proc savePreferences(options: HashSet[string]) = ## assert options.isValid, "Pass an initialized set!" ## # Do stuff here, may crash in release builds! result = not s.data.isNil @@ -72,7 +72,7 @@ proc len*[A](s: HashSet[A]): int = ## ## .. code-block:: ## - ## var values: Set[int] + ## var values: HashSet[int] ## assert(not values.isValid) ## assert values.len == 0 result = s.counter @@ -256,11 +256,13 @@ proc incl*[A](s: var HashSet[A], other: HashSet[A]) = assert other.isValid, "The set `other` needs to be initialized." for item in other: incl(s, item) -template doWhile(a: expr, b: stmt): stmt = +template doWhile(a, b) = while true: b if not a: break +proc default[T](t: typedesc[T]): T {.inline.} = discard + proc excl*[A](s: var HashSet[A], key: A) = ## Excludes `key` from the set `s`. ## @@ -277,12 +279,14 @@ proc excl*[A](s: var HashSet[A], key: A) = var msk = high(s.data) if i >= 0: s.data[i].hcode = 0 + s.data[i].key = default(type(s.data[i].key)) dec(s.counter) while true: # KnuthV3 Algo6.4R adapted for i=i+1 instead of i=i-1 var j = i # The correctness of this depends on (h+1) in nextTry, var r = j # though may be adaptable to other simple sequences. s.data[i].hcode = 0 # mark current EMPTY - doWhile ((i >= r and r > j) or (r > j and j > i) or (j > i and i >= r)): + s.data[i].key = default(type(s.data[i].key)) + doWhile((i >= r and r > j) or (r > j and j > i) or (j > i and i >= r)): i = (i + 1) and msk # increment mod table size if isEmpty(s.data[i].hcode): # end of collision cluster; So all done return @@ -334,7 +338,7 @@ proc init*[A](s: var HashSet[A], initialSize=64) = ## existing values and calling `excl() <#excl,TSet[A],A>`_ on them. Example: ## ## .. code-block :: - ## var a: Set[int] + ## var a: HashSet[int] ## a.init(4) ## a.incl(2) ## a.init @@ -367,7 +371,7 @@ proc toSet*[A](keys: openArray[A]): HashSet[A] = result = initSet[A](rightSize(keys.len)) for key in items(keys): result.incl(key) -template dollarImpl(): stmt {.dirty.} = +template dollarImpl() {.dirty.} = result = "{" for key in items(s): if result.len > 1: result.add(", ") @@ -611,7 +615,7 @@ proc card*[A](s: OrderedSet[A]): int {.inline.} = ## <http://en.wikipedia.org/wiki/Cardinality>`_ of a set. result = s.counter -template forAllOrderedPairs(yieldStmt: stmt) {.dirty, immediate.} = +template forAllOrderedPairs(yieldStmt: untyped) {.dirty.} = var h = s.first while h >= 0: var nxt = s.data[h].next diff --git a/lib/pure/collections/sharedtables.nim b/lib/pure/collections/sharedtables.nim index 20e1bb7a9..17600b272 100644 --- a/lib/pure/collections/sharedtables.nim +++ b/lib/pure/collections/sharedtables.nim @@ -45,6 +45,59 @@ template withLock(t, x: untyped) = x release(t.lock) +template withValue*[A, B](t: var SharedTable[A, B], key: A, + value, body: untyped) = + ## retrieves the value at ``t[key]``. + ## `value` can be modified in the scope of the ``withValue`` call. + ## + ## .. code-block:: nim + ## + ## sharedTable.withValue(key, value) do: + ## # block is executed only if ``key`` in ``t`` + ## # value is threadsafe in block + ## value.name = "username" + ## value.uid = 1000 + ## + acquire(t.lock) + try: + var hc: Hash + var index = rawGet(t, key, hc) + let hasKey = index >= 0 + if hasKey: + var value {.inject.} = addr(t.data[index].val) + body + finally: + release(t.lock) + +template withValue*[A, B](t: var SharedTable[A, B], key: A, + value, body1, body2: untyped) = + ## retrieves the value at ``t[key]``. + ## `value` can be modified in the scope of the ``withValue`` call. + ## + ## .. code-block:: nim + ## + ## sharedTable.withValue(key, value) do: + ## # block is executed only if ``key`` in ``t`` + ## # value is threadsafe in block + ## value.name = "username" + ## value.uid = 1000 + ## do: + ## # block is executed when ``key`` not in ``t`` + ## raise newException(KeyError, "Key not found") + ## + acquire(t.lock) + try: + var hc: Hash + var index = rawGet(t, key, hc) + let hasKey = index >= 0 + if hasKey: + var value {.inject.} = addr(t.data[index].val) + body1 + else: + body2 + finally: + release(t.lock) + proc mget*[A, B](t: var SharedTable[A, B], key: A): var B = ## retrieves the value at ``t[key]``. The value can be modified. ## If `key` is not in `t`, the ``KeyError`` exception is raised. diff --git a/lib/pure/collections/tableimpl.nim b/lib/pure/collections/tableimpl.nim index e4ec05b1c..be3507137 100644 --- a/lib/pure/collections/tableimpl.nim +++ b/lib/pure/collections/tableimpl.nim @@ -72,14 +72,14 @@ proc rawInsert[X, A, B](t: var X, data: var KeyValuePairSeq[A, B], key: A, val: B, hc: Hash, h: Hash) = rawInsertImpl() -template addImpl(enlarge) {.dirty, immediate.} = +template addImpl(enlarge) {.dirty.} = if mustRehash(t.dataLen, t.counter): enlarge(t) var hc: Hash var j = rawGetDeep(t, key, hc) rawInsert(t, t.data, key, val, hc, j) inc(t.counter) -template maybeRehashPutImpl(enlarge) {.dirty, immediate.} = +template maybeRehashPutImpl(enlarge) {.oldimmediate, dirty.} = if mustRehash(t.dataLen, t.counter): enlarge(t) index = rawGetKnownHC(t, key, hc) @@ -87,13 +87,13 @@ template maybeRehashPutImpl(enlarge) {.dirty, immediate.} = rawInsert(t, t.data, key, val, hc, index) inc(t.counter) -template putImpl(enlarge) {.dirty, immediate.} = +template putImpl(enlarge) {.oldimmediate, dirty.} = var hc: Hash var index = rawGet(t, key, hc) if index >= 0: t.data[index].val = val else: maybeRehashPutImpl(enlarge) -template mgetOrPutImpl(enlarge) {.dirty, immediate.} = +template mgetOrPutImpl(enlarge) {.dirty.} = var hc: Hash var index = rawGet(t, key, hc) if index < 0: @@ -102,7 +102,7 @@ template mgetOrPutImpl(enlarge) {.dirty, immediate.} = # either way return modifiable val result = t.data[index].val -template hasKeyOrPutImpl(enlarge) {.dirty, immediate.} = +template hasKeyOrPutImpl(enlarge) {.dirty.} = var hc: Hash var index = rawGet(t, key, hc) if index < 0: @@ -110,18 +110,24 @@ template hasKeyOrPutImpl(enlarge) {.dirty, immediate.} = maybeRehashPutImpl(enlarge) else: result = true -template delImpl() {.dirty, immediate.} = +proc default[T](t: typedesc[T]): T {.inline.} = discard + +template delImpl() {.dirty.} = var hc: Hash var i = rawGet(t, key, hc) let msk = maxHash(t) if i >= 0: t.data[i].hcode = 0 + t.data[i].key = default(type(t.data[i].key)) + t.data[i].val = default(type(t.data[i].val)) dec(t.counter) block outer: while true: # KnuthV3 Algo6.4R adapted for i=i+1 instead of i=i-1 var j = i # The correctness of this depends on (h+1) in nextTry, var r = j # though may be adaptable to other simple sequences. t.data[i].hcode = 0 # mark current EMPTY + t.data[i].key = default(type(t.data[i].key)) + t.data[i].val = default(type(t.data[i].val)) while true: i = (i + 1) and msk # increment mod table size if isEmpty(t.data[i].hcode): # end of collision cluster; So all done @@ -133,3 +139,10 @@ template delImpl() {.dirty, immediate.} = t.data[j] = t.data[i] else: shallowCopy(t.data[j], t.data[i]) # data[j] will be marked EMPTY next loop + +template clearImpl() {.dirty.} = + for i in 0 .. <t.data.len: + t.data[i].hcode = 0 + t.data[i].key = default(type(t.data[i].key)) + t.data[i].val = default(type(t.data[i].val)) + t.counter = 0 diff --git a/lib/pure/collections/tables.nim b/lib/pure/collections/tables.nim index 2ed0d2034..9308095aa 100644 --- a/lib/pure/collections/tables.nim +++ b/lib/pure/collections/tables.nim @@ -16,6 +16,39 @@ ## semantics, this means that ``=`` performs a copy of the hash table. ## For **reference** semantics use the ``Ref`` variant: ``TableRef``, ## ``OrderedTableRef``, ``CountTableRef``. +## To give an example, when `a` is a Table, then `var b = a` gives `b` +## as a new independent table. b is initialised with the contents of `a`. +## Changing `b` does not affect `a` and vice versa: +## +## .. code-block:: +## import tables +## +## var +## a = {1: "one", 2: "two"}.toTable # creates a Table +## b = a +## +## echo a, b # output: {1: one, 2: two}{1: one, 2: two} +## +## b[3] = "three" +## echo a, b # output: {1: one, 2: two}{1: one, 2: two, 3: three} +## echo a == b # output: false +## +## On the other hand, when `a` is a TableRef instead, then changes to `b` also affect `a`. +## Both `a` and `b` reference the same data structure: +## +## .. code-block:: +## import tables +## +## var +## a = {1: "one", 2: "two"}.newTable # creates a TableRef +## b = a +## +## echo a, b # output: {1: one, 2: two}{1: one, 2: two} +## +## b[3] = "three" +## echo a, b # output: {1: one, 2: two, 3: three}{1: one, 2: two, 3: three} +## echo a == b # output: true +## ## ## If you are using simple standard types like ``int`` or ``string`` for the ## keys of the table you won't have any problems, but as soon as you try to use @@ -80,11 +113,15 @@ type {.deprecated: [TTable: Table, PTable: TableRef].} -template maxHash(t): expr {.immediate.} = high(t.data) -template dataLen(t): expr = len(t.data) +template maxHash(t): untyped = high(t.data) +template dataLen(t): untyped = len(t.data) include tableimpl +proc clear*[A, B](t: Table[A, B] | TableRef[A, B]) = + ## Resets the table so that it is empty. + clearImpl() + proc rightSize*(count: Natural): int {.inline.} = ## Return the value of `initialSize` to support `count` items. ## @@ -98,7 +135,7 @@ proc len*[A, B](t: Table[A, B]): int = ## returns the number of keys in `t`. result = t.counter -template get(t, key): untyped {.immediate.} = +template get(t, key): untyped = ## retrieves the value at ``t[key]``. The value can be modified. ## If `key` is not in `t`, the ``KeyError`` exception is raised. mixin rawGet @@ -111,7 +148,7 @@ template get(t, key): untyped {.immediate.} = else: raise newException(KeyError, "key not found") -template getOrDefaultImpl(t, key): untyped {.immediate.} = +template getOrDefaultImpl(t, key): untyped = mixin rawGet var hc: Hash var index = rawGet(t, key, hc) @@ -136,6 +173,51 @@ proc mget*[A, B](t: var Table[A, B], key: A): var B {.deprecated.} = proc getOrDefault*[A, B](t: Table[A, B], key: A): B = getOrDefaultImpl(t, key) +template withValue*[A, B](t: var Table[A, B], key: A, + value, body: untyped) = + ## retrieves the value at ``t[key]``. + ## `value` can be modified in the scope of the ``withValue`` call. + ## + ## .. code-block:: nim + ## + ## sharedTable.withValue(key, value) do: + ## # block is executed only if ``key`` in ``t`` + ## value.name = "username" + ## value.uid = 1000 + ## + mixin rawGet + var hc: Hash + var index = rawGet(t, key, hc) + let hasKey = index >= 0 + if hasKey: + var value {.inject.} = addr(t.data[index].val) + body + +template withValue*[A, B](t: var Table[A, B], key: A, + value, body1, body2: untyped) = + ## retrieves the value at ``t[key]``. + ## `value` can be modified in the scope of the ``withValue`` call. + ## + ## .. code-block:: nim + ## + ## table.withValue(key, value) do: + ## # block is executed only if ``key`` in ``t`` + ## value.name = "username" + ## value.uid = 1000 + ## do: + ## # block is executed when ``key`` not in ``t`` + ## raise newException(KeyError, "Key not found") + ## + mixin rawGet + var hc: Hash + var index = rawGet(t, key, hc) + let hasKey = index >= 0 + if hasKey: + var value {.inject.} = addr(t.data[index].val) + body1 + else: + body2 + iterator allValues*[A, B](t: Table[A, B]; key: A): B = ## iterates over any value in the table `t` that belongs to the given `key`. var h: Hash = hash(key) and high(t.data) @@ -229,7 +311,7 @@ proc toTable*[A, B](pairs: openArray[(A, result = initTable[A, B](rightSize(pairs.len)) for key, val in items(pairs): result[key] = val -template dollarImpl(): stmt {.dirty.} = +template dollarImpl(): untyped {.dirty.} = if t.len == 0: result = "{:}" else: @@ -249,18 +331,17 @@ proc hasKey*[A, B](t: TableRef[A, B], key: A): bool = ## returns true iff `key` is in the table `t`. result = t[].hasKey(key) -template equalsImpl() = +template equalsImpl(t) = if s.counter == t.counter: # different insertion orders mean different 'data' seqs, so we have # to use the slow route here: for key, val in s: - # prefix notation leads to automatic dereference in case of PTable if not t.hasKey(key): return false - if t[key] != val: return false + if t.getOrDefault(key) != val: return false return true proc `==`*[A, B](s, t: Table[A, B]): bool = - equalsImpl() + equalsImpl(t) proc indexBy*[A, B, C](collection: A, index: proc(x: B): C): Table[C, B] = ## Index the collection with the proc provided. @@ -350,7 +431,7 @@ proc `$`*[A, B](t: TableRef[A, B]): string = proc `==`*[A, B](s, t: TableRef[A, B]): bool = if isNil(s): result = isNil(t) elif isNil(t): result = false - else: equalsImpl() + else: equalsImpl(t[]) proc newTableFrom*[A, B, C](collection: A, index: proc(x: B): C): TableRef[C, B] = ## Index the collection with the proc provided. @@ -376,7 +457,13 @@ proc len*[A, B](t: OrderedTable[A, B]): int {.inline.} = ## returns the number of keys in `t`. result = t.counter -template forAllOrderedPairs(yieldStmt: stmt) {.dirty, immediate.} = +proc clear*[A, B](t: OrderedTable[A, B] | OrderedTableRef[A, B]) = + ## Resets the table so that it is empty. + clearImpl() + t.first = -1 + t.last = -1 + +template forAllOrderedPairs(yieldStmt: untyped) {.oldimmediate, dirty.} = var h = t.first while h >= 0: var nxt = t.data[h].next @@ -562,7 +649,7 @@ proc len*[A, B](t: OrderedTableRef[A, B]): int {.inline.} = ## returns the number of keys in `t`. result = t.counter -template forAllOrderedPairs(yieldStmt: stmt) {.dirty, immediate.} = +template forAllOrderedPairs(yieldStmt: untyped) {.oldimmediate, dirty.} = var h = t.first while h >= 0: var nxt = t.data[h].next @@ -663,6 +750,27 @@ proc sort*[A, B](t: OrderedTableRef[A, B], ## contrast to the `sort` for count tables). t[].sort(cmp) +proc del*[A, B](t: var OrderedTable[A, B], key: A) = + ## deletes `key` from ordered hash table `t`. O(n) comlexity. + var prev = -1 + let hc = hash(key) + forAllOrderedPairs: + if t.data[h].hcode == hc: + if t.first == h: + t.first = t.data[h].next + else: + t.data[prev].next = t.data[h].next + var zeroValue : type(t.data[h]) + t.data[h] = zeroValue + dec t.counter + break + else: + prev = h + +proc del*[A, B](t: var OrderedTableRef[A, B], key: A) = + ## deletes `key` from ordered hash table `t`. O(n) comlexity. + t[].del(key) + # ------------------------------ count tables ------------------------------- type @@ -678,6 +786,11 @@ proc len*[A](t: CountTable[A]): int = ## returns the number of keys in `t`. result = t.counter +proc clear*[A](t: CountTable[A] | CountTableRef[A]) = + ## Resets the table so that it is empty. + clearImpl() + t.counter = 0 + iterator pairs*[A](t: CountTable[A]): (A, int) = ## iterates over any (key, value) pair in the table `t`. for h in 0..high(t.data): @@ -711,7 +824,7 @@ proc rawGet[A](t: CountTable[A], key: A): int = h = nextTry(h, high(t.data)) result = -1 - h # < 0 => MISSING; insert idx = -1 - result -template ctget(t, key: untyped): untyped {.immediate.} = +template ctget(t, key: untyped): untyped = var index = rawGet(t, key) if index >= 0: result = t.data[index].val else: @@ -984,6 +1097,26 @@ when isMainModule: s3[p1] = 30_000 s3[p2] = 45_000 + block: # Ordered table should preserve order after deletion + var + s4 = initOrderedTable[int, int]() + s4[1] = 1 + s4[2] = 2 + s4[3] = 3 + + var prev = 0 + for i in s4.values: + doAssert(prev < i) + prev = i + + s4.del(2) + doAssert(2 notin s4) + doAssert(s4.len == 2) + prev = 0 + for i in s4.values: + doAssert(prev < i) + prev = i + var t1 = initCountTable[string]() t2 = initCountTable[string]() @@ -1040,3 +1173,12 @@ when isMainModule: doAssert 0 == t.getOrDefault(testKey) t.inc(testKey,3) doAssert 3 == t.getOrDefault(testKey) + + # Clear tests + var clearTable = newTable[int, string]() + clearTable[42] = "asd" + clearTable[123123] = "piuyqwb " + doAssert clearTable[42] == "asd" + clearTable.clear() + doAssert(not clearTable.hasKey(123123)) + doAssert clearTable.getOrDefault(42) == nil diff --git a/lib/pure/concurrency/cpuload.nim b/lib/pure/concurrency/cpuload.nim index 22598b5c9..b0fd002ed 100644 --- a/lib/pure/concurrency/cpuload.nim +++ b/lib/pure/concurrency/cpuload.nim @@ -79,6 +79,8 @@ proc advice*(s: var ThreadPoolState): ThreadPoolAdvice = inc s.calls when not defined(testing) and isMainModule: + import random + proc busyLoop() = while true: discard random(80) diff --git a/lib/pure/future.nim b/lib/pure/future.nim index 3793edc8b..67975cfcb 100644 --- a/lib/pure/future.nim +++ b/lib/pure/future.nim @@ -29,28 +29,24 @@ proc createProcType(p, b: NimNode): NimNode {.compileTime.} = of nnkExprColonExpr: identDefs.add ident[0] identDefs.add ident[1] - of nnkIdent: + else: identDefs.add newIdentNode("i" & $i) identDefs.add(ident) - else: - error("Incorrect type list in proc type declaration.") identDefs.add newEmptyNode() formalParams.add identDefs - of nnkIdent: + else: var identDefs = newNimNode(nnkIdentDefs) identDefs.add newIdentNode("i0") identDefs.add(p) identDefs.add newEmptyNode() formalParams.add identDefs - else: - error("Incorrect type list in proc type declaration.") result.add formalParams result.add newEmptyNode() #echo(treeRepr(result)) #echo(result.toStrLit()) -macro `=>`*(p, b: expr): expr {.immediate.} = +macro `=>`*(p, b: untyped): untyped = ## Syntax sugar for anonymous procedures. ## ## .. code-block:: nim @@ -111,7 +107,7 @@ macro `=>`*(p, b: expr): expr {.immediate.} = #echo(result.toStrLit()) #return result # TODO: Bug? -macro `->`*(p, b: expr): expr {.immediate.} = +macro `->`*(p, b: untyped): untyped = ## Syntax sugar for procedure types. ## ## .. code-block:: nim @@ -129,7 +125,7 @@ macro `->`*(p, b: expr): expr {.immediate.} = type ListComprehension = object var lc*: ListComprehension -macro `[]`*(lc: ListComprehension, comp, typ: expr): expr = +macro `[]`*(lc: ListComprehension, comp, typ: untyped): untyped = ## List comprehension, returns a sequence. `comp` is the actual list ## comprehension, for example ``x | (x <- 1..10, x mod 2 == 0)``. `typ` is ## the type that will be stored inside the result seq. diff --git a/lib/pure/htmlparser.nim b/lib/pure/htmlparser.nim index d620e816e..1fe0b297b 100644 --- a/lib/pure/htmlparser.nim +++ b/lib/pure/htmlparser.nim @@ -433,7 +433,7 @@ proc htmlTag*(n: XmlNode): HtmlTag = proc htmlTag*(s: string): HtmlTag = ## converts `s` to a ``HtmlTag``. If `s` is no HTML tag, ``tagUnknown`` is ## returned. - let s = if allLower(s): s else: s.toLower + let s = if allLower(s): s else: toLowerAscii(s) result = toHtmlTag(s) proc entityToUtf8*(entity: string): string = @@ -464,12 +464,18 @@ proc untilElementEnd(x: var XmlParser, result: XmlNode, case x.kind of xmlElementStart, xmlElementOpen: case result.htmlTag - of tagLi, tagP, tagDt, tagDd, tagInput, tagOption: - # some tags are common to have no ``</end>``, like ``<li>``: + of tagP, tagInput, tagOption: + # some tags are common to have no ``</end>``, like ``<li>`` but + # allow ``<p>`` in `<dd>`, `<dt>` and ``<li>`` in next case if htmlTag(x.elemName) in {tagLi, tagP, tagDt, tagDd, tagInput, tagOption}: errors.add(expected(x, result)) break + of tagDd, tagDt, tagLi: + if htmlTag(x.elemName) in {tagLi, tagDt, tagDd, tagInput, + tagOption}: + errors.add(expected(x, result)) + break of tagTd, tagTh: if htmlTag(x.elemName) in {tagTr, tagTd, tagTh, tagTfoot, tagThead}: errors.add(expected(x, result)) @@ -513,13 +519,13 @@ proc parse(x: var XmlParser, errors: var seq[string]): XmlNode = errors.add(errorMsg(x)) next(x) of xmlElementStart: - result = newElement(x.elemName.toLower) + result = newElement(toLowerAscii(x.elemName)) next(x) untilElementEnd(x, result, errors) of xmlElementEnd: errors.add(errorMsg(x, "unexpected ending tag: " & x.elemName)) of xmlElementOpen: - result = newElement(x.elemName.toLower) + result = newElement(toLowerAscii(x.elemName)) next(x) result.attrs = newStringTable() while true: diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index 603763386..778ca2cbb 100644 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -79,15 +79,18 @@ ## constructor should be used for this purpose. However, ## currently only basic authentication is supported. -import net, strutils, uri, parseutils, strtabs, base64, os, mimetypes, math +import net, strutils, uri, parseutils, strtabs, base64, os, mimetypes, + math, random, httpcore import asyncnet, asyncdispatch import nativesockets +export httpcore except parseHeader # TODO: The ``except`` doesn't work + type Response* = tuple[ version: string, status: string, - headers: StringTableRef, + headers: HttpHeaders, body: string] Proxy* = ref object @@ -164,7 +167,7 @@ proc parseChunks(s: Socket, timeout: int): string = # Trailer headers will only be sent if the request specifies that we want # them: http://tools.ietf.org/html/rfc2616#section-3.6.1 -proc parseBody(s: Socket, headers: StringTableRef, timeout: int): string = +proc parseBody(s: Socket, headers: HttpHeaders, httpVersion: string, timeout: int): string = result = "" if headers.getOrDefault"Transfer-Encoding" == "chunked": result = parseChunks(s, timeout) @@ -190,7 +193,7 @@ proc parseBody(s: Socket, headers: StringTableRef, timeout: int): string = # -REGION- Connection: Close # (http://tools.ietf.org/html/rfc2616#section-4.4) NR.5 - if headers.getOrDefault"Connection" == "close": + if headers.getOrDefault"Connection" == "close" or httpVersion == "1.0": var buf = "" while true: buf = newString(4000) @@ -204,7 +207,7 @@ proc parseResponse(s: Socket, getBody: bool, timeout: int): Response = var linei = 0 var fullyRead = false var line = "" - result.headers = newStringTable(modeCaseInsensitive) + result.headers = newHttpHeaders() while true: line = "" linei = 0 @@ -239,10 +242,14 @@ proc parseResponse(s: Socket, getBody: bool, timeout: int): Response = inc(linei) # Skip : result.headers[name] = line[linei.. ^1].strip() + # Ensure the server isn't trying to DoS us. + if result.headers.len > headerLimit: + httpError("too many headers") + if not fullyRead: httpError("Connection was closed before full request has been made") if getBody: - result.body = parseBody(s, result.headers, timeout) + result.body = parseBody(s, result.headers, result.version, timeout) else: result.body = "" @@ -335,7 +342,7 @@ proc addFiles*(p: var MultipartData, xs: openarray[tuple[name, file: string]]): var m = newMimetypes() for name, file in xs.items: var contentType: string - let (dir, fName, ext) = splitFile(file) + let (_, fName, ext) = splitFile(file) if ext.len > 0: contentType = m.getMimetype(ext[1..ext.high], nil) p.add(name, readFile(file), fName & ext, contentType) @@ -392,6 +399,59 @@ proc request*(url: string, httpMethod: string, extraHeaders = "", var hostUrl = if proxy == nil: r else: parseUri(url) var headers = substr(httpMethod, len("http")) # TODO: Use generateHeaders further down once it supports proxies. + + var s = newSocket() + defer: s.close() + if s == nil: raiseOSError(osLastError()) + var port = net.Port(80) + if r.scheme == "https": + when defined(ssl): + sslContext.wrapSocket(s) + port = net.Port(443) + else: + raise newException(HttpRequestError, + "SSL support is not available. Cannot connect over SSL.") + if r.port != "": + port = net.Port(r.port.parseInt) + + + # get the socket ready. If we are connecting through a proxy to SSL, + # send the appropiate CONNECT header. If not, simply connect to the proper + # host (which may still be the proxy, for normal HTTP) + if proxy != nil and hostUrl.scheme == "https": + when defined(ssl): + var connectHeaders = "CONNECT " + let targetPort = if hostUrl.port == "": 443 else: hostUrl.port.parseInt + connectHeaders.add(hostUrl.hostname) + connectHeaders.add(":" & $targetPort) + connectHeaders.add(" HTTP/1.1\c\L") + connectHeaders.add("Host: " & hostUrl.hostname & ":" & $targetPort & "\c\L") + if proxy.auth != "": + let auth = base64.encode(proxy.auth, newline = "") + connectHeaders.add("Proxy-Authorization: basic " & auth & "\c\L") + connectHeaders.add("\c\L") + if timeout == -1: + s.connect(r.hostname, port) + else: + s.connect(r.hostname, port, timeout) + + s.send(connectHeaders) + let connectResult = parseResponse(s, false, timeout) + if not connectResult.status.startsWith("200"): + raise newException(HttpRequestError, + "The proxy server rejected a CONNECT request, " & + "so a secure connection could not be established.") + sslContext.wrapConnectedSocket(s, handshakeAsClient) + else: + raise newException(HttpRequestError, "SSL support not available. Cannot connect via proxy over SSL") + else: + if timeout == -1: + s.connect(r.hostname, port) + else: + s.connect(r.hostname, port, timeout) + + + # now that the socket is ready, prepare the headers if proxy == nil: headers.add ' ' if r.path[0] != '/': headers.add '/' @@ -415,29 +475,13 @@ proc request*(url: string, httpMethod: string, extraHeaders = "", add(headers, "Proxy-Authorization: basic " & auth & "\c\L") add(headers, extraHeaders) add(headers, "\c\L") - var s = newSocket() - if s == nil: raiseOSError(osLastError()) - var port = net.Port(80) - if r.scheme == "https": - when defined(ssl): - sslContext.wrapSocket(s) - port = net.Port(443) - else: - raise newException(HttpRequestError, - "SSL support is not available. Cannot connect over SSL.") - if r.port != "": - port = net.Port(r.port.parseInt) - if timeout == -1: - s.connect(r.hostname, port) - else: - s.connect(r.hostname, port, timeout) + # headers are ready. send them, await the result, and close the socket. s.send(headers) if body != "": s.send(body) result = parseResponse(s, httpMethod != "httpHEAD", timeout) - s.close() proc request*(url: string, httpMethod = httpGET, extraHeaders = "", body = "", sslContext = defaultSSLContext, timeout = -1, @@ -455,7 +499,7 @@ proc redirection(status: string): bool = if status.startsWith(i): return true -proc getNewLocation(lastURL: string, headers: StringTableRef): string = +proc getNewLocation(lastURL: string, headers: HttpHeaders): string = result = headers.getOrDefault"Location" if result == "": httpError("location header expected") # Relative URLs. (Not part of the spec, but soon will be.) @@ -624,10 +668,10 @@ proc newAsyncHttpClient*(userAgent = defUserAgent, ## ``sslContext`` specifies the SSL context to use for HTTPS requests. new result result.headers = newStringTable(modeCaseInsensitive) - result.userAgent = defUserAgent + result.userAgent = userAgent result.maxRedirects = maxRedirects when defined(ssl): - result.sslContext = net.SslContext(sslContext) + result.sslContext = sslContext proc close*(client: AsyncHttpClient) = ## Closes any connections held by the HTTP client. @@ -678,7 +722,8 @@ proc parseChunks(client: AsyncHttpClient): Future[string] {.async.} = # them: http://tools.ietf.org/html/rfc2616#section-3.6.1 proc parseBody(client: AsyncHttpClient, - headers: StringTableRef): Future[string] {.async.} = + headers: HttpHeaders, + httpVersion: string): Future[string] {.async.} = result = "" if headers.getOrDefault"Transfer-Encoding" == "chunked": result = await parseChunks(client) @@ -700,7 +745,7 @@ proc parseBody(client: AsyncHttpClient, # -REGION- Connection: Close # (http://tools.ietf.org/html/rfc2616#section-4.4) NR.5 - if headers.getOrDefault"Connection" == "close": + if headers.getOrDefault"Connection" == "close" or httpVersion == "1.0": var buf = "" while true: buf = await client.socket.recvFull(4000) @@ -713,7 +758,7 @@ proc parseResponse(client: AsyncHttpClient, var linei = 0 var fullyRead = false var line = "" - result.headers = newStringTable(modeCaseInsensitive) + result.headers = newHttpHeaders() while true: linei = 0 line = await client.socket.recvLine() @@ -748,10 +793,13 @@ proc parseResponse(client: AsyncHttpClient, inc(linei) # Skip : result.headers[name] = line[linei.. ^1].strip() + if result.headers.len > headerLimit: + httpError("too many headers") + if not fullyRead: httpError("Connection was closed before full request has been made") if getBody: - result.body = await parseBody(client, result.headers) + result.body = await parseBody(client, result.headers, result.version) else: result.body = "" diff --git a/lib/pure/httpcore.nim b/lib/pure/httpcore.nim new file mode 100644 index 000000000..562d16c19 --- /dev/null +++ b/lib/pure/httpcore.nim @@ -0,0 +1,198 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2016 Dominik Picheta +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Contains functionality shared between the ``httpclient`` and +## ``asynchttpserver`` modules. + +import tables, strutils, parseutils + +type + HttpHeaders* = ref object + table*: TableRef[string, seq[string]] + + HttpHeaderValues* = distinct seq[string] + + HttpCode* = enum + Http100 = "100 Continue", + Http101 = "101 Switching Protocols", + Http200 = "200 OK", + Http201 = "201 Created", + Http202 = "202 Accepted", + Http203 = "203 Non-Authoritative Information", + Http204 = "204 No Content", + Http205 = "205 Reset Content", + Http206 = "206 Partial Content", + Http300 = "300 Multiple Choices", + Http301 = "301 Moved Permanently", + Http302 = "302 Found", + Http303 = "303 See Other", + Http304 = "304 Not Modified", + Http305 = "305 Use Proxy", + Http307 = "307 Temporary Redirect", + Http400 = "400 Bad Request", + Http401 = "401 Unauthorized", + Http403 = "403 Forbidden", + Http404 = "404 Not Found", + Http405 = "405 Method Not Allowed", + Http406 = "406 Not Acceptable", + Http407 = "407 Proxy Authentication Required", + Http408 = "408 Request Timeout", + Http409 = "409 Conflict", + Http410 = "410 Gone", + Http411 = "411 Length Required", + Http412 = "412 Precondition Failed", + Http413 = "413 Request Entity Too Large", + Http414 = "414 Request-URI Too Long", + Http415 = "415 Unsupported Media Type", + Http416 = "416 Requested Range Not Satisfiable", + Http417 = "417 Expectation Failed", + Http418 = "418 I'm a teapot", + Http421 = "421 Misdirected Request", + Http422 = "422 Unprocessable Entity", + Http426 = "426 Upgrade Required", + Http428 = "428 Precondition Required", + Http429 = "429 Too Many Requests", + Http431 = "431 Request Header Fields Too Large", + Http451 = "451 Unavailable For Legal Reasons", + Http500 = "500 Internal Server Error", + Http501 = "501 Not Implemented", + Http502 = "502 Bad Gateway", + Http503 = "503 Service Unavailable", + Http504 = "504 Gateway Timeout", + Http505 = "505 HTTP Version Not Supported" + + HttpVersion* = enum + HttpVer11, + HttpVer10 + +const headerLimit* = 10_000 + +proc newHttpHeaders*(): HttpHeaders = + new result + result.table = newTable[string, seq[string]]() + +proc newHttpHeaders*(keyValuePairs: + openarray[tuple[key: string, val: string]]): HttpHeaders = + var pairs: seq[tuple[key: string, val: seq[string]]] = @[] + for pair in keyValuePairs: + pairs.add((pair.key.toLower(), @[pair.val])) + new result + result.table = newTable[string, seq[string]](pairs) + +proc clear*(headers: HttpHeaders) = + headers.table.clear() + +proc `[]`*(headers: HttpHeaders, key: string): HttpHeaderValues = + ## Returns the values associated with the given ``key``. If the returned + ## values are passed to a procedure expecting a ``string``, the first + ## value is automatically picked. If there are + ## no values associated with the key, an exception is raised. + ## + ## To access multiple values of a key, use the overloaded ``[]`` below or + ## to get all of them access the ``table`` field directly. + return headers.table[key.toLower].HttpHeaderValues + +converter toString*(values: HttpHeaderValues): string = + return seq[string](values)[0] + +proc `[]`*(headers: HttpHeaders, key: string, i: int): string = + ## Returns the ``i``'th value associated with the given key. If there are + ## no values associated with the key or the ``i``'th value doesn't exist, + ## an exception is raised. + return headers.table[key.toLower][i] + +proc `[]=`*(headers: HttpHeaders, key, value: string) = + ## Sets the header entries associated with ``key`` to the specified value. + ## Replaces any existing values. + headers.table[key.toLower] = @[value] + +proc `[]=`*(headers: HttpHeaders, key: string, value: seq[string]) = + ## Sets the header entries associated with ``key`` to the specified list of + ## values. + ## Replaces any existing values. + headers.table[key.toLower] = value + +proc add*(headers: HttpHeaders, key, value: string) = + ## Adds the specified value to the specified key. Appends to any existing + ## values associated with the key. + if not headers.table.hasKey(key.toLower): + headers.table[key.toLower] = @[value] + else: + headers.table[key.toLower].add(value) + +iterator pairs*(headers: HttpHeaders): tuple[key, value: string] = + ## Yields each key, value pair. + for k, v in headers.table: + for value in v: + yield (k, value) + +proc contains*(values: HttpHeaderValues, value: string): bool = + ## Determines if ``value`` is one of the values inside ``values``. Comparison + ## is performed without case sensitivity. + for val in seq[string](values): + if val.toLower == value.toLower: return true + +proc hasKey*(headers: HttpHeaders, key: string): bool = + return headers.table.hasKey(key.toLower()) + +proc getOrDefault*(headers: HttpHeaders, key: string, + default = @[""].HttpHeaderValues): HttpHeaderValues = + ## Returns the values associated with the given ``key``. If there are no + ## values associated with the key, then ``default`` is returned. + if headers.hasKey(key): + return headers[key] + else: + return default + +proc len*(headers: HttpHeaders): int = return headers.table.len + +proc parseList(line: string, list: var seq[string], start: int): int = + var i = 0 + var current = "" + while line[start + i] notin {'\c', '\l', '\0'}: + i += line.skipWhitespace(start + i) + i += line.parseUntil(current, {'\c', '\l', ','}, start + i) + list.add(current) + if line[start + i] == ',': + i.inc # Skip , + current.setLen(0) + +proc parseHeader*(line: string): tuple[key: string, value: seq[string]] = + ## Parses a single raw header HTTP line into key value pairs. + ## + ## Used by ``asynchttpserver`` and ``httpclient`` internally and should not + ## be used by you. + result.value = @[] + var i = 0 + i = line.parseUntil(result.key, ':') + inc(i) # skip : + if i < len(line): + i += parseList(line, result.value, i) + else: + result.value = @[] + +proc `==`*(protocol: tuple[orig: string, major, minor: int], + ver: HttpVersion): bool = + let major = + case ver + of HttpVer11, HttpVer10: 1 + let minor = + case ver + of HttpVer11: 1 + of HttpVer10: 0 + result = protocol.major == major and protocol.minor == minor + +when isMainModule: + var test = newHttpHeaders() + test["Connection"] = @["Upgrade", "Close"] + doAssert test["Connection", 0] == "Upgrade" + doAssert test["Connection", 1] == "Close" + test.add("Connection", "Test") + doAssert test["Connection", 2] == "Test" + doAssert "upgrade" in test["Connection"] diff --git a/lib/pure/ioselectors.nim b/lib/pure/ioselectors.nim new file mode 100644 index 000000000..a5d5d2c01 --- /dev/null +++ b/lib/pure/ioselectors.nim @@ -0,0 +1,255 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2016 Eugene Kabanov +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module allows high-level and efficient I/O multiplexing. +## +## Supported OS primitives: ``epoll``, ``kqueue``, ``poll`` and +## Windows ``select``. +## +## To use threadsafe version of this module, it needs to be compiled +## with both ``-d:threadsafe`` and ``--threads:on`` options. +## +## Supported features: files, sockets, pipes, timers, processes, signals +## and user events. +## +## Fully supported OS: MacOSX, FreeBSD, OpenBSD, NetBSD, Linux. +## +## Partially supported OS: Windows (only sockets and user events), +## Solaris (files, sockets, handles and user events). +## +## TODO: ``/dev/poll``, ``event ports`` and filesystem events. + +import os + +const hasThreadSupport = compileOption("threads") and defined(threadsafe) + +const supportedPlatform = defined(macosx) or defined(freebsd) or + defined(netbsd) or defined(openbsd) or + defined(linux) + +const bsdPlatform = defined(macosx) or defined(freebsd) or + defined(netbsd) or defined(openbsd) + + +when defined(nimdoc): + type + Selector*[T] = ref object + ## An object which holds descriptors to be checked for read/write status + + Event* {.pure.} = enum + ## An enum which hold event types + Read, ## Descriptor is available for read + Write, ## Descriptor is available for write + Timer, ## Timer descriptor is completed + Signal, ## Signal is raised + Process, ## Process is finished + Vnode, ## Currently not supported + User, ## User event is raised + Error ## Error happens while waiting, for descriptor + + ReadyKey*[T] = object + ## An object which holds result for descriptor + fd* : int ## file/socket descriptor + events*: set[Event] ## set of events + data*: T ## application-defined data + + SelectEvent* = object + ## An object which holds user defined event + + proc newSelector*[T](): Selector[T] = + ## Creates a new selector + + proc close*[T](s: Selector[T]) = + ## Closes selector + + proc registerHandle*[T](s: Selector[T], fd: SocketHandle, events: set[Event], + data: T) = + ## Registers file/socket descriptor ``fd`` to selector ``s`` + ## with events set in ``events``. The ``data`` is application-defined + ## data, which to be passed when event happens. + + proc updateHandle*[T](s: Selector[T], fd: SocketHandle, events: set[Event]) = + ## Update file/socket descriptor ``fd``, registered in selector + ## ``s`` with new events set ``event``. + + proc registerTimer*[T](s: Selector[T], timeout: int, oneshot: bool, + data: T): int {.discardable.} = + ## Registers timer notification with ``timeout`` in milliseconds + ## to selector ``s``. + ## If ``oneshot`` is ``true`` timer will be notified only once. + ## Set ``oneshot`` to ``false`` if your want periodic notifications. + ## The ``data`` is application-defined data, which to be passed, when + ## time limit expired. + + proc registerSignal*[T](s: Selector[T], signal: int, + data: T): int {.discardable.} = + ## Registers Unix signal notification with ``signal`` to selector + ## ``s``. The ``data`` is application-defined data, which to be + ## passed, when signal raises. + ## + ## This function is not supported for ``Windows``. + + proc registerProcess*[T](s: Selector[T], pid: int, + data: T): int {.discardable.} = + ## Registers process id (pid) notification when process has + ## exited to selector ``s``. + ## The ``data`` is application-defined data, which to be passed, when + ## process with ``pid`` has exited. + + proc registerEvent*[T](s: Selector[T], ev: SelectEvent, data: T) = + ## Registers selector event ``ev`` to selector ``s``. + ## ``data`` application-defined data, which to be passed, when + ## ``ev`` happens. + + proc newSelectEvent*(): SelectEvent = + ## Creates new event ``SelectEvent``. + + proc setEvent*(ev: SelectEvent) = + ## Trigger event ``ev``. + + proc close*(ev: SelectEvent) = + ## Closes selector event ``ev``. + + proc unregister*[T](s: Selector[T], ev: SelectEvent) = + ## Unregisters event ``ev`` from selector ``s``. + + proc unregister*[T](s: Selector[T], fd: int|SocketHandle|cint) = + ## Unregisters file/socket descriptor ``fd`` from selector ``s``. + + proc flush*[T](s: Selector[T]) = + ## Flushes all changes was made to kernel pool/queue. + ## This function is usefull only for BSD and MacOS, because + ## kqueue supports bulk changes to be made. + ## On Linux/Windows and other Posix compatible operation systems, + ## ``flush`` is alias for `discard`. + + proc selectInto*[T](s: Selector[T], timeout: int, + results: var openarray[ReadyKey[T]]): int = + ## Process call waiting for events registered in selector ``s``. + ## The ``timeout`` argument specifies the minimum number of milliseconds + ## the function will be blocked, if no events are not ready. Specifying a + ## timeout of ``-1`` causes function to block indefinitely. + ## All available events will be stored in ``results`` array. + ## + ## Function returns number of triggered events. + + proc select*[T](s: Selector[T], timeout: int): seq[ReadyKey[T]] = + ## Process call waiting for events registered in selector ``s``. + ## The ``timeout`` argument specifies the minimum number of milliseconds + ## the function will be blocked, if no events are not ready. Specifying a + ## timeout of -1 causes function to block indefinitely. + ## + ## Function returns sequence of triggered events. + + template isEmpty*[T](s: Selector[T]): bool = + ## Returns ``true``, if there no registered events or descriptors + ## in selector. + + template withData*[T](s: Selector[T], fd: SocketHandle, value, + body: untyped) = + ## retrieves the application-data assigned with descriptor ``fd`` + ## to ``value``. This ``value`` can be modified in the scope of + ## the ``withData`` call. + ## + ## .. code-block:: nim + ## + ## s.withData(fd, value) do: + ## # block is executed only if ``fd`` registered in selector ``s`` + ## value.uid = 1000 + ## + + template withData*[T](s: Selector[T], fd: SocketHandle, value, + body1, body2: untyped) = + ## retrieves the application-data assigned with descriptor ``fd`` + ## to ``value``. This ``value`` can be modified in the scope of + ## the ``withData`` call. + ## + ## .. code-block:: nim + ## + ## s.withData(fd, value) do: + ## # block is executed only if ``fd`` registered in selector ``s``. + ## value.uid = 1000 + ## do: + ## # block is executed if ``fd`` not registered in selector ``s``. + ## raise + ## + +else: + when hasThreadSupport: + import locks + + type + SharedArray {.unchecked.}[T] = array[0..100, T] + + proc allocSharedArray[T](nsize: int): ptr SharedArray[T] = + result = cast[ptr SharedArray[T]](allocShared0(sizeof(T) * nsize)) + + proc deallocSharedArray[T](sa: ptr SharedArray[T]) = + deallocShared(cast[pointer](sa)) + type + Event* {.pure.} = enum + Read, Write, Timer, Signal, Process, Vnode, User, Error, Oneshot + + ReadyKey*[T] = object + fd* : int + events*: set[Event] + data*: T + + SelectorKey[T] = object + ident: int + events: set[Event] + param: int + key: ReadyKey[T] + + when not defined(windows): + import posix + proc setNonBlocking(fd: cint) {.inline.} = + var x = fcntl(fd, F_GETFL, 0) + if x == -1: + raiseOSError(osLastError()) + else: + var mode = x or O_NONBLOCK + if fcntl(fd, F_SETFL, mode) == -1: + raiseOSError(osLastError()) + + template setKey(s, pident, pkeyfd, pevents, pparam, pdata) = + var skey = addr(s.fds[pident]) + skey.ident = pident + skey.events = pevents + skey.param = pparam + skey.key.fd = pkeyfd + skey.key.data = pdata + + when supportedPlatform: + template blockSignals(newmask: var Sigset, oldmask: var Sigset) = + when hasThreadSupport: + if posix.pthread_sigmask(SIG_BLOCK, newmask, oldmask) == -1: + raiseOSError(osLastError()) + else: + if posix.sigprocmask(SIG_BLOCK, newmask, oldmask) == -1: + raiseOSError(osLastError()) + + template unblockSignals(newmask: var Sigset, oldmask: var Sigset) = + when hasThreadSupport: + if posix.pthread_sigmask(SIG_UNBLOCK, newmask, oldmask) == -1: + raiseOSError(osLastError()) + else: + if posix.sigprocmask(SIG_UNBLOCK, newmask, oldmask) == -1: + raiseOSError(osLastError()) + + when defined(linux): + include ioselects/ioselectors_epoll + elif bsdPlatform: + include ioselects/ioselectors_kqueue + elif defined(windows): + include ioselects/ioselectors_select + elif defined(solaris): + include ioselects/ioselectors_poll # need to replace it with event ports + else: + include ioselects/ioselectors_poll diff --git a/lib/pure/ioselects/ioselectors_epoll.nim b/lib/pure/ioselects/ioselectors_epoll.nim new file mode 100644 index 000000000..92b2cdc07 --- /dev/null +++ b/lib/pure/ioselects/ioselectors_epoll.nim @@ -0,0 +1,461 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2016 Eugene Kabanov +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# This module implements Linux epoll(). + +import posix, times + +# Maximum number of events that can be returned +const MAX_EPOLL_RESULT_EVENTS = 64 + +type + SignalFdInfo* {.importc: "struct signalfd_siginfo", + header: "<sys/signalfd.h>", pure, final.} = object + ssi_signo*: uint32 + ssi_errno*: int32 + ssi_code*: int32 + ssi_pid*: uint32 + ssi_uid*: uint32 + ssi_fd*: int32 + ssi_tid*: uint32 + ssi_band*: uint32 + ssi_overrun*: uint32 + ssi_trapno*: uint32 + ssi_status*: int32 + ssi_int*: int32 + ssi_ptr*: uint64 + ssi_utime*: uint64 + ssi_stime*: uint64 + ssi_addr*: uint64 + pad* {.importc: "__pad".}: array[0..47, uint8] + + eventFdData {.importc: "eventfd_t", + header: "<sys/eventfd.h>", pure, final.} = uint64 + epoll_data {.importc: "union epoll_data", header: "<sys/epoll.h>", + pure, final.} = object + u64 {.importc: "u64".}: uint64 + epoll_event {.importc: "struct epoll_event", + header: "<sys/epoll.h>", pure, final.} = object + events: uint32 # Epoll events + data: epoll_data # User data variable + +const + EPOLL_CTL_ADD = 1 # Add a file descriptor to the interface. + EPOLL_CTL_DEL = 2 # Remove a file descriptor from the interface. + EPOLL_CTL_MOD = 3 # Change file descriptor epoll_event structure. + EPOLLIN = 0x00000001 + EPOLLOUT = 0x00000004 + EPOLLERR = 0x00000008 + EPOLLHUP = 0x00000010 + EPOLLRDHUP = 0x00002000 + EPOLLONESHOT = 1 shl 30 + +proc epoll_create(size: cint): cint + {.importc: "epoll_create", header: "<sys/epoll.h>".} +proc epoll_ctl(epfd: cint; op: cint; fd: cint; event: ptr epoll_event): cint + {.importc: "epoll_ctl", header: "<sys/epoll.h>".} +proc epoll_wait(epfd: cint; events: ptr epoll_event; maxevents: cint; + timeout: cint): cint + {.importc: "epoll_wait", header: "<sys/epoll.h>".} +proc timerfd_create(clock_id: ClockId, flags: cint): cint + {.cdecl, importc: "timerfd_create", header: "<sys/timerfd.h>".} +proc timerfd_settime(ufd: cint, flags: cint, + utmr: var Itimerspec, otmr: var Itimerspec): cint + {.cdecl, importc: "timerfd_settime", header: "<sys/timerfd.h>".} +proc signalfd(fd: cint, mask: var Sigset, flags: cint): cint + {.cdecl, importc: "signalfd", header: "<sys/signalfd.h>".} +proc eventfd(count: cuint, flags: cint): cint + {.cdecl, importc: "eventfd", header: "<sys/eventfd.h>".} +proc ulimit(cmd: cint): clong + {.importc: "ulimit", header: "<ulimit.h>", varargs.} + +when hasThreadSupport: + type + SelectorImpl[T] = object + epollFD : cint + maxFD : int + fds: ptr SharedArray[SelectorKey[T]] + count: int + Selector*[T] = ptr SelectorImpl[T] +else: + type + SelectorImpl[T] = object + epollFD : cint + maxFD : int + fds: seq[SelectorKey[T]] + count: int + Selector*[T] = ref SelectorImpl[T] +type + SelectEventImpl = object + efd: cint + SelectEvent* = ptr SelectEventImpl + +proc newSelector*[T](): Selector[T] = + var maxFD = int(ulimit(4, 0)) + doAssert(maxFD > 0) + + var epollFD = epoll_create(MAX_EPOLL_RESULT_EVENTS) + if epollFD < 0: + raiseOsError(osLastError()) + + when hasThreadSupport: + result = cast[Selector[T]](allocShared0(sizeof(SelectorImpl[T]))) + result.epollFD = epollFD + result.maxFD = maxFD + result.fds = allocSharedArray[SelectorKey[T]](maxFD) + else: + result = Selector[T]() + result.epollFD = epollFD + result.maxFD = maxFD + result.fds = newSeq[SelectorKey[T]](maxFD) + +proc close*[T](s: Selector[T]) = + if posix.close(s.epollFD) != 0: + raiseOSError(osLastError()) + when hasThreadSupport: + deallocSharedArray(s.fds) + deallocShared(cast[pointer](s)) + +proc newSelectEvent*(): SelectEvent = + let fdci = eventfd(0, 0) + if fdci == -1: + raiseOSError(osLastError()) + setNonBlocking(fdci) + result = cast[SelectEvent](allocShared0(sizeof(SelectEventImpl))) + result.efd = fdci + +proc setEvent*(ev: SelectEvent) = + var data : uint64 = 1 + if posix.write(ev.efd, addr data, sizeof(uint64)) == -1: + raiseOSError(osLastError()) + +proc close*(ev: SelectEvent) = + discard posix.close(ev.efd) + deallocShared(cast[pointer](ev)) + +template checkFd(s, f) = + if f >= s.maxFD: + raise newException(ValueError, "Maximum file descriptors exceeded") + +proc registerHandle*[T](s: Selector[T], fd: SocketHandle, + events: set[Event], data: T) = + let fdi = int(fd) + s.checkFd(fdi) + doAssert(s.fds[fdi].ident == 0) + s.setKey(fdi, fdi, events, 0, data) + if events != {}: + var epv = epoll_event(events: EPOLLRDHUP) + epv.data.u64 = fdi.uint + if Event.Read in events: epv.events = epv.events or EPOLLIN + if Event.Write in events: epv.events = epv.events or EPOLLOUT + if epoll_ctl(s.epollFD, EPOLL_CTL_ADD, fdi.cint, addr epv) == -1: + raiseOSError(osLastError()) + inc(s.count) + +proc updateHandle*[T](s: Selector[T], fd: SocketHandle, events: set[Event]) = + let maskEvents = {Event.Timer, Event.Signal, Event.Process, Event.Vnode, + Event.User, Event.Oneshot, Event.Error} + let fdi = int(fd) + s.checkFd(fdi) + var pkey = addr(s.fds[fdi]) + doAssert(pkey.ident != 0) + doAssert(pkey.events * maskEvents == {}) + if pkey.events != events: + var epv = epoll_event(events: EPOLLRDHUP) + epv.data.u64 = fdi.uint + + if Event.Read in events: epv.events = epv.events or EPOLLIN + if Event.Write in events: epv.events = epv.events or EPOLLOUT + + if pkey.events == {}: + if epoll_ctl(s.epollFD, EPOLL_CTL_ADD, fdi.cint, addr epv) == -1: + raiseOSError(osLastError()) + inc(s.count) + else: + if events != {}: + if epoll_ctl(s.epollFD, EPOLL_CTL_MOD, fdi.cint, addr epv) == -1: + raiseOSError(osLastError()) + else: + if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) == -1: + raiseOSError(osLastError()) + dec(s.count) + pkey.events = events + +proc unregister*[T](s: Selector[T], fd: int|SocketHandle) = + let fdi = int(fd) + s.checkFd(fdi) + var pkey = addr(s.fds[fdi]) + doAssert(pkey.ident != 0) + + if pkey.events != {}: + if pkey.events * {Event.Read, Event.Write} != {}: + var epv = epoll_event() + if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) == -1: + raiseOSError(osLastError()) + dec(s.count) + elif Event.Timer in pkey.events: + var epv = epoll_event() + if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) == -1: + raiseOSError(osLastError()) + discard posix.close(fdi.cint) + dec(s.count) + elif Event.Signal in pkey.events: + var epv = epoll_event() + if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) == -1: + raiseOSError(osLastError()) + var nmask, omask: Sigset + discard sigemptyset(nmask) + discard sigemptyset(omask) + discard sigaddset(nmask, cint(s.fds[fdi].param)) + unblockSignals(nmask, omask) + discard posix.close(fdi.cint) + dec(s.count) + elif Event.Process in pkey.events: + var epv = epoll_event() + if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) == -1: + raiseOSError(osLastError()) + var nmask, omask: Sigset + discard sigemptyset(nmask) + discard sigemptyset(omask) + discard sigaddset(nmask, SIGCHLD) + unblockSignals(nmask, omask) + discard posix.close(fdi.cint) + dec(s.count) + pkey.ident = 0 + pkey.events = {} + +proc unregister*[T](s: Selector[T], ev: SelectEvent) = + let fdi = int(ev.efd) + s.checkFd(fdi) + var pkey = addr(s.fds[fdi]) + doAssert(pkey.ident != 0) + doAssert(Event.User in pkey.events) + pkey.ident = 0 + pkey.events = {} + var epv = epoll_event() + if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) == -1: + raiseOSError(osLastError()) + dec(s.count) + +proc registerTimer*[T](s: Selector[T], timeout: int, oneshot: bool, + data: T): int {.discardable.} = + var + new_ts: Itimerspec + old_ts: Itimerspec + let fdi = timerfd_create(CLOCK_MONOTONIC, 0).int + if fdi == -1: + raiseOSError(osLastError()) + setNonBlocking(fdi.cint) + + s.checkFd(fdi) + doAssert(s.fds[fdi].ident == 0) + + var events = {Event.Timer} + var epv = epoll_event(events: EPOLLIN or EPOLLRDHUP) + epv.data.u64 = fdi.uint + if oneshot: + new_ts.it_interval.tv_sec = 0.Time + new_ts.it_interval.tv_nsec = 0 + new_ts.it_value.tv_sec = (timeout div 1_000).Time + new_ts.it_value.tv_nsec = (timeout %% 1_000) * 1_000_000 + incl(events, Event.Oneshot) + epv.events = epv.events or EPOLLONESHOT + else: + new_ts.it_interval.tv_sec = (timeout div 1000).Time + new_ts.it_interval.tv_nsec = (timeout %% 1_000) * 1_000_000 + new_ts.it_value.tv_sec = new_ts.it_interval.tv_sec + new_ts.it_value.tv_nsec = new_ts.it_interval.tv_nsec + + if timerfd_settime(fdi.cint, cint(0), new_ts, old_ts) == -1: + raiseOSError(osLastError()) + if epoll_ctl(s.epollFD, EPOLL_CTL_ADD, fdi.cint, addr epv) == -1: + raiseOSError(osLastError()) + s.setKey(fdi, fdi, events, 0, data) + inc(s.count) + result = fdi + +proc registerSignal*[T](s: Selector[T], signal: int, + data: T): int {.discardable.} = + var + nmask: Sigset + omask: Sigset + + discard sigemptyset(nmask) + discard sigemptyset(omask) + discard sigaddset(nmask, cint(signal)) + blockSignals(nmask, omask) + + let fdi = signalfd(-1, nmask, 0).int + if fdi == -1: + raiseOSError(osLastError()) + setNonBlocking(fdi.cint) + + s.checkFd(fdi) + doAssert(s.fds[fdi].ident == 0) + + var epv = epoll_event(events: EPOLLIN or EPOLLRDHUP) + epv.data.u64 = fdi.uint + if epoll_ctl(s.epollFD, EPOLL_CTL_ADD, fdi.cint, addr epv) == -1: + raiseOSError(osLastError()) + s.setKey(fdi, signal, {Event.Signal}, signal, data) + inc(s.count) + result = fdi + +proc registerProcess*[T](s: Selector, pid: int, + data: T): int {.discardable.} = + var + nmask: Sigset + omask: Sigset + + discard sigemptyset(nmask) + discard sigemptyset(omask) + discard sigaddset(nmask, posix.SIGCHLD) + blockSignals(nmask, omask) + + let fdi = signalfd(-1, nmask, 0).int + if fdi == -1: + raiseOSError(osLastError()) + setNonBlocking(fdi.cint) + + s.checkFd(fdi) + doAssert(s.fds[fdi].ident == 0) + + var epv = epoll_event(events: EPOLLIN or EPOLLRDHUP) + epv.data.u64 = fdi.uint + epv.events = EPOLLIN or EPOLLRDHUP + if epoll_ctl(s.epollFD, EPOLL_CTL_ADD, fdi.cint, addr epv) == -1: + raiseOSError(osLastError()) + s.setKey(fdi, pid, {Event.Process, Event.Oneshot}, pid, data) + inc(s.count) + result = fdi + +proc registerEvent*[T](s: Selector[T], ev: SelectEvent, data: T) = + let fdi = int(ev.efd) + doAssert(s.fds[fdi].ident == 0) + s.setKey(fdi, fdi, {Event.User}, 0, data) + var epv = epoll_event(events: EPOLLIN or EPOLLRDHUP) + epv.data.u64 = ev.efd.uint + if epoll_ctl(s.epollFD, EPOLL_CTL_ADD, ev.efd, addr epv) == -1: + raiseOSError(osLastError()) + inc(s.count) + +proc flush*[T](s: Selector[T]) = + discard + +proc selectInto*[T](s: Selector[T], timeout: int, + results: var openarray[ReadyKey[T]]): int = + var + resTable: array[MAX_EPOLL_RESULT_EVENTS, epoll_event] + maxres = MAX_EPOLL_RESULT_EVENTS + events: set[Event] = {} + i, k: int + + if maxres > len(results): + maxres = len(results) + + let count = epoll_wait(s.epollFD, addr(resTable[0]), maxres.cint, + timeout.cint) + if count < 0: + result = 0 + let err = osLastError() + if cint(err) != EINTR: + raiseOSError(err) + elif count == 0: + result = 0 + else: + i = 0 + k = 0 + while i < count: + let fdi = int(resTable[i].data.u64) + let pevents = resTable[i].events + var skey = addr(s.fds[fdi]) + doAssert(skey.ident != 0) + events = {} + + if (pevents and EPOLLERR) != 0 or (pevents and EPOLLHUP) != 0: + events.incl(Event.Error) + if (pevents and EPOLLOUT) != 0: + events.incl(Event.Write) + if (pevents and EPOLLIN) != 0: + if Event.Read in skey.events: + events.incl(Event.Read) + elif Event.Timer in skey.events: + var data: uint64 = 0 + if posix.read(fdi.cint, addr data, sizeof(uint64)) != sizeof(uint64): + raiseOSError(osLastError()) + events = {Event.Timer} + elif Event.Signal in skey.events: + var data = SignalFdInfo() + if posix.read(fdi.cint, addr data, + sizeof(SignalFdInfo)) != sizeof(SignalFdInfo): + raiseOsError(osLastError()) + events = {Event.Signal} + elif Event.Process in skey.events: + var data = SignalFdInfo() + if posix.read(fdi.cint, addr data, + sizeof(SignalFdInfo)) != sizeof(SignalFdInfo): + raiseOsError(osLastError()) + if cast[int](data.ssi_pid) == skey.param: + events = {Event.Process} + else: + inc(i) + continue + elif Event.User in skey.events: + var data: uint = 0 + if posix.read(fdi.cint, addr data, sizeof(uint)) != sizeof(uint): + let err = osLastError() + if err == OSErrorCode(EAGAIN): + inc(i) + continue + else: + raiseOSError(err) + events = {Event.User} + + skey.key.events = events + results[k] = skey.key + inc(k) + + if Event.Oneshot in skey.events: + var epv = epoll_event() + if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) == -1: + raiseOSError(osLastError()) + discard posix.close(fdi.cint) + skey.ident = 0 + skey.events = {} + dec(s.count) + inc(i) + result = k + +proc select*[T](s: Selector[T], timeout: int): seq[ReadyKey[T]] = + result = newSeq[ReadyKey[T]](MAX_EPOLL_RESULT_EVENTS) + let count = selectInto(s, timeout, result) + result.setLen(count) + +template isEmpty*[T](s: Selector[T]): bool = + (s.count == 0) + +template withData*[T](s: Selector[T], fd: SocketHandle, value, + body: untyped) = + mixin checkFd + let fdi = int(fd) + s.checkFd(fdi) + if s.fds[fdi].ident != 0: + var value = addr(s.fds[fdi].key.data) + body + +template withData*[T](s: Selector[T], fd: SocketHandle, value, body1, + body2: untyped) = + mixin checkFd + let fdi = int(fd) + s.checkFd(fdi) + if s.fds[fdi].ident != 0: + var value = addr(s.fds[fdi].key.data) + body1 + else: + body2 diff --git a/lib/pure/ioselects/ioselectors_kqueue.nim b/lib/pure/ioselects/ioselectors_kqueue.nim new file mode 100644 index 000000000..3e86f19aa --- /dev/null +++ b/lib/pure/ioselects/ioselectors_kqueue.nim @@ -0,0 +1,443 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2016 Eugene Kabanov +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# This module implements BSD kqueue(). + +import posix, times, kqueue + +const + # Maximum number of cached changes. + MAX_KQUEUE_CHANGE_EVENTS = 64 + # Maximum number of events that can be returned. + MAX_KQUEUE_RESULT_EVENTS = 64 + # SIG_IGN and SIG_DFL declared in posix.nim as variables, but we need them + # to be constants and GC-safe. + SIG_DFL = cast[proc(x: cint) {.noconv,gcsafe.}](0) + SIG_IGN = cast[proc(x: cint) {.noconv,gcsafe.}](1) + +when defined(macosx) or defined(freebsd): + when defined(macosx): + const MAX_DESCRIPTORS_ID = 29 # KERN_MAXFILESPERPROC (MacOS) + else: + const MAX_DESCRIPTORS_ID = 27 # KERN_MAXFILESPERPROC (FreeBSD) + proc sysctl(name: ptr cint, namelen: cuint, oldp: pointer, oldplen: ptr int, + newp: pointer, newplen: int): cint + {.importc: "sysctl",header: """#include <sys/types.h> + #include <sys/sysctl.h>"""} +elif defined(netbsd) or defined(openbsd): + # OpenBSD and NetBSD don't have KERN_MAXFILESPERPROC, so we are using + # KERN_MAXFILES, because KERN_MAXFILES is always bigger, + # than KERN_MAXFILESPERPROC. + const MAX_DESCRIPTORS_ID = 7 # KERN_MAXFILES + proc sysctl(name: ptr cint, namelen: cuint, oldp: pointer, oldplen: ptr int, + newp: pointer, newplen: int): cint + {.importc: "sysctl",header: """#include <sys/param.h> + #include <sys/sysctl.h>"""} + +when hasThreadSupport: + type + SelectorImpl[T] = object + kqFD : cint + maxFD : int + changesTable: array[MAX_KQUEUE_CHANGE_EVENTS, KEvent] + changesCount: int + fds: ptr SharedArray[SelectorKey[T]] + count: int + changesLock: Lock + Selector*[T] = ptr SelectorImpl[T] +else: + type + SelectorImpl[T] = object + kqFD : cint + maxFD : int + changesTable: array[MAX_KQUEUE_CHANGE_EVENTS, KEvent] + changesCount: int + fds: seq[SelectorKey[T]] + count: int + Selector*[T] = ref SelectorImpl[T] + +type + SelectEventImpl = object + rfd: cint + wfd: cint +# SelectEvent is declared as `ptr` to be placed in `shared memory`, +# so you can share one SelectEvent handle between threads. +type SelectEvent* = ptr SelectEventImpl + +proc newSelector*[T](): Selector[T] = + var maxFD = 0.cint + var size = sizeof(cint) + var namearr = [1.cint, MAX_DESCRIPTORS_ID.cint] + # Obtain maximum number of file descriptors for process + if sysctl(addr(namearr[0]), 2, cast[pointer](addr maxFD), addr size, + nil, 0) != 0: + raiseOsError(osLastError()) + + var kqFD = kqueue() + if kqFD < 0: + raiseOsError(osLastError()) + + when hasThreadSupport: + result = cast[Selector[T]](allocShared0(sizeof(SelectorImpl[T]))) + result.kqFD = kqFD + result.maxFD = maxFD.int + result.fds = allocSharedArray[SelectorKey[T]](maxFD) + initLock(result.changesLock) + else: + result = Selector[T]() + result.kqFD = kqFD + result.maxFD = maxFD.int + result.fds = newSeq[SelectorKey[T]](maxFD) + +proc close*[T](s: Selector[T]) = + if posix.close(s.kqFD) != 0: + raiseOSError(osLastError()) + when hasThreadSupport: + deinitLock(s.changesLock) + deallocSharedArray(s.fds) + deallocShared(cast[pointer](s)) + +proc newSelectEvent*(): SelectEvent = + var fds: array[2, cint] + if posix.pipe(fds) == -1: + raiseOSError(osLastError()) + setNonBlocking(fds[0]) + setNonBlocking(fds[1]) + result = cast[SelectEvent](allocShared0(sizeof(SelectEventImpl))) + result.rfd = fds[0] + result.wfd = fds[1] + +proc setEvent*(ev: SelectEvent) = + var data: uint64 = 1 + if posix.write(ev.wfd, addr data, sizeof(uint64)) != sizeof(uint64): + raiseOSError(osLastError()) + +proc close*(ev: SelectEvent) = + discard posix.close(cint(ev.rfd)) + discard posix.close(cint(ev.wfd)) + deallocShared(cast[pointer](ev)) + +template checkFd(s, f) = + if f >= s.maxFD: + raise newException(ValueError, "Maximum file descriptors exceeded") + +when hasThreadSupport: + template withChangeLock[T](s: Selector[T], body: untyped) = + acquire(s.changesLock) + {.locks: [s.changesLock].}: + try: + body + finally: + release(s.changesLock) +else: + template withChangeLock(s, body: untyped) = + body + +template modifyKQueue[T](s: Selector[T], nident: uint, nfilter: cshort, + nflags: cushort, nfflags: cuint, ndata: int, + nudata: pointer) = + mixin withChangeLock + s.withChangeLock(): + s.changesTable[s.changesCount] = KEvent(ident: nident, + filter: nfilter, flags: nflags, + fflags: nfflags, data: ndata, + udata: nudata) + inc(s.changesCount) + if s.changesCount == MAX_KQUEUE_CHANGE_EVENTS: + if kevent(s.kqFD, addr(s.changesTable[0]), cint(s.changesCount), + nil, 0, nil) == -1: + raiseOSError(osLastError()) + s.changesCount = 0 + +proc registerHandle*[T](s: Selector[T], fd: SocketHandle, + events: set[Event], data: T) = + let fdi = int(fd) + s.checkFd(fdi) + doAssert(s.fds[fdi].ident == 0) + s.setKey(fdi, fdi, events, 0, data) + if events != {}: + if Event.Read in events: + modifyKQueue(s, fdi.uint, EVFILT_READ, EV_ADD, 0, 0, nil) + inc(s.count) + if Event.Write in events: + modifyKQueue(s, fdi.uint, EVFILT_WRITE, EV_ADD, 0, 0, nil) + inc(s.count) + +proc updateHandle*[T](s: Selector[T], fd: SocketHandle, + events: set[Event]) = + let maskEvents = {Event.Timer, Event.Signal, Event.Process, Event.Vnode, + Event.User, Event.Oneshot, Event.Error} + let fdi = int(fd) + s.checkFd(fdi) + var pkey = addr(s.fds[fdi]) + doAssert(pkey.ident != 0) + doAssert(pkey.events * maskEvents == {}) + + if pkey.events != events: + if (Event.Read in pkey.events) and (Event.Read notin events): + modifyKQueue(s, fdi.uint, EVFILT_READ, EV_DELETE, 0, 0, nil) + dec(s.count) + if (Event.Write in pkey.events) and (Event.Write notin events): + modifyKQueue(s, fdi.uint, EVFILT_WRITE, EV_DELETE, 0, 0, nil) + dec(s.count) + if (Event.Read notin pkey.events) and (Event.Read in events): + modifyKQueue(s, fdi.uint, EVFILT_READ, EV_ADD, 0, 0, nil) + inc(s.count) + if (Event.Write notin pkey.events) and (Event.Write in events): + modifyKQueue(s, fdi.uint, EVFILT_WRITE, EV_ADD, 0, 0, nil) + inc(s.count) + pkey.events = events + +proc registerTimer*[T](s: Selector[T], timeout: int, oneshot: bool, + data: T): int {.discardable.} = + var fdi = posix.socket(posix.AF_INET, posix.SOCK_STREAM, + posix.IPPROTO_TCP).int + if fdi == -1: + raiseOsError(osLastError()) + + s.checkFd(fdi) + doAssert(s.fds[fdi].ident == 0) + + let events = if oneshot: {Event.Timer, Event.Oneshot} else: {Event.Timer} + let flags: cushort = if oneshot: EV_ONESHOT or EV_ADD else: EV_ADD + + s.setKey(fdi, fdi, events, 0, data) + # EVFILT_TIMER on Open/Net(BSD) has granularity of only milliseconds, + # but MacOS and FreeBSD allow use `0` as `fflags` to use milliseconds + # too + modifyKQueue(s, fdi.uint, EVFILT_TIMER, flags, 0, cint(timeout), nil) + inc(s.count) + result = fdi + +proc registerSignal*[T](s: Selector[T], signal: int, + data: T): int {.discardable.} = + var fdi = posix.socket(posix.AF_INET, posix.SOCK_STREAM, + posix.IPPROTO_TCP).int + if fdi == -1: + raiseOsError(osLastError()) + + s.checkFd(fdi) + doAssert(s.fds[fdi].ident == 0) + + s.setKey(fdi, signal, {Event.Signal}, signal, data) + var nmask, omask: Sigset + discard sigemptyset(nmask) + discard sigemptyset(omask) + discard sigaddset(nmask, cint(signal)) + blockSignals(nmask, omask) + # to be compatible with linux semantic we need to "eat" signals + posix.signal(cint(signal), SIG_IGN) + modifyKQueue(s, signal.uint, EVFILT_SIGNAL, EV_ADD, 0, 0, + cast[pointer](fdi)) + inc(s.count) + result = fdi + +proc registerProcess*[T](s: Selector[T], pid: int, + data: T): int {.discardable.} = + var fdi = posix.socket(posix.AF_INET, posix.SOCK_STREAM, + posix.IPPROTO_TCP).int + if fdi == -1: + raiseOsError(osLastError()) + + s.checkFd(fdi) + doAssert(s.fds[fdi].ident == 0) + + var kflags: cushort = EV_ONESHOT or EV_ADD + setKey(s, fdi, pid, {Event.Process, Event.Oneshot}, pid, data) + modifyKQueue(s, pid.uint, EVFILT_PROC, kflags, NOTE_EXIT, 0, + cast[pointer](fdi)) + inc(s.count) + result = fdi + +proc registerEvent*[T](s: Selector[T], ev: SelectEvent, data: T) = + let fdi = ev.rfd.int + doAssert(s.fds[fdi].ident == 0) + setKey(s, fdi, fdi, {Event.User}, 0, data) + modifyKQueue(s, fdi.uint, EVFILT_READ, EV_ADD, 0, 0, nil) + inc(s.count) + +proc unregister*[T](s: Selector[T], fd: int|SocketHandle) = + let fdi = int(fd) + s.checkFd(fdi) + var pkey = addr(s.fds[fdi]) + doAssert(pkey.ident != 0) + + if pkey.events != {}: + if pkey.events * {Event.Read, Event.Write} != {}: + if Event.Read in pkey.events: + modifyKQueue(s, fdi.uint, EVFILT_READ, EV_DELETE, 0, 0, nil) + dec(s.count) + if Event.Write in pkey.events: + modifyKQueue(s, fdi.uint, EVFILT_WRITE, EV_DELETE, 0, 0, nil) + dec(s.count) + elif Event.Timer in pkey.events: + discard posix.close(cint(pkey.key.fd)) + modifyKQueue(s, fdi.uint, EVFILT_TIMER, EV_DELETE, 0, 0, nil) + dec(s.count) + elif Event.Signal in pkey.events: + var nmask, omask: Sigset + var signal = cint(pkey.param) + discard sigemptyset(nmask) + discard sigemptyset(omask) + discard sigaddset(nmask, signal) + unblockSignals(nmask, omask) + posix.signal(signal, SIG_DFL) + discard posix.close(cint(pkey.key.fd)) + modifyKQueue(s, fdi.uint, EVFILT_SIGNAL, EV_DELETE, 0, 0, nil) + dec(s.count) + elif Event.Process in pkey.events: + discard posix.close(cint(pkey.key.fd)) + modifyKQueue(s, fdi.uint, EVFILT_PROC, EV_DELETE, 0, 0, nil) + dec(s.count) + elif Event.User in pkey.events: + modifyKQueue(s, fdi.uint, EVFILT_READ, EV_DELETE, 0, 0, nil) + dec(s.count) + pkey.ident = 0 + pkey.events = {} + +proc unregister*[T](s: Selector[T], ev: SelectEvent) = + let fdi = int(ev.rfd) + s.checkFd(fdi) + var pkey = addr(s.fds[fdi]) + doAssert(pkey.ident != 0) + doAssert(Event.User in pkey.events) + pkey.ident = 0 + pkey.events = {} + modifyKQueue(s, fdi.uint, EVFILT_READ, EV_DELETE, 0, 0, nil) + dec(s.count) + +proc flush*[T](s: Selector[T]) = + s.withChangeLock(): + var tv = Timespec() + if kevent(s.kqFD, addr(s.changesTable[0]), cint(s.changesCount), + nil, 0, addr tv) == -1: + raiseOSError(osLastError()) + s.changesCount = 0 + +proc selectInto*[T](s: Selector[T], timeout: int, + results: var openarray[ReadyKey[T]]): int = + var + tv: Timespec + resTable: array[MAX_KQUEUE_RESULT_EVENTS, KEvent] + ptv = addr tv + maxres = MAX_KQUEUE_RESULT_EVENTS + + if timeout != -1: + if timeout >= 1000: + tv.tv_sec = (timeout div 1_000).Time + tv.tv_nsec = (timeout %% 1_000) * 1_000_000 + else: + tv.tv_sec = 0.Time + tv.tv_nsec = timeout * 1_000_000 + else: + ptv = nil + + if maxres > len(results): + maxres = len(results) + + var count = 0 + s.withChangeLock(): + count = kevent(s.kqFD, addr(s.changesTable[0]), cint(s.changesCount), + addr(resTable[0]), cint(maxres), ptv) + s.changesCount = 0 + + if count < 0: + result = 0 + let err = osLastError() + if cint(err) != EINTR: + raiseOSError(err) + elif count == 0: + result = 0 + else: + var i = 0 + var k = 0 + var pkey: ptr SelectorKey[T] + while i < count: + let kevent = addr(resTable[i]) + if (kevent.flags and EV_ERROR) == 0: + case kevent.filter: + of EVFILT_READ: + pkey = addr(s.fds[kevent.ident.int]) + pkey.key.events = {Event.Read} + if Event.User in pkey.events: + var data: uint64 = 0 + if posix.read(kevent.ident.cint, addr data, + sizeof(uint64)) != sizeof(uint64): + let err = osLastError() + if err == OSErrorCode(EAGAIN): + # someone already consumed event data + inc(i) + continue + else: + raiseOSError(osLastError()) + pkey.key.events = {Event.User} + of EVFILT_WRITE: + pkey = addr(s.fds[kevent.ident.int]) + pkey.key.events = {Event.Write} + of EVFILT_TIMER: + pkey = addr(s.fds[kevent.ident.int]) + if Event.Oneshot in pkey.events: + if posix.close(cint(pkey.ident)) == -1: + raiseOSError(osLastError()) + pkey.ident = 0 + pkey.events = {} + dec(s.count) + pkey.key.events = {Event.Timer} + of EVFILT_VNODE: + pkey = addr(s.fds[kevent.ident.int]) + pkey.key.events = {Event.Vnode} + of EVFILT_SIGNAL: + pkey = addr(s.fds[cast[int](kevent.udata)]) + pkey.key.events = {Event.Signal} + of EVFILT_PROC: + pkey = addr(s.fds[cast[int](kevent.udata)]) + if posix.close(cint(pkey.ident)) == -1: + raiseOSError(osLastError()) + pkey.ident = 0 + pkey.events = {} + dec(s.count) + pkey.key.events = {Event.Process} + else: + raise newException(ValueError, "Unsupported kqueue filter in queue") + + if (kevent.flags and EV_EOF) != 0: + pkey.key.events.incl(Event.Error) + + results[k] = pkey.key + inc(k) + inc(i) + result = k + +proc select*[T](s: Selector[T], timeout: int): seq[ReadyKey[T]] = + result = newSeq[ReadyKey[T]](MAX_KQUEUE_RESULT_EVENTS) + let count = selectInto(s, timeout, result) + result.setLen(count) + +template isEmpty*[T](s: Selector[T]): bool = + (s.count == 0) + +template withData*[T](s: Selector[T], fd: SocketHandle, value, + body: untyped) = + mixin checkFd + let fdi = int(fd) + s.checkFd(fdi) + if s.fds[fdi].ident != 0: + var value = addr(s.fds[fdi].key.data) + body + +template withData*[T](s: Selector[T], fd: SocketHandle, value, body1, + body2: untyped) = + mixin checkFd + let fdi = int(fd) + s.checkFd(fdi) + if s.fds[fdi].ident != 0: + var value = addr(s.fds[fdi].key.data) + body1 + else: + body2 diff --git a/lib/pure/ioselects/ioselectors_poll.nim b/lib/pure/ioselects/ioselectors_poll.nim new file mode 100644 index 000000000..d2a0a1273 --- /dev/null +++ b/lib/pure/ioselects/ioselectors_poll.nim @@ -0,0 +1,295 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2016 Eugene Kabanov +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# This module implements Posix poll(). + +import posix, times + +# Maximum number of events that can be returned +const MAX_POLL_RESULT_EVENTS = 64 + +when hasThreadSupport: + type + SelectorImpl[T] = object + maxFD : int + pollcnt: int + fds: ptr SharedArray[SelectorKey[T]] + pollfds: ptr SharedArray[TPollFd] + count: int + lock: Lock + Selector*[T] = ptr SelectorImpl[T] +else: + type + SelectorImpl[T] = object + maxFD : int + pollcnt: int + fds: seq[SelectorKey[T]] + pollfds: seq[TPollFd] + count: int + Selector*[T] = ref SelectorImpl[T] + +type + SelectEventImpl = object + rfd: cint + wfd: cint + SelectEvent* = ptr SelectEventImpl + +var RLIMIT_NOFILE {.importc: "RLIMIT_NOFILE", + header: "<sys/resource.h>".}: cint +type + rlimit {.importc: "struct rlimit", + header: "<sys/resource.h>", pure, final.} = object + rlim_cur: int + rlim_max: int +proc getrlimit(resource: cint, rlp: var rlimit): cint + {.importc: "getrlimit",header: "<sys/resource.h>".} + +when hasThreadSupport: + template withPollLock[T](s: Selector[T], body: untyped) = + acquire(s.lock) + {.locks: [s.lock].}: + try: + body + finally: + release(s.lock) +else: + template withPollLock(s, body: untyped) = + body + +proc newSelector*[T](): Selector[T] = + var a = rlimit() + if getrlimit(RLIMIT_NOFILE, a) != 0: + raiseOsError(osLastError()) + var maxFD = int(a.rlim_max) + + when hasThreadSupport: + result = cast[Selector[T]](allocShared0(sizeof(SelectorImpl[T]))) + result.maxFD = maxFD + result.fds = allocSharedArray[SelectorKey[T]](maxFD) + result.pollfds = allocSharedArray[TPollFd](maxFD) + initLock(result.lock) + else: + result = Selector[T]() + result.maxFD = maxFD + result.fds = newSeq[SelectorKey[T]](maxFD) + result.pollfds = newSeq[TPollFd](maxFD) + +proc close*[T](s: Selector[T]) = + when hasThreadSupport: + deinitLock(s.lock) + deallocSharedArray(s.fds) + deallocSharedArray(s.pollfds) + deallocShared(cast[pointer](s)) + +template pollAdd[T](s: Selector[T], sock: cint, events: set[Event]) = + withPollLock(s): + var pollev: cshort = 0 + if Event.Read in events: pollev = pollev or POLLIN + if Event.Write in events: pollev = pollev or POLLOUT + s.pollfds[s.pollcnt].fd = cint(sock) + s.pollfds[s.pollcnt].events = pollev + inc(s.count) + inc(s.pollcnt) + +template pollUpdate[T](s: Selector[T], sock: cint, events: set[Event]) = + withPollLock(s): + var i = 0 + var pollev: cshort = 0 + if Event.Read in events: pollev = pollev or POLLIN + if Event.Write in events: pollev = pollev or POLLOUT + + while i < s.pollcnt: + if s.pollfds[i].fd == sock: + s.pollfds[i].events = pollev + break + inc(i) + + if i == s.pollcnt: + raise newException(ValueError, "Descriptor is not registered in queue") + +template pollRemove[T](s: Selector[T], sock: cint) = + withPollLock(s): + var i = 0 + while i < s.pollcnt: + if s.pollfds[i].fd == sock: + if i == s.pollcnt - 1: + s.pollfds[i].fd = 0 + s.pollfds[i].events = 0 + s.pollfds[i].revents = 0 + else: + while i < (s.pollcnt - 1): + s.pollfds[i].fd = s.pollfds[i + 1].fd + s.pollfds[i].events = s.pollfds[i + 1].events + inc(i) + break + inc(i) + dec(s.pollcnt) + dec(s.count) + +template checkFd(s, f) = + if f >= s.maxFD: + raise newException(ValueError, "Maximum file descriptors exceeded") + +proc registerHandle*[T](s: Selector[T], fd: SocketHandle, + events: set[Event], data: T) = + var fdi = int(fd) + s.checkFd(fdi) + doAssert(s.fds[fdi].ident == 0) + s.setKey(fdi, fdi, events, 0, data) + if events != {}: s.pollAdd(fdi.cint, events) + +proc updateHandle*[T](s: Selector[T], fd: SocketHandle, + events: set[Event]) = + let maskEvents = {Event.Timer, Event.Signal, Event.Process, Event.Vnode, + Event.User, Event.Oneshot, Event.Error} + let fdi = int(fd) + s.checkFd(fdi) + var pkey = addr(s.fds[fdi]) + doAssert(pkey.ident != 0) + doAssert(pkey.events * maskEvents == {}) + + if pkey.events != events: + if pkey.events == {}: + s.pollAdd(fd.cint, events) + else: + if events != {}: + s.pollUpdate(fd.cint, events) + else: + s.pollRemove(fd.cint) + pkey.events = events + +proc registerEvent*[T](s: Selector[T], ev: SelectEvent, data: T) = + var fdi = int(ev.rfd) + doAssert(s.fds[fdi].ident == 0) + var events = {Event.User} + setKey(s, fdi, fdi, events, 0, data) + events.incl(Event.Read) + s.pollAdd(fdi.cint, events) + +proc flush*[T](s: Selector[T]) = discard + +proc unregister*[T](s: Selector[T], fd: int|SocketHandle) = + let fdi = int(fd) + s.checkFd(fdi) + var pkey = addr(s.fds[fdi]) + doAssert(pkey.ident != 0) + pkey.ident = 0 + pkey.events = {} + s.pollRemove(fdi.cint) + +proc unregister*[T](s: Selector[T], ev: SelectEvent) = + let fdi = int(ev.rfd) + s.checkFd(fdi) + var pkey = addr(s.fds[fdi]) + doAssert(pkey.ident != 0) + doAssert(Event.User in pkey.events) + pkey.ident = 0 + pkey.events = {} + s.pollRemove(fdi.cint) + +proc newSelectEvent*(): SelectEvent = + var fds: array[2, cint] + if posix.pipe(fds) == -1: + raiseOSError(osLastError()) + setNonBlocking(fds[0]) + setNonBlocking(fds[1]) + result = cast[SelectEvent](allocShared0(sizeof(SelectEventImpl))) + result.rfd = fds[0] + result.wfd = fds[1] + +proc setEvent*(ev: SelectEvent) = + var data: uint64 = 1 + if posix.write(ev.wfd, addr data, sizeof(uint64)) != sizeof(uint64): + raiseOSError(osLastError()) + +proc close*(ev: SelectEvent) = + discard posix.close(cint(ev.rfd)) + discard posix.close(cint(ev.wfd)) + deallocShared(cast[pointer](ev)) + +proc selectInto*[T](s: Selector[T], timeout: int, + results: var openarray[ReadyKey[T]]): int = + var maxres = MAX_POLL_RESULT_EVENTS + if maxres > len(results): + maxres = len(results) + + s.withPollLock(): + let count = posix.poll(addr(s.pollfds[0]), Tnfds(s.pollcnt), timeout) + if count < 0: + result = 0 + let err = osLastError() + if err.cint == EINTR: + discard + else: + raiseOSError(osLastError()) + elif count == 0: + result = 0 + else: + var i = 0 + var k = 0 + var rindex = 0 + while (i < s.pollcnt) and (k < count) and (rindex < maxres): + let revents = s.pollfds[i].revents + if revents != 0: + let fd = s.pollfds[i].fd + var skey = addr(s.fds[fd]) + skey.key.events = {} + + if (revents and POLLIN) != 0: + skey.key.events.incl(Event.Read) + if Event.User in skey.events: + var data: uint64 = 0 + if posix.read(fd, addr data, sizeof(int)) != sizeof(int): + let err = osLastError() + if err != OSErrorCode(EAGAIN): + raiseOSError(osLastError()) + else: + # someone already consumed event data + inc(i) + continue + skey.key.events = {Event.User} + if (revents and POLLOUT) != 0: + skey.key.events.incl(Event.Write) + if (revents and POLLERR) != 0 or (revents and POLLHUP) != 0 or + (revents and POLLNVAL) != 0: + skey.key.events.incl(Event.Error) + results[rindex] = skey.key + s.pollfds[i].revents = 0 + inc(rindex) + inc(k) + inc(i) + result = k + +proc select*[T](s: Selector[T], timeout: int): seq[ReadyKey[T]] = + result = newSeq[ReadyKey[T]](MAX_POLL_RESULT_EVENTS) + let count = selectInto(s, timeout, result) + result.setLen(count) + +template isEmpty*[T](s: Selector[T]): bool = + (s.count == 0) + +template withData*[T](s: Selector[T], fd: SocketHandle, value, + body: untyped) = + mixin checkFd + let fdi = int(fd) + s.checkFd(fdi) + if s.fds[fdi].ident != 0: + var value = addr(s.fds[fdi].key.data) + body + +template withData*[T](s: Selector[T], fd: SocketHandle, value, body1, + body2: untyped) = + mixin checkFd + let fdi = int(fd) + s.checkFd(fdi) + if s.fds[fdi].ident != 0: + var value = addr(s.fds[fdi].key.data) + body1 + else: + body2 diff --git a/lib/pure/ioselects/ioselectors_select.nim b/lib/pure/ioselects/ioselectors_select.nim new file mode 100644 index 000000000..f8099f9a0 --- /dev/null +++ b/lib/pure/ioselects/ioselectors_select.nim @@ -0,0 +1,416 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2016 Eugene Kabanov +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# This module implements Posix and Windows select(). + +import times, nativesockets + +when defined(windows): + import winlean + when defined(gcc): + {.passL: "-lws2_32".} + elif defined(vcc): + {.passL: "ws2_32.lib".} + const platformHeaders = """#include <winsock2.h> + #include <windows.h>""" + const EAGAIN = WSAEWOULDBLOCK +else: + const platformHeaders = """#include <sys/select.h> + #include <sys/time.h> + #include <sys/types.h> + #include <unistd.h>""" +type + Fdset {.importc: "fd_set", header: platformHeaders, pure, final.} = object +var + FD_SETSIZE {.importc: "FD_SETSIZE", header: platformHeaders.}: cint + +proc IOFD_SET(fd: SocketHandle, fdset: ptr Fdset) + {.cdecl, importc: "FD_SET", header: platformHeaders, inline.} +proc IOFD_CLR(fd: SocketHandle, fdset: ptr Fdset) + {.cdecl, importc: "FD_CLR", header: platformHeaders, inline.} +proc IOFD_ZERO(fdset: ptr Fdset) + {.cdecl, importc: "FD_ZERO", header: platformHeaders, inline.} + +when defined(windows): + proc IOFD_ISSET(fd: SocketHandle, fdset: ptr Fdset): cint + {.stdcall, importc: "FD_ISSET", header: platformHeaders, inline.} + proc ioselect(nfds: cint, readFds, writeFds, exceptFds: ptr Fdset, + timeout: ptr Timeval): cint + {.stdcall, importc: "select", header: platformHeaders.} +else: + proc IOFD_ISSET(fd: SocketHandle, fdset: ptr Fdset): cint + {.cdecl, importc: "FD_ISSET", header: platformHeaders, inline.} + proc ioselect(nfds: cint, readFds, writeFds, exceptFds: ptr Fdset, + timeout: ptr Timeval): cint + {.cdecl, importc: "select", header: platformHeaders.} + +when hasThreadSupport: + type + SelectorImpl[T] = object + rSet: FdSet + wSet: FdSet + eSet: FdSet + maxFD: int + fds: ptr SharedArray[SelectorKey[T]] + count: int + lock: Lock + Selector*[T] = ptr SelectorImpl[T] +else: + type + SelectorImpl[T] = object + rSet: FdSet + wSet: FdSet + eSet: FdSet + maxFD: int + fds: seq[SelectorKey[T]] + count: int + Selector*[T] = ref SelectorImpl[T] + +type + SelectEventImpl = object + rsock: SocketHandle + wsock: SocketHandle + SelectEvent* = ptr SelectEventImpl + +when hasThreadSupport: + template withSelectLock[T](s: Selector[T], body: untyped) = + acquire(s.lock) + {.locks: [s.lock].}: + try: + body + finally: + release(s.lock) +else: + template withSelectLock[T](s: Selector[T], body: untyped) = + body + +proc newSelector*[T](): Selector[T] = + when hasThreadSupport: + result = cast[Selector[T]](allocShared0(sizeof(SelectorImpl[T]))) + result.fds = allocSharedArray[SelectorKey[T]](FD_SETSIZE) + initLock result.lock + else: + result = Selector[T]() + result.fds = newSeq[SelectorKey[T]](FD_SETSIZE) + + IOFD_ZERO(addr result.rSet) + IOFD_ZERO(addr result.wSet) + IOFD_ZERO(addr result.eSet) + +proc close*[T](s: Selector[T]) = + when hasThreadSupport: + deallocSharedArray(s.fds) + deallocShared(cast[pointer](s)) + +when defined(windows): + proc newSelectEvent*(): SelectEvent = + var ssock = newNativeSocket() + var wsock = newNativeSocket() + var rsock: SocketHandle = INVALID_SOCKET + var saddr = Sockaddr_in() + + saddr.sin_family = winlean.AF_INET + saddr.sin_port = 0 + saddr.sin_addr.s_addr = INADDR_ANY + if bindAddr(ssock, cast[ptr SockAddr](addr(saddr)), + sizeof(saddr).SockLen) < 0'i32: + raiseOSError(osLastError()) + + if winlean.listen(ssock, 1) == -1: + raiseOSError(osLastError()) + + var namelen = sizeof(saddr).SockLen + if getsockname(ssock, cast[ptr SockAddr](addr(saddr)), + addr(namelen)) == -1'i32: + raiseOSError(osLastError()) + + saddr.sin_addr.s_addr = 0x0100007F + if winlean.connect(wsock, cast[ptr SockAddr](addr(saddr)), + sizeof(saddr).SockLen) == -1: + raiseOSError(osLastError()) + namelen = sizeof(saddr).SockLen + rsock = winlean.accept(ssock, cast[ptr SockAddr](addr(saddr)), + cast[ptr SockLen](addr(namelen))) + if rsock == SocketHandle(-1): + raiseOSError(osLastError()) + + if winlean.closesocket(ssock) == -1: + raiseOSError(osLastError()) + + var mode = clong(1) + if ioctlsocket(rsock, FIONBIO, addr(mode)) == -1: + raiseOSError(osLastError()) + mode = clong(1) + if ioctlsocket(wsock, FIONBIO, addr(mode)) == -1: + raiseOSError(osLastError()) + + result = cast[SelectEvent](allocShared0(sizeof(SelectEventImpl))) + result.rsock = rsock + result.wsock = wsock + + proc setEvent*(ev: SelectEvent) = + var data: int = 1 + if winlean.send(ev.wsock, cast[pointer](addr data), + cint(sizeof(int)), 0) != sizeof(int): + raiseOSError(osLastError()) + + proc close*(ev: SelectEvent) = + discard winlean.closesocket(ev.rsock) + discard winlean.closesocket(ev.wsock) + deallocShared(cast[pointer](ev)) + +else: + proc newSelectEvent*(): SelectEvent = + var fds: array[2, cint] + if posix.pipe(fds) == -1: + raiseOSError(osLastError()) + setNonBlocking(fds[0]) + setNonBlocking(fds[1]) + result = cast[SelectEvent](allocShared0(sizeof(SelectEventImpl))) + result.rsock = SocketHandle(fds[0]) + result.wsock = SocketHandle(fds[1]) + + proc setEvent*(ev: SelectEvent) = + var data: uint64 = 1 + if posix.write(cint(ev.wsock), addr data, sizeof(uint64)) != sizeof(uint64): + raiseOSError(osLastError()) + + proc close*(ev: SelectEvent) = + discard posix.close(cint(ev.rsock)) + discard posix.close(cint(ev.wsock)) + deallocShared(cast[pointer](ev)) + +proc setKey[T](s: Selector[T], fd: SocketHandle, events: set[Event], data: T) = + var i = 0 + let fdi = int(fd) + while i < FD_SETSIZE: + if s.fds[i].ident == 0: + var pkey = addr(s.fds[i]) + pkey.ident = fdi + pkey.events = events + pkey.key.fd = fd.int + pkey.key.events = {} + pkey.key.data = data + break + inc(i) + if i == FD_SETSIZE: + raise newException(ValueError, "Maximum numbers of fds exceeded") + +proc getKey[T](s: Selector[T], fd: SocketHandle): ptr SelectorKey[T] = + var i = 0 + let fdi = int(fd) + while i < FD_SETSIZE: + if s.fds[i].ident == fdi: + result = addr(s.fds[i]) + break + inc(i) + doAssert(i < FD_SETSIZE, "Descriptor not registered in queue") + +proc delKey[T](s: Selector[T], fd: SocketHandle) = + var i = 0 + while i < FD_SETSIZE: + if s.fds[i].ident == fd.int: + s.fds[i].ident = 0 + s.fds[i].events = {} + break + inc(i) + doAssert(i < FD_SETSIZE, "Descriptor not registered in queue") + +proc registerHandle*[T](s: Selector[T], fd: SocketHandle, + events: set[Event], data: T) = + when not defined(windows): + let fdi = int(fd) + s.withSelectLock(): + s.setKey(fd, events, data) + when not defined(windows): + if fdi > s.maxFD: s.maxFD = fdi + if Event.Read in events: + IOFD_SET(fd, addr s.rSet) + inc(s.count) + if Event.Write in events: + IOFD_SET(fd, addr s.wSet) + IOFD_SET(fd, addr s.eSet) + inc(s.count) + +proc registerEvent*[T](s: Selector[T], ev: SelectEvent, data: T) = + when not defined(windows): + let fdi = int(ev.rsock) + s.withSelectLock(): + s.setKey(ev.rsock, {Event.User}, data) + when not defined(windows): + if fdi > s.maxFD: s.maxFD = fdi + IOFD_SET(ev.rsock, addr s.rSet) + inc(s.count) + +proc updateHandle*[T](s: Selector[T], fd: SocketHandle, + events: set[Event]) = + let maskEvents = {Event.Timer, Event.Signal, Event.Process, Event.Vnode, + Event.User, Event.Oneshot, Event.Error} + s.withSelectLock(): + var pkey = s.getKey(fd) + doAssert(pkey.events * maskEvents == {}) + if pkey.events != events: + if (Event.Read in pkey.events) and (Event.Read notin events): + IOFD_CLR(fd, addr s.rSet) + dec(s.count) + if (Event.Write in pkey.events) and (Event.Write notin events): + IOFD_CLR(fd, addr s.wSet) + IOFD_CLR(fd, addr s.eSet) + dec(s.count) + if (Event.Read notin pkey.events) and (Event.Read in events): + IOFD_SET(fd, addr s.rSet) + inc(s.count) + if (Event.Write notin pkey.events) and (Event.Write in events): + IOFD_SET(fd, addr s.wSet) + IOFD_SET(fd, addr s.eSet) + inc(s.count) + pkey.events = events + +proc unregister*[T](s: Selector[T], fd: SocketHandle) = + s.withSelectLock(): + var pkey = s.getKey(fd) + if Event.Read in pkey.events: + IOFD_CLR(fd, addr s.rSet) + dec(s.count) + if Event.Write in pkey.events: + IOFD_CLR(fd, addr s.wSet) + IOFD_CLR(fd, addr s.eSet) + dec(s.count) + s.delKey(fd) + +proc unregister*[T](s: Selector[T], ev: SelectEvent) = + let fd = ev.rsock + s.withSelectLock(): + IOFD_CLR(fd, addr s.rSet) + dec(s.count) + s.delKey(fd) + +proc selectInto*[T](s: Selector[T], timeout: int, + results: var openarray[ReadyKey[T]]): int = + var tv = Timeval() + var ptv = addr tv + var rset, wset, eset: FdSet + + if timeout != -1: + tv.tv_sec = timeout.int32 div 1_000 + tv.tv_usec = (timeout.int32 %% 1_000) * 1_000 + else: + ptv = nil + + s.withSelectLock(): + rset = s.rSet + wset = s.wSet + eset = s.eSet + + var count = ioselect(cint(s.maxFD) + 1, addr(rset), addr(wset), + addr(eset), ptv) + if count < 0: + result = 0 + when defined(windows): + raiseOSError(osLastError()) + else: + let err = osLastError() + if cint(err) != EINTR: + raiseOSError(err) + elif count == 0: + result = 0 + else: + var rindex = 0 + var i = 0 + var k = 0 + + while (i < FD_SETSIZE) and (k < count): + if s.fds[i].ident != 0: + var flag = false + var pkey = addr(s.fds[i]) + pkey.key.events = {} + let fd = SocketHandle(pkey.ident) + if IOFD_ISSET(fd, addr rset) != 0: + if Event.User in pkey.events: + var data: uint64 = 0 + if recv(fd, cast[pointer](addr(data)), + sizeof(uint64).cint, 0) != sizeof(uint64): + let err = osLastError() + if cint(err) != EAGAIN: + raiseOSError(err) + else: + inc(i) + inc(k) + continue + else: + flag = true + pkey.key.events = {Event.User} + else: + flag = true + pkey.key.events = {Event.Read} + if IOFD_ISSET(fd, addr wset) != 0: + pkey.key.events.incl(Event.Write) + if IOFD_ISSET(fd, addr eset) != 0: + pkey.key.events.incl(Event.Error) + flag = true + if flag: + results[rindex] = pkey.key + inc(rindex) + inc(k) + inc(i) + result = rindex + +proc select*[T](s: Selector[T], timeout: int): seq[ReadyKey[T]] = + result = newSeq[ReadyKey[T]](FD_SETSIZE) + var count = selectInto(s, timeout, result) + result.setLen(count) + +proc flush*[T](s: Selector[T]) = discard + +template isEmpty*[T](s: Selector[T]): bool = + (s.count == 0) + +when hasThreadSupport: + template withSelectLock[T](s: Selector[T], body: untyped) = + acquire(s.lock) + {.locks: [s.lock].}: + try: + body + finally: + release(s.lock) +else: + template withSelectLock[T](s: Selector[T], body: untyped) = + body + +template withData*[T](s: Selector[T], fd: SocketHandle, value, + body: untyped) = + mixin withSelectLock + s.withSelectLock(): + var value: ptr T + let fdi = int(fd) + var i = 0 + while i < FD_SETSIZE: + if s.fds[i].ident == fdi: + value = addr(s.fds[i].key.data) + break + inc(i) + if i != FD_SETSIZE: + body + +template withData*[T](s: Selector[T], fd: SocketHandle, value, + body1, body2: untyped) = + mixin withSelectLock + s.withSelectLock(): + var value: ptr T + let fdi = int(fd) + var i = 0 + while i < FD_SETSIZE: + if s.fds[i].ident == fdi: + value = addr(s.fds[i].key.data) + break + inc(i) + if i != FD_SETSIZE: + body1 + else: + body2 diff --git a/lib/pure/json.nim b/lib/pure/json.nim index b0179cd82..19947fbc2 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -20,7 +20,8 @@ ## let ## small_json = """{"test": 1.3, "key2": true}""" ## jobj = parseJson(small_json) -## assert (jobj.kind == JObject) +## assert (jobj.kind == JObject)\ +## jobj["test"] = newJFloat(0.7) # create or update ## echo($jobj["test"].fnum) ## echo($jobj["key2"].bval) ## @@ -49,6 +50,10 @@ ## "age": herAge ## } ## ] +## +## var j2 = %* {"name": "Isaac", "books": ["Robot Dreams"]} +## j2["details"] = %* {"age":35, "pi":3.1415} +## echo j2 import hashes, tables, strutils, lexbase, streams, unicode, macros @@ -56,6 +61,11 @@ import export tables.`$` +when defined(nimJsonGet): + {.pragma: deprecatedGet, deprecated.} +else: + {.pragma: deprecatedGet.} + type JsonEventKind* = enum ## enumeration of all events that may occur when parsing jsonError, ## an error occurred during parsing @@ -116,7 +126,7 @@ type TJsonParser: JsonParser, TTokKind: TokKind].} const - errorMessages: array [JsonError, string] = [ + errorMessages: array[JsonError, string] = [ "no error", "invalid token", "string expected", @@ -129,7 +139,7 @@ const "EOF expected", "expression expected" ] - tokToStr: array [TokKind, string] = [ + tokToStr: array[TokKind, string] = [ "invalid token", "EOF", "string literal", @@ -702,17 +712,28 @@ proc `%`*(b: bool): JsonNode = proc `%`*(keyVals: openArray[tuple[key: string, val: JsonNode]]): JsonNode = ## Generic constructor for JSON data. Creates a new `JObject JsonNode` - new(result) - result.kind = JObject - result.fields = initTable[string, JsonNode](4) + if keyvals.len == 0: return newJArray() + result = newJObject() for key, val in items(keyVals): result.fields[key] = val -proc `%`*(elements: openArray[JsonNode]): JsonNode = +template `%`*(j: JsonNode): JsonNode = j + +proc `%`*[T](elements: openArray[T]): JsonNode = ## Generic constructor for JSON data. Creates a new `JArray JsonNode` - new(result) - result.kind = JArray - newSeq(result.elems, elements.len) - for i, p in pairs(elements): result.elems[i] = p + result = newJArray() + for elem in elements: result.add(%elem) + +proc `%`*(o: object): JsonNode = + ## Generic constructor for JSON data. Creates a new `JObject JsonNode` + result = newJObject() + for k, v in o.fieldPairs: result[k] = %v + +proc `%`*(o: ref object): JsonNode = + ## Generic constructor for JSON data. Creates a new `JObject JsonNode` + if o.isNil: + result = newJNull() + else: + result = %(o[]) proc toJson(x: NimNode): NimNode {.compiletime.} = case x.kind @@ -731,6 +752,9 @@ proc toJson(x: NimNode): NimNode {.compiletime.} = result = newNimNode(nnkTableConstr) x.expectLen(0) + of nnkNilLit: + result = newCall("newJNull") + else: result = x @@ -799,16 +823,23 @@ proc len*(n: JsonNode): int = of JObject: result = n.fields.len else: discard -proc `[]`*(node: JsonNode, name: string): JsonNode {.inline.} = +proc `[]`*(node: JsonNode, name: string): JsonNode {.inline, deprecatedGet.} = ## Gets a field from a `JObject`, which must not be nil. - ## If the value at `name` does not exist, returns nil + ## If the value at `name` does not exist, raises KeyError. + ## + ## **Note:** The behaviour of this procedure changed in version 0.14.0. To + ## get a list of usages and to restore the old behaviour of this procedure, + ## compile with the ``-d:nimJsonGet`` flag. assert(not isNil(node)) assert(node.kind == JObject) - result = node.fields.getOrDefault(name) + when defined(nimJsonGet): + if not node.fields.hasKey(name): return nil + result = node.fields[name] proc `[]`*(node: JsonNode, index: int): JsonNode {.inline.} = ## Gets the node at `index` in an Array. Result is undefined if `index` - ## is out of bounds + ## is out of bounds, but as long as array bound checks are enabled it will + ## result in an exception. assert(not isNil(node)) assert(node.kind == JArray) return node.elems[index] @@ -818,6 +849,16 @@ proc hasKey*(node: JsonNode, key: string): bool = assert(node.kind == JObject) result = node.fields.hasKey(key) +proc contains*(node: JsonNode, key: string): bool = + ## Checks if `key` exists in `node`. + assert(node.kind == JObject) + node.fields.hasKey(key) + +proc contains*(node: JsonNode, val: JsonNode): bool = + ## Checks if `val` exists in array `node`. + assert(node.kind == JArray) + find(node.elems, val) >= 0 + proc existsKey*(node: JsonNode, key: string): bool {.deprecated.} = node.hasKey(key) ## Deprecated for `hasKey` @@ -838,20 +879,28 @@ proc `[]=`*(obj: JsonNode, key: string, val: JsonNode) {.inline.} = proc `{}`*(node: JsonNode, keys: varargs[string]): JsonNode = ## Traverses the node and gets the given value. If any of the - ## keys do not exist, returns nil. Also returns nil if one of the - ## intermediate data structures is not an object + ## keys do not exist, returns ``nil``. Also returns ``nil`` if one of the + ## intermediate data structures is not an object. result = node for key in keys: - if isNil(result) or result.kind!=JObject: + if isNil(result) or result.kind != JObject: return nil - result=result[key] + result = result.fields.getOrDefault(key) + +proc getOrDefault*(node: JsonNode, key: string): JsonNode = + ## Gets a field from a `node`. If `node` is nil or not an object or + ## value at `key` does not exist, returns nil + if not isNil(node) and node.kind == JObject: + result = node.fields.getOrDefault(key) + +template simpleGetOrDefault*{`{}`(node, [key])}(node: JsonNode, key: string): JsonNode = node.getOrDefault(key) proc `{}=`*(node: JsonNode, keys: varargs[string], value: JsonNode) = ## Traverses the node and tries to set the value at the given location - ## to `value` If any of the keys are missing, they are added + ## to ``value``. If any of the keys are missing, they are added. var node = node for i in 0..(keys.len-2): - if isNil(node[keys[i]]): + if not node.hasKey(keys[i]): node[keys[i]] = newJObject() node = node[keys[i]] node[keys[keys.len-1]] = value @@ -972,7 +1021,7 @@ proc toPretty(result: var string, node: JsonNode, indent = 2, ml = true, result.add("null") proc pretty*(node: JsonNode, indent = 2): string = - ## Converts `node` to its JSON Representation, with indentation and + ## Returns a JSON Representation of `node`, with indentation and ## on multiple lines. result = "" toPretty(result, node, indent) @@ -982,7 +1031,9 @@ proc toUgly*(result: var string, node: JsonNode) = ## regard for human readability. Meant to improve ``$`` string ## conversion performance. ## - ## This provides higher efficiency than the ``toPretty`` procedure as it + ## JSON representation is stored in the passed `result` + ## + ## This provides higher efficiency than the ``pretty`` procedure as it ## does **not** attempt to format the resulting JSON to make it human readable. var comma = false case node.kind: @@ -1076,7 +1127,7 @@ proc parseJson(p: var JsonParser): JsonNode = discard getTok(p) while p.tok != tkCurlyRi: if p.tok != tkString: - raiseParseErr(p, "string literal as key expected") + raiseParseErr(p, "string literal as key") var key = p.a discard getTok(p) eat(p, tkColon) @@ -1217,16 +1268,6 @@ when false: # To get that we shall use, obj["json"] when isMainModule: - when not defined(js): - var parsed = parseFile("tests/testdata/jsontest.json") - - try: - discard parsed["key2"][12123] - doAssert(false) - except IndexError: doAssert(true) - - var parsed2 = parseFile("tests/testdata/jsontest2.json") - doAssert(parsed2{"repository", "description"}.str=="IRC Library for Haskell", "Couldn't fetch via multiply nested key using {}") let testJson = parseJson"""{ "a": [1, 2, 3, 4], "b": "asd", "c": "\ud83c\udf83", "d": "\u00E6"}""" # nil passthrough @@ -1314,3 +1355,35 @@ when isMainModule: var j4 = %*{"test": nil} doAssert j4 == %{"test": newJNull()} + + let seqOfNodes = @[%1, %2] + let jSeqOfNodes = %seqOfNodes + doAssert(jSeqOfNodes[1].num == 2) + + type MyObj = object + a, b: int + s: string + f32: float32 + f64: float64 + next: ref MyObj + var m: MyObj + m.s = "hi" + m.a = 5 + let jMyObj = %m + doAssert(jMyObj["a"].num == 5) + doAssert(jMyObj["s"].str == "hi") + + # Test loading of file. + when not defined(js): + echo("99% of tests finished. Going to try loading file.") + var parsed = parseFile("tests/testdata/jsontest.json") + + try: + discard parsed["key2"][12123] + doAssert(false) + except IndexError: doAssert(true) + + var parsed2 = parseFile("tests/testdata/jsontest2.json") + doAssert(parsed2{"repository", "description"}.str=="IRC Library for Haskell", "Couldn't fetch via multiply nested key using {}") + + echo("Tests succeeded!") diff --git a/lib/pure/logging.nim b/lib/pure/logging.nim index f602ce31d..6a27e6af8 100644 --- a/lib/pure/logging.nim +++ b/lib/pure/logging.nim @@ -54,14 +54,15 @@ type lvlAll, ## all levels active lvlDebug, ## debug level (and any above) active lvlInfo, ## info level (and any above) active + lvlNotice, ## info notice (and any above) active lvlWarn, ## warn level (and any above) active lvlError, ## error level (and any above) active lvlFatal, ## fatal level (and any above) active lvlNone ## no levels active const - LevelNames*: array [Level, string] = [ - "DEBUG", "DEBUG", "INFO", "WARN", "ERROR", "FATAL", "NONE" + LevelNames*: array[Level, string] = [ + "DEBUG", "DEBUG", "INFO", "NOTICE", "WARN", "ERROR", "FATAL", "NONE" ] defaultFmtStr* = "$levelname " ## default format string @@ -258,22 +259,47 @@ template log*(level: Level, args: varargs[string, `$`]) = template debug*(args: varargs[string, `$`]) = ## Logs a debug message to all registered handlers. + ## + ## Messages that are useful to the application developer only and are usually + ## turned off in release. log(lvlDebug, args) template info*(args: varargs[string, `$`]) = ## Logs an info message to all registered handlers. + ## + ## Messages that are generated during the normal operation of an application + ## and are of no particular importance. Useful to aggregate for potential + ## later analysis. log(lvlInfo, args) +template notice*(args: varargs[string, `$`]) = + ## Logs an notice message to all registered handlers. + ## + ## Semantically very similar to `info`, but meant to be messages you want to + ## be actively notified about (depending on your application). + ## These could be, for example, grouped by hour and mailed out. + log(lvlNotice, args) + template warn*(args: varargs[string, `$`]) = ## Logs a warning message to all registered handlers. + ## + ## A non-error message that may indicate a potential problem rising or + ## impacted performance. log(lvlWarn, args) template error*(args: varargs[string, `$`]) = ## Logs an error message to all registered handlers. + ## + ## A application-level error condition. For example, some user input generated + ## an exception. The application will continue to run, but functionality or + ## data was impacted, possibly visible to users. log(lvlError, args) template fatal*(args: varargs[string, `$`]) = ## Logs a fatal error message to all registered handlers. + ## + ## A application-level fatal condition. FATAL usually means that the application + ## cannot go on and will exit (but this logging event will not do that for you). log(lvlFatal, args) proc addHandler*(handler: Logger) = diff --git a/lib/pure/matchers.nim b/lib/pure/matchers.nim index 5c28f65a0..7022c21d9 100644 --- a/lib/pure/matchers.nim +++ b/lib/pure/matchers.nim @@ -8,6 +8,10 @@ # ## This module contains various string matchers for email addresses, etc. +## +## **Warning:** This module is deprecated since version 0.14.0. +{.deprecated.} + {.deadCodeElim: on.} {.push debugger:off .} # the user does not want to trace a part diff --git a/lib/pure/math.nim b/lib/pure/math.nim index 84c8d3b11..4ef169b4f 100644 --- a/lib/pure/math.nim +++ b/lib/pure/math.nim @@ -39,8 +39,6 @@ proc fac*(n: int): int {.noSideEffect.} = when defined(Posix) and not defined(haiku): {.passl: "-lm".} -when not defined(js) and not defined(nimscript): - import times const PI* = 3.1415926535897932384626433 ## the circle constant PI (Ludolph's number) @@ -119,192 +117,247 @@ proc sum*[T](x: openArray[T]): T {.noSideEffect.} = ## If `x` is empty, 0 is returned. for i in items(x): result = result + i -proc random*(max: int): int {.benign.} - ## Returns a random number in the range 0..max-1. The sequence of - ## random number is always the same, unless `randomize` is called - ## which initializes the random number generator with a "random" - ## number, i.e. a tickcount. - -proc random*(max: float): float {.benign.} - ## Returns a random number in the range 0..<max. The sequence of - ## random number is always the same, unless `randomize` is called - ## which initializes the random number generator with a "random" - ## number, i.e. a tickcount. This has a 16-bit resolution on windows - ## and a 48-bit resolution on other platforms. - -when not defined(nimscript): - proc randomize*() {.benign.} - ## Initializes the random number generator with a "random" - ## number, i.e. a tickcount. Note: Does nothing for the JavaScript target, - ## as JavaScript does not support this. Nor does it work for NimScript. - -proc randomize*(seed: int) {.benign.} - ## Initializes the random number generator with a specific seed. - ## Note: Does nothing for the JavaScript target, - ## as JavaScript does not support this. - {.push noSideEffect.} when not defined(JS): - proc sqrt*(x: float): float {.importc: "sqrt", header: "<math.h>".} + proc sqrt*(x: float32): float32 {.importc: "sqrtf", header: "<math.h>".} + proc sqrt*(x: float64): float64 {.importc: "sqrt", header: "<math.h>".} ## Computes the square root of `x`. - proc cbrt*(x: float): float {.importc: "cbrt", header: "<math.h>".} + proc cbrt*(x: float32): float32 {.importc: "cbrtf", header: "<math.h>".} + proc cbrt*(x: float64): float64 {.importc: "cbrt", header: "<math.h>".} ## Computes the cubic root of `x` - proc ln*(x: float): float {.importc: "log", header: "<math.h>".} + proc ln*(x: float32): float32 {.importc: "logf", header: "<math.h>".} + proc ln*(x: float64): float64 {.importc: "log", header: "<math.h>".} ## Computes the natural log of `x` - proc log10*(x: float): float {.importc: "log10", header: "<math.h>".} + proc log10*(x: float32): float32 {.importc: "log10f", header: "<math.h>".} + proc log10*(x: float64): float64 {.importc: "log10", header: "<math.h>".} ## Computes the common logarithm (base 10) of `x` - proc log2*(x: float): float = return ln(x) / ln(2.0) + proc log2*[T: float32|float64](x: T): T = return ln(x) / ln(2.0) ## Computes the binary logarithm (base 2) of `x` - proc exp*(x: float): float {.importc: "exp", header: "<math.h>".} + proc exp*(x: float32): float32 {.importc: "expf", header: "<math.h>".} + proc exp*(x: float64): float64 {.importc: "exp", header: "<math.h>".} ## Computes the exponential function of `x` (pow(E, x)) - proc frexp*(x: float, exponent: var int): float {. - importc: "frexp", header: "<math.h>".} - ## Split a number into mantissa and exponent. - ## `frexp` calculates the mantissa m (a float greater than or equal to 0.5 - ## and less than 1) and the integer value n such that `x` (the original - ## float value) equals m * 2**n. frexp stores n in `exponent` and returns - ## m. - - proc round*(x: float): int {.importc: "lrint", header: "<math.h>".} - ## Converts a float to an int by rounding. - - proc arccos*(x: float): float {.importc: "acos", header: "<math.h>".} + proc arccos*(x: float32): float32 {.importc: "acosf", header: "<math.h>".} + proc arccos*(x: float64): float64 {.importc: "acos", header: "<math.h>".} ## Computes the arc cosine of `x` - proc arcsin*(x: float): float {.importc: "asin", header: "<math.h>".} + proc arcsin*(x: float32): float32 {.importc: "asinf", header: "<math.h>".} + proc arcsin*(x: float64): float64 {.importc: "asin", header: "<math.h>".} ## Computes the arc sine of `x` - proc arctan*(x: float): float {.importc: "atan", header: "<math.h>".} + proc arctan*(x: float32): float32 {.importc: "atanf", header: "<math.h>".} + proc arctan*(x: float64): float64 {.importc: "atan", header: "<math.h>".} ## Calculate the arc tangent of `y` / `x` - proc arctan2*(y, x: float): float {.importc: "atan2", header: "<math.h>".} + proc arctan2*(y, x: float32): float32 {.importc: "atan2f", header: "<math.h>".} + proc arctan2*(y, x: float64): float64 {.importc: "atan2", header: "<math.h>".} ## Calculate the arc tangent of `y` / `x`. ## `atan2` returns the arc tangent of `y` / `x`; it produces correct ## results even when the resulting angle is near pi/2 or -pi/2 ## (`x` near 0). - proc cos*(x: float): float {.importc: "cos", header: "<math.h>".} + proc cos*(x: float32): float32 {.importc: "cosf", header: "<math.h>".} + proc cos*(x: float64): float64 {.importc: "cos", header: "<math.h>".} ## Computes the cosine of `x` - proc cosh*(x: float): float {.importc: "cosh", header: "<math.h>".} + + proc cosh*(x: float32): float32 {.importc: "coshf", header: "<math.h>".} + proc cosh*(x: float64): float64 {.importc: "cosh", header: "<math.h>".} ## Computes the hyperbolic cosine of `x` - proc hypot*(x, y: float): float {.importc: "hypot", header: "<math.h>".} + + proc hypot*(x, y: float32): float32 {.importc: "hypotf", header: "<math.h>".} + proc hypot*(x, y: float64): float64 {.importc: "hypot", header: "<math.h>".} ## Computes the hypotenuse of a right-angle triangle with `x` and ## `y` as its base and height. Equivalent to ``sqrt(x*x + y*y)``. - proc sinh*(x: float): float {.importc: "sinh", header: "<math.h>".} + proc sinh*(x: float32): float32 {.importc: "sinhf", header: "<math.h>".} + proc sinh*(x: float64): float64 {.importc: "sinh", header: "<math.h>".} ## Computes the hyperbolic sine of `x` - proc sin*(x: float): float {.importc: "sin", header: "<math.h>".} + proc sin*(x: float32): float32 {.importc: "sinf", header: "<math.h>".} + proc sin*(x: float64): float64 {.importc: "sin", header: "<math.h>".} ## Computes the sine of `x` - proc tan*(x: float): float {.importc: "tan", header: "<math.h>".} + + proc tan*(x: float32): float32 {.importc: "tanf", header: "<math.h>".} + proc tan*(x: float64): float64 {.importc: "tan", header: "<math.h>".} ## Computes the tangent of `x` - proc tanh*(x: float): float {.importc: "tanh", header: "<math.h>".} + proc tanh*(x: float32): float32 {.importc: "tanhf", header: "<math.h>".} + proc tanh*(x: float64): float64 {.importc: "tanh", header: "<math.h>".} ## Computes the hyperbolic tangent of `x` - proc pow*(x, y: float): float {.importc: "pow", header: "<math.h>".} - ## Computes `x` to power of `y`. - proc erf*(x: float): float {.importc: "erf", header: "<math.h>".} + proc pow*(x, y: float32): float32 {.importc: "powf", header: "<math.h>".} + proc pow*(x, y: float64): float64 {.importc: "pow", header: "<math.h>".} + ## computes x to power raised of y. + + proc erf*(x: float32): float32 {.importc: "erff", header: "<math.h>".} + proc erf*(x: float64): float64 {.importc: "erf", header: "<math.h>".} ## The error function - proc erfc*(x: float): float {.importc: "erfc", header: "<math.h>".} + proc erfc*(x: float32): float32 {.importc: "erfcf", header: "<math.h>".} + proc erfc*(x: float64): float64 {.importc: "erfc", header: "<math.h>".} ## The complementary error function - proc lgamma*(x: float): float {.importc: "lgamma", header: "<math.h>".} + proc lgamma*(x: float32): float32 {.importc: "lgammaf", header: "<math.h>".} + proc lgamma*(x: float64): float64 {.importc: "lgamma", header: "<math.h>".} ## Natural log of the gamma function - proc tgamma*(x: float): float {.importc: "tgamma", header: "<math.h>".} + proc tgamma*(x: float32): float32 {.importc: "tgammaf", header: "<math.h>".} + proc tgamma*(x: float64): float64 {.importc: "tgamma", header: "<math.h>".} ## The gamma function - # C procs: - when defined(vcc) and false: - # The "secure" random, available from Windows XP - # https://msdn.microsoft.com/en-us/library/sxtz2fa8.aspx - # Present in some variants of MinGW but not enough to justify - # `when defined(windows)` yet - proc rand_s(val: var cuint) {.importc: "rand_s", header: "<stdlib.h>".} - # To behave like the normal version - proc rand(): cuint = rand_s(result) - else: - proc srand(seed: cint) {.importc: "srand", header: "<stdlib.h>".} - proc rand(): cint {.importc: "rand", header: "<stdlib.h>".} - - when not defined(windows): - proc srand48(seed: clong) {.importc: "srand48", header: "<stdlib.h>".} - proc drand48(): float {.importc: "drand48", header: "<stdlib.h>".} - proc random(max: float): float = - result = drand48() * max - else: - when defined(vcc): # Windows with Visual C - proc random(max: float): float = - # we are hardcoding this because - # importc-ing macros is extremely problematic - # and because the value is publicly documented - # on MSDN and very unlikely to change - # See https://msdn.microsoft.com/en-us/library/296az74e.aspx - const rand_max = 4294967295 # UINT_MAX - result = (float(rand()) / float(rand_max)) * max - proc randomize() = discard - proc randomize(seed: int) = discard - else: # Windows with another compiler - proc random(max: float): float = - # we are hardcoding this because - # importc-ing macros is extremely problematic - # and because the value is publicly documented - # on MSDN and very unlikely to change - const rand_max = 32767 - result = (float(rand()) / float(rand_max)) * max - - when not defined(vcc): # the above code for vcc uses `discard` instead - # this is either not Windows or is Windows without vcc - when not defined(nimscript): - proc randomize() = - randomize(cast[int](epochTime())) - proc randomize(seed: int) = - srand(cint(seed)) # rand_s doesn't use srand - when declared(srand48): srand48(seed) - - proc random(max: int): int = - result = int(rand()) mod max - - proc trunc*(x: float): float {.importc: "trunc", header: "<math.h>".} - ## Truncates `x` to the decimal point - ## - ## .. code-block:: nim - ## echo trunc(PI) # 3.0 - proc floor*(x: float): float {.importc: "floor", header: "<math.h>".} + proc floor*(x: float32): float32 {.importc: "floorf", header: "<math.h>".} + proc floor*(x: float64): float64 {.importc: "floor", header: "<math.h>".} ## Computes the floor function (i.e., the largest integer not greater than `x`) ## ## .. code-block:: nim ## echo floor(-3.5) ## -4.0 - proc ceil*(x: float): float {.importc: "ceil", header: "<math.h>".} + + proc ceil*(x: float32): float32 {.importc: "ceilf", header: "<math.h>".} + proc ceil*(x: float64): float64 {.importc: "ceil", header: "<math.h>".} ## Computes the ceiling function (i.e., the smallest integer not less than `x`) ## ## .. code-block:: nim ## echo ceil(-2.1) ## -2.0 - proc fmod*(x, y: float): float {.importc: "fmod", header: "<math.h>".} + when defined(windows) and defined(vcc): + # MSVC 2010 don't have trunc/truncf + # this implementation was inspired by Go-lang Math.Trunc + proc truncImpl(f: float64): float64 = + const + mask : uint64 = 0x7FF + shift: uint64 = 64 - 12 + bias : uint64 = 0x3FF + + if f < 1: + if f < 0: return -truncImpl(-f) + elif f == 0: return f # Return -0 when f == -0 + else: return 0 + + var x = cast[uint64](f) + let e = (x shr shift) and mask - bias + + # Keep the top 12+e bits, the integer part; clear the rest. + if e < 64-12: + x = x and (not (1'u64 shl (64'u64-12'u64-e) - 1'u64)) + + result = cast[float64](x) + + proc truncImpl(f: float32): float32 = + const + mask : uint32 = 0xFF + shift: uint32 = 32 - 9 + bias : uint32 = 0x7F + + if f < 1: + if f < 0: return -truncImpl(-f) + elif f == 0: return f # Return -0 when f == -0 + else: return 0 + + var x = cast[uint32](f) + let e = (x shr shift) and mask - bias + + # Keep the top 9+e bits, the integer part; clear the rest. + if e < 32-9: + x = x and (not (1'u32 shl (32'u32-9'u32-e) - 1'u32)) + + result = cast[float32](x) + + proc trunc*(x: float64): float64 = + if classify(x) in {fcZero, fcNegZero, fcNan, fcInf, fcNegInf}: return x + result = truncImpl(x) + + proc trunc*(x: float32): float32 = + if classify(x) in {fcZero, fcNegZero, fcNan, fcInf, fcNegInf}: return x + result = truncImpl(x) + + proc round0[T: float32|float64](x: T): T = + ## Windows compilers prior to MSVC 2012 do not implement 'round', + ## 'roundl' or 'roundf'. + result = if x < 0.0: ceil(x - T(0.5)) else: floor(x + T(0.5)) + else: + proc round0(x: float32): float32 {.importc: "roundf", header: "<math.h>".} + proc round0(x: float64): float64 {.importc: "round", header: "<math.h>".} + ## Rounds a float to zero decimal places. Used internally by the round + ## function when the specified number of places is 0. + + proc trunc*(x: float32): float32 {.importc: "truncf", header: "<math.h>".} + proc trunc*(x: float64): float64 {.importc: "trunc", header: "<math.h>".} + ## Truncates `x` to the decimal point + ## + ## .. code-block:: nim + ## echo trunc(PI) # 3.0 + + proc fmod*(x, y: float32): float32 {.importc: "fmodf", header: "<math.h>".} + proc fmod*(x, y: float64): float64 {.importc: "fmod", header: "<math.h>".} ## Computes the remainder of `x` divided by `y` ## ## .. code-block:: nim ## echo fmod(-2.5, 0.3) ## -0.1 else: - proc mathrandom(): float {.importc: "Math.random", nodecl.} - proc floor*(x: float): float {.importc: "Math.floor", nodecl.} - proc ceil*(x: float): float {.importc: "Math.ceil", nodecl.} - proc random(max: int): int = - result = int(floor(mathrandom() * float(max))) - proc random(max: float): float = - result = float(mathrandom() * float(max)) - proc randomize() = discard - proc randomize(seed: int) = discard - - proc sqrt*(x: float): float {.importc: "Math.sqrt", nodecl.} - proc ln*(x: float): float {.importc: "Math.log", nodecl.} - proc log10*(x: float): float = return ln(x) / ln(10.0) - proc log2*(x: float): float = return ln(x) / ln(2.0) - - proc exp*(x: float): float {.importc: "Math.exp", nodecl.} - proc round*(x: float): int {.importc: "Math.round", nodecl.} - proc pow*(x, y: float): float {.importc: "Math.pow", nodecl.} - - proc frexp*(x: float, exponent: var int): float = + proc floor*(x: float32): float32 {.importc: "Math.floor", nodecl.} + proc floor*(x: float64): float64 {.importc: "Math.floor", nodecl.} + proc ceil*(x: float32): float32 {.importc: "Math.ceil", nodecl.} + proc ceil*(x: float64): float64 {.importc: "Math.ceil", nodecl.} + + proc sqrt*(x: float32): float32 {.importc: "Math.sqrt", nodecl.} + proc sqrt*(x: float64): float64 {.importc: "Math.sqrt", nodecl.} + proc ln*(x: float32): float32 {.importc: "Math.log", nodecl.} + proc ln*(x: float64): float64 {.importc: "Math.log", nodecl.} + proc log10*[T: float32|float64](x: T): T = return ln(x) / ln(10.0) + proc log2*[T: float32|float64](x: T): T = return ln(x) / ln(2.0) + + proc exp*(x: float32): float32 {.importc: "Math.exp", nodecl.} + proc exp*(x: float64): float64 {.importc: "Math.exp", nodecl.} + proc round0(x: float): float {.importc: "Math.round", nodecl.} + + proc pow*(x, y: float32): float32 {.importC: "Math.pow", nodecl.} + proc pow*(x, y: float64): float64 {.importc: "Math.pow", nodecl.} + + proc arccos*(x: float32): float32 {.importc: "Math.acos", nodecl.} + proc arccos*(x: float64): float64 {.importc: "Math.acos", nodecl.} + proc arcsin*(x: float32): float32 {.importc: "Math.asin", nodecl.} + proc arcsin*(x: float64): float64 {.importc: "Math.asin", nodecl.} + proc arctan*(x: float32): float32 {.importc: "Math.atan", nodecl.} + proc arctan*(x: float64): float64 {.importc: "Math.atan", nodecl.} + proc arctan2*(y, x: float32): float32 {.importC: "Math.atan2", nodecl.} + proc arctan2*(y, x: float64): float64 {.importc: "Math.atan2", nodecl.} + + proc cos*(x: float32): float32 {.importc: "Math.cos", nodecl.} + proc cos*(x: float64): float64 {.importc: "Math.cos", nodecl.} + proc cosh*(x: float32): float32 = return (exp(x)+exp(-x))*0.5 + proc cosh*(x: float64): float64 = return (exp(x)+exp(-x))*0.5 + proc hypot*[T: float32|float64](x, y: T): T = return sqrt(x*x + y*y) + proc sinh*[T: float32|float64](x: T): T = return (exp(x)-exp(-x))*0.5 + proc sin*(x: float32): float32 {.importc: "Math.sin", nodecl.} + proc sin*(x: float64): float64 {.importc: "Math.sin", nodecl.} + proc tan*(x: float32): float32 {.importc: "Math.tan", nodecl.} + proc tan*(x: float64): float64 {.importc: "Math.tan", nodecl.} + proc tanh*[T: float32|float64](x: T): T = + var y = exp(2.0*x) + return (y-1.0)/(y+1.0) + +proc round*[T: float32|float64](x: T, places: int = 0): T = + ## Round a floating point number. + ## + ## If `places` is 0 (or omitted), round to the nearest integral value + ## following normal mathematical rounding rules (e.g. `round(54.5) -> 55.0`). + ## If `places` is greater than 0, round to the given number of decimal + ## places, e.g. `round(54.346, 2) -> 54.35`. + ## If `places` is negative, round to the left of the decimal place, e.g. + ## `round(537.345, -1) -> 540.0` + if places == 0: + result = round0(x) + else: + var mult = pow(10.0, places.T) + result = round0(x*mult)/mult + +when not defined(JS): + proc frexp*(x: float32, exponent: var int): float32 {. + importc: "frexp", header: "<math.h>".} + proc frexp*(x: float64, exponent: var int): float64 {. + importc: "frexp", header: "<math.h>".} + ## Split a number into mantissa and exponent. + ## `frexp` calculates the mantissa m (a float greater than or equal to 0.5 + ## and less than 1) and the integer value n such that `x` (the original + ## float value) equals m * 2**n. frexp stores n in `exponent` and returns + ## m. +else: + proc frexp*[T: float32|float64](x: T, exponent: var int): T = if x == 0.0: exponent = 0 result = 0.0 @@ -315,20 +368,22 @@ else: exponent = round(ex) result = x / pow(2.0, ex) - proc arccos*(x: float): float {.importc: "Math.acos", nodecl.} - proc arcsin*(x: float): float {.importc: "Math.asin", nodecl.} - proc arctan*(x: float): float {.importc: "Math.atan", nodecl.} - proc arctan2*(y, x: float): float {.importc: "Math.atan2", nodecl.} - - proc cos*(x: float): float {.importc: "Math.cos", nodecl.} - proc cosh*(x: float): float = return (exp(x)+exp(-x))*0.5 - proc hypot*(x, y: float): float = return sqrt(x*x + y*y) - proc sinh*(x: float): float = return (exp(x)-exp(-x))*0.5 - proc sin*(x: float): float {.importc: "Math.sin", nodecl.} - proc tan*(x: float): float {.importc: "Math.tan", nodecl.} - proc tanh*(x: float): float = - var y = exp(2.0*x) - return (y-1.0)/(y+1.0) +proc splitDecimal*[T: float32|float64](x: T): tuple[intpart: T, floatpart: T] = + ## Breaks `x` into an integral and a fractional part. + ## + ## Returns a tuple containing intpart and floatpart representing + ## the integer part and the fractional part respectively. + ## + ## Both parts have the same sign as `x`. Analogous to the `modf` + ## function in C. + var + absolute: T + absolute = abs(x) + result.intpart = floor(absolute) + result.floatpart = absolute - result.intpart + if x < 0: + result.intpart = -result.intpart + result.floatpart = -result.floatpart {.pop.} @@ -340,7 +395,7 @@ proc radToDeg*[T: float32|float64](d: T): T {.inline.} = ## Convert from radians to degrees result = T(d) / RadPerDeg -proc `mod`*(x, y: float): float = +proc `mod`*[T: float32|float64](x, y: T): T = ## Computes the modulo operation for float operators. Equivalent ## to ``x - y * floor(x/y)``. Note that the remainder will always ## have the same sign as the divisor. @@ -349,21 +404,13 @@ proc `mod`*(x, y: float): float = ## echo (4.0 mod -3.1) # -2.2 result = if y == 0.0: x else: x - y * (x/y).floor -proc random*[T](x: Slice[T]): T = - ## For a slice `a .. b` returns a value in the range `a .. b-1`. - result = random(x.b - x.a) + x.a - -proc random*[T](a: openArray[T]): T = - ## returns a random element from the openarray `a`. - result = a[random(a.low..a.len)] - {.pop.} {.pop.} proc `^`*[T](x, y: T): T = ## Computes ``x`` to the power ``y`. ``x`` must be non-negative, use ## `pow <#pow,float,float>` for negative exponents. - assert y >= 0 + assert y >= T(0) var (x, y) = (x, y) result = 1 @@ -391,24 +438,6 @@ proc lcm*[T](x, y: T): T = x div gcd(x, y) * y when isMainModule and not defined(JS): - proc gettime(dummy: ptr cint): cint {.importc: "time", header: "<time.h>".} - - # Verifies random seed initialization. - let seed = gettime(nil) - randomize(seed) - const SIZE = 10 - var buf : array[0..SIZE, int] - # Fill the buffer with random values - for i in 0..SIZE-1: - buf[i] = random(high(int)) - # Check that the second random calls are the same for each position. - randomize(seed) - for i in 0..SIZE-1: - assert buf[i] == random(high(int)), "non deterministic random seeding" - - when not defined(testing): - echo "random values equal after reseeding" - # Check for no side effect annotation proc mySqrt(num: float): float {.noSideEffect.} = return sqrt(num) @@ -418,3 +447,65 @@ when isMainModule and not defined(JS): assert(lgamma(1.0) == 0.0) # ln(1.0) == 0.0 assert(erf(6.0) > erf(5.0)) assert(erfc(6.0) < erfc(5.0)) +when isMainModule: + # Function for approximate comparison of floats + proc `==~`(x, y: float): bool = (abs(x-y) < 1e-9) + + block: # round() tests + # Round to 0 decimal places + doAssert round(54.652) ==~ 55.0 + doAssert round(54.352) ==~ 54.0 + doAssert round(-54.652) ==~ -55.0 + doAssert round(-54.352) ==~ -54.0 + doAssert round(0.0) ==~ 0.0 + # Round to positive decimal places + doAssert round(-547.652, 1) ==~ -547.7 + doAssert round(547.652, 1) ==~ 547.7 + doAssert round(-547.652, 2) ==~ -547.65 + doAssert round(547.652, 2) ==~ 547.65 + # Round to negative decimal places + doAssert round(547.652, -1) ==~ 550.0 + doAssert round(547.652, -2) ==~ 500.0 + doAssert round(547.652, -3) ==~ 1000.0 + doAssert round(547.652, -4) ==~ 0.0 + doAssert round(-547.652, -1) ==~ -550.0 + doAssert round(-547.652, -2) ==~ -500.0 + doAssert round(-547.652, -3) ==~ -1000.0 + doAssert round(-547.652, -4) ==~ 0.0 + + block: # splitDecimal() tests + doAssert splitDecimal(54.674).intpart ==~ 54.0 + doAssert splitDecimal(54.674).floatpart ==~ 0.674 + doAssert splitDecimal(-693.4356).intpart ==~ -693.0 + doAssert splitDecimal(-693.4356).floatpart ==~ -0.4356 + doAssert splitDecimal(0.0).intpart ==~ 0.0 + doAssert splitDecimal(0.0).floatpart ==~ 0.0 + + block: # trunc tests for vcc + doAssert(trunc(-1.1) == -1) + doAssert(trunc(1.1) == 1) + doAssert(trunc(-0.1) == -0) + doAssert(trunc(0.1) == 0) + + #special case + doAssert(classify(trunc(1e1000000)) == fcInf) + doAssert(classify(trunc(-1e1000000)) == fcNegInf) + doAssert(classify(trunc(0.0/0.0)) == fcNan) + doAssert(classify(trunc(0.0)) == fcZero) + + #trick the compiler to produce signed zero + let + f_neg_one = -1.0 + f_zero = 0.0 + f_nan = f_zero / f_zero + + doAssert(classify(trunc(f_neg_one*f_zero)) == fcNegZero) + + doAssert(trunc(-1.1'f32) == -1) + doAssert(trunc(1.1'f32) == 1) + doAssert(trunc(-0.1'f32) == -0) + doAssert(trunc(0.1'f32) == 0) + doAssert(classify(trunc(1e1000000'f32)) == fcInf) + doAssert(classify(trunc(-1e1000000'f32)) == fcNegInf) + doAssert(classify(trunc(f_nan.float32)) == fcNan) + doAssert(classify(trunc(0.0'f32)) == fcZero) diff --git a/lib/pure/memfiles.nim b/lib/pure/memfiles.nim index b9c574944..aa32778c5 100644 --- a/lib/pure/memfiles.nim +++ b/lib/pure/memfiles.nim @@ -42,6 +42,10 @@ type proc mapMem*(m: var MemFile, mode: FileMode = fmRead, mappedSize = -1, offset = 0): pointer = + ## returns a pointer to a mapped portion of MemFile `m` + ## + ## ``mappedSize`` of ``-1`` maps to the whole file, and + ## ``offset`` must be multiples of the PAGE SIZE of your OS var readonly = mode == fmRead when defined(windows): result = mapViewOfFileEx( @@ -68,7 +72,9 @@ proc mapMem*(m: var MemFile, mode: FileMode = fmRead, proc unmapMem*(f: var MemFile, p: pointer, size: int) = ## unmaps the memory region ``(p, <p+size)`` of the mapped file `f`. ## All changes are written back to the file system, if `f` was opened - ## with write access. ``size`` must be of exactly the size that was requested + ## with write access. + ## + ## ``size`` must be of exactly the size that was requested ## via ``mapMem``. when defined(windows): if unmapViewOfFile(p) == 0: raiseOSError(osLastError()) @@ -79,9 +85,17 @@ proc unmapMem*(f: var MemFile, p: pointer, size: int) = proc open*(filename: string, mode: FileMode = fmRead, mappedSize = -1, offset = 0, newFileSize = -1): MemFile = ## opens a memory mapped file. If this fails, ``EOS`` is raised. - ## `newFileSize` can only be set if the file does not exist and is opened - ## with write access (e.g., with fmReadWrite). `mappedSize` and `offset` - ## can be used to map only a slice of the file. Example: + ## + ## ``newFileSize`` can only be set if the file does not exist and is opened + ## with write access (e.g., with fmReadWrite). + ## + ##``mappedSize`` and ``offset`` + ## can be used to map only a slice of the file. + ## + ## ``offset`` must be multiples of the PAGE SIZE of your OS + ## (usually 4K or 8K but is unique to your OS) + ## + ## Example: ## ## .. code-block:: nim ## var @@ -257,12 +271,10 @@ type MemSlice* = object ## represent slice of a MemFile for iteration over deli data*: pointer size*: int -proc c_memcpy(a, b: pointer, n: int) {.importc: "memcpy", header: "<string.h>".} - proc `$`*(ms: MemSlice): string {.inline.} = ## Return a Nim string built from a MemSlice. var buf = newString(ms.size) - c_memcpy(addr(buf[0]), ms.data, ms.size) + copyMem(addr(buf[0]), ms.data, ms.size) buf[ms.size] = '\0' result = buf @@ -287,7 +299,9 @@ iterator memSlices*(mfile: MemFile, delim='\l', eat='\r'): MemSlice {.inline.} = ## iterate over line-like records in a file. However, returned (data,size) ## objects are not Nim strings, bounds checked Nim arrays, or even terminated ## C strings. So, care is required to access the data (e.g., think C mem* - ## functions, not str* functions). Example: + ## functions, not str* functions). + ## + ## Example: ## ## .. code-block:: nim ## var count = 0 @@ -320,7 +334,9 @@ iterator lines*(mfile: MemFile, buf: var TaintedString, delim='\l', eat='\r'): T ## Replace contents of passed buffer with each new line, like ## `readLine(File) <system.html#readLine,File,TaintedString>`_. ## `delim`, `eat`, and delimiting logic is exactly as for - ## `memSlices <#memSlices>`_, but Nim strings are returned. Example: + ## `memSlices <#memSlices>`_, but Nim strings are returned. + ## + ## Example: ## ## .. code-block:: nim ## var buffer: TaintedString = "" @@ -329,7 +345,7 @@ iterator lines*(mfile: MemFile, buf: var TaintedString, delim='\l', eat='\r'): T for ms in memSlices(mfile, delim, eat): buf.setLen(ms.size) - c_memcpy(addr(buf[0]), ms.data, ms.size) + copyMem(addr(buf[0]), ms.data, ms.size) buf[ms.size] = '\0' yield buf @@ -337,7 +353,9 @@ iterator lines*(mfile: MemFile, delim='\l', eat='\r'): TaintedString {.inline.} ## Return each line in a file as a Nim string, like ## `lines(File) <system.html#lines.i,File>`_. ## `delim`, `eat`, and delimiting logic is exactly as for - ## `memSlices <#memSlices>`_, but Nim strings are returned. Example: + ## `memSlices <#memSlices>`_, but Nim strings are returned. + ## + ## Example: ## ## .. code-block:: nim ## for line in lines(memfiles.open("foo")): diff --git a/lib/pure/mersenne.nim b/lib/pure/mersenne.nim index ae0845714..36b597767 100644 --- a/lib/pure/mersenne.nim +++ b/lib/pure/mersenne.nim @@ -1,3 +1,12 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2015 Nim Contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + type MersenneTwister* = object mt: array[0..623, uint32] @@ -5,29 +14,31 @@ type {.deprecated: [TMersenneTwister: MersenneTwister].} -proc newMersenneTwister*(seed: int): MersenneTwister = +proc newMersenneTwister*(seed: uint32): MersenneTwister = result.index = 0 - result.mt[0]= uint32(seed) + result.mt[0] = seed for i in 1..623'u32: - result.mt[i]= (0x6c078965'u32 * (result.mt[i-1] xor (result.mt[i-1] shr 30'u32)) + i) + result.mt[i] = (0x6c078965'u32 * (result.mt[i-1] xor (result.mt[i-1] shr 30'u32)) + i) proc generateNumbers(m: var MersenneTwister) = for i in 0..623: - var y = (m.mt[i] and 0x80000000'u32) + (m.mt[(i+1) mod 624] and 0x7fffffff'u32) + var y = (m.mt[i] and 0x80000000'u32) + + (m.mt[(i+1) mod 624] and 0x7fffffff'u32) m.mt[i] = m.mt[(i+397) mod 624] xor uint32(y shr 1'u32) if (y mod 2'u32) != 0: - m.mt[i] = m.mt[i] xor 0x9908b0df'u32 + m.mt[i] = m.mt[i] xor 0x9908b0df'u32 -proc getNum*(m: var MersenneTwister): int = +proc getNum*(m: var MersenneTwister): uint32 = + ## Returns the next pseudo random number ranging from 0 to high(uint32) if m.index == 0: generateNumbers(m) - var y = m.mt[m.index] - y = y xor (y shr 11'u32) - y = y xor ((7'u32 shl y) and 0x9d2c5680'u32) - y = y xor ((15'u32 shl y) and 0xefc60000'u32) - y = y xor (y shr 18'u32) - m.index = (m.index+1) mod 624 - return int(y) + result = m.mt[m.index] + m.index = (m.index + 1) mod m.mt.len + + result = result xor (result shr 11'u32) + result = result xor ((7'u32 shl result) and 0x9d2c5680'u32) + result = result xor ((15'u32 shl result) and 0xefc60000'u32) + result = result xor (result shr 18'u32) # Test when not defined(testing) and isMainModule: diff --git a/lib/pure/nativesockets.nim b/lib/pure/nativesockets.nim index 043d6d80a..4526afa49 100644 --- a/lib/pure/nativesockets.nim +++ b/lib/pure/nativesockets.nim @@ -27,7 +27,7 @@ else: import posix export fcntl, F_GETFL, O_NONBLOCK, F_SETFL, EAGAIN, EWOULDBLOCK, MSG_NOSIGNAL, EINTR, EINPROGRESS, ECONNRESET, EPIPE, ENETRESET - export Sockaddr_storage + export Sockaddr_storage, Sockaddr_un, Sockaddr_un_path_length export SocketHandle, Sockaddr_in, Addrinfo, INADDR_ANY, SockAddr, SockLen, Sockaddr_in6, @@ -38,7 +38,7 @@ export SOL_SOCKET, SOMAXCONN, SO_ACCEPTCONN, SO_BROADCAST, SO_DEBUG, SO_DONTROUTE, - SO_KEEPALIVE, SO_OOBINLINE, SO_REUSEADDR, + SO_KEEPALIVE, SO_OOBINLINE, SO_REUSEADDR, SO_REUSEPORT, MSG_PEEK when defined(macosx) and not defined(nimdoc): @@ -326,8 +326,13 @@ proc getHostByAddr*(ip: string): Hostent {.tags: [ReadIOEffect].} = cint(AF_INET)) if s == nil: raiseOSError(osLastError()) else: - var s = posix.gethostbyaddr(addr(myaddr), sizeof(myaddr).Socklen, - cint(posix.AF_INET)) + var s = + when defined(android4): + posix.gethostbyaddr(cast[cstring](addr(myaddr)), sizeof(myaddr).cint, + cint(posix.AF_INET)) + else: + posix.gethostbyaddr(addr(myaddr), sizeof(myaddr).Socklen, + cint(posix.AF_INET)) if s == nil: raiseOSError(osLastError(), $hstrerror(h_errno)) diff --git a/lib/pure/net.nim b/lib/pure/net.nim index 330682ca9..bd208761b 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -8,19 +8,77 @@ # ## This module implements a high-level cross-platform sockets interface. +## The procedures implemented in this module are primarily for blocking sockets. +## For asynchronous non-blocking sockets use the ``asyncnet`` module together +## with the ``asyncdispatch`` module. +## +## The first thing you will always need to do in order to start using sockets, +## is to create a new instance of the ``Socket`` type using the ``newSocket`` +## procedure. +## +## SSL +## ==== +## +## In order to use the SSL procedures defined in this module, you will need to +## compile your application with the ``-d:ssl`` flag. +## +## Examples +## ======== +## +## Connecting to a server +## ---------------------- +## +## After you create a socket with the ``newSocket`` procedure, you can easily +## connect it to a server running at a known hostname (or IP address) and port. +## To do so over TCP, use the example below. +## +## .. code-block:: Nim +## var socket = newSocket() +## socket.connect("google.com", Port(80)) +## +## UDP is a connectionless protocol, so UDP sockets don't have to explicitly +## call the ``connect`` procedure. They can simply start sending data +## immediately. +## +## .. code-block:: Nim +## var socket = newSocket() +## socket.sendTo("192.168.0.1", Port(27960), "status\n") +## +## Creating a server +## ----------------- +## +## After you create a socket with the ``newSocket`` procedure, you can create a +## TCP server by calling the ``bindAddr`` and ``listen`` procedures. +## +## .. code-block:: Nim +## var socket = newSocket() +## socket.bindAddr(Port(1234)) +## socket.listen() +## +## You can then begin accepting connections using the ``accept`` procedure. +## +## .. code-block:: Nim +## var client = newSocket() +## var address = "" +## while true: +## socket.acceptAddr(client, address) +## echo("Client connected from: ", address) +## {.deadCodeElim: on.} -import nativesockets, os, strutils, parseutils, times +import nativesockets, os, strutils, parseutils, times, sets export Port, `$`, `==` +export Domain, SockType, Protocol const useWinVersion = defined(Windows) or defined(nimdoc) +const defineSsl = defined(ssl) or defined(nimdoc) -when defined(ssl): +when defineSsl: import openssl # Note: The enumerations are mapped to Window's constants. -when defined(ssl): +when defineSsl: type SslError* = object of Exception @@ -30,7 +88,10 @@ when defined(ssl): SslProtVersion* = enum protSSLv2, protSSLv3, protTLSv1, protSSLv23 - SslContext* = distinct SslCtx + SslContext* = ref object + context*: SslCtx + extraInternalIndex: int + referencedData: HashSet[int] SslAcceptResult* = enum AcceptNoClient = 0, AcceptNoHandshake, AcceptSuccess @@ -38,6 +99,10 @@ when defined(ssl): SslHandshakeType* = enum handshakeAsClient, handshakeAsServer + SslClientGetPskFunc* = proc(hint: string): tuple[identity: string, psk: string] + + SslServerGetPskFunc* = proc(identity: string): string + {.deprecated: [ESSL: SSLError, TSSLCVerifyMode: SSLCVerifyMode, TSSLProtVersion: SSLProtVersion, PSSLContext: SSLContext, TSSLAcceptResult: SSLAcceptResult].} @@ -54,7 +119,7 @@ type currPos: int # current index in buffer bufLen: int # current length of buffer of false: nil - when defined(ssl): + when defineSsl: case isSsl: bool of true: sslHandle: SSLPtr @@ -72,7 +137,7 @@ type SOBool* = enum ## Boolean socket options. OptAcceptConn, OptBroadcast, OptDebug, OptDontRoute, OptKeepAlive, - OptOOBInline, OptReuseAddr + OptOOBInline, OptReuseAddr, OptReusePort ReadLineResult* = enum ## result for readLineAsync ReadFullLine, ReadPartialLine, ReadDisconnected, ReadNone @@ -140,6 +205,10 @@ proc newSocket*(fd: SocketHandle, domain: Domain = AF_INET, if buffered: result.currPos = 0 + # Set SO_NOSIGPIPE on OS X. + when defined(macosx): + setSockOptInt(fd, SOL_SOCKET, SO_NOSIGPIPE, 1) + proc newSocket*(domain, sockType, protocol: cint, buffered = true): Socket = ## Creates a new socket. ## @@ -160,13 +229,18 @@ proc newSocket*(domain: Domain = AF_INET, sockType: SockType = SOCK_STREAM, raiseOSError(osLastError()) result = newSocket(fd, domain, sockType, protocol, buffered) -when defined(ssl): +when defineSsl: CRYPTO_malloc_init() SslLibraryInit() SslLoadErrorStrings() ErrLoadBioStrings() OpenSSL_add_all_algorithms() + type + SslContextExtraInternal = ref object of RootRef + serverGetPskFunc: SslServerGetPskFunc + clientGetPskFunc: SslClientGetPskFunc + proc raiseSSLError*(s = "") = ## Raises a new SSL error. if s != "": @@ -179,6 +253,34 @@ when defined(ssl): var errStr = ErrErrorString(err, nil) raise newException(SSLError, $errStr) + proc getExtraDataIndex*(ctx: SSLContext): int = + ## Retrieves unique index for storing extra data in SSLContext. + result = SSL_CTX_get_ex_new_index(0, nil, nil, nil, nil).int + if result < 0: + raiseSSLError() + + proc getExtraData*(ctx: SSLContext, index: int): RootRef = + ## Retrieves arbitrary data stored inside SSLContext. + if index notin ctx.referencedData: + raise newException(IndexError, "No data with that index.") + let res = ctx.context.SSL_CTX_get_ex_data(index.cint) + if cast[int](res) == 0: + raiseSSLError() + return cast[RootRef](res) + + proc setExtraData*(ctx: SSLContext, index: int, data: RootRef) = + ## Stores arbitrary data inside SSLContext. The unique `index` + ## should be retrieved using getSslContextExtraDataIndex. + if index in ctx.referencedData: + GC_unref(getExtraData(ctx, index)) + + if ctx.context.SSL_CTX_set_ex_data(index.cint, cast[pointer](data)) == -1: + raiseSSLError() + + if index notin ctx.referencedData: + ctx.referencedData.incl(index) + GC_ref(data) + # http://simplestcodings.blogspot.co.uk/2010/08/secure-server-client-using-openssl-in-c.html proc loadCertificates(ctx: SSL_CTX, certFile, keyFile: string) = if certFile != "" and not existsFile(certFile): @@ -201,7 +303,7 @@ when defined(ssl): raiseSSLError("Verification of private key file failed.") proc newContext*(protVersion = protSSLv23, verifyMode = CVerifyPeer, - certFile = "", keyFile = ""): SSLContext = + certFile = "", keyFile = "", cipherList = "ALL"): SSLContext = ## Creates an SSL context. ## ## Protocol version specifies the protocol to use. SSLv2, SSLv3, TLSv1 @@ -222,13 +324,13 @@ when defined(ssl): of protSSLv23: newCTX = SSL_CTX_new(SSLv23_method()) # SSlv2,3 and TLS1 support. of protSSLv2: - raiseSslError("SSLv2 is no longer secure and has been deprecated, use protSSLv3") + raiseSslError("SSLv2 is no longer secure and has been deprecated, use protSSLv23") of protSSLv3: - newCTX = SSL_CTX_new(SSLv3_method()) + raiseSslError("SSLv3 is no longer secure and has been deprecated, use protSSLv23") of protTLSv1: newCTX = SSL_CTX_new(TLSv1_method()) - if newCTX.SSLCTXSetCipherList("ALL") != 1: + if newCTX.SSLCTXSetCipherList(cipherList) != 1: raiseSSLError() case verifyMode of CVerifyPeer: @@ -240,7 +342,87 @@ when defined(ssl): discard newCTX.SSLCTXSetMode(SSL_MODE_AUTO_RETRY) newCTX.loadCertificates(certFile, keyFile) - return SSLContext(newCTX) + + result = SSLContext(context: newCTX, extraInternalIndex: 0, + referencedData: initSet[int]()) + result.extraInternalIndex = getExtraDataIndex(result) + + let extraInternal = new(SslContextExtraInternal) + result.setExtraData(result.extraInternalIndex, extraInternal) + + proc getExtraInternal(ctx: SSLContext): SslContextExtraInternal = + return SslContextExtraInternal(ctx.getExtraData(ctx.extraInternalIndex)) + + proc destroyContext*(ctx: SSLContext) = + ## Free memory referenced by SSLContext. + + # We assume here that OpenSSL's internal indexes increase by 1 each time. + # That means we can assume that the next internal index is the length of + # extra data indexes. + for i in ctx.referencedData: + GC_unref(getExtraData(ctx, i).RootRef) + ctx.context.SSL_CTX_free() + + proc `pskIdentityHint=`*(ctx: SSLContext, hint: string) = + ## Sets the identity hint passed to server. + ## + ## Only used in PSK ciphersuites. + if ctx.context.SSL_CTX_use_psk_identity_hint(hint) <= 0: + raiseSSLError() + + proc clientGetPskFunc*(ctx: SSLContext): SslClientGetPskFunc = + return ctx.getExtraInternal().clientGetPskFunc + + proc pskClientCallback(ssl: SslPtr; hint: cstring; identity: cstring; max_identity_len: cuint; psk: ptr cuchar; + max_psk_len: cuint): cuint {.cdecl.} = + let ctx = SSLContext(context: ssl.SSL_get_SSL_CTX, extraInternalIndex: 0) + let hintString = if hint == nil: nil else: $hint + let (identityString, pskString) = (ctx.clientGetPskFunc)(hintString) + if psk.len.cuint > max_psk_len: + return 0 + if identityString.len.cuint >= max_identity_len: + return 0 + + copyMem(identity, identityString.cstring, pskString.len + 1) # with the last zero byte + copyMem(psk, pskString.cstring, pskString.len) + + return pskString.len.cuint + + proc `clientGetPskFunc=`*(ctx: SSLContext, fun: SslClientGetPskFunc) = + ## Sets function that returns the client identity and the PSK based on identity + ## hint from the server. + ## + ## Only used in PSK ciphersuites. + ctx.getExtraInternal().clientGetPskFunc = fun + assert ctx.extraInternalIndex == 0, + "The pskClientCallback assumes the extraInternalIndex is 0" + ctx.context.SSL_CTX_set_psk_client_callback( + if fun == nil: nil else: pskClientCallback) + + proc serverGetPskFunc*(ctx: SSLContext): SslServerGetPskFunc = + return ctx.getExtraInternal().serverGetPskFunc + + proc pskServerCallback(ssl: SslCtx; identity: cstring; psk: ptr cuchar; max_psk_len: cint): cuint {.cdecl.} = + let ctx = SSLContext(context: ssl.SSL_get_SSL_CTX, extraInternalIndex: 0) + let pskString = (ctx.serverGetPskFunc)($identity) + if psk.len.cint > max_psk_len: + return 0 + copyMem(psk, pskString.cstring, pskString.len) + + return pskString.len.cuint + + proc `serverGetPskFunc=`*(ctx: SSLContext, fun: SslServerGetPskFunc) = + ## Sets function that returns PSK based on the client identity. + ## + ## Only used in PSK ciphersuites. + ctx.getExtraInternal().serverGetPskFunc = fun + ctx.context.SSL_CTX_set_psk_server_callback(if fun == nil: nil + else: pskServerCallback) + + proc getPskIdentity*(socket: Socket): string = + ## Gets the PSK identity provided by the client. + assert socket.isSSL + return $(socket.sslHandle.SSL_get_psk_identity) proc wrapSocket*(ctx: SSLContext, socket: Socket) = ## Wraps a socket in an SSL context. This function effectively turns @@ -255,7 +437,7 @@ when defined(ssl): assert (not socket.isSSL) socket.isSSL = true socket.sslContext = ctx - socket.sslHandle = SSLNew(SSLCTX(socket.sslContext)) + socket.sslHandle = SSLNew(socket.sslContext.context) socket.sslNoHandshake = false socket.sslHasPeekChar = false if socket.sslHandle == nil: @@ -301,7 +483,7 @@ proc socketError*(socket: Socket, err: int = -1, async = false, ## error was caused by no data being available to be read. ## ## If ``err`` is not lower than 0 no exception will be raised. - when defined(ssl): + when defineSsl: if socket.isSSL: if err <= 0: var ret = SSLGetError(socket.sslHandle, err.cint) @@ -334,7 +516,7 @@ proc socketError*(socket: Socket, err: int = -1, async = false, raiseSSLError() else: raiseSSLError("Unknown Error") - if err == -1 and not (when defined(ssl): socket.isSSL else: false): + if err == -1 and not (when defineSsl: socket.isSSL else: false): var lastE = if lastError.int == -1: getSocketError(socket) else: lastError if async: when useWinVersion: @@ -414,7 +596,7 @@ proc acceptAddr*(server: Socket, client: var Socket, address: var string, client.isBuffered = server.isBuffered # Handle SSL. - when defined(ssl): + when defineSsl: if server.isSSL: # We must wrap the client sock in a ssl context. @@ -425,7 +607,7 @@ proc acceptAddr*(server: Socket, client: var Socket, address: var string, # Client socket is set above. address = $inet_ntoa(sockAddress.sin_addr) -when false: #defined(ssl): +when false: #defineSsl: proc acceptAddrSSL*(server: Socket, client: var Socket, address: var string): SSLAcceptResult {. tags: [ReadIOEffect].} = @@ -444,7 +626,7 @@ when false: #defined(ssl): ## ``AcceptNoClient`` will be returned when no client is currently attempting ## to connect. template doHandshake(): stmt = - when defined(ssl): + when defineSsl: if server.isSSL: client.setBlocking(false) # We must wrap the client sock in a ssl context. @@ -495,7 +677,7 @@ proc accept*(server: Socket, client: var Socket, proc close*(socket: Socket) = ## Closes a socket. try: - when defined(ssl): + when defineSsl: if socket.isSSL: ErrClearError() # As we are closing the underlying socket immediately afterwards, @@ -522,6 +704,7 @@ proc toCInt*(opt: SOBool): cint = of OptKeepAlive: SO_KEEPALIVE of OptOOBInline: SO_OOBINLINE of OptReuseAddr: SO_REUSEADDR + of OptReusePort: SO_REUSEPORT proc getSockOpt*(socket: Socket, opt: SOBool, level = SOL_SOCKET): bool {. tags: [ReadIOEffect].} = @@ -547,8 +730,35 @@ proc setSockOpt*(socket: Socket, opt: SOBool, value: bool, level = SOL_SOCKET) { var valuei = cint(if value: 1 else: 0) setSockOptInt(socket.fd, cint(level), toCInt(opt), valuei) +when defined(posix) and not defined(nimdoc): + proc makeUnixAddr(path: string): Sockaddr_un = + result.sun_family = AF_UNIX.toInt + if path.len >= Sockaddr_un_path_length: + raise newException(ValueError, "socket path too long") + copyMem(addr result.sun_path, path.cstring, path.len + 1) + +when defined(posix): + proc connectUnix*(socket: Socket, path: string) = + ## Connects to Unix socket on `path`. + ## This only works on Unix-style systems: Mac OS X, BSD and Linux + when not defined(nimdoc): + var socketAddr = makeUnixAddr(path) + if socket.fd.connect(cast[ptr SockAddr](addr socketAddr), + sizeof(socketAddr).Socklen) != 0'i32: + raiseOSError(osLastError()) + + proc bindUnix*(socket: Socket, path: string) = + ## Binds Unix socket to `path`. + ## This only works on Unix-style systems: Mac OS X, BSD and Linux + when not defined(nimdoc): + var socketAddr = makeUnixAddr(path) + if socket.fd.bindAddr(cast[ptr SockAddr](addr socketAddr), + sizeof(socketAddr).Socklen) != 0'i32: + raiseOSError(osLastError()) + when defined(ssl): - proc handshake*(socket: Socket): bool {.tags: [ReadIOEffect, WriteIOEffect].} = + proc handshake*(socket: Socket): bool + {.tags: [ReadIOEffect, WriteIOEffect], deprecated.} = ## This proc needs to be called on a socket after it connects. This is ## only applicable when using ``connectAsync``. ## This proc performs the SSL handshake. @@ -557,6 +767,8 @@ when defined(ssl): ## ``True`` whenever handshake completed successfully. ## ## A ESSL error is raised on any other errors. + ## + ## **Note:** This procedure is deprecated since version 0.14.0. result = true if socket.isSSL: var ret = SSLConnect(socket.sslHandle) @@ -594,7 +806,7 @@ proc hasDataBuffered*(s: Socket): bool = if s.isBuffered: result = s.bufLen > 0 and s.currPos != s.bufLen - when defined(ssl): + when defineSsl: if s.isSSL and not result: result = s.sslHasPeekChar @@ -608,7 +820,7 @@ proc select(readfd: Socket, timeout = 500): int = proc readIntoBuf(socket: Socket, flags: int32): int = result = 0 - when defined(ssl): + when defineSsl: if socket.isSSL: result = SSLRead(socket.sslHandle, addr(socket.buffer), int(socket.buffer.high)) else: @@ -658,7 +870,7 @@ proc recv*(socket: Socket, data: pointer, size: int): int {.tags: [ReadIOEffect] result = read else: - when defined(ssl): + when defineSsl: if socket.isSSL: if socket.sslHasPeekChar: copyMem(data, addr(socket.sslPeekChar), 1) @@ -696,7 +908,7 @@ proc waitFor(socket: Socket, waited: var float, timeout, size: int, if timeout - int(waited * 1000.0) < 1: raise newException(TimeoutError, "Call to '" & funcName & "' timed out.") - when defined(ssl): + when defineSsl: if socket.isSSL: if socket.hasDataBuffered: # sslPeekChar is present. @@ -764,7 +976,7 @@ proc peekChar(socket: Socket, c: var char): int {.tags: [ReadIOEffect].} = c = socket.buffer[socket.currPos] else: - when defined(ssl): + when defineSsl: if socket.isSSL: if not socket.sslHasPeekChar: result = SSLRead(socket.sslHandle, addr(socket.sslPeekChar), 1) @@ -792,11 +1004,11 @@ proc readLine*(socket: Socket, line: var TaintedString, timeout = -1, ## ## **Warning**: Only the ``SafeDisconn`` flag is currently supported. - template addNLIfEmpty(): stmt = + template addNLIfEmpty() = if line.len == 0: line.string.add("\c\L") - template raiseSockError(): stmt {.dirty, immediate.} = + template raiseSockError() {.dirty.} = let lastError = getSocketError(socket) if flags.isDisconnectionError(lastError): setLen(line.string, 0); return socket.socketError(n, lastError = lastError) @@ -872,7 +1084,7 @@ proc send*(socket: Socket, data: pointer, size: int): int {. ## ## **Note**: This is a low-level version of ``send``. You likely should use ## the version below. - when defined(ssl): + when defineSsl: if socket.isSSL: return SSLWrite(socket.sslHandle, cast[cstring](data), size) @@ -943,7 +1155,7 @@ proc sendTo*(socket: Socket, address: string, port: Port, proc isSsl*(socket: Socket): bool = ## Determines whether ``socket`` is a SSL socket. - when defined(ssl): + when defineSsl: result = socket.isSSL else: result = false @@ -1253,7 +1465,7 @@ proc connect*(socket: Socket, address: string, dealloc(aiList) if not success: raiseOSError(lastError) - when defined(ssl): + when defineSsl: if socket.isSSL: # RFC3546 for SNI specifies that IP addresses are not allowed. if not isIpAddress(address): @@ -1314,8 +1526,10 @@ proc connect*(socket: Socket, address: string, port = Port(0), if selectWrite(s, timeout) != 1: raise newException(TimeoutError, "Call to 'connect' timed out.") else: - when defined(ssl): + when defineSsl and not defined(nimdoc): if socket.isSSL: socket.fd.setBlocking(true) + {.warning[Deprecated]: off.} doAssert socket.handshake() + {.warning[Deprecated]: on.} socket.fd.setBlocking(true) diff --git a/lib/pure/nimprof.nim b/lib/pure/nimprof.nim index 5a7deaab0..4289eb049 100644 --- a/lib/pure/nimprof.nim +++ b/lib/pure/nimprof.nim @@ -27,19 +27,22 @@ const tickCountCorrection = 50_000 when not declared(system.StackTrace): - type StackTrace = array [0..20, cstring] + type StackTrace = object + lines: array[0..20, cstring] + files: array[0..20, cstring] {.deprecated: [TStackTrace: StackTrace].} + proc `[]`*(st: StackTrace, i: int): cstring = st.lines[i] # We use a simple hash table of bounded size to keep track of the stack traces: type ProfileEntry = object total: int st: StackTrace - ProfileData = array [0..64*1024-1, ptr ProfileEntry] + ProfileData = array[0..64*1024-1, ptr ProfileEntry] {.deprecated: [TProfileEntry: ProfileEntry, TProfileData: ProfileData].} proc `==`(a, b: StackTrace): bool = - for i in 0 .. high(a): + for i in 0 .. high(a.lines): if a[i] != b[i]: return false result = true @@ -72,7 +75,7 @@ proc hookAux(st: StackTrace, costs: int) = # this is quite performance sensitive! when withThreads: acquire profilingLock inc totalCalls - var last = high(st) + var last = high(st.lines) while last > 0 and isNil(st[last]): dec last var h = hash(pointer(st[last])) and high(profileData) @@ -178,7 +181,7 @@ proc writeProfile() {.noconv.} = var perProc = initCountTable[string]() for i in 0..entries-1: var dups = initSet[string]() - for ii in 0..high(StackTrace): + for ii in 0..high(StackTrace.lines): let procname = profileData[i].st[ii] if isNil(procname): break let p = $procname @@ -193,10 +196,11 @@ proc writeProfile() {.noconv.} = writeLine(f, "Entry: ", i+1, "/", entries, " Calls: ", profileData[i].total // totalCalls, " [sum: ", sum, "; ", sum // totalCalls, "]") - for ii in 0..high(StackTrace): + for ii in 0..high(StackTrace.lines): let procname = profileData[i].st[ii] + let filename = profileData[i].st.files[ii] if isNil(procname): break - writeLine(f, " ", procname, " ", perProc[$procname] // totalCalls) + writeLine(f, " ", $filename & ": " & $procname, " ", perProc[$procname] // totalCalls) close(f) echo "... done" else: diff --git a/lib/pure/oids.nim b/lib/pure/oids.nim index fca10dab6..e4c97b260 100644 --- a/lib/pure/oids.nim +++ b/lib/pure/oids.nim @@ -74,8 +74,7 @@ proc genOid*(): Oid = var t = gettime(nil) - var i = int32(incr) - atomicInc(incr) + var i = int32(atomicInc(incr)) if fuzz == 0: # racy, but fine semantically: diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 470559e17..1e474f4d4 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -26,7 +26,6 @@ elif defined(posix): else: {.error: "OS module not ported to your operating system!".} -include "system/ansi_c" include ospaths when defined(posix): @@ -37,6 +36,23 @@ when defined(posix): var pathMax {.importc: "PATH_MAX", header: "<stdlib.h>".}: cint +proc c_remove(filename: cstring): cint {. + importc: "remove", header: "<stdio.h>".} +proc c_rename(oldname, newname: cstring): cint {. + importc: "rename", header: "<stdio.h>".} +proc c_system(cmd: cstring): cint {. + importc: "system", header: "<stdlib.h>".} +proc c_strerror(errnum: cint): cstring {. + importc: "strerror", header: "<string.h>".} +proc c_strlen(a: cstring): cint {. + importc: "strlen", header: "<string.h>", noSideEffect.} +proc c_getenv(env: cstring): cstring {. + importc: "getenv", header: "<stdlib.h>".} +proc c_putenv(env: cstring): cint {. + importc: "putenv", header: "<stdlib.h>".} + +var errno {.importc, header: "<errno.h>".}: cint + proc osErrorMsg*(): string {.rtl, extern: "nos$1", deprecated.} = ## Retrieves the operating system's error flag, ``errno``. ## On Windows ``GetLastError`` is checked before ``errno``. @@ -50,18 +66,18 @@ proc osErrorMsg*(): string {.rtl, extern: "nos$1", deprecated.} = if err != 0'i32: when useWinUnicode: var msgbuf: WideCString - if formatMessageW(0x00000100 or 0x00001000 or 0x00000200, + if formatMessageW(0x00000100 or 0x00001000 or 0x00000200 or 0x000000FF, nil, err, 0, addr(msgbuf), 0, nil) != 0'i32: result = $msgbuf if msgbuf != nil: localFree(cast[pointer](msgbuf)) else: var msgbuf: cstring - if formatMessageA(0x00000100 or 0x00001000 or 0x00000200, + if formatMessageA(0x00000100 or 0x00001000 or 0x00000200 or 0x000000FF, nil, err, 0, addr(msgbuf), 0, nil) != 0'i32: result = $msgbuf if msgbuf != nil: localFree(msgbuf) if errno != 0'i32: - result = $os.strerror(errno) + result = $os.c_strerror(errno) {.push warning[deprecated]: off.} proc raiseOSError*(msg: string = "") {.noinline, rtl, extern: "nos$1", @@ -114,7 +130,7 @@ proc osErrorMsg*(errorCode: OSErrorCode): string = if msgbuf != nil: localFree(msgbuf) else: if errorCode != OSErrorCode(0'i32): - result = $os.strerror(errorCode.int32) + result = $os.c_strerror(errorCode.int32) proc raiseOSError*(errorCode: OSErrorCode; additionalInfo = "") {.noinline.} = ## Raises an ``OSError`` exception. The ``errorCode`` will determine the @@ -129,7 +145,7 @@ proc raiseOSError*(errorCode: OSErrorCode; additionalInfo = "") {.noinline.} = if additionalInfo.len == 0: e.msg = osErrorMsg(errorCode) else: - e.msg = additionalInfo & " " & osErrorMsg(errorCode) + e.msg = osErrorMsg(errorCode) & "\nAdditional info: " & additionalInfo if e.msg == "": e.msg = "unknown OS error" raise e @@ -157,24 +173,24 @@ proc osLastError*(): OSErrorCode = when defined(windows): when useWinUnicode: - template wrapUnary(varname, winApiProc, arg: expr) {.immediate.} = + template wrapUnary(varname, winApiProc, arg: untyped) = var varname = winApiProc(newWideCString(arg)) - template wrapBinary(varname, winApiProc, arg, arg2: expr) {.immediate.} = + template wrapBinary(varname, winApiProc, arg, arg2: untyped) = var varname = winApiProc(newWideCString(arg), arg2) proc findFirstFile(a: string, b: var WIN32_FIND_DATA): Handle = result = findFirstFileW(newWideCString(a), b) - template findNextFile(a, b: expr): expr = findNextFileW(a, b) - template getCommandLine(): expr = getCommandLineW() + template findNextFile(a, b: untyped): untyped = findNextFileW(a, b) + template getCommandLine(): untyped = getCommandLineW() - template getFilename(f: expr): expr = + template getFilename(f: untyped): untyped = $cast[WideCString](addr(f.cFilename[0])) else: - template findFirstFile(a, b: expr): expr = findFirstFileA(a, b) - template findNextFile(a, b: expr): expr = findNextFileA(a, b) - template getCommandLine(): expr = getCommandLineA() + template findFirstFile(a, b: untyped): untyped = findFirstFileA(a, b) + template findNextFile(a, b: untyped): untyped = findNextFileA(a, b) + template getCommandLine(): untyped = getCommandLineA() - template getFilename(f: expr): expr = $f.cFilename + template getFilename(f: untyped): untyped = $f.cFilename proc skipFindData(f: WIN32_FIND_DATA): bool {.inline.} = # Note - takes advantage of null delimiter in the cstring @@ -320,7 +336,7 @@ proc setCurrentDir*(newDir: string) {.inline, tags: [].} = proc expandFilename*(filename: string): string {.rtl, extern: "nos$1", tags: [ReadDirEffect].} = - ## Returns the full path of `filename`, raises OSError in case of an error. + ## Returns the full (`absolute`:idx:) path of the file `filename`, raises OSError in case of an error. when defined(windows): const bufsize = 3072'i32 when useWinUnicode: @@ -762,12 +778,26 @@ iterator envPairs*(): tuple[key, value: TaintedString] {.tags: [ReadEnvEffect].} yield (TaintedString(substr(environment[i], 0, p-1)), TaintedString(substr(environment[i], p+1))) -iterator walkFiles*(pattern: string): string {.tags: [ReadDirEffect].} = - ## Iterate over all the files that match the `pattern`. On POSIX this uses - ## the `glob`:idx: call. - ## - ## `pattern` is OS dependent, but at least the "\*.ext" - ## notation is supported. +# Templates for filtering directories and files +when defined(windows): + template isDir(f: WIN32_FIND_DATA): bool = + (f.dwFileAttributes and FILE_ATTRIBUTE_DIRECTORY) != 0'i32 + template isFile(f: WIN32_FIND_DATA): bool = + not isDir(f) +else: + template isDir(f: string): bool = + dirExists(f) + template isFile(f: string): bool = + fileExists(f) + +template defaultWalkFilter(item): bool = + ## Walk filter used to return true on both + ## files and directories + true + +template walkCommon(pattern: string, filter) = + ## Common code for getting the files and directories with the + ## specified `pattern` when defined(windows): var f: WIN32_FIND_DATA @@ -776,8 +806,7 @@ iterator walkFiles*(pattern: string): string {.tags: [ReadDirEffect].} = if res != -1: defer: findClose(res) while true: - if not skipFindData(f) and - (f.dwFileAttributes and FILE_ATTRIBUTE_DIRECTORY) == 0'i32: + if not skipFindData(f) and filter(f): # Windows bug/gotcha: 't*.nim' matches 'tfoo.nims' -.- so we check # that the file extensions have the same length ... let ff = getFilename(f) @@ -799,7 +828,33 @@ iterator walkFiles*(pattern: string): string {.tags: [ReadDirEffect].} = if res == 0: for i in 0.. f.gl_pathc - 1: assert(f.gl_pathv[i] != nil) - yield $f.gl_pathv[i] + let path = $f.gl_pathv[i] + if filter(path): + yield path + +iterator walkPattern*(pattern: string): string {.tags: [ReadDirEffect].} = + ## Iterate over all the files and directories that match the `pattern`. + ## On POSIX this uses the `glob`:idx: call. + ## + ## `pattern` is OS dependent, but at least the "\*.ext" + ## notation is supported. + walkCommon(pattern, defaultWalkFilter) + +iterator walkFiles*(pattern: string): string {.tags: [ReadDirEffect].} = + ## Iterate over all the files that match the `pattern`. On POSIX this uses + ## the `glob`:idx: call. + ## + ## `pattern` is OS dependent, but at least the "\*.ext" + ## notation is supported. + walkCommon(pattern, isFile) + +iterator walkDirs*(pattern: string): string {.tags: [ReadDirEffect].} = + ## Iterate over all the directories that match the `pattern`. + ## On POSIX this uses the `glob`:idx: call. + ## + ## `pattern` is OS dependent, but at least the "\*.ext" + ## notation is supported. + walkCommon(pattern, isDir) type PathComponent* = enum ## Enumeration specifying a path component. @@ -1067,7 +1122,7 @@ proc parseCmdLine*(c: string): seq[string] {. while true: setLen(a, 0) # eat all delimiting whitespace - while c[i] == ' ' or c[i] == '\t' or c [i] == '\l' or c [i] == '\r' : inc(i) + while c[i] == ' ' or c[i] == '\t' or c[i] == '\l' or c[i] == '\r' : inc(i) when defined(windows): # parse a single argument according to the above rules: if c[i] == '\0': break @@ -1447,13 +1502,13 @@ type lastWriteTime*: Time # Time file was last modified/written to. creationTime*: Time # Time file was created. Not supported on all systems! -template rawToFormalFileInfo(rawInfo, formalInfo): expr = +template rawToFormalFileInfo(rawInfo, formalInfo): untyped = ## Transforms the native file info structure into the one nim uses. ## 'rawInfo' is either a 'TBY_HANDLE_FILE_INFORMATION' structure on Windows, ## or a 'Stat' structure on posix when defined(Windows): - template toTime(e): expr = winTimeToUnixTime(rdFileTime(e)) - template merge(a, b): expr = a or (b shl 32) + template toTime(e: FILETIME): untyped {.gensym.} = winTimeToUnixTime(rdFileTime(e)) # local templates default to bind semantics + template merge(a, b): untyped = a or (b shl 32) formalInfo.id.device = rawInfo.dwVolumeSerialNumber formalInfo.id.file = merge(rawInfo.nFileIndexLow, rawInfo.nFileIndexHigh) formalInfo.size = merge(rawInfo.nFileSizeLow, rawInfo.nFileSizeHigh) @@ -1478,7 +1533,7 @@ template rawToFormalFileInfo(rawInfo, formalInfo): expr = else: - template checkAndIncludeMode(rawMode, formalMode: expr) = + template checkAndIncludeMode(rawMode, formalMode: untyped) = if (rawInfo.st_mode and rawMode) != 0'i32: formalInfo.permissions.incl(formalMode) formalInfo.id = (rawInfo.st_dev, rawInfo.st_ino) diff --git a/lib/pure/ospaths.nim b/lib/pure/ospaths.nim index 9fc816f2f..56671ee62 100644 --- a/lib/pure/ospaths.nim +++ b/lib/pure/ospaths.nim @@ -10,7 +10,7 @@ # Included by the ``os`` module but a module in its own right for NimScript # support. -when isMainModule: +when not declared(os): {.pragma: rtl.} import strutils @@ -556,12 +556,20 @@ when declared(getEnv) or defined(nimscript): yield substr(s, first, last-1) inc(last) - proc findExe*(exe: string): string {. + when not defined(windows) and declared(os): + proc checkSymlink(path: string): bool = + var rawInfo: Stat + if lstat(path, rawInfo) < 0'i32: result = false + else: result = S_ISLNK(rawInfo.st_mode) + + proc findExe*(exe: string, followSymlinks: bool = true): string {. tags: [ReadDirEffect, ReadEnvEffect, ReadIOEffect].} = ## Searches for `exe` in the current working directory and then ## in directories listed in the ``PATH`` environment variable. ## Returns "" if the `exe` cannot be found. On DOS-like platforms, `exe` ## is added the `ExeExt <#ExeExt>`_ file extension if it has none. + ## If the system supports symlinks it also resolves them until it + ## meets the actual file. This behavior can be disabled if desired. result = addFileExt(exe, ExeExt) if existsFile(result): return var path = string(getEnv("PATH")) @@ -572,7 +580,25 @@ when declared(getEnv) or defined(nimscript): result else: var x = expandTilde(candidate) / result - if existsFile(x): return x + if existsFile(x): + when not defined(windows) and declared(os): + while followSymlinks: # doubles as if here + if x.checkSymlink: + var r = newString(256) + var len = readlink(x, r, 256) + if len < 0: + raiseOSError(osLastError()) + if len > 256: + r = newString(len+1) + len = readlink(x, r, len) + setLen(r, len) + if isAbsolute(r): + x = r + else: + x = parentDir(x) / r + else: + break + return x result = "" when defined(nimscript) or (defined(nimdoc) and not declared(os)): diff --git a/lib/pure/osproc.nim b/lib/pure/osproc.nim index 38b0ed4a3..7378520e3 100644 --- a/lib/pure/osproc.nim +++ b/lib/pure/osproc.nim @@ -89,7 +89,7 @@ proc quoteShellWindows*(s: string): string {.noSideEffect, rtl, extern: "nosp$1" result.add("\"") proc quoteShellPosix*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".} = - ## Quote s, so it can be safely passed to POSIX shell. + ## Quote ``s``, so it can be safely passed to POSIX shell. ## Based on Python's pipes.quote const safeUnixChars = {'%', '+', '-', '.', '/', '_', ':', '=', '@', '0'..'9', 'A'..'Z', 'a'..'z'} @@ -104,7 +104,7 @@ proc quoteShellPosix*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".} return "'" & s.replace("'", "'\"'\"'") & "'" proc quoteShell*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".} = - ## Quote s, so it can be safely passed to shell. + ## Quote ``s``, so it can be safely passed to shell. when defined(Windows): return quoteShellWindows(s) elif defined(posix): @@ -175,7 +175,11 @@ proc startCmd*(command: string, options: set[ProcessOption] = { result = startProcess(command=command, options=options + {poEvalCommand}) proc close*(p: Process) {.rtl, extern: "nosp$1", tags: [].} - ## When the process has finished executing, cleanup related handles + ## When the process has finished executing, cleanup related handles. + ## + ## **Warning:** If the process has not finished executing, this will forcibly + ## terminate the process. Doing so may result in zombie processes and + ## `pty leaks <http://stackoverflow.com/questions/27021641/how-to-fix-request-failed-on-channel-0>`_. proc suspend*(p: Process) {.rtl, extern: "nosp$1", tags: [].} ## Suspends the process `p`. @@ -400,15 +404,16 @@ when defined(Windows) and not defined(useNimRtl): result = cast[cstring](alloc0(res.len+1)) copyMem(result, cstring(res), res.len) - proc buildEnv(env: StringTableRef): cstring = + proc buildEnv(env: StringTableRef): tuple[str: cstring, len: int] = var L = 0 for key, val in pairs(env): inc(L, key.len + val.len + 2) - result = cast[cstring](alloc0(L+2)) + var str = cast[cstring](alloc0(L+2)) L = 0 for key, val in pairs(env): var x = key & "=" & val - copyMem(addr(result[L]), cstring(x), x.len+1) # copy \0 + copyMem(addr(str[L]), cstring(x), x.len+1) # copy \0 inc(L, x.len+1) + (str, L) #proc open_osfhandle(osh: Handle, mode: int): int {. # importc: "_open_osfhandle", header: "<fcntl.h>".} @@ -526,13 +531,15 @@ when defined(Windows) and not defined(useNimRtl): else: cmdl = buildCommandLine(command, args) var wd: cstring = nil - var e: cstring = nil + var e = (str: nil.cstring, len: -1) if len(workingDir) > 0: wd = workingDir if env != nil: e = buildEnv(env) if poEchoCmd in options: echo($cmdl) when useWinUnicode: var tmp = newWideCString(cmdl) - var ee = newWideCString(e) + var ee = + if e.str.isNil: nil + else: newWideCString(e.str, e.len) var wwd = newWideCString(wd) var flags = NORMAL_PRIORITY_CLASS or CREATE_UNICODE_ENVIRONMENT if poDemon in options: flags = flags or CREATE_NO_WINDOW @@ -549,7 +556,7 @@ when defined(Windows) and not defined(useNimRtl): if poStdErrToStdOut notin options: fileClose(si.hStdError) - if e != nil: dealloc(e) + if e.str != nil: dealloc(e.str) if success == 0: if poInteractive in result.options: close(result) const errInvalidParameter = 87.int @@ -721,7 +728,7 @@ elif not defined(useNimRtl): env: StringTableRef = nil, options: set[ProcessOption] = {poStdErrToStdOut}): Process = var - pStdin, pStdout, pStderr: array [0..1, cint] + pStdin, pStdout, pStderr: array[0..1, cint] new(result) result.options = options result.exitCode = -3 # for ``waitForExit`` @@ -875,8 +882,9 @@ elif not defined(useNimRtl): var error: cint let sizeRead = read(data.pErrorPipe[readIdx], addr error, sizeof(error)) if sizeRead == sizeof(error): - raiseOSError("Could not find command: '$1'. OS error: $2" % - [$data.sysCommand, $strerror(error)]) + raiseOSError(osLastError(), + "Could not find command: '$1'. OS error: $2" % + [$data.sysCommand, $strerror(error)]) return pid @@ -967,16 +975,168 @@ elif not defined(useNimRtl): if kill(p.id, SIGKILL) != 0'i32: raiseOsError(osLastError()) - proc waitForExit(p: Process, timeout: int = -1): int = - #if waitPid(p.id, p.exitCode, 0) == int(p.id): - # ``waitPid`` fails if the process is not running anymore. But then - # ``running`` probably set ``p.exitCode`` for us. Since ``p.exitCode`` is - # initialized with -3, wrong success exit codes are prevented. - if p.exitCode != -3: return p.exitCode - if waitpid(p.id, p.exitCode, 0) < 0: - p.exitCode = -3 - raiseOSError(osLastError()) - result = int(p.exitCode) shr 8 + when defined(macosx) or defined(freebsd) or defined(netbsd) or + defined(openbsd): + import kqueue, times + + proc waitForExit(p: Process, timeout: int = -1): int = + if p.exitCode != -3: return p.exitCode + if timeout == -1: + if waitpid(p.id, p.exitCode, 0) < 0: + p.exitCode = -3 + raiseOSError(osLastError()) + else: + var kqFD = kqueue() + if kqFD == -1: + raiseOSError(osLastError()) + + var kevIn = KEvent(ident: p.id.uint, filter: EVFILT_PROC, + flags: EV_ADD, fflags: NOTE_EXIT) + var kevOut: KEvent + var tmspec: Timespec + + if timeout >= 1000: + tmspec.tv_sec = (timeout div 1_000).Time + tmspec.tv_nsec = (timeout %% 1_000) * 1_000_000 + else: + tmspec.tv_sec = 0.Time + tmspec.tv_nsec = (timeout * 1_000_000) + + try: + while true: + var count = kevent(kqFD, addr(kevIn), 1, addr(kevOut), 1, + addr(tmspec)) + if count < 0: + let err = osLastError() + if err.cint != EINTR: + raiseOSError(osLastError()) + elif count == 0: + # timeout expired, so we trying to kill process + if posix.kill(p.id, SIGKILL) == -1: + raiseOSError(osLastError()) + if waitpid(p.id, p.exitCode, 0) < 0: + p.exitCode = -3 + raiseOSError(osLastError()) + break + else: + if kevOut.ident == p.id.uint and kevOut.filter == EVFILT_PROC: + if waitpid(p.id, p.exitCode, 0) < 0: + p.exitCode = -3 + raiseOSError(osLastError()) + break + else: + raiseOSError(osLastError()) + finally: + discard posix.close(kqFD) + + result = int(p.exitCode) shr 8 + else: + import times + + const + hasThreadSupport = compileOption("threads") and not defined(nimscript) + + proc waitForExit(p: Process, timeout: int = -1): int = + template adjustTimeout(t, s, e: Timespec) = + var diff: int + var b: Timespec + b.tv_sec = e.tv_sec + b.tv_nsec = e.tv_nsec + e.tv_sec = (e.tv_sec - s.tv_sec).Time + if e.tv_nsec >= s.tv_nsec: + e.tv_nsec -= s.tv_nsec + else: + if e.tv_sec == 0.Time: + raise newException(ValueError, "System time was modified") + else: + diff = s.tv_nsec - e.tv_nsec + e.tv_nsec = 1_000_000_000 - diff + t.tv_sec = (t.tv_sec - e.tv_sec).Time + if t.tv_nsec >= e.tv_nsec: + t.tv_nsec -= e.tv_nsec + else: + t.tv_sec = (int(t.tv_sec) - 1).Time + diff = e.tv_nsec - t.tv_nsec + t.tv_nsec = 1_000_000_000 - diff + s.tv_sec = b.tv_sec + s.tv_nsec = b.tv_nsec + + #if waitPid(p.id, p.exitCode, 0) == int(p.id): + # ``waitPid`` fails if the process is not running anymore. But then + # ``running`` probably set ``p.exitCode`` for us. Since ``p.exitCode`` is + # initialized with -3, wrong success exit codes are prevented. + if p.exitCode != -3: return p.exitCode + if timeout == -1: + if waitpid(p.id, p.exitCode, 0) < 0: + p.exitCode = -3 + raiseOSError(osLastError()) + else: + var nmask, omask: Sigset + var sinfo: SigInfo + var stspec, enspec, tmspec: Timespec + + discard sigemptyset(nmask) + discard sigemptyset(omask) + discard sigaddset(nmask, SIGCHLD) + + when hasThreadSupport: + if pthread_sigmask(SIG_BLOCK, nmask, omask) == -1: + raiseOSError(osLastError()) + else: + if sigprocmask(SIG_BLOCK, nmask, omask) == -1: + raiseOSError(osLastError()) + + if timeout >= 1000: + tmspec.tv_sec = (timeout div 1_000).Time + tmspec.tv_nsec = (timeout %% 1_000) * 1_000_000 + else: + tmspec.tv_sec = 0.Time + tmspec.tv_nsec = (timeout * 1_000_000) + + try: + if clock_gettime(CLOCK_REALTIME, stspec) == -1: + raiseOSError(osLastError()) + while true: + let res = sigtimedwait(nmask, sinfo, tmspec) + if res == SIGCHLD: + if sinfo.si_pid == p.id: + if waitpid(p.id, p.exitCode, 0) < 0: + p.exitCode = -3 + raiseOSError(osLastError()) + break + else: + # we have SIGCHLD, but not for process we are waiting, + # so we need to adjust timeout value and continue + if clock_gettime(CLOCK_REALTIME, enspec) == -1: + raiseOSError(osLastError()) + adjustTimeout(tmspec, stspec, enspec) + elif res < 0: + let err = osLastError() + if err.cint == EINTR: + # we have received another signal, so we need to + # adjust timeout and continue + if clock_gettime(CLOCK_REALTIME, enspec) == -1: + raiseOSError(osLastError()) + adjustTimeout(tmspec, stspec, enspec) + elif err.cint == EAGAIN: + # timeout expired, so we trying to kill process + if posix.kill(p.id, SIGKILL) == -1: + raiseOSError(osLastError()) + if waitpid(p.id, p.exitCode, 0) < 0: + p.exitCode = -3 + raiseOSError(osLastError()) + break + else: + raiseOSError(err) + finally: + when hasThreadSupport: + if pthread_sigmask(SIG_UNBLOCK, nmask, omask) == -1: + raiseOSError(osLastError()) + else: + if sigprocmask(SIG_UNBLOCK, nmask, omask) == -1: + raiseOSError(osLastError()) + + result = int(p.exitCode) shr 8 proc peekExitCode(p: Process): int = if p.exitCode != -3: return p.exitCode diff --git a/lib/pure/oswalkdir.nim b/lib/pure/oswalkdir.nim index 000fe25a3..23ca0566a 100644 --- a/lib/pure/oswalkdir.nim +++ b/lib/pure/oswalkdir.nim @@ -1,3 +1,11 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2015 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# ## Compile-time only version for walkDir if you need it at compile-time ## for JavaScript. diff --git a/lib/pure/parsecfg.nim b/lib/pure/parsecfg.nim index 9bcac0a50..c648b0703 100644 --- a/lib/pure/parsecfg.nim +++ b/lib/pure/parsecfg.nim @@ -15,17 +15,78 @@ ## This is an example of how a configuration file may look like: ## -## .. include:: doc/mytest.cfg +## .. include:: ../../doc/mytest.cfg ## :literal: ## The file ``examples/parsecfgex.nim`` demonstrates how to use the ## configuration file parser: ## ## .. code-block:: nim -## :file: examples/parsecfgex.nim - +## :file: ../../examples/parsecfgex.nim +## +## Examples +## -------- +## +## This is an example of a configuration file. +## +## :: +## +## charset = "utf-8" +## [Package] +## name = "hello" +## --threads:on +## [Author] +## name = "lihf8515" +## qq = "10214028" +## email = "lihaifeng@wxm.com" +## +## Creating a configuration file. +## ============================== +## .. code-block:: nim +## +## import parsecfg +## var dict=newConfig() +## dict.setSectionKey("","charset","utf-8") +## dict.setSectionKey("Package","name","hello") +## dict.setSectionKey("Package","--threads","on") +## dict.setSectionKey("Author","name","lihf8515") +## dict.setSectionKey("Author","qq","10214028") +## dict.setSectionKey("Author","email","lihaifeng@wxm.com") +## dict.writeConfig("config.ini") +## +## Reading a configuration file. +## ============================= +## .. code-block:: nim +## +## import parsecfg +## var dict = loadConfig("config.ini") +## var charset = dict.getSectionValue("","charset") +## var threads = dict.getSectionValue("Package","--threads") +## var pname = dict.getSectionValue("Package","name") +## var name = dict.getSectionValue("Author","name") +## var qq = dict.getSectionValue("Author","qq") +## var email = dict.getSectionValue("Author","email") +## echo pname & "\n" & name & "\n" & qq & "\n" & email +## +## Modifying a configuration file. +## =============================== +## .. code-block:: nim +## +## import parsecfg +## var dict = loadConfig("config.ini") +## dict.setSectionKey("Author","name","lhf") +## dict.writeConfig("config.ini") +## +## Deleting a section key in a configuration file. +## =============================================== +## .. code-block:: nim +## +## import parsecfg +## var dict = loadConfig("config.ini") +## dict.delSectionKey("Author","email") +## dict.writeConfig("config.ini") import - hashes, strutils, lexbase, streams + hashes, strutils, lexbase, streams, tables include "system/inclrtl" @@ -70,7 +131,7 @@ type # implementation const - SymChars = {'a'..'z', 'A'..'Z', '0'..'9', '_', '\x80'..'\xFF', '.', '/', '\\'} + SymChars = {'a'..'z', 'A'..'Z', '0'..'9', '_', '\x80'..'\xFF', '.', '/', '\\', '-'} proc rawGetTok(c: var CfgParser, tok: var Token) {.gcsafe.} @@ -359,3 +420,138 @@ proc next*(c: var CfgParser): CfgEvent {.rtl, extern: "npc$1".} = result.kind = cfgError result.msg = errorStr(c, "invalid token: " & c.tok.literal) rawGetTok(c, c.tok) + +# ---------------- Configuration file related operations ---------------- +type + Config* = OrderedTableRef[string, OrderedTableRef[string, string]] + +proc newConfig*(): Config = + ## Create a new configuration table. + ## Useful when wanting to create a configuration file. + result = newOrderedTable[string, OrderedTableRef[string, string]]() + +proc loadConfig*(filename: string): Config = + ## Load the specified configuration file into a new Config instance. + var dict = newOrderedTable[string, OrderedTableRef[string, string]]() + var curSection = "" ## Current section, + ## the default value of the current section is "", + ## which means that the current section is a common + var p: CfgParser + var fileStream = newFileStream(filename, fmRead) + if fileStream != nil: + open(p, fileStream, filename) + while true: + var e = next(p) + case e.kind + of cfgEof: + break + of cfgSectionStart: # Only look for the first time the Section + curSection = e.section + of cfgKeyValuePair: + var t = newOrderedTable[string, string]() + if dict.hasKey(curSection): + t = dict[curSection] + t[e.key] = e.value + dict[curSection] = t + of cfgOption: + var c = newOrderedTable[string, string]() + if dict.hasKey(curSection): + c = dict[curSection] + c["--" & e.key] = e.value + dict[curSection] = c + of cfgError: + break + close(p) + result = dict + +proc replace(s: string): string = + var d = "" + var i = 0 + while i < s.len(): + if s[i] == '\\': + d.add(r"\\") + elif s[i] == '\c' and s[i+1] == '\L': + d.add(r"\n") + inc(i) + elif s[i] == '\c': + d.add(r"\n") + elif s[i] == '\L': + d.add(r"\n") + else: + d.add(s[i]) + inc(i) + result = d + +proc writeConfig*(dict: Config, filename: string) = + ## Writes the contents of the table to the specified configuration file. + ## Note: Comment statement will be ignored. + var file: File + if file.open(filename, fmWrite): + try: + var section, key, value, kv, segmentChar:string + for pair in dict.pairs(): + section = pair[0] + if section != "": ## Not general section + if not allCharsInSet(section, SymChars): ## Non system character + file.writeLine("[\"" & section & "\"]") + else: + file.writeLine("[" & section & "]") + for pair2 in pair[1].pairs(): + key = pair2[0] + value = pair2[1] + if key[0] == '-' and key[1] == '-': ## If it is a command key + segmentChar = ":" + if not allCharsInSet(key[2..key.len()-1], SymChars): + kv.add("--\"") + kv.add(key[2..key.len()-1]) + kv.add("\"") + else: + kv = key + else: + segmentChar = "=" + kv = key + if value != "": ## If the key is not empty + if not allCharsInSet(value, SymChars): + kv.add(segmentChar) + kv.add("\"") + kv.add(replace(value)) + kv.add("\"") + else: + kv.add(segmentChar) + kv.add(value) + file.writeLine(kv) + except: + raise + finally: + file.close() + +proc getSectionValue*(dict: Config, section, key: string): string = + ## Gets the Key value of the specified Section. + if dict.haskey(section): + if dict[section].hasKey(key): + result = dict[section][key] + else: + result = "" + else: + result = "" + +proc setSectionKey*(dict: var Config, section, key, value: string) = + ## Sets the Key value of the specified Section. + var t = newOrderedTable[string, string]() + if dict.hasKey(section): + t = dict[section] + t[key] = value + dict[section] = t + +proc delSection*(dict: var Config, section: string) = + ## Deletes the specified section and all of its sub keys. + dict.del(section) + +proc delSectionKey*(dict: var Config, section, key: string) = + ## Delete the key of the specified section. + if dict.haskey(section): + if dict[section].hasKey(key): + if dict[section].len() == 1: + dict.del(section) + else: + dict[section].del(key) diff --git a/lib/pure/parsecsv.nim b/lib/pure/parsecsv.nim index af51e1201..77b145a73 100644 --- a/lib/pure/parsecsv.nim +++ b/lib/pure/parsecsv.nim @@ -25,6 +25,28 @@ ## echo "##", val, "##" ## close(x) ## +## For CSV files with a header row, the header can be read and then used as a +## reference for item access with `rowEntry <#rowEntry.CsvParser.string>`_: +## +## .. code-block:: nim +## import parsecsv +## import os +## # Prepare a file +## var csv_content = """One,Two,Three,Four +## 1,2,3,4 +## 10,20,30,40 +## 100,200,300,400 +## """ +## writeFile("temp.csv", content) +## +## var p: CsvParser +## p.open("temp.csv") +## p.readHeaderRow() +## while p.readRow(): +## echo "new row: " +## for col in items(p.headers): +## echo "##", col, ":", p.rowEntry(col), "##" +## p.close() import lexbase, streams @@ -37,6 +59,9 @@ type sep, quote, esc: char skipWhite: bool currRow: int + headers*: seq[string] ## The columns that are defined in the csv file + ## (read using `readHeaderRow <#readHeaderRow.CsvParser>`_). + ## Used with `rowEntry <#rowEntry.CsvParser.string>`_). CsvError* = object of IOError ## exception that is raised if ## a parsing error occurs @@ -77,6 +102,15 @@ proc open*(my: var CsvParser, input: Stream, filename: string, my.row = @[] my.currRow = 0 +proc open*(my: var CsvParser, filename: string, + separator = ',', quote = '"', escape = '\0', + skipInitialSpace = false) = + ## same as the other `open` but creates the file stream for you. + var s = newFileStream(filename, fmRead) + if s == nil: my.error(0, "cannot open: " & filename) + open(my, s, filename, separator, + quote, escape, skipInitialSpace) + proc parseField(my: var CsvParser, a: var string) = var pos = my.bufpos var buf = my.buf @@ -131,6 +165,8 @@ proc readRow*(my: var CsvParser, columns = 0): bool = ## reads the next row; if `columns` > 0, it expects the row to have ## exactly this many columns. Returns false if the end of the file ## has been encountered else true. + ## + ## Blank lines are skipped. var col = 0 # current column var oldpos = my.bufpos while my.buf[my.bufpos] != '\0': @@ -166,6 +202,22 @@ proc close*(my: var CsvParser) {.inline.} = ## closes the parser `my` and its associated input stream. lexbase.close(my) +proc readHeaderRow*(my: var CsvParser) = + ## Reads the first row and creates a look-up table for column numbers + ## See also `rowEntry <#rowEntry.CsvParser.string>`_. + var present = my.readRow() + if present: + my.headers = my.row + +proc rowEntry*(my: var CsvParser, entry: string): string = + ## Reads a specified `entry` from the current row. + ## + ## Assumes that `readHeaderRow <#readHeaderRow.CsvParser>`_ has already been + ## called. + var index = my.headers.find(entry) + if index >= 0: + result = my.row[index] + when not defined(testing) and isMainModule: import os var s = newFileStream(paramStr(1), fmRead) @@ -178,3 +230,35 @@ when not defined(testing) and isMainModule: echo "##", val, "##" close(x) +when isMainModule: + import os + import strutils + block: # Tests for reading the header row + var content = "One,Two,Three,Four\n1,2,3,4\n10,20,30,40,\n100,200,300,400\n" + writeFile("temp.csv", content) + + var p: CsvParser + p.open("temp.csv") + p.readHeaderRow() + while p.readRow(): + var zeros = repeat('0', p.currRow-2) + doAssert p.rowEntry("One") == "1" & zeros + doAssert p.rowEntry("Two") == "2" & zeros + doAssert p.rowEntry("Three") == "3" & zeros + doAssert p.rowEntry("Four") == "4" & zeros + p.close() + + when not defined(testing): + var parser: CsvParser + parser.open("temp.csv") + parser.readHeaderRow() + while parser.readRow(): + echo "new row: " + for col in items(parser.headers): + echo "##", col, ":", parser.rowEntry(col), "##" + parser.close() + removeFile("temp.csv") + + # Tidy up + removeFile("temp.csv") + diff --git a/lib/pure/parseutils.nim b/lib/pure/parseutils.nim index 698bde42a..fb7d72182 100644 --- a/lib/pure/parseutils.nim +++ b/lib/pure/parseutils.nim @@ -173,6 +173,22 @@ proc parseUntil*(s: string, token: var string, until: char, result = i-start token = substr(s, start, i-1) +proc parseUntil*(s: string, token: var string, until: string, + start = 0): int {.inline.} = + ## parses a token and stores it in ``token``. Returns + ## the number of the parsed characters or 0 in case of an error. A token + ## consists of any character that comes before the `until` token. + var i = start + while i < s.len: + if s[i] == until[0]: + var u = 1 + while i+u < s.len and u < until.len and s[i+u] == until[u]: + inc u + if u >= until.len: break + inc(i) + result = i-start + token = substr(s, start, i-1) + proc parseWhile*(s: string, token: var string, validChars: set[char], start = 0): int {.inline.} = ## parses a token and stores it in ``token``. Returns @@ -234,6 +250,51 @@ proc parseInt*(s: string, number: var int, start = 0): int {. elif result != 0: number = int(res) +# overflowChecks doesn't work with uint64 +proc rawParseUInt(s: string, b: var uint64, start = 0): int = + var + res = 0'u64 + prev = 0'u64 + i = start + if s[i] == '+': inc(i) # Allow + if s[i] in {'0'..'9'}: + b = 0 + while s[i] in {'0'..'9'}: + prev = res + res = res * 10 + (ord(s[i]) - ord('0')).uint64 + if prev > res: + return 0 # overflowChecks emulation + inc(i) + while s[i] == '_': inc(i) # underscores are allowed and ignored + b = res + result = i - start + +proc parseBiggestUInt*(s: string, number: var uint64, start = 0): int {. + rtl, extern: "npuParseBiggestUInt", noSideEffect.} = + ## parses an unsigned integer starting at `start` and stores the value + ## into `number`. + ## Result is the number of processed chars or 0 if there is no integer + ## or overflow detected. + var res: uint64 + # use 'res' for exception safety (don't write to 'number' in case of an + # overflow exception): + result = rawParseUInt(s, res, start) + number = res + +proc parseUInt*(s: string, number: var uint, start = 0): int {. + rtl, extern: "npuParseUInt", noSideEffect.} = + ## parses an unsigned integer starting at `start` and stores the value + ## into `number`. + ## Result is the number of processed chars or 0 if there is no integer or + ## overflow detected. + var res: uint64 + result = parseBiggestUInt(s, res, start) + if (sizeof(uint) <= 4) and + (res > 0xFFFF_FFFF'u64): + raise newException(OverflowError, "overflow") + elif result != 0: + number = uint(res) + proc parseBiggestFloat*(s: string, number: var BiggestFloat, start = 0): int {. magic: "ParseBiggestFloat", importc: "nimParseBiggestFloat", noSideEffect.} ## parses a float starting at `start` and stores the value into `number`. diff --git a/lib/pure/parsexml.nim b/lib/pure/parsexml.nim index f8b2c3d8d..d16a55302 100644 --- a/lib/pure/parsexml.nim +++ b/lib/pure/parsexml.nim @@ -34,7 +34,7 @@ ## document. ## ## .. code-block:: nim -## :file: examples/htmltitle.nim +## :file: ../../examples/htmltitle.nim ## ## ## Example 2: Retrieve all HTML links @@ -45,7 +45,7 @@ ## an HTML document contains. ## ## .. code-block:: nim -## :file: examples/htmlrefs.nim +## :file: ../../examples/htmlrefs.nim ## import @@ -142,6 +142,9 @@ proc kind*(my: XmlParser): XmlEventKind {.inline.} = template charData*(my: XmlParser): string = ## returns the character data for the events: ``xmlCharData``, ## ``xmlWhitespace``, ``xmlComment``, ``xmlCData``, ``xmlSpecial`` + ## Raises an assertion in debug mode if ``my.kind`` is not one + ## of those events. In release mode, this will not trigger an error + ## but the value returned will not be valid. assert(my.kind in {xmlCharData, xmlWhitespace, xmlComment, xmlCData, xmlSpecial}) my.a @@ -149,31 +152,49 @@ template charData*(my: XmlParser): string = template elementName*(my: XmlParser): string = ## returns the element name for the events: ``xmlElementStart``, ## ``xmlElementEnd``, ``xmlElementOpen`` + ## Raises an assertion in debug mode if ``my.kind`` is not one + ## of those events. In release mode, this will not trigger an error + ## but the value returned will not be valid. assert(my.kind in {xmlElementStart, xmlElementEnd, xmlElementOpen}) my.a template entityName*(my: XmlParser): string = ## returns the entity name for the event: ``xmlEntity`` + ## Raises an assertion in debug mode if ``my.kind`` is not + ## ``xmlEntity``. In release mode, this will not trigger an error + ## but the value returned will not be valid. assert(my.kind == xmlEntity) my.a template attrKey*(my: XmlParser): string = ## returns the attribute key for the event ``xmlAttribute`` + ## Raises an assertion in debug mode if ``my.kind`` is not + ## ``xmlAttribute``. In release mode, this will not trigger an error + ## but the value returned will not be valid. assert(my.kind == xmlAttribute) my.a template attrValue*(my: XmlParser): string = ## returns the attribute value for the event ``xmlAttribute`` + ## Raises an assertion in debug mode if ``my.kind`` is not + ## ``xmlAttribute``. In release mode, this will not trigger an error + ## but the value returned will not be valid. assert(my.kind == xmlAttribute) my.b template piName*(my: XmlParser): string = ## returns the processing instruction name for the event ``xmlPI`` + ## Raises an assertion in debug mode if ``my.kind`` is not + ## ``xmlPI``. In release mode, this will not trigger an error + ## but the value returned will not be valid. assert(my.kind == xmlPI) my.a template piRest*(my: XmlParser): string = ## returns the rest of the processing instruction for the event ``xmlPI`` + ## Raises an assertion in debug mode if ``my.kind`` is not + ## ``xmlPI``. In release mode, this will not trigger an error + ## but the value returned will not be valid. assert(my.kind == xmlPI) my.b diff --git a/lib/pure/pegs.nim b/lib/pure/pegs.nim index fead66de2..5c978a2f8 100644 --- a/lib/pure/pegs.nim +++ b/lib/pure/pegs.nim @@ -12,7 +12,7 @@ ## Matching performance is hopefully competitive with optimized regular ## expression engines. ## -## .. include:: ../doc/pegdocs.txt +## .. include:: ../../doc/pegdocs.txt ## include "system/inclrtl" @@ -659,7 +659,7 @@ proc rawMatch*(s: string, p: Peg, start: int, c: var Captures): int {. of pkSearch: var oldMl = c.ml result = 0 - while start+result < s.len: + while start+result <= s.len: var x = rawMatch(s, p.sons[0], start+result, c) if x >= 0: inc(result, x) @@ -671,7 +671,7 @@ proc rawMatch*(s: string, p: Peg, start: int, c: var Captures): int {. var idx = c.ml # reserve a slot for the subpattern inc(c.ml) result = 0 - while start+result < s.len: + while start+result <= s.len: var x = rawMatch(s, p.sons[0], start+result, c) if x >= 0: if idx < MaxSubpatterns: @@ -962,6 +962,50 @@ proc parallelReplace*(s: string, subs: varargs[ # copy the rest: add(result, substr(s, i)) +proc replace*(s: string, sub: Peg, cb: proc( + match: int, cnt: int, caps: openArray[string]): string): string {. + rtl, extern: "npegs$1cb".}= + ## Replaces `sub` in `s` by the resulting strings from the callback. + ## The callback proc receives the index of the current match (starting with 0), + ## the count of captures and an open array with the captures of each match. Examples: + ## + ## .. code-block:: nim + ## + ## proc handleMatches*(m: int, n: int, c: openArray[string]): string = + ## result = "" + ## if m > 0: + ## result.add ", " + ## result.add case n: + ## of 2: c[0].toLower & ": '" & c[1] & "'" + ## of 1: c[0].toLower & ": ''" + ## else: "" + ## + ## let s = "Var1=key1;var2=Key2; VAR3" + ## echo s.replace(peg"{\ident}('='{\ident})* ';'* \s*", handleMatches) + ## + ## Results in: + ## + ## .. code-block:: nim + ## + ## "var1: 'key1', var2: 'Key2', var3: ''" + result = "" + var i = 0 + var caps: array[0..MaxSubpatterns-1, string] + var c: Captures + var m = 0 + while i < s.len: + c.ml = 0 + var x = rawMatch(s, sub, i, c) + if x <= 0: + add(result, s[i]) + inc(i) + else: + fillMatches(s, caps, c) + add(result, cb(m, c.ml, caps)) + inc(i, x) + inc(m) + add(result, substr(s, i)) + proc transformFile*(infile, outfile: string, subs: varargs[tuple[pattern: Peg, repl: string]]) {. rtl, extern: "npegs$1".} = @@ -1789,3 +1833,22 @@ when isMainModule: assert(str.find(empty_test) == 0) assert(str.match(empty_test)) + + proc handleMatches*(m: int, n: int, c: openArray[string]): string = + result = "" + + if m > 0: + result.add ", " + + result.add case n: + of 2: toLowerAscii(c[0]) & ": '" & c[1] & "'" + of 1: toLowerAscii(c[0]) & ": ''" + else: "" + + assert("Var1=key1;var2=Key2; VAR3". + replace(peg"{\ident}('='{\ident})* ';'* \s*", + handleMatches)=="var1: 'key1', var2: 'Key2', var3: ''") + + + doAssert "test1".match(peg"""{@}$""") + doAssert "test2".match(peg"""{(!$ .)*} $""") diff --git a/lib/pure/punycode.nim b/lib/pure/punycode.nim new file mode 100644 index 000000000..ab6501ed1 --- /dev/null +++ b/lib/pure/punycode.nim @@ -0,0 +1,174 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2016 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +import strutils +import unicode + +# issue #3045 + +const + Base = 36 + TMin = 1 + TMax = 26 + Skew = 38 + Damp = 700 + InitialBias = 72 + InitialN = 128 + Delimiter = '-' + +type + PunyError* = object of Exception + +proc decodeDigit(x: char): int {.raises: [PunyError].} = + if '0' <= x and x <= '9': + result = ord(x) - (ord('0') - 26) + elif 'A' <= x and x <= 'Z': + result = ord(x) - ord('A') + elif 'a' <= x and x <= 'z': + result = ord(x) - ord('a') + else: + raise newException(PunyError, "Bad input") + +proc encodeDigit(digit: int): Rune {.raises: [PunyError].} = + if 0 <= digit and digit < 26: + result = Rune(digit + ord('a')) + elif 26 <= digit and digit < 36: + result = Rune(digit + (ord('0') - 26)) + else: + raise newException(PunyError, "internal error in punycode encoding") + +proc isBasic(c: char): bool = ord(c) < 0x80 +proc isBasic(r: Rune): bool = int(r) < 0x80 + +proc adapt(delta, numPoints: int, first: bool): int = + var d = if first: delta div Damp else: delta div 2 + d += d div numPoints + var k = 0 + while d > ((Base-TMin)*TMax) div 2: + d = d div (Base - TMin) + k += Base + result = k + (Base - TMin + 1) * d div (d + Skew) + +proc encode*(prefix, s: string): string {.raises: [PunyError].} = + ## Encode a string that may contain Unicode. + ## Prepend `prefix` to the result + result = prefix + var (d, n, bias) = (0, InitialN, InitialBias) + var (b, remaining) = (0, 0) + for r in s.runes: + if r.isBasic: + # basic Ascii character + inc b + result.add($r) + else: + # special character + inc remaining + + var h = b + if b > 0: + result.add(Delimiter) # we have some Ascii chars + while remaining != 0: + var m: int = high(int32) + for r in s.runes: + if m > int(r) and int(r) >= n: + m = int(r) + d += (m - n) * (h + 1) + if d < 0: + raise newException(PunyError, "invalid label " & s) + n = m + for r in s.runes: + if int(r) < n: + inc d + if d < 0: + raise newException(PunyError, "invalid label " & s) + continue + if int(r) > n: + continue + var q = d + var k = Base + while true: + var t = k - bias + if t < TMin: + t = TMin + elif t > TMax: + t = TMax + if q < t: + break + result.add($encodeDigit(t + (q - t) mod (Base - t))) + q = (q - t) div (Base - t) + k += Base + result.add($encodeDigit(q)) + bias = adapt(d, h + 1, h == b) + d = 0 + inc h + dec remaining + inc d + inc n + +proc encode*(s: string): string {.raises: [PunyError].} = + ## Encode a string that may contain Unicode. Prefix is empty. + result = encode("", s) + +proc decode*(encoded: string): string {.raises: [PunyError].} = + ## Decode a Punycode-encoded string + var + n = InitialN + i = 0 + bias = InitialBias + var d = rfind(encoded, Delimiter) + result = "" + + if d > 0: + # found Delimiter + for j in 0..<d: + var c = encoded[j] # char + if not c.isBasic: + raise newException(PunyError, "Encoded contains a non-basic char") + result.add(c) # add the character + inc d + else: + d = 0 # set to first index + + while (d < len(encoded)): + var oldi = i + var w = 1 + var k = Base + while true: + if d == len(encoded): + raise newException(PunyError, "Bad input: " & encoded) + var c = encoded[d]; inc d + var digit = int(decodeDigit(c)) + if digit > (high(int32) - i) div w: + raise newException(PunyError, "Too large a value: " & $digit) + i += digit * w + var t: int + if k <= bias: + t = TMin + elif k >= bias + TMax: + t = TMax + else: + t = k - bias + if digit < t: + break + w *= Base - t + k += Base + bias = adapt(i - oldi, runelen(result) + 1, oldi == 0) + + if i div (runelen(result) + 1) > high(int32) - n: + raise newException(PunyError, "Value too large") + + n += i div (runelen(result) + 1) + i = i mod (runelen(result) + 1) + insert(result, $Rune(n), i) + inc i + +when isMainModule: + assert(decode(encode("", "bücher")) == "bücher") + assert(decode(encode("münchen")) == "münchen") + assert encode("xn--", "münchen") == "xn--mnchen-3ya" diff --git a/lib/pure/random.nim b/lib/pure/random.nim new file mode 100644 index 000000000..08da771dc --- /dev/null +++ b/lib/pure/random.nim @@ -0,0 +1,128 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2016 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Nim's standard random number generator. Based on the ``xoroshiro128+`` (xor/rotate/shift/rotate) library. +## * More information: http://xoroshiro.di.unimi.it/ +## * C implementation: http://xoroshiro.di.unimi.it/xoroshiro128plus.c +## + +include "system/inclrtl" +{.push debugger:off.} + +# XXX Expose RandomGenState +when defined(JS): + type ui = uint32 +else: + type ui = uint64 + +type + RandomGenState = object + a0, a1: ui + +when defined(JS): + var state = RandomGenState( + a0: 0x69B4C98Cu32, + a1: 0xFED1DD30u32) # global for backwards compatibility +else: + # racy for multi-threading but good enough for now: + var state = RandomGenState( + a0: 0x69B4C98CB8530805u64, + a1: 0xFED1DD3004688D67CAu64) # global for backwards compatibility + +proc rotl(x, k: ui): ui = + result = (x shl k) or (x shr (ui(64) - k)) + +proc next(s: var RandomGenState): uint64 = + let s0 = s.a0 + var s1 = s.a1 + result = s0 + s1 + s1 = s1 xor s0 + s.a0 = rotl(s0, 55) xor s1 xor (s1 shl 14) # a, b + s.a1 = rotl(s1, 36) # c + +proc skipRandomNumbers(s: var RandomGenState) = + ## This is the jump function for the generator. It is equivalent + ## to 2^64 calls to next(); it can be used to generate 2^64 + ## non-overlapping subsequences for parallel computations. + when defined(JS): + const helper = [0xbeac0467u32, 0xd86b048bu32] + else: + const helper = [0xbeac0467eba5facbu64, 0xd86b048b86aa9922u64] + var + s0 = ui 0 + s1 = ui 0 + for i in 0..high(helper): + for b in 0..< 64: + if (helper[i] and (ui(1) shl ui(b))) != 0: + s0 = s0 xor s.a0 + s1 = s1 xor s.a1 + discard next(s) + s.a0 = s0 + s.a1 = s1 + +proc random*(max: int): int {.benign.} = + ## Returns a random number in the range 0..max-1. The sequence of + ## random number is always the same, unless `randomize` is called + ## which initializes the random number generator with a "random" + ## number, i.e. a tickcount. + result = int(next(state) mod uint64(max)) + +proc random*(max: float): float {.benign.} = + ## Returns a random number in the range 0..<max. The sequence of + ## random number is always the same, unless `randomize` is called + ## which initializes the random number generator with a "random" + ## number, i.e. a tickcount. + let x = next(state) + when defined(JS): + result = (float(x) / float(high(uint32))) * max + else: + let u = (0x3FFu64 shl 52u64) or (x shr 12u64) + result = (cast[float](u) - 1.0) * max + +proc random*[T](x: Slice[T]): T = + ## For a slice `a .. b` returns a value in the range `a .. b-1`. + result = random(x.b - x.a) + x.a + +proc random*[T](a: openArray[T]): T = + ## returns a random element from the openarray `a`. + result = a[random(a.low..a.len)] + +proc randomize*(seed: int) {.benign.} = + ## Initializes the random number generator with a specific seed. + state.a0 = ui(seed shr 16) + state.a1 = ui(seed and 0xffff) + +when not defined(nimscript): + import times + + proc randomize*() {.benign.} = + ## Initializes the random number generator with a "random" + ## number, i.e. a tickcount. Note: Does not work for NimScript. + when defined(JS): + proc getMil(t: Time): int {.importcpp: "getTime", nodecl.} + randomize(getMil times.getTime()) + else: + randomize(int times.getTime()) + +{.pop.} + +when isMainModule: + proc main = + var occur: array[1000, int] + + var x = 8234 + for i in 0..100_000: + x = random(len(occur)) # myrand(x) + inc occur[x] + for i, oc in occur: + if oc < 69: + doAssert false, "too few occurances of " & $i + elif oc > 130: + doAssert false, "too many occurances of " & $i + main() diff --git a/lib/pure/rationals.nim b/lib/pure/rationals.nim index 6fd05dc4b..bf134f2ae 100644 --- a/lib/pure/rationals.nim +++ b/lib/pure/rationals.nim @@ -41,26 +41,26 @@ proc toRational*[T:SomeInteger](x: T): Rational[T] = proc toRationalSub(x: float, n: int): Rational[int] = var - a = 0 - b, c, d = 1 + a = 0'i64 + b, c, d = 1'i64 result = 0 // 1 # rational 0 while b <= n and d <= n: let ac = (a+c) let bd = (b+d) # scale by 1000 so not overflow for high precision - let mediant = (ac/1000) / (bd/1000) + let mediant = (ac.float/1000) / (bd.float/1000) if x == mediant: if bd <= n: - result.num = ac - result.den = bd + result.num = ac.int + result.den = bd.int return result elif d > b: - result.num = c - result.den = d + result.num = c.int + result.den = d.int return result else: - result.num = a - result.den = b + result.num = a.int + result.den = b.int return result elif x > mediant: a = ac @@ -69,8 +69,8 @@ proc toRationalSub(x: float, n: int): Rational[int] = c = ac d = bd if (b > n): - return initRational(c, d) - return initRational(a, b) + return initRational(c.int, d.int) + return initRational(a.int, b.int) proc toRational*(x: float, n: int = high(int)): Rational[int] = ## Calculate the best rational numerator and denominator diff --git a/lib/pure/selectors.nim b/lib/pure/selectors.nim index 89e92c133..098b78c95 100644 --- a/lib/pure/selectors.nim +++ b/lib/pure/selectors.nim @@ -132,11 +132,12 @@ elif defined(linux): s.fds[fd].events = events proc unregister*(s: var Selector, fd: SocketHandle) = - if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fd, nil) != 0: - let err = osLastError() - if err.cint notin {ENOENT, EBADF}: - # TODO: Why do we sometimes get an EBADF? Is this normal? - raiseOSError(err) + if s.fds[fd].events != {}: + if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fd, nil) != 0: + let err = osLastError() + if err.cint notin {ENOENT, EBADF}: + # TODO: Why do we sometimes get an EBADF? Is this normal? + raiseOSError(err) s.fds.del(fd) proc close*(s: var Selector) = diff --git a/lib/pure/smtp.nim b/lib/pure/smtp.nim index b2adac2f3..050712902 100644 --- a/lib/pure/smtp.nim +++ b/lib/pure/smtp.nim @@ -20,9 +20,9 @@ ## var msg = createMessage("Hello from Nim's SMTP", ## "Hello!.\n Is this awesome or what?", ## @["foo@gmail.com"]) -## var smtp = connect("smtp.gmail.com", 465, true, true) -## smtp.auth("username", "password") -## smtp.sendmail("username@gmail.com", @["foo@gmail.com"], $msg) +## var smtpConn = connect("smtp.gmail.com", Port 465, true, true) +## smtpConn.auth("username", "password") +## smtpConn.sendmail("username@gmail.com", @["foo@gmail.com"], $msg) ## ## ## For SSL support this module relies on OpenSSL. If you want to @@ -31,6 +31,8 @@ import net, strutils, strtabs, base64, os import asyncnet, asyncdispatch +export Port + type Smtp* = object sock: Socket @@ -258,8 +260,8 @@ when not defined(testing) and isMainModule: # "Hello, my name is dom96.\n What\'s yours?", @["dominik@localhost"]) #echo(msg) - #var smtp = connect("localhost", 25, False, True) - #smtp.sendmail("root@localhost", @["dominik@localhost"], $msg) + #var smtpConn = connect("localhost", Port 25, false, true) + #smtpConn.sendmail("root@localhost", @["dominik@localhost"], $msg) #echo(decode("a17sm3701420wbe.12")) proc main() {.async.} = diff --git a/lib/pure/streams.nim b/lib/pure/streams.nim index c606b4680..eea06f4ce 100644 --- a/lib/pure/streams.nim +++ b/lib/pure/streams.nim @@ -306,68 +306,68 @@ proc peekLine*(s: Stream): TaintedString = defer: setPosition(s, pos) result = readLine(s) -type - StringStream* = ref StringStreamObj ## a stream that encapsulates a string - StringStreamObj* = object of StreamObj - data*: string - pos: int - -{.deprecated: [PStringStream: StringStream, TStringStream: StringStreamObj].} - -proc ssAtEnd(s: Stream): bool = - var s = StringStream(s) - return s.pos >= s.data.len - -proc ssSetPosition(s: Stream, pos: int) = - var s = StringStream(s) - s.pos = clamp(pos, 0, s.data.len) - -proc ssGetPosition(s: Stream): int = - var s = StringStream(s) - return s.pos - -proc ssReadData(s: Stream, buffer: pointer, bufLen: int): int = - var s = StringStream(s) - result = min(bufLen, s.data.len - s.pos) - if result > 0: - copyMem(buffer, addr(s.data[s.pos]), result) - inc(s.pos, result) - -proc ssPeekData(s: Stream, buffer: pointer, bufLen: int): int = - var s = StringStream(s) - result = min(bufLen, s.data.len - s.pos) - if result > 0: - copyMem(buffer, addr(s.data[s.pos]), result) - -proc ssWriteData(s: Stream, buffer: pointer, bufLen: int) = - var s = StringStream(s) - if bufLen <= 0: - return - if s.pos + bufLen > s.data.len: - setLen(s.data, s.pos + bufLen) - copyMem(addr(s.data[s.pos]), buffer, bufLen) - inc(s.pos, bufLen) - -proc ssClose(s: Stream) = - var s = StringStream(s) - s.data = nil - -proc newStringStream*(s: string = ""): StringStream = - ## creates a new stream from the string `s`. - new(result) - result.data = s - result.pos = 0 - result.closeImpl = ssClose - result.atEndImpl = ssAtEnd - result.setPositionImpl = ssSetPosition - result.getPositionImpl = ssGetPosition - result.readDataImpl = ssReadData - result.peekDataImpl = ssPeekData - result.writeDataImpl = ssWriteData - when not defined(js): type + StringStream* = ref StringStreamObj ## a stream that encapsulates a string + StringStreamObj* = object of StreamObj + data*: string + pos: int + + {.deprecated: [PStringStream: StringStream, TStringStream: StringStreamObj].} + + proc ssAtEnd(s: Stream): bool = + var s = StringStream(s) + return s.pos >= s.data.len + + proc ssSetPosition(s: Stream, pos: int) = + var s = StringStream(s) + s.pos = clamp(pos, 0, s.data.len) + + proc ssGetPosition(s: Stream): int = + var s = StringStream(s) + return s.pos + + proc ssReadData(s: Stream, buffer: pointer, bufLen: int): int = + var s = StringStream(s) + result = min(bufLen, s.data.len - s.pos) + if result > 0: + copyMem(buffer, addr(s.data[s.pos]), result) + inc(s.pos, result) + + proc ssPeekData(s: Stream, buffer: pointer, bufLen: int): int = + var s = StringStream(s) + result = min(bufLen, s.data.len - s.pos) + if result > 0: + copyMem(buffer, addr(s.data[s.pos]), result) + + proc ssWriteData(s: Stream, buffer: pointer, bufLen: int) = + var s = StringStream(s) + if bufLen <= 0: + return + if s.pos + bufLen > s.data.len: + setLen(s.data, s.pos + bufLen) + copyMem(addr(s.data[s.pos]), buffer, bufLen) + inc(s.pos, bufLen) + + proc ssClose(s: Stream) = + var s = StringStream(s) + s.data = nil + + proc newStringStream*(s: string = ""): StringStream = + ## creates a new stream from the string `s`. + new(result) + result.data = s + result.pos = 0 + result.closeImpl = ssClose + result.atEndImpl = ssAtEnd + result.setPositionImpl = ssSetPosition + result.getPositionImpl = ssGetPosition + result.readDataImpl = ssReadData + result.peekDataImpl = ssPeekData + result.writeDataImpl = ssWriteData + + type FileStream* = ref FileStreamObj ## a stream that encapsulates a `File` FileStreamObj* = object of Stream f: File diff --git a/lib/pure/strmisc.nim b/lib/pure/strmisc.nim new file mode 100644 index 000000000..89ef2fcd2 --- /dev/null +++ b/lib/pure/strmisc.nim @@ -0,0 +1,83 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2016 Joey Payne +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module contains various string utility routines that are uncommonly +## used in comparison to `strutils <strutils.html>`_. + +import strutils + +{.deadCodeElim: on.} + +proc expandTabs*(s: string, tabSize: int = 8): string {.noSideEffect, + procvar.} = + ## Expand tab characters in `s` by `tabSize` spaces + + result = newStringOfCap(s.len + s.len shr 2) + var pos = 0 + + template addSpaces(n) = + for j in 0 ..< n: + result.add(' ') + pos += 1 + + for i in 0 ..< len(s): + let c = s[i] + if c == '\t': + let + denominator = if tabSize > 0: tabSize else: 1 + numSpaces = tabSize - pos mod denominator + + addSpaces(numSpaces) + else: + result.add(c) + pos += 1 + if c == '\l': + pos = 0 + +proc partition*(s: string, sep: string, + right: bool = false): (string, string, string) + {.noSideEffect, procvar.} = + ## Split the string at the first or last occurrence of `sep` into a 3-tuple + ## + ## Returns a 3 string tuple of (beforeSep, `sep`, afterSep) or + ## (`s`, "", "") if `sep` is not found and `right` is false or + ## ("", "", `s`) if `sep` is not found and `right` is true + let position = if right: s.rfind(sep) else: s.find(sep) + if position != -1: + return (s[0 ..< position], sep, s[position + sep.len ..< s.len]) + return if right: ("", "", s) else: (s, "", "") + +proc rpartition*(s: string, sep: string): (string, string, string) + {.noSideEffect, procvar.} = + ## Split the string at the last occurrence of `sep` into a 3-tuple + ## + ## Returns a 3 string tuple of (beforeSep, `sep`, afterSep) or + ## ("", "", `s`) if `sep` is not found + return partition(s, sep, right = true) + +when isMainModule: + doAssert expandTabs("\t", 4) == " " + doAssert expandTabs("\tfoo\t", 4) == " foo " + doAssert expandTabs("\tfoo\tbar", 4) == " foo bar" + doAssert expandTabs("\tfoo\tbar\t", 4) == " foo bar " + doAssert expandTabs("", 4) == "" + doAssert expandTabs("", 0) == "" + doAssert expandTabs("\t\t\t", 0) == "" + + doAssert partition("foo:bar", ":") == ("foo", ":", "bar") + doAssert partition("foobarbar", "bar") == ("foo", "bar", "bar") + doAssert partition("foobarbar", "bank") == ("foobarbar", "", "") + doAssert partition("foobarbar", "foo") == ("", "foo", "barbar") + doAssert partition("foofoobar", "bar") == ("foofoo", "bar", "") + + doAssert rpartition("foo:bar", ":") == ("foo", ":", "bar") + doAssert rpartition("foobarbar", "bar") == ("foobar", "bar", "") + doAssert rpartition("foobarbar", "bank") == ("", "", "foobarbar") + doAssert rpartition("foobarbar", "foo") == ("", "foo", "barbar") + doAssert rpartition("foofoobar", "bar") == ("foofoo", "bar", "") diff --git a/lib/pure/strscans.nim b/lib/pure/strscans.nim new file mode 100644 index 000000000..246f018c5 --- /dev/null +++ b/lib/pure/strscans.nim @@ -0,0 +1,522 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2016 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +##[ +This module contains a `scanf`:idx: macro that can be used for extracting +substrings from an input string. This is often easier than regular expressions. +Some examples as an apetizer: + +.. code-block:: nim + # check if input string matches a triple of integers: + const input = "(1,2,4)" + var x, y, z: int + if scanf(input, "($i,$i,$i)", x, y, z): + echo "matches and x is ", x, " y is ", y, " z is ", z + + # check if input string matches an ISO date followed by an identifier followed + # by whitespace and a floating point number: + var year, month, day: int + var identifier: string + var myfloat: float + if scanf(input, "$i-$i-$i $w$s$f", year, month, day, identifier, myfloat): + echo "yes, we have a match!" + +As can be seen from the examples, strings are matched verbatim except for +substrings starting with ``$``. These constructions are available: + +================= ======================================================== +``$i`` Matches an integer. This uses ``parseutils.parseInt``. +``$f`` Matches a floating pointer number. Uses ``parseFloat``. +``$w`` Matches an ASCII identifier: ``[A-Z-a-z_][A-Za-z_0-9]*``. +``$s`` Skips optional whitespace. +``$$`` Matches a single dollar sign. +``$.`` Matches if the end of the input string has been reached. +``$*`` Matches until the token following the ``$*`` was found. + The match is allowed to be of 0 length. +``$+`` Matches until the token following the ``$+`` was found. + The match must consist of at least one char. +``${foo}`` User defined matcher. Uses the proc ``foo`` to perform + the match. See below for more details. +``$[foo]`` Call user defined proc ``foo`` to **skip** some optional + parts in the input string. See below for more details. +================= ======================================================== + +Even though ``$*`` and ``$+`` look similar to the regular expressions ``.*`` +and ``.+`` they work quite differently, there is no non-deterministic +state machine involved and the matches are non-greedy. ``[$*]`` +matches ``[xyz]`` via ``parseutils.parseUntil``. + +Furthermore no backtracking is performed, if parsing fails after a value +has already been bound to a matched subexpression this value is not restored +to its original value. This rarely causes problems in practice and if it does +for you, it's easy enough to bind to a temporary variable first. + + +Startswith vs full match +======================== + +``scanf`` returns true if the input string **starts with** the specified +pattern. If instead it should only return true if theres is also nothing +left in the input, append ``$.`` to your pattern. + + +User definable matchers +======================= + +One very nice advantage over regular expressions is that ``scanf`` is +extensible with ordinary Nim procs. The proc is either enclosed in ``${}`` +or in ``$[]``. ``${}`` matches and binds the result +to a variable (that was passed to the ``scanf`` macro) while ``$[]`` merely +optional tokens. + + +In this example, we define a helper proc ``skipSep`` that skips some separators +which we then use in our scanf pattern to help us in the matching process: + +.. code-block:: nim + + proc someSep(input: string; start: int; seps: set[char] = {':','-','.'}): int = + # Note: The parameters and return value must match to what ``scanf`` requires + result = 0 + while input[start+result] in seps: inc result + + if scanf(input, "$w${someSep}$w", key, value): + ... + +It also possible to pass arguments to a user definable matcher: + +.. code-block:: nim + + proc ndigits(input: string; start: int; intVal: var int; n: int): int = + # matches exactly ``n`` digits. Matchers need to return 0 if nothing + # matched or otherwise the number of processed chars. + var x = 0 + var i = 0 + while i < n and i+start < input.len and input[i+start] in {'0'..'9'}: + x = x * 10 + input[i+start].ord - '0'.ord + inc i + # only overwrite if we had a match + if i == n: + result = n + intVal = x + + # match an ISO date extracting year, month, day at the same time. + # Also ensure the input ends after the ISO date: + var year, month, day: int + if scanf("2013-01-03", "${ndigits(4)}-${ndigits(2)}-${ndigits(2)}$.", year, month, day): + ... + +]## + + +import macros, parseutils + +proc conditionsToIfChain(n, idx, res: NimNode; start: int): NimNode = + assert n.kind == nnkStmtList + if start >= n.len: return newAssignment(res, newLit true) + var ifs: NimNode = nil + if n[start+1].kind == nnkEmpty: + ifs = conditionsToIfChain(n, idx, res, start+3) + else: + ifs = newIfStmt((n[start+1], + newTree(nnkStmtList, newCall(bindSym"inc", idx, n[start+2]), + conditionsToIfChain(n, idx, res, start+3)))) + result = newTree(nnkStmtList, n[start], ifs) + +proc notZero(x: NimNode): NimNode = newCall(bindSym"!=", x, newLit 0) + +proc buildUserCall(x: string; args: varargs[NimNode]): NimNode = + let y = parseExpr(x) + result = newTree(nnkCall) + if y.kind in nnkCallKinds: result.add y[0] + else: result.add y + for a in args: result.add a + if y.kind in nnkCallKinds: + for i in 1..<y.len: result.add y[i] + +macro scanf*(input: string; pattern: static[string]; results: varargs[typed]): bool = + ## See top level documentation of his module of how ``scanf`` works. + template matchBind(parser) {.dirty.} = + var resLen = genSym(nskLet, "resLen") + conds.add newLetStmt(resLen, newCall(bindSym(parser), input, results[i], idx)) + conds.add resLen.notZero + conds.add resLen + + var i = 0 + var p = 0 + var idx = genSym(nskVar, "idx") + var res = genSym(nskVar, "res") + result = newTree(nnkStmtListExpr, newVarStmt(idx, newLit 0), newVarStmt(res, newLit false)) + var conds = newTree(nnkStmtList) + var fullMatch = false + while p < pattern.len: + if pattern[p] == '$': + inc p + case pattern[p] + of '$': + var resLen = genSym(nskLet, "resLen") + conds.add newLetStmt(resLen, newCall(bindSym"skip", input, newLit($pattern[p]), idx)) + conds.add resLen.notZero + conds.add resLen + of 'w': + if i < results.len or getType(results[i]).typeKind != ntyString: + matchBind "parseIdent" + else: + error("no string var given for $w") + inc i + of 'i': + if i < results.len or getType(results[i]).typeKind != ntyInt: + matchBind "parseInt" + else: + error("no int var given for $d") + inc i + of 'f': + if i < results.len or getType(results[i]).typeKind != ntyFloat: + matchBind "parseFloat" + else: + error("no float var given for $f") + inc i + of 's': + conds.add newCall(bindSym"inc", idx, newCall(bindSym"skipWhitespace", input, idx)) + conds.add newEmptyNode() + conds.add newEmptyNode() + of '.': + if p == pattern.len-1: + fullMatch = true + else: + error("invalid format string") + of '*', '+': + if i < results.len or getType(results[i]).typeKind != ntyString: + var min = ord(pattern[p] == '+') + var q=p+1 + var token = "" + while q < pattern.len and pattern[q] != '$': + token.add pattern[q] + inc q + var resLen = genSym(nskLet, "resLen") + conds.add newLetStmt(resLen, newCall(bindSym"parseUntil", input, results[i], newLit(token), idx)) + conds.add newCall(bindSym"!=", resLen, newLit min) + conds.add resLen + else: + error("no string var given for $" & pattern[p]) + inc i + of '{': + inc p + var nesting = 0 + let start = p + while true: + case pattern[p] + of '{': inc nesting + of '}': + if nesting == 0: break + dec nesting + of '\0': error("expected closing '}'") + else: discard + inc p + let expr = pattern.substr(start, p-1) + if i < results.len: + var resLen = genSym(nskLet, "resLen") + conds.add newLetStmt(resLen, buildUserCall(expr, input, results[i], idx)) + conds.add newCall(bindSym"!=", resLen, newLit 0) + conds.add resLen + else: + error("no var given for $" & expr) + inc i + of '[': + inc p + var nesting = 0 + let start = p + while true: + case pattern[p] + of '[': inc nesting + of ']': + if nesting == 0: break + dec nesting + of '\0': error("expected closing ']'") + else: discard + inc p + let expr = pattern.substr(start, p-1) + conds.add newCall(bindSym"inc", idx, buildUserCall(expr, input, idx)) + conds.add newEmptyNode() + conds.add newEmptyNode() + else: error("invalid format string") + inc p + else: + var token = "" + while p < pattern.len and pattern[p] != '$': + token.add pattern[p] + inc p + var resLen = genSym(nskLet, "resLen") + conds.add newLetStmt(resLen, newCall(bindSym"skip", input, newLit(token), idx)) + conds.add resLen.notZero + conds.add resLen + result.add conditionsToIfChain(conds, idx, res, 0) + if fullMatch: + result.add newCall(bindSym"and", res, + newCall(bindSym">=", idx, newCall(bindSym"len", input))) + else: + result.add res + +template atom*(input: string; idx: int; c: char): bool = + ## Used in scanp for the matching of atoms (usually chars). + input[idx] == c + +template atom*(input: string; idx: int; s: set[char]): bool = + input[idx] in s + +#template prepare*(input: string): int = 0 +template success*(x: int): bool = x != 0 + +template nxt*(input: string; idx, step: int = 1) = inc(idx, step) + +macro scanp*(input, idx: typed; pattern: varargs[untyped]): bool = + ## See top level documentation of his module of how ``scanp`` works. + type StmtTriple = tuple[init, cond, action: NimNode] + + template interf(x): untyped = bindSym(x, brForceOpen) + + proc toIfChain(n: seq[StmtTriple]; idx, res: NimNode; start: int): NimNode = + if start >= n.len: return newAssignment(res, newLit true) + var ifs: NimNode = nil + if n[start].cond.kind == nnkEmpty: + ifs = toIfChain(n, idx, res, start+1) + else: + ifs = newIfStmt((n[start].cond, + newTree(nnkStmtList, n[start].action, + toIfChain(n, idx, res, start+1)))) + result = newTree(nnkStmtList, n[start].init, ifs) + + proc attach(x, attached: NimNode): NimNode = + if attached == nil: x + else: newStmtList(attached, x) + + proc placeholder(n, x, j: NimNode): NimNode = + if n.kind == nnkPrefix and n[0].eqIdent("$"): + let n1 = n[1] + if n1.eqIdent"_" or n1.eqIdent"current": + result = newTree(nnkBracketExpr, x, j) + elif n1.eqIdent"input": + result = x + elif n1.eqIdent"i" or n1.eqIdent"index": + result = j + else: + error("unknown pattern " & repr(n)) + else: + result = copyNimNode(n) + for i in 0 ..< n.len: + result.add placeholder(n[i], x, j) + + proc atm(it, input, idx, attached: NimNode): StmtTriple = + template `!!`(x): untyped = attach(x, attached) + case it.kind + of nnkIdent: + var resLen = genSym(nskLet, "resLen") + result = (newLetStmt(resLen, newCall(it, input, idx)), + newCall(interf"success", resLen), + !!newCall(interf"nxt", input, idx, resLen)) + of nnkCallKinds: + # *{'A'..'Z'} !! s.add(!_) + template buildWhile(init, cond, action): untyped = + while true: + init + if not cond: break + action + + # (x) a # bind action a to (x) + if it[0].kind == nnkPar and it.len == 2: + result = atm(it[0], input, idx, placeholder(it[1], input, idx)) + elif it.kind == nnkInfix and it[0].eqIdent"->": + # bind matching to some action: + result = atm(it[1], input, idx, placeholder(it[2], input, idx)) + elif it.kind == nnkInfix and it[0].eqIdent"as": + let cond = if it[1].kind in nnkCallKinds: placeholder(it[1], input, idx) + else: newCall(it[1], input, idx) + result = (newLetStmt(it[2], cond), + newCall(interf"success", it[2]), + !!newCall(interf"nxt", input, idx, it[2])) + elif it.kind == nnkPrefix and it[0].eqIdent"*": + let (init, cond, action) = atm(it[1], input, idx, attached) + result = (getAst(buildWhile(init, cond, action)), + newEmptyNode(), newEmptyNode()) + elif it.kind == nnkPrefix and it[0].eqIdent"+": + # x+ is the same as xx* + result = atm(newTree(nnkPar, it[1], newTree(nnkPrefix, ident"*", it[1])), + input, idx, attached) + elif it.kind == nnkPrefix and it[0].eqIdent"?": + # optional. + let (init, cond, action) = atm(it[1], input, idx, attached) + if cond.kind == nnkEmpty: + error("'?' operator applied to a non-condition") + else: + result = (newTree(nnkStmtList, init, newIfStmt((cond, action))), + newEmptyNode(), newEmptyNode()) + elif it.kind == nnkPrefix and it[0].eqIdent"~": + # not operator + let (init, cond, action) = atm(it[1], input, idx, attached) + if cond.kind == nnkEmpty: + error("'~' operator applied to a non-condition") + else: + result = (init, newCall(bindSym"not", cond), action) + elif it.kind == nnkInfix and it[0].eqIdent"|": + let a = atm(it[1], input, idx, attached) + let b = atm(it[2], input, idx, attached) + if a.cond.kind == nnkEmpty or b.cond.kind == nnkEmpty: + error("'|' operator applied to a non-condition") + else: + result = (newStmtList(a.init, + newIfStmt((a.cond, a.action), (newTree(nnkStmtListExpr, b.init, b.cond), b.action))), + newEmptyNode(), newEmptyNode()) + elif it.kind == nnkInfix and it[0].eqIdent"^*": + # a ^* b is rewritten to: (a *(b a))? + #exprList = expr ^+ comma + template tmp(a, b): untyped = ?(a, *(b, a)) + result = atm(getAst(tmp(it[1], it[2])), input, idx, attached) + + elif it.kind == nnkInfix and it[0].eqIdent"^+": + # a ^* b is rewritten to: (a +(b a))? + template tmp(a, b): untyped = (a, *(b, a)) + result = atm(getAst(tmp(it[1], it[2])), input, idx, attached) + elif it.kind == nnkCommand and it.len == 2 and it[0].eqIdent"pred": + # enforce that the wrapped call is interpreted as a predicate, not a non-terminal: + result = (newEmptyNode(), placeholder(it[1], input, idx), newEmptyNode()) + else: + var resLen = genSym(nskLet, "resLen") + result = (newLetStmt(resLen, placeholder(it, input, idx)), + newCall(interf"success", resLen), !!newCall(interf"nxt", input, idx, resLen)) + of nnkStrLit..nnkTripleStrLit: + var resLen = genSym(nskLet, "resLen") + result = (newLetStmt(resLen, newCall(interf"skip", input, it, idx)), + newCall(interf"success", resLen), !!newCall(interf"nxt", input, idx, resLen)) + of nnkCurly, nnkAccQuoted, nnkCharLit: + result = (newEmptyNode(), newCall(interf"atom", input, idx, it), !!newCall(interf"nxt", input, idx)) + of nnkCurlyExpr: + if it.len == 3 and it[1].kind == nnkIntLit and it[2].kind == nnkIntLit: + var h = newTree(nnkPar, it[0]) + for count in 2..it[1].intVal: h.add(it[0]) + for count in it[1].intVal .. it[2].intVal-1: h.add(newTree(nnkPrefix, ident"?", it[0])) + result = atm(h, input, idx, attached) + elif it.len == 2 and it[1].kind == nnkIntLit: + var h = newTree(nnkPar, it[0]) + for count in 2..it[1].intVal: h.add(it[0]) + result = atm(h, input, idx, attached) + else: + error("invalid pattern") + of nnkPar: + if it.len == 1: + result = atm(it[0], input, idx, attached) + else: + # concatenation: + var conds: seq[StmtTriple] = @[] + for x in it: conds.add atm(x, input, idx, attached) + var res = genSym(nskVar, "res") + result = (newStmtList(newVarStmt(res, newLit false), + toIfChain(conds, idx, res, 0)), res, newEmptyNode()) + else: + error("invalid pattern") + + #var idx = genSym(nskVar, "idx") + var res = genSym(nskVar, "res") + result = newTree(nnkStmtListExpr, #newVarStmt(idx, newCall(interf"prepare", input)), + newVarStmt(res, newLit false)) + var conds: seq[StmtTriple] = @[] + for it in pattern: + conds.add atm(it, input, idx, nil) + result.add toIfChain(conds, idx, res, 0) + result.add res + when defined(debugScanp): + echo repr result + + +when isMainModule: + proc twoDigits(input: string; x: var int; start: int): int = + if input[start] == '0' and input[start+1] == '0': + result = 2 + x = 13 + else: + result = 0 + + proc someSep(input: string; start: int; seps: set[char] = {';',',','-','.'}): int = + result = 0 + while input[start+result] in seps: inc result + + proc demangle(s: string; res: var string; start: int): int = + while s[result+start] in {'_', '@'}: inc result + res = "" + while result+start < s.len and s[result+start] > ' ' and s[result+start] != '_': + res.add s[result+start] + inc result + while result+start < s.len and s[result+start] > ' ': + inc result + + proc parseGDB(resp: string): seq[string] = + const + digits = {'0'..'9'} + hexdigits = digits + {'a'..'f', 'A'..'F'} + whites = {' ', '\t', '\C', '\L'} + result = @[] + var idx = 0 + while true: + var prc = "" + var info = "" + if scanp(resp, idx, *`whites`, '#', *`digits`, +`whites`, ?("0x", *`hexdigits`, " in "), + demangle($input, prc, $index), *`whites`, '(', * ~ ')', ')', + *`whites`, "at ", +(~{'\C', '\L', '\0'} -> info.add($_)) ): + result.add prc & " " & info + else: + break + + var key, val: string + var intval: int + var floatval: float + doAssert scanf("abc:: xyz 89 33.25", "$w$s::$s$w$s$i $f", key, val, intval, floatVal) + doAssert key == "abc" + doAssert val == "xyz" + doAssert intval == 89 + doAssert floatVal == 33.25 + + let xx = scanf("$abc", "$$$i", intval) + doAssert xx == false + + + let xx2 = scanf("$1234", "$$$i", intval) + doAssert xx2 + + let yy = scanf(";.--Breakpoint00 [output]", "$[someSep]Breakpoint${twoDigits}$[someSep({';','.','-'})] [$+]$.", intVal, key) + doAssert yy + doAssert key == "output" + doAssert intVal == 13 + + var ident = "" + var idx = 0 + let zz = scanp("foobar x x x xWZ", idx, +{'a'..'z'} -> add(ident, $_), *(*{' ', '\t'}, "x"), ~'U', "Z") + doAssert zz + doAssert ident == "foobar" + + const digits = {'0'..'9'} + var year = 0 + var idx2 = 0 + if scanp("201655-8-9", idx2, `digits`{4,6} -> (year = year * 10 + ord($_) - ord('0')), "-8", "-9"): + doAssert year == 201655 + + const gdbOut = """ + #0 @foo_96013_1208911747@8 (x0=...) + at c:/users/anwender/projects/nim/temp.nim:11 + #1 0x00417754 in tempInit000 () at c:/users/anwender/projects/nim/temp.nim:13 + #2 0x0041768d in NimMainInner () + at c:/users/anwender/projects/nim/lib/system.nim:2605 + #3 0x004176b1 in NimMain () + at c:/users/anwender/projects/nim/lib/system.nim:2613 + #4 0x004176db in main (argc=1, args=0x712cc8, env=0x711ca8) + at c:/users/anwender/projects/nim/lib/system.nim:2620""" + const result = @["foo c:/users/anwender/projects/nim/temp.nim:11", + "tempInit000 c:/users/anwender/projects/nim/temp.nim:13", + "NimMainInner c:/users/anwender/projects/nim/lib/system.nim:2605", + "NimMain c:/users/anwender/projects/nim/lib/system.nim:2613", + "main c:/users/anwender/projects/nim/lib/system.nim:2620"] + doAssert parseGDB(gdbOut) == result diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index f2c1e77e1..bfc32bc71 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -14,6 +14,8 @@ ## <backends.html#the-javascript-target>`_. import parseutils +from math import pow, round, floor, log10 +from algorithm import reverse {.deadCodeElim: on.} @@ -24,6 +26,12 @@ include "system/inclrtl" {.pop.} +# Support old split with set[char] +when defined(nimOldSplit): + {.pragma: deprecatedSplit, deprecated.} +else: + {.pragma: deprecatedSplit.} + type CharSet* {.deprecated.} = set[char] # for compatibility with Nim {.deprecated: [TCharSet: CharSet].} @@ -62,8 +70,8 @@ const ## doAssert "01234".find(invalid) == -1 ## doAssert "01A34".find(invalid) == 2 -proc isAlpha*(c: char): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsAlphaChar".}= +proc isAlphaAscii*(c: char): bool {.noSideEffect, procvar, + rtl, extern: "nsuIsAlphaAsciiChar".}= ## Checks whether or not `c` is alphabetical. ## ## This checks a-z, A-Z ASCII characters only. @@ -83,27 +91,27 @@ proc isDigit*(c: char): bool {.noSideEffect, procvar, ## This checks 0-9 ASCII characters only. return c in Digits -proc isSpace*(c: char): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsSpaceChar".}= +proc isSpaceAscii*(c: char): bool {.noSideEffect, procvar, + rtl, extern: "nsuIsSpaceAsciiChar".}= ## Checks whether or not `c` is a whitespace character. return c in Whitespace -proc isLower*(c: char): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsLowerChar".}= +proc isLowerAscii*(c: char): bool {.noSideEffect, procvar, + rtl, extern: "nsuIsLowerAsciiChar".}= ## Checks whether or not `c` is a lower case character. ## ## This checks ASCII characters only. return c in {'a'..'z'} -proc isUpper*(c: char): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsUpperChar".}= +proc isUpperAscii*(c: char): bool {.noSideEffect, procvar, + rtl, extern: "nsuIsUpperAsciiChar".}= ## Checks whether or not `c` is an upper case character. ## ## This checks ASCII characters only. return c in {'A'..'Z'} -proc isAlpha*(s: string): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsAlphaStr".}= +proc isAlphaAscii*(s: string): bool {.noSideEffect, procvar, + rtl, extern: "nsuIsAlphaAsciiStr".}= ## Checks whether or not `s` is alphabetical. ## ## This checks a-z, A-Z ASCII characters only. @@ -115,7 +123,7 @@ proc isAlpha*(s: string): bool {.noSideEffect, procvar, result = true for c in s: - result = c.isAlpha() and result + result = c.isAlphaAscii() and result proc isAlphaNumeric*(s: string): bool {.noSideEffect, procvar, rtl, extern: "nsuIsAlphaNumericStr".}= @@ -147,8 +155,8 @@ proc isDigit*(s: string): bool {.noSideEffect, procvar, for c in s: result = c.isDigit() and result -proc isSpace*(s: string): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsSpaceStr".}= +proc isSpaceAscii*(s: string): bool {.noSideEffect, procvar, + rtl, extern: "nsuIsSpaceAsciiStr".}= ## Checks whether or not `s` is completely whitespace. ## ## Returns true if all characters in `s` are whitespace @@ -158,10 +166,11 @@ proc isSpace*(s: string): bool {.noSideEffect, procvar, result = true for c in s: - result = c.isSpace() and result + if not c.isSpaceAscii(): + return false -proc isLower*(s: string): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsLowerStr".}= +proc isLowerAscii*(s: string): bool {.noSideEffect, procvar, + rtl, extern: "nsuIsLowerAsciiStr".}= ## Checks whether or not `s` contains all lower case characters. ## ## This checks ASCII characters only. @@ -172,10 +181,10 @@ proc isLower*(s: string): bool {.noSideEffect, procvar, result = true for c in s: - result = c.isLower() and result + result = c.isLowerAscii() and result -proc isUpper*(s: string): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsUpperStr".}= +proc isUpperAscii*(s: string): bool {.noSideEffect, procvar, + rtl, extern: "nsuIsUpperAsciiStr".}= ## Checks whether or not `s` contains all upper case characters. ## ## This checks ASCII characters only. @@ -186,10 +195,10 @@ proc isUpper*(s: string): bool {.noSideEffect, procvar, result = true for c in s: - result = c.isUpper() and result + result = c.isUpperAscii() and result -proc toLower*(c: char): char {.noSideEffect, procvar, - rtl, extern: "nsuToLowerChar".} = +proc toLowerAscii*(c: char): char {.noSideEffect, procvar, + rtl, extern: "nsuToLowerAsciiChar".} = ## Converts `c` into lower case. ## ## This works only for the letters ``A-Z``. See `unicode.toLower @@ -200,8 +209,8 @@ proc toLower*(c: char): char {.noSideEffect, procvar, else: result = c -proc toLower*(s: string): string {.noSideEffect, procvar, - rtl, extern: "nsuToLowerStr".} = +proc toLowerAscii*(s: string): string {.noSideEffect, procvar, + rtl, extern: "nsuToLowerAsciiStr".} = ## Converts `s` into lower case. ## ## This works only for the letters ``A-Z``. See `unicode.toLower @@ -209,10 +218,10 @@ proc toLower*(s: string): string {.noSideEffect, procvar, ## character. result = newString(len(s)) for i in 0..len(s) - 1: - result[i] = toLower(s[i]) + result[i] = toLowerAscii(s[i]) -proc toUpper*(c: char): char {.noSideEffect, procvar, - rtl, extern: "nsuToUpperChar".} = +proc toUpperAscii*(c: char): char {.noSideEffect, procvar, + rtl, extern: "nsuToUpperAsciiChar".} = ## Converts `c` into upper case. ## ## This works only for the letters ``A-Z``. See `unicode.toUpper @@ -223,8 +232,8 @@ proc toUpper*(c: char): char {.noSideEffect, procvar, else: result = c -proc toUpper*(s: string): string {.noSideEffect, procvar, - rtl, extern: "nsuToUpperStr".} = +proc toUpperAscii*(s: string): string {.noSideEffect, procvar, + rtl, extern: "nsuToUpperAsciiStr".} = ## Converts `s` into upper case. ## ## This works only for the letters ``A-Z``. See `unicode.toUpper @@ -232,14 +241,145 @@ proc toUpper*(s: string): string {.noSideEffect, procvar, ## character. result = newString(len(s)) for i in 0..len(s) - 1: - result[i] = toUpper(s[i]) + result[i] = toUpperAscii(s[i]) + +proc capitalizeAscii*(s: string): string {.noSideEffect, procvar, + rtl, extern: "nsuCapitalizeAscii".} = + ## Converts the first character of `s` into upper case. + ## + ## This works only for the letters ``A-Z``. + result = toUpperAscii(s[0]) & substr(s, 1) + +proc isSpace*(c: char): bool {.noSideEffect, procvar, + rtl, deprecated, extern: "nsuIsSpaceChar".}= + ## Checks whether or not `c` is a whitespace character. + ## + ## **Deprecated since version 0.15.0**: use ``isSpaceAscii`` instead. + isSpaceAscii(c) + +proc isLower*(c: char): bool {.noSideEffect, procvar, + rtl, deprecated, extern: "nsuIsLowerChar".}= + ## Checks whether or not `c` is a lower case character. + ## + ## This checks ASCII characters only. + ## + ## **Deprecated since version 0.15.0**: use ``isLowerAscii`` instead. + isLowerAscii(c) + +proc isUpper*(c: char): bool {.noSideEffect, procvar, + rtl, deprecated, extern: "nsuIsUpperChar".}= + ## Checks whether or not `c` is an upper case character. + ## + ## This checks ASCII characters only. + ## + ## **Deprecated since version 0.15.0**: use ``isUpperAscii`` instead. + isUpperAscii(c) + +proc isAlpha*(c: char): bool {.noSideEffect, procvar, + rtl, deprecated, extern: "nsuIsAlphaChar".}= + ## Checks whether or not `c` is alphabetical. + ## + ## This checks a-z, A-Z ASCII characters only. + ## + ## **Deprecated since version 0.15.0**: use ``isAlphaAscii`` instead. + isAlphaAscii(c) + +proc isAlpha*(s: string): bool {.noSideEffect, procvar, + rtl, deprecated, extern: "nsuIsAlphaStr".}= + ## Checks whether or not `s` is alphabetical. + ## + ## This checks a-z, A-Z ASCII characters only. + ## Returns true if all characters in `s` are + ## alphabetic and there is at least one character + ## in `s`. + ## + ## **Deprecated since version 0.15.0**: use ``isAlphaAscii`` instead. + isAlphaAscii(s) + +proc isSpace*(s: string): bool {.noSideEffect, procvar, + rtl, deprecated, extern: "nsuIsSpaceStr".}= + ## Checks whether or not `s` is completely whitespace. + ## + ## Returns true if all characters in `s` are whitespace + ## characters and there is at least one character in `s`. + ## + ## **Deprecated since version 0.15.0**: use ``isSpaceAscii`` instead. + isSpaceAscii(s) + +proc isLower*(s: string): bool {.noSideEffect, procvar, + rtl, deprecated, extern: "nsuIsLowerStr".}= + ## Checks whether or not `s` contains all lower case characters. + ## + ## This checks ASCII characters only. + ## Returns true if all characters in `s` are lower case + ## and there is at least one character in `s`. + ## + ## **Deprecated since version 0.15.0**: use ``isLowerAscii`` instead. + isLowerAscii(s) + +proc isUpper*(s: string): bool {.noSideEffect, procvar, + rtl, deprecated, extern: "nsuIsUpperStr".}= + ## Checks whether or not `s` contains all upper case characters. + ## + ## This checks ASCII characters only. + ## Returns true if all characters in `s` are upper case + ## and there is at least one character in `s`. + ## + ## **Deprecated since version 0.15.0**: use ``isUpperAscii`` instead. + isUpperAscii(s) + +proc toLower*(c: char): char {.noSideEffect, procvar, + rtl, deprecated, extern: "nsuToLowerChar".} = + ## Converts `c` into lower case. + ## + ## This works only for the letters ``A-Z``. See `unicode.toLower + ## <unicode.html#toLower>`_ for a version that works for any Unicode + ## character. + ## + ## **Deprecated since version 0.15.0**: use ``toLowerAscii`` instead. + toLowerAscii(c) + +proc toLower*(s: string): string {.noSideEffect, procvar, + rtl, deprecated, extern: "nsuToLowerStr".} = + ## Converts `s` into lower case. + ## + ## This works only for the letters ``A-Z``. See `unicode.toLower + ## <unicode.html#toLower>`_ for a version that works for any Unicode + ## character. + ## + ## **Deprecated since version 0.15.0**: use ``toLowerAscii`` instead. + toLowerAscii(s) + +proc toUpper*(c: char): char {.noSideEffect, procvar, + rtl, deprecated, extern: "nsuToUpperChar".} = + ## Converts `c` into upper case. + ## + ## This works only for the letters ``A-Z``. See `unicode.toUpper + ## <unicode.html#toUpper>`_ for a version that works for any Unicode + ## character. + ## + ## **Deprecated since version 0.15.0**: use ``toUpperAscii`` instead. + toUpperAscii(c) + +proc toUpper*(s: string): string {.noSideEffect, procvar, + rtl, deprecated, extern: "nsuToUpperStr".} = + ## Converts `s` into upper case. + ## + ## This works only for the letters ``A-Z``. See `unicode.toUpper + ## <unicode.html#toUpper>`_ for a version that works for any Unicode + ## character. + ## + ## **Deprecated since version 0.15.0**: use ``toUpperAscii`` instead. + toUpperAscii(s) proc capitalize*(s: string): string {.noSideEffect, procvar, - rtl, extern: "nsuCapitalize".} = + rtl, deprecated, extern: "nsuCapitalize".} = ## Converts the first character of `s` into upper case. ## ## This works only for the letters ``A-Z``. - result = toUpper(s[0]) & substr(s, 1) + ## + ## **Deprecated since version 0.15.0**: use ``capitalizeAscii`` instead. + capitalizeAscii(s) proc normalize*(s: string): string {.noSideEffect, procvar, rtl, extern: "nsuNormalize".} = @@ -268,7 +408,7 @@ proc cmpIgnoreCase*(a, b: string): int {.noSideEffect, var i = 0 var m = min(a.len, b.len) while i < m: - result = ord(toLower(a[i])) - ord(toLower(b[i])) + result = ord(toLowerAscii(a[i])) - ord(toLowerAscii(b[i])) if result != 0: return inc(i) result = a.len - b.len @@ -289,8 +429,8 @@ proc cmpIgnoreStyle*(a, b: string): int {.noSideEffect, while true: while a[i] == '_': inc(i) while b[j] == '_': inc(j) # BUGFIX: typo - var aa = toLower(a[i]) - var bb = toLower(b[j]) + var aa = toLowerAscii(a[i]) + var bb = toLowerAscii(b[j]) result = ord(aa) - ord(bb) if result != 0 or aa == '\0': break inc(i) @@ -324,16 +464,77 @@ proc toOctal*(c: char): string {.noSideEffect, rtl, extern: "nsuToOctal".} = result[i] = chr(val mod 8 + ord('0')) val = val div 8 -iterator split*(s: string, seps: set[char] = Whitespace): string = +proc isNilOrEmpty*(s: string): bool {.noSideEffect, procvar, rtl, extern: "nsuIsNilOrEmpty".} = + ## Checks if `s` is nil or empty. + result = len(s) == 0 + +proc isNilOrWhitespace*(s: string): bool {.noSideEffect, procvar, rtl, extern: "nsuIsNilOrWhitespace".} = + ## Checks if `s` is nil or consists entirely of whitespace characters. + if len(s) == 0: + return true + + result = true + for c in s: + if not c.isSpaceAscii(): + return false + +proc substrEq(s: string, pos: int, substr: string): bool = + var i = 0 + var length = substr.len + while i < length and s[pos+i] == substr[i]: + inc i + + return i == length + +# --------- Private templates for different split separators ----------- + +template stringHasSep(s: string, index: int, seps: set[char]): bool = + s[index] in seps + +template stringHasSep(s: string, index: int, sep: char): bool = + s[index] == sep + +template stringHasSep(s: string, index: int, sep: string): bool = + s.substrEq(index, sep) + +template splitCommon(s, sep, maxsplit, sepLen) = + ## Common code for split procedures + var last = 0 + var splits = maxsplit + + if len(s) > 0: + while last <= len(s): + var first = last + while last < len(s) and not stringHasSep(s, last, sep): + inc(last) + if splits == 0: last = len(s) + yield substr(s, first, last-1) + if splits == 0: break + dec(splits) + inc(last, sepLen) + +template oldSplit(s, seps, maxsplit) = + var last = 0 + var splits = maxsplit + assert(not ('\0' in seps)) + while last < len(s): + while s[last] in seps: inc(last) + var first = last + while last < len(s) and s[last] notin seps: inc(last) + if first <= last-1: + if splits == 0: last = len(s) + yield substr(s, first, last-1) + if splits == 0: break + dec(splits) + +iterator split*(s: string, seps: set[char] = Whitespace, + maxsplit: int = -1): string = ## Splits the string `s` into substrings using a group of separators. ## - ## Substrings are separated by a substring containing only `seps`. Note - ## that whole sequences of characters found in ``seps`` will be counted as - ## a single split point and leading/trailing separators will be ignored. - ## The following example: + ## Substrings are separated by a substring containing only `seps`. ## ## .. code-block:: nim - ## for word in split(" this is an example "): + ## for word in split("this\lis an\texample"): ## writeLine(stdout, word) ## ## ...generates this output: @@ -347,7 +548,7 @@ iterator split*(s: string, seps: set[char] = Whitespace): string = ## And the following code: ## ## .. code-block:: nim - ## for word in split(";;this;is;an;;example;;;", {';'}): + ## for word in split("this:is;an$example", {';', ':', '$'}): ## writeLine(stdout, word) ## ## ...produces the same output as the first example. The code: @@ -368,22 +569,26 @@ iterator split*(s: string, seps: set[char] = Whitespace): string = ## "08" ## "08.398990" ## - var last = 0 - assert(not ('\0' in seps)) - while last < len(s): - while s[last] in seps: inc(last) - var first = last - while last < len(s) and s[last] notin seps: inc(last) # BUGFIX! - if first <= last-1: - yield substr(s, first, last-1) + when defined(nimOldSplit): + oldSplit(s, seps, maxsplit) + else: + splitCommon(s, seps, maxsplit, 1) + +iterator splitWhitespace*(s: string): string = + ## Splits at whitespace. + oldSplit(s, Whitespace, -1) -iterator split*(s: string, sep: char): string = +proc splitWhitespace*(s: string): seq[string] {.noSideEffect, + rtl, extern: "nsuSplitWhitespace".} = + ## The same as the `splitWhitespace <#splitWhitespace.i,string>`_ + ## iterator, but is a proc that returns a sequence of substrings. + accumulateResult(splitWhitespace(s)) + +iterator split*(s: string, sep: char, maxsplit: int = -1): string = ## Splits the string `s` into substrings using a single separator. ## ## Substrings are separated by the character `sep`. - ## Unlike the version of the iterator which accepts a set of separator - ## characters, this proc will not coalesce groups of the - ## separator, returning a string for each found character. The code: + ## The code: ## ## .. code-block:: nim ## for word in split(";;this;is;an;;example;;;", ';'): @@ -403,28 +608,118 @@ iterator split*(s: string, sep: char): string = ## "" ## "" ## - var last = 0 - assert('\0' != sep) - if len(s) > 0: - # `<=` is correct here for the edge cases! - while last <= len(s): - var first = last - while last < len(s) and s[last] != sep: inc(last) - yield substr(s, first, last-1) - inc(last) + splitCommon(s, sep, maxsplit, 1) -iterator split*(s: string, sep: string): string = +iterator split*(s: string, sep: string, maxsplit: int = -1): string = ## Splits the string `s` into substrings using a string separator. ## ## Substrings are separated by the string `sep`. - var last = 0 + ## The code: + ## + ## .. code-block:: nim + ## for word in split("thisDATAisDATAcorrupted", "DATA"): + ## writeLine(stdout, word) + ## + ## Results in: + ## + ## .. code-block:: + ## "this" + ## "is" + ## "corrupted" + ## + + splitCommon(s, sep, maxsplit, sep.len) + +template rsplitCommon(s, sep, maxsplit, sepLen) = + ## Common code for rsplit functions + var + last = s.len - 1 + first = last + splits = maxsplit + startPos = 0 + if len(s) > 0: - while last <= len(s): - var first = last - while last < len(s) and s.substr(last, last + <sep.len) != sep: - inc(last) - yield substr(s, first, last-1) - inc(last, sep.len) + # go to -1 in order to get separators at the beginning + while first >= -1: + while first >= 0 and not stringHasSep(s, first, sep): + dec(first) + + if splits == 0: + # No more splits means set first to the beginning + first = -1 + + if first == -1: + startPos = 0 + else: + startPos = first + sepLen + + yield substr(s, startPos, last) + + if splits == 0: + break + + dec(splits) + dec(first) + + last = first + +iterator rsplit*(s: string, seps: set[char] = Whitespace, + maxsplit: int = -1): string = + ## Splits the string `s` into substrings from the right using a + ## string separator. Works exactly the same as `split iterator + ## <#split.i,string,char>`_ except in reverse order. + ## + ## .. code-block:: nim + ## for piece in "foo bar".rsplit(WhiteSpace): + ## echo piece + ## + ## Results in: + ## + ## .. code-block:: nim + ## "bar" + ## "foo" + ## + ## Substrings are separated from the right by the set of chars `seps` + + rsplitCommon(s, seps, maxsplit, 1) + +iterator rsplit*(s: string, sep: char, + maxsplit: int = -1): string = + ## Splits the string `s` into substrings from the right using a + ## string separator. Works exactly the same as `split iterator + ## <#split.i,string,char>`_ except in reverse order. + ## + ## .. code-block:: nim + ## for piece in "foo:bar".rsplit(':'): + ## echo piece + ## + ## Results in: + ## + ## .. code-block:: nim + ## "bar" + ## "foo" + ## + ## Substrings are separated from the right by the char `sep` + rsplitCommon(s, sep, maxsplit, 1) + +iterator rsplit*(s: string, sep: string, maxsplit: int = -1, + keepSeparators: bool = false): string = + ## Splits the string `s` into substrings from the right using a + ## string separator. Works exactly the same as `split iterator + ## <#split.i,string,string>`_ except in reverse order. + ## + ## .. code-block:: nim + ## for piece in "foothebar".rsplit("the"): + ## echo piece + ## + ## Results in: + ## + ## .. code-block:: nim + ## "bar" + ## "foo" + ## + ## Substrings are separated from the right by the string `sep` + rsplitCommon(s, sep, maxsplit, sep.len) iterator splitLines*(s: string): string = ## Splits the string `s` into its containing lines. @@ -493,25 +788,92 @@ proc countLines*(s: string): int {.noSideEffect, else: discard inc i -proc split*(s: string, seps: set[char] = Whitespace): seq[string] {. +proc split*(s: string, seps: set[char] = Whitespace, maxsplit: int = -1): seq[string] {. noSideEffect, rtl, extern: "nsuSplitCharSet".} = ## The same as the `split iterator <#split.i,string,set[char]>`_, but is a ## proc that returns a sequence of substrings. - accumulateResult(split(s, seps)) + accumulateResult(split(s, seps, maxsplit)) -proc split*(s: string, sep: char): seq[string] {.noSideEffect, +proc split*(s: string, sep: char, maxsplit: int = -1): seq[string] {.noSideEffect, rtl, extern: "nsuSplitChar".} = ## The same as the `split iterator <#split.i,string,char>`_, but is a proc ## that returns a sequence of substrings. - accumulateResult(split(s, sep)) + accumulateResult(split(s, sep, maxsplit)) -proc split*(s: string, sep: string): seq[string] {.noSideEffect, +proc split*(s: string, sep: string, maxsplit: int = -1): seq[string] {.noSideEffect, rtl, extern: "nsuSplitString".} = ## Splits the string `s` into substrings using a string separator. ## ## Substrings are separated by the string `sep`. This is a wrapper around the ## `split iterator <#split.i,string,string>`_. - accumulateResult(split(s, sep)) + accumulateResult(split(s, sep, maxsplit)) + +proc rsplit*(s: string, seps: set[char] = Whitespace, + maxsplit: int = -1): seq[string] + {.noSideEffect, rtl, extern: "nsuRSplitCharSet".} = + ## The same as the `rsplit iterator <#rsplit.i,string,set[char]>`_, but is a + ## proc that returns a sequence of substrings. + ## + ## A possible common use case for `rsplit` is path manipulation, + ## particularly on systems that don't use a common delimiter. + ## + ## For example, if a system had `#` as a delimiter, you could + ## do the following to get the tail of the path: + ## + ## .. code-block:: nim + ## var tailSplit = rsplit("Root#Object#Method#Index", {'#'}, maxsplit=1) + ## + ## Results in `tailSplit` containing: + ## + ## .. code-block:: nim + ## @["Root#Object#Method", "Index"] + ## + accumulateResult(rsplit(s, seps, maxsplit)) + result.reverse() + +proc rsplit*(s: string, sep: char, maxsplit: int = -1): seq[string] + {.noSideEffect, rtl, extern: "nsuRSplitChar".} = + ## The same as the `split iterator <#rsplit.i,string,char>`_, but is a proc + ## that returns a sequence of substrings. + ## + ## A possible common use case for `rsplit` is path manipulation, + ## particularly on systems that don't use a common delimiter. + ## + ## For example, if a system had `#` as a delimiter, you could + ## do the following to get the tail of the path: + ## + ## .. code-block:: nim + ## var tailSplit = rsplit("Root#Object#Method#Index", '#', maxsplit=1) + ## + ## Results in `tailSplit` containing: + ## + ## .. code-block:: nim + ## @["Root#Object#Method", "Index"] + ## + accumulateResult(rsplit(s, sep, maxsplit)) + result.reverse() + +proc rsplit*(s: string, sep: string, maxsplit: int = -1): seq[string] + {.noSideEffect, rtl, extern: "nsuRSplitString".} = + ## The same as the `split iterator <#rsplit.i,string,string>`_, but is a proc + ## that returns a sequence of substrings. + ## + ## A possible common use case for `rsplit` is path manipulation, + ## particularly on systems that don't use a common delimiter. + ## + ## For example, if a system had `#` as a delimiter, you could + ## do the following to get the tail of the path: + ## + ## .. code-block:: nim + ## var tailSplit = rsplit("Root#Object#Method#Index", "#", maxsplit=1) + ## + ## Results in `tailSplit` containing: + ## + ## .. code-block:: nim + ## @["Root#Object#Method", "Index"] + ## + accumulateResult(rsplit(s, sep, maxsplit)) + result.reverse() proc toHex*(x: BiggestInt, len: Positive): string {.noSideEffect, rtl, extern: "nsuToHex".} = @@ -530,6 +892,10 @@ proc toHex*(x: BiggestInt, len: Positive): string {.noSideEffect, # handle negative overflow if n == 0 and x < 0: n = -1 +proc toHex*[T](x: T): string = + ## Shortcut for ``toHex(x, T.sizeOf * 2)`` + toHex(x, T.sizeOf * 2) + proc intToStr*(x: int, minchars: Positive = 1): string {.noSideEffect, rtl, extern: "nsuIntToStr".} = ## Converts `x` to its decimal representation. @@ -560,6 +926,24 @@ proc parseBiggestInt*(s: string): BiggestInt {.noSideEffect, procvar, if L != s.len or L == 0: raise newException(ValueError, "invalid integer: " & s) +proc parseUInt*(s: string): uint {.noSideEffect, procvar, + rtl, extern: "nsuParseUInt".} = + ## Parses a decimal unsigned integer value contained in `s`. + ## + ## If `s` is not a valid integer, `ValueError` is raised. + var L = parseutils.parseUInt(s, result, 0) + if L != s.len or L == 0: + raise newException(ValueError, "invalid unsigned integer: " & s) + +proc parseBiggestUInt*(s: string): uint64 {.noSideEffect, procvar, + rtl, extern: "nsuParseBiggestUInt".} = + ## Parses a decimal unsigned integer value contained in `s`. + ## + ## If `s` is not a valid integer, `ValueError` is raised. + var L = parseutils.parseBiggestUInt(s, result, 0) + if L != s.len or L == 0: + raise newException(ValueError, "invalid unsigned integer: " & s) + proc parseFloat*(s: string): float {.noSideEffect, procvar, rtl, extern: "nsuParseFloat".} = ## Parses a decimal floating point value contained in `s`. If `s` is not @@ -651,7 +1035,7 @@ proc repeat*(s: string, n: Natural): string {.noSideEffect, result = newStringOfCap(n * s.len) for i in 1..n: result.add(s) -template spaces*(n: Natural): string = repeat(' ',n) +template spaces*(n: Natural): string = repeat(' ', n) ## Returns a String with `n` space characters. You can use this proc ## to left align strings. Example: ## @@ -766,8 +1150,7 @@ proc indent*(s: string, count: Natural, padding: string = " "): string {.noSideEffect, rtl, extern: "nsuIndent".} = ## Indents each line in ``s`` by ``count`` amount of ``padding``. ## - ## **Note:** This currently does not preserve the specific new line characters - ## used. + ## **Note:** This does not preserve the new line characters used in ``s``. result = "" var i = 0 for line in s.splitLines(): @@ -778,32 +1161,39 @@ proc indent*(s: string, count: Natural, padding: string = " "): string result.add(line) i.inc -proc unindent*(s: string, eatAllIndent = false): string {. - noSideEffect, rtl, extern: "nsuUnindent".} = - ## Unindents `s`. - result = newStringOfCap(s.len) +proc unindent*(s: string, count: Natural, padding: string = " "): string + {.noSideEffect, rtl, extern: "nsuUnindent".} = + ## Unindents each line in ``s`` by ``count`` amount of ``padding``. + ## + ## **Note:** This does not preserve the new line characters used in ``s``. + result = "" var i = 0 - var pattern = true - var indent = 0 - while s[i] == ' ': inc i - var level = if i == 0: -1 else: i - while i < s.len: - if s[i] == ' ': - if i > 0 and s[i-1] in {'\l', '\c'}: - pattern = true - indent = 0 - if pattern: - inc(indent) - if indent > level and not eatAllIndent: - result.add(s[i]) - if level < 0: level = indent - else: - # a space somewhere: do not delete - result.add(s[i]) - else: - pattern = false - result.add(s[i]) - inc i + for line in s.splitLines(): + if i != 0: + result.add("\n") + var indentCount = 0 + for j in 0..<count.int: + indentCount.inc + if line[j .. j + <padding.len] != padding: + indentCount = j + break + result.add(line[indentCount*padding.len .. ^1]) + i.inc + +proc unindent*(s: string): string + {.noSideEffect, rtl, extern: "nsuUnindentAll".} = + ## Removes all indentation composed of whitespace from each line in ``s``. + ## + ## For example: + ## + ## .. code-block:: nim + ## const x = """ + ## Hello + ## There + ## """.unindent() + ## + ## doAssert x == "Hello\nThere\n" + unindent(s, 1000) # TODO: Passing a 1000 is a bit hackish. proc startsWith*(s, prefix: string): bool {.noSideEffect, rtl, extern: "nsuStartsWith".} = @@ -816,6 +1206,10 @@ proc startsWith*(s, prefix: string): bool {.noSideEffect, if s[i] != prefix[i]: return false inc(i) +proc startsWith*(s: string, prefix: char): bool {.noSideEffect, inline.} = + ## Returns true iff ``s`` starts with ``prefix``. + result = s[0] == prefix + proc endsWith*(s, suffix: string): bool {.noSideEffect, rtl, extern: "nsuEndsWith".} = ## Returns true iff ``s`` ends with ``suffix``. @@ -828,6 +1222,10 @@ proc endsWith*(s, suffix: string): bool {.noSideEffect, inc(i) if suffix[i] == '\0': return true +proc endsWith*(s: string, suffix: char): bool {.noSideEffect, inline.} = + ## Returns true iff ``s`` ends with ``suffix``. + result = s[s.high] == suffix + proc continuesWith*(s, substr: string, start: Natural): bool {.noSideEffect, rtl, extern: "nsuContinuesWith".} = ## Returns true iff ``s`` continues with ``substr`` at position ``start``. @@ -981,6 +1379,34 @@ proc rfind*(s: string, sub: char, start: int = -1): int {.noSideEffect, if sub == s[i]: return i return -1 +proc center*(s: string, width: int, fillChar: char = ' '): string {. + noSideEffect, rtl, extern: "nsuCenterString".} = + ## Return the contents of `s` centered in a string `width` long using + ## `fillChar` as padding. + ## + ## The original string is returned if `width` is less than or equal + ## to `s.len`. + if width <= s.len: + return s + + result = newString(width) + + # Left padding will be one fillChar + # smaller if there are an odd number + # of characters + let + charsLeft = (width - s.len) + leftPadding = charsLeft div 2 + + for i in 0 ..< width: + if i >= leftPadding and i < leftPadding + s.len: + # we are where the string should be located + result[i] = s[i-leftPadding] + else: + # we are either before or after where + # the string s should go + result[i] = fillChar + proc count*(s: string, sub: string, overlapping: bool = false): int {. noSideEffect, rtl, extern: "nsuCountString".} = ## Count the occurrences of a substring `sub` in the string `s`. @@ -1420,28 +1846,216 @@ proc formatFloat*(f: float, format: FloatFormatMode = ffDefault, ## after the decimal point for Nim's ``float`` type. result = formatBiggestFloat(f, format, precision, decimalSep) -proc formatSize*(bytes: BiggestInt, decimalSep = '.'): string = - ## Rounds and formats `bytes`. Examples: +proc trimZeros*(x: var string) {.noSideEffect.} = + ## Trim trailing zeros from a formatted floating point + ## value (`x`). Modifies the passed value. + var spl: seq[string] + if x.contains('.') or x.contains(','): + if x.contains('e'): + spl= x.split('e') + x = spl[0] + while x[x.high] == '0': + x.setLen(x.len-1) + if x[x.high] in [',', '.']: + x.setLen(x.len-1) + if spl.len > 0: + x &= "e" & spl[1] + +type + BinaryPrefixMode* = enum ## the different names for binary prefixes + bpIEC, # use the IEC/ISO standard prefixes such as kibi + bpColloquial # use the colloquial kilo, mega etc + +proc formatSize*(bytes: int64, + decimalSep = '.', + prefix = bpIEC, + includeSpace = false): string {.noSideEffect.} = + ## Rounds and formats `bytes`. + ## + ## By default, uses the IEC/ISO standard binary prefixes, so 1024 will be + ## formatted as 1KiB. Set prefix to `bpColloquial` to use the colloquial + ## names from the SI standard (e.g. k for 1000 being reused as 1024). + ## + ## `includeSpace` can be set to true to include the (SI preferred) space + ## between the number and the unit (e.g. 1 KiB). + ## + ## Examples: ## ## .. code-block:: nim ## - ## formatSize(1'i64 shl 31 + 300'i64) == "2.204GB" - ## formatSize(4096) == "4KB" - ## - template frmt(a, b, c: expr): expr = - let bs = $b - insertSep($a) & decimalSep & bs.substr(0, 2) & c - let gigabytes = bytes shr 30 - let megabytes = bytes shr 20 - let kilobytes = bytes shr 10 - if gigabytes != 0: - result = frmt(gigabytes, megabytes, "GB") - elif megabytes != 0: - result = frmt(megabytes, kilobytes, "MB") - elif kilobytes != 0: - result = frmt(kilobytes, bytes, "KB") + ## formatSize((1'i64 shl 31) + (300'i64 shl 20)) == "2.293GiB" + ## formatSize((2.234*1024*1024).int) == "2.234MiB" + ## formatSize(4096, includeSpace=true) == "4 KiB" + ## formatSize(4096, prefix=bpColloquial, includeSpace=true) == "4 kB" + ## formatSize(4096) == "4KiB" + ## formatSize(5_378_934, prefix=bpColloquial, decimalSep=',') == "5,13MB" + ## + const iecPrefixes = ["", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi", "Yi"] + const collPrefixes = ["", "k", "M", "G", "T", "P", "E", "Z", "Y"] + var + xb: int64 = bytes + fbytes: float + last_xb: int64 = bytes + matchedIndex: int + prefixes: array[9, string] + if prefix == bpColloquial: + prefixes = collPrefixes else: - result = insertSep($bytes) & "B" + prefixes = iecPrefixes + + # Iterate through prefixes seeing if value will be greater than + # 0 in each case + for index in 1..<prefixes.len: + last_xb = xb + xb = bytes div (1'i64 shl (index*10)) + matchedIndex = index + if xb == 0: + xb = last_xb + matchedIndex = index - 1 + break + # xb has the integer number for the latest value; index should be correct + fbytes = bytes.float / (1'i64 shl (matchedIndex*10)).float + result = formatFloat(fbytes, format=ffDecimal, precision=3, decimalSep=decimalSep) + result.trimZeros() + if includeSpace: + result &= " " + result &= prefixes[matchedIndex] + result &= "B" + +proc formatEng*(f: BiggestFloat, + precision: range[0..32] = 10, + trim: bool = true, + siPrefix: bool = false, + unit: string = nil, + decimalSep = '.'): string {.noSideEffect.} = + ## Converts a floating point value `f` to a string using engineering notation. + ## + ## Numbers in of the range -1000.0<f<1000.0 will be formatted without an + ## exponent. Numbers outside of this range will be formatted as a + ## significand in the range -1000.0<f<1000.0 and an exponent that will always + ## be an integer multiple of 3, corresponding with the SI prefix scale k, M, + ## G, T etc for numbers with an absolute value greater than 1 and m, μ, n, p + ## etc for numbers with an absolute value less than 1. + ## + ## The default configuration (`trim=true` and `precision=10`) shows the + ## **shortest** form that precisely (up to a maximum of 10 decimal places) + ## displays the value. For example, 4.100000 will be displayed as 4.1 (which + ## is mathematically identical) whereas 4.1000003 will be displayed as + ## 4.1000003. + ## + ## If `trim` is set to true, trailing zeros will be removed; if false, the + ## number of digits specified by `precision` will always be shown. + ## + ## `precision` can be used to set the number of digits to be shown after the + ## decimal point or (if `trim` is true) the maximum number of digits to be + ## shown. + ## + ## .. code-block:: nim + ## + ## formatEng(0, 2, trim=false) == "0.00" + ## formatEng(0, 2) == "0" + ## formatEng(0.053, 0) == "53e-3" + ## formatEng(52731234, 2) == "52.73e6" + ## formatEng(-52731234, 2) == "-52.73e6" + ## + ## If `siPrefix` is set to true, the number will be displayed with the SI + ## prefix corresponding to the exponent. For example 4100 will be displayed + ## as "4.1 k" instead of "4.1e3". Note that `u` is used for micro- in place + ## of the greek letter mu (μ) as per ISO 2955. Numbers with an absolute + ## value outside of the range 1e-18<f<1000e18 (1a<f<1000E) will be displayed + ## with an exponent rather than an SI prefix, regardless of whether + ## `siPrefix` is true. + ## + ## If `unit` is not nil, the provided unit will be appended to the string + ## (with a space as required by the SI standard). This behaviour is slightly + ## different to appending the unit to the result as the location of the space + ## is altered depending on whether there is an exponent. + ## + ## .. code-block:: nim + ## + ## formatEng(4100, siPrefix=true, unit="V") == "4.1 kV" + ## formatEng(4.1, siPrefix=true, unit="V") == "4.1 V" + ## formatEng(4.1, siPrefix=true) == "4.1" # Note lack of space + ## formatEng(4100, siPrefix=true) == "4.1 k" + ## formatEng(4.1, siPrefix=true, unit="") == "4.1 " # Space with unit="" + ## formatEng(4100, siPrefix=true, unit="") == "4.1 k" + ## formatEng(4100) == "4.1e3" + ## formatEng(4100, unit="V") == "4.1e3 V" + ## formatEng(4100, unit="") == "4.1e3 " # Space with unit="" + ## + ## `decimalSep` is used as the decimal separator + var + absolute: BiggestFloat + significand: BiggestFloat + fexponent: BiggestFloat + exponent: int + splitResult: seq[string] + suffix: string = "" + proc getPrefix(exp: int): char = + ## Get the SI prefix for a given exponent + ## + ## Assumes exponent is a multiple of 3; returns ' ' if no prefix found + const siPrefixes = ['a','f','p','n','u','m',' ','k','M','G','T','P','E'] + var index: int = (exp div 3) + 6 + result = ' ' + if index in low(siPrefixes)..high(siPrefixes): + result = siPrefixes[index] + + # Most of the work is done with the sign ignored, so get the absolute value + absolute = abs(f) + significand = f + + if absolute == 0.0: + # Simple case: just format it and force the exponent to 0 + exponent = 0 + result = significand.formatBiggestFloat(ffDecimal, precision, decimalSep='.') + else: + # Find the best exponent that's a multiple of 3 + fexponent = round(floor(log10(absolute))) + fexponent = 3.0 * round(floor(fexponent / 3.0)) + # Adjust the significand for the new exponent + significand /= pow(10.0, fexponent) + + # Round the significand and check whether it has affected + # the exponent + significand = round(significand, precision) + absolute = abs(significand) + if absolute >= 1000.0: + significand *= 0.001 + fexponent += 3 + # Components of the result: + result = significand.formatBiggestFloat(ffDecimal, precision, decimalSep='.') + exponent = fexponent.int() + + splitResult = result.split('.') + result = splitResult[0] + # result should have at most one decimal character + if splitResult.len() > 1: + # If trim is set, we get rid of trailing zeros. Don't use trimZeros here as + # we can be a bit more efficient through knowledge that there will never be + # an exponent in this part. + if trim: + while splitResult[1].endsWith("0"): + # Trim last character + splitResult[1].setLen(splitResult[1].len-1) + if splitResult[1].len() > 0: + result &= decimalSep & splitResult[1] + else: + result &= decimalSep & splitResult[1] + + # Combine the results accordingly + if siPrefix and exponent != 0: + var p = getPrefix(exponent) + if p != ' ': + suffix = " " & p + exponent = 0 # Exponent replaced by SI prefix + if suffix == "" and unit != nil: + suffix = " " + if unit != nil: + suffix &= unit + if exponent != 0: + result &= "e" & $exponent + result &= suffix proc findNormalized(x: string, inArray: openArray[string]): int = var i = 0 @@ -1637,9 +2251,14 @@ when isMainModule: ["1,0e-11", "1,0e-011"] doAssert "$# $3 $# $#" % ["a", "b", "c"] == "a c b c" - when not defined(testing): - echo formatSize(1'i64 shl 31 + 300'i64) # == "4,GB" - echo formatSize(1'i64 shl 31) + + block: # formatSize tests + doAssert formatSize((1'i64 shl 31) + (300'i64 shl 20)) == "2.293GiB" + doAssert formatSize((2.234*1024*1024).int) == "2.234MiB" + doAssert formatSize(4096) == "4KiB" + doAssert formatSize(4096, prefix=bpColloquial, includeSpace=true) == "4 kB" + doAssert formatSize(4096, includeSpace=true) == "4 KiB" + doAssert formatSize(5_378_934, prefix=bpColloquial, decimalSep=',') == "5,13MB" doAssert "$animal eats $food." % ["animal", "The cat", "food", "fish"] == "The cat eats fish." @@ -1652,6 +2271,11 @@ when isMainModule: doAssert parseEnum("invalid enum value", enC) == enC + doAssert center("foo", 13) == " foo " + doAssert center("foo", 0) == "foo" + doAssert center("foo", 3, fillChar = 'a') == "foo" + doAssert center("foo", 10, fillChar = '\t') == "\t\t\tfoo\t\t\t\t" + doAssert count("foofoofoo", "foofoo") == 1 doAssert count("foofoofoo", "foofoo", overlapping = true) == 2 doAssert count("foofoofoo", 'f') == 3 @@ -1668,13 +2292,13 @@ when isMainModule: doAssert " foo\n bar".indent(4, "Q") == "QQQQ foo\nQQQQ bar" - doAssert isAlpha('r') - doAssert isAlpha('A') - doAssert(not isAlpha('$')) + doAssert isAlphaAscii('r') + doAssert isAlphaAscii('A') + doAssert(not isAlphaAscii('$')) - doAssert isAlpha("Rasp") - doAssert isAlpha("Args") - doAssert(not isAlpha("$Tomato")) + doAssert isAlphaAscii("Rasp") + doAssert isAlphaAscii("Args") + doAssert(not isAlphaAscii("$Tomato")) doAssert isAlphaNumeric('3') doAssert isAlphaNumeric('R') @@ -1693,35 +2317,131 @@ when isMainModule: doAssert(not isDigit("12.33")) doAssert(not isDigit("A45b")) - doAssert isSpace('\t') - doAssert isSpace('\l') - doAssert(not isSpace('A')) - - doAssert isSpace("\t\l \v\r\f") - doAssert isSpace(" ") - doAssert(not isSpace("ABc \td")) + doAssert isSpaceAscii('\t') + doAssert isSpaceAscii('\l') + doAssert(not isSpaceAscii('A')) + + doAssert isSpaceAscii("\t\l \v\r\f") + doAssert isSpaceAscii(" ") + doAssert(not isSpaceAscii("ABc \td")) + + doAssert(isNilOrEmpty("")) + doAssert(isNilOrEmpty(nil)) + doAssert(not isNilOrEmpty("test")) + doAssert(not isNilOrEmpty(" ")) + + doAssert(isNilOrWhitespace("")) + doAssert(isNilOrWhitespace(nil)) + doAssert(isNilOrWhitespace(" ")) + doAssert(isNilOrWhitespace("\t\l \v\r\f")) + doAssert(not isNilOrWhitespace("ABc \td")) + + doAssert isLowerAscii('a') + doAssert isLowerAscii('z') + doAssert(not isLowerAscii('A')) + doAssert(not isLowerAscii('5')) + doAssert(not isLowerAscii('&')) + + doAssert isLowerAscii("abcd") + doAssert(not isLowerAscii("abCD")) + doAssert(not isLowerAscii("33aa")) + + doAssert isUpperAscii('A') + doAssert(not isUpperAscii('b')) + doAssert(not isUpperAscii('5')) + doAssert(not isUpperAscii('%')) + + doAssert isUpperAscii("ABC") + doAssert(not isUpperAscii("AAcc")) + doAssert(not isUpperAscii("A#$")) + + doAssert rsplit("foo bar", seps=Whitespace) == @["foo", "bar"] + doAssert rsplit(" foo bar", seps=Whitespace, maxsplit=1) == @[" foo", "bar"] + doAssert rsplit(" foo bar ", seps=Whitespace, maxsplit=1) == @[" foo bar", ""] + doAssert rsplit(":foo:bar", sep=':') == @["", "foo", "bar"] + doAssert rsplit(":foo:bar", sep=':', maxsplit=2) == @["", "foo", "bar"] + doAssert rsplit(":foo:bar", sep=':', maxsplit=3) == @["", "foo", "bar"] + doAssert rsplit("foothebar", sep="the") == @["foo", "bar"] - doAssert isLower('a') - doAssert isLower('z') - doAssert(not isLower('A')) - doAssert(not isLower('5')) - doAssert(not isLower('&')) - - doAssert isLower("abcd") - doAssert(not isLower("abCD")) - doAssert(not isLower("33aa")) - - doAssert isUpper('A') - doAssert(not isUpper('b')) - doAssert(not isUpper('5')) - doAssert(not isUpper('%')) - - doAssert isUpper("ABC") - doAssert(not isUpper("AAcc")) - doAssert(not isUpper("A#$")) doAssert(unescape(r"\x013", "", "") == "\x013") doAssert join(["foo", "bar", "baz"]) == "foobarbaz" doAssert join(@["foo", "bar", "baz"], ", ") == "foo, bar, baz" doAssert join([1, 2, 3]) == "123" doAssert join(@[1, 2, 3], ", ") == "1, 2, 3" + + doAssert """~~!!foo +~~!!bar +~~!!baz""".unindent(2, "~~!!") == "foo\nbar\nbaz" + + doAssert """~~!!foo +~~!!bar +~~!!baz""".unindent(2, "~~!!aa") == "~~!!foo\n~~!!bar\n~~!!baz" + doAssert """~~foo +~~ bar +~~ baz""".unindent(4, "~") == "foo\n bar\n baz" + doAssert """foo +bar + baz + """.unindent(4) == "foo\nbar\nbaz\n" + doAssert """foo + bar + baz + """.unindent(2) == "foo\n bar\n baz\n" + doAssert """foo + bar + baz + """.unindent(100) == "foo\nbar\nbaz\n" + + doAssert """foo + foo + bar + """.unindent() == "foo\nfoo\nbar\n" + + let s = " this is an example " + let s2 = ":this;is;an:example;;" + + doAssert s.split() == @["", "this", "is", "an", "example", "", ""] + doAssert s2.split(seps={':', ';'}) == @["", "this", "is", "an", "example", "", ""] + doAssert s.split(maxsplit=4) == @["", "this", "is", "an", "example "] + doAssert s.split(' ', maxsplit=1) == @["", "this is an example "] + doAssert s.split(" ", maxsplit=4) == @["", "this", "is", "an", "example "] + + block: # formatEng tests + doAssert formatEng(0, 2, trim=false) == "0.00" + doAssert formatEng(0, 2) == "0" + doAssert formatEng(53, 2, trim=false) == "53.00" + doAssert formatEng(0.053, 2, trim=false) == "53.00e-3" + doAssert formatEng(0.053, 4, trim=false) == "53.0000e-3" + doAssert formatEng(0.053, 4, trim=true) == "53e-3" + doAssert formatEng(0.053, 0) == "53e-3" + doAssert formatEng(52731234) == "52.731234e6" + doAssert formatEng(-52731234) == "-52.731234e6" + doAssert formatEng(52731234, 1) == "52.7e6" + doAssert formatEng(-52731234, 1) == "-52.7e6" + doAssert formatEng(52731234, 1, decimalSep=',') == "52,7e6" + doAssert formatEng(-52731234, 1, decimalSep=',') == "-52,7e6" + + doAssert formatEng(4100, siPrefix=true, unit="V") == "4.1 kV" + doAssert formatEng(4.1, siPrefix=true, unit="V") == "4.1 V" + doAssert formatEng(4.1, siPrefix=true) == "4.1" # Note lack of space + doAssert formatEng(4100, siPrefix=true) == "4.1 k" + doAssert formatEng(4.1, siPrefix=true, unit="") == "4.1 " # Includes space + doAssert formatEng(4100, siPrefix=true, unit="") == "4.1 k" + doAssert formatEng(4100) == "4.1e3" + doAssert formatEng(4100, unit="V") == "4.1e3 V" + doAssert formatEng(4100, unit="") == "4.1e3 " # Space with unit="" + # Don't use SI prefix as number is too big + doAssert formatEng(3.1e22, siPrefix=true, unit="a") == "31e21 a" + # Don't use SI prefix as number is too small + doAssert formatEng(3.1e-25, siPrefix=true, unit="A") == "310e-27 A" + + block: # startsWith / endsWith char tests + var s = "abcdef" + doAssert s.startsWith('a') + doAssert s.startsWith('b') == false + doAssert s.endsWith('f') + doAssert s.endsWith('a') == false + doAssert s.endsWith('\0') == false + + #echo("strutils tests passed") diff --git a/lib/pure/subexes.nim b/lib/pure/subexes.nim index 5824ace81..351b3c086 100644 --- a/lib/pure/subexes.nim +++ b/lib/pure/subexes.nim @@ -9,7 +9,7 @@ ## Nim support for `substitution expressions`:idx: (`subex`:idx:). ## -## .. include:: ../doc/subexes.txt +## .. include:: ../../doc/subexes.txt ## {.push debugger:off .} # the user does not want to trace a part @@ -390,11 +390,11 @@ when isMainModule: doAssert(("type MyEnum* = enum\n $', '2i'\n '{..}" % ["fieldA", "fieldB", "FiledClkad", "fieldD", "fieldE", "longishFieldName"]).replace(" \n", "\n") == - strutils.unindent """ + strutils.unindent(""" type MyEnum* = enum fieldA, fieldB, FiledClkad, fieldD, - fieldE, longishFieldName""") + fieldE, longishFieldName""", 6)) doAssert subex"$1($', '{2..})" % ["f", "a", "b", "c"] == "f(a, b, c)" @@ -404,8 +404,8 @@ when isMainModule: doAssert((subex("type\n Enum = enum\n $', '40c'\n '{..}") % [ "fieldNameA", "fieldNameB", "fieldNameC", "fieldNameD"]).replace(" \n", "\n") == - strutils.unindent """ + strutils.unindent(""" type Enum = enum fieldNameA, fieldNameB, fieldNameC, - fieldNameD""") + fieldNameD""", 6)) diff --git a/lib/pure/terminal.nim b/lib/pure/terminal.nim index 60f064e7c..1f34ec07e 100644 --- a/lib/pure/terminal.nim +++ b/lib/pure/terminal.nim @@ -384,7 +384,7 @@ proc setForegroundColor*(f: File, fg: ForegroundColor, bright=false) = var old = getAttributes(h) and not 0x0007 if bright: old = old or FOREGROUND_INTENSITY - const lookup: array [ForegroundColor, int] = [ + const lookup: array[ForegroundColor, int] = [ 0, (FOREGROUND_RED), (FOREGROUND_GREEN), @@ -406,7 +406,7 @@ proc setBackgroundColor*(f: File, bg: BackgroundColor, bright=false) = var old = getAttributes(h) and not 0x0070 if bright: old = old or BACKGROUND_INTENSITY - const lookup: array [BackgroundColor, int] = [ + const lookup: array[BackgroundColor, int] = [ 0, (BACKGROUND_RED), (BACKGROUND_GREEN), @@ -493,17 +493,21 @@ template styledEcho*(args: varargs[expr]): expr = ## Echoes styles arguments to stdout using ``styledWriteLine``. callStyledEcho(args) -when defined(nimdoc): - proc getch*(): char = - ## Read a single character from the terminal, blocking until it is entered. - ## The character is not printed to the terminal. This is not available for - ## Windows. - discard -elif not defined(windows): - proc getch*(): char = - ## Read a single character from the terminal, blocking until it is entered. - ## The character is not printed to the terminal. This is not available for - ## Windows. +proc getch*(): char = + ## Read a single character from the terminal, blocking until it is entered. + ## The character is not printed to the terminal. + when defined(windows): + let fd = getStdHandle(STD_INPUT_HANDLE) + var keyEvent = KEY_EVENT_RECORD() + var numRead: cint + while true: + # Block until character is entered + doAssert(waitForSingleObject(fd, INFINITE) == WAIT_OBJECT_0) + doAssert(readConsoleInput(fd, addr(keyEvent), 1, addr(numRead)) != 0) + if numRead == 0 or keyEvent.eventType != 1 or keyEvent.bKeyDown == 0: + continue + return char(keyEvent.uChar) + else: let fd = getFileHandle(stdin) var oldMode: Termios discard fd.tcgetattr(addr oldMode) diff --git a/lib/pure/times.nim b/lib/pure/times.nim index 29ae52d47..d6eb29e1c 100644 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -64,8 +64,9 @@ when defined(posix) and not defined(JS): proc posix_gettimeofday(tp: var Timeval, unused: pointer = nil) {. importc: "gettimeofday", header: "<sys/time.h>".} + when not defined(freebsd) and not defined(netbsd) and not defined(openbsd): + var timezone {.importc, header: "<time.h>".}: int var - timezone {.importc, header: "<time.h>".}: int tzname {.importc, header: "<time.h>" .}: array[0..1, cstring] # we also need tzset() to make sure that tzname is initialized proc tzset() {.importc, header: "<time.h>".} @@ -94,7 +95,8 @@ elif defined(windows): elif defined(JS): type - Time* {.importc.} = object + Time* = ref TimeObj + TimeObj {.importc.} = object getDay: proc (): int {.tags: [], raises: [], benign.} getFullYear: proc (): int {.tags: [], raises: [], benign.} getHours: proc (): int {.tags: [], raises: [], benign.} @@ -182,7 +184,16 @@ proc getGMTime*(t: Time): TimeInfo {.tags: [TimeEffect], raises: [], benign.} ## converts the calendar time `t` to broken-down time representation, ## expressed in Coordinated Universal Time (UTC). -proc timeInfoToTime*(timeInfo: TimeInfo): Time {.tags: [], benign.} +proc timeInfoToTime*(timeInfo: TimeInfo): Time {.tags: [], benign, deprecated.} + ## converts a broken-down time structure to + ## calendar time representation. The function ignores the specified + ## contents of the structure members `weekday` and `yearday` and recomputes + ## them from the other information in the broken-down time structure. + ## + ## **Warning:** This procedure is deprecated since version 0.14.0. + ## Use ``toTime`` instead. + +proc toTime*(timeInfo: TimeInfo): Time {.tags: [], benign.} ## converts a broken-down time structure to ## calendar time representation. The function ignores the specified ## contents of the structure members `weekday` and `yearday` and recomputes @@ -356,16 +367,19 @@ proc `+`*(a: TimeInfo, interval: TimeInterval): TimeInfo = ## ## **Note:** This has been only briefly tested and it may not be ## very accurate. - let t = toSeconds(timeInfoToTime(a)) + let t = toSeconds(toTime(a)) let secs = toSeconds(a, interval) - result = getLocalTime(fromSeconds(t + secs)) + if a.tzname == "UTC": + result = getGMTime(fromSeconds(t + secs)) + else: + result = getLocalTime(fromSeconds(t + secs)) proc `-`*(a: TimeInfo, interval: TimeInterval): TimeInfo = ## subtracts ``interval`` time from TimeInfo ``a``. ## ## **Note:** This has been only briefly tested, it is inaccurate especially ## when you subtract so much that you reach the Julian calendar. - let t = toSeconds(timeInfoToTime(a)) + let t = toSeconds(toTime(a)) var intval: TimeInterval intval.milliseconds = - interval.milliseconds intval.seconds = - interval.seconds @@ -375,7 +389,10 @@ proc `-`*(a: TimeInfo, interval: TimeInterval): TimeInfo = intval.months = - interval.months intval.years = - interval.years let secs = toSeconds(a, intval) - result = getLocalTime(fromSeconds(t + secs)) + if a.tzname == "UTC": + result = getGMTime(fromSeconds(t + secs)) + else: + result = getLocalTime(fromSeconds(t + secs)) proc miliseconds*(t: TimeInterval): int {.deprecated.} = t.milliseconds @@ -407,18 +424,32 @@ when not defined(JS): when not defined(JS): # C wrapper: + when defined(freebsd) or defined(netbsd) or defined(openbsd): + type + StructTM {.importc: "struct tm", final.} = object + second {.importc: "tm_sec".}, + minute {.importc: "tm_min".}, + hour {.importc: "tm_hour".}, + monthday {.importc: "tm_mday".}, + month {.importc: "tm_mon".}, + year {.importc: "tm_year".}, + weekday {.importc: "tm_wday".}, + yearday {.importc: "tm_yday".}, + isdst {.importc: "tm_isdst".}: cint + gmtoff {.importc: "tm_gmtoff".}: clong + else: + type + StructTM {.importc: "struct tm", final.} = object + second {.importc: "tm_sec".}, + minute {.importc: "tm_min".}, + hour {.importc: "tm_hour".}, + monthday {.importc: "tm_mday".}, + month {.importc: "tm_mon".}, + year {.importc: "tm_year".}, + weekday {.importc: "tm_wday".}, + yearday {.importc: "tm_yday".}, + isdst {.importc: "tm_isdst".}: cint type - StructTM {.importc: "struct tm", final.} = object - second {.importc: "tm_sec".}, - minute {.importc: "tm_min".}, - hour {.importc: "tm_hour".}, - monthday {.importc: "tm_mday".}, - month {.importc: "tm_mon".}, - year {.importc: "tm_year".}, - weekday {.importc: "tm_wday".}, - yearday {.importc: "tm_yday".}, - isdst {.importc: "tm_isdst".}: cint - TimeInfoPtr = ptr StructTM Clock {.importc: "clock_t".} = distinct int @@ -446,30 +477,53 @@ when not defined(JS): # our own procs on top of that: proc tmToTimeInfo(tm: StructTM, local: bool): TimeInfo = const - weekDays: array [0..6, WeekDay] = [ + weekDays: array[0..6, WeekDay] = [ dSun, dMon, dTue, dWed, dThu, dFri, dSat] - TimeInfo(second: int(tm.second), - minute: int(tm.minute), - hour: int(tm.hour), - monthday: int(tm.monthday), - month: Month(tm.month), - year: tm.year + 1900'i32, - weekday: weekDays[int(tm.weekday)], - yearday: int(tm.yearday), - isDST: tm.isdst > 0, - tzname: if local: - if tm.isdst > 0: - getTzname().DST + when defined(freebsd) or defined(netbsd) or defined(openbsd): + TimeInfo(second: int(tm.second), + minute: int(tm.minute), + hour: int(tm.hour), + monthday: int(tm.monthday), + month: Month(tm.month), + year: tm.year + 1900'i32, + weekday: weekDays[int(tm.weekday)], + yearday: int(tm.yearday), + isDST: tm.isdst > 0, + tzname: if local: + if tm.isdst > 0: + getTzname().DST + else: + getTzname().nonDST + else: + "UTC", + # BSD stores in `gmtoff` offset east of UTC in seconds, + # but posix systems using west of UTC in seconds + timezone: if local: -(tm.gmtoff) else: 0 + ) + else: + TimeInfo(second: int(tm.second), + minute: int(tm.minute), + hour: int(tm.hour), + monthday: int(tm.monthday), + month: Month(tm.month), + year: tm.year + 1900'i32, + weekday: weekDays[int(tm.weekday)], + yearday: int(tm.yearday), + isDST: tm.isdst > 0, + tzname: if local: + if tm.isdst > 0: + getTzname().DST + else: + getTzname().nonDST else: - getTzname().nonDST - else: - "UTC", - timezone: if local: getTimezone() else: 0 - ) + "UTC", + timezone: if local: getTimezone() else: 0 + ) + proc timeInfoToTM(t: TimeInfo): StructTM = const - weekDays: array [WeekDay, int8] = [1'i8,2'i8,3'i8,4'i8,5'i8,6'i8,0'i8] + weekDays: array[WeekDay, int8] = [1'i8,2'i8,3'i8,4'i8,5'i8,6'i8,0'i8] result.second = t.second result.minute = t.minute result.hour = t.hour @@ -517,6 +571,11 @@ when not defined(JS): # because the header of mktime is broken in my version of libc return mktime(timeInfoToTM(cTimeInfo)) + proc toTime(timeInfo: TimeInfo): Time = + var cTimeInfo = timeInfo # for C++ we have to make a copy, + # because the header of mktime is broken in my version of libc + return mktime(timeInfoToTM(cTimeInfo)) + proc toStringTillNL(p: cstring): string = result = "" var i = 0 @@ -550,7 +609,14 @@ when not defined(JS): return ($tzname[0], $tzname[1]) proc getTimezone(): int = - return timezone + when defined(freebsd) or defined(netbsd) or defined(openbsd): + var a = timec(nil) + let lt = localtime(addr(a)) + # BSD stores in `gmtoff` offset east of UTC in seconds, + # but posix systems using west of UTC in seconds + return -(lt.gmtoff) + else: + return timezone proc fromSeconds(since1970: float): Time = Time(since1970) @@ -586,7 +652,7 @@ elif defined(JS): return newDate() const - weekDays: array [0..6, WeekDay] = [ + weekDays: array[0..6, WeekDay] = [ dSun, dMon, dTue, dWed, dThu, dFri, dSat] proc getLocalTime(t: Time): TimeInfo = @@ -618,7 +684,16 @@ elif defined(JS): result.setFullYear(timeInfo.year) result.setDate(timeInfo.monthday) - proc `$`(timeInfo: TimeInfo): string = return $(timeInfoToTime(timeInfo)) + proc toTime*(timeInfo: TimeInfo): Time = + result = internGetTime() + result.setSeconds(timeInfo.second) + result.setMinutes(timeInfo.minute) + result.setHours(timeInfo.hour) + result.setMonth(ord(timeInfo.month)) + result.setFullYear(timeInfo.year) + result.setDate(timeInfo.monthday) + + proc `$`(timeInfo: TimeInfo): string = return $(toTime(timeInfo)) proc `$`(time: Time): string = return $time.toLocaleString() proc `-` (a, b: Time): int64 = @@ -708,24 +783,24 @@ proc years*(y: int): TimeInterval {.inline.} = proc `+=`*(t: var Time, ti: TimeInterval) = ## modifies `t` by adding the interval `ti` - t = timeInfoToTime(getLocalTime(t) + ti) + t = toTime(getLocalTime(t) + ti) proc `+`*(t: Time, ti: TimeInterval): Time = ## adds the interval `ti` to Time `t` ## by converting to localTime, adding the interval, and converting back ## ## ``echo getTime() + 1.day`` - result = timeInfoToTime(getLocalTime(t) + ti) + result = toTime(getLocalTime(t) + ti) proc `-=`*(t: var Time, ti: TimeInterval) = ## modifies `t` by subtracting the interval `ti` - t = timeInfoToTime(getLocalTime(t) - ti) + t = toTime(getLocalTime(t) - ti) proc `-`*(t: Time, ti: TimeInterval): Time = ## adds the interval `ti` to Time `t` ## ## ``echo getTime() - 1.day`` - result = timeInfoToTime(getLocalTime(t) - ti) + result = toTime(getLocalTime(t) - ti) proc formatToken(info: TimeInfo, token: string, buf: var string) = ## Helper of the format proc to parse individual tokens. @@ -923,21 +998,14 @@ proc parseToken(info: var TimeInfo; token, value: string; j: var int) = info.monthday = value[j..j+1].parseInt() j += 2 of "ddd": - case value[j..j+2].toLower() - of "sun": - info.weekday = dSun - of "mon": - info.weekday = dMon - of "tue": - info.weekday = dTue - of "wed": - info.weekday = dWed - of "thu": - info.weekday = dThu - of "fri": - info.weekday = dFri - of "sat": - info.weekday = dSat + case value[j..j+2].toLowerAscii() + of "sun": info.weekday = dSun + of "mon": info.weekday = dMon + of "tue": info.weekday = dTue + of "wed": info.weekday = dWed + of "thu": info.weekday = dThu + of "fri": info.weekday = dFri + of "sat": info.weekday = dSat else: raise newException(ValueError, "Couldn't parse day of week (ddd), got: " & value[j..j+2]) @@ -991,31 +1059,19 @@ proc parseToken(info: var TimeInfo; token, value: string; j: var int) = j += 2 info.month = Month(month-1) of "MMM": - case value[j..j+2].toLower(): - of "jan": - info.month = mJan - of "feb": - info.month = mFeb - of "mar": - info.month = mMar - of "apr": - info.month = mApr - of "may": - info.month = mMay - of "jun": - info.month = mJun - of "jul": - info.month = mJul - of "aug": - info.month = mAug - of "sep": - info.month = mSep - of "oct": - info.month = mOct - of "nov": - info.month = mNov - of "dec": - info.month = mDec + case value[j..j+2].toLowerAscii(): + of "jan": info.month = mJan + of "feb": info.month = mFeb + of "mar": info.month = mMar + of "apr": info.month = mApr + of "may": info.month = mMay + of "jun": info.month = mJun + of "jul": info.month = mJul + of "aug": info.month = mAug + of "sep": info.month = mSep + of "oct": info.month = mOct + of "nov": info.month = mNov + of "dec": info.month = mDec else: raise newException(ValueError, "Couldn't parse month (MMM), got: " & value) @@ -1112,7 +1168,7 @@ proc parseToken(info: var TimeInfo; token, value: string; j: var int) = "Couldn't parse timezone offset (zzz), got: " & value[j]) j += 6 of "ZZZ": - info.tzname = value[j..j+2].toUpper() + info.tzname = value[j..j+2].toUpperAscii() j += 3 else: # Ignore the token and move forward in the value string by the same length @@ -1193,7 +1249,7 @@ proc parse*(value, layout: string): TimeInfo = parseToken(info, token, value, j) token = "" # Reset weekday as it might not have been provided and the default may be wrong - info.weekday = getLocalTime(timeInfoToTime(info)).weekday + info.weekday = getLocalTime(toTime(info)).weekday return info # Leap year calculations are adapted from: @@ -1204,7 +1260,7 @@ proc parse*(value, layout: string): TimeInfo = proc countLeapYears*(yearSpan: int): int = ## Returns the number of leap years spanned by a given number of years. ## - ## Note: for leap years, start date is assumed to be 1 AD. + ## **Note:** For leap years, start date is assumed to be 1 AD. ## counts the number of leap years up to January 1st of a given year. ## Keep in mind that if specified year is a leap year, the leap day ## has not happened before January 1st of that year. @@ -1239,13 +1295,14 @@ proc getDayOfWeek*(day, month, year: int): WeekDay = y = year - a m = month + (12*a) - 2 d = (day + y + (y div 4) - (y div 100) + (y div 400) + (31*m) div 12) mod 7 - # The value of d is 0 for a Sunday, 1 for a Monday, 2 for a Tuesday, etc. so we must correct - # for the WeekDay type. + # The value of d is 0 for a Sunday, 1 for a Monday, 2 for a Tuesday, etc. + # so we must correct for the WeekDay type. if d == 0: return dSun result = (d-1).WeekDay proc getDayOfWeekJulian*(day, month, year: int): WeekDay = - ## Returns the day of the week enum from day, month and year, according to the Julian calendar. + ## Returns the day of the week enum from day, month and year, + ## according to the Julian calendar. # Day & month start from one. let a = (14 - month) div 12 @@ -1254,8 +1311,11 @@ proc getDayOfWeekJulian*(day, month, year: int): WeekDay = d = (5 + day + y + (y div 4) + (31*m) div 12) mod 7 result = d.WeekDay -proc timeToTimeInfo*(t: Time): TimeInfo = +proc timeToTimeInfo*(t: Time): TimeInfo {.deprecated.} = ## Converts a Time to TimeInfo. + ## + ## **Warning:** This procedure is deprecated since version 0.14.0. + ## Use ``getLocalTime`` or ``getGMTime`` instead. let secs = t.toSeconds().int daysSinceEpoch = secs div secondsInDay @@ -1286,34 +1346,21 @@ proc timeToTimeInfo*(t: Time): TimeInfo = s = daySeconds mod secondsInMin result = TimeInfo(year: y, yearday: yd, month: m, monthday: md, weekday: wd, hour: h, minute: mi, second: s) -proc timeToTimeInterval*(t: Time): TimeInterval = +proc timeToTimeInterval*(t: Time): TimeInterval {.deprecated.} = ## Converts a Time to a TimeInterval. + ## + ## **Warning:** This procedure is deprecated since version 0.14.0. + ## Use ``toTimeInterval`` instead. + # Milliseconds not available from Time + var tInfo = t.getLocalTime() + initInterval(0, tInfo.second, tInfo.minute, tInfo.hour, tInfo.weekday.ord, tInfo.month.ord, tInfo.year) - let - secs = t.toSeconds().int - daysSinceEpoch = secs div secondsInDay - (yearsSinceEpoch, daysRemaining) = countYearsAndDays(daysSinceEpoch) - daySeconds = secs mod secondsInDay - - result.years = yearsSinceEpoch + epochStartYear - - var - mon = mJan - days = daysRemaining - daysInMonth = getDaysInMonth(mon, result.years) - - # calculate month and day remainder - while days > daysInMonth and mon <= mDec: - days -= daysInMonth - mon.inc - daysInMonth = getDaysInMonth(mon, result.years) - - result.months = mon.int + 1 # month is 1 indexed int - result.days = days - result.hours = daySeconds div secondsInHour + 1 - result.minutes = (daySeconds mod secondsInHour) div secondsInMin - result.seconds = daySeconds mod secondsInMin +proc toTimeInterval*(t: Time): TimeInterval = + ## Converts a Time to a TimeInterval. # Milliseconds not available from Time + var tInfo = t.getLocalTime() + initInterval(0, tInfo.second, tInfo.minute, tInfo.hour, tInfo.weekday.ord, tInfo.month.ord, tInfo.year) + when isMainModule: # this is testing non-exported function diff --git a/lib/pure/unicode.nim b/lib/pure/unicode.nim index 45f52eb7f..0cbe8de7a 100644 --- a/lib/pure/unicode.nim +++ b/lib/pure/unicode.nim @@ -14,7 +14,7 @@ include "system/inclrtl" type - RuneImpl = int # underlying type of Rune + RuneImpl = int32 # underlying type of Rune Rune* = distinct RuneImpl ## type that can hold any Unicode character Rune16* = distinct int16 ## 16 bit Unicode character @@ -135,45 +135,62 @@ proc runeAt*(s: string, i: Natural): Rune = ## Returns the unicode character in ``s`` at byte index ``i`` fastRuneAt(s, i, result, false) -proc toUTF8*(c: Rune): string {.rtl, extern: "nuc$1".} = - ## Converts a rune into its UTF-8 representation +template fastToUTF8Copy*(c: Rune, s: var string, pos: int, doInc = true) = + ## Copies UTF-8 representation of `c` into the preallocated string `s` + ## starting at position `pos`. If `doInc == true`, `pos` is incremented + ## by the number of bytes that have been processed. + ## + ## To be the most efficient, make sure `s` is preallocated + ## with an additional amount equal to the byte length of + ## `c`. var i = RuneImpl(c) if i <=% 127: - result = newString(1) - result[0] = chr(i) + s.setLen(pos+1) + s[pos+0] = chr(i) + when doInc: inc(pos) elif i <=% 0x07FF: - result = newString(2) - result[0] = chr((i shr 6) or 0b110_00000) - result[1] = chr((i and ones(6)) or 0b10_0000_00) + s.setLen(pos+2) + s[pos+0] = chr((i shr 6) or 0b110_00000) + s[pos+1] = chr((i and ones(6)) or 0b10_0000_00) + when doInc: inc(pos, 2) elif i <=% 0xFFFF: - result = newString(3) - result[0] = chr(i shr 12 or 0b1110_0000) - result[1] = chr(i shr 6 and ones(6) or 0b10_0000_00) - result[2] = chr(i and ones(6) or 0b10_0000_00) + s.setLen(pos+3) + s[pos+0] = chr(i shr 12 or 0b1110_0000) + s[pos+1] = chr(i shr 6 and ones(6) or 0b10_0000_00) + s[pos+2] = chr(i and ones(6) or 0b10_0000_00) + when doInc: inc(pos, 3) elif i <=% 0x001FFFFF: - result = newString(4) - result[0] = chr(i shr 18 or 0b1111_0000) - result[1] = chr(i shr 12 and ones(6) or 0b10_0000_00) - result[2] = chr(i shr 6 and ones(6) or 0b10_0000_00) - result[3] = chr(i and ones(6) or 0b10_0000_00) + s.setLen(pos+4) + s[pos+0] = chr(i shr 18 or 0b1111_0000) + s[pos+1] = chr(i shr 12 and ones(6) or 0b10_0000_00) + s[pos+2] = chr(i shr 6 and ones(6) or 0b10_0000_00) + s[pos+3] = chr(i and ones(6) or 0b10_0000_00) + when doInc: inc(pos, 4) elif i <=% 0x03FFFFFF: - result = newString(5) - result[0] = chr(i shr 24 or 0b111110_00) - result[1] = chr(i shr 18 and ones(6) or 0b10_0000_00) - result[2] = chr(i shr 12 and ones(6) or 0b10_0000_00) - result[3] = chr(i shr 6 and ones(6) or 0b10_0000_00) - result[4] = chr(i and ones(6) or 0b10_0000_00) + s.setLen(pos+5) + s[pos+0] = chr(i shr 24 or 0b111110_00) + s[pos+1] = chr(i shr 18 and ones(6) or 0b10_0000_00) + s[pos+2] = chr(i shr 12 and ones(6) or 0b10_0000_00) + s[pos+3] = chr(i shr 6 and ones(6) or 0b10_0000_00) + s[pos+4] = chr(i and ones(6) or 0b10_0000_00) + when doInc: inc(pos, 5) elif i <=% 0x7FFFFFFF: - result = newString(6) - result[0] = chr(i shr 30 or 0b1111110_0) - result[1] = chr(i shr 24 and ones(6) or 0b10_0000_00) - result[2] = chr(i shr 18 and ones(6) or 0b10_0000_00) - result[3] = chr(i shr 12 and ones(6) or 0b10_0000_00) - result[4] = chr(i shr 6 and ones(6) or 0b10_0000_00) - result[5] = chr(i and ones(6) or 0b10_0000_00) + s.setLen(pos+6) + s[pos+0] = chr(i shr 30 or 0b1111110_0) + s[pos+1] = chr(i shr 24 and ones(6) or 0b10_0000_00) + s[pos+2] = chr(i shr 18 and ones(6) or 0b10_0000_00) + s[pos+3] = chr(i shr 12 and ones(6) or 0b10_0000_00) + s[pos+4] = chr(i shr 6 and ones(6) or 0b10_0000_00) + s[pos+5] = chr(i and ones(6) or 0b10_0000_00) + when doInc: inc(pos, 6) else: discard # error, exception? +proc toUTF8*(c: Rune): string {.rtl, extern: "nuc$1".} = + ## Converts a rune into its UTF-8 representation + result = "" + fastToUTF8Copy(c, result, 0, false) + proc `$`*(rune: Rune): string = ## Converts a Rune to a string rune.toUTF8 @@ -183,6 +200,105 @@ proc `$`*(runes: seq[Rune]): string = result = "" for rune in runes: result.add(rune.toUTF8) +proc runeOffset*(s: string, pos:Natural, start: Natural = 0): int = + ## Returns the byte position of unicode character + ## at position pos in s with an optional start byte position. + ## returns the special value -1 if it runs out of the string + ## + ## Beware: This can lead to unoptimized code and slow execution! + ## Most problems are solve more efficient by using an iterator + ## or conversion to a seq of Rune. + var + i = 0 + o = start + while i < pos: + o += runeLenAt(s, o) + if o >= s.len: + return -1 + inc i + return o + +proc runeAtPos*(s: string, pos: int): Rune = + ## Returns the unicode character at position pos + ## + ## Beware: This can lead to unoptimized code and slow execution! + ## Most problems are solve more efficient by using an iterator + ## or conversion to a seq of Rune. + fastRuneAt(s, runeOffset(s, pos), result, false) + +proc runeStrAtPos*(s: string, pos: Natural): string = + ## Returns the unicode character at position pos as UTF8 String + ## + ## Beware: This can lead to unoptimized code and slow execution! + ## Most problems are solve more efficient by using an iterator + ## or conversion to a seq of Rune. + let o = runeOffset(s, pos) + s[o.. (o+runeLenAt(s, o)-1)] + +proc runeReverseOffset*(s: string, rev:Positive): (int, int) = + ## Returns a tuple with the the byte offset of the + ## unicode character at position ``rev`` in s counting + ## from the end (starting with 1) and the total + ## number of runes in the string. Returns a negative value + ## for offset if there are to few runes in the string to + ## satisfy the request. + ## + ## Beware: This can lead to unoptimized code and slow execution! + ## Most problems are solve more efficient by using an iterator + ## or conversion to a seq of Rune. + var + a = rev.int + o = 0 + x = 0 + while o < s.len: + let r = runeLenAt(s, o) + o += r + if a < 0: + x += r + dec a + + if a > 0: + return (-a, rev.int-a) + return (x, -a+rev.int) + +proc runeSubStr*(s: string, pos:int, len:int = int.high): string = + ## Returns the UTF-8 substring starting at codepoint pos + ## with len codepoints. If pos or len is negativ they count from + ## the end of the string. If len is not given it means the longest + ## possible string. + ## + ## (Needs some examples) + if pos < 0: + let (o, rl) = runeReverseOffset(s, -pos) + if len >= rl: + result = s[o.. s.len-1] + elif len < 0: + let e = rl + len + if e < 0: + result = "" + else: + result = s[o.. runeOffset(s, e-(rl+pos) , o)-1] + else: + result = s[o.. runeOffset(s, len, o)-1] + else: + let o = runeOffset(s, pos) + if o < 0: + result = "" + elif len == int.high: + result = s[o.. s.len-1] + elif len < 0: + let (e, rl) = runeReverseOffset(s, -len) + discard rl + if e <= 0: + result = "" + else: + result = s[o.. e-1] + else: + var e = runeOffset(s, len, o) + if e < 0: + e = s.len + result = s[o.. e-1] + const alphaRanges = [ 0x00d8, 0x00f6, # - @@ -1148,7 +1264,7 @@ const 0x01f1, 501, # 0x01f3, 499] # -proc binarySearch(c: RuneImpl, tab: openArray[RuneImpl], len, stride: int): int = +proc binarySearch(c: RuneImpl, tab: openArray[int], len, stride: int): int = var n = len var t = 0 while n > 1: @@ -1253,8 +1369,210 @@ proc isCombining*(c: Rune): bool {.rtl, extern: "nuc$1", procvar.} = (c >= 0x20d0 and c <= 0x20ff) or (c >= 0xfe20 and c <= 0xfe2f)) +template runeCheck(s, runeProc) = + ## Common code for rune.isLower, rune.isUpper, etc + result = if len(s) == 0: false else: true + + var + i = 0 + rune: Rune + + while i < len(s) and result: + fastRuneAt(s, i, rune, doInc=true) + result = runeProc(rune) and result + +proc isUpper*(s: string): bool {.noSideEffect, procvar, + rtl, extern: "nuc$1Str".} = + ## Returns true iff `s` contains all upper case unicode characters. + runeCheck(s, isUpper) + +proc isLower*(s: string): bool {.noSideEffect, procvar, + rtl, extern: "nuc$1Str".} = + ## Returns true iff `s` contains all lower case unicode characters. + runeCheck(s, isLower) + +proc isAlpha*(s: string): bool {.noSideEffect, procvar, + rtl, extern: "nuc$1Str".} = + ## Returns true iff `s` contains all alphabetic unicode characters. + runeCheck(s, isAlpha) + +proc isSpace*(s: string): bool {.noSideEffect, procvar, + rtl, extern: "nuc$1Str".} = + ## Returns true iff `s` contains all whitespace unicode characters. + runeCheck(s, isWhiteSpace) + +template convertRune(s, runeProc) = + ## Convert runes in `s` using `runeProc` as the converter. + result = newString(len(s)) + + var + i = 0 + lastIndex = 0 + rune: Rune + + while i < len(s): + lastIndex = i + fastRuneAt(s, i, rune, doInc=true) + rune = runeProc(rune) + + rune.fastToUTF8Copy(result, lastIndex) + +proc toUpper*(s: string): string {.noSideEffect, procvar, + rtl, extern: "nuc$1Str".} = + ## Converts `s` into upper-case unicode characters. + convertRune(s, toUpper) + +proc toLower*(s: string): string {.noSideEffect, procvar, + rtl, extern: "nuc$1Str".} = + ## Converts `s` into lower-case unicode characters. + convertRune(s, toLower) + +proc swapCase*(s: string): string {.noSideEffect, procvar, + rtl, extern: "nuc$1".} = + ## Swaps the case of unicode characters in `s` + ## + ## Returns a new string such that the cases of all unicode characters + ## are swapped if possible + + var + i = 0 + lastIndex = 0 + rune: Rune + + result = newString(len(s)) + + while i < len(s): + lastIndex = i + + fastRuneAt(s, i, rune) + + if rune.isUpper(): + rune = rune.toLower() + elif rune.isLower(): + rune = rune.toUpper() + + rune.fastToUTF8Copy(result, lastIndex) + +proc capitalize*(s: string): string {.noSideEffect, procvar, + rtl, extern: "nuc$1".} = + ## Converts the first character of `s` into an upper-case unicode character. + if len(s) == 0: + return s + + var + rune: Rune + i = 0 + + fastRuneAt(s, i, rune, doInc=true) + + result = $toUpper(rune) & substr(s, i) + +proc translate*(s: string, replacements: proc(key: string): string): string {. + rtl, extern: "nuc$1".} = + ## Translates words in a string using the `replacements` proc to substitute + ## words inside `s` with their replacements + ## + ## `replacements` is any proc that takes a word and returns + ## a new word to fill it's place. + + # Allocate memory for the new string based on the old one. + # If the new string length is less than the old, no allocations + # will be needed. If the new string length is greater than the + # old, then maybe only one allocation is needed + result = newStringOfCap(s.len) + + var + index = 0 + lastIndex = 0 + wordStart = 0 + inWord = false + rune: Rune + + while index < len(s): + lastIndex = index + + fastRuneAt(s, index, rune) + + let whiteSpace = rune.isWhiteSpace() + + if whiteSpace and inWord: + # If we've reached the end of a word + let word = s[wordStart ..< lastIndex] + result.add(replacements(word)) + result.add($rune) + + inWord = false + elif not whiteSpace and not inWord: + # If we've hit a non space character and + # are not currently in a word, track + # the starting index of the word + inWord = true + wordStart = lastIndex + elif whiteSpace: + result.add($rune) + + if wordStart < len(s) and inWord: + # Get the trailing word at the end + let word = s[wordStart .. ^1] + result.add(replacements(word)) + +proc title*(s: string): string {.noSideEffect, procvar, + rtl, extern: "nuc$1".} = + ## Converts `s` to a unicode title. + ## + ## Returns a new string such that the first character + ## in each word inside `s` is capitalized + + var + i = 0 + lastIndex = 0 + rune: Rune + + result = newString(len(s)) + + var firstRune = true + + while i < len(s): + lastIndex = i + + fastRuneAt(s, i, rune) + + if not rune.isWhiteSpace() and firstRune: + rune = rune.toUpper() + firstRune = false + elif rune.isWhiteSpace(): + firstRune = true + + rune.fastToUTF8Copy(result, lastIndex) + +proc isTitle*(s: string): bool {.noSideEffect, procvar, + rtl, extern: "nuc$1Str".}= + ## Checks whether or not `s` is a unicode title. + ## + ## Returns true if the first character in each word inside `s` + ## are upper case and there is at least one character in `s`. + if s.len() == 0: + return false + + result = true + + var + i = 0 + rune: Rune + + var firstRune = true + + while i < len(s) and result: + fastRuneAt(s, i, rune, doInc=true) + + if not rune.isWhiteSpace() and firstRune: + result = rune.isUpper() and result + firstRune = false + elif rune.isWhiteSpace(): + firstRune = true + iterator runes*(s: string): Rune = - ## Iterates over any unicode character of the string ``s`` + ## Iterates over any unicode character of the string ``s`` returning runes var i = 0 result: Rune @@ -1262,6 +1580,14 @@ iterator runes*(s: string): Rune = fastRuneAt(s, i, result, true) yield result +iterator utf8*(s: string): string = + ## Iterates over any unicode character of the string ``s`` returning utf8 values + var o = 0 + while o < s.len: + let n = runeLenAt(s, o) + yield s[o.. (o+n-1)] + o += n + proc toRunes*(s: string): seq[Rune] = ## Obtains a sequence containing the Runes in ``s`` result = newSeq[Rune]() @@ -1352,6 +1678,101 @@ when isMainModule: compared = (someString == $someRunes) doAssert compared == true + proc test_replacements(word: string): string = + case word + of "two": + return "2" + of "foo": + return "BAR" + of "βeta": + return "beta" + of "alpha": + return "αlpha" + else: + return "12345" + + doAssert translate("two not alpha foo βeta", test_replacements) == "2 12345 αlpha BAR beta" + doAssert translate(" two not foo βeta ", test_replacements) == " 2 12345 BAR beta " + + doAssert title("foo bar") == "Foo Bar" + doAssert title("αlpha βeta γamma") == "Αlpha Βeta Γamma" + doAssert title("") == "" + + doAssert capitalize("βeta") == "Βeta" + doAssert capitalize("foo") == "Foo" + doAssert capitalize("") == "" + + doAssert isTitle("Foo") + doAssert(not isTitle("Foo bar")) + doAssert(not isTitle("αlpha Βeta")) + doAssert(isTitle("Αlpha Βeta Γamma")) + doAssert(not isTitle("fFoo")) + + doAssert swapCase("FooBar") == "fOObAR" + doAssert swapCase(" ") == " " + doAssert swapCase("Αlpha Βeta Γamma") == "αLPHA βETA γAMMA" + doAssert swapCase("a✓B") == "A✓b" + doAssert swapCase("") == "" + + doAssert isAlpha("r") + doAssert isAlpha("α") + doAssert(not isAlpha("$")) + doAssert(not isAlpha("")) + + doAssert isAlpha("Βeta") + doAssert isAlpha("Args") + doAssert(not isAlpha("$Foo✓")) + + doAssert isSpace("\t") + doAssert isSpace("\l") + doAssert(not isSpace("Β")) + doAssert(not isSpace("Βeta")) + + doAssert isSpace("\t\l \v\r\f") + doAssert isSpace(" ") + doAssert(not isSpace("")) + doAssert(not isSpace("ΑΓc \td")) + + doAssert isLower("a") + doAssert isLower("γ") + doAssert(not isLower("Γ")) + doAssert(not isLower("4")) + doAssert(not isLower("")) + + doAssert isLower("abcdγ") + doAssert(not isLower("abCDΓ")) + doAssert(not isLower("33aaΓ")) + + doAssert isUpper("Γ") + doAssert(not isUpper("b")) + doAssert(not isUpper("α")) + doAssert(not isUpper("✓")) + doAssert(not isUpper("")) + + doAssert isUpper("ΑΒΓ") + doAssert(not isUpper("AAccβ")) + doAssert(not isUpper("A#$β")) + + doAssert toUpper("Γ") == "Γ" + doAssert toUpper("b") == "B" + doAssert toUpper("α") == "Α" + doAssert toUpper("✓") == "✓" + doAssert toUpper("") == "" + + doAssert toUpper("ΑΒΓ") == "ΑΒΓ" + doAssert toUpper("AAccβ") == "AACCΒ" + doAssert toUpper("A✓$β") == "A✓$Β" + + doAssert toLower("a") == "a" + doAssert toLower("γ") == "γ" + doAssert toLower("Γ") == "γ" + doAssert toLower("4") == "4" + doAssert toLower("") == "" + + doAssert toLower("abcdγ") == "abcdγ" + doAssert toLower("abCDΓ") == "abcdγ" + doAssert toLower("33aaΓ") == "33aaγ" + doAssert reversed("Reverse this!") == "!siht esreveR" doAssert reversed("先秦兩漢") == "漢兩秦先" doAssert reversed("as⃝df̅") == "f̅ds⃝a" @@ -1360,3 +1781,44 @@ when isMainModule: const test = "as⃝" doAssert lastRune(test, test.len-1)[1] == 3 doAssert graphemeLen("è", 0) == 2 + + # test for rune positioning and runeSubStr() + let s = "Hänsel ««: 10,00€" + + var t = "" + for c in s.utf8: + t.add c + + doAssert(s == t) + + doAssert(runeReverseOffset(s, 1) == (20, 18)) + doAssert(runeReverseOffset(s, 19) == (-1, 18)) + + doAssert(runeStrAtPos(s, 0) == "H") + doAssert(runeSubStr(s, 0, 1) == "H") + doAssert(runeStrAtPos(s, 10) == ":") + doAssert(runeSubStr(s, 10, 1) == ":") + doAssert(runeStrAtPos(s, 9) == "«") + doAssert(runeSubStr(s, 9, 1) == "«") + doAssert(runeStrAtPos(s, 17) == "€") + doAssert(runeSubStr(s, 17, 1) == "€") + # echo runeStrAtPos(s, 18) # index error + + doAssert(runeSubStr(s, 0) == "Hänsel ««: 10,00€") + doAssert(runeSubStr(s, -18) == "Hänsel ««: 10,00€") + doAssert(runeSubStr(s, 10) == ": 10,00€") + doAssert(runeSubStr(s, 18) == "") + doAssert(runeSubStr(s, 0, 10) == "Hänsel ««") + + doAssert(runeSubStr(s, 12) == "10,00€") + doAssert(runeSubStr(s, -6) == "10,00€") + + doAssert(runeSubStr(s, 12, 5) == "10,00") + doAssert(runeSubStr(s, 12, -1) == "10,00") + doAssert(runeSubStr(s, -6, 5) == "10,00") + doAssert(runeSubStr(s, -6, -1) == "10,00") + + doAssert(runeSubStr(s, 0, 100) == "Hänsel ««: 10,00€") + doAssert(runeSubStr(s, -100, 100) == "Hänsel ««: 10,00€") + doAssert(runeSubStr(s, 0, -100) == "") + doAssert(runeSubStr(s, 100, -100) == "") diff --git a/lib/pure/unittest.nim b/lib/pure/unittest.nim index aca9d51e2..92ddc3e75 100644 --- a/lib/pure/unittest.nim +++ b/lib/pure/unittest.nim @@ -41,7 +41,11 @@ when not defined(ECMAScript): import terminal type - TestStatus* = enum OK, FAILED ## The status of a test when it is done. + TestStatus* = enum ## The status of a test when it is done. + OK, + FAILED, + SKIPPED + OutputLevel* = enum ## The output verbosity of the tests. PRINT_ALL, ## Print as much as possible. PRINT_FAILURES, ## Print only the failed tests. @@ -73,7 +77,7 @@ checkpoints = @[] proc shouldRun(testName: string): bool = result = true -template suite*(name: expr, body: stmt): stmt {.immediate, dirty.} = +template suite*(name, body) {.dirty.} = ## Declare a test suite identified by `name` with optional ``setup`` ## and/or ``teardown`` section. ## @@ -102,13 +106,13 @@ template suite*(name: expr, body: stmt): stmt {.immediate, dirty.} = ## [OK] 2 + 2 = 4 ## [OK] (2 + -2) != 4 block: - template setup(setupBody: stmt): stmt {.immediate, dirty.} = + template setup(setupBody: untyped) {.dirty.} = var testSetupIMPLFlag = true - template testSetupIMPL: stmt {.immediate, dirty.} = setupBody + template testSetupIMPL: untyped {.dirty.} = setupBody - template teardown(teardownBody: stmt): stmt {.immediate, dirty.} = + template teardown(teardownBody: untyped) {.dirty.} = var testTeardownIMPLFlag = true - template testTeardownIMPL: stmt {.immediate, dirty.} = teardownBody + template testTeardownIMPL: untyped {.dirty.} = teardownBody body @@ -120,14 +124,18 @@ proc testDone(name: string, s: TestStatus) = template rawPrint() = echo("[", $s, "] ", name) when not defined(ECMAScript): if colorOutput and not defined(ECMAScript): - var color = (if s == OK: fgGreen else: fgRed) + var color = case s + of OK: fgGreen + of FAILED: fgRed + of SKIPPED: fgYellow + else: fgWhite styledEcho styleBright, color, "[", $s, "] ", fgWhite, name else: rawPrint() else: rawPrint() -template test*(name: expr, body: stmt): stmt {.immediate, dirty.} = +template test*(name, body) {.dirty.} = ## Define a single test case identified by `name`. ## ## .. code-block:: nim @@ -203,7 +211,22 @@ template fail* = checkpoints = @[] -macro check*(conditions: stmt): stmt {.immediate.} = +template skip* = + ## Makes test to be skipped. Should be used directly + ## in case when it is not possible to perform test + ## for reasons depending on outer environment, + ## or certain application logic conditions or configurations. + ## + ## .. code-block:: nim + ## + ## if not isGLConextCreated(): + ## skip() + bind checkpoints + + testStatusIMPL = SKIPPED + checkpoints = @[] + +macro check*(conditions: untyped): untyped = ## Verify if a statement or a list of statements is true. ## A helpful error message and set checkpoints are printed out on ## failure (if ``outputLevel`` is not ``PRINT_NONE``). @@ -236,31 +259,34 @@ macro check*(conditions: stmt): stmt {.immediate.} = proc inspectArgs(exp: NimNode): NimNode = result = copyNimTree(exp) - for i in countup(1, exp.len - 1): - if exp[i].kind notin nnkLiterals: - inc counter - var arg = newIdentNode(":p" & $counter) - var argStr = exp[i].toStrLit - var paramAst = exp[i] - if exp[i].kind == nnkIdent: - argsPrintOuts.add getAst(print(argStr, paramAst)) - if exp[i].kind in nnkCallKinds: - var callVar = newIdentNode(":c" & $counter) - argsAsgns.add getAst(asgn(callVar, paramAst)) - result[i] = callVar - argsPrintOuts.add getAst(print(argStr, callVar)) - if exp[i].kind == nnkExprEqExpr: - # ExprEqExpr - # Ident !"v" - # IntLit 2 - result[i] = exp[i][1] - if exp[i].typekind notin {ntyTypeDesc}: - argsAsgns.add getAst(asgn(arg, paramAst)) - argsPrintOuts.add getAst(print(argStr, arg)) - if exp[i].kind != nnkExprEqExpr: - result[i] = arg - else: - result[i][1] = arg + if exp[0].kind == nnkIdent and + $exp[0] in ["and", "or", "not", "in", "notin", "==", "<=", + ">=", "<", ">", "!=", "is", "isnot"]: + for i in countup(1, exp.len - 1): + if exp[i].kind notin nnkLiterals: + inc counter + var arg = newIdentNode(":p" & $counter) + var argStr = exp[i].toStrLit + var paramAst = exp[i] + if exp[i].kind == nnkIdent: + argsPrintOuts.add getAst(print(argStr, paramAst)) + if exp[i].kind in nnkCallKinds: + var callVar = newIdentNode(":c" & $counter) + argsAsgns.add getAst(asgn(callVar, paramAst)) + result[i] = callVar + argsPrintOuts.add getAst(print(argStr, callVar)) + if exp[i].kind == nnkExprEqExpr: + # ExprEqExpr + # Ident !"v" + # IntLit 2 + result[i] = exp[i][1] + if exp[i].typekind notin {ntyTypeDesc}: + argsAsgns.add getAst(asgn(arg, paramAst)) + argsPrintOuts.add getAst(print(argStr, arg)) + if exp[i].kind != nnkExprEqExpr: + result[i] = arg + else: + result[i][1] = arg case checked.kind of nnkCallKinds: @@ -292,7 +318,7 @@ macro check*(conditions: stmt): stmt {.immediate.} = result = getAst(rewrite(checked, checked.lineinfo, checked.toStrLit)) -template require*(conditions: stmt): stmt {.immediate.} = +template require*(conditions: untyped) = ## Same as `check` except any failed test causes the program to quit ## immediately. Any teardown statements are not executed and the failed ## test output is not generated. @@ -302,7 +328,7 @@ template require*(conditions: stmt): stmt {.immediate.} = check conditions abortOnError = savedAbortOnError -macro expect*(exceptions: varargs[expr], body: stmt): stmt {.immediate.} = +macro expect*(exceptions: varargs[typed], body: untyped): untyped = ## Test if `body` raises an exception found in the passed `exceptions`. ## The test passes if the raised exception is part of the acceptable ## exceptions. Otherwise, it fails. @@ -310,7 +336,7 @@ macro expect*(exceptions: varargs[expr], body: stmt): stmt {.immediate.} = ## ## .. code-block:: nim ## - ## import math + ## import math, random ## proc defectiveRobot() = ## randomize() ## case random(1..4) diff --git a/lib/pure/xmldom.nim b/lib/pure/xmldom.nim index 6cf837f25..559f45348 100644 --- a/lib/pure/xmldom.nim +++ b/lib/pure/xmldom.nim @@ -51,6 +51,9 @@ const # Illegal characters illegalChars = {'>', '<', '&', '"'} + # standard xml: attribute names + # see https://www.w3.org/XML/1998/namespace + stdattrnames = ["lang", "space", "base", "id"] type Feature = tuple[name: string, version: string] @@ -229,12 +232,15 @@ proc createAttributeNS*(doc: PDocument, namespaceURI: string, qualifiedName: str raise newException(EInvalidCharacterErr, "Invalid character") # Exceptions if qualifiedName.contains(':'): + let qfnamespaces = qualifiedName.toLower().split(':') if isNil(namespaceURI): raise newException(ENamespaceErr, "When qualifiedName contains a prefix namespaceURI cannot be nil") - elif qualifiedName.split(':')[0].toLower() == "xml" and namespaceURI != "http://www.w3.org/XML/1998/namespace": + elif qfnamespaces[0] == "xml" and + namespaceURI != "http://www.w3.org/XML/1998/namespace" and + qfnamespaces[1] notin stdattrnames: raise newException(ENamespaceErr, "When the namespace prefix is \"xml\" namespaceURI has to be \"http://www.w3.org/XML/1998/namespace\"") - elif qualifiedName.split(':')[1].toLower() == "xmlns" and namespaceURI != "http://www.w3.org/2000/xmlns/": + elif qfnamespaces[1] == "xmlns" and namespaceURI != "http://www.w3.org/2000/xmlns/": raise newException(ENamespaceErr, "When the namespace prefix is \"xmlns\" namespaceURI has to be \"http://www.w3.org/2000/xmlns/\"") @@ -305,9 +311,12 @@ proc createElement*(doc: PDocument, tagName: string): PElement = proc createElementNS*(doc: PDocument, namespaceURI: string, qualifiedName: string): PElement = ## Creates an element of the given qualified name and namespace URI. if qualifiedName.contains(':'): + let qfnamespaces = qualifiedName.toLower().split(':') if isNil(namespaceURI): raise newException(ENamespaceErr, "When qualifiedName contains a prefix namespaceURI cannot be nil") - elif qualifiedName.split(':')[0].toLower() == "xml" and namespaceURI != "http://www.w3.org/XML/1998/namespace": + elif qfnamespaces[0] == "xml" and + namespaceURI != "http://www.w3.org/XML/1998/namespace" and + qfnamespaces[1] notin stdattrnames: raise newException(ENamespaceErr, "When the namespace prefix is \"xml\" namespaceURI has to be \"http://www.w3.org/XML/1998/namespace\"") diff --git a/lib/stdlib.nimble b/lib/stdlib.nimble index e8bb364f1..4b0066ee8 100644 --- a/lib/stdlib.nimble +++ b/lib/stdlib.nimble @@ -1,6 +1,6 @@ [Package] name = "stdlib" -version = "0.13.0" +version = "0.14.3" author = "Dominik Picheta" description = "Nim's standard library." license = "MIT" diff --git a/lib/system.nim b/lib/system.nim index da37512d6..6af5dc01f 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -66,8 +66,10 @@ type `ref`* {.magic: Pointer.}[T] ## built-in generic traced pointer type `nil` {.magic: "Nil".} - expr* {.magic: Expr.} ## meta type to denote an expression (for templates) - stmt* {.magic: Stmt.} ## meta type to denote a statement (for templates) + expr* {.magic: Expr, deprecated.} ## meta type to denote an expression (for templates) + ## **Deprecated** since version 0.15. Use ``untyped`` instead. + stmt* {.magic: Stmt, deprecated.} ## meta type to denote a statement (for templates) + ## **Deprecated** since version 0.15. Use ``typed`` instead. typedesc* {.magic: TypeDesc.} ## meta type to denote a type description void* {.magic: "VoidType".} ## meta type to denote the absence of any type auto* {.magic: Expr.} ## meta type for automatic type determination @@ -302,8 +304,7 @@ proc `==` *(x, y: pointer): bool {.magic: "EqRef", noSideEffect.} ## echo (a == b) # true due to the special meaning of `nil`/0 as a pointer proc `==` *(x, y: string): bool {.magic: "EqStr", noSideEffect.} ## Checks for equality between two `string` variables -proc `==` *(x, y: cstring): bool {.magic: "EqCString", noSideEffect.} - ## Checks for equality between two `cstring` variables + proc `==` *(x, y: char): bool {.magic: "EqCh", noSideEffect.} ## Checks for equality between two `char` variables proc `==` *(x, y: bool): bool {.magic: "EqB", noSideEffect.} @@ -339,15 +340,15 @@ proc `<` *[T](x, y: ref T): bool {.magic: "LtPtr", noSideEffect.} proc `<` *[T](x, y: ptr T): bool {.magic: "LtPtr", noSideEffect.} proc `<` *(x, y: pointer): bool {.magic: "LtPtr", noSideEffect.} -template `!=` * (x, y: expr): expr {.immediate.} = +template `!=` * (x, y: untyped): untyped = ## unequals operator. This is a shorthand for ``not (x == y)``. not (x == y) -template `>=` * (x, y: expr): expr {.immediate.} = +template `>=` * (x, y: untyped): untyped = ## "is greater or equals" operator. This is the same as ``y <= x``. y <= x -template `>` * (x, y: expr): expr {.immediate.} = +template `>` * (x, y: untyped): untyped = ## "is greater" operator. This is the same as ``y < x``. y < x @@ -588,6 +589,9 @@ proc sizeof*[T](x: T): int {.magic: "SizeOf", noSideEffect.} ## that one never needs to know ``x``'s size. As a special semantic rule, ## ``x`` may also be a type identifier (``sizeof(int)`` is valid). ## + ## Limitations: If used within nim VM context ``sizeof`` will only work + ## for simple types. + ## ## .. code-block:: nim ## sizeof('A') #=> 1 ## sizeof(2) #=> 8 @@ -667,6 +671,12 @@ proc newSeq*[T](len = 0.Natural): seq[T] = ## #inputStrings[3] = "out of bounds" newSeq(result, len) +proc newSeqOfCap*[T](cap: Natural): seq[T] {. + magic: "NewSeqOfCap", noSideEffect.} = + ## creates a new sequence of type ``seq[T]`` with length 0 and capacity + ## ``cap``. + discard + proc len*[TOpenArray: openArray|varargs](x: TOpenArray): int {. magic: "LengthOpenArray", noSideEffect.} proc len*(x: string): int {.magic: "LengthStr", noSideEffect.} @@ -1090,13 +1100,13 @@ proc contains*[T](s: Slice[T], value: T): bool {.noSideEffect, inline.} = ## assert((1..3).contains(4) == false) result = s.a <= value and value <= s.b -template `in` * (x, y: expr): expr {.immediate, dirty.} = contains(y, x) +template `in` * (x, y: untyped): untyped {.dirty.} = contains(y, x) ## Sugar for contains ## ## .. code-block:: Nim ## assert(1 in (1..3) == true) ## assert(5 in (1..3) == false) -template `notin` * (x, y: expr): expr {.immediate, dirty.} = not contains(y, x) +template `notin` * (x, y: untyped): untyped {.dirty.} = not contains(y, x) ## Sugar for not containing ## ## .. code-block:: Nim @@ -1115,7 +1125,7 @@ proc `is` *[T, S](x: T, y: S): bool {.magic: "Is", noSideEffect.} ## ## assert(test[int](3) == 3) ## assert(test[string]("xyz") == 0) -template `isnot` *(x, y: expr): expr {.immediate.} = not (x is y) +template `isnot` *(x, y: untyped): untyped = not (x is y) ## Negated version of `is`. Equivalent to ``not(x is y)``. proc `of` *[T, S](x: T, y: S): bool {.magic: "Of", noSideEffect.} @@ -1291,7 +1301,7 @@ const when hasThreadSupport and defined(tcc) and not compileOption("tlsEmulation"): # tcc doesn't support TLS {.error: "``--tlsEmulation:on`` must be used when using threads with tcc backend".} - + when defined(boehmgc): when defined(windows): const boehmLib = "boehmgc.dll" @@ -1521,7 +1531,7 @@ type # these work for most platforms: ## This is the same as the type ``unsigned long long`` in *C*. cstringArray* {.importc: "char**", nodecl.} = ptr - array [0..ArrayDummySize, cstring] + array[0..ArrayDummySize, cstring] ## This is binary compatible to the type ``char**`` in *C*. The array's ## high value is large enough to disable bounds checking in practice. ## Use `cstringArrayToSeq` to convert it into a ``seq[string]``. @@ -1591,34 +1601,32 @@ proc substr*(s: string, first, last: int): string {. ## is used instead: This means ``substr`` can also be used to `cut`:idx: ## or `limit`:idx: a string's length. -when not defined(nimscript): - proc zeroMem*(p: pointer, size: Natural) {.importc, noDecl, benign.} +when not defined(nimscript) and not defined(JS): + proc zeroMem*(p: pointer, size: Natural) {.inline, benign.} ## overwrites the contents of the memory at ``p`` with the value 0. ## Exactly ``size`` bytes will be overwritten. Like any procedure ## dealing with raw memory this is *unsafe*. - proc copyMem*(dest, source: pointer, size: Natural) {. - importc: "memcpy", header: "<string.h>", benign.} + proc copyMem*(dest, source: pointer, size: Natural) {.inline, benign.} ## copies the contents from the memory at ``source`` to the memory ## at ``dest``. Exactly ``size`` bytes will be copied. The memory ## regions may not overlap. Like any procedure dealing with raw ## memory this is *unsafe*. - proc moveMem*(dest, source: pointer, size: Natural) {. - importc: "memmove", header: "<string.h>", benign.} + proc moveMem*(dest, source: pointer, size: Natural) {.inline, benign.} ## copies the contents from the memory at ``source`` to the memory ## at ``dest``. Exactly ``size`` bytes will be copied. The memory ## regions may overlap, ``moveMem`` handles this case appropriately ## and is thus somewhat more safe than ``copyMem``. Like any procedure ## dealing with raw memory this is still *unsafe*, though. - proc equalMem*(a, b: pointer, size: Natural): bool {. - importc: "equalMem", noDecl, noSideEffect.} + proc equalMem*(a, b: pointer, size: Natural): bool {.inline, noSideEffect.} ## compares the memory blocks ``a`` and ``b``. ``size`` bytes will ## be compared. If the blocks are equal, true is returned, false ## otherwise. Like any procedure dealing with raw memory this is ## *unsafe*. +when not defined(nimscript): when hasAlloc: proc alloc*(size: Natural): pointer {.noconv, rtl, tags: [], benign.} ## allocates a new memory block with at least ``size`` bytes. The @@ -1736,11 +1744,18 @@ proc swap*[T](a, b: var T) {.magic: "Swap", noSideEffect.} ## swaps the values `a` and `b`. This is often more efficient than ## ``tmp = a; a = b; b = tmp``. Particularly useful for sorting algorithms. -template `>=%` *(x, y: expr): expr {.immediate.} = y <=% x +when not defined(js) and not defined(booting) and defined(nimTrMacros): + template swapRefsInArray*{swap(arr[a], arr[b])}(arr: openarray[ref], a, b: int) = + # Optimize swapping of array elements if they are refs. Default swap + # implementation will cause unsureAsgnRef to be emitted which causes + # unnecessary slow down in this case. + swap(cast[ptr pointer](addr arr[a])[], cast[ptr pointer](addr arr[b])[]) + +template `>=%` *(x, y: untyped): untyped = y <=% x ## treats `x` and `y` as unsigned and compares them. ## Returns true iff ``unsigned(x) >= unsigned(y)``. -template `>%` *(x, y: expr): expr {.immediate.} = y <% x +template `>%` *(x, y: untyped): untyped = y <% x ## treats `x` and `y` as unsigned and compares them. ## Returns true iff ``unsigned(x) > unsigned(y)``. @@ -1807,10 +1822,10 @@ const NimMajor*: int = 0 ## is the major number of Nim's version. - NimMinor*: int = 13 + NimMinor*: int = 14 ## is the minor number of Nim's version. - NimPatch*: int = 1 + NimPatch*: int = 3 ## is the patch number of Nim's version. NimVersion*: string = $NimMajor & "." & $NimMinor & "." & $NimPatch @@ -1868,7 +1883,7 @@ iterator countdown*[T](a, b: T, step = 1): T {.inline.} = yield res dec(res, step) -template countupImpl(incr: stmt) {.immediate, dirty.} = +template countupImpl(incr: untyped) {.oldimmediate, dirty.} = when T is IntLikeForCount: var res = int(a) while res <= int(b): @@ -2163,7 +2178,7 @@ proc `&` *[T](x: T, y: seq[T]): seq[T] {.noSideEffect.} = result[i+1] = y[i] when not defined(nimscript): - when not defined(JS): + when not defined(JS) or defined(nimphp): proc seqToPtr[T](x: seq[T]): pointer {.inline, nosideeffect.} = result = cast[pointer](x) else: @@ -2293,12 +2308,15 @@ proc `$`*[T: tuple|object](x: T): string = if not firstElement: result.add(", ") result.add(name) result.add(": ") - when compiles(value.isNil): - if value.isNil: result.add "nil" - else: result.add($value) + when compiles($value): + when compiles(value.isNil): + if value.isNil: result.add "nil" + else: result.add($value) + else: + result.add($value) + firstElement = false else: - result.add($value) - firstElement = false + result.add("...") result.add(")") proc collectionToString[T: set | seq](x: T, b, e: string): string = @@ -2511,7 +2529,7 @@ template newException*(exceptn: typedesc, message: string): expr = e when hostOS == "standalone": - include panicoverride + include "$projectpath/panicoverride" when not declared(sysFatal): when hostOS == "standalone": @@ -2564,15 +2582,11 @@ else: when not defined(JS): #and not defined(nimscript): {.push stack_trace: off, profiler:off.} - when not ( - defined(nimscript) or - defined(boehmgc) or - defined(gogc) or - (defined(nogc) and defined(useMalloc))): - proc initAllocator() {.inline.} - - when not defined(nimscript) and not defined(nogc): - proc initGC() + when not defined(nimscript) and (not defined(nogc) or not defined(useMalloc)): + when not defined(gcStack): + proc initGC() + when not defined(boehmgc) and not defined(gogc) and not defined(gcStack): + proc initAllocator() {.inline.} proc initStackBottom() {.inline, compilerproc.} = # WARNING: This is very fragile! An array size of 8 does not work on my @@ -2617,11 +2631,21 @@ when not defined(JS): #and not defined(nimscript): {.deprecated: [TFile: File, TFileHandle: FileHandle, TFileMode: FileMode].} - when not defined(nimscript): - include "system/ansi_c" + include "system/ansi_c" + when not defined(nimscript): proc cmp(x, y: string): int = result = int(c_strcmp(x, y)) + + proc zeroMem(p: pointer, size: Natural) = + c_memset(p, 0, size) + proc copyMem(dest, source: pointer, size: Natural) = + c_memcpy(dest, source, size) + proc moveMem(dest, source: pointer, size: Natural) = + c_memmove(dest, source, size) + proc equalMem(a, b: pointer, size: Natural): bool = + c_memcmp(a, b, size) == 0 + else: proc cmp(x, y: string): int = if x < y: result = -1 @@ -2629,28 +2653,20 @@ when not defined(JS): #and not defined(nimscript): else: result = 0 when defined(nimscript): + proc readFile*(filename: string): string {.tags: [ReadIOEffect], benign.} + ## Opens a file named `filename` for reading. + ## + ## Then calls `readAll <#readAll>`_ and closes the file afterwards. + ## Returns the string. Raises an IO exception in case of an error. If + ## you need to call this inside a compile time macro you can use + ## `staticRead <#staticRead>`_. + proc writeFile*(filename, content: string) {.tags: [WriteIOEffect], benign.} ## Opens a file named `filename` for writing. Then writes the ## `content` completely to the file and closes the file afterwards. ## Raises an IO exception in case of an error. when not defined(nimscript) and hostOS != "standalone": - when defined(windows): - # work-around C's sucking abstraction: - # BUGFIX: stdin and stdout should be binary files! - proc setmode(handle, mode: int) {.importc: "setmode", - header: "<io.h>".} - proc fileno(f: C_TextFileStar): int {.importc: "fileno", - header: "<fcntl.h>".} - var - O_BINARY {.importc: "O_BINARY", nodecl.}: int - - # we use binary mode on Windows: - setmode(fileno(c_stdin), O_BINARY) - setmode(fileno(c_stdout), O_BINARY) - - when defined(endb): - proc endbStep() # text file handling: var @@ -2661,6 +2677,21 @@ when not defined(JS): #and not defined(nimscript): stderr* {.importc: "stderr", header: "<stdio.h>".}: File ## The standard error stream. + when defined(windows): + # work-around C's sucking abstraction: + # BUGFIX: stdin and stdout should be binary files! + proc c_setmode(handle, mode: cint) {.importc: "_setmode", + header: "<io.h>".} + var + O_BINARY {.importc: "O_BINARY", nodecl.}: cint + + # we use binary mode on Windows: + c_setmode(c_fileno(stdin), O_BINARY) + c_setmode(c_fileno(stdout), O_BINARY) + + when defined(endb): + proc endbStep() + when defined(useStdoutAsStdmsg): template stdmsg*: File = stdout else: @@ -2699,17 +2730,19 @@ when not defined(JS): #and not defined(nimscript): ## ## Default mode is readonly. Returns true iff the file could be reopened. - proc close*(f: File) {.importc: "fclose", header: "<stdio.h>", tags: [].} + proc setStdIoUnbuffered*() {.tags: [], benign.} + ## Configures `stdin`, `stdout` and `stderr` to be unbuffered. + + proc close*(f: File) {.tags: [].} ## Closes the file. proc endOfFile*(f: File): bool {.tags: [], benign.} ## Returns true iff `f` is at the end. - proc readChar*(f: File): char {. - importc: "fgetc", header: "<stdio.h>", tags: [ReadIOEffect].} + proc readChar*(f: File): char {.tags: [ReadIOEffect].} ## Reads a single character from the stream `f`. - proc flushFile*(f: File) {. - importc: "fflush", header: "<stdio.h>", tags: [WriteIOEffect].} + + proc flushFile*(f: File) {.tags: [WriteIOEffect].} ## Flushes `f`'s buffer. proc readAll*(file: File): TaintedString {.tags: [ReadIOEffect], benign.} @@ -2779,6 +2812,9 @@ when not defined(JS): #and not defined(nimscript): ## reads `len` bytes into the buffer `a` starting at ``a[start]``. Returns ## the actual number of bytes that have been read which may be less than ## `len` (if not as many bytes are remaining), but not greater. + ## + ## **Warning:** The buffer `a` must be pre-allocated. This can be done + ## using, for example, ``newString``. proc readBuffer*(f: File, buffer: pointer, len: Natural): int {. tags: [ReadIOEffect], benign.} @@ -2812,8 +2848,7 @@ when not defined(JS): #and not defined(nimscript): ## retrieves the current position of the file pointer that is used to ## read from the file `f`. The file's first byte has the index zero. - proc getFileHandle*(f: File): FileHandle {.importc: "fileno", - header: "<stdio.h>"} + proc getFileHandle*(f: File): FileHandle ## returns the OS file handle of the file ``f``. This is only useful for ## platform specific programming. @@ -2879,6 +2914,7 @@ when not defined(JS): #and not defined(nimscript): when declared(initAllocator): initAllocator() when hasThreadSupport: + const insideRLocksModule = false include "system/syslocks" when hostOS != "standalone": include "system/threads" elif not defined(nogc) and not defined(nimscript): @@ -3031,34 +3067,6 @@ when not defined(JS): #and not defined(nimscript): {.pop.} # stacktrace when not defined(nimscript): - proc likely*(val: bool): bool {.importc: "likely", nodecl, nosideeffect.} - ## Hints the optimizer that `val` is likely going to be true. - ## - ## You can use this proc to decorate a branch condition. On certain - ## platforms this can help the processor predict better which branch is - ## going to be run. Example: - ## - ## .. code-block:: nim - ## for value in inputValues: - ## if likely(value <= 100): - ## process(value) - ## else: - ## echo "Value too big!" - - proc unlikely*(val: bool): bool {.importc: "unlikely", nodecl, nosideeffect.} - ## Hints the optimizer that `val` is likely going to be false. - ## - ## You can use this proc to decorate a branch condition. On certain - ## platforms this can help the processor predict better which branch is - ## going to be run. Example: - ## - ## .. code-block:: nim - ## for value in inputValues: - ## if unlikely(value > 100): - ## echo "Value too big!" - ## else: - ## process(value) - proc rawProc*[T: proc](x: T): pointer {.noSideEffect, inline.} = ## retrieves the raw proc pointer of the closure `x`. This is ## useful for interfacing closures with C. @@ -3126,11 +3134,63 @@ proc quit*(errormsg: string, errorcode = QuitFailure) {.noReturn.} = {.pop.} # checks {.pop.} # hints +when not defined(JS): + proc likely_proc(val: bool): bool {.importc: "likely", nodecl, nosideeffect.} + proc unlikely_proc(val: bool): bool {.importc: "unlikely", nodecl, nosideeffect.} + +template likely*(val: bool): bool = + ## Hints the optimizer that `val` is likely going to be true. + ## + ## You can use this template to decorate a branch condition. On certain + ## platforms this can help the processor predict better which branch is + ## going to be run. Example: + ## + ## .. code-block:: nim + ## for value in inputValues: + ## if likely(value <= 100): + ## process(value) + ## else: + ## echo "Value too big!" + ## + ## On backends without branch prediction (JS and the nimscript VM), this + ## template will not affect code execution. + when nimvm: + val + else: + when defined(JS): + val + else: + likely_proc(val) + +template unlikely*(val: bool): bool = + ## Hints the optimizer that `val` is likely going to be false. + ## + ## You can use this proc to decorate a branch condition. On certain + ## platforms this can help the processor predict better which branch is + ## going to be run. Example: + ## + ## .. code-block:: nim + ## for value in inputValues: + ## if unlikely(value > 100): + ## echo "Value too big!" + ## else: + ## process(value) + ## + ## On backends without branch prediction (JS and the nimscript VM), this + ## template will not affect code execution. + when nimvm: + val + else: + when defined(JS): + val + else: + unlikely_proc(val) + proc `/`*(x, y: int): float {.inline, noSideEffect.} = ## integer division that results in a float. result = toFloat(x) / toFloat(y) -template spliceImpl(s, a, L, b: expr): stmt {.immediate.} = +template spliceImpl(s, a, L, b: untyped): untyped = # make room for additional elements or cut: var slen = s.len var shift = b.len - L @@ -3172,7 +3232,7 @@ proc `[]`*[Idx, T](a: array[Idx, T], x: Slice[int]): seq[T] = when low(a) < 0: {.error: "Slicing for arrays with negative indices is unsupported.".} var L = x.b - x.a + 1 - newSeq(result, L) + result = newSeq[T](L) for i in 0.. <L: result[i] = a[i + x.a] proc `[]=`*[Idx, T](a: var array[Idx, T], x: Slice[int], b: openArray[T]) = @@ -3308,7 +3368,11 @@ proc astToStr*[T](x: T): string {.magic: "AstToStr", noSideEffect.} proc instantiationInfo*(index = -1, fullPaths = false): tuple[ filename: string, line: int] {. magic: "InstantiationInfo", noSideEffect.} - ## provides access to the compiler's instantiation stack line information. + ## provides access to the compiler's instantiation stack line information + ## of a template. + ## + ## While similar to the `caller info`:idx: of other languages, it is determined + ## at compile time. ## ## This proc is mostly useful for meta programming (eg. ``assert`` template) ## to retrieve information about the current filename and line number. @@ -3412,7 +3476,7 @@ iterator mitems*(a: var string): var char {.inline.} = when not defined(nimhygiene): {.pragma: inject.} -template onFailedAssert*(msg: expr, code: stmt): stmt {.dirty, immediate.} = +template onFailedAssert*(msg, code: untyped): untyped {.dirty.} = ## Sets an assertion failure handler that will intercept any assert ## statements following `onFailedAssert` in the current module scope. ## @@ -3425,7 +3489,7 @@ template onFailedAssert*(msg: expr, code: stmt): stmt {.dirty, immediate.} = ## e.lineinfo = instantiationInfo(-2) ## raise e ## - template failedAssertImpl(msgIMPL: string): stmt {.dirty.} = + template failedAssertImpl(msgIMPL: string): untyped {.dirty.} = let msg = msgIMPL code @@ -3578,7 +3642,43 @@ proc xlen*[T](x: seq[T]): int {.magic: "XLenSeq", noSideEffect.} = ## This is an optimization that rarely makes sense. discard + +proc `==` *(x, y: cstring): bool {.magic: "EqCString", noSideEffect, + inline.} = + ## Checks for equality between two `cstring` variables. + proc strcmp(a, b: cstring): cint {.noSideEffect, + importc, header: "<string.h>".} + if pointer(x) == pointer(y): result = true + elif x.isNil or y.isNil: result = false + else: result = strcmp(x, y) == 0 + +template closureScope*(body: untyped): untyped = + ## Useful when creating a closure in a loop to capture local loop variables by + ## their current iteration values. Example: + ## + ## .. code-block:: nim + ## var myClosure : proc() + ## # without closureScope: + ## for i in 0 .. 5: + ## let j = i + ## if j == 3: + ## myClosure = proc() = echo j + ## myClosure() # outputs 5. `j` is changed after closure creation + ## # with closureScope: + ## for i in 0 .. 5: + ## closureScope: # Everything in this scope is locked after closure creation + ## let j = i + ## if j == 3: + ## myClosure = proc() = echo j + ## myClosure() # outputs 3 + (proc() = body)() + {.pop.} #{.push warning[GcMem]: off, warning[Uninit]: off.} when defined(nimconfig): include "system/nimscript" + +when defined(windows) and appType == "console" and not defined(nimconfig): + proc setConsoleOutputCP(codepage: cint): cint {.stdcall, dynlib: "kernel32", + importc: "SetConsoleOutputCP".} + discard setConsoleOutputCP(65001) # 65001 - utf-8 codepage diff --git a/lib/system/alloc.nim b/lib/system/alloc.nim index e0fd53b7b..bed9fd906 100644 --- a/lib/system/alloc.nim +++ b/lib/system/alloc.nim @@ -27,15 +27,14 @@ const type PTrunk = ptr Trunk - Trunk {.final.} = object + Trunk = object next: PTrunk # all nodes are connected with this pointer key: int # start address at bit 0 bits: array[0..IntsPerTrunk-1, int] # a bit vector TrunkBuckets = array[0..255, PTrunk] - IntSet {.final.} = object + IntSet = object data: TrunkBuckets -{.deprecated: [TIntSet: IntSet, TTrunk: Trunk, TTrunkBuckets: TrunkBuckets].} type AlignType = BiggestFloat @@ -64,8 +63,6 @@ type next, prev: PBigChunk # chunks of the same (or bigger) size align: int data: AlignType # start of usable memory -{.deprecated: [TAlignType: AlignType, TFreeCell: FreeCell, TBaseChunk: BaseChunk, - TBigChunk: BigChunk, TSmallChunk: SmallChunk].} template smallChunkOverhead(): expr = sizeof(SmallChunk)-sizeof(AlignType) template bigChunkOverhead(): expr = sizeof(BigChunk)-sizeof(AlignType) @@ -79,18 +76,18 @@ template bigChunkOverhead(): expr = sizeof(BigChunk)-sizeof(AlignType) type PLLChunk = ptr LLChunk - LLChunk {.pure.} = object ## *low-level* chunk + LLChunk = object ## *low-level* chunk size: int # remaining size acc: int # accumulator next: PLLChunk # next low-level chunk; only needed for dealloc PAvlNode = ptr AvlNode - AvlNode {.pure, final.} = object + AvlNode = object link: array[0..1, PAvlNode] # Left (0) and right (1) links key, upperBound: int level: int - MemRegion {.final, pure.} = object + MemRegion = object minLargeObj, maxLargeObj: int freeSmallChunks: array[0..SmallChunkSize div MemAlign-1, PSmallChunk] llmem: PLLChunk @@ -99,6 +96,7 @@ type freeChunksList: PBigChunk # XXX make this a datastructure with O(1) access chunkStarts: IntSet root, deleted, last, freeAvlNodes: PAvlNode + locked: bool # if locked, we cannot free pages. {.deprecated: [TLLChunk: LLChunk, TAvlNode: AvlNode, TMemRegion: MemRegion].} # shared: @@ -234,7 +232,8 @@ proc isSmallChunk(c: PChunk): bool {.inline.} = proc chunkUnused(c: PChunk): bool {.inline.} = result = not c.used -iterator allObjects(m: MemRegion): pointer {.inline.} = +iterator allObjects(m: var MemRegion): pointer {.inline.} = + m.locked = true for s in elements(m.chunkStarts): # we need to check here again as it could have been modified: if s in m.chunkStarts: @@ -252,6 +251,7 @@ iterator allObjects(m: MemRegion): pointer {.inline.} = else: let c = cast[PBigChunk](c) yield addr(c.data) + m.locked = false proc iterToProc*(iter: typed, envType: typedesc; procName: untyped) {. magic: "Plugin", compileTime.} @@ -311,7 +311,7 @@ proc freeOsChunks(a: var MemRegion, p: pointer, size: int) = osDeallocPages(p, size) decCurrMem(a, size) dec(a.freeMem, size) - #c_fprintf(c_stdout, "[Alloc] back to OS: %ld\n", size) + #c_fprintf(stdout, "[Alloc] back to OS: %ld\n", size) proc isAccessible(a: MemRegion, p: pointer): bool {.inline.} = result = contains(a.chunkStarts, pageIndex(p)) @@ -324,9 +324,9 @@ proc contains[T](list, x: T): bool = proc writeFreeList(a: MemRegion) = var it = a.freeChunksList - c_fprintf(c_stdout, "freeChunksList: %p\n", it) + c_fprintf(stdout, "freeChunksList: %p\n", it) while it != nil: - c_fprintf(c_stdout, "it: %p, next: %p, prev: %p\n", + c_fprintf(stdout, "it: %p, next: %p, prev: %p\n", it, it.next, it.prev) it = it.next @@ -385,7 +385,7 @@ proc freeBigChunk(a: var MemRegion, c: PBigChunk) = excl(a.chunkStarts, pageIndex(c)) c = cast[PBigChunk](le) - if c.size < ChunkOsReturn or doNotUnmap: + if c.size < ChunkOsReturn or doNotUnmap or a.locked: incl(a, a.chunkStarts, pageIndex(c)) updatePrevSize(a, c, c.size) listAdd(a.freeChunksList, c) @@ -442,26 +442,29 @@ proc getSmallChunk(a: var MemRegion): PSmallChunk = # ----------------------------------------------------------------------------- proc isAllocatedPtr(a: MemRegion, p: pointer): bool {.benign.} -proc allocInv(a: MemRegion): bool = - ## checks some (not all yet) invariants of the allocator's data structures. - for s in low(a.freeSmallChunks)..high(a.freeSmallChunks): - var c = a.freeSmallChunks[s] - while not (c == nil): - if c.next == c: - echo "[SYSASSERT] c.next == c" - return false - if not (c.size == s * MemAlign): - echo "[SYSASSERT] c.size != s * MemAlign" - return false - var it = c.freeList - while not (it == nil): - if not (it.zeroField == 0): - echo "[SYSASSERT] it.zeroField != 0" - c_printf("%ld %p\n", it.zeroField, it) +when true: + template allocInv(a: MemRegion): bool = true +else: + proc allocInv(a: MemRegion): bool = + ## checks some (not all yet) invariants of the allocator's data structures. + for s in low(a.freeSmallChunks)..high(a.freeSmallChunks): + var c = a.freeSmallChunks[s] + while not (c == nil): + if c.next == c: + echo "[SYSASSERT] c.next == c" return false - it = it.next - c = c.next - result = true + if not (c.size == s * MemAlign): + echo "[SYSASSERT] c.size != s * MemAlign" + return false + var it = c.freeList + while not (it == nil): + if not (it.zeroField == 0): + echo "[SYSASSERT] it.zeroField != 0" + c_printf("%ld %p\n", it.zeroField, it) + return false + it = it.next + c = c.next + result = true proc rawAlloc(a: var MemRegion, requestedSize: int): pointer = sysAssert(allocInv(a), "rawAlloc: begin") @@ -469,7 +472,7 @@ proc rawAlloc(a: var MemRegion, requestedSize: int): pointer = sysAssert(requestedSize >= sizeof(FreeCell), "rawAlloc: requested size too small") var size = roundup(requestedSize, MemAlign) sysAssert(size >= requestedSize, "insufficient allocated size!") - #c_fprintf(c_stdout, "alloc; size: %ld; %ld\n", requestedSize, size) + #c_fprintf(stdout, "alloc; size: %ld; %ld\n", requestedSize, size) if size <= SmallChunkSize-smallChunkOverhead(): # allocate a small block: for small chunks, we use only its next pointer var s = size div MemAlign @@ -490,7 +493,7 @@ proc rawAlloc(a: var MemRegion, requestedSize: int): pointer = sysAssert(allocInv(a), "rawAlloc: begin c != nil") sysAssert c.next != c, "rawAlloc 5" #if c.size != size: - # c_fprintf(c_stdout, "csize: %lld; size %lld\n", c.size, size) + # c_fprintf(stdout, "csize: %lld; size %lld\n", c.size, size) sysAssert c.size == size, "rawAlloc 6" if c.freeList == nil: sysAssert(c.acc + smallChunkOverhead() + size <= SmallChunkSize, diff --git a/lib/system/ansi_c.nim b/lib/system/ansi_c.nim index 1abd8466d..a8e358229 100644 --- a/lib/system/ansi_c.nim +++ b/lib/system/ansi_c.nim @@ -13,60 +13,43 @@ {.push hints:off} -proc c_strcmp(a, b: cstring): cint {.header: "<string.h>", - noSideEffect, importc: "strcmp".} -proc c_memcmp(a, b: cstring, size: int): cint {.header: "<string.h>", - noSideEffect, importc: "memcmp".} -proc c_memcpy(a, b: cstring, size: int) {.header: "<string.h>", importc: "memcpy".} -proc c_strlen(a: cstring): int {.header: "<string.h>", - noSideEffect, importc: "strlen".} -proc c_memset(p: pointer, value: cint, size: int) {. - header: "<string.h>", importc: "memset".} - -when not declared(File): - type - C_TextFile {.importc: "FILE", header: "<stdio.h>", - final, incompleteStruct.} = object - C_BinaryFile {.importc: "FILE", header: "<stdio.h>", - final, incompleteStruct.} = object - C_TextFileStar = ptr C_TextFile - C_BinaryFileStar = ptr C_BinaryFile -else: - type - C_TextFileStar = File - C_BinaryFileStar = File +proc c_memchr(s: pointer, c: cint, n: csize): pointer {. + importc: "memchr", header: "<string.h>".} +proc c_memcmp(a, b: pointer, size: csize): cint {. + importc: "memcmp", header: "<string.h>", noSideEffect.} +proc c_memcpy(a, b: pointer, size: csize): pointer {. + importc: "memcpy", header: "<string.h>", discardable.} +proc c_memmove(a, b: pointer, size: csize): pointer {. + importc: "memmove", header: "<string.h>",discardable.} +proc c_memset(p: pointer, value: cint, size: csize): pointer {. + importc: "memset", header: "<string.h>", discardable.} +proc c_strcmp(a, b: cstring): cint {. + importc: "strcmp", header: "<string.h>", noSideEffect.} type C_JmpBuf {.importc: "jmp_buf", header: "<setjmp.h>".} = object -when not defined(vm): - var - c_stdin {.importc: "stdin", nodecl.}: C_TextFileStar - c_stdout {.importc: "stdout", nodecl.}: C_TextFileStar - c_stderr {.importc: "stderr", nodecl.}: C_TextFileStar - -# constants faked as variables: -when not declared(SIGINT): +when defined(windows): + const + SIGABRT = cint(22) + SIGFPE = cint(8) + SIGILL = cint(4) + SIGINT = cint(2) + SIGSEGV = cint(11) + SIGTERM = cint(15) +elif defined(macosx) or defined(linux) or defined(freebsd) or + defined(openbsd) or defined(netbsd) or defined(solaris): + const + SIGABRT = cint(6) + SIGFPE = cint(8) + SIGILL = cint(4) + SIGINT = cint(2) + SIGSEGV = cint(11) + SIGTERM = cint(15) + SIGPIPE = cint(13) +else: when NoFakeVars: - when defined(windows): - const - SIGABRT = cint(22) - SIGFPE = cint(8) - SIGILL = cint(4) - SIGINT = cint(2) - SIGSEGV = cint(11) - SIGTERM = cint(15) - elif defined(macosx) or defined(linux): - const - SIGABRT = cint(6) - SIGFPE = cint(8) - SIGILL = cint(4) - SIGINT = cint(2) - SIGSEGV = cint(11) - SIGTERM = cint(15) - SIGPIPE = cint(13) - else: - {.error: "SIGABRT not ported to your platform".} + {.error: "SIGABRT not ported to your platform".} else: var SIGINT {.importc: "SIGINT", nodecl.}: cint @@ -78,10 +61,7 @@ when not declared(SIGINT): var SIGPIPE {.importc: "SIGPIPE", nodecl.}: cint when defined(macosx): - when NoFakeVars: - const SIGBUS = cint(10) - else: - var SIGBUS {.importc: "SIGBUS", nodecl.}: cint + const SIGBUS = cint(10) else: template SIGBUS: expr = SIGSEGV @@ -103,72 +83,27 @@ else: proc c_setjmp(jmpb: C_JmpBuf): cint {. header: "<setjmp.h>", importc: "setjmp".} -proc c_signal(sign: cint, handler: proc (a: cint) {.noconv.}) {. - importc: "signal", header: "<signal.h>".} -proc c_raise(sign: cint) {.importc: "raise", header: "<signal.h>".} +type c_sighandler_t = proc (a: cint) {.noconv.} +proc c_signal(sign: cint, handler: proc (a: cint) {.noconv.}): c_sighandler_t {. + importc: "signal", header: "<signal.h>", discardable.} -proc c_fputs(c: cstring, f: C_TextFileStar) {.importc: "fputs", - header: "<stdio.h>".} -proc c_fgets(c: cstring, n: int, f: C_TextFileStar): cstring {. - importc: "fgets", header: "<stdio.h>".} -proc c_fgetc(stream: C_TextFileStar): int {.importc: "fgetc", - header: "<stdio.h>".} -proc c_ungetc(c: int, f: C_TextFileStar) {.importc: "ungetc", - header: "<stdio.h>".} -proc c_putc(c: char, stream: C_TextFileStar) {.importc: "putc", - header: "<stdio.h>".} -proc c_fprintf(f: C_TextFileStar, frmt: cstring) {. - importc: "fprintf", header: "<stdio.h>", varargs.} -proc c_printf(frmt: cstring) {. - importc: "printf", header: "<stdio.h>", varargs.} +proc c_fprintf(f: File, frmt: cstring): cint {. + importc: "fprintf", header: "<stdio.h>", varargs, discardable.} +proc c_printf(frmt: cstring): cint {. + importc: "printf", header: "<stdio.h>", varargs, discardable.} -proc c_fopen(filename, mode: cstring): C_TextFileStar {. - importc: "fopen", header: "<stdio.h>".} -proc c_fclose(f: C_TextFileStar) {.importc: "fclose", header: "<stdio.h>".} - -proc c_sprintf(buf, frmt: cstring): cint {.header: "<stdio.h>", - importc: "sprintf", varargs, noSideEffect.} +proc c_sprintf(buf, frmt: cstring): cint {. + importc: "sprintf", header: "<stdio.h>", varargs, noSideEffect.} # we use it only in a way that cannot lead to security issues -proc c_fread(buf: pointer, size, n: int, f: C_BinaryFileStar): int {. - importc: "fread", header: "<stdio.h>".} -proc c_fseek(f: C_BinaryFileStar, offset: clong, whence: int): int {. - importc: "fseek", header: "<stdio.h>".} - -proc c_fwrite(buf: pointer, size, n: int, f: C_BinaryFileStar): int {. - importc: "fwrite", header: "<stdio.h>".} - -proc c_exit(errorcode: cint) {.importc: "exit", header: "<stdlib.h>".} -proc c_ferror(stream: C_TextFileStar): bool {. - importc: "ferror", header: "<stdio.h>".} -proc c_fflush(stream: C_TextFileStar) {.importc: "fflush", header: "<stdio.h>".} -proc c_abort() {.importc: "abort", header: "<stdlib.h>".} -proc c_feof(stream: C_TextFileStar): bool {. - importc: "feof", header: "<stdio.h>".} +proc c_fileno(f: File): cint {. + importc: "fileno", header: "<fcntl.h>".} -proc c_malloc(size: int): pointer {.importc: "malloc", header: "<stdlib.h>".} -proc c_free(p: pointer) {.importc: "free", header: "<stdlib.h>".} -proc c_realloc(p: pointer, newsize: int): pointer {. +proc c_malloc(size: csize): pointer {. + importc: "malloc", header: "<stdlib.h>".} +proc c_free(p: pointer) {. + importc: "free", header: "<stdlib.h>".} +proc c_realloc(p: pointer, newsize: csize): pointer {. importc: "realloc", header: "<stdlib.h>".} -when hostOS != "standalone": - when not declared(errno): - when defined(NimrodVM): - var vmErrnoWrapper {.importc.}: ptr cint - template errno: expr = - bind vmErrnoWrapper - vmErrnoWrapper[] - else: - var errno {.importc, header: "<errno.h>".}: cint ## error variable -proc strerror(errnum: cint): cstring {.importc, header: "<string.h>".} - -proc c_remove(filename: cstring): cint {. - importc: "remove", header: "<stdio.h>".} -proc c_rename(oldname, newname: cstring): cint {. - importc: "rename", header: "<stdio.h>".} - -proc c_system(cmd: cstring): cint {.importc: "system", header: "<stdlib.h>".} -proc c_getenv(env: cstring): cstring {.importc: "getenv", header: "<stdlib.h>".} -proc c_putenv(env: cstring): cint {.importc: "putenv", header: "<stdlib.h>".} - {.pop} diff --git a/lib/system/assign.nim b/lib/system/assign.nim index 55d7572e2..231a20d86 100644 --- a/lib/system/assign.nim +++ b/lib/system/assign.nim @@ -211,14 +211,14 @@ proc genericReset(dest: pointer, mt: PNimType) = zeroMem(dest, mt.size) # set raw bits to zero proc selectBranch(discVal, L: int, - a: ptr array [0..0x7fff, ptr TNimNode]): ptr TNimNode = + a: ptr array[0..0x7fff, ptr TNimNode]): ptr TNimNode = result = a[L] # a[L] contains the ``else`` part (but may be nil) if discVal <% L: var x = a[discVal] if x != nil: result = x proc FieldDiscriminantCheck(oldDiscVal, newDiscVal: int, - a: ptr array [0..0x7fff, ptr TNimNode], + a: ptr array[0..0x7fff, ptr TNimNode], L: int) {.compilerProc.} = var oldBranch = selectBranch(oldDiscVal, L, a) var newBranch = selectBranch(newDiscVal, L, a) diff --git a/lib/system/chcks.nim b/lib/system/chcks.nim index 6caf99d27..27a3708ea 100644 --- a/lib/system/chcks.nim +++ b/lib/system/chcks.nim @@ -51,7 +51,6 @@ proc chckRangeF(x, a, b: float): float = proc chckNil(p: pointer) = if p == nil: sysFatal(ValueError, "attempt to write to a nil address") - #c_raise(SIGSEGV) proc chckObj(obj, subclass: PNimType) {.compilerproc.} = # checks if obj is of type subclass: diff --git a/lib/system/debugger.nim b/lib/system/debugger.nim index b18c61755..cc6919d36 100644 --- a/lib/system/debugger.nim +++ b/lib/system/debugger.nim @@ -108,8 +108,8 @@ proc fileMatches(c, bp: cstring): bool = # and the character for the suffix does not exist or # is one of: \ / : # depending on the OS case does not matter! - var blen: int = c_strlen(bp) - var clen: int = c_strlen(c) + var blen: int = bp.len + var clen: int = c.len if blen > clen: return false # check for \ / : if clen-blen-1 >= 0 and c[clen-blen-1] notin {'\\', '/', ':'}: @@ -159,7 +159,7 @@ type {.deprecated: [THash: Hash, TWatchpoint: Watchpoint].} var - watchpoints: array [0..99, Watchpoint] + watchpoints: array[0..99, Watchpoint] watchpointsLen: int proc `!&`(h: Hash, val: int): Hash {.inline.} = diff --git a/lib/system/deepcopy.nim b/lib/system/deepcopy.nim index 03230e541..5445a067c 100644 --- a/lib/system/deepcopy.nim +++ b/lib/system/deepcopy.nim @@ -36,7 +36,7 @@ proc copyDeepString(src: NimString): NimString {.inline.} = if src != nil: result = rawNewStringNoInit(src.len) result.len = src.len - c_memcpy(result.data, src.data, src.len + 1) + copyMem(addr(result.data), addr(src.data), src.len + 1) proc genericDeepCopyAux(dest, src: pointer, mt: PNimType) = var @@ -124,7 +124,9 @@ proc genericDeepCopyAux(dest, src: pointer, mt: PNimType) = copyMem(dest, src, mt.size) proc genericDeepCopy(dest, src: pointer, mt: PNimType) {.compilerProc.} = + GC_disable() genericDeepCopyAux(dest, src, mt) + GC_enable() proc genericSeqDeepCopy(dest, src: pointer, mt: PNimType) {.compilerProc.} = # also invoked for 'string' diff --git a/lib/system/dyncalls.nim b/lib/system/dyncalls.nim index 3b3d1f87d..0a994efac 100644 --- a/lib/system/dyncalls.nim +++ b/lib/system/dyncalls.nim @@ -52,11 +52,14 @@ when defined(posix): # # c stuff: - var - RTLD_NOW {.importc: "RTLD_NOW", header: "<dlfcn.h>".}: int + when defined(linux) or defined(macosx): + const RTLD_NOW = cint(2) + else: + var + RTLD_NOW {.importc: "RTLD_NOW", header: "<dlfcn.h>".}: cint proc dlclose(lib: LibHandle) {.importc, header: "<dlfcn.h>".} - proc dlopen(path: cstring, mode: int): LibHandle {. + proc dlopen(path: cstring, mode: cint): LibHandle {. importc, header: "<dlfcn.h>".} proc dlsym(lib: LibHandle, name: cstring): ProcAddr {. importc, header: "<dlfcn.h>".} @@ -109,9 +112,30 @@ elif defined(windows) or defined(dos): proc nimGetProcAddr(lib: LibHandle, name: cstring): ProcAddr = result = getProcAddress(cast[THINSTANCE](lib), name) if result != nil: return - var decorated: array[250, char] + const decorated_length = 250 + var decorated: array[decorated_length, char] + decorated[0] = '_' + var m = 1 + while m < (decorated_length - 5): + if name[m - 1] == '\x00': break + decorated[m] = name[m - 1] + inc(m) + decorated[m] = '@' for i in countup(0, 50): - discard csprintf(decorated, "_%s@%ld", name, i*4) + var k = i * 4 + if k div 100 == 0: + if k div 10 == 0: + m = m + 1 + else: + m = m + 2 + else: + m = m + 3 + decorated[m + 1] = '\x00' + while true: + decorated[m] = chr(ord('0') + (k %% 10)) + dec(m) + k = k div 10 + if k == 0: break result = getProcAddress(cast[THINSTANCE](lib), decorated) if result != nil: return procAddrError(name) diff --git a/lib/system/endb.nim b/lib/system/endb.nim index b2cc5624b..d4d10a52c 100644 --- a/lib/system/endb.nim +++ b/lib/system/endb.nim @@ -329,14 +329,14 @@ proc dbgStackFrame(s: cstring, start: int, currFrame: PFrame) = proc readLine(f: File, line: var StaticStr): bool = while true: - var c = fgetc(f) + var c = c_fgetc(f) if c < 0'i32: if line.len > 0: break else: return false if c == 10'i32: break # LF if c == 13'i32: # CR - c = fgetc(f) # is the next char LF? - if c != 10'i32: ungetc(c, f) # no, put the character back + c = c_fgetc(f) # is the next char LF? + if c != 10'i32: discard c_ungetc(c, f) # no, put the character back break add line, chr(int(c)) result = true @@ -475,7 +475,7 @@ proc dbgWriteStackTrace(f: PFrame) = it = f i = 0 total = 0 - tempFrames: array [0..127, PFrame] + tempFrames: array[0..127, PFrame] # setup long head: while it != nil and i <= high(tempFrames)-firstCalls: tempFrames[i] = it diff --git a/lib/system/excpt.nim b/lib/system/excpt.nim index 948f87410..b89729850 100644 --- a/lib/system/excpt.nim +++ b/lib/system/excpt.nim @@ -90,13 +90,13 @@ when defined(nativeStacktrace) and nativeStackTraceSupported: when not hasThreadSupport: var - tempAddresses: array [0..127, pointer] # should not be alloc'd on stack + tempAddresses: array[0..127, pointer] # should not be alloc'd on stack tempDlInfo: TDl_info proc auxWriteStackTraceWithBacktrace(s: var string) = when hasThreadSupport: var - tempAddresses: array [0..127, pointer] # but better than a threadvar + tempAddresses: array[0..127, pointer] # but better than a threadvar tempDlInfo: TDl_info # This is allowed to be expensive since it only happens during crashes # (but this way you don't need manual stack tracing) @@ -124,12 +124,12 @@ when defined(nativeStacktrace) and nativeStackTraceSupported: when not hasThreadSupport: var - tempFrames: array [0..127, PFrame] # should not be alloc'd on stack + tempFrames: array[0..127, PFrame] # should not be alloc'd on stack proc auxWriteStackTrace(f: PFrame, s: var string) = when hasThreadSupport: var - tempFrames: array [0..127, PFrame] # but better than a threadvar + tempFrames: array[0..127, PFrame] # but better than a threadvar const firstCalls = 32 var @@ -250,12 +250,12 @@ proc raiseExceptionAux(e: ref Exception) = inc L, slen template add(buf, s: expr) = xadd(buf, s, s.len) - var buf: array [0..2000, char] + var buf: array[0..2000, char] var L = 0 add(buf, "Error: unhandled exception: ") if not isNil(e.msg): add(buf, e.msg) add(buf, " [") - xadd(buf, e.name, c_strlen(e.name)) + xadd(buf, e.name, e.name.len) add(buf, "]\n") showErrorMessage(buf) quitOrDebug() diff --git a/lib/system/gc.nim b/lib/system/gc.nim index 4f461b5c3..ba6b2fcf9 100644 --- a/lib/system/gc.nim +++ b/lib/system/gc.nim @@ -1,7 +1,7 @@ # # # Nim's Runtime Library -# (c) Copyright 2015 Andreas Rumpf +# (c) Copyright 2016 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -9,13 +9,8 @@ # Garbage Collector # -# The basic algorithm is *Deferred Reference Counting* with cycle detection. -# This is achieved by combining a Deutsch-Bobrow garbage collector -# together with Christoper's partial mark-sweep garbage collector. -# -# Special care has been taken to avoid recursion as far as possible to avoid -# stack overflows when traversing deep datastructures. It is well-suited -# for soft real time applications (like games). +# Refcounting + Mark&Sweep. Complex algorithms avoided. +# Been there, done that, didn't work. when defined(nimCoroutines): import arch @@ -30,7 +25,7 @@ const # this seems to be a good value withRealTime = defined(useRealtimeGC) useMarkForDebug = defined(gcGenerational) - useBackupGc = false # use a simple M&S GC to collect + useBackupGc = true # use a simple M&S GC to collect # cycles instead of the complex # algorithm @@ -55,8 +50,8 @@ type WalkOp = enum waMarkGlobal, # part of the backup/debug mark&sweep waMarkPrecise, # part of the backup/debug mark&sweep - waZctDecRef, waPush, waCycleDecRef, waMarkGray, waScan, waScanBlack, - waCollectWhite #, waDebug + waZctDecRef, waPush + #, waDebug Finalizer {.compilerproc.} = proc (self: pointer) {.nimcall, benign.} # A ref type can have a finalizer that is called before the object's @@ -87,7 +82,6 @@ type idGenerator: int zct: CellSeq # the zero count table decStack: CellSeq # cells in the stack that are to decref again - cycleRoots: CellSet tempStack: CellSeq # temporary stack for recursion elimination recGcLock: int # prevent recursion via finalizers; no thread lock when withRealTime: @@ -96,6 +90,7 @@ type stat: GcStat when useMarkForDebug or useBackupGc: marked: CellSet + additionalRoots: CellSeq # dummy roots for GC_ref/unref when hasThreadSupport: toDispose: SharedList[pointer] @@ -136,9 +131,6 @@ proc usrToCell(usr: pointer): PCell {.inline.} = # convert pointer to userdata to object (=pointer to refcount) result = cast[PCell](cast[ByteAddress](usr)-%ByteAddress(sizeof(Cell))) -proc canBeCycleRoot(c: PCell): bool {.inline.} = - result = ntfAcyclic notin c.typ.flags - proc extGetCellType(c: pointer): PNimType {.compilerproc.} = # used for code generation concerning debugging result = usrToCell(c).typ @@ -161,10 +153,10 @@ proc writeCell(msg: cstring, c: PCell) = var kind = -1 if c.typ != nil: kind = ord(c.typ.kind) when leakDetector: - c_fprintf(c_stdout, "[GC] %s: %p %d rc=%ld from %s(%ld)\n", + c_fprintf(stdout, "[GC] %s: %p %d rc=%ld from %s(%ld)\n", msg, c, kind, c.refcount shr rcShift, c.filename, c.line) else: - c_fprintf(c_stdout, "[GC] %s: %p %d rc=%ld; color=%ld\n", + c_fprintf(stdout, "[GC] %s: %p %d rc=%ld; color=%ld\n", msg, c, kind, c.refcount shr rcShift, c.color) template gcTrace(cell, state: expr): stmt {.immediate.} = @@ -200,14 +192,16 @@ proc prepareDealloc(cell: PCell) = (cast[Finalizer](cell.typ.finalizer))(cellToUsr(cell)) dec(gch.recGcLock) +template beforeDealloc(gch: var GcHeap; c: PCell; msg: typed) = + when false: + for i in 0..gch.decStack.len-1: + if gch.decStack.d[i] == c: + sysAssert(false, msg) + proc rtlAddCycleRoot(c: PCell) {.rtl, inl.} = # we MUST access gch as a global here, because this crosses DLL boundaries! when hasThreadSupport and hasSharedHeap: acquireSys(HeapLock) - when cycleGC: - if c.color != rcPurple: - c.setColor(rcPurple) - incl(gch.cycleRoots, c) when hasThreadSupport and hasSharedHeap: releaseSys(HeapLock) @@ -224,22 +218,30 @@ proc decRef(c: PCell) {.inline.} = gcAssert(c.refcount >=% rcIncrement, "decRef") if --c.refcount: rtlAddZCT(c) - elif canbeCycleRoot(c): - # unfortunately this is necessary here too, because a cycle might just - # have been broken up and we could recycle it. - rtlAddCycleRoot(c) - #writeCell("decRef", c) proc incRef(c: PCell) {.inline.} = gcAssert(isAllocatedPtr(gch.region, c), "incRef: interiorPtr") c.refcount = c.refcount +% rcIncrement # and not colorMask #writeCell("incRef", c) - if canbeCycleRoot(c): - rtlAddCycleRoot(c) -proc nimGCref(p: pointer) {.compilerProc, inline.} = incRef(usrToCell(p)) -proc nimGCunref(p: pointer) {.compilerProc, inline.} = decRef(usrToCell(p)) +proc nimGCref(p: pointer) {.compilerProc.} = + # we keep it from being collected by pretending it's not even allocated: + add(gch.additionalRoots, usrToCell(p)) + incRef(usrToCell(p)) + +proc nimGCunref(p: pointer) {.compilerProc.} = + let cell = usrToCell(p) + var L = gch.additionalRoots.len-1 + var i = L + let d = gch.additionalRoots.d + while i >= 0: + if d[i] == cell: + d[i] = d[L] + dec gch.additionalRoots.len + break + dec(i) + decRef(usrToCell(p)) proc GC_addCycleRoot*[T](p: ref T) {.inline.} = ## adds 'p' to the cycle candidate set for the cycle collector. It is @@ -306,10 +308,10 @@ proc initGC() = # init the rt init(gch.zct) init(gch.tempStack) - init(gch.cycleRoots) init(gch.decStack) when useMarkForDebug or useBackupGc: init(gch.marked) + init(gch.additionalRoots) when hasThreadSupport: gch.toDispose = initSharedList[pointer]() @@ -451,10 +453,13 @@ proc rawNewObj(typ: PNimType, size: int, gch: var GcHeap): pointer = gcAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "newObj: 2") # now it is buffered in the ZCT res.typ = typ - when leakDetector and not hasThreadSupport: - if framePtr != nil and framePtr.prev != nil: - res.filename = framePtr.prev.filename - res.line = framePtr.prev.line + when leakDetector: + res.filename = nil + res.line = 0 + when not hasThreadSupport: + if framePtr != nil and framePtr.prev != nil: + res.filename = framePtr.prev.filename + res.line = framePtr.prev.line # refcount is zero, color is black, but mark it to be in the ZCT res.refcount = ZctFlag sysAssert(isAllocatedPtr(gch.region, res), "newObj: 3") @@ -502,10 +507,13 @@ proc newObjRC1(typ: PNimType, size: int): pointer {.compilerRtl.} = sysAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "newObj: 2") # now it is buffered in the ZCT res.typ = typ - when leakDetector and not hasThreadSupport: - if framePtr != nil and framePtr.prev != nil: - res.filename = framePtr.prev.filename - res.line = framePtr.prev.line + when leakDetector: + res.filename = nil + res.line = 0 + when not hasThreadSupport: + if framePtr != nil and framePtr.prev != nil: + res.filename = framePtr.prev.filename + res.line = framePtr.prev.line res.refcount = rcIncrement # refcount is 1 sysAssert(isAllocatedPtr(gch.region, res), "newObj: 3") when logGC: writeCell("new cell", res) @@ -563,7 +571,7 @@ proc growObj(old: pointer, newsize: int, gch: var GcHeap): pointer = d[j] = res break dec(j) - if canbeCycleRoot(ol): excl(gch.cycleRoots, ol) + beforeDealloc(gch, ol, "growObj stack trash") rawDealloc(gch.region, ol) else: # we split the old refcount in 2 parts. XXX This is still not entirely @@ -597,54 +605,12 @@ proc freeCyclicCell(gch: var GcHeap, c: PCell) = when logGC: writeCell("cycle collector dealloc cell", c) when reallyDealloc: sysAssert(allocInv(gch.region), "free cyclic cell") + beforeDealloc(gch, c, "freeCyclicCell: stack trash") rawDealloc(gch.region, c) else: gcAssert(c.typ != nil, "freeCyclicCell") zeroMem(c, sizeof(Cell)) -proc markGray(s: PCell) = - if s.color != rcGray: - setColor(s, rcGray) - forAllChildren(s, waMarkGray) - -proc scanBlack(s: PCell) = - s.setColor(rcBlack) - forAllChildren(s, waScanBlack) - -proc scan(s: PCell) = - if s.color == rcGray: - if s.refcount >=% rcIncrement: - scanBlack(s) - else: - s.setColor(rcWhite) - forAllChildren(s, waScan) - -proc collectWhite(s: PCell) = - # This is a hacky way to deal with the following problem (bug #1796) - # Consider this content in cycleRoots: - # x -> a; y -> a where 'a' is an acyclic object so not included in - # cycleRoots itself. Then 'collectWhite' used to free 'a' twice. The - # 'isAllocatedPtr' check prevents this. This also means we do not need - # to query 's notin gch.cycleRoots' at all. - if isAllocatedPtr(gch.region, s) and s.color == rcWhite: - s.setColor(rcBlack) - forAllChildren(s, waCollectWhite) - freeCyclicCell(gch, s) - -proc markRoots(gch: var GcHeap) = - var tabSize = 0 - for s in elements(gch.cycleRoots): - #writeCell("markRoot", s) - inc tabSize - if s.color == rcPurple and s.refcount >=% rcIncrement: - markGray(s) - else: - excl(gch.cycleRoots, s) - # (s.color == rcBlack and rc == 0) as 1 condition: - if s.refcount == 0: - freeCyclicCell(gch, s) - gch.stat.cycleTableSize = max(gch.stat.cycleTableSize, tabSize) - when useBackupGc: proc sweep(gch: var GcHeap) = for x in allObjects(gch.region): @@ -666,16 +632,8 @@ when useMarkForDebug or useBackupGc: proc markGlobals(gch: var GcHeap) = for i in 0 .. < globalMarkersLen: globalMarkers[i]() - - proc stackMarkS(gch: var GcHeap, p: pointer) {.inline.} = - # the addresses are not as cells on the stack, so turn them to cells: - var cell = usrToCell(p) - var c = cast[ByteAddress](cell) - if c >% PageSize: - # fast check: does it look like a cell? - var objStart = cast[PCell](interiorAllocatedPtr(gch.region, cell)) - if objStart != nil: - markS(gch, objStart) + let d = gch.additionalRoots.d + for i in 0 .. < gch.additionalRoots.len: markS(gch, d[i]) when logGC: var @@ -697,7 +655,7 @@ when logGC: else: writeCell("cell {", s) forAllChildren(s, waDebug) - c_fprintf(c_stdout, "}\n") + c_fprintf(stdout, "}\n") proc doOperation(p: pointer, op: WalkOp) = if p == nil: return @@ -708,7 +666,7 @@ proc doOperation(p: pointer, op: WalkOp) = case op of waZctDecRef: #if not isAllocatedPtr(gch.region, c): - # c_fprintf(c_stdout, "[GC] decref bug: %p", c) + # c_fprintf(stdout, "[GC] decref bug: %p", c) gcAssert(isAllocatedPtr(gch.region, c), "decRef: waZctDecRef") gcAssert(c.refcount >=% rcIncrement, "doOperation 2") #c.refcount = c.refcount -% rcIncrement @@ -717,19 +675,6 @@ proc doOperation(p: pointer, op: WalkOp) = #if c.refcount <% rcIncrement: addZCT(gch.zct, c) of waPush: add(gch.tempStack, c) - of waCycleDecRef: - gcAssert(c.refcount >=% rcIncrement, "doOperation 3") - c.refcount = c.refcount -% rcIncrement - of waMarkGray: - gcAssert(c.refcount >=% rcIncrement, "waMarkGray") - c.refcount = c.refcount -% rcIncrement - markGray(c) - of waScan: scan(c) - of waScanBlack: - c.refcount = c.refcount +% rcIncrement - if c.color != rcBlack: - scanBlack(c) - of waCollectWhite: collectWhite(c) of waMarkGlobal: when useMarkForDebug or useBackupGc: when hasThreadSupport: @@ -748,14 +693,6 @@ proc nimGCvisit(d: pointer, op: int) {.compilerRtl.} = proc collectZCT(gch: var GcHeap): bool {.benign.} -when useMarkForDebug or useBackupGc: - proc markStackAndRegistersForSweep(gch: var GcHeap) {.noinline, cdecl, - benign.} - -proc collectRoots(gch: var GcHeap) = - for s in elements(gch.cycleRoots): - collectWhite(s) - proc collectCycles(gch: var GcHeap) = when hasThreadSupport: for c in gch.toDispose: @@ -764,33 +701,12 @@ proc collectCycles(gch: var GcHeap) = while gch.zct.len > 0: discard collectZCT(gch) when useBackupGc: cellsetReset(gch.marked) - markStackAndRegistersForSweep(gch) - markGlobals(gch) - sweep(gch) - else: - markRoots(gch) - # scanRoots: - for s in elements(gch.cycleRoots): scan(s) - collectRoots(gch) - - cellsetReset(gch.cycleRoots) - # alive cycles need to be kept in 'cycleRoots' if they are referenced - # from the stack; otherwise the write barrier will add the cycle root again - # anyway: - when false: var d = gch.decStack.d - var cycleRootsLen = 0 for i in 0..gch.decStack.len-1: - var c = d[i] - gcAssert isAllocatedPtr(gch.region, c), "addBackStackRoots" - gcAssert c.refcount >=% rcIncrement, "addBackStackRoots: dead cell" - if canBeCycleRoot(c): - #if c notin gch.cycleRoots: - inc cycleRootsLen - incl(gch.cycleRoots, c) - gcAssert c.typ != nil, "addBackStackRoots 2" - if cycleRootsLen != 0: - cfprintf(cstdout, "cycle roots: %ld\n", cycleRootsLen) + sysAssert isAllocatedPtr(gch.region, d[i]), "collectCycles" + markS(gch, d[i]) + markGlobals(gch) + sweep(gch) proc gcMark(gch: var GcHeap, p: pointer) {.inline.} = # the addresses are not as cells on the stack, so turn them to cells: @@ -812,31 +728,11 @@ proc gcMark(gch: var GcHeap, p: pointer) {.inline.} = add(gch.decStack, cell) sysAssert(allocInv(gch.region), "gcMark end") -proc markThreadStacks(gch: var GcHeap) = - when hasThreadSupport and hasSharedHeap: - {.error: "not fully implemented".} - var it = threadList - while it != nil: - # mark registers: - for i in 0 .. high(it.registers): gcMark(gch, it.registers[i]) - var sp = cast[ByteAddress](it.stackBottom) - var max = cast[ByteAddress](it.stackTop) - # XXX stack direction? - # XXX unroll this loop: - while sp <=% max: - gcMark(gch, cast[ppointer](sp)[]) - sp = sp +% sizeof(pointer) - it = it.next - include gc_common proc markStackAndRegisters(gch: var GcHeap) {.noinline, cdecl.} = forEachStackSlot(gch, gcMark) -when useMarkForDebug or useBackupGc: - proc markStackAndRegistersForSweep(gch: var GcHeap) = - forEachStackSlot(gch, stackMarkS) - proc collectZCT(gch: var GcHeap): bool = # Note: Freeing may add child objects to the ZCT! So essentially we do # deep freeing, which is bad for incremental operation. In order to @@ -866,8 +762,6 @@ proc collectZCT(gch: var GcHeap): bool = # as this might be too slow. # In any case, it should be removed from the ZCT. But not # freed. **KEEP THIS IN MIND WHEN MAKING THIS INCREMENTAL!** - when cycleGC: - if canbeCycleRoot(c): excl(gch.cycleRoots, c) when logGC: writeCell("zct dealloc cell", c) gcTrace(c, csZctFreed) # We are about to free the object, call the finalizer BEFORE its @@ -877,6 +771,7 @@ proc collectZCT(gch: var GcHeap): bool = forAllChildren(c, waZctDecRef) when reallyDealloc: sysAssert(allocInv(gch.region), "collectZCT: rawDealloc") + beforeDealloc(gch, c, "collectZCT: stack trash") rawDealloc(gch.region, c) else: sysAssert(c.typ != nil, "collectZCT 2") @@ -915,7 +810,6 @@ proc collectCTBody(gch: var GcHeap) = sysAssert(gch.decStack.len == 0, "collectCT") prepareForInteriorPointerChecking(gch.region) markStackAndRegisters(gch) - markThreadStacks(gch) gch.stat.maxStackCells = max(gch.stat.maxStackCells, gch.decStack.len) inc(gch.stat.stackScans) if collectZCT(gch): @@ -935,12 +829,7 @@ proc collectCTBody(gch: var GcHeap) = gch.stat.maxPause = max(gch.stat.maxPause, duration) when defined(reportMissedDeadlines): if gch.maxPause > 0 and duration > gch.maxPause: - c_fprintf(c_stdout, "[GC] missed deadline: %ld\n", duration) - -when useMarkForDebug or useBackupGc: - proc markForDebug(gch: var GcHeap) = - markStackAndRegistersForSweep(gch) - markGlobals(gch) + c_fprintf(stdout, "[GC] missed deadline: %ld\n", duration) when defined(nimCoroutines): proc currentStackSizes(): int = @@ -980,7 +869,19 @@ when withRealTime: collectCTBody(gch) release(gch) - proc GC_step*(us: int, strongAdvice = false) = GC_step(gch, us, strongAdvice) + proc GC_step*(us: int, strongAdvice = false, stackSize = -1) {.noinline.} = + var stackTop {.volatile.}: pointer + let prevStackBottom = gch.stackBottom + if stackSize >= 0: + stackTop = addr(stackTop) + when stackIncreases: + gch.stackBottom = cast[pointer]( + cast[ByteAddress](stackTop) - sizeof(pointer) * 6 - stackSize) + else: + gch.stackBottom = cast[pointer]( + cast[ByteAddress](stackTop) + sizeof(pointer) * 6 + stackSize) + GC_step(gch, us, strongAdvice) + gch.stackBottom = prevStackBottom when not defined(useNimRtl): proc GC_disable() = @@ -1023,7 +924,7 @@ when not defined(useNimRtl): "[GC] max threshold: " & $gch.stat.maxThreshold & "\n" & "[GC] zct capacity: " & $gch.zct.cap & "\n" & "[GC] max cycle table size: " & $gch.stat.cycleTableSize & "\n" & - "[GC] max pause time [ms]: " & $(gch.stat.maxPause div 1000_000) + "[GC] max pause time [ms]: " & $(gch.stat.maxPause div 1000_000) & "\n" when defined(nimCoroutines): result = result & "[GC] number of stacks: " & $gch.stack.len & "\n" for stack in items(gch.stack): diff --git a/lib/system/gc2.nim b/lib/system/gc2.nim index 6c44d509e..089c9c915 100644 --- a/lib/system/gc2.nim +++ b/lib/system/gc2.nim @@ -97,6 +97,8 @@ type additionalRoots: CellSeq # dummy roots for GC_ref/unref spaceIter: ObjectSpaceIter dumpHeapFile: File # File that is used for GC_dumpHeap + when hasThreadSupport: + toDispose: SharedList[pointer] var gch {.rtlThreadVar.}: GcHeap @@ -119,6 +121,8 @@ proc initGC() = init(gch.decStack) init(gch.additionalRoots) init(gch.greyStack) + when hasThreadSupport: + gch.toDispose = initSharedList[pointer]() # Which color to use for new objects is tricky: When we're marking, # they have to be *white* so that everything is marked that is only @@ -185,7 +189,7 @@ proc writeCell(file: File; msg: cstring, c: PCell) = msg, id, kind, c.refcount shr rcShift, col) proc writeCell(msg: cstring, c: PCell) = - c_stdout.writeCell(msg, c) + stdout.writeCell(msg, c) proc myastToStr[T](x: T): string {.magic: "AstToStr", noSideEffect.} @@ -259,7 +263,7 @@ proc nimGCunref(p: pointer) {.compilerProc.} = template markGrey(x: PCell) = if x.color != 1-gch.black and gch.phase == Phase.Marking: if not isAllocatedPtr(gch.region, x): - c_fprintf(c_stdout, "[GC] markGrey proc: %p\n", x) + c_fprintf(stdout, "[GC] markGrey proc: %p\n", x) #GC_dumpHeap() sysAssert(false, "wtf") x.setColor(rcGrey) @@ -675,7 +679,7 @@ proc sweep(gch: var GcHeap): bool = takeStartTime(100) #echo "loop start" let white = 1-gch.black - #cfprintf(cstdout, "black is %d\n", black) + #c_fprintf(stdout, "black is %d\n", black) while true: let x = allObjectsAsProc(gch.region, addr gch.spaceIter) if gch.spaceIter.state < 0: break @@ -715,7 +719,7 @@ proc markIncremental(gch: var GcHeap): bool = while L[] > 0: var c = gch.greyStack.d[0] if not isAllocatedPtr(gch.region, c): - c_fprintf(c_stdout, "[GC] not allocated anymore: %p\n", c) + c_fprintf(stdout, "[GC] not allocated anymore: %p\n", c) #GC_dumpHeap() sysAssert(false, "wtf") @@ -760,7 +764,7 @@ proc doOperation(p: pointer, op: WalkOp) = case op of waZctDecRef: #if not isAllocatedPtr(gch.region, c): - # c_fprintf(c_stdout, "[GC] decref bug: %p", c) + # c_fprintf(stdout, "[GC] decref bug: %p", c) gcAssert(isAllocatedPtr(gch.region, c), "decRef: waZctDecRef") gcAssert(c.refcount >=% rcIncrement, "doOperation 2") #c.refcount = c.refcount -% rcIncrement @@ -779,14 +783,14 @@ proc doOperation(p: pointer, op: WalkOp) = else: #gcAssert(isAllocatedPtr(gch.region, c), "doOperation: waMarkGlobal") if not isAllocatedPtr(gch.region, c): - c_fprintf(c_stdout, "[GC] not allocated anymore: MarkGlobal %p\n", c) + c_fprintf(stdout, "[GC] not allocated anymore: MarkGlobal %p\n", c) #GC_dumpHeap() sysAssert(false, "wtf") handleRoot() discard allocInv(gch.region) of waMarkGrey: if not isAllocatedPtr(gch.region, c): - c_fprintf(c_stdout, "[GC] not allocated anymore: MarkGrey %p\n", c) + c_fprintf(stdout, "[GC] not allocated anymore: MarkGrey %p\n", c) #GC_dumpHeap() sysAssert(false, "wtf") if c.color == 1-gch.black: @@ -800,6 +804,10 @@ proc nimGCvisit(d: pointer, op: int) {.compilerRtl.} = proc collectZCT(gch: var GcHeap): bool {.benign.} proc collectCycles(gch: var GcHeap): bool = + when hasThreadSupport: + for c in gch.toDispose: + nimGCunref(c) + # ensure the ZCT 'color' is not used: while gch.zct.len > 0: discard collectZCT(gch) @@ -808,7 +816,7 @@ proc collectCycles(gch: var GcHeap): bool = gch.phase = Phase.Marking markGlobals(gch) - cfprintf(stdout, "collectCycles: introduced bug E %ld\n", gch.phase) + c_fprintf(stdout, "collectCycles: introduced bug E %ld\n", gch.phase) discard allocInv(gch.region) of Phase.Marking: # since locals do not have a write barrier, we need @@ -922,7 +930,7 @@ proc collectCTBody(gch: var GcHeap) = gch.stat.maxPause = max(gch.stat.maxPause, duration) when defined(reportMissedDeadlines): if gch.maxPause > 0 and duration > gch.maxPause: - c_fprintf(c_stdout, "[GC] missed deadline: %ld\n", duration) + c_fprintf(stdout, "[GC] missed deadline: %ld\n", duration) when defined(nimCoroutines): proc currentStackSizes(): int = @@ -956,7 +964,19 @@ when withRealTime: strongAdvice: collectCTBody(gch) - proc GC_step*(us: int, strongAdvice = false) = GC_step(gch, us, strongAdvice) + proc GC_step*(us: int, strongAdvice = false, stackSize = -1) {.noinline.} = + var stackTop {.volatile.}: pointer + let prevStackBottom = gch.stackBottom + if stackSize >= 0: + stackTop = addr(stackTop) + when stackIncreases: + gch.stackBottom = cast[pointer]( + cast[ByteAddress](stackTop) - sizeof(pointer) * 6 - stackSize) + else: + gch.stackBottom = cast[pointer]( + cast[ByteAddress](stackTop) + sizeof(pointer) * 6 + stackSize) + GC_step(gch, us, strongAdvice) + gch.stackBottom = prevStackBottom when not defined(useNimRtl): proc GC_disable() = diff --git a/lib/system/gc_common.nim b/lib/system/gc_common.nim index 4807bb6f8..7a1b88c84 100644 --- a/lib/system/gc_common.nim +++ b/lib/system/gc_common.nim @@ -63,7 +63,7 @@ when defined(nimCoroutines): stack.next = gch.stack gch.stack.prev = stack gch.stack = stack - # c_fprintf(c_stdout, "[GC] added stack 0x%016X\n", starts) + # c_fprintf(stdout, "[GC] added stack 0x%016X\n", starts) proc GC_removeStack*(starts: pointer) {.cdecl, exportc.} = var stack = gch.stack @@ -143,7 +143,7 @@ else: when not defined(useNimRtl): {.push stack_trace: off.} proc setStackBottom(theStackBottom: pointer) = - #c_fprintf(c_stdout, "stack bottom: %p;\n", theStackBottom) + #c_fprintf(stdout, "stack bottom: %p;\n", theStackBottom) # the first init must be the one that defines the stack bottom: when defined(nimCoroutines): GC_addStack(theStackBottom) @@ -152,7 +152,7 @@ when not defined(useNimRtl): else: var a = cast[ByteAddress](theStackBottom) # and not PageMask - PageSize*2 var b = cast[ByteAddress](gch.stackBottom) - #c_fprintf(c_stdout, "old: %p new: %p;\n",gch.stackBottom,theStackBottom) + #c_fprintf(stdout, "old: %p new: %p;\n",gch.stackBottom,theStackBottom) when stackIncreases: gch.stackBottom = cast[pointer](min(a, b)) else: @@ -239,7 +239,7 @@ else: # We use a jmp_buf buffer that is in the C stack. # Used to traverse the stack and registers assuming # that 'setjmp' will save registers in the C stack. - type PStackSlice = ptr array [0..7, pointer] + type PStackSlice = ptr array[0..7, pointer] var registers {.noinit.}: Registers getRegisters(registers) for i in registers.low .. registers.high: @@ -277,7 +277,7 @@ else: # We use a jmp_buf buffer that is in the C stack. # Used to traverse the stack and registers assuming # that 'setjmp' will save registers in the C stack. - type PStackSlice = ptr array [0..7, pointer] + type PStackSlice = ptr array[0..7, pointer] var registers {.noinit.}: C_JmpBuf if c_setjmp(registers) == 0'i32: # To fill the C stack with registers. var max = cast[ByteAddress](gch.stackBottom) diff --git a/lib/system/gc_ms.nim b/lib/system/gc_ms.nim index c764571b1..ec69f6e5f 100644 --- a/lib/system/gc_ms.nim +++ b/lib/system/gc_ms.nim @@ -28,6 +28,9 @@ template mulThreshold(x): expr {.immediate.} = x * 2 when defined(memProfiler): proc nimProfile(requestedSize: int) + +when hasThreadSupport: + import sharedlist type WalkOp = enum diff --git a/lib/system/gc_stack.nim b/lib/system/gc_stack.nim index 3a5c5594a..5f72b8959 100644 --- a/lib/system/gc_stack.nim +++ b/lib/system/gc_stack.nim @@ -8,7 +8,23 @@ # "Stack GC" for embedded devices or ultra performance requirements. -include osalloc +when defined(nimphpext): + proc roundup(x, v: int): int {.inline.} = + result = (x + (v-1)) and not (v-1) + proc emalloc(size: int): pointer {.importc: "_emalloc".} + proc efree(mem: pointer) {.importc: "_efree".} + + proc osAllocPages(size: int): pointer {.inline.} = + emalloc(size) + + proc osTryAllocPages(size: int): pointer {.inline.} = + emalloc(size) + + proc osDeallocPages(p: pointer, size: int) {.inline.} = + efree(p) + +else: + include osalloc # We manage memory as a thread local stack. Since the allocation pointer # is detached from the control flow pointer, this model is vastly more @@ -36,33 +52,34 @@ type BaseChunk = object next: Chunk size: int - head, last: ptr ObjHeader # first and last object in chunk that + head, tail: ptr ObjHeader # first and last object in chunk that # has a finalizer attached to it type StackPtr = object - chunk: pointer + bump: pointer remaining: int current: Chunk MemRegion* = object remaining: int - chunk: pointer - head, last: Chunk + bump: pointer + head, tail: Chunk nextChunkSize, totalSize: int hole: ptr Hole # we support individual freeing - lock: SysLock + when hasThreadSupport: + lock: SysLock var - region {.threadVar.}: MemRegion + tlRegion {.threadVar.}: MemRegion template withRegion*(r: MemRegion; body: untyped) = - let oldRegion = region - region = r + let oldRegion = tlRegion + tlRegion = r try: body finally: - region = oldRegion + tlRegion = oldRegion template inc(p: pointer, s: int) = p = cast[pointer](cast[int](p) +% s) @@ -71,7 +88,7 @@ template `+!`(p: pointer, s: int): pointer = cast[pointer](cast[int](p) +% s) template `-!`(p: pointer, s: int): pointer = - cast[pointer](cast[int](p) +% s) + cast[pointer](cast[int](p) -% s) proc allocSlowPath(r: var MemRegion; size: int) = # we need to ensure that the underlying linked list @@ -84,7 +101,7 @@ proc allocSlowPath(r: var MemRegion; size: int) = r.nextChunkSize = if r.totalSize < 64 * 1024: PageSize*4 else: r.nextChunkSize*2 - var s = align(size+sizeof(BaseChunk), PageSize) + var s = roundup(size+sizeof(BaseChunk), PageSize) var fresh: Chunk if s > r.nextChunkSize: fresh = cast[Chunk](osAllocPages(s)) @@ -97,22 +114,26 @@ proc allocSlowPath(r: var MemRegion; size: int) = else: s = r.nextChunkSize fresh.size = s - fresh.final = nil - r.totalSize += s - let old = r.last + fresh.head = nil + fresh.tail = nil + fresh.next = nil + inc r.totalSize, s + let old = r.tail if old == nil: r.head = fresh else: - r.last.next = fresh - r.chunk = fresh +! sizeof(BaseChunk) - r.last = fresh + r.tail.next = fresh + r.bump = fresh +! sizeof(BaseChunk) + r.tail = fresh r.remaining = s - sizeof(BaseChunk) proc alloc(r: var MemRegion; size: int): pointer {.inline.} = - if unlikely(r.remaining < size): allocSlowPath(r, size) + if size > r.remaining: + allocSlowPath(r, size) + sysAssert(size <= r.remaining, "size <= r.remaining") dec(r.remaining, size) - result = r.chunk - inc r.chunk, size + result = r.bump + inc r.bump, size proc runFinalizers(c: Chunk) = var it = c.head @@ -120,228 +141,245 @@ proc runFinalizers(c: Chunk) = # indivually freed objects with finalizer stay in the list, but # their typ is nil then: if it.typ != nil and it.typ.finalizer != nil: - (cast[Finalizer](cell.typ.finalizer))(cell+!sizeof(ObjHeader)) - it = it.next + (cast[Finalizer](it.typ.finalizer))(it+!sizeof(ObjHeader)) + it = it.nextFinal -proc dealloc(r: var MemRegion; p: pointer) = - let it = p-!sizeof(ObjHeader) - if it.typ != nil and it.typ.finalizer != nil: - (cast[Finalizer](cell.typ.finalizer))(p) - it.typ = nil +when false: + proc dealloc(r: var MemRegion; p: pointer) = + let it = cast[ptr ObjHeader](p-!sizeof(ObjHeader)) + if it.typ != nil and it.typ.finalizer != nil: + (cast[Finalizer](it.typ.finalizer))(p) + it.typ = nil -proc deallocAll(head: Chunk) = +proc deallocAll(r: var MemRegion; head: Chunk) = var it = head while it != nil: + let nxt = it.next runFinalizers(it) + dec r.totalSize, it.size osDeallocPages(it, it.size) - it = it.next + it = nxt proc deallocAll*(r: var MemRegion) = - deallocAll(r.head) + deallocAll(r, r.head) zeroMem(addr r, sizeof r) proc obstackPtr*(r: MemRegion): StackPtr = - result.chunk = r.chunk + result.bump = r.bump result.remaining = r.remaining - result.current = r.last + result.current = r.tail + +template computeRemaining(r): untyped = + r.tail.size -% (cast[int](r.bump) -% cast[int](r.tail)) -proc setObstackPtr*(r: MemRegion; sp: StackPtr) = +proc setObstackPtr*(r: var MemRegion; sp: StackPtr) = # free everything after 'sp': if sp.current != nil: - deallocAll(sp.current.next) - r.chunk = sp.chunk + deallocAll(r, sp.current.next) + sp.current.next = nil + else: + deallocAll(r, r.head) + r.head = nil + r.bump = sp.bump + r.tail = sp.current r.remaining = sp.remaining - r.last = sp.current + +proc obstackPtr*(): StackPtr = tlRegion.obstackPtr() +proc setObstackPtr*(sp: StackPtr) = tlRegion.setObstackPtr(sp) +proc deallocAll*() = tlRegion.deallocAll() + +proc deallocOsPages(r: var MemRegion) = r.deallocAll() proc joinRegion*(dest: var MemRegion; src: MemRegion) = # merging is not hard. if dest.head.isNil: dest.head = src.head else: - dest.last.next = src.head - dest.last = src.last - dest.chunk = src.chunk + dest.tail.next = src.head + dest.tail = src.tail + dest.bump = src.bump dest.remaining = src.remaining dest.nextChunkSize = max(dest.nextChunkSize, src.nextChunkSize) - dest.totalSize += src.totalSize - if dest.hole.size < src.hole.size: - dest.hole = src.hole + inc dest.totalSize, src.totalSize proc isOnHeap*(r: MemRegion; p: pointer): bool = - # the last chunk is the largest, so check it first. It's also special + # the tail chunk is the largest, so check it first. It's also special # in that contains the current bump pointer: - if r.last >= p and p < r.chunk: + if r.tail >= p and p < r.bump: return true var it = r.head - while it != r.last: + while it != r.tail: if it >= p and p <= it+!it.size: return true it = it.next -proc isInteriorPointer(r: MemRegion; p: pointer): pointer = - discard " we cannot patch stack pointers anyway!" +when false: + # essential feature for later: copy data over from one region to another -type - PointerStackChunk = object - next, prev: ptr PointerStackChunk - len: int - data: array[128, pointer] + proc isInteriorPointer(r: MemRegion; p: pointer): pointer = + discard " we cannot patch stack pointers anyway!" -template head(s: PointerStackChunk): untyped = s.prev -template tail(s: PointerStackChunk): untyped = s.next + type + PointerStackChunk = object + next, prev: ptr PointerStackChunk + len: int + data: array[128, pointer] -include chains + template head(s: PointerStackChunk): untyped = s.prev + template tail(s: PointerStackChunk): untyped = s.next -proc push(r: var MemRegion; s: var PointerStackChunk; x: pointer) = - if s.len < high(s.data): - s.data[s.len] = x - inc s.len - else: - let fresh = cast[ptr PointerStackChunk](alloc(r, sizeof(PointerStackChunk))) - fresh.len = 1 - fresh.data[0] = x - fresh.next = nil - fresh.prev = nil - append(s, fresh) - - -proc genericDeepCopyAux(dr: var MemRegion; stack: var PointerStackChunk; - dest, src: pointer, mt: PNimType) {.benign.} -proc genericDeepCopyAux(dr: var MemRegion; stack: var PointerStackChunk; - dest, src: pointer, n: ptr TNimNode) {.benign.} = - var - d = cast[ByteAddress](dest) - s = cast[ByteAddress](src) - case n.kind - of nkSlot: - genericDeepCopyAux(cast[pointer](d +% n.offset), - cast[pointer](s +% n.offset), n.typ) - of nkList: - for i in 0..n.len-1: - genericDeepCopyAux(dest, src, n.sons[i]) - of nkCase: - var dd = selectBranch(dest, n) - var m = selectBranch(src, n) - # reset if different branches are in use; note different branches also - # imply that's not self-assignment (``x = x``)! - if m != dd and dd != nil: - genericResetAux(dest, dd) - copyMem(cast[pointer](d +% n.offset), cast[pointer](s +% n.offset), - n.typ.size) - if m != nil: - genericDeepCopyAux(dest, src, m) - of nkNone: sysAssert(false, "genericDeepCopyAux") - -proc copyDeepString(dr: var MemRegion; stack: var PointerStackChunk; src: NimString): NimString {.inline.} = - result = rawNewStringNoInit(dr, src.len) - result.len = src.len - c_memcpy(result.data, src.data, src.len + 1) - -proc genericDeepCopyAux(dr: var MemRegion; stack: var PointerStackChunk; - dest, src: pointer, mt: PNimType) = - var - d = cast[ByteAddress](dest) - s = cast[ByteAddress](src) - sysAssert(mt != nil, "genericDeepCopyAux 2") - case mt.kind - of tyString: - var x = cast[PPointer](dest) - var s2 = cast[PPointer](s)[] - if s2 == nil: - x[] = nil - else: - x[] = copyDeepString(cast[NimString](s2)) - of tySequence: - var s2 = cast[PPointer](src)[] - var seq = cast[PGenericSeq](s2) - var x = cast[PPointer](dest) - if s2 == nil: - x[] = nil - return - sysAssert(dest != nil, "genericDeepCopyAux 3") - x[] = newSeq(mt, seq.len) - var dst = cast[ByteAddress](cast[PPointer](dest)[]) - for i in 0..seq.len-1: - genericDeepCopyAux(dr, stack, - cast[pointer](dst +% i*% mt.base.size +% GenericSeqSize), - cast[pointer](cast[ByteAddress](s2) +% i *% mt.base.size +% - GenericSeqSize), - mt.base) - of tyObject: - # we need to copy m_type field for tyObject, as it could be empty for - # sequence reallocations: - var pint = cast[ptr PNimType](dest) - pint[] = cast[ptr PNimType](src)[] - if mt.base != nil: - genericDeepCopyAux(dr, stack, dest, src, mt.base) - genericDeepCopyAux(dr, stack, dest, src, mt.node) - of tyTuple: - genericDeepCopyAux(dr, stack, dest, src, mt.node) - of tyArray, tyArrayConstr: - for i in 0..(mt.size div mt.base.size)-1: - genericDeepCopyAux(dr, stack, - cast[pointer](d +% i*% mt.base.size), - cast[pointer](s +% i*% mt.base.size), mt.base) - of tyRef: - let s2 = cast[PPointer](src)[] - if s2 == nil: - cast[PPointer](dest)[] = nil + include chains + + proc push(r: var MemRegion; s: var PointerStackChunk; x: pointer) = + if s.len < high(s.data): + s.data[s.len] = x + inc s.len else: - # we modify the header of the cell temporarily; instead of the type - # field we store a forwarding pointer. XXX This is bad when the cloning - # fails due to OOM etc. - let x = usrToCell(s2) - let forw = cast[int](x.typ) - if (forw and 1) == 1: - # we stored a forwarding pointer, so let's use that: - let z = cast[pointer](forw and not 1) - unsureAsgnRef(cast[PPointer](dest), z) + let fresh = cast[ptr PointerStackChunk](alloc(r, sizeof(PointerStackChunk))) + fresh.len = 1 + fresh.data[0] = x + fresh.next = nil + fresh.prev = nil + append(s, fresh) + + + proc genericDeepCopyAux(dr: var MemRegion; stack: var PointerStackChunk; + dest, src: pointer, mt: PNimType) {.benign.} + proc genericDeepCopyAux(dr: var MemRegion; stack: var PointerStackChunk; + dest, src: pointer, n: ptr TNimNode) {.benign.} = + var + d = cast[ByteAddress](dest) + s = cast[ByteAddress](src) + case n.kind + of nkSlot: + genericDeepCopyAux(cast[pointer](d +% n.offset), + cast[pointer](s +% n.offset), n.typ) + of nkList: + for i in 0..n.len-1: + genericDeepCopyAux(dest, src, n.sons[i]) + of nkCase: + var dd = selectBranch(dest, n) + var m = selectBranch(src, n) + # reset if different branches are in use; note different branches also + # imply that's not self-assignment (``x = x``)! + if m != dd and dd != nil: + genericResetAux(dest, dd) + copyMem(cast[pointer](d +% n.offset), cast[pointer](s +% n.offset), + n.typ.size) + if m != nil: + genericDeepCopyAux(dest, src, m) + of nkNone: sysAssert(false, "genericDeepCopyAux") + + proc copyDeepString(dr: var MemRegion; stack: var PointerStackChunk; src: NimString): NimString {.inline.} = + result = rawNewStringNoInit(dr, src.len) + result.len = src.len + copyMem(result.data, src.data, src.len + 1) + + proc genericDeepCopyAux(dr: var MemRegion; stack: var PointerStackChunk; + dest, src: pointer, mt: PNimType) = + var + d = cast[ByteAddress](dest) + s = cast[ByteAddress](src) + sysAssert(mt != nil, "genericDeepCopyAux 2") + case mt.kind + of tyString: + var x = cast[PPointer](dest) + var s2 = cast[PPointer](s)[] + if s2 == nil: + x[] = nil else: - let realType = x.typ - let z = newObj(realType, realType.base.size) - - unsureAsgnRef(cast[PPointer](dest), z) - x.typ = cast[PNimType](cast[int](z) or 1) - genericDeepCopyAux(dr, stack, z, s2, realType.base) - x.typ = realType - else: - copyMem(dest, src, mt.size) - -proc joinAliveDataFromRegion*(dest: var MemRegion; src: var MemRegion; - root: pointer): pointer = - # we mark the alive data and copy only alive data over to 'dest'. - # This is O(liveset) but it nicely compacts memory, so it's fine. - # We use the 'typ' field as a forwarding pointer. The forwarding - # pointers have bit 0 set, so we can disambiguate them. - # We allocate a temporary stack in 'src' that we later free: - var s: PointerStackChunk - s.len = 1 - s.data[0] = root - while s.len > 0: - var p: pointer - if s.tail == nil: - p = s.data[s.len-1] - dec s.len + x[] = copyDeepString(cast[NimString](s2)) + of tySequence: + var s2 = cast[PPointer](src)[] + var seq = cast[PGenericSeq](s2) + var x = cast[PPointer](dest) + if s2 == nil: + x[] = nil + return + sysAssert(dest != nil, "genericDeepCopyAux 3") + x[] = newSeq(mt, seq.len) + var dst = cast[ByteAddress](cast[PPointer](dest)[]) + for i in 0..seq.len-1: + genericDeepCopyAux(dr, stack, + cast[pointer](dst +% i*% mt.base.size +% GenericSeqSize), + cast[pointer](cast[ByteAddress](s2) +% i *% mt.base.size +% + GenericSeqSize), + mt.base) + of tyObject: + # we need to copy m_type field for tyObject, as it could be empty for + # sequence reallocations: + var pint = cast[ptr PNimType](dest) + pint[] = cast[ptr PNimType](src)[] + if mt.base != nil: + genericDeepCopyAux(dr, stack, dest, src, mt.base) + genericDeepCopyAux(dr, stack, dest, src, mt.node) + of tyTuple: + genericDeepCopyAux(dr, stack, dest, src, mt.node) + of tyArray, tyArrayConstr: + for i in 0..(mt.size div mt.base.size)-1: + genericDeepCopyAux(dr, stack, + cast[pointer](d +% i*% mt.base.size), + cast[pointer](s +% i*% mt.base.size), mt.base) + of tyRef: + let s2 = cast[PPointer](src)[] + if s2 == nil: + cast[PPointer](dest)[] = nil + else: + # we modify the header of the cell temporarily; instead of the type + # field we store a forwarding pointer. XXX This is bad when the cloning + # fails due to OOM etc. + let x = usrToCell(s2) + let forw = cast[int](x.typ) + if (forw and 1) == 1: + # we stored a forwarding pointer, so let's use that: + let z = cast[pointer](forw and not 1) + unsureAsgnRef(cast[PPointer](dest), z) + else: + let realType = x.typ + let z = newObj(realType, realType.base.size) + + unsureAsgnRef(cast[PPointer](dest), z) + x.typ = cast[PNimType](cast[int](z) or 1) + genericDeepCopyAux(dr, stack, z, s2, realType.base) + x.typ = realType else: - p = s.tail.data[s.tail.len-1] - dec s.tail.len - if s.tail.len == 0: - unlink(s, s.tail) + copyMem(dest, src, mt.size) + + proc joinAliveDataFromRegion*(dest: var MemRegion; src: var MemRegion; + root: pointer): pointer = + # we mark the alive data and copy only alive data over to 'dest'. + # This is O(liveset) but it nicely compacts memory, so it's fine. + # We use the 'typ' field as a forwarding pointer. The forwarding + # pointers have bit 0 set, so we can disambiguate them. + # We allocate a temporary stack in 'src' that we later free: + var s: PointerStackChunk + s.len = 1 + s.data[0] = root + while s.len > 0: + var p: pointer + if s.tail == nil: + p = s.data[s.len-1] + dec s.len + else: + p = s.tail.data[s.tail.len-1] + dec s.tail.len + if s.tail.len == 0: + unlink(s, s.tail) proc rawNewObj(r: var MemRegion, typ: PNimType, size: int): pointer = var res = cast[ptr ObjHeader](alloc(r, size + sizeof(ObjHeader))) res.typ = typ if typ.finalizer != nil: - res.nextFinal = r.chunk.head - r.chunk.head = res + res.nextFinal = r.head.head + r.head.head = res result = res +! sizeof(ObjHeader) proc newObj(typ: PNimType, size: int): pointer {.compilerRtl.} = - result = rawNewObj(typ, size, region) + result = rawNewObj(tlRegion, typ, size) zeroMem(result, size) when defined(memProfiler): nimProfile(size) proc newObjNoInit(typ: PNimType, size: int): pointer {.compilerRtl.} = - result = rawNewObj(typ, size, region) + result = rawNewObj(tlRegion, typ, size) when defined(memProfiler): nimProfile(size) proc newSeq(typ: PNimType, len: int): pointer {.compilerRtl.} = @@ -351,7 +389,7 @@ proc newSeq(typ: PNimType, len: int): pointer {.compilerRtl.} = cast[PGenericSeq](result).reserved = len proc newObjRC1(typ: PNimType, size: int): pointer {.compilerRtl.} = - result = rawNewObj(typ, size, gch) + result = rawNewObj(tlRegion, typ, size) zeroMem(result, size) proc newSeqRC1(typ: PNimType, len: int): pointer {.compilerRtl.} = @@ -360,23 +398,70 @@ proc newSeqRC1(typ: PNimType, len: int): pointer {.compilerRtl.} = cast[PGenericSeq](result).len = len cast[PGenericSeq](result).reserved = len -proc growObj(old: pointer, newsize: int, gch: var GcHeap): pointer = - collectCT(gch) - var ol = usrToCell(old) - sysAssert(ol.typ != nil, "growObj: 1") - gcAssert(ol.typ.kind in {tyString, tySequence}, "growObj: 2") - - var res = cast[PCell](rawAlloc(gch.region, newsize + sizeof(Cell))) - var elemSize = 1 - if ol.typ.kind != tyString: elemSize = ol.typ.base.size - - var oldsize = cast[PGenericSeq](old).len*elemSize + GenericSeqSize - copyMem(res, ol, oldsize + sizeof(Cell)) - zeroMem(cast[pointer](cast[ByteAddress](res)+% oldsize +% sizeof(Cell)), - newsize-oldsize) - sysAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "growObj: 3") - result = cellToUsr(res) +proc growObj(region: var MemRegion; old: pointer, newsize: int): pointer = + let typ = cast[ptr ObjHeader](old -! sizeof(ObjHeader)).typ + result = rawNewObj(region, typ, newsize) + let elemSize = if typ.kind == tyString: 1 else: typ.base.size + let oldsize = cast[PGenericSeq](old).len*elemSize + GenericSeqSize + copyMem(result, old, oldsize) + zeroMem(result +! oldsize, newsize-oldsize) proc growObj(old: pointer, newsize: int): pointer {.rtl.} = - result = growObj(old, newsize, region) - + result = growObj(tlRegion, old, newsize) + +proc unsureAsgnRef(dest: PPointer, src: pointer) {.compilerproc, inline.} = + dest[] = src +proc asgnRef(dest: PPointer, src: pointer) {.compilerproc, inline.} = + dest[] = src +proc asgnRefNoCycle(dest: PPointer, src: pointer) {.compilerproc, inline.} = + dest[] = src + +proc alloc(size: Natural): pointer = + result = c_malloc(size) + if result == nil: raiseOutOfMem() +proc alloc0(size: Natural): pointer = + result = alloc(size) + zeroMem(result, size) +proc realloc(p: pointer, newsize: Natural): pointer = + result = c_realloc(p, newsize) + if result == nil: raiseOutOfMem() +proc dealloc(p: pointer) = c_free(p) + +proc alloc0(r: var MemRegion; size: Natural): pointer = + # ignore the region. That is correct for the channels module + # but incorrect in general. XXX + result = alloc0(size) + +proc dealloc(r: var MemRegion; p: pointer) = dealloc(p) + +proc allocShared(size: Natural): pointer = + result = c_malloc(size) + if result == nil: raiseOutOfMem() +proc allocShared0(size: Natural): pointer = + result = alloc(size) + zeroMem(result, size) +proc reallocShared(p: pointer, newsize: Natural): pointer = + result = c_realloc(p, newsize) + if result == nil: raiseOutOfMem() +proc deallocShared(p: pointer) = c_free(p) + +when hasThreadSupport: + proc getFreeSharedMem(): int = 0 + proc getTotalSharedMem(): int = 0 + proc getOccupiedSharedMem(): int = 0 + +proc GC_disable() = discard +proc GC_enable() = discard +proc GC_fullCollect() = discard +proc GC_setStrategy(strategy: GC_Strategy) = discard +proc GC_enableMarkAndSweep() = discard +proc GC_disableMarkAndSweep() = discard +proc GC_getStatistics(): string = return "" + +proc getOccupiedMem(): int = + result = tlRegion.totalSize - tlRegion.remaining +proc getFreeMem(): int = tlRegion.remaining +proc getTotalMem(): int = + result = tlRegion.totalSize + +proc setStackBottom(theStackBottom: pointer) = discard diff --git a/lib/system/hti.nim b/lib/system/hti.nim index bfb13059e..984f888cb 100644 --- a/lib/system/hti.nim +++ b/lib/system/hti.nim @@ -71,7 +71,7 @@ type typ: ptr TNimType name: cstring len: int - sons: ptr array [0..0x7fff, ptr TNimNode] + sons: ptr array[0..0x7fff, ptr TNimNode] TNimTypeFlag = enum ntfNoRefs = 0, # type contains no tyRef, tySequence, tyString diff --git a/lib/system/inclrtl.nim b/lib/system/inclrtl.nim index 3caeefcbc..f9e6754ef 100644 --- a/lib/system/inclrtl.nim +++ b/lib/system/inclrtl.nim @@ -19,6 +19,11 @@ when not defined(nimNewShared): {.pragma: gcsafe.} +when not defined(nimImmediateDeprecated): + {.pragma: oldimmediate, immediate.} +else: + {.pragma: oldimmediate.} + when defined(createNimRtl): when defined(useNimRtl): {.error: "Cannot create and use nimrtl at the same time!".} diff --git a/lib/system/jssys.nim b/lib/system/jssys.nim index 1b98883b9..9c8a18bfe 100644 --- a/lib/system/jssys.nim +++ b/lib/system/jssys.nim @@ -65,7 +65,7 @@ proc auxWriteStackTrace(f: PCallFrame): string = it = f i = 0 total = 0 - tempFrames: array [0..63, TempFrame] + tempFrames: array[0..63, TempFrame] while it != nil and i <= high(tempFrames): tempFrames[i].procname = it.procname tempFrames[i].line = it.line @@ -97,6 +97,8 @@ proc rawWriteStackTrace(): string = else: result = "No stack traceback available\n" +proc getStackTrace*(): string = rawWriteStackTrace() + proc unhandledException(e: ref Exception) {. compilerproc, asmNoStackFrame.} = when NimStackTrace: @@ -119,7 +121,10 @@ proc raiseException(e: ref Exception, ename: cstring) {. when not defined(noUnhandledHandler): if excHandler == 0: unhandledException(e) - asm "throw `e`;" + when defined(nimphp): + asm """throw new Exception($`e`["message"]);""" + else: + asm "throw `e`;" proc reraiseException() {.compilerproc, asmNoStackFrame.} = if lastJSError == nil: @@ -243,8 +248,12 @@ proc toJSStr(s: string): cstring {.asmNoStackFrame, compilerproc.} = for (var i = 0; i < len; ++i) { if (nonAsciiPart !== null) { var offset = (i - nonAsciiOffset) * 2; + var code = `s`[i].toString(16); + if (code.length == 1) { + code = "0"+code; + } nonAsciiPart[offset] = "%"; - nonAsciiPart[offset + 1] = `s`[i].toString(16); + nonAsciiPart[offset + 1] = code; } else if (`s`[i] < 128) asciiPart[i] = fcc(`s`[i]); @@ -729,16 +738,19 @@ proc genericReset(x: JSRef, ti: PNimType): JSRef {.compilerproc.} = else: discard -proc arrayConstr(len: int, value: JSRef, typ: PNimType): JSRef {. - asmNoStackFrame, compilerproc.} = - # types are fake - when defined(nimphp): +when defined(nimphp): + proc arrayConstr(len: int, value: string, typ: string): JSRef {. + asmNoStackFrame, compilerproc.} = + # types are fake asm """ $result = array(); for ($i = 0; $i < `len`; $i++) $result[] = `value`; return $result; """ - else: +else: + proc arrayConstr(len: int, value: JSRef, typ: PNimType): JSRef {. + asmNoStackFrame, compilerproc.} = + # types are fake asm """ var result = new Array(`len`); for (var i = 0; i < `len`; ++i) result[i] = nimCopy(null, `value`, `typ`); diff --git a/lib/system/mmdisp.nim b/lib/system/mmdisp.nim index 5e576f0a3..186349152 100644 --- a/lib/system/mmdisp.nim +++ b/lib/system/mmdisp.nim @@ -300,7 +300,7 @@ elif defined(gogc): proc setStackBottom(theStackBottom: pointer) = discard proc alloc(size: Natural): pointer = - result = cmalloc(size) + result = c_malloc(size) if result == nil: raiseOutOfMem() proc alloc0(size: Natural): pointer = @@ -308,13 +308,13 @@ elif defined(gogc): zeroMem(result, size) proc realloc(p: pointer, newsize: Natural): pointer = - result = crealloc(p, newsize) + result = c_realloc(p, newsize) if result == nil: raiseOutOfMem() - proc dealloc(p: pointer) = cfree(p) + proc dealloc(p: pointer) = c_free(p) proc allocShared(size: Natural): pointer = - result = cmalloc(size) + result = c_malloc(size) if result == nil: raiseOutOfMem() proc allocShared0(size: Natural): pointer = @@ -322,10 +322,10 @@ elif defined(gogc): zeroMem(result, size) proc reallocShared(p: pointer, newsize: Natural): pointer = - result = crealloc(p, newsize) + result = c_realloc(p, newsize) if result == nil: raiseOutOfMem() - proc deallocShared(p: pointer) = cfree(p) + proc deallocShared(p: pointer) = c_free(p) when hasThreadSupport: proc getFreeSharedMem(): int = discard @@ -354,6 +354,12 @@ elif defined(gogc): cast[PGenericSeq](result).reserved = len cast[PGenericSeq](result).elemSize = typ.base.size + proc nimNewSeqOfCap(typ: PNimType, cap: int): pointer {.compilerproc.} = + result = newObj(typ, cap * typ.base.size + GenericSeqSize) + cast[PGenericSeq](result).len = 0 + cast[PGenericSeq](result).reserved = cap + cast[PGenericSeq](result).elemSize = typ.base.size + proc growObj(old: pointer, newsize: int): pointer = # the Go GC doesn't have a realloc var @@ -389,26 +395,41 @@ elif defined(nogc) and defined(useMalloc): when not defined(useNimRtl): proc alloc(size: Natural): pointer = - result = cmalloc(size) - if result == nil: raiseOutOfMem() + var x = c_malloc(size + sizeof(size)) + if x == nil: raiseOutOfMem() + + cast[ptr int](x)[] = size + result = cast[pointer](cast[int](x) + sizeof(size)) + proc alloc0(size: Natural): pointer = result = alloc(size) zeroMem(result, size) proc realloc(p: pointer, newsize: Natural): pointer = - result = crealloc(p, newsize) - if result == nil: raiseOutOfMem() - proc dealloc(p: pointer) = cfree(p) + var x = cast[pointer](cast[int](p) - sizeof(newsize)) + let oldsize = cast[ptr int](x)[] + + x = c_realloc(x, newsize + sizeof(newsize)) + + if x == nil: raiseOutOfMem() + + cast[ptr int](x)[] = newsize + result = cast[pointer](cast[int](x) + sizeof(newsize)) + + if newsize > oldsize: + zeroMem(cast[pointer](cast[int](result) + oldsize), newsize - oldsize) + + proc dealloc(p: pointer) = c_free(cast[pointer](cast[int](p) - sizeof(int))) proc allocShared(size: Natural): pointer = - result = cmalloc(size) + result = c_malloc(size) if result == nil: raiseOutOfMem() proc allocShared0(size: Natural): pointer = result = alloc(size) zeroMem(result, size) proc reallocShared(p: pointer, newsize: Natural): pointer = - result = crealloc(p, newsize) + result = c_realloc(p, newsize) if result == nil: raiseOutOfMem() - proc deallocShared(p: pointer) = cfree(p) + proc deallocShared(p: pointer) = c_free(p) proc GC_disable() = discard proc GC_enable() = discard @@ -432,6 +453,7 @@ elif defined(nogc) and defined(useMalloc): result = newObj(typ, addInt(mulInt(len, typ.base.size), GenericSeqSize)) cast[PGenericSeq](result).len = len cast[PGenericSeq](result).reserved = len + proc newObjNoInit(typ: PNimType, size: int): pointer = result = alloc(size) @@ -491,6 +513,7 @@ elif defined(nogc): result = newObj(typ, addInt(mulInt(len, typ.base.size), GenericSeqSize)) cast[PGenericSeq](result).len = len cast[PGenericSeq](result).reserved = len + proc growObj(old: pointer, newsize: int): pointer = result = realloc(old, newsize) @@ -511,11 +534,12 @@ elif defined(nogc): include "system/cellsets" else: - include "system/alloc" + when not defined(gcStack): + include "system/alloc" - include "system/cellsets" - when not leakDetector and not useCellIds: - sysAssert(sizeof(Cell) == sizeof(FreeCell), "sizeof FreeCell") + include "system/cellsets" + when not leakDetector and not useCellIds: + sysAssert(sizeof(Cell) == sizeof(FreeCell), "sizeof FreeCell") when compileOption("gc", "v2"): include "system/gc2" elif defined(gcStack): @@ -529,4 +553,10 @@ else: else: include "system/gc" +when not declared(nimNewSeqOfCap): + proc nimNewSeqOfCap(typ: PNimType, cap: int): pointer {.compilerproc.} = + result = newObj(typ, addInt(mulInt(cap, typ.base.size), GenericSeqSize)) + cast[PGenericSeq](result).len = 0 + cast[PGenericSeq](result).reserved = cap + {.pop.} diff --git a/lib/system/nimscript.nim b/lib/system/nimscript.nim index d587d772f..fc6b8c99d 100644 --- a/lib/system/nimscript.nim +++ b/lib/system/nimscript.nim @@ -39,6 +39,9 @@ proc getCurrentDir(): string = builtin proc rawExec(cmd: string): int {.tags: [ExecIOEffect], raises: [OSError].} = builtin +proc warningImpl(arg, orig: string) = discard +proc hintImpl(arg, orig: string) = discard + proc paramStr*(i: int): string = ## Retrieves the ``i``'th command line parameter. builtin @@ -52,6 +55,31 @@ proc switch*(key: string, val="") = ## example ``switch("checks", "on")``. builtin +proc warning*(name: string; val: bool) = + ## Disables or enables a specific warning. + let v = if val: "on" else: "off" + warningImpl(name & "]:" & v, "warning[" & name & "]:" & v) + +proc hint*(name: string; val: bool) = + ## Disables or enables a specific hint. + let v = if val: "on" else: "off" + hintImpl(name & "]:" & v, "hint[" & name & "]:" & v) + +proc patchFile*(package, filename, replacement: string) = + ## Overrides the location of a given file belonging to the + ## passed package. + ## If the ``replacement`` is not an absolute path, the path + ## is interpreted to be local to the Nimscript file that contains + ## the call to ``patchFile``, Nim's ``--path`` is not used at all + ## to resolve the filename! + ## + ## Example: + ## + ## .. code-block:: nim + ## + ## patchFile("stdlib", "asyncdispatch", "patches/replacement") + discard + proc getCommand*(): string = ## Gets the Nim command that the compiler has been invoked with, for example ## "c", "js", "build", "help". @@ -94,6 +122,10 @@ proc existsDir*(dir: string): bool = ## An alias for ``dirExists``. dirExists(dir) +proc selfExe*(): string = + ## Returns the currently running nim or nimble executable. + builtin + proc toExe*(filename: string): string = ## On Windows adds ".exe" to `filename`, else returns `filename` unmodified. (when defined(windows): filename & ".exe" else: filename) @@ -180,6 +212,15 @@ proc exec*(command: string, input: string, cache = "") {. log "exec: " & command: echo staticExec(command, input, cache) +proc selfExec*(command: string) = + ## Executes an external command with the current nim/nimble executable. + ## ``Command`` must not contain the "nim " part. + let c = selfExe() & " " & command + log "exec: " & c: + if rawExec(c) != 0: + raise newException(OSError, "FAILED: " & c) + checkOsError() + proc put*(key, value: string) = ## Sets a configuration 'key' like 'gcc.options.always' to its value. builtin @@ -206,7 +247,7 @@ proc cd*(dir: string) {.raises: [OSError].} = ## ## The change is permanent for the rest of the execution, since this is just ## a shortcut for `os.setCurrentDir() - ## <http://nim-lang.org/os.html#setCurrentDir,string>`_ . Use the `withDir() + ## <http://nim-lang.org/docs/os.html#setCurrentDir,string>`_ . Use the `withDir() ## <#withDir>`_ template if you want to perform a temporary change only. setCurrentDir(dir) checkOsError() diff --git a/lib/system/osalloc.nim b/lib/system/osalloc.nim index 78410d716..b07a362a0 100644 --- a/lib/system/osalloc.nim +++ b/lib/system/osalloc.nim @@ -68,11 +68,11 @@ when defined(emscripten): mmapDescr.realSize = realSize mmapDescr.realPointer = realPointer - c_fprintf(c_stdout, "[Alloc] size %d %d realSize:%d realPos:%d\n", block_size, cast[int](result), realSize, cast[int](realPointer)) + #c_fprintf(stdout, "[Alloc] size %d %d realSize:%d realPos:%d\n", block_size, cast[int](result), realSize, cast[int](realPointer)) proc osTryAllocPages(size: int): pointer = osAllocPages(size) - proc osDeallocPages(p: pointer, size: int) {.inline} = + proc osDeallocPages(p: pointer, size: int) {.inline.} = var mmapDescrPos = cast[ByteAddress](p) -% sizeof(EmscriptenMMapBlock) var mmapDescr = cast[EmscriptenMMapBlock](mmapDescrPos) munmap(mmapDescr.realPointer, mmapDescr.realSize) @@ -87,6 +87,8 @@ elif defined(posix): const MAP_ANONYMOUS = 0x1000 elif defined(solaris): const MAP_ANONYMOUS = 0x100 + elif defined(linux): + const MAP_ANONYMOUS = 0x20 else: var MAP_ANONYMOUS {.importc: "MAP_ANONYMOUS", header: "<sys/mman.h>".}: cint @@ -107,7 +109,7 @@ elif defined(posix): MAP_PRIVATE or MAP_ANONYMOUS, -1, 0) if result == cast[pointer](-1): result = nil - proc osDeallocPages(p: pointer, size: int) {.inline} = + proc osDeallocPages(p: pointer, size: int) {.inline.} = when reallyOsDealloc: discard munmap(p, size) elif defined(windows): @@ -148,8 +150,9 @@ elif defined(windows): #VirtualFree(p, size, MEM_DECOMMIT) elif hostOS == "standalone": + const StandaloneHeapSize {.intdefine.}: int = 1024 * PageSize var - theHeap: array[1024*PageSize, float64] # 'float64' for alignment + theHeap: array[StandaloneHeapSize, float64] # 'float64' for alignment bumpPointer = cast[int](addr theHeap) proc osAllocPages(size: int): pointer {.inline.} = diff --git a/lib/system/profiler.nim b/lib/system/profiler.nim index ae8ff4e19..7146500d9 100644 --- a/lib/system/profiler.nim +++ b/lib/system/profiler.nim @@ -19,10 +19,14 @@ const MaxTraceLen = 20 # tracking the last 20 calls is enough type - StackTrace* = array [0..MaxTraceLen-1, cstring] + StackTrace* = object + lines*: array[0..MaxTraceLen-1, cstring] + files*: array[0..MaxTraceLen-1, cstring] ProfilerHook* = proc (st: StackTrace) {.nimcall.} {.deprecated: [TStackTrace: StackTrace, TProfilerHook: ProfilerHook].} +proc `[]`*(st: StackTrace, i: int): cstring = st.lines[i] + proc captureStackTrace(f: PFrame, st: var StackTrace) = const firstCalls = 5 @@ -30,9 +34,10 @@ proc captureStackTrace(f: PFrame, st: var StackTrace) = it = f i = 0 total = 0 - while it != nil and i <= high(st)-(firstCalls-1): + while it != nil and i <= high(st.lines)-(firstCalls-1): # the (-1) is for the "..." entry - st[i] = it.procname + st.lines[i] = it.procname + st.files[i] = it.filename inc(i) inc(total) it = it.prev @@ -43,10 +48,12 @@ proc captureStackTrace(f: PFrame, st: var StackTrace) = for j in 1..total-i-(firstCalls-1): if b != nil: b = b.prev if total != i: - st[i] = "..." + st.lines[i] = "..." + st.files[i] = "..." inc(i) - while b != nil and i <= high(st): - st[i] = b.procname + while b != nil and i <= high(st.lines): + st.lines[i] = b.procname + st.files[i] = b.filename inc(i) b = b.prev diff --git a/lib/system/repr.nim b/lib/system/repr.nim index 4da4781ef..cf7d6d7a9 100644 --- a/lib/system/repr.nim +++ b/lib/system/repr.nim @@ -16,7 +16,7 @@ proc reprInt(x: int64): string {.compilerproc.} = return $x proc reprFloat(x: float): string {.compilerproc.} = return $x proc reprPointer(x: pointer): string {.compilerproc.} = - var buf: array [0..59, char] + var buf: array[0..59, char] discard c_sprintf(buf, "%p", x) return $buf @@ -24,7 +24,7 @@ proc `$`(x: uint64): string = if x == 0: result = "0" else: - var buf: array [60, char] + var buf: array[60, char] var i = 0 var n = x while n != 0: @@ -73,23 +73,20 @@ proc reprChar(x: char): string {.compilerRtl.} = add result, "\'" proc reprEnum(e: int, typ: PNimType): string {.compilerRtl.} = - # we read an 'int' but this may have been too large, so mask the other bits: - let b = (sizeof(int)-typ.size)*8 # bits - let m = 1 shl (b-1) # mask - var o = e and ((1 shl b)-1) # clear upper bits - o = (o xor m) - m # sign extend - # XXX we need a proper narrowing based on signedness here - #e and ((1 shl (typ.size*8)) - 1) + ## Return string representation for enumeration values + var n = typ.node if ntfEnumHole notin typ.flags: - if o <% typ.node.len: - return $typ.node.sons[o].name + let o = e - n.sons[0].offset + if o >= 0 and o <% typ.node.len: + return $n.sons[o].name else: # ugh we need a slow linear search: - var n = typ.node var s = n.sons for i in 0 .. n.len-1: - if s[i].offset == o: return $s[i].name - result = $o & " (invalid data!)" + if s[i].offset == e: + return $s[i].name + + result = $e & " (invalid data!)" type PByteArray = ptr array[0.. 0xffff, int8] diff --git a/lib/system/sets.nim b/lib/system/sets.nim index 66877de30..53d222468 100644 --- a/lib/system/sets.nim +++ b/lib/system/sets.nim @@ -10,7 +10,7 @@ # set handling type - NimSet = array [0..4*2048-1, uint8] + NimSet = array[0..4*2048-1, uint8] {.deprecated: [TNimSet: NimSet].} proc countBits32(n: int32): int {.compilerproc.} = diff --git a/lib/system/sysio.nim b/lib/system/sysio.nim index d0bba6775..3e9657ce0 100644 --- a/lib/system/sysio.nim +++ b/lib/system/sysio.nim @@ -16,54 +16,56 @@ # of the standard library! -proc fputs(c: cstring, f: File) {.importc: "fputs", header: "<stdio.h>", - tags: [WriteIOEffect].} -proc fgets(c: cstring, n: int, f: File): cstring {. +proc c_fdopen(filehandle: cint, mode: cstring): File {. + importc: "fdopen", header: "<stdio.h>".} +proc c_fputs(c: cstring, f: File): cint {. + importc: "fputs", header: "<stdio.h>", tags: [WriteIOEffect].} +proc c_fgets(c: cstring, n: cint, f: File): cstring {. importc: "fgets", header: "<stdio.h>", tags: [ReadIOEffect].} -proc fgetc(stream: File): cint {.importc: "fgetc", header: "<stdio.h>", - tags: [ReadIOEffect].} -proc ungetc(c: cint, f: File) {.importc: "ungetc", header: "<stdio.h>", - tags: [].} -proc putc(c: char, stream: File) {.importc: "putc", header: "<stdio.h>", - tags: [WriteIOEffect].} -proc fprintf(f: File, frmt: cstring) {.importc: "fprintf", - header: "<stdio.h>", varargs, tags: [WriteIOEffect].} -proc strlen(c: cstring): int {. - importc: "strlen", header: "<string.h>", tags: [].} +proc c_fgetc(stream: File): cint {. + importc: "fgetc", header: "<stdio.h>", tags: [ReadIOEffect].} +proc c_ungetc(c: cint, f: File): cint {. + importc: "ungetc", header: "<stdio.h>", tags: [].} +proc c_putc(c: cint, stream: File): cint {. + importc: "putc", header: "<stdio.h>", tags: [WriteIOEffect].} +proc c_fflush(f: File): cint {. + importc: "fflush", header: "<stdio.h>".} +proc c_fclose(f: File): cint {. + importc: "fclose", header: "<stdio.h>".} # C routine that is used here: -proc fread(buf: pointer, size, n: int, f: File): int {. +proc c_fread(buf: pointer, size, n: csize, f: File): csize {. importc: "fread", header: "<stdio.h>", tags: [ReadIOEffect].} -proc fseek(f: File, offset: clong, whence: int): int {. +proc c_fseek(f: File, offset: clong, whence: cint): cint {. importc: "fseek", header: "<stdio.h>", tags: [].} -proc ftell(f: File): int {.importc: "ftell", header: "<stdio.h>", tags: [].} -proc ferror(f: File): int {.importc: "ferror", header: "<stdio.h>", tags: [].} -proc setvbuf(stream: File, buf: pointer, typ, size: cint): cint {. - importc, header: "<stdio.h>", tags: [].} -proc memchr(s: pointer, c: cint, n: csize): pointer {. - importc: "memchr", header: "<string.h>", tags: [].} -proc memset(s: pointer, c: cint, n: csize) {. - header: "<string.h>", importc: "memset", tags: [].} -proc fwrite(buf: pointer, size, n: int, f: File): int {. - importc: "fwrite", noDecl.} +proc c_ftell(f: File): clong {. + importc: "ftell", header: "<stdio.h>", tags: [].} +proc c_ferror(f: File): cint {. + importc: "ferror", header: "<stdio.h>", tags: [].} +proc c_setvbuf(f: File, buf: pointer, mode: cint, size: csize): cint {. + importc: "setvbuf", header: "<stdio.h>", tags: [].} +proc c_fwrite(buf: pointer, size, n: csize, f: File): cint {. + importc: "fwrite", header: "<stdio.h>".} proc raiseEIO(msg: string) {.noinline, noreturn.} = sysFatal(IOError, msg) {.push stackTrace:off, profiler:off.} proc readBuffer(f: File, buffer: pointer, len: Natural): int = - result = fread(buffer, 1, len, f) + result = c_fread(buffer, 1, len, f) proc readBytes(f: File, a: var openArray[int8|uint8], start, len: Natural): int = result = readBuffer(f, addr(a[start]), len) proc readChars(f: File, a: var openArray[char], start, len: Natural): int = + if (start + len) > len(a): + raiseEIO("buffer overflow: (start+len) > length of openarray buffer") result = readBuffer(f, addr(a[start]), len) -proc write(f: File, c: cstring) = fputs(c, f) +proc write(f: File, c: cstring) = discard c_fputs(c, f) proc writeBuffer(f: File, buffer: pointer, len: Natural): int = - result = fwrite(buffer, 1, len, f) + result = c_fwrite(buffer, 1, len, f) proc writeBytes(f: File, a: openArray[int8|uint8], start, len: Natural): int = var x = cast[ptr array[0..1000_000_000, int8]](a) @@ -95,23 +97,28 @@ else: const BufSize = 4000 +proc close*(f: File) = discard c_fclose(f) +proc readChar*(f: File): char = result = char(c_fgetc(f)) +proc flushFile*(f: File) = discard c_fflush(f) +proc getFileHandle*(f: File): FileHandle = c_fileno(f) + proc readLine(f: File, line: var TaintedString): bool = var pos = 0 # Use the currently reserved space for a first try when defined(nimscript): - var space = 80 + var space: cint = 80 else: - var space = cast[PGenericSeq](line.string).space + var space: cint = cint(cast[PGenericSeq](line.string).space) line.string.setLen(space) while true: # memset to \l so that we can tell how far fgets wrote, even on EOF, where # fgets doesn't append an \l - memset(addr line.string[pos], '\l'.ord, space) - if fgets(addr line.string[pos], space, f) == nil: + c_memset(addr line.string[pos], '\l'.ord, space) + if c_fgets(addr line.string[pos], space, f) == nil: line.string.setLen(0) return false - let m = memchr(addr line.string[pos], '\l'.ord, space) + let m = c_memchr(addr line.string[pos], '\l'.ord, space) if m != nil: # \l found: Could be our own or the one by fgets, in any case, we're done var last = cast[ByteAddress](m) - cast[ByteAddress](addr line.string[0]) @@ -140,23 +147,23 @@ proc readLine(f: File): TaintedString = proc write(f: File, i: int) = when sizeof(int) == 8: - fprintf(f, "%lld", i) + c_fprintf(f, "%lld", i) else: - fprintf(f, "%ld", i) + c_fprintf(f, "%ld", i) proc write(f: File, i: BiggestInt) = when sizeof(BiggestInt) == 8: - fprintf(f, "%lld", i) + c_fprintf(f, "%lld", i) else: - fprintf(f, "%ld", i) + c_fprintf(f, "%ld", i) proc write(f: File, b: bool) = if b: write(f, "true") else: write(f, "false") -proc write(f: File, r: float32) = fprintf(f, "%g", r) -proc write(f: File, r: BiggestFloat) = fprintf(f, "%g", r) +proc write(f: File, r: float32) = c_fprintf(f, "%g", r) +proc write(f: File, r: BiggestFloat) = c_fprintf(f, "%g", r) -proc write(f: File, c: char) = putc(c, f) +proc write(f: File, c: char) = discard c_putc(ord(c), f) proc write(f: File, a: varargs[string, `$`]) = for x in items(a): write(f, x) @@ -176,15 +183,15 @@ proc readAllBuffer(file: File): string = proc rawFileSize(file: File): int = # this does not raise an error opposed to `getFileSize` - var oldPos = ftell(file) - discard fseek(file, 0, 2) # seek the end of the file - result = ftell(file) - discard fseek(file, clong(oldPos), 0) + var oldPos = c_ftell(file) + discard c_fseek(file, 0, 2) # seek the end of the file + result = c_ftell(file) + discard c_fseek(file, clong(oldPos), 0) proc endOfFile(f: File): bool = # do not blame me; blame the ANSI C standard this is so brain-damaged - var c = fgetc(f) - ungetc(c, f) + var c = c_fgetc(f) + discard c_ungetc(c, f) return c < 0'i32 proc readAllFile(file: File, len: int): string = @@ -195,7 +202,7 @@ proc readAllFile(file: File, len: int): string = if endOfFile(file): if bytes < len: result.setLen(bytes) - elif ferror(file) != 0: + elif c_ferror(file) != 0: raiseEIO("error while reading from file") else: # We read all the bytes but did not reach the EOF @@ -234,8 +241,7 @@ when declared(stdout): # interface to the C procs: -when (defined(windows) and not defined(useWinAnsi)) or defined(nimdoc): - include "system/widestrs" +include "system/widestrs" when defined(windows) and not defined(useWinAnsi): when defined(cpp): @@ -265,23 +271,40 @@ else: importc: "freopen", nodecl.} const - FormatOpen: array [FileMode, string] = ["rb", "wb", "w+b", "r+b", "ab"] + FormatOpen: array[FileMode, string] = ["rb", "wb", "w+b", "r+b", "ab"] #"rt", "wt", "w+t", "r+t", "at" # we always use binary here as for Nim the OS line ending # should not be translated. when defined(posix) and not defined(nimscript): - type - Mode {.importc: "mode_t", header: "<sys/types.h>".} = cint + when defined(linux) and defined(amd64): + type + Mode {.importc: "mode_t", header: "<sys/types.h>".} = cint + + # fillers ensure correct size & offsets + Stat {.importc: "struct stat", + header: "<sys/stat.h>", final, pure.} = object ## struct stat + filler_1: array[24, char] + st_mode: Mode ## Mode of file + filler_2: array[144 - 24 - 4, char] + + proc S_ISDIR(m: Mode): bool = + ## Test for a directory. + (m and 0o170000) == 0o40000 + + else: + type + Mode {.importc: "mode_t", header: "<sys/types.h>".} = cint - Stat {.importc: "struct stat", - header: "<sys/stat.h>", final, pure.} = object ## struct stat - st_mode: Mode ## Mode of file + Stat {.importc: "struct stat", + header: "<sys/stat.h>", final, pure.} = object ## struct stat + st_mode: Mode ## Mode of file - proc S_ISDIR(m: Mode): bool {.importc, header: "<sys/stat.h>".} - ## Test for a directory. + proc S_ISDIR(m: Mode): bool {.importc, header: "<sys/stat.h>".} + ## Test for a directory. - proc fstat(a1: cint, a2: var Stat): cint {.importc, header: "<sys/stat.h>".} + proc c_fstat(a1: cint, a2: var Stat): cint {. + importc: "fstat", header: "<sys/stat.h>".} proc open(f: var File, filename: string, mode: FileMode = fmRead, @@ -294,45 +317,38 @@ proc open(f: var File, filename: string, # be opened. var f2 = cast[File](p) var res: Stat - if fstat(getFileHandle(f2), res) >= 0'i32 and S_ISDIR(res.st_mode): + if c_fstat(getFileHandle(f2), res) >= 0'i32 and S_ISDIR(res.st_mode): close(f2) return false result = true f = cast[File](p) if bufSize > 0 and bufSize <= high(cint).int: - discard setvbuf(f, nil, IOFBF, bufSize.cint) + discard c_setvbuf(f, nil, IOFBF, bufSize.cint) elif bufSize == 0: - discard setvbuf(f, nil, IONBF, 0) + discard c_setvbuf(f, nil, IONBF, 0) proc reopen(f: File, filename: string, mode: FileMode = fmRead): bool = var p: pointer = freopen(filename, FormatOpen[mode], f) result = p != nil -proc fdopen(filehandle: FileHandle, mode: cstring): File {. - importc: "fdopen", header: "<stdio.h>".} - proc open(f: var File, filehandle: FileHandle, mode: FileMode): bool = - f = fdopen(filehandle, FormatOpen[mode]) + f = c_fdopen(filehandle, FormatOpen[mode]) result = f != nil proc setFilePos(f: File, pos: int64) = - if fseek(f, clong(pos), 0) != 0: + if c_fseek(f, clong(pos), 0) != 0: raiseEIO("cannot set file position") proc getFilePos(f: File): int64 = - result = ftell(f) + result = c_ftell(f) if result < 0: raiseEIO("cannot retrieve file position") proc getFileSize(f: File): int64 = var oldPos = getFilePos(f) - discard fseek(f, 0, 2) # seek the end of the file + discard c_fseek(f, 0, 2) # seek the end of the file result = getFilePos(f) setFilePos(f, oldPos) -when not declared(close): - proc close(f: File) {. - importc: "fclose", header: "<stdio.h>", tags: [].} - proc readFile(filename: string): TaintedString = var f: File if open(f, filename): @@ -353,4 +369,12 @@ proc writeFile(filename, content: string) = else: sysFatal(IOError, "cannot open: ", filename) +proc setStdIoUnbuffered() = + when declared(stdout): + discard c_setvbuf(stdout, nil, IONBF, 0) + when declared(stderr): + discard c_setvbuf(stderr, nil, IONBF, 0) + when declared(stdin): + discard c_setvbuf(stdin, nil, IONBF, 0) + {.pop.} diff --git a/lib/system/syslocks.nim b/lib/system/syslocks.nim index 6dcdfff0d..c3e23052b 100644 --- a/lib/system/syslocks.nim +++ b/lib/system/syslocks.nim @@ -14,39 +14,41 @@ when defined(Windows): type Handle = int - SysLock {.final, pure.} = object # CRITICAL_SECTION in WinApi + + SysLock {.importc: "CRITICAL_SECTION", + header: "<windows.h>", final, pure.} = object # CRITICAL_SECTION in WinApi DebugInfo: pointer LockCount: int32 RecursionCount: int32 OwningThread: int LockSemaphore: int - Reserved: int32 + SpinCount: int SysCond = Handle {.deprecated: [THandle: Handle, TSysLock: SysLock, TSysCond: SysCond].} - proc initSysLock(L: var SysLock) {.stdcall, noSideEffect, - dynlib: "kernel32", importc: "InitializeCriticalSection".} + proc initSysLock(L: var SysLock) {.importc: "InitializeCriticalSection", + header: "<windows.h>".} ## Initializes the lock `L`. - proc tryAcquireSysAux(L: var SysLock): int32 {.stdcall, noSideEffect, - dynlib: "kernel32", importc: "TryEnterCriticalSection".} + proc tryAcquireSysAux(L: var SysLock): int32 {.importc: "TryEnterCriticalSection", + header: "<windows.h>".} ## Tries to acquire the lock `L`. proc tryAcquireSys(L: var SysLock): bool {.inline.} = result = tryAcquireSysAux(L) != 0'i32 - proc acquireSys(L: var SysLock) {.stdcall, noSideEffect, - dynlib: "kernel32", importc: "EnterCriticalSection".} + proc acquireSys(L: var SysLock) {.importc: "EnterCriticalSection", + header: "<windows.h>".} ## Acquires the lock `L`. - proc releaseSys(L: var SysLock) {.stdcall, noSideEffect, - dynlib: "kernel32", importc: "LeaveCriticalSection".} + proc releaseSys(L: var SysLock) {.importc: "LeaveCriticalSection", + header: "<windows.h>".} ## Releases the lock `L`. - proc deinitSys(L: var SysLock) {.stdcall, noSideEffect, - dynlib: "kernel32", importc: "DeleteCriticalSection".} + proc deinitSys(L: var SysLock) {.importc: "DeleteCriticalSection", + header: "<windows.h>".} proc createEvent(lpEventAttributes: pointer, bManualReset, bInitialState: int32, @@ -86,17 +88,16 @@ else: #include <pthread.h>""".} = object SysLockType = distinct cint - proc SysLockType_Reentrant: SysLockType = - {.emit: "`result` = PTHREAD_MUTEX_RECURSIVE;".} - proc initSysLock(L: var SysLock, attr: ptr SysLockAttr = nil) {. importc: "pthread_mutex_init", header: "<pthread.h>", noSideEffect.} - proc initSysLockAttr(a: var SysLockAttr) {. - importc: "pthread_mutexattr_init", header: "<pthread.h>", noSideEffect.} - - proc setSysLockType(a: var SysLockAttr, t: SysLockType) {. - importc: "pthread_mutexattr_settype", header: "<pthread.h>", noSideEffect.} + when insideRLocksModule: + proc SysLockType_Reentrant: SysLockType = + {.emit: "`result` = PTHREAD_MUTEX_RECURSIVE;".} + proc initSysLockAttr(a: var SysLockAttr) {. + importc: "pthread_mutexattr_init", header: "<pthread.h>", noSideEffect.} + proc setSysLockType(a: var SysLockAttr, t: SysLockType) {. + importc: "pthread_mutexattr_settype", header: "<pthread.h>", noSideEffect.} proc acquireSys(L: var SysLock) {.noSideEffect, importc: "pthread_mutex_lock", header: "<pthread.h>".} @@ -111,14 +112,14 @@ else: proc deinitSys(L: var SysLock) {.noSideEffect, importc: "pthread_mutex_destroy", header: "<pthread.h>".} - proc initSysCond(cond: var SysCond, cond_attr: pointer = nil) {. - importc: "pthread_cond_init", header: "<pthread.h>", noSideEffect.} - proc waitSysCond(cond: var SysCond, lock: var SysLock) {. - importc: "pthread_cond_wait", header: "<pthread.h>", noSideEffect.} - proc signalSysCond(cond: var SysCond) {. - importc: "pthread_cond_signal", header: "<pthread.h>", noSideEffect.} - - proc deinitSysCond(cond: var SysCond) {.noSideEffect, - importc: "pthread_cond_destroy", header: "<pthread.h>".} + when not insideRLocksModule: + proc initSysCond(cond: var SysCond, cond_attr: pointer = nil) {. + importc: "pthread_cond_init", header: "<pthread.h>", noSideEffect.} + proc waitSysCond(cond: var SysCond, lock: var SysLock) {. + importc: "pthread_cond_wait", header: "<pthread.h>", noSideEffect.} + proc signalSysCond(cond: var SysCond) {. + importc: "pthread_cond_signal", header: "<pthread.h>", noSideEffect.} + proc deinitSysCond(cond: var SysCond) {.noSideEffect, + importc: "pthread_cond_destroy", header: "<pthread.h>".} {.pop.} diff --git a/lib/system/sysstr.nim b/lib/system/sysstr.nim index e2137e8f4..eb3d276e0 100644 --- a/lib/system/sysstr.nim +++ b/lib/system/sysstr.nim @@ -30,7 +30,7 @@ proc eqStrings(a, b: NimString): bool {.inline, compilerProc.} = if a == b: return true if a == nil or b == nil: return false return a.len == b.len and - c_memcmp(a.data, b.data, a.len) == 0'i32 + equalMem(addr(a.data), addr(b.data), a.len) when declared(allocAtomic): template allocStr(size: expr): expr = @@ -71,7 +71,7 @@ proc copyStrLast(s: NimString, start, last: int): NimString {.compilerProc.} = if len > 0: result = rawNewStringNoInit(len) result.len = len - c_memcpy(result.data, addr(s.data[start]), len) + copyMem(addr(result.data), addr(s.data[start]), len) result.data[len] = '\0' else: result = rawNewString(len) @@ -82,10 +82,11 @@ proc copyStr(s: NimString, start: int): NimString {.compilerProc.} = proc toNimStr(str: cstring, len: int): NimString {.compilerProc.} = result = rawNewStringNoInit(len) result.len = len - c_memcpy(result.data, str, len + 1) + copyMem(addr(result.data), str, len + 1) proc cstrToNimstr(str: cstring): NimString {.compilerRtl.} = - result = toNimStr(str, c_strlen(str)) + if str == nil: NimString(nil) + else: toNimStr(str, str.len) proc copyString(src: NimString): NimString {.compilerRtl.} = if src != nil: @@ -94,7 +95,7 @@ proc copyString(src: NimString): NimString {.compilerRtl.} = else: result = rawNewStringNoInit(src.len) result.len = src.len - c_memcpy(result.data, src.data, src.len + 1) + copyMem(addr(result.data), addr(src.data), src.len + 1) proc copyStringRC1(src: NimString): NimString {.compilerRtl.} = if src != nil: @@ -107,7 +108,7 @@ proc copyStringRC1(src: NimString): NimString {.compilerRtl.} = else: result = rawNewStringNoInit(src.len) result.len = src.len - c_memcpy(result.data, src.data, src.len + 1) + copyMem(addr(result.data), addr(src.data), src.len + 1) proc hashString(s: string): int {.compilerproc.} = @@ -177,7 +178,7 @@ proc resizeString(dest: NimString, addlen: int): NimString {.compilerRtl.} = # DO NOT UPDATE LEN YET: dest.len = newLen proc appendString(dest, src: NimString) {.compilerproc, inline.} = - c_memcpy(addr(dest.data[dest.len]), src.data, src.len + 1) + copyMem(addr(dest.data[dest.len]), addr(src.data), src.len + 1) inc(dest.len, src.len) proc appendChar(dest: NimString, c: char) {.compilerproc, inline.} = @@ -228,7 +229,8 @@ proc setLengthSeq(seq: PGenericSeq, elemSize, newLen: int): PGenericSeq {. elif newLen < result.len: # we need to decref here, otherwise the GC leaks! when not defined(boehmGC) and not defined(nogc) and - not defined(gcMarkAndSweep) and not defined(gogc): + not defined(gcMarkAndSweep) and not defined(gogc) and + not defined(gcStack): when false: # compileOption("gc", "v2"): for i in newLen..result.len-1: let len0 = gch.tempStack.len @@ -300,46 +302,42 @@ proc nimFloatToStr(f: float): string {.compilerproc.} = else: result = $buf -proc strtod(buf: cstring, endptr: ptr cstring): float64 {.importc, - header: "<stdlib.h>", noSideEffect.} - -var decimalPoint: char - -proc getDecimalPoint(): char = - result = decimalPoint - if result == '\0': - if strtod("0,5", nil) == 0.5: result = ',' - else: result = '.' - # yes this is threadsafe in practice, spare me: - decimalPoint = result +proc c_strtod(buf: cstring, endptr: ptr cstring): float64 {. + importc: "strtod", header: "<stdlib.h>", noSideEffect.} const IdentChars = {'a'..'z', 'A'..'Z', '0'..'9', '_'} + powtens = [ 1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, + 1e10, 1e11, 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19, + 1e20, 1e21, 1e22] proc nimParseBiggestFloat(s: string, number: var BiggestFloat, start = 0): int {.compilerProc.} = - # This routine leverages `strtod()` for the non-trivial task of - # parsing floating point numbers correctly. Because `strtod()` is - # locale-dependent with respect to the radix character, we create - # a copy where the decimal point is replaced with the locale's - # radix character. + # This routine attempt to parse float that can parsed quickly. + # ie whose integer part can fit inside a 53bits integer. + # their real exponent must also be <= 22. If the float doesn't follow + # these restrictions, transform the float into this form: + # INTEGER * 10 ^ exponent and leave the work to standard `strtod()`. + # This avoid the problems of decimal character portability. + # see: http://www.exploringbinary.com/fast-path-decimal-to-floating-point-conversion/ var i = start sign = 1.0 - t: array[500, char] # flaviu says: 325 is the longest reasonable literal - ti = 0 - hasdigits = false - - template addToBuf(c) = - if ti < t.high: - t[ti] = c; inc(ti) + kdigits, fdigits = 0 + exponent: int + integer: uint64 + fraction: uint64 + frac_exponent= 0 + exp_sign = 1 + first_digit = -1 + has_sign = false # Sign? if s[i] == '+' or s[i] == '-': + has_sign = true if s[i] == '-': sign = -1.0 - t[ti] = s[i] - inc(i); inc(ti) + inc(i) # NaN? if s[i] == 'N' or s[i] == 'n': @@ -359,40 +357,111 @@ proc nimParseBiggestFloat(s: string, number: var BiggestFloat, return i+3 - start return 0 + if s[i] in {'0'..'9'}: + first_digit = (s[i].ord - '0'.ord) # Integer part? while s[i] in {'0'..'9'}: - hasdigits = true - addToBuf(s[i]) - inc(i); + inc(kdigits) + integer = integer * 10'u64 + (s[i].ord - '0'.ord).uint64 + inc(i) while s[i] == '_': inc(i) # Fractional part? if s[i] == '.': - addToBuf(getDecimalPoint()) inc(i) + # if no integer part, Skip leading zeros + if kdigits <= 0: + while s[i] == '0': + inc(frac_exponent) + inc(i) + while s[i] == '_': inc(i) + + if first_digit == -1 and s[i] in {'0'..'9'}: + first_digit = (s[i].ord - '0'.ord) + # get fractional part while s[i] in {'0'..'9'}: - hasdigits = true - addToBuf(s[i]) + inc(fdigits) + inc(frac_exponent) + integer = integer * 10'u64 + (s[i].ord - '0'.ord).uint64 inc(i) while s[i] == '_': inc(i) - if not hasdigits: + + # if has no digits: return error + if kdigits + fdigits <= 0 and + (i == start or # no char consumed (empty string). + (i == start + 1 and has_sign)): # or only '+' or '- return 0 - # Exponent? if s[i] in {'e', 'E'}: - addToBuf(s[i]) inc(i) - if s[i] in {'+', '-'}: - addToBuf(s[i]) + if s[i] == '+' or s[i] == '-': + if s[i] == '-': + exp_sign = -1 + inc(i) if s[i] notin {'0'..'9'}: return 0 while s[i] in {'0'..'9'}: - addToBuf(s[i]) + exponent = exponent * 10 + (ord(s[i]) - ord('0')) inc(i) - while s[i] == '_': inc(i) - number = strtod(t, nil) + while s[i] == '_': inc(i) # underscores are allowed and ignored + + var real_exponent = exp_sign*exponent - frac_exponent + let exp_negative = real_exponent < 0 + var abs_exponent = abs(real_exponent) + + # if exponent greater than can be represented: +/- zero or infinity + if abs_exponent > 999: + if exp_negative: + number = 0.0*sign + else: + number = Inf*sign + return i - start + + # if integer is representable in 53 bits: fast path + # max fast path integer is 1<<53 - 1 or 8999999999999999 (16 digits) + if kdigits + fdigits <= 16 and first_digit <= 8: + # max float power of ten with set bits above the 53th bit is 10^22 + if abs_exponent <= 22: + if exp_negative: + number = sign * integer.float / powtens[abs_exponent] + else: + number = sign * integer.float * powtens[abs_exponent] + return i - start + + # if exponent is greater try to fit extra exponent above 22 by multiplying + # integer part is there is space left. + let slop = 15 - kdigits - fdigits + if abs_exponent <= 22 + slop and not exp_negative: + number = sign * integer.float * powtens[slop] * powtens[abs_exponent-slop] + return i - start + + # if failed: slow path with strtod. + var t: array[500, char] # flaviu says: 325 is the longest reasonable literal + var ti = 0 + let maxlen = t.high - "e+000".len # reserve enough space for exponent + result = i - start + i = start + # re-parse without error checking, any error should be handled by the code above. + while s[i] in {'0'..'9','+','-'}: + if ti < maxlen: + t[ti] = s[i]; inc(ti) + inc(i) + while s[i] in {'.', '_'}: # skip underscore and decimal point + inc(i) + + # insert exponent + t[ti] = 'E'; inc(ti) + t[ti] = if exp_negative: '-' else: '+'; inc(ti) + inc(ti, 3) + + # insert adjusted exponent + t[ti-1] = ('0'.ord + abs_exponent mod 10).char; abs_exponent = abs_exponent div 10 + t[ti-2] = ('0'.ord + abs_exponent mod 10).char; abs_exponent = abs_exponent div 10 + t[ti-3] = ('0'.ord + abs_exponent mod 10).char + + number = c_strtod(t, nil) proc nimInt64ToStr(x: int64): string {.compilerRtl.} = result = newString(sizeof(x)*4) diff --git a/lib/system/threads.nim b/lib/system/threads.nim index bdb737e35..583e3ae86 100644 --- a/lib/system/threads.nim +++ b/lib/system/threads.nim @@ -24,7 +24,7 @@ ## import locks ## ## var -## thr: array [0..4, Thread[tuple[a,b: int]]] +## thr: array[0..4, Thread[tuple[a,b: int]]] ## L: Lock ## ## proc threadFunc(interval: tuple[a,b: int]) {.thread.} = @@ -51,7 +51,7 @@ const when defined(windows): type - SysThread = Handle + SysThread* = Handle WinThreadProc = proc (x: pointer): int32 {.stdcall.} {.deprecated: [TSysThread: SysThread, TWinThreadProc: WinThreadProc].} @@ -117,16 +117,21 @@ else: schedh = "#define _GNU_SOURCE\n#include <sched.h>" pthreadh = "#define _GNU_SOURCE\n#include <pthread.h>" + when defined(linux): + type Time = clong + else: + type Time = int + type - SysThread {.importc: "pthread_t", header: "<sys/types.h>", + SysThread* {.importc: "pthread_t", header: "<sys/types.h>", final, pure.} = object Pthread_attr {.importc: "pthread_attr_t", header: "<sys/types.h>", final, pure.} = object Timespec {.importc: "struct timespec", header: "<time.h>", final, pure.} = object - tv_sec: int - tv_nsec: int + tv_sec: Time + tv_nsec: clong {.deprecated: [TSysThread: SysThread, Tpthread_attr: PThreadAttr, Ttimespec: Timespec].} @@ -193,7 +198,7 @@ when emulatedThreadVars: # allocations are needed. Currently less than 7K are used on a 64bit machine. # We use ``float`` for proper alignment: type - ThreadLocalStorage = array [0..1_000, float] + ThreadLocalStorage = array[0..1_000, float] PGcThread = ptr GcThread GcThread {.pure, inheritable.} = object @@ -301,7 +306,7 @@ type ## a pointer as a thread ID. {.deprecated: [TThread: Thread, TThreadId: ThreadId].} -when not defined(boehmgc) and not hasSharedHeap and not defined(gogc): +when not defined(boehmgc) and not hasSharedHeap and not defined(gogc) and not defined(gcstack): proc deallocOsPages() when defined(boehmgc): @@ -331,7 +336,7 @@ else: proc threadProcWrapStackFrame[TArg](thrd: ptr Thread[TArg]) = when defined(boehmgc): boehmGC_call_with_stack_base(threadProcWrapDispatch[TArg], thrd) - elif not defined(nogc) and not defined(gogc): + elif not defined(nogc) and not defined(gogc) and not defined(gcstack): var p {.volatile.}: proc(a: ptr Thread[TArg]) {.nimcall.} = threadProcWrapDispatch[TArg] when not hasSharedHeap: @@ -374,6 +379,10 @@ proc running*[TArg](t: Thread[TArg]): bool {.inline.} = ## returns true if `t` is running. result = t.dataFn != nil +proc handle*[TArg](t: Thread[TArg]): SysThread {.inline.} = + ## returns the thread handle of `t`. + result = t.sys + when hostOS == "windows": proc joinThread*[TArg](t: Thread[TArg]) {.inline.} = ## waits for the thread `t` to finish. diff --git a/lib/system/timers.nim b/lib/system/timers.nim index ac8418824..129a7d092 100644 --- a/lib/system/timers.nim +++ b/lib/system/timers.nim @@ -61,7 +61,7 @@ elif defined(posixRealtime): final, pure.} = object ## struct timespec tv_sec: int ## Seconds. tv_nsec: int ## Nanoseconds. - {.deprecated: [TClockid: Clickid, TTimeSpec: TimeSpec].} + {.deprecated: [TClockid: Clockid, TTimeSpec: TimeSpec].} var CLOCK_REALTIME {.importc: "CLOCK_REALTIME", header: "<time.h>".}: Clockid @@ -78,11 +78,16 @@ elif defined(posixRealtime): else: # fallback Posix implementation: + when defined(linux): + type Time = clong + else: + type Time = int + type Timeval {.importc: "struct timeval", header: "<sys/select.h>", final, pure.} = object ## struct timeval - tv_sec: int ## Seconds. - tv_usec: int ## Microseconds. + tv_sec: Time ## Seconds. + tv_usec: clong ## Microseconds. {.deprecated: [Ttimeval: Timeval].} proc posix_gettimeofday(tp: var Timeval, unused: pointer = nil) {. importc: "gettimeofday", header: "<sys/time.h>".} diff --git a/lib/system/widestrs.nim b/lib/system/widestrs.nim index 5a30a7c0f..578bebe80 100644 --- a/lib/system/widestrs.nim +++ b/lib/system/widestrs.nim @@ -73,11 +73,11 @@ template fastRuneAt(s: cstring, i: int, result: expr, doInc = true) = result = 0xFFFD when doInc: inc(i) -iterator runes(s: cstring): int = +iterator runes(s: cstring, L: int): int = var i = 0 result: int - while s[i] != '\0': + while i < L: fastRuneAt(s, i, result, true) yield result @@ -85,7 +85,7 @@ proc newWideCString*(source: cstring, L: int): WideCString = unsafeNew(result, L * 4 + 2) #result = cast[wideCString](alloc(L * 4 + 2)) var d = 0 - for ch in runes(source): + for ch in runes(source, L): if ch <=% UNI_MAX_BMP: if ch >=% UNI_SUR_HIGH_START and ch <=% UNI_SUR_LOW_END: result[d] = UNI_REPLACEMENT_CHAR @@ -104,12 +104,7 @@ proc newWideCString*(source: cstring, L: int): WideCString = proc newWideCString*(s: cstring): WideCString = if s.isNil: return nil - when not declared(c_strlen): - proc c_strlen(a: cstring): int {. - header: "<string.h>", noSideEffect, importc: "strlen".} - - let L = c_strlen(s) - result = newWideCString(s, L) + result = newWideCString(s, s.len) proc newWideCString*(s: string): WideCString = result = newWideCString(s, s.len) diff --git a/lib/upcoming/asyncdispatch.nim b/lib/upcoming/asyncdispatch.nim new file mode 100644 index 000000000..19c9815d2 --- /dev/null +++ b/lib/upcoming/asyncdispatch.nim @@ -0,0 +1,2154 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2015 Dominik Picheta +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +include "system/inclrtl" + +import os, oids, tables, strutils, macros, times, heapqueue + +import nativesockets, net, queues + +export Port, SocketFlag + +#{.injectStmt: newGcInvariant().} + +## AsyncDispatch +## ************* +## +## This module implements asynchronous IO. This includes a dispatcher, +## a ``Future`` type implementation, and an ``async`` macro which allows +## asynchronous code to be written in a synchronous style with the ``await`` +## keyword. +## +## The dispatcher acts as a kind of event loop. You must call ``poll`` on it +## (or a function which does so for you such as ``waitFor`` or ``runForever``) +## in order to poll for any outstanding events. The underlying implementation +## is based on epoll on Linux, IO Completion Ports on Windows and select on +## other operating systems. +## +## The ``poll`` function will not, on its own, return any events. Instead +## an appropriate ``Future`` object will be completed. A ``Future`` is a +## type which holds a value which is not yet available, but which *may* be +## available in the future. You can check whether a future is finished +## by using the ``finished`` function. When a future is finished it means that +## either the value that it holds is now available or it holds an error instead. +## The latter situation occurs when the operation to complete a future fails +## with an exception. You can distinguish between the two situations with the +## ``failed`` function. +## +## Future objects can also store a callback procedure which will be called +## automatically once the future completes. +## +## Futures therefore can be thought of as an implementation of the proactor +## pattern. In this +## pattern you make a request for an action, and once that action is fulfilled +## a future is completed with the result of that action. Requests can be +## made by calling the appropriate functions. For example: calling the ``recv`` +## function will create a request for some data to be read from a socket. The +## future which the ``recv`` function returns will then complete once the +## requested amount of data is read **or** an exception occurs. +## +## Code to read some data from a socket may look something like this: +## +## .. code-block::nim +## var future = socket.recv(100) +## future.callback = +## proc () = +## echo(future.read) +## +## All asynchronous functions returning a ``Future`` will not block. They +## will not however return immediately. An asynchronous function will have +## code which will be executed before an asynchronous request is made, in most +## cases this code sets up the request. +## +## In the above example, the ``recv`` function will return a brand new +## ``Future`` instance once the request for data to be read from the socket +## is made. This ``Future`` instance will complete once the requested amount +## of data is read, in this case it is 100 bytes. The second line sets a +## callback on this future which will be called once the future completes. +## All the callback does is write the data stored in the future to ``stdout``. +## The ``read`` function is used for this and it checks whether the future +## completes with an error for you (if it did it will simply raise the +## error), if there is no error however it returns the value of the future. +## +## Asynchronous procedures +## ----------------------- +## +## Asynchronous procedures remove the pain of working with callbacks. They do +## this by allowing you to write asynchronous code the same way as you would +## write synchronous code. +## +## An asynchronous procedure is marked using the ``{.async.}`` pragma. +## When marking a procedure with the ``{.async.}`` pragma it must have a +## ``Future[T]`` return type or no return type at all. If you do not specify +## a return type then ``Future[void]`` is assumed. +## +## Inside asynchronous procedures ``await`` can be used to call any +## procedures which return a +## ``Future``; this includes asynchronous procedures. When a procedure is +## "awaited", the asynchronous procedure it is awaited in will +## suspend its execution +## until the awaited procedure's Future completes. At which point the +## asynchronous procedure will resume its execution. During the period +## when an asynchronous procedure is suspended other asynchronous procedures +## will be run by the dispatcher. +## +## The ``await`` call may be used in many contexts. It can be used on the right +## hand side of a variable declaration: ``var data = await socket.recv(100)``, +## in which case the variable will be set to the value of the future +## automatically. It can be used to await a ``Future`` object, and it can +## be used to await a procedure returning a ``Future[void]``: +## ``await socket.send("foobar")``. +## +## Discarding futures +## ------------------ +## +## Futures should **never** be discarded. This is because they may contain +## errors. If you do not care for the result of a Future then you should +## use the ``asyncCheck`` procedure instead of the ``discard`` keyword. +## +## Examples +## -------- +## +## For examples take a look at the documentation for the modules implementing +## asynchronous IO. A good place to start is the +## `asyncnet module <asyncnet.html>`_. +## +## Limitations/Bugs +## ---------------- +## +## * The effect system (``raises: []``) does not work with async procedures. +## * Can't await in a ``except`` body +## * Forward declarations for async procs are broken, +## link includes workaround: https://github.com/nim-lang/Nim/issues/3182. +## * FutureVar[T] needs to be completed manually. + +# TODO: Check if yielded future is nil and throw a more meaningful exception + +# -- Futures + +type + FutureBase* = ref object of RootObj ## Untyped future. + cb: proc () {.closure,gcsafe.} + finished: bool + error*: ref Exception ## Stored exception + errorStackTrace*: string + when not defined(release): + stackTrace: string ## For debugging purposes only. + id: int + fromProc: string + + Future*[T] = ref object of FutureBase ## Typed future. + value: T ## Stored value + + FutureVar*[T] = distinct Future[T] + + FutureError* = object of Exception + cause*: FutureBase + +{.deprecated: [PFutureBase: FutureBase, PFuture: Future].} + +when not defined(release): + var currentID = 0 + +proc callSoon*(cbproc: proc ()) {.gcsafe.} + +proc newFuture*[T](fromProc: string = "unspecified"): Future[T] = + ## Creates a new future. + ## + ## Specifying ``fromProc``, which is a string specifying the name of the proc + ## that this future belongs to, is a good habit as it helps with debugging. + new(result) + result.finished = false + when not defined(release): + result.stackTrace = getStackTrace() + result.id = currentID + result.fromProc = fromProc + currentID.inc() + +proc newFutureVar*[T](fromProc = "unspecified"): FutureVar[T] = + ## Create a new ``FutureVar``. This Future type is ideally suited for + ## situations where you want to avoid unnecessary allocations of Futures. + ## + ## Specifying ``fromProc``, which is a string specifying the name of the proc + ## that this future belongs to, is a good habit as it helps with debugging. + result = FutureVar[T](newFuture[T](fromProc)) + +proc clean*[T](future: FutureVar[T]) = + ## Resets the ``finished`` status of ``future``. + Future[T](future).finished = false + Future[T](future).error = nil + +proc checkFinished[T](future: Future[T]) = + ## Checks whether `future` is finished. If it is then raises a + ## ``FutureError``. + when not defined(release): + if future.finished: + var msg = "" + msg.add("An attempt was made to complete a Future more than once. ") + msg.add("Details:") + msg.add("\n Future ID: " & $future.id) + msg.add("\n Created in proc: " & future.fromProc) + msg.add("\n Stack trace to moment of creation:") + msg.add("\n" & indent(future.stackTrace.strip(), 4)) + when T is string: + msg.add("\n Contents (string): ") + msg.add("\n" & indent(future.value.repr, 4)) + msg.add("\n Stack trace to moment of secondary completion:") + msg.add("\n" & indent(getStackTrace().strip(), 4)) + var err = newException(FutureError, msg) + err.cause = future + raise err + +proc complete*[T](future: Future[T], val: T) = + ## Completes ``future`` with value ``val``. + #assert(not future.finished, "Future already finished, cannot finish twice.") + checkFinished(future) + assert(future.error == nil) + future.value = val + future.finished = true + if future.cb != nil: + future.cb() + +proc complete*(future: Future[void]) = + ## Completes a void ``future``. + #assert(not future.finished, "Future already finished, cannot finish twice.") + checkFinished(future) + assert(future.error == nil) + future.finished = true + if future.cb != nil: + future.cb() + +proc complete*[T](future: FutureVar[T]) = + ## Completes a ``FutureVar``. + template fut: expr = Future[T](future) + checkFinished(fut) + assert(fut.error == nil) + fut.finished = true + if fut.cb != nil: + fut.cb() + +proc fail*[T](future: Future[T], error: ref Exception) = + ## Completes ``future`` with ``error``. + #assert(not future.finished, "Future already finished, cannot finish twice.") + checkFinished(future) + future.finished = true + future.error = error + future.errorStackTrace = + if getStackTrace(error) == "": getStackTrace() else: getStackTrace(error) + if future.cb != nil: + future.cb() + else: + # This is to prevent exceptions from being silently ignored when a future + # is discarded. + # TODO: This may turn out to be a bad idea. + # Turns out this is a bad idea. + #raise error + discard + +proc `callback=`*(future: FutureBase, cb: proc () {.closure,gcsafe.}) = + ## Sets the callback proc to be called when the future completes. + ## + ## If future has already completed then ``cb`` will be called immediately. + ## + ## **Note**: You most likely want the other ``callback`` setter which + ## passes ``future`` as a param to the callback. + future.cb = cb + if future.finished: + callSoon(future.cb) + +proc `callback=`*[T](future: Future[T], + cb: proc (future: Future[T]) {.closure,gcsafe.}) = + ## Sets the callback proc to be called when the future completes. + ## + ## If future has already completed then ``cb`` will be called immediately. + future.callback = proc () = cb(future) + +proc injectStacktrace[T](future: Future[T]) = + # TODO: Come up with something better. + when not defined(release): + var msg = "" + msg.add("\n " & future.fromProc & "'s lead up to read of failed Future:") + + if not future.errorStackTrace.isNil and future.errorStackTrace != "": + msg.add("\n" & indent(future.errorStackTrace.strip(), 4)) + else: + msg.add("\n Empty or nil stack trace.") + future.error.msg.add(msg) + +proc read*[T](future: Future[T]): T = + ## Retrieves the value of ``future``. Future must be finished otherwise + ## this function will fail with a ``ValueError`` exception. + ## + ## If the result of the future is an error then that error will be raised. + if future.finished: + if future.error != nil: + injectStacktrace(future) + raise future.error + when T isnot void: + return future.value + else: + # TODO: Make a custom exception type for this? + raise newException(ValueError, "Future still in progress.") + +proc readError*[T](future: Future[T]): ref Exception = + ## Retrieves the exception stored in ``future``. + ## + ## An ``ValueError`` exception will be thrown if no exception exists + ## in the specified Future. + if future.error != nil: return future.error + else: + raise newException(ValueError, "No error in future.") + +proc mget*[T](future: FutureVar[T]): var T = + ## Returns a mutable value stored in ``future``. + ## + ## Unlike ``read``, this function will not raise an exception if the + ## Future has not been finished. + result = Future[T](future).value + +proc finished*[T](future: Future[T]): bool = + ## Determines whether ``future`` has completed. + ## + ## ``True`` may indicate an error or a value. Use ``failed`` to distinguish. + future.finished + +proc failed*(future: FutureBase): bool = + ## Determines whether ``future`` completed with an error. + return future.error != nil + +proc asyncCheck*[T](future: Future[T]) = + ## Sets a callback on ``future`` which raises an exception if the future + ## finished with an error. + ## + ## This should be used instead of ``discard`` to discard void futures. + future.callback = + proc () = + if future.failed: + injectStacktrace(future) + raise future.error + +proc `and`*[T, Y](fut1: Future[T], fut2: Future[Y]): Future[void] = + ## Returns a future which will complete once both ``fut1`` and ``fut2`` + ## complete. + var retFuture = newFuture[void]("asyncdispatch.`and`") + fut1.callback = + proc () = + if fut2.finished: retFuture.complete() + fut2.callback = + proc () = + if fut1.finished: retFuture.complete() + return retFuture + +proc `or`*[T, Y](fut1: Future[T], fut2: Future[Y]): Future[void] = + ## Returns a future which will complete once either ``fut1`` or ``fut2`` + ## complete. + var retFuture = newFuture[void]("asyncdispatch.`or`") + proc cb() = + if not retFuture.finished: retFuture.complete() + fut1.callback = cb + fut2.callback = cb + return retFuture + +proc all*[T](futs: varargs[Future[T]]): auto = + ## Returns a future which will complete once + ## all futures in ``futs`` complete. + ## + ## If the awaited futures are not ``Future[void]``, the returned future + ## will hold the values of all awaited futures in a sequence. + ## + ## If the awaited futures *are* ``Future[void]``, + ## this proc returns ``Future[void]``. + + when T is void: + var + retFuture = newFuture[void]("asyncdispatch.all") + completedFutures = 0 + + let totalFutures = len(futs) + + for fut in futs: + fut.callback = proc(f: Future[T]) = + inc(completedFutures) + + if completedFutures == totalFutures: + retFuture.complete() + + return retFuture + + else: + var + retFuture = newFuture[seq[T]]("asyncdispatch.all") + retValues = newSeq[T](len(futs)) + completedFutures = 0 + + for i, fut in futs: + proc setCallback(i: int) = + fut.callback = proc(f: Future[T]) = + retValues[i] = f.read() + inc(completedFutures) + + if completedFutures == len(retValues): + retFuture.complete(retValues) + + setCallback(i) + + return retFuture + +type + PDispatcherBase = ref object of RootRef + timers: HeapQueue[tuple[finishAt: float, fut: Future[void]]] + callbacks: Queue[proc ()] + +proc processTimers(p: PDispatcherBase) {.inline.} = + while p.timers.len > 0 and epochTime() >= p.timers[0].finishAt: + p.timers.pop().fut.complete() + +proc processPendingCallbacks(p: PDispatcherBase) = + while p.callbacks.len > 0: + var cb = p.callbacks.dequeue() + cb() + +proc adjustedTimeout(p: PDispatcherBase, timeout: int): int {.inline.} = + # If dispatcher has active timers this proc returns the timeout + # of the nearest timer. Returns `timeout` otherwise. + result = timeout + if p.timers.len > 0: + let timerTimeout = p.timers[0].finishAt + let curTime = epochTime() + if timeout == -1 or (curTime + (timeout / 1000)) > timerTimeout: + result = int((timerTimeout - curTime) * 1000) + if result < 0: result = 0 + +when defined(windows) or defined(nimdoc): + import winlean, sets, hashes + type + CompletionKey = ULONG_PTR + + CompletionData* = object + fd*: AsyncFD # TODO: Rename this. + cb*: proc (fd: AsyncFD, bytesTransferred: Dword, + errcode: OSErrorCode) {.closure,gcsafe.} + cell*: ForeignCell # we need this `cell` to protect our `cb` environment, + # when using RegisterWaitForSingleObject, because + # waiting is done in different thread. + + PDispatcher* = ref object of PDispatcherBase + ioPort: Handle + handles: HashSet[AsyncFD] + + CustomOverlapped = object of OVERLAPPED + data*: CompletionData + + PCustomOverlapped* = ref CustomOverlapped + + AsyncFD* = distinct int + + PostCallbackData = object + ioPort: Handle + handleFd: AsyncFD + waitFd: Handle + ovl: PCustomOverlapped + PostCallbackDataPtr = ptr PostCallbackData + + AsyncEventImpl = object + hEvent: Handle + hWaiter: Handle + pcd: PostCallbackDataPtr + AsyncEvent* = ptr AsyncEventImpl + + Callback = proc (fd: AsyncFD): bool {.closure,gcsafe.} + {.deprecated: [TCompletionKey: CompletionKey, TAsyncFD: AsyncFD, + TCustomOverlapped: CustomOverlapped, TCompletionData: CompletionData].} + + proc hash(x: AsyncFD): Hash {.borrow.} + proc `==`*(x: AsyncFD, y: AsyncFD): bool {.borrow.} + + proc newDispatcher*(): PDispatcher = + ## Creates a new Dispatcher instance. + new result + result.ioPort = createIoCompletionPort(INVALID_HANDLE_VALUE, 0, 0, 1) + result.handles = initSet[AsyncFD]() + result.timers.newHeapQueue() + result.callbacks = initQueue[proc ()](64) + + var gDisp{.threadvar.}: PDispatcher ## Global dispatcher + proc getGlobalDispatcher*(): PDispatcher = + ## Retrieves the global thread-local dispatcher. + if gDisp.isNil: gDisp = newDispatcher() + result = gDisp + + proc register*(fd: AsyncFD) = + ## Registers ``fd`` with the dispatcher. + let p = getGlobalDispatcher() + if createIoCompletionPort(fd.Handle, p.ioPort, + cast[CompletionKey](fd), 1) == 0: + raiseOSError(osLastError()) + p.handles.incl(fd) + + proc verifyPresence(fd: AsyncFD) = + ## Ensures that file descriptor has been registered with the dispatcher. + let p = getGlobalDispatcher() + if fd notin p.handles: + raise newException(ValueError, + "Operation performed on a socket which has not been registered with" & + " the dispatcher yet.") + + proc poll*(timeout = 500) = + ## Waits for completion events and processes them. + let p = getGlobalDispatcher() + if p.handles.len == 0 and p.timers.len == 0 and p.callbacks.len == 0: + raise newException(ValueError, + "No handles or timers registered in dispatcher.") + + let at = p.adjustedTimeout(timeout) + var llTimeout = + if at == -1: winlean.INFINITE + else: at.int32 + + var lpNumberOfBytesTransferred: Dword + var lpCompletionKey: ULONG_PTR + var customOverlapped: PCustomOverlapped + let res = getQueuedCompletionStatus(p.ioPort, + addr lpNumberOfBytesTransferred, addr lpCompletionKey, + cast[ptr POVERLAPPED](addr customOverlapped), llTimeout).bool + + # http://stackoverflow.com/a/12277264/492186 + # TODO: http://www.serverframework.com/handling-multiple-pending-socket-read-and-write-operations.html + if res: + # This is useful for ensuring the reliability of the overlapped struct. + assert customOverlapped.data.fd == lpCompletionKey.AsyncFD + + customOverlapped.data.cb(customOverlapped.data.fd, + lpNumberOfBytesTransferred, OSErrorCode(-1)) + + # If cell.data != nil, then system.protect(rawEnv(cb)) was called, + # so we need to dispose our `cb` environment, because it is not needed + # anymore. + if customOverlapped.data.cell.data != nil: + system.dispose(customOverlapped.data.cell) + + GC_unref(customOverlapped) + else: + let errCode = osLastError() + if customOverlapped != nil: + assert customOverlapped.data.fd == lpCompletionKey.AsyncFD + customOverlapped.data.cb(customOverlapped.data.fd, + lpNumberOfBytesTransferred, errCode) + if customOverlapped.data.cell.data != nil: + system.dispose(customOverlapped.data.cell) + GC_unref(customOverlapped) + else: + if errCode.int32 == WAIT_TIMEOUT: + # Timed out + discard + else: raiseOSError(errCode) + + # Timer processing. + processTimers(p) + # Callback queue processing + processPendingCallbacks(p) + + var acceptEx*: WSAPROC_ACCEPTEX + var connectEx*: WSAPROC_CONNECTEX + var getAcceptExSockAddrs*: WSAPROC_GETACCEPTEXSOCKADDRS + + proc initPointer(s: SocketHandle, fun: var pointer, guid: var GUID): bool = + # Ref: https://github.com/powdahound/twisted/blob/master/twisted/internet/iocpreactor/iocpsupport/winsock_pointers.c + var bytesRet: Dword + fun = nil + result = WSAIoctl(s, SIO_GET_EXTENSION_FUNCTION_POINTER, addr guid, + sizeof(GUID).Dword, addr fun, sizeof(pointer).Dword, + addr bytesRet, nil, nil) == 0 + + proc initAll() = + let dummySock = newNativeSocket() + if dummySock == INVALID_SOCKET: + raiseOSError(osLastError()) + var fun: pointer = nil + if not initPointer(dummySock, fun, WSAID_CONNECTEX): + raiseOSError(osLastError()) + connectEx = cast[WSAPROC_CONNECTEX](fun) + if not initPointer(dummySock, fun, WSAID_ACCEPTEX): + raiseOSError(osLastError()) + acceptEx = cast[WSAPROC_ACCEPTEX](fun) + if not initPointer(dummySock, fun, WSAID_GETACCEPTEXSOCKADDRS): + raiseOSError(osLastError()) + getAcceptExSockAddrs = cast[WSAPROC_GETACCEPTEXSOCKADDRS](fun) + close(dummySock) + + proc connect*(socket: AsyncFD, address: string, port: Port, + domain = nativesockets.AF_INET): Future[void] = + ## Connects ``socket`` to server at ``address:port``. + ## + ## Returns a ``Future`` which will complete when the connection succeeds + ## or an error occurs. + verifyPresence(socket) + var retFuture = newFuture[void]("connect") + # Apparently ``ConnectEx`` expects the socket to be initially bound: + var saddr: Sockaddr_in + saddr.sin_family = int16(toInt(domain)) + saddr.sin_port = 0 + saddr.sin_addr.s_addr = INADDR_ANY + if bindAddr(socket.SocketHandle, cast[ptr SockAddr](addr(saddr)), + sizeof(saddr).SockLen) < 0'i32: + raiseOSError(osLastError()) + + var aiList = getAddrInfo(address, port, domain) + var success = false + var lastError: OSErrorCode + var it = aiList + while it != nil: + # "the OVERLAPPED structure must remain valid until the I/O completes" + # http://blogs.msdn.com/b/oldnewthing/archive/2011/02/02/10123392.aspx + var ol = PCustomOverlapped() + GC_ref(ol) + ol.data = CompletionData(fd: socket, cb: + proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + if not retFuture.finished: + if errcode == OSErrorCode(-1): + retFuture.complete() + else: + retFuture.fail(newException(OSError, osErrorMsg(errcode))) + ) + + var ret = connectEx(socket.SocketHandle, it.ai_addr, + sizeof(Sockaddr_in).cint, nil, 0, nil, + cast[POVERLAPPED](ol)) + if ret: + # Request to connect completed immediately. + success = true + retFuture.complete() + # We don't deallocate ``ol`` here because even though this completed + # immediately poll will still be notified about its completion and it will + # free ``ol``. + break + else: + lastError = osLastError() + if lastError.int32 == ERROR_IO_PENDING: + # In this case ``ol`` will be deallocated in ``poll``. + success = true + break + else: + GC_unref(ol) + success = false + it = it.ai_next + + dealloc(aiList) + if not success: + retFuture.fail(newException(OSError, osErrorMsg(lastError))) + return retFuture + + proc recv*(socket: AsyncFD, size: int, + flags = {SocketFlag.SafeDisconn}): Future[string] = + ## Reads **up to** ``size`` bytes from ``socket``. Returned future will + ## complete once all the data requested is read, a part of the data has been + ## read, or the socket has disconnected in which case the future will + ## complete with a value of ``""``. + ## + ## **Warning**: The ``Peek`` socket flag is not supported on Windows. + + + # Things to note: + # * When WSARecv completes immediately then ``bytesReceived`` is very + # unreliable. + # * Still need to implement message-oriented socket disconnection, + # '\0' in the message currently signifies a socket disconnect. Who + # knows what will happen when someone sends that to our socket. + verifyPresence(socket) + assert SocketFlag.Peek notin flags, "Peek not supported on Windows." + + var retFuture = newFuture[string]("recv") + var dataBuf: TWSABuf + dataBuf.buf = cast[cstring](alloc0(size)) + dataBuf.len = size.ULONG + + var bytesReceived: Dword + var flagsio = flags.toOSFlags().Dword + var ol = PCustomOverlapped() + GC_ref(ol) + ol.data = CompletionData(fd: socket, cb: + proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + if not retFuture.finished: + if errcode == OSErrorCode(-1): + if bytesCount == 0 and dataBuf.buf[0] == '\0': + retFuture.complete("") + else: + var data = newString(bytesCount) + assert bytesCount <= size + copyMem(addr data[0], addr dataBuf.buf[0], bytesCount) + retFuture.complete($data) + else: + if flags.isDisconnectionError(errcode): + retFuture.complete("") + else: + retFuture.fail(newException(OSError, osErrorMsg(errcode))) + if dataBuf.buf != nil: + dealloc dataBuf.buf + dataBuf.buf = nil + ) + + let ret = WSARecv(socket.SocketHandle, addr dataBuf, 1, addr bytesReceived, + addr flagsio, cast[POVERLAPPED](ol), nil) + if ret == -1: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + if dataBuf.buf != nil: + dealloc dataBuf.buf + dataBuf.buf = nil + GC_unref(ol) + if flags.isDisconnectionError(err): + retFuture.complete("") + else: + retFuture.fail(newException(OSError, osErrorMsg(err))) + elif ret == 0: + # Request completed immediately. + if bytesReceived != 0: + var data = newString(bytesReceived) + assert bytesReceived <= size + copyMem(addr data[0], addr dataBuf.buf[0], bytesReceived) + retFuture.complete($data) + else: + if hasOverlappedIoCompleted(cast[POVERLAPPED](ol)): + retFuture.complete("") + return retFuture + + proc recvInto*(socket: AsyncFD, buf: cstring, size: int, + flags = {SocketFlag.SafeDisconn}): Future[int] = + ## Reads **up to** ``size`` bytes from ``socket`` into ``buf``, which must + ## at least be of that size. Returned future will complete once all the + ## data requested is read, a part of the data has been read, or the socket + ## has disconnected in which case the future will complete with a value of + ## ``0``. + ## + ## **Warning**: The ``Peek`` socket flag is not supported on Windows. + + + # Things to note: + # * When WSARecv completes immediately then ``bytesReceived`` is very + # unreliable. + # * Still need to implement message-oriented socket disconnection, + # '\0' in the message currently signifies a socket disconnect. Who + # knows what will happen when someone sends that to our socket. + verifyPresence(socket) + assert SocketFlag.Peek notin flags, "Peek not supported on Windows." + + var retFuture = newFuture[int]("recvInto") + + #buf[] = '\0' + var dataBuf: TWSABuf + dataBuf.buf = buf + dataBuf.len = size.ULONG + + var bytesReceived: Dword + var flagsio = flags.toOSFlags().Dword + var ol = PCustomOverlapped() + GC_ref(ol) + ol.data = CompletionData(fd: socket, cb: + proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + if not retFuture.finished: + if errcode == OSErrorCode(-1): + if bytesCount == 0 and dataBuf.buf[0] == '\0': + retFuture.complete(0) + else: + retFuture.complete(bytesCount) + else: + if flags.isDisconnectionError(errcode): + retFuture.complete(0) + else: + retFuture.fail(newException(OSError, osErrorMsg(errcode))) + if dataBuf.buf != nil: + dataBuf.buf = nil + ) + + let ret = WSARecv(socket.SocketHandle, addr dataBuf, 1, addr bytesReceived, + addr flagsio, cast[POVERLAPPED](ol), nil) + if ret == -1: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + if dataBuf.buf != nil: + dataBuf.buf = nil + GC_unref(ol) + if flags.isDisconnectionError(err): + retFuture.complete(0) + else: + retFuture.fail(newException(OSError, osErrorMsg(err))) + elif ret == 0: + # Request completed immediately. + if bytesReceived != 0: + assert bytesReceived <= size + retFuture.complete(bytesReceived) + else: + if hasOverlappedIoCompleted(cast[POVERLAPPED](ol)): + retFuture.complete(bytesReceived) + return retFuture + + proc send*(socket: AsyncFD, data: string, + flags = {SocketFlag.SafeDisconn}): Future[void] = + ## Sends ``data`` to ``socket``. The returned future will complete once all + ## data has been sent. + verifyPresence(socket) + var retFuture = newFuture[void]("send") + + var dataBuf: TWSABuf + dataBuf.buf = data # since this is not used in a callback, this is fine + dataBuf.len = data.len.ULONG + + var bytesReceived, lowFlags: Dword + var ol = PCustomOverlapped() + GC_ref(ol) + ol.data = CompletionData(fd: socket, cb: + proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + if not retFuture.finished: + if errcode == OSErrorCode(-1): + retFuture.complete() + else: + if flags.isDisconnectionError(errcode): + retFuture.complete() + else: + retFuture.fail(newException(OSError, osErrorMsg(errcode))) + ) + + let ret = WSASend(socket.SocketHandle, addr dataBuf, 1, addr bytesReceived, + lowFlags, cast[POVERLAPPED](ol), nil) + if ret == -1: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + GC_unref(ol) + if flags.isDisconnectionError(err): + retFuture.complete() + else: + retFuture.fail(newException(OSError, osErrorMsg(err))) + else: + retFuture.complete() + # We don't deallocate ``ol`` here because even though this completed + # immediately poll will still be notified about its completion and it will + # free ``ol``. + return retFuture + + proc sendTo*(socket: AsyncFD, data: pointer, size: int, saddr: ptr SockAddr, + saddrLen: Socklen, + flags = {SocketFlag.SafeDisconn}): Future[void] = + ## Sends ``data`` to specified destination ``saddr``, using + ## socket ``socket``. The returned future will complete once all data + ## has been sent. + verifyPresence(socket) + var retFuture = newFuture[void]("sendTo") + var dataBuf: TWSABuf + dataBuf.buf = cast[cstring](data) + dataBuf.len = size.ULONG + var bytesSent = 0.Dword + var lowFlags = 0.Dword + + # we will preserve address in our stack + var staddr: array[128, char] # SOCKADDR_STORAGE size is 128 bytes + var stalen: cint = cint(saddrLen) + zeroMem(addr(staddr[0]), 128) + copyMem(addr(staddr[0]), saddr, saddrLen) + + var ol = PCustomOverlapped() + GC_ref(ol) + ol.data = CompletionData(fd: socket, cb: + proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + if not retFuture.finished: + if errcode == OSErrorCode(-1): + retFuture.complete() + else: + retFuture.fail(newException(OSError, osErrorMsg(errcode))) + ) + + let ret = WSASendTo(socket.SocketHandle, addr dataBuf, 1, addr bytesSent, + lowFlags, cast[ptr SockAddr](addr(staddr[0])), + stalen, cast[POVERLAPPED](ol), nil) + if ret == -1: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + GC_unref(ol) + retFuture.fail(newException(OSError, osErrorMsg(err))) + else: + retFuture.complete() + # We don't deallocate ``ol`` here because even though this completed + # immediately poll will still be notified about its completion and it will + # free ``ol``. + return retFuture + + proc recvFromInto*(socket: AsyncFD, data: pointer, size: int, + saddr: ptr SockAddr, saddrLen: ptr SockLen, + flags = {SocketFlag.SafeDisconn}): Future[int] = + ## Receives a datagram data from ``socket`` into ``buf``, which must + ## be at least of size ``size``, address of datagram's sender will be + ## stored into ``saddr`` and ``saddrLen``. Returned future will complete + ## once one datagram has been received, and will return size of packet + ## received. + verifyPresence(socket) + var retFuture = newFuture[int]("recvFromInto") + + var dataBuf = TWSABuf(buf: cast[cstring](data), len: size.ULONG) + + var bytesReceived = 0.Dword + var lowFlags = 0.Dword + + var ol = PCustomOverlapped() + GC_ref(ol) + ol.data = CompletionData(fd: socket, cb: + proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + if not retFuture.finished: + if errcode == OSErrorCode(-1): + assert bytesCount <= size + retFuture.complete(bytesCount) + else: + # datagram sockets don't have disconnection, + # so we can just raise an exception + retFuture.fail(newException(OSError, osErrorMsg(errcode))) + ) + + let res = WSARecvFrom(socket.SocketHandle, addr dataBuf, 1, + addr bytesReceived, addr lowFlags, + saddr, cast[ptr cint](saddrLen), + cast[POVERLAPPED](ol), nil) + if res == -1: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + GC_unref(ol) + retFuture.fail(newException(OSError, osErrorMsg(err))) + else: + # Request completed immediately. + if bytesReceived != 0: + assert bytesReceived <= size + retFuture.complete(bytesReceived) + else: + if hasOverlappedIoCompleted(cast[POVERLAPPED](ol)): + retFuture.complete(bytesReceived) + return retFuture + + proc acceptAddr*(socket: AsyncFD, flags = {SocketFlag.SafeDisconn}): + Future[tuple[address: string, client: AsyncFD]] = + ## Accepts a new connection. Returns a future containing the client socket + ## corresponding to that connection and the remote address of the client. + ## The future will complete when the connection is successfully accepted. + ## + ## The resulting client socket is automatically registered to the + ## dispatcher. + ## + ## The ``accept`` call may result in an error if the connecting socket + ## disconnects during the duration of the ``accept``. If the ``SafeDisconn`` + ## flag is specified then this error will not be raised and instead + ## accept will be called again. + verifyPresence(socket) + var retFuture = newFuture[tuple[address: string, client: AsyncFD]]("acceptAddr") + + var clientSock = newNativeSocket() + if clientSock == osInvalidSocket: raiseOSError(osLastError()) + + const lpOutputLen = 1024 + var lpOutputBuf = newString(lpOutputLen) + var dwBytesReceived: Dword + let dwReceiveDataLength = 0.Dword # We don't want any data to be read. + let dwLocalAddressLength = Dword(sizeof (Sockaddr_in) + 16) + let dwRemoteAddressLength = Dword(sizeof(Sockaddr_in) + 16) + + template completeAccept(): stmt {.immediate, dirty.} = + var listenSock = socket + let setoptRet = setsockopt(clientSock, SOL_SOCKET, + SO_UPDATE_ACCEPT_CONTEXT, addr listenSock, + sizeof(listenSock).SockLen) + if setoptRet != 0: raiseOSError(osLastError()) + + var localSockaddr, remoteSockaddr: ptr SockAddr + var localLen, remoteLen: int32 + getAcceptExSockaddrs(addr lpOutputBuf[0], dwReceiveDataLength, + dwLocalAddressLength, dwRemoteAddressLength, + addr localSockaddr, addr localLen, + addr remoteSockaddr, addr remoteLen) + register(clientSock.AsyncFD) + # TODO: IPv6. Check ``sa_family``. http://stackoverflow.com/a/9212542/492186 + retFuture.complete( + (address: $inet_ntoa(cast[ptr Sockaddr_in](remoteSockAddr).sin_addr), + client: clientSock.AsyncFD) + ) + + template failAccept(errcode): stmt = + if flags.isDisconnectionError(errcode): + var newAcceptFut = acceptAddr(socket, flags) + newAcceptFut.callback = + proc () = + if newAcceptFut.failed: + retFuture.fail(newAcceptFut.readError) + else: + retFuture.complete(newAcceptFut.read) + else: + retFuture.fail(newException(OSError, osErrorMsg(errcode))) + + var ol = PCustomOverlapped() + GC_ref(ol) + ol.data = CompletionData(fd: socket, cb: + proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + if not retFuture.finished: + if errcode == OSErrorCode(-1): + completeAccept() + else: + failAccept(errcode) + ) + + # http://msdn.microsoft.com/en-us/library/windows/desktop/ms737524%28v=vs.85%29.aspx + let ret = acceptEx(socket.SocketHandle, clientSock, addr lpOutputBuf[0], + dwReceiveDataLength, + dwLocalAddressLength, + dwRemoteAddressLength, + addr dwBytesReceived, cast[POVERLAPPED](ol)) + + if not ret: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + failAccept(err) + GC_unref(ol) + else: + completeAccept() + # We don't deallocate ``ol`` here because even though this completed + # immediately poll will still be notified about its completion and it will + # free ``ol``. + + return retFuture + + proc newAsyncNativeSocket*(domain, sockType, protocol: cint): AsyncFD = + ## Creates a new socket and registers it with the dispatcher implicitly. + result = newNativeSocket(domain, sockType, protocol).AsyncFD + result.SocketHandle.setBlocking(false) + register(result) + + proc newAsyncNativeSocket*(domain: Domain = nativesockets.AF_INET, + sockType: SockType = SOCK_STREAM, + protocol: Protocol = IPPROTO_TCP): AsyncFD = + ## Creates a new socket and registers it with the dispatcher implicitly. + result = newNativeSocket(domain, sockType, protocol).AsyncFD + result.SocketHandle.setBlocking(false) + register(result) + + proc closeSocket*(socket: AsyncFD) = + ## Closes a socket and ensures that it is unregistered. + socket.SocketHandle.close() + getGlobalDispatcher().handles.excl(socket) + + proc unregister*(fd: AsyncFD) = + ## Unregisters ``fd``. + getGlobalDispatcher().handles.excl(fd) + + {.push stackTrace:off.} + proc waitableCallback(param: pointer, + timerOrWaitFired: WINBOOL): void {.stdcall.} = + var p = cast[PostCallbackDataPtr](param) + discard postQueuedCompletionStatus(p.ioPort, timerOrWaitFired.Dword, + ULONG_PTR(p.handleFd), + cast[pointer](p.ovl)) + {.pop.} + + template registerWaitableEvent(mask) = + let p = getGlobalDispatcher() + var flags = (WT_EXECUTEINWAITTHREAD or WT_EXECUTEONLYONCE).Dword + var hEvent = wsaCreateEvent() + if hEvent == 0: + raiseOSError(osLastError()) + var pcd = cast[PostCallbackDataPtr](allocShared0(sizeof(PostCallbackData))) + pcd.ioPort = p.ioPort + pcd.handleFd = fd + var ol = PCustomOverlapped() + GC_ref(ol) + + ol.data = CompletionData(fd: fd, cb: + proc(fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + # we excluding our `fd` because cb(fd) can register own handler + # for this `fd` + p.handles.excl(fd) + # unregisterWait() is called before callback, because appropriate + # winsockets function can re-enable event. + # https://msdn.microsoft.com/en-us/library/windows/desktop/ms741576(v=vs.85).aspx + if unregisterWait(pcd.waitFd) == 0: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + raiseOSError(osLastError()) + if cb(fd): + # callback returned `true`, so we free all allocated resources + deallocShared(cast[pointer](pcd)) + if not wsaCloseEvent(hEvent): + raiseOSError(osLastError()) + # pcd.ovl will be unrefed in poll(). + else: + # callback returned `false` we need to continue + if p.handles.contains(fd): + # new callback was already registered with `fd`, so we free all + # allocated resources. This happens because in callback `cb` + # addRead/addWrite was called with same `fd`. + deallocShared(cast[pointer](pcd)) + if not wsaCloseEvent(hEvent): + raiseOSError(osLastError()) + else: + # we need to include `fd` again + p.handles.incl(fd) + # and register WaitForSingleObject again + if not registerWaitForSingleObject(addr(pcd.waitFd), hEvent, + cast[WAITORTIMERCALLBACK](waitableCallback), + cast[pointer](pcd), INFINITE, flags): + # pcd.ovl will be unrefed in poll() + discard wsaCloseEvent(hEvent) + deallocShared(cast[pointer](pcd)) + raiseOSError(osLastError()) + else: + # we ref pcd.ovl one more time, because it will be unrefed in + # poll() + GC_ref(pcd.ovl) + ) + # We need to protect our callback environment value, so GC will not free it + # accidentally. + ol.data.cell = system.protect(rawEnv(ol.data.cb)) + + # This is main part of `hacky way` is using WSAEventSelect, so `hEvent` + # will be signaled when appropriate `mask` events will be triggered. + if wsaEventSelect(fd.SocketHandle, hEvent, mask) != 0: + GC_unref(ol) + deallocShared(cast[pointer](pcd)) + discard wsaCloseEvent(hEvent) + raiseOSError(osLastError()) + + pcd.ovl = ol + if not registerWaitForSingleObject(addr(pcd.waitFd), hEvent, + cast[WAITORTIMERCALLBACK](waitableCallback), + cast[pointer](pcd), INFINITE, flags): + GC_unref(ol) + deallocShared(cast[pointer](pcd)) + discard wsaCloseEvent(hEvent) + raiseOSError(osLastError()) + p.handles.incl(fd) + + proc addRead*(fd: AsyncFD, cb: Callback) = + ## Start watching the file descriptor for read availability and then call + ## the callback ``cb``. + ## + ## This is not ``pure`` mechanism for Windows Completion Ports (IOCP), + ## so if you can avoid it, please do it. Use `addRead` only if really + ## need it (main usecase is adaptation of `unix like` libraries to be + ## asynchronous on Windows). + ## If you use this function, you dont need to use asyncdispatch.recv() + ## or asyncdispatch.accept(), because they are using IOCP, please use + ## nativesockets.recv() and nativesockets.accept() instead. + ## + ## Be sure your callback ``cb`` returns ``true``, if you want to remove + ## watch of `read` notifications, and ``false``, if you want to continue + ## receiving notifies. + registerWaitableEvent(FD_READ or FD_ACCEPT or FD_OOB or FD_CLOSE) + + proc addWrite*(fd: AsyncFD, cb: Callback) = + ## Start watching the file descriptor for write availability and then call + ## the callback ``cb``. + ## + ## This is not ``pure`` mechanism for Windows Completion Ports (IOCP), + ## so if you can avoid it, please do it. Use `addWrite` only if really + ## need it (main usecase is adaptation of `unix like` libraries to be + ## asynchronous on Windows). + ## If you use this function, you dont need to use asyncdispatch.send() + ## or asyncdispatch.connect(), because they are using IOCP, please use + ## nativesockets.send() and nativesockets.connect() instead. + ## + ## Be sure your callback ``cb`` returns ``true``, if you want to remove + ## watch of `write` notifications, and ``false``, if you want to continue + ## receiving notifies. + registerWaitableEvent(FD_WRITE or FD_CONNECT or FD_CLOSE) + + template registerWaitableHandle(p, hEvent, flags, pcd, handleCallback) = + let handleFD = AsyncFD(hEvent) + pcd.ioPort = p.ioPort + pcd.handleFd = handleFD + var ol = PCustomOverlapped() + GC_ref(ol) + ol.data = CompletionData(fd: handleFD, cb: handleCallback) + # We need to protect our callback environment value, so GC will not free it + # accidentally. + ol.data.cell = system.protect(rawEnv(ol.data.cb)) + + pcd.ovl = ol + if not registerWaitForSingleObject(addr(pcd.waitFd), hEvent, + cast[WAITORTIMERCALLBACK](waitableCallback), + cast[pointer](pcd), INFINITE, flags): + GC_unref(ol) + deallocShared(cast[pointer](pcd)) + discard wsaCloseEvent(hEvent) + raiseOSError(osLastError()) + p.handles.incl(handleFD) + + proc addTimer*(timeout: int, oneshot: bool, cb: Callback) = + ## Registers callback ``cb`` to be called when timer expired. + ## ``timeout`` - timeout value in milliseconds. + ## ``oneshot`` - `true`, to generate only one timeout event, `false`, to + ## generate timeout events periodically. + + doAssert(timeout > 0) + let p = getGlobalDispatcher() + + var hEvent = createEvent(nil, 1, 0, nil) + if hEvent == INVALID_HANDLE_VALUE: + raiseOSError(osLastError()) + + var pcd = cast[PostCallbackDataPtr](allocShared0(sizeof(PostCallbackData))) + var flags = WT_EXECUTEINWAITTHREAD.Dword + if oneshot: flags = flags or WT_EXECUTEONLYONCE + + proc timercb(fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + let res = cb(fd) + if res or oneshot: + if unregisterWait(pcd.waitFd) == 0: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + raiseOSError(osLastError()) + discard closeHandle(hEvent) + deallocShared(cast[pointer](pcd)) + p.handles.excl(fd) + + registerWaitableHandle(p, hEvent, flags, pcd, timercb) + + proc addProcess*(pid: int, cb: Callback) = + ## Registers callback ``cb`` to be called when process with pid ``pid`` + ## exited. + let p = getGlobalDispatcher() + let procFlags = SYNCHRONIZE + var hProcess = openProcess(procFlags, 0, pid.Dword) + if hProcess == INVALID_HANDLE_VALUE: + raiseOSError(osLastError()) + + var pcd = cast[PostCallbackDataPtr](allocShared0(sizeof(PostCallbackData))) + var flags = WT_EXECUTEINWAITTHREAD.Dword + + proc proccb(fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + if unregisterWait(pcd.waitFd) == 0: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + raiseOSError(osLastError()) + discard closeHandle(hProcess) + deallocShared(cast[pointer](pcd)) + p.handles.excl(fd) + discard cb(fd) + + registerWaitableHandle(p, hProcess, flags, pcd, proccb) + + proc newAsyncEvent*(): AsyncEvent = + ## Creates new ``AsyncEvent`` object. + var sa = SECURITY_ATTRIBUTES( + nLength: sizeof(SECURITY_ATTRIBUTES).cint, + bInheritHandle: 1 + ) + var event = createEvent(addr(sa), 0'i32, 0'i32, nil) + if event == INVALID_HANDLE_VALUE: + raiseOSError(osLastError()) + result = cast[AsyncEvent](allocShared0(sizeof(AsyncEventImpl))) + + proc setEvent*(ev: AsyncEvent) = + ## Set event ``ev`` to signaled state. + if setEvent(ev.hEvent) == 0: + raiseOSError(osLastError()) + + proc close*(ev: AsyncEvent) = + ## Closes event ``ev``. + if ev.hWaiter != 0: + let p = getGlobalDispatcher() + if unregisterWait(ev.hWaiter) == 0: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + raiseOSError(osLastError()) + p.handles.excl(AsyncFD(ev.hEvent)) + + if closeHandle(ev.hEvent) == 0: + raiseOSError(osLastError()) + deallocShared(cast[pointer](ev)) + + proc addEvent*(ev: AsyncEvent, cb: Callback) = + ## Registers callback ``cb`` to be called when ``ev`` will be signaled + if ev.hWaiter != 0: + raise newException(ValueError, "Event is already registered!") + + let p = getGlobalDispatcher() + let hEvent = ev.hEvent + + var pcd = cast[PostCallbackDataPtr](allocShared0(sizeof(PostCallbackData))) + var flags = WT_EXECUTEINWAITTHREAD.Dword + + proc eventcb(fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + if cb(fd): + if unregisterWait(pcd.waitFd) == 0: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + raiseOSError(osLastError()) + ev.hWaiter = 0 + deallocShared(cast[pointer](pcd)) + p.handles.excl(fd) + + registerWaitableHandle(p, hEvent, flags, pcd, eventcb) + ev.hWaiter = pcd.waitFd + + initAll() +else: + import ioselectors + when defined(windows): + import winlean + const + EINTR = WSAEINPROGRESS + EINPROGRESS = WSAEINPROGRESS + EWOULDBLOCK = WSAEWOULDBLOCK + EAGAIN = EINPROGRESS + MSG_NOSIGNAL = 0 + else: + from posix import EINTR, EAGAIN, EINPROGRESS, EWOULDBLOCK, MSG_PEEK, + MSG_NOSIGNAL + + const supportedPlatform = defined(linux) or defined(freebsd) or + defined(netbsd) or defined(openbsd) or + defined(macosx) + + type + AsyncFD* = distinct cint + Callback = proc (fd: AsyncFD): bool {.closure,gcsafe.} + + AsyncData = object + readCB: Callback + writeCB: Callback + + AsyncEvent* = SelectEvent + + PDispatcher* = ref object of PDispatcherBase + selector: Selector[AsyncData] + {.deprecated: [TAsyncFD: AsyncFD, TCallback: Callback].} + + proc `==`*(x, y: AsyncFD): bool {.borrow.} + + proc newDispatcher*(): PDispatcher = + new result + result.selector = newSelector[AsyncData]() + result.timers.newHeapQueue() + result.callbacks = initQueue[proc ()](64) + + var gDisp{.threadvar.}: PDispatcher ## Global dispatcher + proc getGlobalDispatcher*(): PDispatcher = + if gDisp.isNil: gDisp = newDispatcher() + result = gDisp + + proc register*(fd: AsyncFD) = + let p = getGlobalDispatcher() + var data = AsyncData() + p.selector.registerHandle(fd.SocketHandle, {}, data) + + proc newAsyncNativeSocket*(domain: cint, sockType: cint, + protocol: cint): AsyncFD = + result = newNativeSocket(domain, sockType, protocol).AsyncFD + result.SocketHandle.setBlocking(false) + when defined(macosx): + result.SocketHandle.setSockOptInt(SOL_SOCKET, SO_NOSIGPIPE, 1) + register(result) + + proc newAsyncNativeSocket*(domain: Domain = AF_INET, + sockType: SockType = SOCK_STREAM, + protocol: Protocol = IPPROTO_TCP): AsyncFD = + result = newNativeSocket(domain, sockType, protocol).AsyncFD + result.SocketHandle.setBlocking(false) + when defined(macosx): + result.SocketHandle.setSockOptInt(SOL_SOCKET, SO_NOSIGPIPE, 1) + register(result) + + proc closeSocket*(sock: AsyncFD) = + let disp = getGlobalDispatcher() + disp.selector.unregister(sock.SocketHandle) + sock.SocketHandle.close() + + proc unregister*(fd: AsyncFD) = + getGlobalDispatcher().selector.unregister(fd.SocketHandle) + + # proc unregister*(ev: AsyncEvent) = + # getGlobalDispatcher().selector.unregister(SelectEvent(ev)) + + proc addRead*(fd: AsyncFD, cb: Callback) = + let p = getGlobalDispatcher() + withData(p.selector, fd.SocketHandle, adata) do: + adata.readCB = cb + do: + raise newException(ValueError, "File descriptor not registered.") + p.selector.updateHandle(fd.SocketHandle, {Event.Read}) + + proc addWrite*(fd: AsyncFD, cb: Callback) = + let p = getGlobalDispatcher() + withData(p.selector, fd.SocketHandle, adata) do: + adata.writeCB = cb + do: + raise newException(ValueError, "File descriptor not registered.") + p.selector.updateHandle(fd.SocketHandle, {Event.Write}) + + proc poll*(timeout = 500) = + var keys: array[64, ReadyKey[AsyncData]] + + let p = getGlobalDispatcher() + when supportedPlatform: + let customSet = {Event.Timer, Event.Signal, Event.Process, + Event.Vnode, Event.User} + + if p.selector.isEmpty() and p.timers.len == 0 and p.callbacks.len == 0: + raise newException(ValueError, + "No handles or timers registered in dispatcher.") + + if not p.selector.isEmpty(): + var count = p.selector.selectInto(p.adjustedTimeout(timeout), keys) + var i = 0 + while i < count: + var update = false + var fd = keys[i].fd.SocketHandle + let events = keys[i].events + + if Event.Read in events: + let cb = keys[i].data.readCB + doAssert(cb != nil) + if cb(fd.AsyncFD): + p.selector.withData(fd, adata) do: + if adata.readCB == cb: + adata.readCB = nil + update = true + + if Event.Write in events: + let cb = keys[i].data.writeCB + doAssert(cb != nil) + if cb(fd.AsyncFD): + p.selector.withData(fd, adata) do: + if adata.writeCB == cb: + adata.writeCB = nil + update = true + + when supportedPlatform: + if (customSet * events) != {}: + let cb = keys[i].data.readCB + doAssert(cb != nil) + if cb(fd.AsyncFD): + p.selector.withData(fd, adata) do: + if adata.readCB == cb: + adata.readCB = nil + p.selector.unregister(fd) + + if update: + var newEvents: set[Event] = {} + p.selector.withData(fd, adata) do: + if adata.readCB != nil: incl(newEvents, Event.Read) + if adata.writeCB != nil: incl(newEvents, Event.Write) + p.selector.updateHandle(fd, newEvents) + inc(i) + + # Timer processing. + processTimers(p) + # Callback queue processing + processPendingCallbacks(p) + + proc connect*(socket: AsyncFD, address: string, port: Port, + domain = AF_INET): Future[void] = + var retFuture = newFuture[void]("connect") + + proc cb(fd: AsyncFD): bool = + var ret = SocketHandle(fd).getSockOptInt(cint(SOL_SOCKET), cint(SO_ERROR)) + if ret == 0: + # We have connected. + retFuture.complete() + return true + elif ret == EINTR: + # interrupted, keep waiting + return false + else: + retFuture.fail(newException(OSError, osErrorMsg(OSErrorCode(ret)))) + return true + + assert getSockDomain(socket.SocketHandle) == domain + var aiList = getAddrInfo(address, port, domain) + var success = false + var lastError: OSErrorCode + var it = aiList + while it != nil: + var ret = connect(socket.SocketHandle, it.ai_addr, it.ai_addrlen.Socklen) + if ret == 0: + # Request to connect completed immediately. + success = true + retFuture.complete() + break + else: + lastError = osLastError() + if lastError.int32 == EINTR or lastError.int32 == EINPROGRESS: + success = true + addWrite(socket, cb) + break + else: + success = false + it = it.ai_next + + dealloc(aiList) + if not success: + retFuture.fail(newException(OSError, osErrorMsg(lastError))) + return retFuture + + proc recv*(socket: AsyncFD, size: int, + flags = {SocketFlag.SafeDisconn}): Future[string] = + var retFuture = newFuture[string]("recv") + + var readBuffer = newString(size) + + proc cb(sock: AsyncFD): bool = + result = true + let res = recv(sock.SocketHandle, addr readBuffer[0], size.cint, + flags.toOSFlags()) + if res < 0: + let lastError = osLastError() + if lastError.int32 notin {EINTR, EWOULDBLOCK, EAGAIN}: + if flags.isDisconnectionError(lastError): + retFuture.complete("") + else: + retFuture.fail(newException(OSError, osErrorMsg(lastError))) + else: + result = false # We still want this callback to be called. + elif res == 0: + # Disconnected + retFuture.complete("") + else: + readBuffer.setLen(res) + retFuture.complete(readBuffer) + # TODO: The following causes a massive slowdown. + #if not cb(socket): + addRead(socket, cb) + return retFuture + + proc recvInto*(socket: AsyncFD, buf: cstring, size: int, + flags = {SocketFlag.SafeDisconn}): Future[int] = + var retFuture = newFuture[int]("recvInto") + + proc cb(sock: AsyncFD): bool = + result = true + let res = recv(sock.SocketHandle, buf, size.cint, + flags.toOSFlags()) + if res < 0: + let lastError = osLastError() + if lastError.int32 notin {EINTR, EWOULDBLOCK, EAGAIN}: + if flags.isDisconnectionError(lastError): + retFuture.complete(0) + else: + retFuture.fail(newException(OSError, osErrorMsg(lastError))) + else: + result = false # We still want this callback to be called. + else: + retFuture.complete(res) + # TODO: The following causes a massive slowdown. + #if not cb(socket): + addRead(socket, cb) + return retFuture + + proc send*(socket: AsyncFD, data: string, + flags = {SocketFlag.SafeDisconn}): Future[void] = + var retFuture = newFuture[void]("send") + + var written = 0 + + proc cb(sock: AsyncFD): bool = + result = true + let netSize = data.len-written + var d = data.cstring + let res = send(sock.SocketHandle, addr d[written], netSize.cint, + MSG_NOSIGNAL) + if res < 0: + let lastError = osLastError() + if lastError.int32 notin {EINTR, EWOULDBLOCK, EAGAIN}: + if flags.isDisconnectionError(lastError): + retFuture.complete() + else: + retFuture.fail(newException(OSError, osErrorMsg(lastError))) + else: + result = false # We still want this callback to be called. + else: + written.inc(res) + if res != netSize: + result = false # We still have data to send. + else: + retFuture.complete() + # TODO: The following causes crashes. + #if not cb(socket): + addWrite(socket, cb) + return retFuture + + proc sendTo*(socket: AsyncFD, data: pointer, size: int, saddr: ptr SockAddr, + saddrLen: SockLen, + flags = {SocketFlag.SafeDisconn}): Future[void] = + ## Sends ``data`` of size ``size`` in bytes to specified destination + ## (``saddr`` of size ``saddrLen`` in bytes, using socket ``socket``. + ## The returned future will complete once all data has been sent. + var retFuture = newFuture[void]("sendTo") + + # we will preserve address in our stack + var staddr: array[128, char] # SOCKADDR_STORAGE size is 128 bytes + var stalen = saddrLen + zeroMem(addr(staddr[0]), 128) + copyMem(addr(staddr[0]), saddr, saddrLen) + + proc cb(sock: AsyncFD): bool = + result = true + let res = sendto(sock.SocketHandle, data, size, MSG_NOSIGNAL, + cast[ptr SockAddr](addr(staddr[0])), stalen) + if res < 0: + let lastError = osLastError() + if lastError.int32 notin {EINTR, EWOULDBLOCK, EAGAIN}: + retFuture.fail(newException(OSError, osErrorMsg(lastError))) + else: + result = false # We still want this callback to be called. + else: + retFuture.complete() + + addWrite(socket, cb) + return retFuture + + proc recvFromInto*(socket: AsyncFD, data: pointer, size: int, + saddr: ptr SockAddr, saddrLen: ptr SockLen, + flags = {SocketFlag.SafeDisconn}): Future[int] = + ## Receives a datagram data from ``socket`` into ``data``, which must + ## be at least of size ``size`` in bytes, address of datagram's sender + ## will be stored into ``saddr`` and ``saddrLen``. Returned future will + ## complete once one datagram has been received, and will return size + ## of packet received. + var retFuture = newFuture[int]("recvFromInto") + proc cb(sock: AsyncFD): bool = + result = true + let res = recvfrom(sock.SocketHandle, data, size.cint, flags.toOSFlags(), + saddr, saddrLen) + if res < 0: + let lastError = osLastError() + if lastError.int32 notin {EINTR, EWOULDBLOCK, EAGAIN}: + retFuture.fail(newException(OSError, osErrorMsg(lastError))) + else: + result = false + else: + retFuture.complete(res) + addRead(socket, cb) + return retFuture + + proc acceptAddr*(socket: AsyncFD, flags = {SocketFlag.SafeDisconn}): + Future[tuple[address: string, client: AsyncFD]] = + var retFuture = newFuture[tuple[address: string, + client: AsyncFD]]("acceptAddr") + proc cb(sock: AsyncFD): bool = + result = true + var sockAddress: Sockaddr_storage + var addrLen = sizeof(sockAddress).Socklen + var client = accept(sock.SocketHandle, + cast[ptr SockAddr](addr(sockAddress)), addr(addrLen)) + if client == osInvalidSocket: + let lastError = osLastError() + assert lastError.int32 notin {EWOULDBLOCK, EAGAIN} + if lastError.int32 == EINTR: + return false + else: + if flags.isDisconnectionError(lastError): + return false + else: + retFuture.fail(newException(OSError, osErrorMsg(lastError))) + else: + register(client.AsyncFD) + retFuture.complete((getAddrString(cast[ptr SockAddr](addr sockAddress)), + client.AsyncFD)) + addRead(socket, cb) + return retFuture + + when supportedPlatform: + + proc addTimer*(timeout: int, oneshot: bool, cb: Callback) = + ## Start watching for timeout expiration, and then call the + ## callback ``cb``. + ## ``timeout`` - time in milliseconds, + ## ``oneshot`` - if ``true`` only one event will be dispatched, + ## if ``false`` continuous events every ``timeout`` milliseconds. + let p = getGlobalDispatcher() + var data = AsyncData(readCB: cb) + p.selector.registerTimer(timeout, oneshot, data) + + proc addSignal*(signal: int, cb: Callback) = + ## Start watching signal ``signal``, and when signal appears, call the + ## callback ``cb``. + let p = getGlobalDispatcher() + var data = AsyncData(readCB: cb) + p.selector.registerSignal(signal, data) + + proc addProcess*(pid: int, cb: Callback) = + ## Start watching for process exit with pid ``pid``, and then call + ## the callback ``cb``. + let p = getGlobalDispatcher() + var data = AsyncData(readCB: cb) + p.selector.registerProcess(pid, data) + + proc newAsyncEvent*(): AsyncEvent = + ## Creates new ``AsyncEvent``. + result = AsyncEvent(ioselectors.newSelectEvent()) + + proc setEvent*(ev: AsyncEvent) = + ## Sets new ``AsyncEvent`` to signaled state. + ioselectors.setEvent(SelectEvent(ev)) + + proc close*(ev: AsyncEvent) = + ## Closes ``AsyncEvent`` + ioselectors.close(SelectEvent(ev)) + + proc addEvent*(ev: AsyncEvent, cb: Callback) = + ## Start watching for event ``ev``, and call callback ``cb``, when + ## ev will be set to signaled state. + let p = getGlobalDispatcher() + var data = AsyncData(readCB: cb) + p.selector.registerEvent(SelectEvent(ev), data) + +proc sleepAsync*(ms: int): Future[void] = + ## Suspends the execution of the current async procedure for the next + ## ``ms`` milliseconds. + var retFuture = newFuture[void]("sleepAsync") + let p = getGlobalDispatcher() + p.timers.push((epochTime() + (ms / 1000), retFuture)) + return retFuture + +proc withTimeout*[T](fut: Future[T], timeout: int): Future[bool] = + ## Returns a future which will complete once ``fut`` completes or after + ## ``timeout`` milliseconds has elapsed. + ## + ## If ``fut`` completes first the returned future will hold true, + ## otherwise, if ``timeout`` milliseconds has elapsed first, the returned + ## future will hold false. + + var retFuture = newFuture[bool]("asyncdispatch.`withTimeout`") + var timeoutFuture = sleepAsync(timeout) + fut.callback = + proc () = + if not retFuture.finished: retFuture.complete(true) + timeoutFuture.callback = + proc () = + if not retFuture.finished: retFuture.complete(false) + return retFuture + +proc accept*(socket: AsyncFD, + flags = {SocketFlag.SafeDisconn}): Future[AsyncFD] = + ## Accepts a new connection. Returns a future containing the client socket + ## corresponding to that connection. + ## The future will complete when the connection is successfully accepted. + var retFut = newFuture[AsyncFD]("accept") + var fut = acceptAddr(socket, flags) + fut.callback = + proc (future: Future[tuple[address: string, client: AsyncFD]]) = + assert future.finished + if future.failed: + retFut.fail(future.error) + else: + retFut.complete(future.read.client) + return retFut + +# -- Await Macro + +proc skipUntilStmtList(node: NimNode): NimNode {.compileTime.} = + # Skips a nest of StmtList's. + result = node + if node[0].kind == nnkStmtList: + result = skipUntilStmtList(node[0]) + +proc skipStmtList(node: NimNode): NimNode {.compileTime.} = + result = node + if node[0].kind == nnkStmtList: + result = node[0] + +template createCb(retFutureSym, iteratorNameSym, + name: expr): stmt {.immediate.} = + var nameIterVar = iteratorNameSym + #{.push stackTrace: off.} + proc cb {.closure,gcsafe.} = + try: + if not nameIterVar.finished: + var next = nameIterVar() + if next == nil: + assert retFutureSym.finished, "Async procedure's (" & + name & ") return Future was not finished." + else: + next.callback = cb + except: + if retFutureSym.finished: + # Take a look at tasyncexceptions for the bug which this fixes. + # That test explains it better than I can here. + raise + else: + retFutureSym.fail(getCurrentException()) + cb() + #{.pop.} +proc generateExceptionCheck(futSym, + tryStmt, rootReceiver, fromNode: NimNode): NimNode {.compileTime.} = + if tryStmt.kind == nnkNilLit: + result = rootReceiver + else: + var exceptionChecks: seq[tuple[cond, body: NimNode]] = @[] + let errorNode = newDotExpr(futSym, newIdentNode("error")) + for i in 1 .. <tryStmt.len: + let exceptBranch = tryStmt[i] + if exceptBranch[0].kind == nnkStmtList: + exceptionChecks.add((newIdentNode("true"), exceptBranch[0])) + else: + var exceptIdentCount = 0 + var ifCond: NimNode + for i in 0 .. <exceptBranch.len: + let child = exceptBranch[i] + if child.kind == nnkIdent: + let cond = infix(errorNode, "of", child) + if exceptIdentCount == 0: + ifCond = cond + else: + ifCond = infix(ifCond, "or", cond) + else: + break + exceptIdentCount.inc + + expectKind(exceptBranch[exceptIdentCount], nnkStmtList) + exceptionChecks.add((ifCond, exceptBranch[exceptIdentCount])) + # -> -> else: raise futSym.error + exceptionChecks.add((newIdentNode("true"), + newNimNode(nnkRaiseStmt).add(errorNode))) + # Read the future if there is no error. + # -> else: futSym.read + let elseNode = newNimNode(nnkElse, fromNode) + elseNode.add newNimNode(nnkStmtList, fromNode) + elseNode[0].add rootReceiver + + let ifBody = newStmtList() + ifBody.add newCall(newIdentNode("setCurrentException"), errorNode) + ifBody.add newIfStmt(exceptionChecks) + ifBody.add newCall(newIdentNode("setCurrentException"), newNilLit()) + + result = newIfStmt( + (newDotExpr(futSym, newIdentNode("failed")), ifBody) + ) + result.add elseNode + +template useVar(result: var NimNode, futureVarNode: NimNode, valueReceiver, + rootReceiver: expr, fromNode: NimNode) = + ## Params: + ## futureVarNode: The NimNode which is a symbol identifying the Future[T] + ## variable to yield. + ## fromNode: Used for better debug information (to give context). + ## valueReceiver: The node which defines an expression that retrieves the + ## future's value. + ## + ## rootReceiver: ??? TODO + # -> yield future<x> + result.add newNimNode(nnkYieldStmt, fromNode).add(futureVarNode) + # -> future<x>.read + valueReceiver = newDotExpr(futureVarNode, newIdentNode("read")) + result.add generateExceptionCheck(futureVarNode, tryStmt, rootReceiver, + fromNode) + +template createVar(result: var NimNode, futSymName: string, + asyncProc: NimNode, + valueReceiver, rootReceiver: expr, + fromNode: NimNode) = + result = newNimNode(nnkStmtList, fromNode) + var futSym = genSym(nskVar, "future") + result.add newVarStmt(futSym, asyncProc) # -> var future<x> = y + useVar(result, futSym, valueReceiver, rootReceiver, fromNode) + +proc processBody(node, retFutureSym: NimNode, + subTypeIsVoid: bool, + tryStmt: NimNode): NimNode {.compileTime.} = + #echo(node.treeRepr) + result = node + case node.kind + of nnkReturnStmt: + result = newNimNode(nnkStmtList, node) + if node[0].kind == nnkEmpty: + if not subTypeIsVoid: + result.add newCall(newIdentNode("complete"), retFutureSym, + newIdentNode("result")) + else: + result.add newCall(newIdentNode("complete"), retFutureSym) + else: + result.add newCall(newIdentNode("complete"), retFutureSym, + node[0].processBody(retFutureSym, subTypeIsVoid, tryStmt)) + + result.add newNimNode(nnkReturnStmt, node).add(newNilLit()) + return # Don't process the children of this return stmt + of nnkCommand, nnkCall: + if node[0].kind == nnkIdent and node[0].ident == !"await": + case node[1].kind + of nnkIdent, nnkInfix, nnkDotExpr: + # await x + # await x or y + result = newNimNode(nnkYieldStmt, node).add(node[1]) # -> yield x + of nnkCall, nnkCommand: + # await foo(p, x) + # await foo p, x + var futureValue: NimNode + result.createVar("future" & $node[1][0].toStrLit, node[1], futureValue, + futureValue, node) + else: + error("Invalid node kind in 'await', got: " & $node[1].kind) + elif node.len > 1 and node[1].kind == nnkCommand and + node[1][0].kind == nnkIdent and node[1][0].ident == !"await": + # foo await x + var newCommand = node + result.createVar("future" & $node[0].toStrLit, node[1][1], newCommand[1], + newCommand, node) + + of nnkVarSection, nnkLetSection: + case node[0][2].kind + of nnkCommand: + if node[0][2][0].kind == nnkIdent and node[0][2][0].ident == !"await": + # var x = await y + var newVarSection = node # TODO: Should this use copyNimNode? + result.createVar("future" & $node[0][0].ident, node[0][2][1], + newVarSection[0][2], newVarSection, node) + else: discard + of nnkAsgn: + case node[1].kind + of nnkCommand: + if node[1][0].ident == !"await": + # x = await y + var newAsgn = node + result.createVar("future" & $node[0].toStrLit, node[1][1], newAsgn[1], newAsgn, node) + else: discard + of nnkDiscardStmt: + # discard await x + if node[0].kind == nnkCommand and node[0][0].kind == nnkIdent and + node[0][0].ident == !"await": + var newDiscard = node + result.createVar("futureDiscard_" & $toStrLit(node[0][1]), node[0][1], + newDiscard[0], newDiscard, node) + of nnkTryStmt: + # try: await x; except: ... + result = newNimNode(nnkStmtList, node) + template wrapInTry(n, tryBody: expr) = + var temp = n + n[0] = tryBody + tryBody = temp + + # Transform ``except`` body. + # TODO: Could we perform some ``await`` transformation here to get it + # working in ``except``? + tryBody[1] = processBody(n[1], retFutureSym, subTypeIsVoid, nil) + + proc processForTry(n: NimNode, i: var int, + res: NimNode): bool {.compileTime.} = + ## Transforms the body of the tryStmt. Does not transform the + ## body in ``except``. + ## Returns true if the tryStmt node was transformed into an ifStmt. + result = false + var skipped = n.skipStmtList() + while i < skipped.len: + var processed = processBody(skipped[i], retFutureSym, + subTypeIsVoid, n) + + # Check if we transformed the node into an exception check. + # This suggests skipped[i] contains ``await``. + if processed.kind != skipped[i].kind or processed.len != skipped[i].len: + processed = processed.skipUntilStmtList() + expectKind(processed, nnkStmtList) + expectKind(processed[2][1], nnkElse) + i.inc + + if not processForTry(n, i, processed[2][1][0]): + # We need to wrap the nnkElse nodes back into a tryStmt. + # As they are executed if an exception does not happen + # inside the awaited future. + # The following code will wrap the nodes inside the + # original tryStmt. + wrapInTry(n, processed[2][1][0]) + + res.add processed + result = true + else: + res.add skipped[i] + i.inc + var i = 0 + if not processForTry(node, i, result): + # If the tryStmt hasn't been transformed we can just put the body + # back into it. + wrapInTry(node, result) + return + else: discard + + for i in 0 .. <result.len: + result[i] = processBody(result[i], retFutureSym, subTypeIsVoid, nil) + +proc getName(node: NimNode): string {.compileTime.} = + case node.kind + of nnkPostfix: + return $node[1].ident + of nnkIdent: + return $node.ident + of nnkEmpty: + return "anonymous" + else: + error("Unknown name.") + +proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = + ## This macro transforms a single procedure into a closure iterator. + ## The ``async`` macro supports a stmtList holding multiple async procedures. + if prc.kind notin {nnkProcDef, nnkLambda}: + error("Cannot transform this node kind into an async proc." & + " Proc definition or lambda node expected.") + + hint("Processing " & prc[0].getName & " as an async proc.") + + let returnType = prc[3][0] + var baseType: NimNode + # Verify that the return type is a Future[T] + if returnType.kind == nnkBracketExpr: + let fut = repr(returnType[0]) + if fut != "Future": + error("Expected return type of 'Future' got '" & fut & "'") + baseType = returnType[1] + elif returnType.kind in nnkCallKinds and $returnType[0] == "[]": + let fut = repr(returnType[1]) + if fut != "Future": + error("Expected return type of 'Future' got '" & fut & "'") + baseType = returnType[2] + elif returnType.kind == nnkEmpty: + baseType = returnType + else: + error("Expected return type of 'Future' got '" & repr(returnType) & "'") + + let subtypeIsVoid = returnType.kind == nnkEmpty or + (baseType.kind == nnkIdent and returnType[1].ident == !"void") + + var outerProcBody = newNimNode(nnkStmtList, prc[6]) + + # -> var retFuture = newFuture[T]() + var retFutureSym = genSym(nskVar, "retFuture") + var subRetType = + if returnType.kind == nnkEmpty: newIdentNode("void") + else: baseType + outerProcBody.add( + newVarStmt(retFutureSym, + newCall( + newNimNode(nnkBracketExpr, prc[6]).add( + newIdentNode(!"newFuture"), # TODO: Strange bug here? Remove the `!`. + subRetType), + newLit(prc[0].getName)))) # Get type from return type of this proc + + # -> iterator nameIter(): FutureBase {.closure.} = + # -> {.push warning[resultshadowed]: off.} + # -> var result: T + # -> {.pop.} + # -> <proc_body> + # -> complete(retFuture, result) + var iteratorNameSym = genSym(nskIterator, $prc[0].getName & "Iter") + var procBody = prc[6].processBody(retFutureSym, subtypeIsVoid, nil) + if not subtypeIsVoid: + procBody.insert(0, newNimNode(nnkPragma).add(newIdentNode("push"), + newNimNode(nnkExprColonExpr).add(newNimNode(nnkBracketExpr).add( + newIdentNode("warning"), newIdentNode("resultshadowed")), + newIdentNode("off")))) # -> {.push warning[resultshadowed]: off.} + + procBody.insert(1, newNimNode(nnkVarSection, prc[6]).add( + newIdentDefs(newIdentNode("result"), baseType))) # -> var result: T + + procBody.insert(2, newNimNode(nnkPragma).add( + newIdentNode("pop"))) # -> {.pop.}) + + procBody.add( + newCall(newIdentNode("complete"), + retFutureSym, newIdentNode("result"))) # -> complete(retFuture, result) + else: + # -> complete(retFuture) + procBody.add(newCall(newIdentNode("complete"), retFutureSym)) + + var closureIterator = newProc(iteratorNameSym, [newIdentNode("FutureBase")], + procBody, nnkIteratorDef) + closureIterator[4] = newNimNode(nnkPragma, prc[6]).add(newIdentNode("closure")) + outerProcBody.add(closureIterator) + + # -> createCb(retFuture) + #var cbName = newIdentNode("cb") + var procCb = newCall(bindSym"createCb", retFutureSym, iteratorNameSym, + newStrLitNode(prc[0].getName)) + outerProcBody.add procCb + + # -> return retFuture + outerProcBody.add newNimNode(nnkReturnStmt, prc[6][prc[6].len-1]).add(retFutureSym) + + result = prc + + # Remove the 'async' pragma. + for i in 0 .. <result[4].len: + if result[4][i].kind == nnkIdent and result[4][i].ident == !"async": + result[4].del(i) + result[4] = newEmptyNode() + if subtypeIsVoid: + # Add discardable pragma. + if returnType.kind == nnkEmpty: + # Add Future[void] + result[3][0] = parseExpr("Future[void]") + + result[6] = outerProcBody + + #echo(treeRepr(result)) + #if prc[0].getName == "testInfix": + # echo(toStrLit(result)) + +macro async*(prc: stmt): stmt {.immediate.} = + ## Macro which processes async procedures into the appropriate + ## iterators and yield statements. + if prc.kind == nnkStmtList: + for oneProc in prc: + result = newStmtList() + result.add asyncSingleProc(oneProc) + else: + result = asyncSingleProc(prc) + +proc recvLine*(socket: AsyncFD): Future[string] {.async.} = + ## Reads a line of data from ``socket``. Returned future will complete once + ## a full line is read or an error occurs. + ## + ## If a full line is read ``\r\L`` is not + ## added to ``line``, however if solely ``\r\L`` is read then ``line`` + ## will be set to it. + ## + ## If the socket is disconnected, ``line`` will be set to ``""``. + ## + ## If the socket is disconnected in the middle of a line (before ``\r\L`` + ## is read) then line will be set to ``""``. + ## The partial line **will be lost**. + ## + ## **Warning**: This assumes that lines are delimited by ``\r\L``. + ## + ## **Note**: This procedure is mostly used for testing. You likely want to + ## use ``asyncnet.recvLine`` instead. + + template addNLIfEmpty(): stmt = + if result.len == 0: + result.add("\c\L") + + result = "" + var c = "" + while true: + c = await recv(socket, 1) + if c.len == 0: + return "" + if c == "\r": + c = await recv(socket, 1) + assert c == "\l" + addNLIfEmpty() + return + elif c == "\L": + addNLIfEmpty() + return + add(result, c) + +proc callSoon*(cbproc: proc ()) = + ## Schedule `cbproc` to be called as soon as possible. + ## The callback is called when control returns to the event loop. + getGlobalDispatcher().callbacks.enqueue(cbproc) + +proc runForever*() = + ## Begins a never ending global dispatcher poll loop. + while true: + poll() + +proc waitFor*[T](fut: Future[T]): T = + ## **Blocks** the current thread until the specified future completes. + while not fut.finished: + poll() + + fut.read diff --git a/lib/windows/winlean.nim b/lib/windows/winlean.nim index 5e899dd1a..b766ead9c 100644 --- a/lib/windows/winlean.nim +++ b/lib/windows/winlean.nim @@ -36,6 +36,8 @@ type DWORD* = int32 PDWORD* = ptr DWORD LPINT* = ptr int32 + ULONG_PTR* = uint + PULONG_PTR* = ptr uint HDC* = Handle HGLRC* = Handle @@ -92,7 +94,7 @@ type dwMinorVersion*: DWORD dwBuildNumber*: DWORD dwPlatformId*: DWORD - szCSDVersion*: array[0..127, WinChar]; + szCSDVersion*: array[0..127, WinChar] {.deprecated: [THandle: Handle, TSECURITY_ATTRIBUTES: SECURITY_ATTRIBUTES, TSTARTUPINFO: STARTUPINFO, TPROCESS_INFORMATION: PROCESS_INFORMATION, @@ -440,6 +442,8 @@ type sa_family*: int16 # unsigned sa_data: array[0..13, char] + PSockAddr = ptr SockAddr + InAddr* {.importc: "IN_ADDR", header: "winsock2.h".} = object s_addr*: uint32 # IP address @@ -451,7 +455,7 @@ type sin_zero*: array[0..7, char] In6_addr* {.importc: "IN6_ADDR", header: "winsock2.h".} = object - bytes*: array[0..15, char] + bytes* {.importc: "u.Byte".}: array[0..15, char] Sockaddr_in6* {.importc: "SOCKADDR_IN6", header: "ws2tcpip.h".} = object @@ -516,6 +520,8 @@ var SO_DEBUG* {.importc, header: "winsock2.h".}: cint ## turn on debugging info recording SO_ACCEPTCONN* {.importc, header: "winsock2.h".}: cint # socket has had listen() SO_REUSEADDR* {.importc, header: "winsock2.h".}: cint # allow local address reuse + SO_REUSEPORT* {.importc: "SO_REUSEADDR", header: "winsock2.h".}: cint # allow port reuse. Since Windows does not really support it, mapped to SO_REUSEADDR. This shouldn't cause problems. + SO_KEEPALIVE* {.importc, header: "winsock2.h".}: cint # keep connections alive SO_DONTROUTE* {.importc, header: "winsock2.h".}: cint # just use interface addresses SO_BROADCAST* {.importc, header: "winsock2.h".}: cint # permit sending of broadcast msgs @@ -744,7 +750,7 @@ type D1*: int32 D2*: int16 D3*: int16 - D4*: array [0..7, int8] + D4*: array[0..7, int8] {.deprecated: [TOVERLAPPED: OVERLAPPED, TGUID: GUID].} const @@ -757,22 +763,29 @@ const WSAENETRESET* = 10052 WSAETIMEDOUT* = 10060 ERROR_NETNAME_DELETED* = 64 + STATUS_PENDING* = 0x103 proc createIoCompletionPort*(FileHandle: Handle, ExistingCompletionPort: Handle, - CompletionKey: DWORD, + CompletionKey: ULONG_PTR, NumberOfConcurrentThreads: DWORD): Handle{.stdcall, dynlib: "kernel32", importc: "CreateIoCompletionPort".} proc getQueuedCompletionStatus*(CompletionPort: Handle, - lpNumberOfBytesTransferred: PDWORD, lpCompletionKey: PULONG, + lpNumberOfBytesTransferred: PDWORD, lpCompletionKey: PULONG_PTR, lpOverlapped: ptr POVERLAPPED, dwMilliseconds: DWORD): WINBOOL{.stdcall, dynlib: "kernel32", importc: "GetQueuedCompletionStatus".} -proc getOverlappedResult*(hFile: Handle, lpOverlapped: OVERLAPPED, +proc getOverlappedResult*(hFile: Handle, lpOverlapped: POVERLAPPED, lpNumberOfBytesTransferred: var DWORD, bWait: WINBOOL): WINBOOL{. stdcall, dynlib: "kernel32", importc: "GetOverlappedResult".} +# this is copy of HasOverlappedIoCompleted() macro from <winbase.h> +# because we have declared own OVERLAPPED structure with member names not +# compatible with original names. +template hasOverlappedIoCompleted*(lpOverlapped): bool = + (cast[uint](lpOverlapped.internal) != STATUS_PENDING) + const IOC_OUT* = 0x40000000 IOC_IN* = 0x80000000 @@ -812,11 +825,23 @@ proc WSARecv*(s: SocketHandle, buf: ptr TWSABuf, bufCount: DWORD, completionProc: POVERLAPPED_COMPLETION_ROUTINE): cint {. stdcall, importc: "WSARecv", dynlib: "Ws2_32.dll".} +proc WSARecvFrom*(s: SocketHandle, buf: ptr TWSABuf, bufCount: DWORD, + bytesReceived: PDWORD, flags: PDWORD, name: ptr SockAddr, + namelen: ptr cint, lpOverlapped: POVERLAPPED, + completionProc: POVERLAPPED_COMPLETION_ROUTINE): cint {. + stdcall, importc: "WSARecvFrom", dynlib: "Ws2_32.dll".} + proc WSASend*(s: SocketHandle, buf: ptr TWSABuf, bufCount: DWORD, bytesSent: PDWORD, flags: DWORD, lpOverlapped: POVERLAPPED, completionProc: POVERLAPPED_COMPLETION_ROUTINE): cint {. stdcall, importc: "WSASend", dynlib: "Ws2_32.dll".} +proc WSASendTo*(s: SocketHandle, buf: ptr TWSABuf, bufCount: DWORD, + bytesSent: PDWORD, flags: DWORD, name: ptr SockAddr, + namelen: cint, lpOverlapped: POVERLAPPED, + completionProc: POVERLAPPED_COMPLETION_ROUTINE): cint {. + stdcall, importc: "WSASendTo", dynlib: "Ws2_32.dll".} + proc get_osfhandle*(fd:FileHandle): Handle {. importc: "_get_osfhandle", header:"<io.h>".} @@ -878,3 +903,136 @@ proc inet_ntop*(family: cint, paddr: pointer, pStringBuffer: cstring, result = inet_ntop_real(family, paddr, pStringBuffer, stringBufSize) else: result = inet_ntop_emulated(family, paddr, pStringBuffer, stringBufSize) + +type + WSAPROC_ACCEPTEX* = proc (sListenSocket: SocketHandle, + sAcceptSocket: SocketHandle, + lpOutputBuffer: pointer, dwReceiveDataLength: DWORD, + dwLocalAddressLength: DWORD, + dwRemoteAddressLength: DWORD, + lpdwBytesReceived: ptr DWORD, + lpOverlapped: POVERLAPPED): bool {. + stdcall,gcsafe.} + + WSAPROC_CONNECTEX* = proc (s: SocketHandle, name: ptr SockAddr, namelen: cint, + lpSendBuffer: pointer, dwSendDataLength: DWORD, + lpdwBytesSent: ptr DWORD, + lpOverlapped: POVERLAPPED): bool {. + stdcall,gcsafe.} + + WSAPROC_GETACCEPTEXSOCKADDRS* = proc(lpOutputBuffer: pointer, + dwReceiveDataLength: DWORD, + dwLocalAddressLength: DWORD, + dwRemoteAddressLength: DWORD, + LocalSockaddr: ptr PSockAddr, + LocalSockaddrLength: ptr cint, + RemoteSockaddr: ptr PSockAddr, + RemoteSockaddrLength: ptr cint) {. + stdcall,gcsafe.} + +const + WT_EXECUTEDEFAULT* = 0x00000000'i32 + WT_EXECUTEINIOTHREAD* = 0x00000001'i32 + WT_EXECUTEINUITHREAD* = 0x00000002'i32 + WT_EXECUTEINWAITTHREAD* = 0x00000004'i32 + WT_EXECUTEONLYONCE* = 0x00000008'i32 + WT_EXECUTELONGFUNCTION* = 0x00000010'i32 + WT_EXECUTEINTIMERTHREAD* = 0x00000020'i32 + WT_EXECUTEINPERSISTENTIOTHREAD* = 0x00000040'i32 + WT_EXECUTEINPERSISTENTTHREAD* = 0x00000080'i32 + WT_TRANSFER_IMPERSONATION* = 0x00000100'i32 + PROCESS_TERMINATE* = 0x00000001'i32 + PROCESS_CREATE_THREAD* = 0x00000002'i32 + PROCESS_SET_SESSIONID* = 0x00000004'i32 + PROCESS_VM_OPERATION* = 0x00000008'i32 + PROCESS_VM_READ* = 0x00000010'i32 + PROCESS_VM_WRITE* = 0x00000020'i32 + PROCESS_DUP_HANDLE* = 0x00000040'i32 + PROCESS_CREATE_PROCESS* = 0x00000080'i32 + PROCESS_SET_QUOTA* = 0x00000100'i32 + PROCESS_SET_INFORMATION* = 0x00000200'i32 + PROCESS_QUERY_INFORMATION* = 0x00000400'i32 + PROCESS_SUSPEND_RESUME* = 0x00000800'i32 + PROCESS_QUERY_LIMITED_INFORMATION* = 0x00001000'i32 + PROCESS_SET_LIMITED_INFORMATION* = 0x00002000'i32 +type + WAITORTIMERCALLBACK* = proc(para1: pointer, para2: int32): void {.stdcall.} + +proc postQueuedCompletionStatus*(CompletionPort: HANDLE, + dwNumberOfBytesTransferred: DWORD, + dwCompletionKey: ULONG_PTR, + lpOverlapped: pointer): bool + {.stdcall, dynlib: "kernel32", importc: "PostQueuedCompletionStatus".} + +proc registerWaitForSingleObject*(phNewWaitObject: ptr Handle, hObject: Handle, + Callback: WAITORTIMERCALLBACK, + Context: pointer, + dwMilliseconds: ULONG, + dwFlags: ULONG): bool + {.stdcall, dynlib: "kernel32", importc: "RegisterWaitForSingleObject".} + +proc unregisterWait*(WaitHandle: HANDLE): DWORD + {.stdcall, dynlib: "kernel32", importc: "UnregisterWait".} + +proc openProcess*(dwDesiredAccess: DWORD, bInheritHandle: WINBOOL, + dwProcessId: DWORD): Handle + {.stdcall, dynlib: "kernel32", importc: "OpenProcess".} + +when defined(useWinAnsi): + proc createEvent*(lpEventAttributes: ptr SECURITY_ATTRIBUTES, + bManualReset: DWORD, bInitialState: DWORD, + lpName: cstring): Handle + {.stdcall, dynlib: "kernel32", importc: "CreateEventA".} +else: + proc createEvent*(lpEventAttributes: ptr SECURITY_ATTRIBUTES, + bManualReset: DWORD, bInitialState: DWORD, + lpName: ptr Utf16Char): Handle + {.stdcall, dynlib: "kernel32", importc: "CreateEventW".} + +proc setEvent*(hEvent: Handle): cint + {.stdcall, dynlib: "kernel32", importc: "SetEvent".} + +const + FD_READ* = 0x00000001'i32 + FD_WRITE* = 0x00000002'i32 + FD_OOB* = 0x00000004'i32 + FD_ACCEPT* = 0x00000008'i32 + FD_CONNECT* = 0x00000010'i32 + FD_CLOSE* = 0x00000020'i32 + FD_QQS* = 0x00000040'i32 + FD_GROUP_QQS* = 0x00000080'i32 + FD_ROUTING_INTERFACE_CHANGE* = 0x00000100'i32 + FD_ADDRESS_LIST_CHANGE* = 0x00000200'i32 + FD_ALL_EVENTS* = 0x000003FF'i32 + +proc wsaEventSelect*(s: SocketHandle, hEventObject: Handle, + lNetworkEvents: clong): cint + {.stdcall, importc: "WSAEventSelect", dynlib: "ws2_32.dll".} + +proc wsaCreateEvent*(): Handle + {.stdcall, importc: "WSACreateEvent", dynlib: "ws2_32.dll".} + +proc wsaCloseEvent*(hEvent: Handle): bool + {.stdcall, importc: "WSACloseEvent", dynlib: "ws2_32.dll".} + +proc wsaResetEvent*(hEvent: Handle): bool + {.stdcall, importc: "WSAResetEvent", dynlib: "ws2_32.dll".} + +type + KEY_EVENT_RECORD* {.final, pure.} = object + eventType*: int16 + bKeyDown*: WINBOOL + wRepeatCount*: int16 + wVirtualKeyCode*: int16 + wVirtualScanCode*: int16 + uChar*: int16 + dwControlKeyState*: DWORD + +when defined(useWinAnsi): + proc readConsoleInput*(hConsoleInput: Handle, lpBuffer: pointer, nLength: cint, + lpNumberOfEventsRead: ptr cint): cint + {.stdcall, dynlib: "kernel32", importc: "ReadConsoleInputA".} +else: + proc readConsoleInput*(hConsoleInput: Handle, lpBuffer: pointer, nLength: cint, + lpNumberOfEventsRead: ptr cint): cint + {.stdcall, dynlib: "kernel32", importc: "ReadConsoleInputW".} \ No newline at end of file diff --git a/lib/wrappers/openssl.nim b/lib/wrappers/openssl.nim index 05843e2d3..9dad7e489 100644 --- a/lib/wrappers/openssl.nim +++ b/lib/wrappers/openssl.nim @@ -197,6 +197,7 @@ proc TLSv1_method*(): PSSL_METHOD{.cdecl, dynlib: DLLSSLName, importc.} proc SSL_new*(context: SslCtx): SslPtr{.cdecl, dynlib: DLLSSLName, importc.} proc SSL_free*(ssl: SslPtr){.cdecl, dynlib: DLLSSLName, importc.} +proc SSL_get_SSL_CTX*(ssl: SslPtr): SslCtx {.cdecl, dynlib: DLLSSLName, importc.} proc SSL_CTX_new*(meth: PSSL_METHOD): SslCtx{.cdecl, dynlib: DLLSSLName, importc.} proc SSL_CTX_load_verify_locations*(ctx: SslCtx, CAfile: cstring, @@ -216,6 +217,10 @@ proc SSL_CTX_use_PrivateKey_file*(ctx: SslCtx, proc SSL_CTX_check_private_key*(ctx: SslCtx): cInt{.cdecl, dynlib: DLLSSLName, importc.} +proc SSL_CTX_get_ex_new_index*(argl: clong, argp: pointer, new_func: pointer, dup_func: pointer, free_func: pointer): cint {.cdecl, dynlib: DLLSSLName, importc.} +proc SSL_CTX_set_ex_data*(ssl: SslCtx, idx: cint, arg: pointer): cint {.cdecl, dynlib: DLLSSLName, importc.} +proc SSL_CTX_get_ex_data*(ssl: SslCtx, idx: cint): pointer {.cdecl, dynlib: DLLSSLName, importc.} + proc SSL_set_fd*(ssl: SslPtr, fd: SocketHandle): cint{.cdecl, dynlib: DLLSSLName, importc.} proc SSL_shutdown*(ssl: SslPtr): cInt{.cdecl, dynlib: DLLSSLName, importc.} @@ -260,7 +265,7 @@ proc OpenSSL_add_all_algorithms*(){.cdecl, dynlib: DLLUtilName, importc: "OPENSS proc OPENSSL_config*(configName: cstring){.cdecl, dynlib: DLLSSLName, importc.} -when not useWinVersion: +when not useWinVersion and not defined(macosx): proc CRYPTO_set_mem_functions(a,b,c: pointer){.cdecl, dynlib: DLLUtilName, importc.} @@ -314,6 +319,25 @@ proc SSL_CTX_set_tlsext_servername_arg*(ctx: SslCtx, arg: pointer): int = ## Set the pointer to be used in the callback registered to ``SSL_CTX_set_tlsext_servername_callback``. result = SSL_CTX_ctrl(ctx, SSL_CTRL_SET_TLSEXT_SERVERNAME_ARG, 0, arg) +type + PskClientCallback* = proc (ssl: SslPtr; + hint: cstring; identity: cstring; max_identity_len: cuint; psk: ptr cuchar; + max_psk_len: cuint): cuint {.cdecl.} + + PskServerCallback* = proc (ssl: SslPtr; + identity: cstring; psk: ptr cuchar; max_psk_len: cint): cuint {.cdecl.} + +proc SSL_CTX_set_psk_client_callback*(ctx: SslCtx; callback: PskClientCallback) {.cdecl, dynlib: DLLSSLName, importc.} + ## Set callback called when OpenSSL needs PSK (for client). + +proc SSL_CTX_set_psk_server_callback*(ctx: SslCtx; callback: PskServerCallback) {.cdecl, dynlib: DLLSSLName, importc.} + ## Set callback called when OpenSSL needs PSK (for server). + +proc SSL_CTX_use_psk_identity_hint*(ctx: SslCtx; hint: cstring): cint {.cdecl, dynlib: DLLSSLName, importc.} + ## Set PSK identity hint to use. + +proc SSL_get_psk_identity*(ssl: SslPtr): cstring {.cdecl, dynlib: DLLSSLName, importc.} + ## Get PSK identity. proc bioNew*(b: PBIO_METHOD): BIO{.cdecl, dynlib: DLLUtilName, importc: "BIO_new".} proc bioFreeAll*(b: BIO){.cdecl, dynlib: DLLUtilName, importc: "BIO_free_all".} @@ -532,7 +556,7 @@ proc md5_File* (file: string): string {.raises: [IOError,Exception].} = result = hexStr(buf) -proc md5_Str* (str:string): string {.raises:[IOError].} = +proc md5_Str*(str:string): string = ##Generate MD5 hash for a string. Result is a 32 character #hex string with lowercase characters var |