diff options
Diffstat (limited to 'lib')
42 files changed, 1444 insertions, 412 deletions
diff --git a/lib/core/locks.nim b/lib/core/locks.nim index 66e0ab520..fbe9c8acf 100644 --- a/lib/core/locks.nim +++ b/lib/core/locks.nim @@ -9,6 +9,7 @@ ## This module contains Nim's support for locks and condition vars. +const insideRLocksModule = false include "system/syslocks" type @@ -63,4 +64,4 @@ template withLock*(a: Lock, body: untyped) = try: body finally: - a.release() \ No newline at end of file + a.release() diff --git a/lib/core/macros.nim b/lib/core/macros.nim index 4522e0fc6..678982a52 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -796,17 +796,17 @@ proc infix*(a: NimNode; op: string; proc unpackPostfix*(node: NimNode): tuple[node: NimNode; op: string] {. compileTime.} = node.expectKind nnkPostfix - result = (node[0], $node[1]) + result = (node[1], $node[0]) proc unpackPrefix*(node: NimNode): tuple[node: NimNode; op: string] {. compileTime.} = node.expectKind nnkPrefix - result = (node[0], $node[1]) + result = (node[1], $node[0]) proc unpackInfix*(node: NimNode): tuple[left: NimNode; op: string; right: NimNode] {.compileTime.} = assert node.kind == nnkInfix - result = (node[0], $node[1], node[2]) + result = (node[1], $node[0], node[2]) proc copy*(node: NimNode): NimNode {.compileTime.} = ## An alias for copyNimTree(). diff --git a/lib/core/rlocks.nim b/lib/core/rlocks.nim index 14f04592b..4710d6cf1 100644 --- a/lib/core/rlocks.nim +++ b/lib/core/rlocks.nim @@ -9,6 +9,7 @@ ## This module contains Nim's support for reentrant locks. +const insideRLocksModule = true include "system/syslocks" type diff --git a/lib/deprecated/pure/sockets.nim b/lib/deprecated/pure/sockets.nim index f7d0950d8..20e6d9364 100644 --- a/lib/deprecated/pure/sockets.nim +++ b/lib/deprecated/pure/sockets.nim @@ -206,6 +206,18 @@ proc htons*(x: int16): int16 = ## order, this is a no-op; otherwise, it performs a 2-byte swap operation. result = sockets.ntohs(x) +template ntohl(x: uint32): expr = + cast[uint32](sockets.ntohl(cast[int32](x))) + +template ntohs(x: uint16): expr = + cast[uint16](sockets.ntohs(cast[int16](x))) + +template htonl(x: uint32): expr = + sockets.ntohl(x) + +template htons(x: uint16): expr = + sockets.ntohs(x) + when defined(Posix): proc toInt(domain: Domain): cint = case domain @@ -451,7 +463,7 @@ proc bindAddr*(socket: Socket, port = Port(0), address = "") {. name.sin_family = int16(ord(AF_INET)) else: name.sin_family = posix.AF_INET - name.sin_port = sockets.htons(int16(port)) + name.sin_port = sockets.htons(uint16(port)) name.sin_addr.s_addr = sockets.htonl(INADDR_ANY) if bindSocket(socket.fd, cast[ptr SockAddr](addr(name)), sizeof(name).SockLen) < 0'i32: @@ -834,7 +846,7 @@ proc connect*(socket: Socket, address: string, port = Port(0), when false: var s: TSockAddrIn s.sin_addr.s_addr = inet_addr(address) - s.sin_port = sockets.htons(int16(port)) + s.sin_port = sockets.htons(uint16(port)) when defined(windows): s.sin_family = toU16(ord(af)) else: diff --git a/lib/impure/rdstdin.nim b/lib/impure/rdstdin.nim index 469bb69c5..f722a6b39 100644 --- a/lib/impure/rdstdin.nim +++ b/lib/impure/rdstdin.nim @@ -105,7 +105,8 @@ else: proc readLineFromStdin*(prompt: string): TaintedString {. tags: [ReadIOEffect, WriteIOEffect].} = var buffer = linenoise.readLine(prompt) - if isNil(buffer): quit(0) + if isNil(buffer): + raise newException(IOError, "Linenoise returned nil") result = TaintedString($buffer) if result.string.len > 0: historyAdd(buffer) @@ -114,12 +115,12 @@ else: proc readLineFromStdin*(prompt: string, line: var TaintedString): bool {. tags: [ReadIOEffect, WriteIOEffect].} = var buffer = linenoise.readLine(prompt) - if isNil(buffer): quit(0) + if isNil(buffer): + raise newException(IOError, "Linenoise returned nil") line = TaintedString($buffer) if line.string.len > 0: historyAdd(buffer) linenoise.free(buffer) - # XXX how to determine CTRL+D? result = true proc readPasswordFromStdin*(prompt: string, password: var TaintedString): diff --git a/lib/impure/ssl.nim b/lib/impure/ssl.nim index 721e5ce51..e3312d792 100644 --- a/lib/impure/ssl.nim +++ b/lib/impure/ssl.nim @@ -9,6 +9,9 @@ ## This module provides an easy to use sockets-style ## nim interface to the OpenSSL library. +## +## **Warning:** This module is deprecated, use the SSL procedures defined in +## the ``net`` module instead. {.deprecated.} diff --git a/lib/nimbase.h b/lib/nimbase.h index 5a4f403b6..f531f3c49 100644 --- a/lib/nimbase.h +++ b/lib/nimbase.h @@ -222,6 +222,8 @@ __clang__ /* ----------------------------------------------------------------------- */ +#define COMMA , + #include <limits.h> #include <stddef.h> diff --git a/lib/posix/posix.nim b/lib/posix/posix.nim index 40b48f992..19b068b60 100644 --- a/lib/posix/posix.nim +++ b/lib/posix/posix.nim @@ -35,6 +35,15 @@ const hasSpawnH = not defined(haiku) # should exist for every Posix system nowadays hasAioH = defined(linux) +when defined(linux): + # On Linux: + # timer_{create,delete,settime,gettime}, + # clock_{getcpuclockid, getres, gettime, nanosleep, settime} lives in librt + {.passL: "-lrt".} +when defined(solaris): + # On Solaris hstrerror lives in libresolv + {.passL: "-lresolv".} + when false: const C_IRUSR = 0c000400 ## Read by owner. @@ -486,11 +495,11 @@ type l_onoff*: cint ## Indicates whether linger option is enabled. l_linger*: cint ## Linger time, in seconds. - InPort* = int16 ## unsigned! - InAddrScalar* = int32 ## unsigned! + InPort* = uint16 + InAddrScalar* = uint32 InAddrT* {.importc: "in_addr_t", pure, final, - header: "<netinet/in.h>".} = int32 ## unsigned! + header: "<netinet/in.h>".} = uint32 InAddr* {.importc: "struct in_addr", pure, final, header: "<netinet/in.h>".} = object ## struct in_addr @@ -2309,9 +2318,9 @@ proc strftime*(a1: cstring, a2: int, a3: cstring, a4: var Tm): int {.importc, header: "<time.h>".} proc strptime*(a1, a2: cstring, a3: var Tm): cstring {.importc, header: "<time.h>".} proc time*(a1: var Time): Time {.importc, header: "<time.h>".} -proc timer_create*(a1: var ClockId, a2: var SigEvent, +proc timer_create*(a1: ClockId, a2: var SigEvent, a3: var Timer): cint {.importc, header: "<time.h>".} -proc timer_delete*(a1: var Timer): cint {.importc, header: "<time.h>".} +proc timer_delete*(a1: Timer): cint {.importc, header: "<time.h>".} proc timer_gettime*(a1: Timer, a2: var Itimerspec): cint {. importc, header: "<time.h>".} proc timer_getoverrun*(a1: Timer): cint {.importc, header: "<time.h>".} @@ -2618,3 +2627,17 @@ proc utimes*(path: cstring, times: ptr array [2, Timeval]): int {. ## Returns zero on success. ## ## For more information read http://www.unix.com/man-page/posix/3/utimes/. + +proc handle_signal(sig: cint, handler: proc (a: cint) {.noconv.}) {.importc: "signal", header: "<signal.h>".} + +template onSignal*(signals: varargs[cint], body: untyped): stmt = + ## Setup code to be executed when Unix signals are received. Example: + ## from posix import SIGINT, SIGTERM + ## onSignal(SIGINT, SIGTERM): + ## echo "bye" + + for s in signals: + handle_signal(s, + proc (sig: cint) {.noconv.} = + body + ) diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index cc337452f..01b53cb12 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -1316,6 +1316,23 @@ proc generateExceptionCheck(futSym, ) result.add elseNode +template useVar(result: var NimNode, futureVarNode: NimNode, valueReceiver, + rootReceiver: expr, fromNode: NimNode) = + ## Params: + ## futureVarNode: The NimNode which is a symbol identifying the Future[T] + ## variable to yield. + ## fromNode: Used for better debug information (to give context). + ## valueReceiver: The node which defines an expression that retrieves the + ## future's value. + ## + ## rootReceiver: ??? TODO + # -> yield future<x> + result.add newNimNode(nnkYieldStmt, fromNode).add(futureVarNode) + # -> future<x>.read + valueReceiver = newDotExpr(futureVarNode, newIdentNode("read")) + result.add generateExceptionCheck(futureVarNode, tryStmt, rootReceiver, + fromNode) + template createVar(result: var NimNode, futSymName: string, asyncProc: NimNode, valueReceiver, rootReceiver: expr, @@ -1323,9 +1340,7 @@ template createVar(result: var NimNode, futSymName: string, result = newNimNode(nnkStmtList, fromNode) var futSym = genSym(nskVar, "future") result.add newVarStmt(futSym, asyncProc) # -> var future<x> = y - result.add newNimNode(nnkYieldStmt, fromNode).add(futSym) # -> yield future<x> - valueReceiver = newDotExpr(futSym, newIdentNode("read")) # -> future<x>.read - result.add generateExceptionCheck(futSym, tryStmt, rootReceiver, fromNode) + useVar(result, futSym, valueReceiver, rootReceiver, fromNode) proc processBody(node, retFutureSym: NimNode, subTypeIsVoid: bool, @@ -1352,7 +1367,11 @@ proc processBody(node, retFutureSym: NimNode, case node[1].kind of nnkIdent, nnkInfix: # await x - result = newNimNode(nnkYieldStmt, node).add(node[1]) # -> yield x + result = newNimNode(nnkStmtList, node) + var futureValue: NimNode + result.useVar(node[1], futureValue, futureValue, node) + # -> yield x + # -> x.read() of nnkCall, nnkCommand: # await foo(p, x) var futureValue: NimNode @@ -1550,6 +1569,7 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = for i in 0 .. <result[4].len: if result[4][i].kind == nnkIdent and result[4][i].ident == !"async": result[4].del(i) + result[4] = newEmptyNode() if subtypeIsVoid: # Add discardable pragma. if returnType.kind == nnkEmpty: @@ -1559,7 +1579,7 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = result[6] = outerProcBody #echo(treeRepr(result)) - #if prc[0].getName == "hubConnectionLoop": + #if prc[0].getName == "g": # echo(toStrLit(result)) macro async*(prc: stmt): stmt {.immediate.} = diff --git a/lib/pure/asyncfile.nim b/lib/pure/asyncfile.nim index c7b9fac18..c91d833fc 100644 --- a/lib/pure/asyncfile.nim +++ b/lib/pure/asyncfile.nim @@ -162,7 +162,7 @@ proc read*(f: AsyncFile, size: int): Future[string] = # Request completed immediately. var bytesRead: DWord let overlappedRes = getOverlappedResult(f.fd.Handle, - cast[POverlapped](ol)[], bytesRead, false.WinBool) + cast[POverlapped](ol), bytesRead, false.WinBool) if not overlappedRes.bool: let err = osLastError() if err.int32 == ERROR_HANDLE_EOF: @@ -282,7 +282,7 @@ proc write*(f: AsyncFile, data: string): Future[void] = # Request completed immediately. var bytesWritten: DWord let overlappedRes = getOverlappedResult(f.fd.Handle, - cast[POverlapped](ol)[], bytesWritten, false.WinBool) + cast[POverlapped](ol), bytesWritten, false.WinBool) if not overlappedRes.bool: retFuture.fail(newException(OSError, osErrorMsg(osLastError()))) else: diff --git a/lib/pure/asyncnet.nim b/lib/pure/asyncnet.nim index 6b19a48be..748778566 100644 --- a/lib/pure/asyncnet.nim +++ b/lib/pure/asyncnet.nim @@ -11,7 +11,7 @@ ## asynchronous dispatcher defined in the ``asyncdispatch`` module. ## ## SSL -## --- +## ---- ## ## SSL can be enabled by compiling with the ``-d:ssl`` flag. ## @@ -62,7 +62,9 @@ import os export SOBool -when defined(ssl): +const defineSsl = defined(ssl) or defined(nimdoc) + +when defineSsl: import openssl type @@ -79,7 +81,7 @@ type of false: nil case isSsl: bool of true: - when defined(ssl): + when defineSsl: sslHandle: SslPtr sslContext: SslContext bioIn: BIO @@ -125,7 +127,7 @@ proc newAsyncSocket*(domain, sockType, protocol: cint, Domain(domain), SockType(sockType), Protocol(protocol), buffered) -when defined(ssl): +when defineSsl: proc getSslError(handle: SslPtr, err: cint): cint = assert err < 0 var ret = SSLGetError(handle, err.cint) @@ -186,7 +188,7 @@ proc connect*(socket: AsyncSocket, address: string, port: Port) {.async.} = ## or an error occurs. await connect(socket.fd.AsyncFD, address, port, socket.domain) if socket.isSsl: - when defined(ssl): + when defineSsl: let flags = {SocketFlag.SafeDisconn} sslSetConnectState(socket.sslHandle) sslLoop(socket, flags, sslDoHandshake(socket.sslHandle)) @@ -197,7 +199,7 @@ template readInto(buf: cstring, size: int, socket: AsyncSocket, ## this is a template and not a proc. var res = 0 if socket.isSsl: - when defined(ssl): + when defineSsl: # SSL mode. sslLoop(socket, flags, sslRead(socket.sslHandle, buf, size.cint)) @@ -274,7 +276,7 @@ proc send*(socket: AsyncSocket, data: string, ## data has been sent. assert socket != nil if socket.isSsl: - when defined(ssl): + when defineSsl: var copy = data sslLoop(socket, flags, sslWrite(socket.sslHandle, addr copy[0], copy.len.cint)) @@ -426,9 +428,6 @@ proc recvLine*(socket: AsyncSocket, ## ## **Warning**: ``recvLine`` on unbuffered sockets assumes that the protocol ## uses ``\r\L`` to delimit a new line. - template addNLIfEmpty(): stmt = - if result.len == 0: - result.add("\c\L") assert SocketFlag.Peek notin flags ## TODO: # TODO: Optimise this @@ -456,7 +455,8 @@ proc bindAddr*(socket: AsyncSocket, port = Port(0), address = "") {. of AF_INET6: realaddr = "::" of AF_INET: realaddr = "0.0.0.0" else: - raiseOSError("Unknown socket address family and no address specified to bindAddr") + raise newException(ValueError, + "Unknown socket address family and no address specified to bindAddr") var aiList = getAddrInfo(realaddr, port, socket.domain) if bindAddr(socket.fd, aiList.ai_addr, aiList.ai_addrlen.Socklen) < 0'i32: @@ -468,7 +468,7 @@ proc close*(socket: AsyncSocket) = ## Closes the socket. defer: socket.fd.AsyncFD.closeSocket() - when defined(ssl): + when defineSsl: if socket.isSSL: let res = SslShutdown(socket.sslHandle) SSLFree(socket.sslHandle) @@ -478,7 +478,7 @@ proc close*(socket: AsyncSocket) = raiseSslError() socket.closed = true # TODO: Add extra debugging checks for this. -when defined(ssl): +when defineSsl: proc wrapSocket*(ctx: SslContext, socket: AsyncSocket) = ## Wraps a socket in an SSL context. This function effectively turns ## ``socket`` into an SSL socket. diff --git a/lib/pure/base64.nim b/lib/pure/base64.nim index 32d37ce02..2fc0b1c5e 100644 --- a/lib/pure/base64.nim +++ b/lib/pure/base64.nim @@ -90,6 +90,8 @@ template encodeInternal(s: expr, lineLen: int, newLine: string): stmt {.immediat if r+4 != result.len: setLen(result, r+4) else: + if r != result.len: + setLen(result, r) #assert(r == result.len) discard @@ -162,4 +164,3 @@ when isMainModule: "asure.", longText] for t in items(tests): assert decode(encode(t)) == t - diff --git a/lib/pure/collections/chains.nim b/lib/pure/collections/chains.nim new file mode 100644 index 000000000..6b2ecd272 --- /dev/null +++ b/lib/pure/collections/chains.nim @@ -0,0 +1,44 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2016 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Template based implementation of singly and doubly linked lists. +## The involved types should have 'prev' or 'next' fields and the +## list header should have 'head' or 'tail' fields. + +template prepend*(header, node) = + when compiles(header.head): + when compiles(node.prev): + if header.head != nil: + header.head.prev = node + node.next = header.head + header.head = node + when compiles(header.tail): + if header.tail == nil: + header.tail = node + +template append*(header, node) = + when compiles(header.head): + if header.head == nil: + header.head = node + when compiles(header.tail): + when compiles(node.prev): + node.prev = header.tail + if header.tail != nil: + header.tail.next = node + header.tail = node + +template unlink*(header, node) = + if node.next != nil: + node.next.prev = node.prev + if node.prev != nil: + node.prev.next = node.next + if header.head == node: + header.head = node.prev + if header.tail == node: + header.tail = node.next diff --git a/lib/pure/collections/sharedlist.nim b/lib/pure/collections/sharedlist.nim new file mode 100644 index 000000000..e93ceb02f --- /dev/null +++ b/lib/pure/collections/sharedlist.nim @@ -0,0 +1,95 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2015 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Shared list support. + +{.push stackTrace:off.} + +import + locks + +const + ElemsPerNode = 100 + +type + SharedListNode[A] = ptr object + next: SharedListNode[A] + dataLen: int + d: array[ElemsPerNode, A] + + SharedList*[A] = object ## generic shared list + head, tail: SharedListNode[A] + lock*: Lock + +template withLock(t, x: untyped) = + acquire(t.lock) + x + release(t.lock) + +proc iterAndMutate*[A](x: var SharedList[A]; action: proc(x: A): bool) = + ## iterates over the list. If 'action' returns true, the + ## current item is removed from the list. + withLock(x): + var n = x.head + while n != nil: + var i = 0 + while i < n.dataLen: + # action can add new items at the end, so release the lock: + release(x.lock) + if action(n.d[i]): + acquire(x.lock) + let t = x.tail + n.d[i] = t.d[t.dataLen] + dec t.dataLen + else: + acquire(x.lock) + inc i + n = n.next + +iterator items*[A](x: var SharedList[A]): A = + withLock(x): + var it = x.head + while it != nil: + for i in 0..it.dataLen-1: + yield it.d[i] + it = it.next + +proc add*[A](x: var SharedList[A]; y: A) = + withLock(x): + var node: SharedListNode[A] + if x.tail == nil or x.tail.dataLen == ElemsPerNode: + node = cast[type node](allocShared0(sizeof(node[]))) + node.next = x.tail + x.tail = node + if x.head == nil: x.head = node + else: + node = x.tail + node.d[node.dataLen] = y + inc(node.dataLen) + +proc initSharedList*[A](): SharedList[A] = + initLock result.lock + result.head = nil + result.tail = nil + +proc clear*[A](t: var SharedList[A]) = + withLock(t): + var it = t.head + while it != nil: + let nxt = it.next + deallocShared(it) + it = nxt + t.head = nil + t.tail = nil + +proc deinitSharedList*[A](t: var SharedList[A]) = + clear(t) + deinitLock t.lock + +{.pop.} diff --git a/lib/pure/collections/tables.nim b/lib/pure/collections/tables.nim index 2ed0d2034..58a789e76 100644 --- a/lib/pure/collections/tables.nim +++ b/lib/pure/collections/tables.nim @@ -663,6 +663,27 @@ proc sort*[A, B](t: OrderedTableRef[A, B], ## contrast to the `sort` for count tables). t[].sort(cmp) +proc del*[A, B](t: var OrderedTable[A, B], key: A) = + ## deletes `key` from ordered hash table `t`. O(n) comlexity. + var prev = -1 + let hc = hash(key) + forAllOrderedPairs: + if t.data[h].hcode == hc: + if t.first == h: + t.first = t.data[h].next + else: + t.data[prev].next = t.data[h].next + var zeroValue : type(t.data[h]) + t.data[h] = zeroValue + dec t.counter + break + else: + prev = h + +proc del*[A, B](t: var OrderedTableRef[A, B], key: A) = + ## deletes `key` from ordered hash table `t`. O(n) comlexity. + t[].del(key) + # ------------------------------ count tables ------------------------------- type @@ -984,6 +1005,26 @@ when isMainModule: s3[p1] = 30_000 s3[p2] = 45_000 + block: # Ordered table should preserve order after deletion + var + s4 = initOrderedTable[int, int]() + s4[1] = 1 + s4[2] = 2 + s4[3] = 3 + + var prev = 0 + for i in s4.values: + doAssert(prev < i) + prev = i + + s4.del(2) + doAssert(2 notin s4) + doAssert(s4.len == 2) + prev = 0 + for i in s4.values: + doAssert(prev < i) + prev = i + var t1 = initCountTable[string]() t2 = initCountTable[string]() diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index 603763386..f473e47d1 100644 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -335,7 +335,7 @@ proc addFiles*(p: var MultipartData, xs: openarray[tuple[name, file: string]]): var m = newMimetypes() for name, file in xs.items: var contentType: string - let (dir, fName, ext) = splitFile(file) + let (_, fName, ext) = splitFile(file) if ext.len > 0: contentType = m.getMimetype(ext[1..ext.high], nil) p.add(name, readFile(file), fName & ext, contentType) @@ -627,7 +627,7 @@ proc newAsyncHttpClient*(userAgent = defUserAgent, result.userAgent = defUserAgent result.maxRedirects = maxRedirects when defined(ssl): - result.sslContext = net.SslContext(sslContext) + result.sslContext = sslContext proc close*(client: AsyncHttpClient) = ## Closes any connections held by the HTTP client. diff --git a/lib/pure/json.nim b/lib/pure/json.nim index 30004da84..b9da8a0dd 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -56,6 +56,11 @@ import export tables.`$` +when defined(nimJsonGet): + {.pragma: deprecatedGet, deprecated.} +else: + {.pragma: deprecatedGet.} + type JsonEventKind* = enum ## enumeration of all events that may occur when parsing jsonError, ## an error occurred during parsing @@ -787,7 +792,7 @@ proc hash*(n: JsonNode): Hash = proc hash*(n: Table[string, JsonNode]): Hash = for key, val in n: - result = result !& hash(key) !& hash(val) + result = result xor (hash(key) !& hash(val)) result = !$result proc len*(n: JsonNode): int = @@ -799,16 +804,23 @@ proc len*(n: JsonNode): int = of JObject: result = n.fields.len else: discard -proc `[]`*(node: JsonNode, name: string): JsonNode {.inline.} = +proc `[]`*(node: JsonNode, name: string): JsonNode {.inline, deprecatedGet.} = ## Gets a field from a `JObject`, which must not be nil. - ## If the value at `name` does not exist, returns nil + ## If the value at `name` does not exist, raises KeyError. + ## + ## **Note:** The behaviour of this procedure changed in version 0.14.0. To + ## get a list of usages and to restore the old behaviour of this procedure, + ## compile with the ``-d:nimJsonGet`` flag. assert(not isNil(node)) assert(node.kind == JObject) - result = node.fields.getOrDefault(name) + when defined(nimJsonGet): + if not node.fields.hasKey(name): return nil + result = node.fields[name] proc `[]`*(node: JsonNode, index: int): JsonNode {.inline.} = ## Gets the node at `index` in an Array. Result is undefined if `index` - ## is out of bounds + ## is out of bounds, but as long as array bound checks are enabled it will + ## result in an exception. assert(not isNil(node)) assert(node.kind == JArray) return node.elems[index] @@ -838,20 +850,28 @@ proc `[]=`*(obj: JsonNode, key: string, val: JsonNode) {.inline.} = proc `{}`*(node: JsonNode, keys: varargs[string]): JsonNode = ## Traverses the node and gets the given value. If any of the - ## keys do not exist, returns nil. Also returns nil if one of the - ## intermediate data structures is not an object + ## keys do not exist, returns ``nil``. Also returns ``nil`` if one of the + ## intermediate data structures is not an object. result = node for key in keys: - if isNil(result) or result.kind!=JObject: + if isNil(result) or result.kind != JObject: return nil - result=result[key] + result = result.fields.getOrDefault(key) + +proc getOrDefault*(node: JsonNode, key: string): JsonNode = + ## Gets a field from a `node`. If `node` is nil or not an object or + ## value at `key` does not exist, returns nil + if not isNil(node) and node.kind == JObject: + result = node.fields.getOrDefault(key) + +template simpleGetOrDefault*{`{}`(node, [key])}(node: JsonNode, key: string): JsonNode = node.getOrDefault(key) proc `{}=`*(node: JsonNode, keys: varargs[string], value: JsonNode) = ## Traverses the node and tries to set the value at the given location - ## to `value` If any of the keys are missing, they are added + ## to ``value``. If any of the keys are missing, they are added. var node = node for i in 0..(keys.len-2): - if isNil(node[keys[i]]): + if not node.hasKey(keys[i]): node[keys[i]] = newJObject() node = node[keys[i]] node[keys[keys.len-1]] = value @@ -1217,16 +1237,6 @@ when false: # To get that we shall use, obj["json"] when isMainModule: - when not defined(js): - var parsed = parseFile("tests/testdata/jsontest.json") - - try: - discard parsed["key2"][12123] - doAssert(false) - except IndexError: doAssert(true) - - var parsed2 = parseFile("tests/testdata/jsontest2.json") - doAssert(parsed2{"repository", "description"}.str=="IRC Library for Haskell", "Couldn't fetch via multiply nested key using {}") let testJson = parseJson"""{ "a": [1, 2, 3, 4], "b": "asd", "c": "\ud83c\udf83", "d": "\u00E6"}""" # nil passthrough @@ -1314,3 +1324,19 @@ when isMainModule: var j4 = %*{"test": nil} doAssert j4 == %{"test": newJNull()} + + echo("99% of tests finished. Going to try loading file.") + + # Test loading of file. + when not defined(js): + var parsed = parseFile("tests/testdata/jsontest.json") + + try: + discard parsed["key2"][12123] + doAssert(false) + except IndexError: doAssert(true) + + var parsed2 = parseFile("tests/testdata/jsontest2.json") + doAssert(parsed2{"repository", "description"}.str=="IRC Library for Haskell", "Couldn't fetch via multiply nested key using {}") + + echo("Tests succeeded!") diff --git a/lib/pure/matchers.nim b/lib/pure/matchers.nim index 5c28f65a0..7022c21d9 100644 --- a/lib/pure/matchers.nim +++ b/lib/pure/matchers.nim @@ -8,6 +8,10 @@ # ## This module contains various string matchers for email addresses, etc. +## +## **Warning:** This module is deprecated since version 0.14.0. +{.deprecated.} + {.deadCodeElim: on.} {.push debugger:off .} # the user does not want to trace a part diff --git a/lib/pure/mersenne.nim b/lib/pure/mersenne.nim index ae0845714..afa343086 100644 --- a/lib/pure/mersenne.nim +++ b/lib/pure/mersenne.nim @@ -5,29 +5,31 @@ type {.deprecated: [TMersenneTwister: MersenneTwister].} -proc newMersenneTwister*(seed: int): MersenneTwister = +proc newMersenneTwister*(seed: uint32): MersenneTwister = result.index = 0 - result.mt[0]= uint32(seed) + result.mt[0] = seed for i in 1..623'u32: - result.mt[i]= (0x6c078965'u32 * (result.mt[i-1] xor (result.mt[i-1] shr 30'u32)) + i) + result.mt[i] = (0x6c078965'u32 * (result.mt[i-1] xor (result.mt[i-1] shr 30'u32)) + i) proc generateNumbers(m: var MersenneTwister) = for i in 0..623: - var y = (m.mt[i] and 0x80000000'u32) + (m.mt[(i+1) mod 624] and 0x7fffffff'u32) + var y = (m.mt[i] and 0x80000000'u32) + + (m.mt[(i+1) mod 624] and 0x7fffffff'u32) m.mt[i] = m.mt[(i+397) mod 624] xor uint32(y shr 1'u32) if (y mod 2'u32) != 0: - m.mt[i] = m.mt[i] xor 0x9908b0df'u32 + m.mt[i] = m.mt[i] xor 0x9908b0df'u32 -proc getNum*(m: var MersenneTwister): int = +proc getNum*(m: var MersenneTwister): uint32 = + ## Returns the next pseudo random number ranging from 0 to high(uint32) if m.index == 0: generateNumbers(m) - var y = m.mt[m.index] - y = y xor (y shr 11'u32) - y = y xor ((7'u32 shl y) and 0x9d2c5680'u32) - y = y xor ((15'u32 shl y) and 0xefc60000'u32) - y = y xor (y shr 18'u32) - m.index = (m.index+1) mod 624 - return int(y) + result = m.mt[m.index] + m.index = (m.index + 1) mod m.mt.len + + result = result xor (result shr 11'u32) + result = result xor ((7'u32 shl result) and 0x9d2c5680'u32) + result = result xor ((15'u32 shl result) and 0xefc60000'u32) + result = result xor (result shr 18'u32) # Test when not defined(testing) and isMainModule: diff --git a/lib/pure/nativesockets.nim b/lib/pure/nativesockets.nim index 3951b46a3..043d6d80a 100644 --- a/lib/pure/nativesockets.nim +++ b/lib/pure/nativesockets.nim @@ -219,31 +219,67 @@ proc getAddrInfo*(address: string, port: Port, domain: Domain = AF_INET, proc dealloc*(ai: ptr AddrInfo) = freeaddrinfo(ai) -proc ntohl*(x: int32): int32 = - ## Converts 32-bit integers from network to host byte order. +proc ntohl*(x: uint32): uint32 = + ## Converts 32-bit unsigned integers from network to host byte order. ## On machines where the host byte order is the same as network byte order, ## this is a no-op; otherwise, it performs a 4-byte swap operation. when cpuEndian == bigEndian: result = x - else: result = (x shr 24'i32) or - (x shr 8'i32 and 0xff00'i32) or - (x shl 8'i32 and 0xff0000'i32) or - (x shl 24'i32) + else: result = (x shr 24'u32) or + (x shr 8'u32 and 0xff00'u32) or + (x shl 8'u32 and 0xff0000'u32) or + (x shl 24'u32) -proc ntohs*(x: int16): int16 = - ## Converts 16-bit integers from network to host byte order. On machines - ## where the host byte order is the same as network byte order, this is - ## a no-op; otherwise, it performs a 2-byte swap operation. +template ntohl*(x: int32): expr {.deprecated.} = + ## Converts 32-bit integers from network to host byte order. + ## On machines where the host byte order is the same as network byte order, + ## this is a no-op; otherwise, it performs a 4-byte swap operation. + ## **Warning**: This template is deprecated since 0.14.0, IPv4 + ## addresses are now treated as unsigned integers. Please use the unsigned + ## version of this template. + cast[int32](ntohl(cast[uint32](x))) + +proc ntohs*(x: uint16): uint16 = + ## Converts 16-bit unsigned integers from network to host byte order. On + ## machines where the host byte order is the same as network byte order, + ## this is a no-op; otherwise, it performs a 2-byte swap operation. when cpuEndian == bigEndian: result = x - else: result = (x shr 8'i16) or (x shl 8'i16) - -template htonl*(x: int32): expr = + else: result = (x shr 8'u16) or (x shl 8'u16) + +template ntohs*(x: int16): expr {.deprecated.} = + ## Converts 16-bit integers from network to host byte order. On + ## machines where the host byte order is the same as network byte order, + ## this is a no-op; otherwise, it performs a 2-byte swap operation. + ## **Warning**: This template is deprecated since 0.14.0, where port + ## numbers became unsigned integers. Please use the unsigned version of + ## this template. + cast[int16](ntohs(cast[uint16](x))) + +template htonl*(x: int32): expr {.deprecated.} = ## Converts 32-bit integers from host to network byte order. On machines ## where the host byte order is the same as network byte order, this is ## a no-op; otherwise, it performs a 4-byte swap operation. + ## **Warning**: This template is deprecated since 0.14.0, IPv4 + ## addresses are now treated as unsigned integers. Please use the unsigned + ## version of this template. nativesockets.ntohl(x) -template htons*(x: int16): expr = - ## Converts 16-bit positive integers from host to network byte order. +template htonl*(x: uint32): expr = + ## Converts 32-bit unsigned integers from host to network byte order. On + ## machines where the host byte order is the same as network byte order, + ## this is a no-op; otherwise, it performs a 4-byte swap operation. + nativesockets.ntohl(x) + +template htons*(x: int16): expr {.deprecated.} = + ## Converts 16-bit integers from host to network byte order. + ## On machines where the host byte order is the same as network byte + ## order, this is a no-op; otherwise, it performs a 2-byte swap operation. + ## **Warning**: This template is deprecated since 0.14.0, where port + ## numbers became unsigned integers. Please use the unsigned version of + ## this template. + nativesockets.ntohs(x) + +template htons*(x: uint16): expr = + ## Converts 16-bit unsigned integers from host to network byte order. ## On machines where the host byte order is the same as network byte ## order, this is a no-op; otherwise, it performs a 2-byte swap operation. nativesockets.ntohs(x) diff --git a/lib/pure/net.nim b/lib/pure/net.nim index b9764edd3..5de6667dd 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -8,19 +8,76 @@ # ## This module implements a high-level cross-platform sockets interface. +## The procedures implemented in this module are primarily for blocking sockets. +## For asynchronous non-blocking sockets use the ``asyncnet`` module together +## with the ``asyncdispatch`` module. +## +## The first thing you will always need to do in order to start using sockets, +## is to create a new instance of the ``Socket`` type using the ``newSocket`` +## procedure. +## +## SSL +## ==== +## +## In order to use the SSL procedures defined in this module, you will need to +## compile your application with the ``-d:ssl`` flag. +## +## Examples +## ======== +## +## Connecting to a server +## ---------------------- +## +## After you create a socket with the ``newSocket`` procedure, you can easily +## connect it to a server running at a known hostname (or IP address) and port. +## To do so over TCP, use the example below. +## +## .. code-block:: Nim +## var socket = newSocket() +## socket.connect("google.com", Port(80)) +## +## UDP is a connectionless protocol, so UDP sockets don't have to explicitly +## call the ``connect`` procedure. They can simply start sending data +## immediately. +## +## .. code-block:: Nim +## var socket = newSocket() +## socket.sendTo("192.168.0.1", Port(27960), "status\n") +## +## Creating a server +## ----------------- +## +## After you create a socket with the ``newSocket`` procedure, you can create a +## TCP server by calling the ``bindAddr`` and ``listen`` procedures. +## +## .. code-block:: Nim +## var socket = newSocket() +## socket.bindAddr(Port(1234)) +## socket.listen() +## +## You can then begin accepting connections using the ``accept`` procedure. +## +## .. code-block:: Nim +## var client = newSocket() +## var address = "" +## while true: +## socket.acceptAddr(client, address) +## echo("Client connected from: ", address) +## {.deadCodeElim: on.} import nativesockets, os, strutils, parseutils, times export Port, `$`, `==` const useWinVersion = defined(Windows) or defined(nimdoc) +const defineSsl = defined(ssl) or defined(nimdoc) -when defined(ssl): +when defineSsl: import openssl # Note: The enumerations are mapped to Window's constants. -when defined(ssl): +when defineSsl: type SslError* = object of Exception @@ -54,7 +111,7 @@ type currPos: int # current index in buffer bufLen: int # current length of buffer of false: nil - when defined(ssl): + when defineSsl: case isSsl: bool of true: sslHandle: SSLPtr @@ -160,7 +217,7 @@ proc newSocket*(domain: Domain = AF_INET, sockType: SockType = SOCK_STREAM, raiseOSError(osLastError()) result = newSocket(fd, domain, sockType, protocol, buffered) -when defined(ssl): +when defineSsl: CRYPTO_malloc_init() SslLibraryInit() SslLoadErrorStrings() @@ -222,9 +279,9 @@ when defined(ssl): of protSSLv23: newCTX = SSL_CTX_new(SSLv23_method()) # SSlv2,3 and TLS1 support. of protSSLv2: - raiseSslError("SSLv2 is no longer secure and has been deprecated, use protSSLv3") + raiseSslError("SSLv2 is no longer secure and has been deprecated, use protSSLv23") of protSSLv3: - newCTX = SSL_CTX_new(SSLv3_method()) + raiseSslError("SSLv3 is no longer secure and has been deprecated, use protSSLv23") of protTLSv1: newCTX = SSL_CTX_new(TLSv1_method()) @@ -301,7 +358,7 @@ proc socketError*(socket: Socket, err: int = -1, async = false, ## error was caused by no data being available to be read. ## ## If ``err`` is not lower than 0 no exception will be raised. - when defined(ssl): + when defineSsl: if socket.isSSL: if err <= 0: var ret = SSLGetError(socket.sslHandle, err.cint) @@ -334,7 +391,7 @@ proc socketError*(socket: Socket, err: int = -1, async = false, raiseSSLError() else: raiseSSLError("Unknown Error") - if err == -1 and not (when defined(ssl): socket.isSSL else: false): + if err == -1 and not (when defineSsl: socket.isSSL else: false): var lastE = if lastError.int == -1: getSocketError(socket) else: lastError if async: when useWinVersion: @@ -368,7 +425,7 @@ proc bindAddr*(socket: Socket, port = Port(0), address = "") {. name.sin_family = toInt(AF_INET).int16 else: name.sin_family = toInt(AF_INET) - name.sin_port = htons(int16(port)) + name.sin_port = htons(port.uint16) name.sin_addr.s_addr = htonl(INADDR_ANY) if bindAddr(socket.fd, cast[ptr SockAddr](addr(name)), sizeof(name).SockLen) < 0'i32: @@ -414,7 +471,7 @@ proc acceptAddr*(server: Socket, client: var Socket, address: var string, client.isBuffered = server.isBuffered # Handle SSL. - when defined(ssl): + when defineSsl: if server.isSSL: # We must wrap the client sock in a ssl context. @@ -425,7 +482,7 @@ proc acceptAddr*(server: Socket, client: var Socket, address: var string, # Client socket is set above. address = $inet_ntoa(sockAddress.sin_addr) -when false: #defined(ssl): +when false: #defineSsl: proc acceptAddrSSL*(server: Socket, client: var Socket, address: var string): SSLAcceptResult {. tags: [ReadIOEffect].} = @@ -444,7 +501,7 @@ when false: #defined(ssl): ## ``AcceptNoClient`` will be returned when no client is currently attempting ## to connect. template doHandshake(): stmt = - when defined(ssl): + when defineSsl: if server.isSSL: client.setBlocking(false) # We must wrap the client sock in a ssl context. @@ -495,7 +552,7 @@ proc accept*(server: Socket, client: var Socket, proc close*(socket: Socket) = ## Closes a socket. try: - when defined(ssl): + when defineSsl: if socket.isSSL: ErrClearError() # As we are closing the underlying socket immediately afterwards, @@ -547,8 +604,9 @@ proc setSockOpt*(socket: Socket, opt: SOBool, value: bool, level = SOL_SOCKET) { var valuei = cint(if value: 1 else: 0) setSockOptInt(socket.fd, cint(level), toCInt(opt), valuei) -when defined(ssl): - proc handshake*(socket: Socket): bool {.tags: [ReadIOEffect, WriteIOEffect].} = +when defineSsl: + proc handshake*(socket: Socket): bool + {.tags: [ReadIOEffect, WriteIOEffect], deprecated.} = ## This proc needs to be called on a socket after it connects. This is ## only applicable when using ``connectAsync``. ## This proc performs the SSL handshake. @@ -557,6 +615,8 @@ when defined(ssl): ## ``True`` whenever handshake completed successfully. ## ## A ESSL error is raised on any other errors. + ## + ## **Note:** This procedure is deprecated since version 0.14.0. result = true if socket.isSSL: var ret = SSLConnect(socket.sslHandle) @@ -594,7 +654,7 @@ proc hasDataBuffered*(s: Socket): bool = if s.isBuffered: result = s.bufLen > 0 and s.currPos != s.bufLen - when defined(ssl): + when defineSsl: if s.isSSL and not result: result = s.sslHasPeekChar @@ -608,7 +668,7 @@ proc select(readfd: Socket, timeout = 500): int = proc readIntoBuf(socket: Socket, flags: int32): int = result = 0 - when defined(ssl): + when defineSsl: if socket.isSSL: result = SSLRead(socket.sslHandle, addr(socket.buffer), int(socket.buffer.high)) else: @@ -658,7 +718,7 @@ proc recv*(socket: Socket, data: pointer, size: int): int {.tags: [ReadIOEffect] result = read else: - when defined(ssl): + when defineSsl: if socket.isSSL: if socket.sslHasPeekChar: copyMem(data, addr(socket.sslPeekChar), 1) @@ -696,7 +756,7 @@ proc waitFor(socket: Socket, waited: var float, timeout, size: int, if timeout - int(waited * 1000.0) < 1: raise newException(TimeoutError, "Call to '" & funcName & "' timed out.") - when defined(ssl): + when defineSsl: if socket.isSSL: if socket.hasDataBuffered: # sslPeekChar is present. @@ -764,7 +824,7 @@ proc peekChar(socket: Socket, c: var char): int {.tags: [ReadIOEffect].} = c = socket.buffer[socket.currPos] else: - when defined(ssl): + when defineSsl: if socket.isSSL: if not socket.sslHasPeekChar: result = SSLRead(socket.sslHandle, addr(socket.sslPeekChar), 1) @@ -872,7 +932,7 @@ proc send*(socket: Socket, data: pointer, size: int): int {. ## ## **Note**: This is a low-level version of ``send``. You likely should use ## the version below. - when defined(ssl): + when defineSsl: if socket.isSSL: return SSLWrite(socket.sslHandle, cast[cstring](data), size) @@ -943,7 +1003,7 @@ proc sendTo*(socket: Socket, address: string, port: Port, proc isSsl*(socket: Socket): bool = ## Determines whether ``socket`` is a SSL socket. - when defined(ssl): + when defineSsl: result = socket.isSSL else: result = false @@ -1253,7 +1313,7 @@ proc connect*(socket: Socket, address: string, dealloc(aiList) if not success: raiseOSError(lastError) - when defined(ssl): + when defineSsl: if socket.isSSL: # RFC3546 for SNI specifies that IP addresses are not allowed. if not isIpAddress(address): @@ -1314,8 +1374,10 @@ proc connect*(socket: Socket, address: string, port = Port(0), if selectWrite(s, timeout) != 1: raise newException(TimeoutError, "Call to 'connect' timed out.") else: - when defined(ssl): + when defineSsl: if socket.isSSL: socket.fd.setBlocking(true) + {.warning[Deprecated]: off.} doAssert socket.handshake() + {.warning[Deprecated]: on.} socket.fd.setBlocking(true) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 470559e17..dee227c69 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -129,7 +129,7 @@ proc raiseOSError*(errorCode: OSErrorCode; additionalInfo = "") {.noinline.} = if additionalInfo.len == 0: e.msg = osErrorMsg(errorCode) else: - e.msg = additionalInfo & " " & osErrorMsg(errorCode) + e.msg = osErrorMsg(errorCode) & "\nAdditional info: " & additionalInfo if e.msg == "": e.msg = "unknown OS error" raise e @@ -1452,7 +1452,7 @@ template rawToFormalFileInfo(rawInfo, formalInfo): expr = ## 'rawInfo' is either a 'TBY_HANDLE_FILE_INFORMATION' structure on Windows, ## or a 'Stat' structure on posix when defined(Windows): - template toTime(e): expr = winTimeToUnixTime(rdFileTime(e)) + template toTime(e: FILETIME): expr {.gensym.} = winTimeToUnixTime(rdFileTime(e)) # local templates default to bind semantics template merge(a, b): expr = a or (b shl 32) formalInfo.id.device = rawInfo.dwVolumeSerialNumber formalInfo.id.file = merge(rawInfo.nFileIndexLow, rawInfo.nFileIndexHigh) diff --git a/lib/pure/parseutils.nim b/lib/pure/parseutils.nim index 698bde42a..3e25eba22 100644 --- a/lib/pure/parseutils.nim +++ b/lib/pure/parseutils.nim @@ -234,6 +234,47 @@ proc parseInt*(s: string, number: var int, start = 0): int {. elif result != 0: number = int(res) +# overflowChecks doesn't work with uint64 +proc rawParseUInt(s: string, b: var uint64, start = 0): int = + var + res = 0'u64 + prev = 0'u64 + i = start + if s[i] == '+': inc(i) # Allow + if s[i] in {'0'..'9'}: + b = 0 + while s[i] in {'0'..'9'}: + prev = res + res = res * 10 + (ord(s[i]) - ord('0')).uint64 + if prev > res: + return 0 # overflowChecks emulation + inc(i) + while s[i] == '_': inc(i) # underscores are allowed and ignored + b = res + result = i - start + +proc parseBiggestUInt*(s: string, number: var uint64, start = 0): int {. + rtl, extern: "npuParseBiggestUInt", noSideEffect.} = + ## parses an unsigned integer starting at `start` and stores the value into `number`. + ## Result is the number of processed chars or 0 if there is no integer or overflow detected. + var res: uint64 + # use 'res' for exception safety (don't write to 'number' in case of an + # overflow exception): + result = rawParseUInt(s, res, start) + number = res + +proc parseUInt*(s: string, number: var uint, start = 0): int {. + rtl, extern: "npuParseUInt", noSideEffect.} = + ## parses an unsigned integer starting at `start` and stores the value into `number`. + ## Result is the number of processed chars or 0 if there is no integer or overflow detected. + var res: uint64 + result = parseBiggestUInt(s, res, start) + if (sizeof(uint) <= 4) and + (res > 0xFFFF_FFFF'u64): + raise newException(OverflowError, "overflow") + elif result != 0: + number = uint(res) + proc parseBiggestFloat*(s: string, number: var BiggestFloat, start = 0): int {. magic: "ParseBiggestFloat", importc: "nimParseBiggestFloat", noSideEffect.} ## parses a float starting at `start` and stores the value into `number`. diff --git a/lib/pure/smtp.nim b/lib/pure/smtp.nim index b2adac2f3..050712902 100644 --- a/lib/pure/smtp.nim +++ b/lib/pure/smtp.nim @@ -20,9 +20,9 @@ ## var msg = createMessage("Hello from Nim's SMTP", ## "Hello!.\n Is this awesome or what?", ## @["foo@gmail.com"]) -## var smtp = connect("smtp.gmail.com", 465, true, true) -## smtp.auth("username", "password") -## smtp.sendmail("username@gmail.com", @["foo@gmail.com"], $msg) +## var smtpConn = connect("smtp.gmail.com", Port 465, true, true) +## smtpConn.auth("username", "password") +## smtpConn.sendmail("username@gmail.com", @["foo@gmail.com"], $msg) ## ## ## For SSL support this module relies on OpenSSL. If you want to @@ -31,6 +31,8 @@ import net, strutils, strtabs, base64, os import asyncnet, asyncdispatch +export Port + type Smtp* = object sock: Socket @@ -258,8 +260,8 @@ when not defined(testing) and isMainModule: # "Hello, my name is dom96.\n What\'s yours?", @["dominik@localhost"]) #echo(msg) - #var smtp = connect("localhost", 25, False, True) - #smtp.sendmail("root@localhost", @["dominik@localhost"], $msg) + #var smtpConn = connect("localhost", Port 25, false, true) + #smtpConn.sendmail("root@localhost", @["dominik@localhost"], $msg) #echo(decode("a17sm3701420wbe.12")) proc main() {.async.} = diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index f2c1e77e1..a5a4ee509 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -560,6 +560,24 @@ proc parseBiggestInt*(s: string): BiggestInt {.noSideEffect, procvar, if L != s.len or L == 0: raise newException(ValueError, "invalid integer: " & s) +proc parseUInt*(s: string): uint {.noSideEffect, procvar, + rtl, extern: "nsuParseUInt".} = + ## Parses a decimal unsigned integer value contained in `s`. + ## + ## If `s` is not a valid integer, `ValueError` is raised. + var L = parseutils.parseUInt(s, result, 0) + if L != s.len or L == 0: + raise newException(ValueError, "invalid unsigned integer: " & s) + +proc parseBiggestUInt*(s: string): uint64 {.noSideEffect, procvar, + rtl, extern: "nsuParseBiggestUInt".} = + ## Parses a decimal unsigned integer value contained in `s`. + ## + ## If `s` is not a valid integer, `ValueError` is raised. + var L = parseutils.parseBiggestUInt(s, result, 0) + if L != s.len or L == 0: + raise newException(ValueError, "invalid unsigned integer: " & s) + proc parseFloat*(s: string): float {.noSideEffect, procvar, rtl, extern: "nsuParseFloat".} = ## Parses a decimal floating point value contained in `s`. If `s` is not @@ -766,8 +784,7 @@ proc indent*(s: string, count: Natural, padding: string = " "): string {.noSideEffect, rtl, extern: "nsuIndent".} = ## Indents each line in ``s`` by ``count`` amount of ``padding``. ## - ## **Note:** This currently does not preserve the specific new line characters - ## used. + ## **Note:** This does not preserve the new line characters used in ``s``. result = "" var i = 0 for line in s.splitLines(): @@ -778,32 +795,39 @@ proc indent*(s: string, count: Natural, padding: string = " "): string result.add(line) i.inc -proc unindent*(s: string, eatAllIndent = false): string {. - noSideEffect, rtl, extern: "nsuUnindent".} = - ## Unindents `s`. - result = newStringOfCap(s.len) +proc unindent*(s: string, count: Natural, padding: string = " "): string + {.noSideEffect, rtl, extern: "nsuUnindent".} = + ## Unindents each line in ``s`` by ``count`` amount of ``padding``. + ## + ## **Note:** This does not preserve the new line characters used in ``s``. + result = "" var i = 0 - var pattern = true - var indent = 0 - while s[i] == ' ': inc i - var level = if i == 0: -1 else: i - while i < s.len: - if s[i] == ' ': - if i > 0 and s[i-1] in {'\l', '\c'}: - pattern = true - indent = 0 - if pattern: - inc(indent) - if indent > level and not eatAllIndent: - result.add(s[i]) - if level < 0: level = indent - else: - # a space somewhere: do not delete - result.add(s[i]) - else: - pattern = false - result.add(s[i]) - inc i + for line in s.splitLines(): + if i != 0: + result.add("\n") + var indentCount = 0 + for j in 0..<count.int: + indentCount.inc + if line[j .. j + <padding.len] != padding: + indentCount = j + break + result.add(line[indentCount*padding.len .. ^1]) + i.inc + +proc unindent*(s: string): string + {.noSideEffect, rtl, extern: "nsuUnindentAll".} = + ## Removes all indentation composed of whitespace from each line in ``s``. + ## + ## For example: + ## + ## .. code-block:: nim + ## const x = """ + ## Hello + ## There + ## """.unindent() + ## + ## doAssert x == "Hello\nThere\n" + unindent(s, 1000) # TODO: Passing a 1000 is a bit hackish. proc startsWith*(s, prefix: string): bool {.noSideEffect, rtl, extern: "nsuStartsWith".} = @@ -1725,3 +1749,33 @@ when isMainModule: doAssert join(@["foo", "bar", "baz"], ", ") == "foo, bar, baz" doAssert join([1, 2, 3]) == "123" doAssert join(@[1, 2, 3], ", ") == "1, 2, 3" + + doAssert """~~!!foo +~~!!bar +~~!!baz""".unindent(2, "~~!!") == "foo\nbar\nbaz" + + doAssert """~~!!foo +~~!!bar +~~!!baz""".unindent(2, "~~!!aa") == "~~!!foo\n~~!!bar\n~~!!baz" + doAssert """~~foo +~~ bar +~~ baz""".unindent(4, "~") == "foo\n bar\n baz" + doAssert """foo +bar + baz + """.unindent(4) == "foo\nbar\nbaz\n" + doAssert """foo + bar + baz + """.unindent(2) == "foo\n bar\n baz\n" + doAssert """foo + bar + baz + """.unindent(100) == "foo\nbar\nbaz\n" + + doAssert """foo + foo + bar + """.unindent() == "foo\nfoo\nbar\n" + + echo("strutils tests passed") diff --git a/lib/pure/subexes.nim b/lib/pure/subexes.nim index 5824ace81..22f29b77c 100644 --- a/lib/pure/subexes.nim +++ b/lib/pure/subexes.nim @@ -390,11 +390,11 @@ when isMainModule: doAssert(("type MyEnum* = enum\n $', '2i'\n '{..}" % ["fieldA", "fieldB", "FiledClkad", "fieldD", "fieldE", "longishFieldName"]).replace(" \n", "\n") == - strutils.unindent """ + strutils.unindent(""" type MyEnum* = enum fieldA, fieldB, FiledClkad, fieldD, - fieldE, longishFieldName""") + fieldE, longishFieldName""", 6)) doAssert subex"$1($', '{2..})" % ["f", "a", "b", "c"] == "f(a, b, c)" @@ -404,8 +404,8 @@ when isMainModule: doAssert((subex("type\n Enum = enum\n $', '40c'\n '{..}") % [ "fieldNameA", "fieldNameB", "fieldNameC", "fieldNameD"]).replace(" \n", "\n") == - strutils.unindent """ + strutils.unindent(""" type Enum = enum fieldNameA, fieldNameB, fieldNameC, - fieldNameD""") + fieldNameD""", 6)) diff --git a/lib/pure/times.nim b/lib/pure/times.nim index 29ae52d47..e0ee884a8 100644 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -182,7 +182,16 @@ proc getGMTime*(t: Time): TimeInfo {.tags: [TimeEffect], raises: [], benign.} ## converts the calendar time `t` to broken-down time representation, ## expressed in Coordinated Universal Time (UTC). -proc timeInfoToTime*(timeInfo: TimeInfo): Time {.tags: [], benign.} +proc timeInfoToTime*(timeInfo: TimeInfo): Time {.tags: [], benign, deprecated.} + ## converts a broken-down time structure to + ## calendar time representation. The function ignores the specified + ## contents of the structure members `weekday` and `yearday` and recomputes + ## them from the other information in the broken-down time structure. + ## + ## **Warning:** This procedure is deprecated since version 0.14.0. + ## Use ``toTime`` instead. + +proc toTime*(timeInfo: TimeInfo): Time {.tags: [], benign.} ## converts a broken-down time structure to ## calendar time representation. The function ignores the specified ## contents of the structure members `weekday` and `yearday` and recomputes @@ -356,7 +365,7 @@ proc `+`*(a: TimeInfo, interval: TimeInterval): TimeInfo = ## ## **Note:** This has been only briefly tested and it may not be ## very accurate. - let t = toSeconds(timeInfoToTime(a)) + let t = toSeconds(toTime(a)) let secs = toSeconds(a, interval) result = getLocalTime(fromSeconds(t + secs)) @@ -365,7 +374,7 @@ proc `-`*(a: TimeInfo, interval: TimeInterval): TimeInfo = ## ## **Note:** This has been only briefly tested, it is inaccurate especially ## when you subtract so much that you reach the Julian calendar. - let t = toSeconds(timeInfoToTime(a)) + let t = toSeconds(toTime(a)) var intval: TimeInterval intval.milliseconds = - interval.milliseconds intval.seconds = - interval.seconds @@ -517,6 +526,11 @@ when not defined(JS): # because the header of mktime is broken in my version of libc return mktime(timeInfoToTM(cTimeInfo)) + proc toTime(timeInfo: TimeInfo): Time = + var cTimeInfo = timeInfo # for C++ we have to make a copy, + # because the header of mktime is broken in my version of libc + return mktime(timeInfoToTM(cTimeInfo)) + proc toStringTillNL(p: cstring): string = result = "" var i = 0 @@ -618,7 +632,16 @@ elif defined(JS): result.setFullYear(timeInfo.year) result.setDate(timeInfo.monthday) - proc `$`(timeInfo: TimeInfo): string = return $(timeInfoToTime(timeInfo)) + proc toTime*(timeInfo: TimeInfo): Time = + result = internGetTime() + result.setSeconds(timeInfo.second) + result.setMinutes(timeInfo.minute) + result.setHours(timeInfo.hour) + result.setMonth(ord(timeInfo.month)) + result.setFullYear(timeInfo.year) + result.setDate(timeInfo.monthday) + + proc `$`(timeInfo: TimeInfo): string = return $(toTime(timeInfo)) proc `$`(time: Time): string = return $time.toLocaleString() proc `-` (a, b: Time): int64 = @@ -708,24 +731,24 @@ proc years*(y: int): TimeInterval {.inline.} = proc `+=`*(t: var Time, ti: TimeInterval) = ## modifies `t` by adding the interval `ti` - t = timeInfoToTime(getLocalTime(t) + ti) + t = toTime(getLocalTime(t) + ti) proc `+`*(t: Time, ti: TimeInterval): Time = ## adds the interval `ti` to Time `t` ## by converting to localTime, adding the interval, and converting back ## ## ``echo getTime() + 1.day`` - result = timeInfoToTime(getLocalTime(t) + ti) + result = toTime(getLocalTime(t) + ti) proc `-=`*(t: var Time, ti: TimeInterval) = ## modifies `t` by subtracting the interval `ti` - t = timeInfoToTime(getLocalTime(t) - ti) + t = toTime(getLocalTime(t) - ti) proc `-`*(t: Time, ti: TimeInterval): Time = ## adds the interval `ti` to Time `t` ## ## ``echo getTime() - 1.day`` - result = timeInfoToTime(getLocalTime(t) - ti) + result = toTime(getLocalTime(t) - ti) proc formatToken(info: TimeInfo, token: string, buf: var string) = ## Helper of the format proc to parse individual tokens. @@ -1193,7 +1216,7 @@ proc parse*(value, layout: string): TimeInfo = parseToken(info, token, value, j) token = "" # Reset weekday as it might not have been provided and the default may be wrong - info.weekday = getLocalTime(timeInfoToTime(info)).weekday + info.weekday = getLocalTime(toTime(info)).weekday return info # Leap year calculations are adapted from: @@ -1204,7 +1227,7 @@ proc parse*(value, layout: string): TimeInfo = proc countLeapYears*(yearSpan: int): int = ## Returns the number of leap years spanned by a given number of years. ## - ## Note: for leap years, start date is assumed to be 1 AD. + ## **Note:** For leap years, start date is assumed to be 1 AD. ## counts the number of leap years up to January 1st of a given year. ## Keep in mind that if specified year is a leap year, the leap day ## has not happened before January 1st of that year. @@ -1239,13 +1262,14 @@ proc getDayOfWeek*(day, month, year: int): WeekDay = y = year - a m = month + (12*a) - 2 d = (day + y + (y div 4) - (y div 100) + (y div 400) + (31*m) div 12) mod 7 - # The value of d is 0 for a Sunday, 1 for a Monday, 2 for a Tuesday, etc. so we must correct - # for the WeekDay type. + # The value of d is 0 for a Sunday, 1 for a Monday, 2 for a Tuesday, etc. + # so we must correct for the WeekDay type. if d == 0: return dSun result = (d-1).WeekDay proc getDayOfWeekJulian*(day, month, year: int): WeekDay = - ## Returns the day of the week enum from day, month and year, according to the Julian calendar. + ## Returns the day of the week enum from day, month and year, + ## according to the Julian calendar. # Day & month start from one. let a = (14 - month) div 12 @@ -1254,8 +1278,11 @@ proc getDayOfWeekJulian*(day, month, year: int): WeekDay = d = (5 + day + y + (y div 4) + (31*m) div 12) mod 7 result = d.WeekDay -proc timeToTimeInfo*(t: Time): TimeInfo = +proc timeToTimeInfo*(t: Time): TimeInfo {.deprecated.} = ## Converts a Time to TimeInfo. + ## + ## **Warning:** This procedure is deprecated since version 0.14.0. + ## Use ``getLocalTime`` or ``getGMTime`` instead. let secs = t.toSeconds().int daysSinceEpoch = secs div secondsInDay @@ -1286,34 +1313,21 @@ proc timeToTimeInfo*(t: Time): TimeInfo = s = daySeconds mod secondsInMin result = TimeInfo(year: y, yearday: yd, month: m, monthday: md, weekday: wd, hour: h, minute: mi, second: s) -proc timeToTimeInterval*(t: Time): TimeInterval = +proc timeToTimeInterval*(t: Time): TimeInterval {.deprecated.} = ## Converts a Time to a TimeInterval. + ## + ## **Warning:** This procedure is deprecated since version 0.14.0. + ## Use ``toTimeInterval`` instead. + # Milliseconds not available from Time + var tInfo = t.getLocalTime() + initInterval(0, tInfo.second, tInfo.minute, tInfo.hour, tInfo.weekday.ord, tInfo.month.ord, tInfo.year) - let - secs = t.toSeconds().int - daysSinceEpoch = secs div secondsInDay - (yearsSinceEpoch, daysRemaining) = countYearsAndDays(daysSinceEpoch) - daySeconds = secs mod secondsInDay - - result.years = yearsSinceEpoch + epochStartYear - - var - mon = mJan - days = daysRemaining - daysInMonth = getDaysInMonth(mon, result.years) - - # calculate month and day remainder - while days > daysInMonth and mon <= mDec: - days -= daysInMonth - mon.inc - daysInMonth = getDaysInMonth(mon, result.years) - - result.months = mon.int + 1 # month is 1 indexed int - result.days = days - result.hours = daySeconds div secondsInHour + 1 - result.minutes = (daySeconds mod secondsInHour) div secondsInMin - result.seconds = daySeconds mod secondsInMin +proc toTimeInterval*(t: Time): TimeInterval = + ## Converts a Time to a TimeInterval. # Milliseconds not available from Time + var tInfo = t.getLocalTime() + initInterval(0, tInfo.second, tInfo.minute, tInfo.hour, tInfo.weekday.ord, tInfo.month.ord, tInfo.year) + when isMainModule: # this is testing non-exported function diff --git a/lib/pure/unittest.nim b/lib/pure/unittest.nim index aca9d51e2..aca9d51e2 100755..100644 --- a/lib/pure/unittest.nim +++ b/lib/pure/unittest.nim diff --git a/lib/system.nim b/lib/system.nim index fefabe53f..5d6fcf868 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -1291,7 +1291,7 @@ const when hasThreadSupport and defined(tcc) and not compileOption("tlsEmulation"): # tcc doesn't support TLS {.error: "``--tlsEmulation:on`` must be used when using threads with tcc backend".} - + when defined(boehmgc): when defined(windows): const boehmLib = "boehmgc.dll" @@ -2565,8 +2565,9 @@ when not defined(JS): #and not defined(nimscript): {.push stack_trace: off, profiler:off.} when not defined(nimscript) and not defined(nogc): - proc initGC() - when not defined(boehmgc) and not defined(useMalloc) and not defined(gogc): + when not defined(gcStack): + proc initGC() + when not defined(boehmgc) and not defined(useMalloc) and not defined(gogc) and not defined(gcStack): proc initAllocator() {.inline.} proc initStackBottom() {.inline, compilerproc.} = @@ -2774,6 +2775,9 @@ when not defined(JS): #and not defined(nimscript): ## reads `len` bytes into the buffer `a` starting at ``a[start]``. Returns ## the actual number of bytes that have been read which may be less than ## `len` (if not as many bytes are remaining), but not greater. + ## + ## **Warning:** The buffer `a` must be pre-allocated. This can be done + ## using, for example, ``newString``. proc readBuffer*(f: File, buffer: pointer, len: Natural): int {. tags: [ReadIOEffect], benign.} @@ -2874,6 +2878,7 @@ when not defined(JS): #and not defined(nimscript): when declared(initAllocator): initAllocator() when hasThreadSupport: + const insideRLocksModule = false include "system/syslocks" when hostOS != "standalone": include "system/threads" elif not defined(nogc) and not defined(nimscript): diff --git a/lib/system/alloc.nim b/lib/system/alloc.nim index c0b697238..e0fd53b7b 100644 --- a/lib/system/alloc.nim +++ b/lib/system/alloc.nim @@ -13,155 +13,7 @@ # - make searching for block O(1) {.push profiler:off.} -proc roundup(x, v: int): int {.inline.} = - result = (x + (v-1)) and not (v-1) - sysAssert(result >= x, "roundup: result < x") - #return ((-x) and (v-1)) +% x - -sysAssert(roundup(14, PageSize) == PageSize, "invalid PageSize") -sysAssert(roundup(15, 8) == 16, "roundup broken") -sysAssert(roundup(65, 8) == 72, "roundup broken 2") - -# ------------ platform specific chunk allocation code ----------------------- - -# some platforms have really weird unmap behaviour: unmap(blockStart, PageSize) -# really frees the whole block. Happens for Linux/PowerPC for example. Amd64 -# and x86 are safe though; Windows is special because MEM_RELEASE can only be -# used with a size of 0. We also allow unmapping to be turned off with -# -d:nimAllocNoUnmap: -const doNotUnmap = not (defined(amd64) or defined(i386)) or - defined(windows) or defined(nimAllocNoUnmap) - - -when defined(emscripten): - const - PROT_READ = 1 # page can be read - PROT_WRITE = 2 # page can be written - MAP_PRIVATE = 2'i32 # Changes are private - - var MAP_ANONYMOUS {.importc: "MAP_ANONYMOUS", header: "<sys/mman.h>".}: cint - type - PEmscriptenMMapBlock = ptr EmscriptenMMapBlock - EmscriptenMMapBlock {.pure, inheritable.} = object - realSize: int # size of previous chunk; for coalescing - realPointer: pointer # if < PageSize it is a small chunk - - proc mmap(adr: pointer, len: int, prot, flags, fildes: cint, - off: int): pointer {.header: "<sys/mman.h>".} - - proc munmap(adr: pointer, len: int): cint {.header: "<sys/mman.h>".} - - proc osAllocPages(block_size: int): pointer {.inline.} = - let realSize = block_size + sizeof(EmscriptenMMapBlock) + PageSize + 1 - result = mmap(nil, realSize, PROT_READ or PROT_WRITE, - MAP_PRIVATE or MAP_ANONYMOUS, -1, 0) - if result == nil or result == cast[pointer](-1): - raiseOutOfMem() - - let realPointer = result - let pos = cast[int](result) - - # Convert pointer to PageSize correct one. - var new_pos = cast[ByteAddress](pos) +% (PageSize - (pos %% PageSize)) - if (new_pos-pos)< sizeof(EmscriptenMMapBlock): - new_pos = new_pos +% PageSize - result = cast[pointer](new_pos) - - var mmapDescrPos = cast[ByteAddress](result) -% sizeof(EmscriptenMMapBlock) - - var mmapDescr = cast[EmscriptenMMapBlock](mmapDescrPos) - mmapDescr.realSize = realSize - mmapDescr.realPointer = realPointer - - c_fprintf(c_stdout, "[Alloc] size %d %d realSize:%d realPos:%d\n", block_size, cast[int](result), realSize, cast[int](realPointer)) - - proc osDeallocPages(p: pointer, size: int) {.inline} = - var mmapDescrPos = cast[ByteAddress](p) -% sizeof(EmscriptenMMapBlock) - var mmapDescr = cast[EmscriptenMMapBlock](mmapDescrPos) - discard munmap(mmapDescr.realPointer, mmapDescr.realSize) - -elif defined(posix): - const - PROT_READ = 1 # page can be read - PROT_WRITE = 2 # page can be written - MAP_PRIVATE = 2'i32 # Changes are private - - when defined(macosx) or defined(bsd): - const MAP_ANONYMOUS = 0x1000 - elif defined(solaris): - const MAP_ANONYMOUS = 0x100 - elif defined(linux): - const MAP_ANONYMOUS = 0x20'i32 - else: - var - MAP_ANONYMOUS {.importc: "MAP_ANONYMOUS", header: "<sys/mman.h>".}: cint - - proc mmap(adr: pointer, len: int, prot, flags, fildes: cint, - off: int): pointer {.header: "<sys/mman.h>".} - - proc munmap(adr: pointer, len: int): cint {.header: "<sys/mman.h>".} - - proc osAllocPages(size: int): pointer {.inline.} = - result = mmap(nil, size, PROT_READ or PROT_WRITE, - MAP_PRIVATE or MAP_ANONYMOUS, -1, 0) - if result == nil or result == cast[pointer](-1): - raiseOutOfMem() - - proc osDeallocPages(p: pointer, size: int) {.inline} = - when reallyOsDealloc: discard munmap(p, size) - -elif defined(windows): - const - MEM_RESERVE = 0x2000 - MEM_COMMIT = 0x1000 - MEM_TOP_DOWN = 0x100000 - PAGE_READWRITE = 0x04 - - MEM_DECOMMIT = 0x4000 - MEM_RELEASE = 0x8000 - - proc virtualAlloc(lpAddress: pointer, dwSize: int, flAllocationType, - flProtect: int32): pointer {. - header: "<windows.h>", stdcall, importc: "VirtualAlloc".} - - proc virtualFree(lpAddress: pointer, dwSize: int, - dwFreeType: int32) {.header: "<windows.h>", stdcall, - importc: "VirtualFree".} - - proc osAllocPages(size: int): pointer {.inline.} = - result = virtualAlloc(nil, size, MEM_RESERVE or MEM_COMMIT, - PAGE_READWRITE) - if result == nil: raiseOutOfMem() - - proc osDeallocPages(p: pointer, size: int) {.inline.} = - # according to Microsoft, 0 is the only correct value for MEM_RELEASE: - # This means that the OS has some different view over how big the block is - # that we want to free! So, we cannot reliably release the memory back to - # Windows :-(. We have to live with MEM_DECOMMIT instead. - # Well that used to be the case but MEM_DECOMMIT fragments the address - # space heavily, so we now treat Windows as a strange unmap target. - when reallyOsDealloc: virtualFree(p, 0, MEM_RELEASE) - #VirtualFree(p, size, MEM_DECOMMIT) - -elif hostOS == "standalone": - var - theHeap: array[1024*PageSize, float64] # 'float64' for alignment - bumpPointer = cast[int](addr theHeap) - - proc osAllocPages(size: int): pointer {.inline.} = - if size+bumpPointer < cast[int](addr theHeap) + sizeof(theHeap): - result = cast[pointer](bumpPointer) - inc bumpPointer, size - else: - raiseOutOfMem() - - proc osDeallocPages(p: pointer, size: int) {.inline.} = - if bumpPointer-size == cast[int](p): - dec bumpPointer, size -else: - {.error: "Port memory manager to your platform".} - -# --------------------- end of non-portable code ----------------------------- +include osalloc # We manage *chunks* of memory. Each chunk is a multiple of the page size. # Each chunk starts at an address that is divisible by the page size. Chunks diff --git a/lib/system/dyncalls.nim b/lib/system/dyncalls.nim index 6dc8999d1..61777e514 100644 --- a/lib/system/dyncalls.nim +++ b/lib/system/dyncalls.nim @@ -23,16 +23,16 @@ proc rawWrite(f: File, s: string) = proc nimLoadLibraryError(path: string) = # carefully written to avoid memory allocation: - stdout.rawWrite("could not load: ") - stdout.rawWrite(path) - stdout.rawWrite("\n") + stderr.rawWrite("could not load: ") + stderr.rawWrite(path) + stderr.rawWrite("\n") quit(1) proc procAddrError(name: cstring) {.noinline.} = # carefully written to avoid memory allocation: - stdout.rawWrite("could not import: ") - stdout.write(name) - stdout.rawWrite("\n") + stderr.rawWrite("could not import: ") + stderr.write(name) + stderr.rawWrite("\n") quit(1) # this code was inspired from Lua's source code: @@ -71,7 +71,7 @@ when defined(posix): when defined(nimDebugDlOpen): let error = dlerror() if error != nil: - c_fprintf(c_stdout, "%s\n", error) + c_fprintf(c_stderr, "%s\n", error) proc nimGetProcAddr(lib: LibHandle, name: cstring): ProcAddr = result = dlsym(lib, name) @@ -109,9 +109,31 @@ elif defined(windows) or defined(dos): proc nimGetProcAddr(lib: LibHandle, name: cstring): ProcAddr = result = getProcAddress(cast[THINSTANCE](lib), name) if result != nil: return + const decorated_length = 250 + var decorated: array[decorated_length, char] + decorated[0] = '_' + var m = 1 + while m < (decorated_length - 5): + if name[m - 1] == '\x00': break + decorated[m] = name[m - 1] + inc(m) + decorated[m] = '@' for i in countup(0, 50): - var decorated = "_" & $name & "@" & $(i * 4) - result = getProcAddress(cast[THINSTANCE](lib), cstring(decorated)) + var k = i * 4 + if k div 100 == 0: + if k div 10 == 0: + m = m + 1 + else: + m = m + 2 + else: + m = m + 3 + decorated[m + 1] = '\x00' + while true: + decorated[m] = chr(ord('0') + (k %% 10)) + dec(m) + k = k div 10 + if k == 0: break + result = getProcAddress(cast[THINSTANCE](lib), decorated) if result != nil: return procAddrError(name) diff --git a/lib/system/gc.nim b/lib/system/gc.nim index d8390ca14..4f461b5c3 100644 --- a/lib/system/gc.nim +++ b/lib/system/gc.nim @@ -39,6 +39,9 @@ when withRealTime and not declared(getTicks): when defined(memProfiler): proc nimProfile(requestedSize: int) {.benign.} +when hasThreadSupport: + import sharedlist + const rcIncrement = 0b1000 # so that lowest 3 bits are not touched rcBlack = 0b000 # cell is colored black; in use or free @@ -93,6 +96,9 @@ type stat: GcStat when useMarkForDebug or useBackupGc: marked: CellSet + when hasThreadSupport: + toDispose: SharedList[pointer] + {.deprecated: [TWalkOp: WalkOp, TFinalizer: Finalizer, TGcHeap: GcHeap, TGcStat: GcStat].} var @@ -304,6 +310,8 @@ proc initGC() = init(gch.decStack) when useMarkForDebug or useBackupGc: init(gch.marked) + when hasThreadSupport: + gch.toDispose = initSharedList[pointer]() when useMarkForDebug or useBackupGc: type @@ -749,6 +757,9 @@ proc collectRoots(gch: var GcHeap) = collectWhite(s) proc collectCycles(gch: var GcHeap) = + when hasThreadSupport: + for c in gch.toDispose: + nimGCunref(c) # ensure the ZCT 'color' is not used: while gch.zct.len > 0: discard collectZCT(gch) when useBackupGc: diff --git a/lib/system/gc_common.nim b/lib/system/gc_common.nim index a4676d26e..4807bb6f8 100644 --- a/lib/system/gc_common.nim +++ b/lib/system/gc_common.nim @@ -7,6 +7,31 @@ # distribution, for details about the copyright. # +type + ForeignCell* = object + data*: pointer + owner: ptr GcHeap + +proc protect*(x: pointer): ForeignCell = + nimGCref(x) + result.data = x + result.owner = addr(gch) + +proc dispose*(x: ForeignCell) = + when hasThreadSupport: + # if we own it we can free it directly: + if x.owner == addr(gch): + nimGCunref(x.data) + else: + x.owner.toDispose.add(x.data) + else: + nimGCunref(x.data) + +proc isNotForeign*(x: ForeignCell): bool = + ## returns true if 'x' belongs to the calling thread. + ## No deep copy has to be performed then. + x.owner == addr(gch) + proc len(stack: ptr GcStack): int = if stack == nil: return 0 diff --git a/lib/system/gc_ms.nim b/lib/system/gc_ms.nim index d1aecb7a2..ec69f6e5f 100644 --- a/lib/system/gc_ms.nim +++ b/lib/system/gc_ms.nim @@ -28,6 +28,9 @@ template mulThreshold(x): expr {.immediate.} = x * 2 when defined(memProfiler): proc nimProfile(requestedSize: int) + +when hasThreadSupport: + import sharedlist type WalkOp = enum @@ -68,6 +71,8 @@ type recGcLock: int # prevent recursion via finalizers; no thread lock region: MemRegion # garbage collected region stat: GcStat + when hasThreadSupport: + toDispose: SharedList[pointer] additionalRoots: CellSeq # dummy roots for GC_ref/unref {.deprecated: [TWalkOp: WalkOp, TFinalizer: Finalizer, TGcStat: GcStat, TGlobalMarkerProc: GlobalMarkerProc, TGcHeap: GcHeap].} @@ -179,6 +184,8 @@ proc initGC() = when withBitvectors: init(gch.allocated) init(gch.marked) + when hasThreadSupport: + gch.toDispose = initSharedList[pointer]() proc forAllSlotsAux(dest: pointer, n: ptr TNimNode, op: WalkOp) {.benign.} = var d = cast[ByteAddress](dest) @@ -321,6 +328,9 @@ proc growObj(old: pointer, newsize: int): pointer {.rtl.} = # ----------------- collector ----------------------------------------------- proc mark(gch: var GcHeap, c: PCell) = + when hasThreadSupport: + for c in gch.toDispose: + nimGCunref(c) when withBitvectors: incl(gch.marked, c) gcAssert gch.tempStack.len == 0, "stack not empty!" diff --git a/lib/system/gc_stack.nim b/lib/system/gc_stack.nim new file mode 100644 index 000000000..c251a4d0b --- /dev/null +++ b/lib/system/gc_stack.nim @@ -0,0 +1,439 @@ +# +# Nim's Runtime Library +# (c) Copyright 2016 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# "Stack GC" for embedded devices or ultra performance requirements. + +include osalloc + +# We manage memory as a thread local stack. Since the allocation pointer +# is detached from the control flow pointer, this model is vastly more +# useful than the traditional programming model while almost as safe. +# Individual objects can also be deleted but no coalescing is performed. +# Stacks can also be moved from one thread to another. + +# We also support 'finalizers'. + +type + Finalizer {.compilerproc.} = proc (self: pointer) {.nimcall, benign.} + # A ref type can have a finalizer that is called before the object's + # storage is freed. + + AlignType = BiggestFloat + ObjHeader = object + typ: PNimType + nextFinal: ptr ObjHeader # next object with finalizer + + Hole = object # stacks can have holes. Otherwise 'growObj' would be insane. + zeroTyp: pointer # overlaid with 'typ' field. Always 'nil'. + size: int # size of the free slot + + Chunk = ptr BaseChunk + BaseChunk = object + next: Chunk + size: int + head, tail: ptr ObjHeader # first and last object in chunk that + # has a finalizer attached to it + +type + StackPtr = object + bump: pointer + remaining: int + current: Chunk + + MemRegion* = object + remaining: int + bump: pointer + head, tail: Chunk + nextChunkSize, totalSize: int + hole: ptr Hole # we support individual freeing + when hasThreadSupport: + lock: SysLock + +var + tlRegion {.threadVar.}: MemRegion + +template withRegion*(r: MemRegion; body: untyped) = + let oldRegion = tlRegion + tlRegion = r + try: + body + finally: + tlRegion = oldRegion + +template inc(p: pointer, s: int) = + p = cast[pointer](cast[int](p) +% s) + +template `+!`(p: pointer, s: int): pointer = + cast[pointer](cast[int](p) +% s) + +template `-!`(p: pointer, s: int): pointer = + cast[pointer](cast[int](p) -% s) + +proc allocSlowPath(r: var MemRegion; size: int) = + # we need to ensure that the underlying linked list + # stays small. Say we want to grab 16GB of RAM with some + # exponential growth function. So we allocate 16KB, then + # 32 KB, 64 KB, 128KB, 256KB, 512KB, 1MB, 2MB, 4MB, + # 8MB, 16MB, 32MB, 64MB, 128MB, 512MB, 1GB, 2GB, 4GB, 8GB, + # 16GB --> list contains only 20 elements! That's reasonable. + if (r.totalSize and 1) == 0: + r.nextChunkSize = + if r.totalSize < 64 * 1024: PageSize*4 + else: r.nextChunkSize*2 + var s = roundup(size+sizeof(BaseChunk), PageSize) + var fresh: Chunk + if s > r.nextChunkSize: + fresh = cast[Chunk](osAllocPages(s)) + else: + fresh = cast[Chunk](osTryAllocPages(r.nextChunkSize)) + if fresh == nil: + fresh = cast[Chunk](osAllocPages(s)) + # lowest bit in totalSize is the "don't increase nextChunkSize" + inc r.totalSize + else: + s = r.nextChunkSize + fresh.size = s + fresh.head = nil + fresh.tail = nil + inc r.totalSize, s + let old = r.tail + if old == nil: + r.head = fresh + else: + r.tail.next = fresh + r.bump = fresh +! sizeof(BaseChunk) + r.tail = fresh + r.remaining = s - sizeof(BaseChunk) + +proc alloc(r: var MemRegion; size: int): pointer {.inline.} = + if size > r.remaining: + allocSlowPath(r, size) + sysAssert(size <= r.remaining, "size <= r.remaining") + dec(r.remaining, size) + result = r.bump + inc r.bump, size + +proc runFinalizers(c: Chunk) = + var it = c.head + while it != nil: + # indivually freed objects with finalizer stay in the list, but + # their typ is nil then: + if it.typ != nil and it.typ.finalizer != nil: + (cast[Finalizer](it.typ.finalizer))(it+!sizeof(ObjHeader)) + it = it.nextFinal + +proc dealloc(r: var MemRegion; p: pointer) = + let it = cast[ptr ObjHeader](p-!sizeof(ObjHeader)) + if it.typ != nil and it.typ.finalizer != nil: + (cast[Finalizer](it.typ.finalizer))(p) + it.typ = nil + +proc deallocAll(r: var MemRegion; head: Chunk) = + var it = head + while it != nil: + let nxt = it.next + runFinalizers(it) + dec r.totalSize, it.size + osDeallocPages(it, it.size) + it = nxt + +proc deallocAll*(r: var MemRegion) = + deallocAll(r, r.head) + zeroMem(addr r, sizeof r) + +proc obstackPtr*(r: MemRegion): StackPtr = + result.bump = r.bump + result.remaining = r.remaining + result.current = r.tail + +template computeRemaining(r): untyped = + r.tail.size -% (cast[int](r.bump) -% cast[int](r.tail)) + +proc setObstackPtr*(r: var MemRegion; sp: StackPtr) = + # free everything after 'sp': + if sp.current != nil: + deallocAll(r, sp.current.next) + sp.current.next = nil + else: + deallocAll(r, r.head) + r.head = nil + r.bump = sp.bump + r.tail = sp.current + r.remaining = sp.remaining + +proc obstackPtr*(): StackPtr = tlRegion.obstackPtr() +proc setObstackPtr*(sp: StackPtr) = tlRegion.setObstackPtr(sp) + +proc joinRegion*(dest: var MemRegion; src: MemRegion) = + # merging is not hard. + if dest.head.isNil: + dest.head = src.head + else: + dest.tail.next = src.head + dest.tail = src.tail + dest.bump = src.bump + dest.remaining = src.remaining + dest.nextChunkSize = max(dest.nextChunkSize, src.nextChunkSize) + inc dest.totalSize, src.totalSize + +proc isOnHeap*(r: MemRegion; p: pointer): bool = + # the tail chunk is the largest, so check it first. It's also special + # in that contains the current bump pointer: + if r.tail >= p and p < r.bump: + return true + var it = r.head + while it != r.tail: + if it >= p and p <= it+!it.size: return true + it = it.next + +when false: + # essential feature for later: copy data over from one region to another + + proc isInteriorPointer(r: MemRegion; p: pointer): pointer = + discard " we cannot patch stack pointers anyway!" + + type + PointerStackChunk = object + next, prev: ptr PointerStackChunk + len: int + data: array[128, pointer] + + template head(s: PointerStackChunk): untyped = s.prev + template tail(s: PointerStackChunk): untyped = s.next + + include chains + + proc push(r: var MemRegion; s: var PointerStackChunk; x: pointer) = + if s.len < high(s.data): + s.data[s.len] = x + inc s.len + else: + let fresh = cast[ptr PointerStackChunk](alloc(r, sizeof(PointerStackChunk))) + fresh.len = 1 + fresh.data[0] = x + fresh.next = nil + fresh.prev = nil + append(s, fresh) + + + proc genericDeepCopyAux(dr: var MemRegion; stack: var PointerStackChunk; + dest, src: pointer, mt: PNimType) {.benign.} + proc genericDeepCopyAux(dr: var MemRegion; stack: var PointerStackChunk; + dest, src: pointer, n: ptr TNimNode) {.benign.} = + var + d = cast[ByteAddress](dest) + s = cast[ByteAddress](src) + case n.kind + of nkSlot: + genericDeepCopyAux(cast[pointer](d +% n.offset), + cast[pointer](s +% n.offset), n.typ) + of nkList: + for i in 0..n.len-1: + genericDeepCopyAux(dest, src, n.sons[i]) + of nkCase: + var dd = selectBranch(dest, n) + var m = selectBranch(src, n) + # reset if different branches are in use; note different branches also + # imply that's not self-assignment (``x = x``)! + if m != dd and dd != nil: + genericResetAux(dest, dd) + copyMem(cast[pointer](d +% n.offset), cast[pointer](s +% n.offset), + n.typ.size) + if m != nil: + genericDeepCopyAux(dest, src, m) + of nkNone: sysAssert(false, "genericDeepCopyAux") + + proc copyDeepString(dr: var MemRegion; stack: var PointerStackChunk; src: NimString): NimString {.inline.} = + result = rawNewStringNoInit(dr, src.len) + result.len = src.len + c_memcpy(result.data, src.data, src.len + 1) + + proc genericDeepCopyAux(dr: var MemRegion; stack: var PointerStackChunk; + dest, src: pointer, mt: PNimType) = + var + d = cast[ByteAddress](dest) + s = cast[ByteAddress](src) + sysAssert(mt != nil, "genericDeepCopyAux 2") + case mt.kind + of tyString: + var x = cast[PPointer](dest) + var s2 = cast[PPointer](s)[] + if s2 == nil: + x[] = nil + else: + x[] = copyDeepString(cast[NimString](s2)) + of tySequence: + var s2 = cast[PPointer](src)[] + var seq = cast[PGenericSeq](s2) + var x = cast[PPointer](dest) + if s2 == nil: + x[] = nil + return + sysAssert(dest != nil, "genericDeepCopyAux 3") + x[] = newSeq(mt, seq.len) + var dst = cast[ByteAddress](cast[PPointer](dest)[]) + for i in 0..seq.len-1: + genericDeepCopyAux(dr, stack, + cast[pointer](dst +% i*% mt.base.size +% GenericSeqSize), + cast[pointer](cast[ByteAddress](s2) +% i *% mt.base.size +% + GenericSeqSize), + mt.base) + of tyObject: + # we need to copy m_type field for tyObject, as it could be empty for + # sequence reallocations: + var pint = cast[ptr PNimType](dest) + pint[] = cast[ptr PNimType](src)[] + if mt.base != nil: + genericDeepCopyAux(dr, stack, dest, src, mt.base) + genericDeepCopyAux(dr, stack, dest, src, mt.node) + of tyTuple: + genericDeepCopyAux(dr, stack, dest, src, mt.node) + of tyArray, tyArrayConstr: + for i in 0..(mt.size div mt.base.size)-1: + genericDeepCopyAux(dr, stack, + cast[pointer](d +% i*% mt.base.size), + cast[pointer](s +% i*% mt.base.size), mt.base) + of tyRef: + let s2 = cast[PPointer](src)[] + if s2 == nil: + cast[PPointer](dest)[] = nil + else: + # we modify the header of the cell temporarily; instead of the type + # field we store a forwarding pointer. XXX This is bad when the cloning + # fails due to OOM etc. + let x = usrToCell(s2) + let forw = cast[int](x.typ) + if (forw and 1) == 1: + # we stored a forwarding pointer, so let's use that: + let z = cast[pointer](forw and not 1) + unsureAsgnRef(cast[PPointer](dest), z) + else: + let realType = x.typ + let z = newObj(realType, realType.base.size) + + unsureAsgnRef(cast[PPointer](dest), z) + x.typ = cast[PNimType](cast[int](z) or 1) + genericDeepCopyAux(dr, stack, z, s2, realType.base) + x.typ = realType + else: + copyMem(dest, src, mt.size) + + proc joinAliveDataFromRegion*(dest: var MemRegion; src: var MemRegion; + root: pointer): pointer = + # we mark the alive data and copy only alive data over to 'dest'. + # This is O(liveset) but it nicely compacts memory, so it's fine. + # We use the 'typ' field as a forwarding pointer. The forwarding + # pointers have bit 0 set, so we can disambiguate them. + # We allocate a temporary stack in 'src' that we later free: + var s: PointerStackChunk + s.len = 1 + s.data[0] = root + while s.len > 0: + var p: pointer + if s.tail == nil: + p = s.data[s.len-1] + dec s.len + else: + p = s.tail.data[s.tail.len-1] + dec s.tail.len + if s.tail.len == 0: + unlink(s, s.tail) + +proc rawNewObj(r: var MemRegion, typ: PNimType, size: int): pointer = + var res = cast[ptr ObjHeader](alloc(r, size + sizeof(ObjHeader))) + res.typ = typ + if typ.finalizer != nil: + res.nextFinal = r.head.head + r.head.head = res + result = res +! sizeof(ObjHeader) + +proc newObj(typ: PNimType, size: int): pointer {.compilerRtl.} = + result = rawNewObj(tlRegion, typ, size) + zeroMem(result, size) + when defined(memProfiler): nimProfile(size) + +proc newObjNoInit(typ: PNimType, size: int): pointer {.compilerRtl.} = + result = rawNewObj(tlRegion, typ, size) + when defined(memProfiler): nimProfile(size) + +proc newSeq(typ: PNimType, len: int): pointer {.compilerRtl.} = + let size = addInt(mulInt(len, typ.base.size), GenericSeqSize) + result = newObj(typ, size) + cast[PGenericSeq](result).len = len + cast[PGenericSeq](result).reserved = len + +proc newObjRC1(typ: PNimType, size: int): pointer {.compilerRtl.} = + result = rawNewObj(tlRegion, typ, size) + zeroMem(result, size) + +proc newSeqRC1(typ: PNimType, len: int): pointer {.compilerRtl.} = + let size = addInt(mulInt(len, typ.base.size), GenericSeqSize) + result = newObj(typ, size) + cast[PGenericSeq](result).len = len + cast[PGenericSeq](result).reserved = len + +proc growObj(region: var MemRegion; old: pointer, newsize: int): pointer = + let typ = cast[ptr ObjHeader](old -! sizeof(ObjHeader)).typ + result = rawNewObj(region, typ, newsize) + let elemSize = if typ.kind == tyString: 1 else: typ.base.size + let oldsize = cast[PGenericSeq](old).len*elemSize + GenericSeqSize + copyMem(result, old, oldsize) + zeroMem(result +! oldsize, newsize-oldsize) + +proc growObj(old: pointer, newsize: int): pointer {.rtl.} = + result = growObj(tlRegion, old, newsize) + +proc unsureAsgnRef(dest: PPointer, src: pointer) {.compilerproc, inline.} = + dest[] = src +proc asgnRef(dest: PPointer, src: pointer) {.compilerproc, inline.} = + dest[] = src +proc asgnRefNoCycle(dest: PPointer, src: pointer) {.compilerproc, inline.} = + dest[] = src + +proc alloc(size: Natural): pointer = + result = cmalloc(size) + if result == nil: raiseOutOfMem() +proc alloc0(size: Natural): pointer = + result = alloc(size) + zeroMem(result, size) +proc realloc(p: pointer, newsize: Natural): pointer = + result = crealloc(p, newsize) + if result == nil: raiseOutOfMem() +proc dealloc(p: pointer) = cfree(p) + +proc allocShared(size: Natural): pointer = + result = cmalloc(size) + if result == nil: raiseOutOfMem() +proc allocShared0(size: Natural): pointer = + result = alloc(size) + zeroMem(result, size) +proc reallocShared(p: pointer, newsize: Natural): pointer = + result = crealloc(p, newsize) + if result == nil: raiseOutOfMem() +proc deallocShared(p: pointer) = cfree(p) + +when hasThreadSupport: + proc getFreeSharedMem(): int = 0 + proc getTotalSharedMem(): int = 0 + proc getOccupiedSharedMem(): int = 0 + +proc GC_disable() = discard +proc GC_enable() = discard +proc GC_fullCollect() = discard +proc GC_setStrategy(strategy: GC_Strategy) = discard +proc GC_enableMarkAndSweep() = discard +proc GC_disableMarkAndSweep() = discard +proc GC_getStatistics(): string = return "" + +proc getOccupiedMem(): int = + result = tlRegion.totalSize - tlRegion.remaining +proc getFreeMem(): int = tlRegion.remaining +proc getTotalMem(): int = + result = tlRegion.totalSize + +proc setStackBottom(theStackBottom: pointer) = discard diff --git a/lib/system/mmdisp.nim b/lib/system/mmdisp.nim index 76c1b476e..d7010a1a3 100644 --- a/lib/system/mmdisp.nim +++ b/lib/system/mmdisp.nim @@ -526,13 +526,17 @@ elif defined(nogc): include "system/cellsets" else: - include "system/alloc" + when not defined(gcStack): + include "system/alloc" - include "system/cellsets" - when not leakDetector and not useCellIds: - sysAssert(sizeof(Cell) == sizeof(FreeCell), "sizeof FreeCell") + include "system/cellsets" + when not leakDetector and not useCellIds: + sysAssert(sizeof(Cell) == sizeof(FreeCell), "sizeof FreeCell") when compileOption("gc", "v2"): include "system/gc2" + elif defined(gcStack): + # XXX due to bootstrapping reasons, we cannot use compileOption("gc", "stack") here + include "system/gc_stack" elif defined(gcMarkAndSweep): # XXX use 'compileOption' here include "system/gc_ms" diff --git a/lib/system/osalloc.nim b/lib/system/osalloc.nim new file mode 100644 index 000000000..78410d716 --- /dev/null +++ b/lib/system/osalloc.nim @@ -0,0 +1,171 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2016 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +proc roundup(x, v: int): int {.inline.} = + result = (x + (v-1)) and not (v-1) + sysAssert(result >= x, "roundup: result < x") + #return ((-x) and (v-1)) +% x + +sysAssert(roundup(14, PageSize) == PageSize, "invalid PageSize") +sysAssert(roundup(15, 8) == 16, "roundup broken") +sysAssert(roundup(65, 8) == 72, "roundup broken 2") + +# ------------ platform specific chunk allocation code ----------- + +# some platforms have really weird unmap behaviour: +# unmap(blockStart, PageSize) +# really frees the whole block. Happens for Linux/PowerPC for example. Amd64 +# and x86 are safe though; Windows is special because MEM_RELEASE can only be +# used with a size of 0. We also allow unmapping to be turned off with +# -d:nimAllocNoUnmap: +const doNotUnmap = not (defined(amd64) or defined(i386)) or + defined(windows) or defined(nimAllocNoUnmap) + + +when defined(emscripten): + const + PROT_READ = 1 # page can be read + PROT_WRITE = 2 # page can be written + MAP_PRIVATE = 2'i32 # Changes are private + + var MAP_ANONYMOUS {.importc: "MAP_ANONYMOUS", header: "<sys/mman.h>".}: cint + type + PEmscriptenMMapBlock = ptr EmscriptenMMapBlock + EmscriptenMMapBlock {.pure, inheritable.} = object + realSize: int # size of previous chunk; for coalescing + realPointer: pointer # if < PageSize it is a small chunk + + proc mmap(adr: pointer, len: int, prot, flags, fildes: cint, + off: int): pointer {.header: "<sys/mman.h>".} + + proc munmap(adr: pointer, len: int) {.header: "<sys/mman.h>".} + + proc osAllocPages(block_size: int): pointer {.inline.} = + let realSize = block_size + sizeof(EmscriptenMMapBlock) + PageSize + 1 + result = mmap(nil, realSize, PROT_READ or PROT_WRITE, + MAP_PRIVATE or MAP_ANONYMOUS, -1, 0) + if result == nil or result == cast[pointer](-1): + raiseOutOfMem() + + let realPointer = result + let pos = cast[int](result) + + # Convert pointer to PageSize correct one. + var new_pos = cast[ByteAddress](pos) +% (PageSize - (pos %% PageSize)) + if (new_pos-pos)< sizeof(EmscriptenMMapBlock): + new_pos = new_pos +% PageSize + result = cast[pointer](new_pos) + + var mmapDescrPos = cast[ByteAddress](result) -% sizeof(EmscriptenMMapBlock) + + var mmapDescr = cast[EmscriptenMMapBlock](mmapDescrPos) + mmapDescr.realSize = realSize + mmapDescr.realPointer = realPointer + + c_fprintf(c_stdout, "[Alloc] size %d %d realSize:%d realPos:%d\n", block_size, cast[int](result), realSize, cast[int](realPointer)) + + proc osTryAllocPages(size: int): pointer = osAllocPages(size) + + proc osDeallocPages(p: pointer, size: int) {.inline} = + var mmapDescrPos = cast[ByteAddress](p) -% sizeof(EmscriptenMMapBlock) + var mmapDescr = cast[EmscriptenMMapBlock](mmapDescrPos) + munmap(mmapDescr.realPointer, mmapDescr.realSize) + +elif defined(posix): + const + PROT_READ = 1 # page can be read + PROT_WRITE = 2 # page can be written + MAP_PRIVATE = 2'i32 # Changes are private + + when defined(macosx) or defined(bsd): + const MAP_ANONYMOUS = 0x1000 + elif defined(solaris): + const MAP_ANONYMOUS = 0x100 + else: + var + MAP_ANONYMOUS {.importc: "MAP_ANONYMOUS", header: "<sys/mman.h>".}: cint + + proc mmap(adr: pointer, len: int, prot, flags, fildes: cint, + off: int): pointer {.header: "<sys/mman.h>".} + + proc munmap(adr: pointer, len: int): cint {.header: "<sys/mman.h>".} + + proc osAllocPages(size: int): pointer {.inline.} = + result = mmap(nil, size, PROT_READ or PROT_WRITE, + MAP_PRIVATE or MAP_ANONYMOUS, -1, 0) + if result == nil or result == cast[pointer](-1): + raiseOutOfMem() + + proc osTryAllocPages(size: int): pointer {.inline.} = + result = mmap(nil, size, PROT_READ or PROT_WRITE, + MAP_PRIVATE or MAP_ANONYMOUS, -1, 0) + if result == cast[pointer](-1): result = nil + + proc osDeallocPages(p: pointer, size: int) {.inline} = + when reallyOsDealloc: discard munmap(p, size) + +elif defined(windows): + const + MEM_RESERVE = 0x2000 + MEM_COMMIT = 0x1000 + MEM_TOP_DOWN = 0x100000 + PAGE_READWRITE = 0x04 + + MEM_DECOMMIT = 0x4000 + MEM_RELEASE = 0x8000 + + proc virtualAlloc(lpAddress: pointer, dwSize: int, flAllocationType, + flProtect: int32): pointer {. + header: "<windows.h>", stdcall, importc: "VirtualAlloc".} + + proc virtualFree(lpAddress: pointer, dwSize: int, + dwFreeType: int32) {.header: "<windows.h>", stdcall, + importc: "VirtualFree".} + + proc osAllocPages(size: int): pointer {.inline.} = + result = virtualAlloc(nil, size, MEM_RESERVE or MEM_COMMIT, + PAGE_READWRITE) + if result == nil: raiseOutOfMem() + + proc osTryAllocPages(size: int): pointer {.inline.} = + result = virtualAlloc(nil, size, MEM_RESERVE or MEM_COMMIT, + PAGE_READWRITE) + + proc osDeallocPages(p: pointer, size: int) {.inline.} = + # according to Microsoft, 0 is the only correct value for MEM_RELEASE: + # This means that the OS has some different view over how big the block is + # that we want to free! So, we cannot reliably release the memory back to + # Windows :-(. We have to live with MEM_DECOMMIT instead. + # Well that used to be the case but MEM_DECOMMIT fragments the address + # space heavily, so we now treat Windows as a strange unmap target. + when reallyOsDealloc: virtualFree(p, 0, MEM_RELEASE) + #VirtualFree(p, size, MEM_DECOMMIT) + +elif hostOS == "standalone": + var + theHeap: array[1024*PageSize, float64] # 'float64' for alignment + bumpPointer = cast[int](addr theHeap) + + proc osAllocPages(size: int): pointer {.inline.} = + if size+bumpPointer < cast[int](addr theHeap) + sizeof(theHeap): + result = cast[pointer](bumpPointer) + inc bumpPointer, size + else: + raiseOutOfMem() + + proc osTryAllocPages(size: int): pointer {.inline.} = + if size+bumpPointer < cast[int](addr theHeap) + sizeof(theHeap): + result = cast[pointer](bumpPointer) + inc bumpPointer, size + + proc osDeallocPages(p: pointer, size: int) {.inline.} = + if bumpPointer-size == cast[int](p): + dec bumpPointer, size +else: + {.error: "Port memory manager to your platform".} diff --git a/lib/system/sysio.nim b/lib/system/sysio.nim index d0bba6775..3c34215ac 100644 --- a/lib/system/sysio.nim +++ b/lib/system/sysio.nim @@ -58,6 +58,8 @@ proc readBytes(f: File, a: var openArray[int8|uint8], start, len: Natural): int result = readBuffer(f, addr(a[start]), len) proc readChars(f: File, a: var openArray[char], start, len: Natural): int = + if (start + len) > len(a): + raiseEIO("buffer overflow: (start+len) > length of openarray buffer") result = readBuffer(f, addr(a[start]), len) proc write(f: File, c: cstring) = fputs(c, f) diff --git a/lib/system/syslocks.nim b/lib/system/syslocks.nim index 1551a4121..c3e23052b 100644 --- a/lib/system/syslocks.nim +++ b/lib/system/syslocks.nim @@ -9,42 +9,46 @@ # Low level system locks and condition vars. +{.push stackTrace: off.} + when defined(Windows): type Handle = int - SysLock {.final, pure.} = object # CRITICAL_SECTION in WinApi + + SysLock {.importc: "CRITICAL_SECTION", + header: "<windows.h>", final, pure.} = object # CRITICAL_SECTION in WinApi DebugInfo: pointer LockCount: int32 RecursionCount: int32 OwningThread: int LockSemaphore: int - Reserved: int32 + SpinCount: int SysCond = Handle {.deprecated: [THandle: Handle, TSysLock: SysLock, TSysCond: SysCond].} - proc initSysLock(L: var SysLock) {.stdcall, noSideEffect, - dynlib: "kernel32", importc: "InitializeCriticalSection".} + proc initSysLock(L: var SysLock) {.importc: "InitializeCriticalSection", + header: "<windows.h>".} ## Initializes the lock `L`. - proc tryAcquireSysAux(L: var SysLock): int32 {.stdcall, noSideEffect, - dynlib: "kernel32", importc: "TryEnterCriticalSection".} + proc tryAcquireSysAux(L: var SysLock): int32 {.importc: "TryEnterCriticalSection", + header: "<windows.h>".} ## Tries to acquire the lock `L`. proc tryAcquireSys(L: var SysLock): bool {.inline.} = result = tryAcquireSysAux(L) != 0'i32 - proc acquireSys(L: var SysLock) {.stdcall, noSideEffect, - dynlib: "kernel32", importc: "EnterCriticalSection".} + proc acquireSys(L: var SysLock) {.importc: "EnterCriticalSection", + header: "<windows.h>".} ## Acquires the lock `L`. - proc releaseSys(L: var SysLock) {.stdcall, noSideEffect, - dynlib: "kernel32", importc: "LeaveCriticalSection".} + proc releaseSys(L: var SysLock) {.importc: "LeaveCriticalSection", + header: "<windows.h>".} ## Releases the lock `L`. - proc deinitSys(L: var SysLock) {.stdcall, noSideEffect, - dynlib: "kernel32", importc: "DeleteCriticalSection".} + proc deinitSys(L: var SysLock) {.importc: "DeleteCriticalSection", + header: "<windows.h>".} proc createEvent(lpEventAttributes: pointer, bManualReset, bInitialState: int32, @@ -84,17 +88,16 @@ else: #include <pthread.h>""".} = object SysLockType = distinct cint - proc SysLockType_Reentrant: SysLockType = - {.emit: "`result` = PTHREAD_MUTEX_RECURSIVE;".} - proc initSysLock(L: var SysLock, attr: ptr SysLockAttr = nil) {. importc: "pthread_mutex_init", header: "<pthread.h>", noSideEffect.} - proc initSysLockAttr(a: var SysLockAttr) {. - importc: "pthread_mutexattr_init", header: "<pthread.h>", noSideEffect.} - - proc setSysLockType(a: var SysLockAttr, t: SysLockType) {. - importc: "pthread_mutexattr_settype", header: "<pthread.h>", noSideEffect.} + when insideRLocksModule: + proc SysLockType_Reentrant: SysLockType = + {.emit: "`result` = PTHREAD_MUTEX_RECURSIVE;".} + proc initSysLockAttr(a: var SysLockAttr) {. + importc: "pthread_mutexattr_init", header: "<pthread.h>", noSideEffect.} + proc setSysLockType(a: var SysLockAttr, t: SysLockType) {. + importc: "pthread_mutexattr_settype", header: "<pthread.h>", noSideEffect.} proc acquireSys(L: var SysLock) {.noSideEffect, importc: "pthread_mutex_lock", header: "<pthread.h>".} @@ -109,12 +112,14 @@ else: proc deinitSys(L: var SysLock) {.noSideEffect, importc: "pthread_mutex_destroy", header: "<pthread.h>".} - proc initSysCond(cond: var SysCond, cond_attr: pointer = nil) {. - importc: "pthread_cond_init", header: "<pthread.h>", noSideEffect.} - proc waitSysCond(cond: var SysCond, lock: var SysLock) {. - importc: "pthread_cond_wait", header: "<pthread.h>", noSideEffect.} - proc signalSysCond(cond: var SysCond) {. - importc: "pthread_cond_signal", header: "<pthread.h>", noSideEffect.} - - proc deinitSysCond(cond: var SysCond) {.noSideEffect, - importc: "pthread_cond_destroy", header: "<pthread.h>".} + when not insideRLocksModule: + proc initSysCond(cond: var SysCond, cond_attr: pointer = nil) {. + importc: "pthread_cond_init", header: "<pthread.h>", noSideEffect.} + proc waitSysCond(cond: var SysCond, lock: var SysLock) {. + importc: "pthread_cond_wait", header: "<pthread.h>", noSideEffect.} + proc signalSysCond(cond: var SysCond) {. + importc: "pthread_cond_signal", header: "<pthread.h>", noSideEffect.} + proc deinitSysCond(cond: var SysCond) {.noSideEffect, + importc: "pthread_cond_destroy", header: "<pthread.h>".} + +{.pop.} diff --git a/lib/system/sysstr.nim b/lib/system/sysstr.nim index e2137e8f4..f8b93a2c3 100644 --- a/lib/system/sysstr.nim +++ b/lib/system/sysstr.nim @@ -228,7 +228,8 @@ proc setLengthSeq(seq: PGenericSeq, elemSize, newLen: int): PGenericSeq {. elif newLen < result.len: # we need to decref here, otherwise the GC leaks! when not defined(boehmGC) and not defined(nogc) and - not defined(gcMarkAndSweep) and not defined(gogc): + not defined(gcMarkAndSweep) and not defined(gogc) and + not defined(gcStack): when false: # compileOption("gc", "v2"): for i in newLen..result.len-1: let len0 = gch.tempStack.len diff --git a/lib/windows/winlean.nim b/lib/windows/winlean.nim index 5bd9846c9..a08a067fa 100644 --- a/lib/windows/winlean.nim +++ b/lib/windows/winlean.nim @@ -412,7 +412,7 @@ const FD_SETSIZE* = 64 MSG_PEEK* = 2 - INADDR_ANY* = 0 + INADDR_ANY* = 0'u32 INADDR_LOOPBACK* = 0x7F000001 INADDR_BROADCAST* = -1 INADDR_NONE* = -1 @@ -441,12 +441,12 @@ type sa_data: array[0..13, char] InAddr* {.importc: "IN_ADDR", header: "winsock2.h".} = object - s_addr*: int32 # IP address + s_addr*: uint32 # IP address Sockaddr_in* {.importc: "SOCKADDR_IN", header: "winsock2.h".} = object sin_family*: int16 - sin_port*: int16 # unsigned + sin_port*: uint16 sin_addr*: InAddr sin_zero*: array[0..7, char] @@ -456,7 +456,7 @@ type Sockaddr_in6* {.importc: "SOCKADDR_IN6", header: "ws2tcpip.h".} = object sin6_family*: int16 - sin6_port*: int16 # unsigned + sin6_port*: uint16 sin6_flowinfo*: int32 # unsigned sin6_addr*: In6_addr sin6_scope_id*: int32 # unsigned @@ -590,7 +590,7 @@ proc getnameinfo*(a1: ptr SockAddr, a2: SockLen, a6: SockLen, a7: cint): cint {. stdcall, importc: "getnameinfo", dynlib: ws2dll.} -proc inet_addr*(cp: cstring): int32 {. +proc inet_addr*(cp: cstring): uint32 {. stdcall, importc: "inet_addr", dynlib: ws2dll.} proc WSAFDIsSet(s: SocketHandle, set: var TFdSet): bool {. @@ -769,7 +769,7 @@ proc getQueuedCompletionStatus*(CompletionPort: Handle, dwMilliseconds: DWORD): WINBOOL{.stdcall, dynlib: "kernel32", importc: "GetQueuedCompletionStatus".} -proc getOverlappedResult*(hFile: Handle, lpOverlapped: OVERLAPPED, +proc getOverlappedResult*(hFile: Handle, lpOverlapped: POVERLAPPED, lpNumberOfBytesTransferred: var DWORD, bWait: WINBOOL): WINBOOL{. stdcall, dynlib: "kernel32", importc: "GetOverlappedResult".} @@ -840,30 +840,30 @@ if ws2 != nil: proc WSAAddressToStringA(pAddr: ptr SockAddr, addrSize: DWORD, unused: pointer, pBuff: cstring, pBuffSize: ptr DWORD): cint {.stdcall, importc, dynlib: ws2dll.} proc inet_ntop_emulated(family: cint, paddr: pointer, pStringBuffer: cstring, stringBufSize: int32): cstring {.stdcall.} = - case family - of AF_INET: - var sa: Sockaddr_in - sa.sin_family = AF_INET - sa.sin_addr = cast[ptr InAddr](paddr)[] - var bs = stringBufSize.DWORD - let r = WSAAddressToStringA(cast[ptr SockAddr](sa.addr), sa.sizeof.DWORD, nil, pStringBuffer, bs.addr) - if r != 0: - result = nil - else: - result = pStringBuffer - of AF_INET6: - var sa: Sockaddr_in6 - sa.sin6_family = AF_INET6 - sa.sin6_addr = cast[ptr In6_addr](paddr)[] - var bs = stringBufSize.DWORD - let r = WSAAddressToStringA(cast[ptr SockAddr](sa.addr), sa.sizeof.DWORD, nil, pStringBuffer, bs.addr) - if r != 0: - result = nil - else: - result = pStringBuffer + case family + of AF_INET: + var sa: Sockaddr_in + sa.sin_family = AF_INET + sa.sin_addr = cast[ptr InAddr](paddr)[] + var bs = stringBufSize.DWORD + let r = WSAAddressToStringA(cast[ptr SockAddr](sa.addr), sa.sizeof.DWORD, nil, pStringBuffer, bs.addr) + if r != 0: + result = nil else: - setLastError(ERROR_BAD_ARGUMENTS) + result = pStringBuffer + of AF_INET6: + var sa: Sockaddr_in6 + sa.sin6_family = AF_INET6 + sa.sin6_addr = cast[ptr In6_addr](paddr)[] + var bs = stringBufSize.DWORD + let r = WSAAddressToStringA(cast[ptr SockAddr](sa.addr), sa.sizeof.DWORD, nil, pStringBuffer, bs.addr) + if r != 0: result = nil + else: + result = pStringBuffer + else: + setLastError(ERROR_BAD_ARGUMENTS) + result = nil proc inet_ntop*(family: cint, paddr: pointer, pStringBuffer: cstring, stringBufSize: int32): cstring {.stdcall.} = diff --git a/lib/wrappers/openssl.nim b/lib/wrappers/openssl.nim index 05843e2d3..635d52a64 100644 --- a/lib/wrappers/openssl.nim +++ b/lib/wrappers/openssl.nim @@ -260,7 +260,7 @@ proc OpenSSL_add_all_algorithms*(){.cdecl, dynlib: DLLUtilName, importc: "OPENSS proc OPENSSL_config*(configName: cstring){.cdecl, dynlib: DLLSSLName, importc.} -when not useWinVersion: +when not useWinVersion and not defined(macosx): proc CRYPTO_set_mem_functions(a,b,c: pointer){.cdecl, dynlib: DLLUtilName, importc.} @@ -532,7 +532,7 @@ proc md5_File* (file: string): string {.raises: [IOError,Exception].} = result = hexStr(buf) -proc md5_Str* (str:string): string {.raises:[IOError].} = +proc md5_Str*(str:string): string = ##Generate MD5 hash for a string. Result is a 32 character #hex string with lowercase characters var |