diff options
Diffstat (limited to 'lib/pure')
90 files changed, 5381 insertions, 5348 deletions
diff --git a/lib/pure/actors.nim b/lib/pure/actors.nim index f2c50ce4c..294c24741 100644 --- a/lib/pure/actors.nim +++ b/lib/pure/actors.nim @@ -200,7 +200,7 @@ template schedule = if minIdx >= 0: p.actors[minIdx].i.send(t) else: - raise newException(EDeadThread, "cannot send message; thread died") + raise newException(DeadThreadError, "cannot send message; thread died") proc spawn*[TIn, TOut](p: var TActorPool[TIn, TOut], input: TIn, action: proc (input: TIn): TOut {.thread.} @@ -221,7 +221,7 @@ proc spawn*[TIn](p: var TActorPool[TIn, void], input: TIn, setupTask() schedule() -when isMainModule: +when not defined(testing) and isMainModule: var a: TActorPool[int, void] createActorPool(a) diff --git a/lib/pure/algorithm.nim b/lib/pure/algorithm.nim index 0358a9a81..f7ccb9234 100644 --- a/lib/pure/algorithm.nim +++ b/lib/pure/algorithm.nim @@ -11,20 +11,20 @@ type SortOrder* = enum ## sort order - Descending, Ascending + Descending, Ascending {.deprecated: [TSortOrder: SortOrder].} -proc `*`*(x: int, order: SortOrder): int {.inline.} = +proc `*`*(x: int, order: SortOrder): int {.inline.} = ## flips `x` if ``order == Descending``; ## if ``order == Ascending`` then `x` is returned. - ## `x` is supposed to be the result of a comparator, ie ``< 0`` for + ## `x` is supposed to be the result of a comparator, ie ``< 0`` for ## *less than*, ``== 0`` for *equal*, ``> 0`` for *greater than*. var y = order.ord - 1 result = (x xor y) - y -proc reverse*[T](a: var openArray[T], first, last: int) = +proc reverse*[T](a: var openArray[T], first, last: Natural) = ## reverses the array ``a[first..last]``. var x = first var y = last @@ -37,11 +37,11 @@ proc reverse*[T](a: var openArray[T]) = ## reverses the array `a`. reverse(a, 0, a.high) -proc reversed*[T](a: openArray[T], first, last: int): seq[T] = +proc reversed*[T](a: openArray[T], first, last: Natural): seq[T] = ## returns the reverse of the array `a[first..last]`. result = newSeq[T](last - first + 1) - var x = first - var y = last + var x = first.int + var y = last.int while x <= last: result[x] = a[y] dec(y) @@ -73,14 +73,15 @@ const onlySafeCode = true proc lowerBound*[T](a: openArray[T], key: T, cmp: proc(x,y: T): int {.closure.}): int = - ## same as binarySearch except that if key is not in `a` then this + ## same as binarySearch except that if key is not in `a` then this ## returns the location where `key` would be if it were. In other - ## words if you have a sorted sequence and you call insert(thing, elm, lowerBound(thing, elm)) - ## the sequence will still be sorted + ## words if you have a sorted sequence and you call + ## insert(thing, elm, lowerBound(thing, elm)) + ## the sequence will still be sorted. + ## + ## `cmp` is the comparator function to use, the expected return values are + ## the same as that of system.cmp. ## - ## `cmp` is the comparator function to use, the expected return values are the same as - ## that of system.cmp - ## ## example:: ## ## var arr = @[1,2,3,5,6,7,8,9] @@ -102,9 +103,9 @@ proc lowerBound*[T](a: openArray[T], key: T, cmp: proc(x,y: T): int {.closure.}) count = step proc lowerBound*[T](a: openArray[T], key: T): int = lowerBound(a, key, cmp[T]) -proc merge[T](a, b: var openArray[T], lo, m, hi: int, +proc merge[T](a, b: var openArray[T], lo, m, hi: int, cmp: proc (x, y: T): int {.closure.}, order: SortOrder) = - template `<-` (a, b: expr) = + template `<-` (a, b: expr) = when false: a = b elif onlySafeCode: @@ -151,10 +152,10 @@ proc merge[T](a, b: var openArray[T], lo, m, hi: int, proc sort*[T](a: var openArray[T], cmp: proc (x, y: T): int {.closure.}, order = SortOrder.Ascending) = - ## Default Nim sort. The sorting is guaranteed to be stable and + ## Default Nim sort. The sorting is guaranteed to be stable and ## the worst case is guaranteed to be O(n log n). ## The current implementation uses an iterative - ## mergesort to achieve this. It uses a temporary sequence of + ## mergesort to achieve this. It uses a temporary sequence of ## length ``a.len div 2``. Currently Nim does not support a ## sensible default argument for ``cmp``, so you have to provide one ## of your own. However, the ``system.cmp`` procs can be used: @@ -187,6 +188,48 @@ proc sort*[T](a: var openArray[T], dec(m, s*2) s = s*2 +proc sorted*[T](a: openArray[T], cmp: proc(x, y: T): int {.closure.}, + order = SortOrder.Ascending): seq[T] = + ## returns `a` sorted by `cmp` in the specified `order`. + result = newSeq[T](a.len) + for i in 0 .. a.high: + result[i] = a[i] + sort(result, cmp, order) + +template sortedByIt*(seq1, op: expr): expr = + ## Convenience template around the ``sorted`` proc to reduce typing. + ## + ## The template injects the ``it`` variable which you can use directly in an + ## expression. Example: + ## + ## .. code-block:: nim + ## + ## type Person = tuple[name: string, age: int] + ## var + ## p1: Person = (name: "p1", age: 60) + ## p2: Person = (name: "p2", age: 20) + ## p3: Person = (name: "p3", age: 30) + ## p4: Person = (name: "p4", age: 30) + ## + ## people = @[p1,p2,p4,p3] + ## + ## echo people.sortedByIt(it.name) + ## + ## Because the underlying ``cmp()`` is defined for tuples you can do + ## a nested sort like in the following example: + ## + ## .. code-block:: nim + ## + ## echo people.sortedByIt((it.age, it.name)) + ## + var result {.gensym.} = sorted(seq1, proc(x, y: type(seq1[0])): int = + var it {.inject.} = x + let a = op + it = y + let b = op + result = cmp(a, b)) + result + proc product*[T](x: openArray[seq[T]]): seq[seq[T]] = ## produces the Cartesian product of the array. Warning: complexity ## may explode. @@ -210,13 +253,72 @@ proc product*[T](x: openArray[seq[T]]): seq[seq[T]] = while true: while indexes[index] == -1: indexes[index] = initial[index] - index +=1 + index += 1 if index == x.len: return - indexes[index] -=1 + indexes[index] -= 1 for ni, i in indexes: next[ni] = x[ni][i] var res: seq[T] shallowCopy(res, next) result.add(res) index = 0 - indexes[index] -=1 + indexes[index] -= 1 + +proc nextPermutation*[T](x: var openarray[T]): bool {.discardable.} = + ## Calculates the next lexicographic permutation, directly modifying ``x``. + ## The result is whether a permutation happened, otherwise we have reached + ## the last-ordered permutation. + ## + ## .. code-block:: nim + ## + ## var v = @[0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + ## v.nextPermutation() + ## echo v + if x.len < 2: + return false + + var i = x.high + while i > 0 and x[i-1] >= x[i]: + dec i + + if i == 0: + return false + + var j = x.high + while j >= i and x[j] <= x[i-1]: + dec j + + swap x[j], x[i-1] + x.reverse(i, x.high) + + result = true + +proc prevPermutation*[T](x: var openarray[T]): bool {.discardable.} = + ## Calculates the previous lexicographic permutation, directly modifying + ## ``x``. The result is whether a permutation happened, otherwise we have + ## reached the first-ordered permutation. + ## + ## .. code-block:: nim + ## + ## var v = @[0, 1, 2, 3, 4, 5, 6, 7, 9, 8] + ## v.prevPermutation() + ## echo v + if x.len < 2: + return false + + var i = x.high + while i > 0 and x[i-1] <= x[i]: + dec i + + if i == 0: + return false + + x.reverse(i, x.high) + + var j = x.high + while j >= i and x[j-1] < x[i-1]: + dec j + + swap x[i-1], x[j] + + result = true diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index 4bdce9cfb..a4d7a1632 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -1,7 +1,7 @@ # # # Nim's Runtime Library -# (c) Copyright 2014 Dominik Picheta +# (c) Copyright 2015 Dominik Picheta # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -121,7 +121,7 @@ export Port, SocketFlag ## ## Limitations/Bugs ## ---------------- -## +## ## * ``except`` statement (without `try`) does not work inside async procedures. ## * The effect system (``raises: []``) does not work with async procedures. ## * Can't await in a ``except`` body @@ -379,7 +379,7 @@ when defined(windows) or defined(nimdoc): if p.handles.len == 0 and p.timers.len == 0: raise newException(ValueError, "No handles or timers registered in dispatcher.") - + let llTimeout = if timeout == -1: winlean.INFINITE else: timeout.int32 @@ -436,12 +436,12 @@ when defined(windows) or defined(nimdoc): if not initPointer(dummySock, getAcceptExSockAddrsPtr, WSAID_GETACCEPTEXSOCKADDRS): raiseOSError(osLastError()) - proc connectEx(s: SocketHandle, name: ptr SockAddr, namelen: cint, + proc connectEx(s: SocketHandle, name: ptr SockAddr, namelen: cint, lpSendBuffer: pointer, dwSendDataLength: Dword, lpdwBytesSent: PDword, lpOverlapped: POVERLAPPED): bool = if connectExPtr.isNil: raise newException(ValueError, "Need to initialise ConnectEx().") let fun = - cast[proc (s: SocketHandle, name: ptr SockAddr, namelen: cint, + cast[proc (s: SocketHandle, name: ptr SockAddr, namelen: cint, lpSendBuffer: pointer, dwSendDataLength: Dword, lpdwBytesSent: PDword, lpOverlapped: POVERLAPPED): bool {.stdcall,gcsafe.}](connectExPtr) @@ -475,7 +475,7 @@ when defined(windows) or defined(nimdoc): dwRemoteAddressLength: Dword, LocalSockaddr: ptr ptr SockAddr, LocalSockaddrLength: LPInt, RemoteSockaddr: ptr ptr SockAddr, RemoteSockaddrLength: LPInt) {.stdcall,gcsafe.}](getAcceptExSockAddrsPtr) - + fun(lpOutputBuffer, dwReceiveDataLength, dwLocalAddressLength, dwRemoteAddressLength, LocalSockaddr, LocalSockaddrLength, RemoteSockaddr, RemoteSockaddrLength) @@ -514,7 +514,7 @@ when defined(windows) or defined(nimdoc): else: retFuture.fail(newException(OSError, osErrorMsg(errcode))) ) - + var ret = connectEx(socket.SocketHandle, it.ai_addr, sizeof(Sockaddr_in).cint, nil, 0, nil, cast[POVERLAPPED](ol)) @@ -565,7 +565,7 @@ when defined(windows) or defined(nimdoc): var dataBuf: TWSABuf dataBuf.buf = cast[cstring](alloc0(size)) dataBuf.len = size - + var bytesReceived: Dword var flagsio = flags.toOSFlags().Dword var ol = PCustomOverlapped() @@ -606,15 +606,15 @@ when defined(windows) or defined(nimdoc): retFuture.fail(newException(OSError, osErrorMsg(err))) elif ret == 0 and bytesReceived == 0 and dataBuf.buf[0] == '\0': # We have to ensure that the buffer is empty because WSARecv will tell - # us immediatelly when it was disconnected, even when there is still + # us immediately when it was disconnected, even when there is still # data in the buffer. # We want to give the user as much data as we can. So we only return # the empty string (which signals a disconnection) when there is # nothing left to read. retFuture.complete("") - # TODO: "For message-oriented sockets, where a zero byte message is often - # allowable, a failure with an error code of WSAEDISCON is used to - # indicate graceful closure." + # TODO: "For message-oriented sockets, where a zero byte message is often + # allowable, a failure with an error code of WSAEDISCON is used to + # indicate graceful closure." # ~ http://msdn.microsoft.com/en-us/library/ms741688%28v=vs.85%29.aspx else: # Request to read completed immediately. @@ -634,6 +634,93 @@ when defined(windows) or defined(nimdoc): # free ``ol``. return retFuture + proc recvInto*(socket: TAsyncFD, buf: cstring, size: int, + flags = {SocketFlag.SafeDisconn}): Future[int] = + ## Reads **up to** ``size`` bytes from ``socket`` into ``buf``, which must + ## at least be of that size. Returned future will complete once all the + ## data requested is read, a part of the data has been read, or the socket + ## has disconnected in which case the future will complete with a value of + ## ``0``. + ## + ## **Warning**: The ``Peek`` socket flag is not supported on Windows. + + + # Things to note: + # * When WSARecv completes immediately then ``bytesReceived`` is very + # unreliable. + # * Still need to implement message-oriented socket disconnection, + # '\0' in the message currently signifies a socket disconnect. Who + # knows what will happen when someone sends that to our socket. + verifyPresence(socket) + assert SocketFlag.Peek notin flags, "Peek not supported on Windows." + + var retFuture = newFuture[int]("recvInto") + + #buf[] = '\0' + var dataBuf: TWSABuf + dataBuf.buf = buf + dataBuf.len = size + + var bytesReceived: Dword + var flagsio = flags.toOSFlags().Dword + var ol = PCustomOverlapped() + GC_ref(ol) + ol.data = TCompletionData(fd: socket, cb: + proc (fd: TAsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + if not retFuture.finished: + if errcode == OSErrorCode(-1): + if bytesCount == 0 and dataBuf.buf[0] == '\0': + retFuture.complete(0) + else: + retFuture.complete(bytesCount) + else: + if flags.isDisconnectionError(errcode): + retFuture.complete(0) + else: + retFuture.fail(newException(OSError, osErrorMsg(errcode))) + if dataBuf.buf != nil: + dataBuf.buf = nil + ) + + let ret = WSARecv(socket.SocketHandle, addr dataBuf, 1, addr bytesReceived, + addr flagsio, cast[POVERLAPPED](ol), nil) + if ret == -1: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + if dataBuf.buf != nil: + dataBuf.buf = nil + GC_unref(ol) + if flags.isDisconnectionError(err): + retFuture.complete(0) + else: + retFuture.fail(newException(OSError, osErrorMsg(err))) + elif ret == 0 and bytesReceived == 0 and dataBuf.buf[0] == '\0': + # We have to ensure that the buffer is empty because WSARecv will tell + # us immediately when it was disconnected, even when there is still + # data in the buffer. + # We want to give the user as much data as we can. So we only return + # the empty string (which signals a disconnection) when there is + # nothing left to read. + retFuture.complete(0) + # TODO: "For message-oriented sockets, where a zero byte message is often + # allowable, a failure with an error code of WSAEDISCON is used to + # indicate graceful closure." + # ~ http://msdn.microsoft.com/en-us/library/ms741688%28v=vs.85%29.aspx + else: + # Request to read completed immediately. + # From my tests bytesReceived isn't reliable. + let realSize = + if bytesReceived == 0: + size + else: + bytesReceived + assert realSize <= size + retFuture.complete(realSize) + # We don't deallocate ``ol`` here because even though this completed + # immediately poll will still be notified about its completion and it will + # free ``ol``. + return retFuture + proc send*(socket: TAsyncFD, data: string, flags = {SocketFlag.SafeDisconn}): Future[void] = ## Sends ``data`` to ``socket``. The returned future will complete once all @@ -748,7 +835,7 @@ when defined(windows) or defined(nimdoc): # http://msdn.microsoft.com/en-us/library/windows/desktop/ms737524%28v=vs.85%29.aspx let ret = acceptEx(socket.SocketHandle, clientSock, addr lpOutputBuf[0], - dwReceiveDataLength, + dwReceiveDataLength, dwLocalAddressLength, dwRemoteAddressLength, addr dwBytesReceived, cast[POVERLAPPED](ol)) @@ -803,7 +890,7 @@ else: else: from posix import EINTR, EAGAIN, EINPROGRESS, EWOULDBLOCK, MSG_PEEK, MSG_NOSIGNAL - + type TAsyncFD* = distinct cint TCallback = proc (fd: TAsyncFD): bool {.closure,gcsafe.} @@ -841,6 +928,8 @@ else: proc newAsyncRawSocket*(domain: cint, typ: cint, protocol: cint): TAsyncFD = result = newRawSocket(domain, typ, protocol).TAsyncFD result.SocketHandle.setBlocking(false) + when defined(macosx): + result.SocketHandle.setSockOptInt(SOL_SOCKET, SO_NOSIGPIPE, 1) register(result) proc newAsyncRawSocket*(domain: Domain = AF_INET, @@ -848,8 +937,10 @@ else: protocol: Protocol = IPPROTO_TCP): TAsyncFD = result = newRawSocket(domain, typ, protocol).TAsyncFD result.SocketHandle.setBlocking(false) + when defined(macosx): + result.SocketHandle.setSockOptInt(SOL_SOCKET, SO_NOSIGPIPE, 1) register(result) - + proc closeSocket*(sock: TAsyncFD) = let disp = getGlobalDispatcher() sock.SocketHandle.close() @@ -864,20 +955,24 @@ else: raise newException(ValueError, "File descriptor not registered.") p.selector[fd.SocketHandle].data.PData.readCBs.add(cb) update(fd, p.selector[fd.SocketHandle].events + {EvRead}) - + proc addWrite*(fd: TAsyncFD, cb: TCallback) = let p = getGlobalDispatcher() if fd.SocketHandle notin p.selector: raise newException(ValueError, "File descriptor not registered.") p.selector[fd.SocketHandle].data.PData.writeCBs.add(cb) update(fd, p.selector[fd.SocketHandle].events + {EvWrite}) - + proc poll*(timeout = 500) = let p = getGlobalDispatcher() for info in p.selector.select(timeout): let data = PData(info.key.data) assert data.fd == info.key.fd.TAsyncFD #echo("In poll ", data.fd.cint) + if EvError in info.events: + closeSocket(data.fd) + continue + if EvRead in info.events: # Callback may add items to ``data.readCBs`` which causes issues if # we are iterating over ``data.readCBs`` at the same time. We therefore @@ -888,7 +983,7 @@ else: if not cb(data.fd): # Callback wants to be called again. data.readCBs.add(cb) - + if EvWrite in info.events: let currentCBs = data.writeCBs data.writeCBs = @[] @@ -896,7 +991,7 @@ else: if not cb(data.fd): # Callback wants to be called again. data.writeCBs.add(cb) - + if info.key in p.selector: var newEvents: set[Event] if data.readCBs.len != 0: newEvents = {EvRead} @@ -909,16 +1004,16 @@ else: discard processTimers(p) - + proc connect*(socket: TAsyncFD, address: string, port: Port, af = AF_INET): Future[void] = var retFuture = newFuture[void]("connect") - + proc cb(fd: TAsyncFD): bool = # We have connected. retFuture.complete() return true - + var aiList = getAddrInfo(address, port, af) var success = false var lastError: OSErrorCode @@ -948,14 +1043,13 @@ else: proc recv*(socket: TAsyncFD, size: int, flags = {SocketFlag.SafeDisconn}): Future[string] = var retFuture = newFuture[string]("recv") - + var readBuffer = newString(size) proc cb(sock: TAsyncFD): bool = result = true let res = recv(sock.SocketHandle, addr readBuffer[0], size.cint, flags.toOSFlags()) - #echo("recv cb res: ", res) if res < 0: let lastError = osLastError() if lastError.int32 notin {EINTR, EWOULDBLOCK, EAGAIN}: @@ -976,12 +1070,36 @@ else: addRead(socket, cb) return retFuture + proc recvInto*(socket: TAsyncFD, buf: cstring, size: int, + flags = {SocketFlag.SafeDisconn}): Future[int] = + var retFuture = newFuture[int]("recvInto") + + proc cb(sock: TAsyncFD): bool = + result = true + let res = recv(sock.SocketHandle, buf, size.cint, + flags.toOSFlags()) + if res < 0: + let lastError = osLastError() + if lastError.int32 notin {EINTR, EWOULDBLOCK, EAGAIN}: + if flags.isDisconnectionError(lastError): + retFuture.complete(0) + else: + retFuture.fail(newException(OSError, osErrorMsg(lastError))) + else: + result = false # We still want this callback to be called. + else: + retFuture.complete(res) + # TODO: The following causes a massive slowdown. + #if not cb(socket): + addRead(socket, cb) + return retFuture + proc send*(socket: TAsyncFD, data: string, flags = {SocketFlag.SafeDisconn}): Future[void] = var retFuture = newFuture[void]("send") - + var written = 0 - + proc cb(sock: TAsyncFD): bool = result = true let netSize = data.len-written @@ -1060,6 +1178,17 @@ proc accept*(socket: TAsyncFD, # -- Await Macro +proc skipUntilStmtList(node: NimNode): NimNode {.compileTime.} = + # Skips a nest of StmtList's. + result = node + if node[0].kind == nnkStmtList: + result = skipUntilStmtList(node[0]) + +proc skipStmtList(node: NimNode): NimNode {.compileTime.} = + result = node + if node[0].kind == nnkStmtList: + result = node[0] + template createCb(retFutureSym, iteratorNameSym, name: expr): stmt {.immediate.} = var nameIterVar = iteratorNameSym @@ -1083,11 +1212,11 @@ template createCb(retFutureSym, iteratorNameSym, cb() #{.pop.} proc generateExceptionCheck(futSym, - tryStmt, rootReceiver, fromNode: PNimrodNode): PNimrodNode {.compileTime.} = + tryStmt, rootReceiver, fromNode: NimNode): NimNode {.compileTime.} = if tryStmt.kind == nnkNilLit: result = rootReceiver else: - var exceptionChecks: seq[tuple[cond, body: PNimrodNode]] = @[] + var exceptionChecks: seq[tuple[cond, body: NimNode]] = @[] let errorNode = newDotExpr(futSym, newIdentNode("error")) for i in 1 .. <tryStmt.len: let exceptBranch = tryStmt[i] @@ -1095,7 +1224,7 @@ proc generateExceptionCheck(futSym, exceptionChecks.add((newIdentNode("true"), exceptBranch[0])) else: var exceptIdentCount = 0 - var ifCond: PNimrodNode + var ifCond: NimNode for i in 0 .. <exceptBranch.len: let child = exceptBranch[i] if child.kind == nnkIdent: @@ -1129,10 +1258,10 @@ proc generateExceptionCheck(futSym, ) result.add elseNode -template createVar(result: var PNimrodNode, futSymName: string, - asyncProc: PNimrodNode, +template createVar(result: var NimNode, futSymName: string, + asyncProc: NimNode, valueReceiver, rootReceiver: expr, - fromNode: PNimrodNode) = + fromNode: NimNode) = result = newNimNode(nnkStmtList, fromNode) var futSym = genSym(nskVar, "future") result.add newVarStmt(futSym, asyncProc) # -> var future<x> = y @@ -1140,9 +1269,9 @@ template createVar(result: var PNimrodNode, futSymName: string, valueReceiver = newDotExpr(futSym, newIdentNode("read")) # -> future<x>.read result.add generateExceptionCheck(futSym, tryStmt, rootReceiver, fromNode) -proc processBody(node, retFutureSym: PNimrodNode, +proc processBody(node, retFutureSym: NimNode, subTypeIsVoid: bool, - tryStmt: PNimrodNode): PNimrodNode {.compileTime.} = + tryStmt: NimNode): NimNode {.compileTime.} = #echo(node.treeRepr) result = node case node.kind @@ -1168,7 +1297,7 @@ proc processBody(node, retFutureSym: PNimrodNode, result = newNimNode(nnkYieldStmt, node).add(node[1]) # -> yield x of nnkCall, nnkCommand: # await foo(p, x) - var futureValue: PNimrodNode + var futureValue: NimNode result.createVar("future" & $node[1][0].toStrLit, node[1], futureValue, futureValue, node) else: @@ -1207,33 +1336,60 @@ proc processBody(node, retFutureSym: PNimrodNode, of nnkTryStmt: # try: await x; except: ... result = newNimNode(nnkStmtList, node) - proc processForTry(n: PNimrodNode, i: var int, - res: PNimrodNode): bool {.compileTime.} = + template wrapInTry(n, tryBody: expr) = + var temp = n + n[0] = tryBody + tryBody = temp + + # Transform ``except`` body. + # TODO: Could we perform some ``await`` transformation here to get it + # working in ``except``? + tryBody[1] = processBody(n[1], retFutureSym, subTypeIsVoid, nil) + + proc processForTry(n: NimNode, i: var int, + res: NimNode): bool {.compileTime.} = + ## Transforms the body of the tryStmt. Does not transform the + ## body in ``except``. + ## Returns true if the tryStmt node was transformed into an ifStmt. result = false - while i < n[0].len: - var processed = processBody(n[0][i], retFutureSym, subTypeIsVoid, n) - if processed.kind != n[0][i].kind or processed.len != n[0][i].len: + var skipped = n.skipStmtList() + while i < skipped.len: + var processed = processBody(skipped[i], retFutureSym, + subTypeIsVoid, n) + + # Check if we transformed the node into an exception check. + # This suggests skipped[i] contains ``await``. + if processed.kind != skipped[i].kind or processed.len != skipped[i].len: + processed = processed.skipUntilStmtList() expectKind(processed, nnkStmtList) expectKind(processed[2][1], nnkElse) i.inc - discard processForTry(n, i, processed[2][1][0]) + + if not processForTry(n, i, processed[2][1][0]): + # We need to wrap the nnkElse nodes back into a tryStmt. + # As they are executed if an exception does not happen + # inside the awaited future. + # The following code will wrap the nodes inside the + # original tryStmt. + wrapInTry(n, processed[2][1][0]) + res.add processed result = true else: - res.add n[0][i] + res.add skipped[i] i.inc var i = 0 if not processForTry(node, i, result): - var temp = node - temp[0] = result - result = temp + # If the tryStmt hasn't been transformed we can just put the body + # back into it. + wrapInTry(node, result) return else: discard for i in 0 .. <result.len: - result[i] = processBody(result[i], retFutureSym, subTypeIsVoid, tryStmt) + result[i] = processBody(result[i], retFutureSym, subTypeIsVoid, nil) -proc getName(node: PNimrodNode): string {.compileTime.} = +proc getName(node: NimNode): string {.compileTime.} = case node.kind of nnkPostfix: return $node[1].ident @@ -1273,29 +1429,40 @@ macro async*(prc: stmt): stmt {.immediate.} = if returnType.kind == nnkEmpty: newIdentNode("void") else: returnType[1] outerProcBody.add( - newVarStmt(retFutureSym, + newVarStmt(retFutureSym, newCall( newNimNode(nnkBracketExpr, prc[6]).add( newIdentNode(!"newFuture"), # TODO: Strange bug here? Remove the `!`. subRetType), newLit(prc[0].getName)))) # Get type from return type of this proc - - # -> iterator nameIter(): FutureBase {.closure.} = + + # -> iterator nameIter(): FutureBase {.closure.} = + # -> {.push warning[resultshadowed]: off.} # -> var result: T + # -> {.pop.} # -> <proc_body> # -> complete(retFuture, result) var iteratorNameSym = genSym(nskIterator, $prc[0].getName & "Iter") var procBody = prc[6].processBody(retFutureSym, subtypeIsVoid, nil) if not subtypeIsVoid: - procBody.insert(0, newNimNode(nnkVarSection, prc[6]).add( + procBody.insert(0, newNimNode(nnkPragma).add(newIdentNode("push"), + newNimNode(nnkExprColonExpr).add(newNimNode(nnkBracketExpr).add( + newIdentNode("warning"), newIdentNode("resultshadowed")), + newIdentNode("off")))) # -> {.push warning[resultshadowed]: off.} + + procBody.insert(1, newNimNode(nnkVarSection, prc[6]).add( newIdentDefs(newIdentNode("result"), returnType[1]))) # -> var result: T + + procBody.insert(2, newNimNode(nnkPragma).add( + newIdentNode("pop"))) # -> {.pop.}) + procBody.add( newCall(newIdentNode("complete"), retFutureSym, newIdentNode("result"))) # -> complete(retFuture, result) else: # -> complete(retFuture) procBody.add(newCall(newIdentNode("complete"), retFutureSym)) - + var closureIterator = newProc(iteratorNameSym, [newIdentNode("FutureBase")], procBody, nnkIteratorDef) closureIterator[4] = newNimNode(nnkPragma, prc[6]).add(newIdentNode("closure")) @@ -1309,7 +1476,7 @@ macro async*(prc: stmt): stmt {.immediate.} = # -> return retFuture outerProcBody.add newNimNode(nnkReturnStmt, prc[6][prc[6].len-1]).add(retFutureSym) - + result = prc # Remove the 'async' pragma. @@ -1325,7 +1492,7 @@ macro async*(prc: stmt): stmt {.immediate.} = result[6] = outerProcBody #echo(treeRepr(result)) - #if prc[0].getName == "catch": + #if prc[0].getName == "test": # echo(toStrLit(result)) proc recvLine*(socket: TAsyncFD): Future[string] {.async.} = @@ -1335,7 +1502,7 @@ proc recvLine*(socket: TAsyncFD): Future[string] {.async.} = ## If a full line is read ``\r\L`` is not ## added to ``line``, however if solely ``\r\L`` is read then ``line`` ## will be set to it. - ## + ## ## If the socket is disconnected, ``line`` will be set to ``""``. ## ## If the socket is disconnected in the middle of a line (before ``\r\L`` @@ -1346,7 +1513,7 @@ proc recvLine*(socket: TAsyncFD): Future[string] {.async.} = ## ## **Note**: This procedure is mostly used for testing. You likely want to ## use ``asyncnet.recvLine`` instead. - + template addNLIfEmpty(): stmt = if result.len == 0: result.add("\c\L") diff --git a/lib/pure/asyncdispatch.nimrod.cfg b/lib/pure/asyncdispatch.nim.cfg index e88f8eec3..e88f8eec3 100644 --- a/lib/pure/asyncdispatch.nimrod.cfg +++ b/lib/pure/asyncdispatch.nim.cfg diff --git a/lib/pure/asyncfile.nim b/lib/pure/asyncfile.nim index c09cb5a35..25e121183 100644 --- a/lib/pure/asyncfile.nim +++ b/lib/pure/asyncfile.nim @@ -1,13 +1,13 @@ # # # Nim's Runtime Library -# (c) Copyright 2014 Dominik Picheta +# (c) Copyright 2015 Dominik Picheta # # See the file "copying.txt", included in this # distribution, for details about the copyright. # -## This module implements asynchronous file handling. +## This module implements asynchronous file reading and writing. ## ## .. code-block:: Nim ## import asyncfile, asyncdispatch, os @@ -24,18 +24,18 @@ import asyncdispatch, os -when defined(windows): +when defined(windows) or defined(nimdoc): import winlean else: import posix type - AsyncFile = ref object + AsyncFile* = ref object fd: TAsyncFd offset: int64 -when defined(windows): - proc getDesiredAccess(mode: TFileMode): int32 = +when defined(windows) or defined(nimdoc): + proc getDesiredAccess(mode: FileMode): int32 = case mode of fmRead: result = GENERIC_READ @@ -44,7 +44,7 @@ when defined(windows): of fmReadWrite, fmReadWriteExisting: result = GENERIC_READ or GENERIC_WRITE - proc getCreationDisposition(mode: TFileMode, filename: string): int32 = + proc getCreationDisposition(mode: FileMode, filename: string): int32 = case mode of fmRead, fmReadWriteExisting: OPEN_EXISTING @@ -54,7 +54,7 @@ when defined(windows): else: CREATE_NEW else: - proc getPosixFlags(mode: TFileMode): cint = + proc getPosixFlags(mode: FileMode): cint = case mode of fmRead: result = O_RDONLY @@ -70,18 +70,18 @@ else: proc getFileSize(f: AsyncFile): int64 = ## Retrieves the specified file's size. - when defined(windows): + when defined(windows) or defined(nimdoc): var high: DWord let low = getFileSize(f.fd.THandle, addr high) if low == INVALID_FILE_SIZE: - raiseOSError() + raiseOSError(osLastError()) return (high shl 32) or low proc openAsync*(filename: string, mode = fmRead): AsyncFile = ## Opens a file specified by the path in ``filename`` using ## the specified ``mode`` asynchronously. new result - when defined(windows): + when defined(windows) or defined(nimdoc): let flags = FILE_FLAG_OVERLAPPED or FILE_ATTRIBUTE_NORMAL let desiredAccess = getDesiredAccess(mode) let creationDisposition = getCreationDisposition(mode, filename) @@ -95,7 +95,7 @@ proc openAsync*(filename: string, mode = fmRead): AsyncFile = nil, creationDisposition, flags, 0).TAsyncFd if result.fd.THandle == INVALID_HANDLE_VALUE: - raiseOSError() + raiseOSError(osLastError()) register(result.fd) @@ -108,7 +108,7 @@ proc openAsync*(filename: string, mode = fmRead): AsyncFile = let perm = S_IRUSR or S_IWUSR or S_IRGRP or S_IWGRP or S_IROTH result.fd = open(filename, flags, perm).TAsyncFD if result.fd.cint == -1: - raiseOSError() + raiseOSError(osLastError()) register(result.fd) @@ -120,7 +120,7 @@ proc read*(f: AsyncFile, size: int): Future[string] = ## returned. var retFuture = newFuture[string]("asyncfile.read") - when defined(windows): + when defined(windows) or defined(nimdoc): var buffer = alloc0(size) var ol = PCustomOverlapped() @@ -185,7 +185,7 @@ proc read*(f: AsyncFile, size: int): Future[string] = if res < 0: let lastError = osLastError() if lastError.int32 != EAGAIN: - retFuture.fail(newException(EOS, osErrorMsg(lastError))) + retFuture.fail(newException(OSError, osErrorMsg(lastError))) else: result = false # We still want this callback to be called. elif res == 0: @@ -224,10 +224,10 @@ proc setFilePos*(f: AsyncFile, pos: int64) = ## Sets the position of the file pointer that is used for read/write ## operations. The file's first byte has the index zero. f.offset = pos - when not defined(windows): + when not defined(windows) and not defined(nimdoc): let ret = lseek(f.fd.cint, pos, SEEK_SET) if ret == -1: - raiseOSError() + raiseOSError(osLastError()) proc readAll*(f: AsyncFile): Future[string] {.async.} = ## Reads all data from the specified file. @@ -245,7 +245,7 @@ proc write*(f: AsyncFile, data: string): Future[void] = ## specified file. var retFuture = newFuture[void]("asyncfile.write") var copy = data - when defined(windows): + when defined(windows) or defined(nimdoc): var buffer = alloc0(data.len) copyMem(buffer, addr copy[0], data.len) @@ -299,7 +299,7 @@ proc write*(f: AsyncFile, data: string): Future[void] = if res < 0: let lastError = osLastError() if lastError.int32 != EAGAIN: - retFuture.fail(newException(EOS, osErrorMsg(lastError))) + retFuture.fail(newException(OSError, osErrorMsg(lastError))) else: result = false # We still want this callback to be called. else: @@ -316,10 +316,10 @@ proc write*(f: AsyncFile, data: string): Future[void] = proc close*(f: AsyncFile) = ## Closes the file specified. - when defined(windows): + when defined(windows) or defined(nimdoc): if not closeHandle(f.fd.THandle).bool: - raiseOSError() + raiseOSError(osLastError()) else: if close(f.fd.cint) == -1: - raiseOSError() + raiseOSError(osLastError()) diff --git a/lib/pure/asyncftpclient.nim b/lib/pure/asyncftpclient.nim index fc38dc31a..daf69d59f 100644 --- a/lib/pure/asyncftpclient.nim +++ b/lib/pure/asyncftpclient.nim @@ -1,7 +1,7 @@ # # # Nim's Runtime Library -# (c) Copyright 2014 Dominik Picheta +# (c) Copyright 2015 Dominik Picheta # See the file "copying.txt", included in this # distribution, for details about the copyright. # @@ -61,8 +61,8 @@ proc pasv(ftp: AsyncFtpClient) {.async.} = assertReply(pasvMsg, "227") var betweenParens = captureBetween(pasvMsg.string, '(', ')') var nums = betweenParens.split(',') - var ip = nums[0.. -3] - var port = nums[-2.. -1] + var ip = nums[0.. ^3] + var port = nums[^2.. ^1] var properPort = port[0].parseInt()*256+port[1].parseInt() await ftp.dsock.connect(ip.join("."), Port(properPort.toU16)) ftp.dsockConnected = true @@ -149,7 +149,7 @@ proc createDir*(ftp: AsyncFtpClient, dir: string, recursive = false){.async.} = assertReply reply, "257" proc chmod*(ftp: AsyncFtpClient, path: string, - permissions: set[TFilePermission]) {.async.} = + permissions: set[FilePermission]) {.async.} = ## Changes permission of ``path`` to ``permissions``. var userOctal = 0 var groupOctal = 0 @@ -188,7 +188,7 @@ proc retrText*(ftp: AsyncFtpClient, file: string): Future[string] {.async.} = result = await ftp.getLines() -proc getFile(ftp: AsyncFtpClient, file: TFile, total: BiggestInt, +proc getFile(ftp: AsyncFtpClient, file: File, total: BiggestInt, onProgressChanged: ProgressChangedProc) {.async.} = assert ftp.dsockConnected var progress = 0 @@ -240,7 +240,7 @@ proc retrFile*(ftp: AsyncFtpClient, file, dest: string, await getFile(ftp, destFile, fileSize, onProgressChanged) -proc doUpload(ftp: AsyncFtpClient, file: TFile, +proc doUpload(ftp: AsyncFtpClient, file: File, onProgressChanged: ProgressChangedProc) {.async.} = assert ftp.dsockConnected @@ -300,7 +300,7 @@ proc newAsyncFtpClient*(address: string, port = Port(21), result.dsockConnected = false result.csock = newAsyncSocket() -when isMainModule: +when not defined(testing) and isMainModule: var ftp = newAsyncFtpClient("example.com", user = "test", pass = "test") proc main(ftp: AsyncFtpClient) {.async.} = await ftp.connect() diff --git a/lib/pure/asynchttpserver.nim b/lib/pure/asynchttpserver.nim index 0b18e6bcc..279cedb5d 100644 --- a/lib/pure/asynchttpserver.nim +++ b/lib/pure/asynchttpserver.nim @@ -1,7 +1,7 @@ # # # Nim's Runtime Library -# (c) Copyright 2014 Dominik Picheta +# (c) Copyright 2015 Dominik Picheta # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -17,12 +17,13 @@ ## as the response body. ## ## .. code-block::nim +## import asynchttpserver, asyncdispatch +## ## var server = newAsyncHttpServer() ## proc cb(req: Request) {.async.} = ## await req.respond(Http200, "Hello World") ## -## asyncCheck server.serve(Port(8080), cb) -## runForever() +## waitFor server.serve(Port(8080), cb) import strtabs, asyncnet, asyncdispatch, parseutils, uri, strutils type @@ -107,22 +108,19 @@ proc sendHeaders*(req: Request, headers: StringTableRef): Future[void] = addHeaders(msg, headers) return req.client.send(msg) -proc respond*(req: Request, code: HttpCode, - content: string, headers = newStringTable()) {.async.} = +proc respond*(req: Request, code: HttpCode, content: string, + headers: StringTableRef = nil): Future[void] = ## Responds to the request with the specified ``HttpCode``, headers and ## content. ## ## This procedure will **not** close the client socket. - var customHeaders = headers - customHeaders["Content-Length"] = $content.len var msg = "HTTP/1.1 " & $code & "\c\L" - msg.addHeaders(customHeaders) - await req.client.send(msg & "\c\L" & content) -proc newRequest(): Request = - result.headers = newStringTable(modeCaseInsensitive) - result.hostname = "" - result.body = "" + if headers != nil: + msg.addHeaders(headers) + msg.add("Content-Length: " & $content.len & "\c\L\c\L") + msg.add(content) + result = req.client.send(msg) proc parseHeader(line: string): tuple[key, value: string] = var i = 0 @@ -147,59 +145,65 @@ proc sendStatus(client: AsyncSocket, status: string): Future[void] = proc processClient(client: AsyncSocket, address: string, callback: proc (request: Request): Future[void] {.closure, gcsafe.}) {.async.} = + var request: Request + request.url = initUri() + request.headers = newStringTable(modeCaseInsensitive) + var line = newStringOfCap(80) + var key, value = "" + while not client.isClosed: # GET /path HTTP/1.1 # Header: val # \n - var request = newRequest() - request.hostname = address + request.headers.clear(modeCaseInsensitive) + request.hostname.shallowCopy(address) assert client != nil request.client = client # First line - GET /path HTTP/1.1 - let line = await client.recvLine() # TODO: Timeouts. + line.setLen(0) + await client.recvLineInto(addr line) # TODO: Timeouts. if line == "": client.close() return - let lineParts = line.split(' ') - if lineParts.len != 3: - await request.respond(Http400, "Invalid request. Got: " & line) - continue - let reqMethod = lineParts[0] - let path = lineParts[1] - let protocol = lineParts[2] + var i = 0 + for linePart in line.split(' '): + case i + of 0: request.reqMethod.shallowCopy(linePart.normalize) + of 1: parseUri(linePart, request.url) + of 2: + try: + request.protocol = parseProtocol(linePart) + except ValueError: + asyncCheck request.respond(Http400, + "Invalid request protocol. Got: " & linePart) + continue + else: + await request.respond(Http400, "Invalid request. Got: " & line) + continue + inc i # Headers - var i = 0 while true: i = 0 - let headerLine = await client.recvLine() - if headerLine == "": - client.close(); return - if headerLine == "\c\L": break - # TODO: Compiler crash - #let (key, value) = parseHeader(headerLine) - let kv = parseHeader(headerLine) - request.headers[kv.key] = kv.value + line.setLen(0) + await client.recvLineInto(addr line) - request.reqMethod = reqMethod - request.url = parseUri(path) - try: - request.protocol = protocol.parseProtocol() - except ValueError: - asyncCheck request.respond(Http400, "Invalid request protocol. Got: " & - protocol) - continue + if line == "": + client.close(); return + if line == "\c\L": break + let (key, value) = parseHeader(line) + request.headers[key] = value - if reqMethod.normalize == "post": + if request.reqMethod == "post": # Check for Expect header if request.headers.hasKey("Expect"): if request.headers["Expect"].toLower == "100-continue": await client.sendStatus("100 Continue") else: await client.sendStatus("417 Expectation Failed") - + # Read the body # - Check for Content-length header if request.headers.hasKey("Content-Length"): @@ -213,11 +217,11 @@ proc processClient(client: AsyncSocket, address: string, await request.respond(Http400, "Bad Request. No Content-Length.") continue - case reqMethod.normalize + case request.reqMethod of "get", "post", "head", "put", "delete", "trace", "options", "connect", "patch": await callback(request) else: - await request.respond(Http400, "Invalid request method. Got: " & reqMethod) + await request.respond(Http400, "Invalid request method. Got: " & request.reqMethod) # Persistent connections if (request.protocol == HttpVer11 and @@ -245,7 +249,7 @@ proc serve*(server: AsyncHttpServer, port: Port, server.socket.setSockOpt(OptReuseAddr, true) server.socket.bindAddr(port, address) server.socket.listen() - + while true: # TODO: Causes compiler crash. #var (address, client) = await server.socket.acceptAddr() @@ -258,7 +262,7 @@ proc close*(server: AsyncHttpServer) = ## Terminates the async http server instance. server.socket.close() -when isMainModule: +when not defined(testing) and isMainModule: proc main = var server = newAsyncHttpServer() proc cb(req: Request) {.async.} = diff --git a/lib/pure/asyncio.nim b/lib/pure/asyncio.nim index d40c3849e..6ae2c608b 100644 --- a/lib/pure/asyncio.nim +++ b/lib/pure/asyncio.nim @@ -18,8 +18,8 @@ import sockets, os ## This module implements an asynchronous event loop together with asynchronous ## sockets which use this event loop. ## It is akin to Python's asyncore module. Many modules that use sockets -## have an implementation for this module, those modules should all have a -## ``register`` function which you should use to add the desired objects to a +## have an implementation for this module, those modules should all have a +## ``register`` function which you should use to add the desired objects to a ## dispatcher which you created so ## that you can receive the events associated with that module's object. ## @@ -27,19 +27,19 @@ import sockets, os ## function in a while loop. ## ## **Note:** Most modules have tasks which need to be ran regularly, this is -## why you should not call ``poll`` with a infinite timeout, or even a +## why you should not call ``poll`` with a infinite timeout, or even a ## very long one. In most cases the default timeout is fine. ## ## **Note:** This module currently only supports select(), this is limited by ## FD_SETSIZE, which is usually 1024. So you may only be able to use 1024 ## sockets at a time. -## +## ## Most (if not all) modules that use asyncio provide a userArg which is passed ## on with the events. The type that you set userArg to must be inheriting from ## ``RootObj``! ## -## **Note:** If you want to provide async ability to your module please do not -## use the ``Delegate`` object, instead use ``AsyncSocket``. It is possible +## **Note:** If you want to provide async ability to your module please do not +## use the ``Delegate`` object, instead use ``AsyncSocket``. It is possible ## that in the future this type's fields will not be exported therefore breaking ## your code. ## @@ -59,11 +59,11 @@ import sockets, os ## socket which will give you the client which is connecting. You should then ## set any events that you want to use on that client and add it to your dispatcher ## using the ``register`` procedure. -## +## ## An example ``handleAccept`` follows: -## +## ## .. code-block:: nim -## +## ## var disp = newDispatcher() ## ... ## proc handleAccept(s: AsyncSocket) = @@ -74,7 +74,7 @@ import sockets, os ## client.handleRead = ... ## disp.register(client) ## ... -## +## ## For client sockets you should only be interested in the ``handleRead`` and ## ``handleConnect`` events. The former gets called whenever the socket has ## received messages and can be read from and the latter gets called whenever @@ -83,14 +83,14 @@ import sockets, os ## ## Getting a blocking client from an AsyncSocket ## ============================================= -## +## ## If you need a asynchronous server socket but you wish to process the clients ## synchronously then you can use the ``getSocket`` converter to get ## a ``Socket`` from the ``AsyncSocket`` object, this can then be combined ## with ``accept`` like so: ## ## .. code-block:: nim -## +## ## proc handleAccept(s: AsyncSocket) = ## var client: Socket ## getSocket(s).accept(client) @@ -113,11 +113,11 @@ type handleWrite*: proc (h: RootRef) {.nimcall, gcsafe.} handleError*: proc (h: RootRef) {.nimcall, gcsafe.} hasDataBuffered*: proc (h: RootRef): bool {.nimcall, gcsafe.} - + open*: bool task*: proc (h: RootRef) {.nimcall, gcsafe.} mode*: FileMode - + Delegate* = ref DelegateObj Dispatcher* = ref DispatcherObj @@ -144,7 +144,7 @@ type deleg: Delegate SocketStatus* = enum - SockIdle, SockConnecting, SockConnected, SockListening, SockClosed, + SockIdle, SockConnecting, SockConnected, SockListening, SockClosed, SockUDPBound {.deprecated: [TDelegate: DelegateObj, PDelegate: Delegate, @@ -176,8 +176,8 @@ proc newAsyncSocket(): AsyncSocket = result.lineBuffer = "".TaintedString result.sendBuffer = "" -proc asyncSocket*(domain: Domain = AF_INET, typ: SockType = SOCK_STREAM, - protocol: Protocol = IPPROTO_TCP, +proc asyncSocket*(domain: Domain = AF_INET, typ: SockType = SOCK_STREAM, + protocol: Protocol = IPPROTO_TCP, buffered = true): AsyncSocket = ## Initialises an AsyncSocket object. If a socket cannot be initialised ## EOS is raised. @@ -236,7 +236,7 @@ proc asyncSockHandleWrite(h: RootRef) = if AsyncSocket(h).socket.isSSL and not AsyncSocket(h).socket.gotHandshake: return - + if AsyncSocket(h).info == SockConnecting: AsyncSocket(h).handleConnect(AsyncSocket(h)) AsyncSocket(h).info = SockConnected @@ -256,10 +256,10 @@ proc asyncSockHandleWrite(h: RootRef) = # do nothing instead. discard elif bytesSent != sock.sendBuffer.len: - sock.sendBuffer = sock.sendBuffer[bytesSent .. -1] + sock.sendBuffer = sock.sendBuffer[bytesSent .. ^1] elif bytesSent == sock.sendBuffer.len: sock.sendBuffer = "" - + if AsyncSocket(h).handleWrite != nil: AsyncSocket(h).handleWrite(AsyncSocket(h)) except OSError: @@ -284,7 +284,7 @@ when defined(ssl): else: # handshake will set socket's ``sslNoHandshake`` field. discard AsyncSocket(h).socket.handshake() - + proc asyncSockTask(h: RootRef) = when defined(ssl): @@ -377,9 +377,9 @@ proc acceptAddr*(server: AsyncSocket, client: var AsyncSocket, if c == invalidSocket: raiseSocketError(server.socket) c.setBlocking(false) # TODO: Needs to be tested. - + # deleg.open is set in ``toDelegate``. - + client.socket = c client.lineBuffer = "".TaintedString client.sendBuffer = "" @@ -393,7 +393,7 @@ proc accept*(server: AsyncSocket, client: var AsyncSocket) = proc acceptAddr*(server: AsyncSocket): tuple[sock: AsyncSocket, address: string] {.deprecated.} = ## Equivalent to ``sockets.acceptAddr``. - ## + ## ## **Deprecated since version 0.9.0:** Please use the function above. var client = newAsyncSocket() var address: string = "" @@ -441,17 +441,17 @@ proc isConnected*(s: AsyncSocket): bool = ## Determines whether ``s`` is connected. return s.info == SockConnected proc isListening*(s: AsyncSocket): bool = - ## Determines whether ``s`` is listening for incoming connections. + ## Determines whether ``s`` is listening for incoming connections. return s.info == SockListening proc isConnecting*(s: AsyncSocket): bool = - ## Determines whether ``s`` is connecting. + ## Determines whether ``s`` is connecting. return s.info == SockConnecting proc isClosed*(s: AsyncSocket): bool = ## Determines whether ``s`` has been closed. return s.info == SockClosed proc isSendDataBuffered*(s: AsyncSocket): bool = ## Determines whether ``s`` has data waiting to be sent, i.e. whether this - ## socket's sendBuffer contains data. + ## socket's sendBuffer contains data. return s.sendBuffer.len != 0 proc setHandleWrite*(s: AsyncSocket, @@ -550,7 +550,7 @@ proc send*(sock: AsyncSocket, data: string) = sock.sendBuffer.add(data) sock.deleg.mode = fmReadWrite elif bytesSent != data.len: - sock.sendBuffer.add(data[bytesSent .. -1]) + sock.sendBuffer.add(data[bytesSent .. ^1]) sock.deleg.mode = fmReadWrite proc timeValFromMilliseconds(timeout = 500): Timeval = @@ -561,10 +561,10 @@ proc timeValFromMilliseconds(timeout = 500): Timeval = proc createFdSet(fd: var TFdSet, s: seq[Delegate], m: var int) = FD_ZERO(fd) - for i in items(s): + for i in items(s): m = max(m, int(i.fd)) FD_SET(i.fd, fd) - + proc pruneSocketSet(s: var seq[Delegate], fd: var TFdSet) = var i = 0 var L = s.len @@ -576,16 +576,16 @@ proc pruneSocketSet(s: var seq[Delegate], fd: var TFdSet) = inc(i) setLen(s, L) -proc select(readfds, writefds, exceptfds: var seq[Delegate], +proc select(readfds, writefds, exceptfds: var seq[Delegate], timeout = 500): int = var tv {.noInit.}: Timeval = timeValFromMilliseconds(timeout) - + var rd, wr, ex: TFdSet var m = 0 createFdSet(rd, readfds, m) createFdSet(wr, writefds, m) createFdSet(ex, exceptfds, m) - + if timeout != -1: result = int(select(cint(m+1), addr(rd), addr(wr), addr(ex), addr(tv))) else: @@ -599,7 +599,7 @@ proc poll*(d: Dispatcher, timeout: int = 500): bool = ## This function checks for events on all the delegates in the `PDispatcher`. ## It then proceeds to call the correct event handler. ## - ## This function returns ``True`` if there are file descriptors that are still + ## This function returns ``True`` if there are file descriptors that are still ## open, otherwise ``False``. File descriptors that have been ## closed are immediately removed from the dispatcher automatically. ## @@ -611,7 +611,7 @@ proc poll*(d: Dispatcher, timeout: int = 500): bool = var readDg, writeDg, errorDg: seq[Delegate] = @[] var len = d.delegates.len var dc = 0 - + while dc < len: let deleg = d.delegates[dc] if (deleg.mode != fmWrite or deleg.mode != fmAppend) and deleg.open: @@ -625,20 +625,20 @@ proc poll*(d: Dispatcher, timeout: int = 500): bool = # File/socket has been closed. Remove it from dispatcher. d.delegates[dc] = d.delegates[len-1] dec len - + d.delegates.setLen(len) - + var hasDataBufferedCount = 0 for d in d.delegates: if d.hasDataBuffered(d.deleVal): hasDataBufferedCount.inc() d.handleRead(d.deleVal) if hasDataBufferedCount > 0: return true - + if readDg.len() == 0 and writeDg.len() == 0: ## TODO: Perhaps this shouldn't return if errorDg has something? return false - + if select(readDg, writeDg, errorDg, timeout) != 0: for i in 0..len(d.delegates)-1: if i > len(d.delegates)-1: break # One delegate might've been removed. @@ -651,7 +651,7 @@ proc poll*(d: Dispatcher, timeout: int = 500): bool = deleg.handleWrite(deleg.deleVal) if deleg notin errorDg: deleg.handleError(deleg.deleVal) - + # Execute tasks for i in items(d.delegates): i.task(i.deleVal) @@ -660,11 +660,11 @@ proc len*(disp: Dispatcher): int = ## Retrieves the amount of delegates in ``disp``. return disp.delegates.len -when isMainModule: +when not defined(testing) and isMainModule: proc testConnect(s: AsyncSocket, no: int) = echo("Connected! " & $no) - + proc testRead(s: AsyncSocket, no: int) = echo("Reading! " & $no) var data = "" @@ -675,38 +675,38 @@ when isMainModule: echo(data) echo("Finished reading! " & $no) - proc testAccept(s: AsyncSocket, disp: PDispatcher, no: int) = + proc testAccept(s: AsyncSocket, disp: Dispatcher, no: int) = echo("Accepting client! " & $no) var client: AsyncSocket new(client) var address = "" s.acceptAddr(client, address) echo("Accepted ", address) - client.handleRead = + client.handleRead = proc (s: AsyncSocket) = testRead(s, 2) disp.register(client) proc main = var d = newDispatcher() - + var s = asyncSocket() - s.connect("amber.tenthbit.net", TPort(6667)) - s.handleConnect = + s.connect("amber.tenthbit.net", Port(6667)) + s.handleConnect = proc (s: AsyncSocket) = testConnect(s, 1) - s.handleRead = + s.handleRead = proc (s: AsyncSocket) = testRead(s, 1) d.register(s) - + var server = asyncSocket() server.handleAccept = - proc (s: AsyncSocket) = + proc (s: AsyncSocket) = testAccept(s, d, 78) - server.bindAddr(TPort(5555)) + server.bindAddr(Port(5555)) server.listen() d.register(server) - + while d.poll(-1): discard main() diff --git a/lib/pure/asyncnet.nim b/lib/pure/asyncnet.nim index 76b2bc46c..aadbde824 100644 --- a/lib/pure/asyncnet.nim +++ b/lib/pure/asyncnet.nim @@ -1,7 +1,7 @@ # # # Nim's Runtime Library -# (c) Copyright 2014 Dominik Picheta +# (c) Copyright 2015 Dominik Picheta # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -24,7 +24,7 @@ ## ## Chat server ## ^^^^^^^^^^^ -## +## ## The following example demonstrates a simple chat server. ## ## .. code-block::nim @@ -182,26 +182,30 @@ proc connect*(socket: AsyncSocket, address: string, port: Port, sslSetConnectState(socket.sslHandle) sslLoop(socket, flags, sslDoHandshake(socket.sslHandle)) -proc readInto(buf: cstring, size: int, socket: AsyncSocket, - flags: set[SocketFlag]): Future[int] {.async.} = +template readInto(buf: cstring, size: int, socket: AsyncSocket, + flags: set[SocketFlag]): int = + ## Reads **up to** ``size`` bytes from ``socket`` into ``buf``. Note that + ## this is a template and not a proc. + var res = 0 if socket.isSsl: when defined(ssl): # SSL mode. sslLoop(socket, flags, sslRead(socket.sslHandle, buf, size.cint)) - result = opResult + res = opResult else: - var data = await recv(socket.fd.TAsyncFD, size, flags) - if data.len != 0: - copyMem(buf, addr data[0], data.len) + var recvIntoFut = recvInto(socket.fd.TAsyncFD, buf, size, flags) + yield recvIntoFut # Not in SSL mode. - result = data.len + res = recvIntoFut.read() + res -proc readIntoBuf(socket: AsyncSocket, - flags: set[SocketFlag]): Future[int] {.async.} = - result = await readInto(addr socket.buffer[0], BufferSize, socket, flags) +template readIntoBuf(socket: AsyncSocket, + flags: set[SocketFlag]): int = + var size = readInto(addr socket.buffer[0], BufferSize, socket, flags) socket.currPos = 0 - socket.bufLen = result + socket.bufLen = size + size proc recv*(socket: AsyncSocket, size: int, flags = {SocketFlag.SafeDisconn}): Future[string] {.async.} = @@ -222,10 +226,11 @@ proc recv*(socket: AsyncSocket, size: int, ## to be read then the future will complete with a value of ``""``. if socket.isBuffered: result = newString(size) + shallow(result) let originalBufPos = socket.currPos if socket.bufLen == 0: - let res = await socket.readIntoBuf(flags - {SocketFlag.Peek}) + let res = socket.readIntoBuf(flags - {SocketFlag.Peek}) if res == 0: result.setLen(0) return @@ -236,7 +241,7 @@ proc recv*(socket: AsyncSocket, size: int, if SocketFlag.Peek in flags: # We don't want to get another buffer if we're peeking. break - let res = await socket.readIntoBuf(flags - {SocketFlag.Peek}) + let res = socket.readIntoBuf(flags - {SocketFlag.Peek}) if res == 0: break @@ -251,7 +256,7 @@ proc recv*(socket: AsyncSocket, size: int, result.setLen(read) else: result = newString(size) - let read = await readInto(addr result[0], size, socket, flags) + let read = readInto(addr result[0], size, socket, flags) result.setLen(read) proc send*(socket: AsyncSocket, data: string, @@ -302,15 +307,14 @@ proc accept*(socket: AsyncSocket, retFut.complete(future.read.client) return retFut -proc recvLine*(socket: AsyncSocket, - flags = {SocketFlag.SafeDisconn}): Future[string] {.async.} = - ## Reads a line of data from ``socket``. Returned future will complete once - ## a full line is read or an error occurs. +proc recvLineInto*(socket: AsyncSocket, resString: ptr string, + flags = {SocketFlag.SafeDisconn}) {.async.} = + ## Reads a line of data from ``socket`` into ``resString``. ## ## If a full line is read ``\r\L`` is not ## added to ``line``, however if solely ``\r\L`` is read then ``line`` ## will be set to it. - ## + ## ## If the socket is disconnected, ``line`` will be set to ``""``. ## ## If the socket is disconnected in the middle of a line (before ``\r\L`` @@ -318,27 +322,32 @@ proc recvLine*(socket: AsyncSocket, ## The partial line **will be lost**. ## ## **Warning**: The ``Peek`` flag is not yet implemented. - ## - ## **Warning**: ``recvLine`` on unbuffered sockets assumes that the protocol - ## uses ``\r\L`` to delimit a new line. - template addNLIfEmpty(): stmt = - if result.len == 0: - result.add("\c\L") + ## + ## **Warning**: ``recvLineInto`` on unbuffered sockets assumes that the + ## protocol uses ``\r\L`` to delimit a new line. + ## + ## **Warning**: ``recvLineInto`` currently uses a raw pointer to a string for + ## performance reasons. This will likely change soon to use FutureVars. assert SocketFlag.Peek notin flags ## TODO: + result = newFuture[void]("asyncnet.recvLineInto") + + template addNLIfEmpty(): stmt = + if resString[].len == 0: + resString[].add("\c\L") + if socket.isBuffered: - result = "" if socket.bufLen == 0: - let res = await socket.readIntoBuf(flags) + let res = socket.readIntoBuf(flags) if res == 0: return var lastR = false while true: if socket.currPos >= socket.bufLen: - let res = await socket.readIntoBuf(flags) + let res = socket.readIntoBuf(flags) if res == 0: - result = "" - break + resString[].setLen(0) + return case socket.buffer[socket.currPos] of '\r': @@ -353,24 +362,53 @@ proc recvLine*(socket: AsyncSocket, socket.currPos.inc() return else: - result.add socket.buffer[socket.currPos] + resString[].add socket.buffer[socket.currPos] socket.currPos.inc() else: - result = "" var c = "" while true: - c = await recv(socket, 1, flags) + let recvFut = recv(socket, 1, flags) + c = recvFut.read() if c.len == 0: - return "" + resString[].setLen(0) + return if c == "\r": - c = await recv(socket, 1, flags) # Skip \L + let recvFut = recv(socket, 1, flags) # Skip \L + c = recvFut.read() assert c == "\L" addNLIfEmpty() return elif c == "\L": addNLIfEmpty() return - add(result.string, c) + resString[].add c + +proc recvLine*(socket: AsyncSocket, + flags = {SocketFlag.SafeDisconn}): Future[string] {.async.} = + ## Reads a line of data from ``socket``. Returned future will complete once + ## a full line is read or an error occurs. + ## + ## If a full line is read ``\r\L`` is not + ## added to ``line``, however if solely ``\r\L`` is read then ``line`` + ## will be set to it. + ## + ## If the socket is disconnected, ``line`` will be set to ``""``. + ## + ## If the socket is disconnected in the middle of a line (before ``\r\L`` + ## is read) then line will be set to ``""``. + ## The partial line **will be lost**. + ## + ## **Warning**: The ``Peek`` flag is not yet implemented. + ## + ## **Warning**: ``recvLine`` on unbuffered sockets assumes that the protocol + ## uses ``\r\L`` to delimit a new line. + template addNLIfEmpty(): stmt = + if result.len == 0: + result.add("\c\L") + assert SocketFlag.Peek notin flags ## TODO: + + result = "" + await socket.recvLineInto(addr result, flags) proc listen*(socket: AsyncSocket, backlog = SOMAXCONN) {.tags: [ReadIOEffect].} = ## Marks ``socket`` as accepting connections. @@ -458,7 +496,7 @@ proc isClosed*(socket: AsyncSocket): bool = ## Determines whether the socket has been closed. return socket.closed -when isMainModule: +when not defined(testing) and isMainModule: type TestCases = enum HighClient, LowClient, LowServer @@ -500,11 +538,11 @@ when isMainModule: proc (future: Future[void]) = echo("Send") client.close() - + var f = accept(sock) f.callback = onAccept - + var f = accept(sock) f.callback = onAccept runForever() - + diff --git a/lib/pure/basic2d.nim b/lib/pure/basic2d.nim index f2fc1566b..a344cd053 100644 --- a/lib/pure/basic2d.nim +++ b/lib/pure/basic2d.nim @@ -18,7 +18,7 @@ import strutils ## ## Quick start example: ## -## # Create a matrix wich first rotates, then scales and at last translates +## # Create a matrix which first rotates, then scales and at last translates ## ## var m:TMatrix2d=rotate(DEG90) & scale(2.0) & move(100.0,200.0) ## @@ -256,7 +256,7 @@ proc `$`* (t:TMatrix2d):string {.noInit.} = proc isUniform*(t:TMatrix2d,tol=1.0e-6):bool= ## Checks if the transform is uniform, that is - ## perpendicular axes of equal lenght, which means (for example) + ## perpendicular axes of equal length, which means (for example) ## it cannot transform a circle into an ellipse. ## `tol` is used as tolerance for both equal length comparison ## and perp. comparison. @@ -305,7 +305,7 @@ proc equals*(m1:TMatrix2d,m2:TMatrix2d,tol=1.0e-6):bool= abs(m1.ty-m2.ty)<=tol proc `=~`*(m1,m2:TMatrix2d):bool= - ## Checks if `m1`and `m2` is aproximately equal, using a + ## Checks if `m1`and `m2` is approximately equal, using a ## tolerance of 1e-6. equals(m1,m2) diff --git a/lib/pure/basic3d.nim b/lib/pure/basic3d.nim index c00764fc5..5a943dd05 100644 --- a/lib/pure/basic3d.nim +++ b/lib/pure/basic3d.nim @@ -16,33 +16,33 @@ import times ## Vectors are implemented as direction vectors, ie. when transformed with a matrix ## the translation part of matrix is ignored. The coordinate system used is ## right handed, because its compatible with 2d coordinate system (rotation around -## zaxis equals 2d rotation). +## zaxis equals 2d rotation). ## Operators `+` , `-` , `*` , `/` , `+=` , `-=` , `*=` and `/=` are implemented ## for vectors and scalars. ## ## ## Quick start example: -## -## # Create a matrix wich first rotates, then scales and at last translates -## +## +## # Create a matrix which first rotates, then scales and at last translates +## ## var m:TMatrix3d=rotate(PI,vector3d(1,1,2.5)) & scale(2.0) & move(100.0,200.0,300.0) -## +## ## # Create a 3d point at (100,150,200) and a vector (5,2,3) -## -## var pt:TPoint3d=point3d(100.0,150.0,200.0) -## +## +## var pt:TPoint3d=point3d(100.0,150.0,200.0) +## ## var vec:TVector3d=vector3d(5.0,2.0,3.0) -## -## +## +## ## pt &= m # transforms pt in place -## +## ## var pt2:TPoint3d=pt & m #concatenates pt with m and returns a new point -## +## ## var vec2:TVector3d=vec & m #concatenates vec with m and returns a new vector -type +type TMatrix3d* =object ## Implements a row major 3d matrix, which means ## transformations are applied the order they are concatenated. @@ -53,12 +53,12 @@ type ## [ tx ty tz tw ] ax*,ay*,az*,aw*, bx*,by*,bz*,bw*, cx*,cy*,cz*,cw*, tx*,ty*,tz*,tw*:float TPoint3d* = object - ## Implements a non-homegeneous 2d point stored as + ## Implements a non-homegeneous 2d point stored as ## an `x` , `y` and `z` coordinate. x*,y*,z*:float - TVector3d* = object - ## Implements a 3d **direction vector** stored as - ## an `x` , `y` and `z` coordinate. Direction vector means, + TVector3d* = object + ## Implements a 3d **direction vector** stored as + ## an `x` , `y` and `z` coordinate. Direction vector means, ## that when transforming a vector with a matrix, the translational ## part of the matrix is ignored. x*,y*,z*:float @@ -67,7 +67,7 @@ type # Some forward declarations proc matrix3d*(ax,ay,az,aw,bx,by,bz,bw,cx,cy,cz,cw,tx,ty,tz,tw:float):TMatrix3d {.noInit.} - ## Creates a new 4x4 3d transformation matrix. + ## Creates a new 4x4 3d transformation matrix. ## `ax` , `ay` , `az` is the local x axis. ## `bx` , `by` , `bz` is the local y axis. ## `cx` , `cy` , `cz` is the local z axis. @@ -76,7 +76,7 @@ proc vector3d*(x,y,z:float):TVector3d {.noInit,inline.} ## Returns a new 3d vector (`x`,`y`,`z`) proc point3d*(x,y,z:float):TPoint3d {.noInit,inline.} ## Returns a new 4d point (`x`,`y`,`z`) -proc tryNormalize*(v:var TVector3d):bool +proc tryNormalize*(v:var TVector3d):bool ## Modifies `v` to have a length of 1.0, keeping its angle. ## If `v` has zero length (and thus no angle), it is left unmodified and false is ## returned, otherwise true is returned. @@ -85,7 +85,7 @@ proc tryNormalize*(v:var TVector3d):bool let IDMATRIX*:TMatrix3d=matrix3d( - 1.0,0.0,0.0,0.0, + 1.0,0.0,0.0,0.0, 0.0,1.0,0.0,0.0, 0.0,0.0,1.0,0.0, 0.0,0.0,0.0,1.0) @@ -114,20 +114,20 @@ proc safeArccos(v:float):float= ## due to rounding issues return arccos(clamp(v,-1.0,1.0)) -template makeBinOpVector(s:expr)= +template makeBinOpVector(s:expr)= ## implements binary operators + , - , * and / for vectors - proc s*(a,b:TVector3d):TVector3d {.inline,noInit.} = + proc s*(a,b:TVector3d):TVector3d {.inline,noInit.} = vector3d(s(a.x,b.x),s(a.y,b.y),s(a.z,b.z)) - proc s*(a:TVector3d,b:float):TVector3d {.inline,noInit.} = + proc s*(a:TVector3d,b:float):TVector3d {.inline,noInit.} = vector3d(s(a.x,b),s(a.y,b),s(a.z,b)) - proc s*(a:float,b:TVector3d):TVector3d {.inline,noInit.} = + proc s*(a:float,b:TVector3d):TVector3d {.inline,noInit.} = vector3d(s(a,b.x),s(a,b.y),s(a,b.z)) - -template makeBinOpAssignVector(s:expr)= + +template makeBinOpAssignVector(s:expr)= ## implements inplace binary operators += , -= , /= and *= for vectors - proc s*(a:var TVector3d,b:TVector3d) {.inline.} = + proc s*(a:var TVector3d,b:TVector3d) {.inline.} = s(a.x,b.x) ; s(a.y,b.y) ; s(a.z,b.z) - proc s*(a:var TVector3d,b:float) {.inline.} = + proc s*(a:var TVector3d,b:float) {.inline.} = s(a.x,b) ; s(a.y,b) ; s(a.z,b) @@ -188,20 +188,20 @@ proc scale*(s:float):TMatrix3d {.noInit.} = proc scale*(s:float,org:TPoint3d):TMatrix3d {.noInit.} = ## Returns a new scaling matrix using, `org` as scale origin. - result.setElements(s,0,0,0, 0,s,0,0, 0,0,s,0, + result.setElements(s,0,0,0, 0,s,0,0, 0,0,s,0, org.x-s*org.x,org.y-s*org.y,org.z-s*org.z,1.0) proc stretch*(sx,sy,sz:float):TMatrix3d {.noInit.} = ## Returns new a stretch matrix, which is a ## scale matrix with non uniform scale in x,y and z. result.setElements(sx,0,0,0, 0,sy,0,0, 0,0,sz,0, 0,0,0,1) - + proc stretch*(sx,sy,sz:float,org:TPoint3d):TMatrix3d {.noInit.} = ## Returns a new stretch matrix, which is a ## scale matrix with non uniform scale in x,y and z. ## `org` is used as stretch origin. result.setElements(sx,0,0,0, 0,sy,0,0, 0,0,sz,0, org.x-sx*org.x,org.y-sy*org.y,org.z-sz*org.z,1) - + proc move*(dx,dy,dz:float):TMatrix3d {.noInit.} = ## Returns a new translation matrix. result.setElements(1,0,0,0, 0,1,0,0, 0,0,1,0, dx,dy,dz,1) @@ -235,7 +235,7 @@ proc rotate*(angle:float,axis:TVector3d):TMatrix3d {.noInit.}= uvomc=normax.x*normax.y*omc uwomc=normax.x*normax.z*omc vwomc=normax.y*normax.z*omc - + result.setElements( u2+(1.0-u2)*cs, uvomc+wsi, uwomc-vsi, 0.0, uvomc-wsi, v2+(1.0-v2)*cs, vwomc+usi, 0.0, @@ -248,11 +248,11 @@ proc rotate*(angle:float,org:TPoint3d,axis:TVector3d):TMatrix3d {.noInit.}= # see PDF document http://inside.mines.edu/~gmurray/ArbitraryAxisRotation/ArbitraryAxisRotation.pdf # for how this is computed - + var normax=axis if not normax.tryNormalize: #simplifies matrix computation below a lot raise newException(DivByZeroError,"Cannot rotate around zero length axis") - + let u=normax.x v=normax.y @@ -272,7 +272,7 @@ proc rotate*(angle:float,org:TPoint3d,axis:TVector3d):TMatrix3d {.noInit.}= uvomc=normax.x*normax.y*omc uwomc=normax.x*normax.z*omc vwomc=normax.y*normax.z*omc - + result.setElements( u2+(v2+w2)*cs, uvomc+wsi, uwomc-vsi, 0.0, uvomc-wsi, v2+(u2+w2)*cs, vwomc+usi, 0.0, @@ -305,7 +305,7 @@ proc rotateY*(angle:float):TMatrix3d {.noInit.}= 0,1,0,0, s,0,c,0, 0,0,0,1) - + proc rotateZ*(angle:float):TMatrix3d {.noInit.}= ## Creates a matrix that rotates around the z-axis with `angle` radians, ## which is also called a 'yaw' matrix. @@ -317,19 +317,19 @@ proc rotateZ*(angle:float):TMatrix3d {.noInit.}= -s,c,0,0, 0,0,1,0, 0,0,0,1) - + proc isUniform*(m:TMatrix3d,tol=1.0e-6):bool= - ## Checks if the transform is uniform, that is - ## perpendicular axes of equal lenght, which means (for example) + ## Checks if the transform is uniform, that is + ## perpendicular axes of equal length, which means (for example) ## it cannot transform a sphere into an ellipsoid. - ## `tol` is used as tolerance for both equal length comparison + ## `tol` is used as tolerance for both equal length comparison ## and perpendicular comparison. - + #dot product=0 means perpendicular coord. system, check xaxis vs yaxis and xaxis vs zaxis if abs(m.ax*m.bx+m.ay*m.by+m.az*m.bz)<=tol and # x vs y abs(m.ax*m.cx+m.ay*m.cy+m.az*m.cz)<=tol and #x vs z abs(m.bx*m.cx+m.by*m.cy+m.bz*m.cz)<=tol: #y vs z - + #subtract squared lengths of axes to check if uniform scaling: let sqxlen=(m.ax*m.ax+m.ay*m.ay+m.az*m.az) @@ -340,16 +340,16 @@ proc isUniform*(m:TMatrix3d,tol=1.0e-6):bool= return false - + proc mirror*(planeperp:TVector3d):TMatrix3d {.noInit.}= ## Creates a matrix that mirrors over the plane that has `planeperp` as normal, ## and passes through origo. `planeperp` does not need to be normalized. - + # https://en.wikipedia.org/wiki/Transformation_matrix var n=planeperp if not n.tryNormalize: raise newException(DivByZeroError,"Cannot mirror over a plane with a zero length normal") - + let a=n.x b=n.y @@ -357,7 +357,7 @@ proc mirror*(planeperp:TVector3d):TMatrix3d {.noInit.}= ab=a*b ac=a*c bc=b*c - + result.setElements( 1-2*a*a , -2*ab,-2*ac,0, -2*ab , 1-2*b*b, -2*bc, 0, @@ -376,7 +376,7 @@ proc mirror*(org:TPoint3d,planeperp:TVector3d):TMatrix3d {.noInit.}= var n=planeperp if not n.tryNormalize: raise newException(DivByZeroError,"Cannot mirror over a plane with a zero length normal") - + let a=n.x b=n.y @@ -390,7 +390,7 @@ proc mirror*(org:TPoint3d,planeperp:TVector3d):TMatrix3d {.noInit.}= tx=org.x ty=org.y tz=org.z - + result.setElements( 1-2*aa , -2*ab,-2*ac,0, -2*ab , 1-2*bb, -2*bc, 0, @@ -402,8 +402,8 @@ proc mirror*(org:TPoint3d,planeperp:TVector3d):TMatrix3d {.noInit.}= proc determinant*(m:TMatrix3d):float= ## Computes the determinant of matrix `m`. - - # This computation is gotten from ratsimp(optimize(determinant(m))) + + # This computation is gotten from ratsimp(optimize(determinant(m))) # in maxima CAS let O1=m.cx*m.tw-m.cw*m.tx @@ -423,10 +423,10 @@ proc inverse*(m:TMatrix3d):TMatrix3d {.noInit.}= ## Computes the inverse of matrix `m`. If the matrix ## determinant is zero, thus not invertible, a EDivByZero ## will be raised. - + # this computation comes from optimize(invert(m)) in maxima CAS - - let + + let det=m.determinant O2=m.cy*m.tw-m.cw*m.ty O3=m.cz*m.tw-m.cw*m.tz @@ -464,7 +464,7 @@ proc inverse*(m:TMatrix3d):TMatrix3d {.noInit.}= proc equals*(m1:TMatrix3d,m2:TMatrix3d,tol=1.0e-6):bool= ## Checks if all elements of `m1`and `m2` is equal within ## a given tolerance `tol`. - return + return abs(m1.ax-m2.ax)<=tol and abs(m1.ay-m2.ay)<=tol and abs(m1.az-m2.az)<=tol and @@ -483,14 +483,14 @@ proc equals*(m1:TMatrix3d,m2:TMatrix3d,tol=1.0e-6):bool= abs(m1.tw-m2.tw)<=tol proc `=~`*(m1,m2:TMatrix3d):bool= - ## Checks if `m1` and `m2` is aproximately equal, using a + ## Checks if `m1` and `m2` is approximately equal, using a ## tolerance of 1e-6. equals(m1,m2) - + proc transpose*(m:TMatrix3d):TMatrix3d {.noInit.}= ## Returns the transpose of `m` result.setElements(m.ax,m.bx,m.cx,m.tx,m.ay,m.by,m.cy,m.ty,m.az,m.bz,m.cz,m.tz,m.aw,m.bw,m.cw,m.tw) - + proc getXAxis*(m:TMatrix3d):TVector3d {.noInit.}= ## Gets the local x axis of `m` result.x=m.ax @@ -509,26 +509,26 @@ proc getZAxis*(m:TMatrix3d):TVector3d {.noInit.}= result.y=m.cy result.z=m.cz - + proc `$`*(m:TMatrix3d):string= ## String representation of `m` - return rtos(m.ax) & "," & rtos(m.ay) & "," &rtos(m.az) & "," & rtos(m.aw) & - "\n" & rtos(m.bx) & "," & rtos(m.by) & "," &rtos(m.bz) & "," & rtos(m.bw) & - "\n" & rtos(m.cx) & "," & rtos(m.cy) & "," &rtos(m.cz) & "," & rtos(m.cw) & - "\n" & rtos(m.tx) & "," & rtos(m.ty) & "," &rtos(m.tz) & "," & rtos(m.tw) - + return rtos(m.ax) & "," & rtos(m.ay) & "," & rtos(m.az) & "," & rtos(m.aw) & + "\n" & rtos(m.bx) & "," & rtos(m.by) & "," & rtos(m.bz) & "," & rtos(m.bw) & + "\n" & rtos(m.cx) & "," & rtos(m.cy) & "," & rtos(m.cz) & "," & rtos(m.cw) & + "\n" & rtos(m.tx) & "," & rtos(m.ty) & "," & rtos(m.tz) & "," & rtos(m.tw) + proc apply*(m:TMatrix3d, x,y,z:var float, translate=false)= ## Applies transformation `m` onto `x` , `y` , `z` , optionally ## using the translation part of the matrix. - let + let oldx=x oldy=y oldz=z - + x=m.cx*oldz+m.bx*oldy+m.ax*oldx y=m.cy*oldz+m.by*oldy+m.ay*oldx z=m.cz*oldz+m.bz*oldy+m.az*oldx - + if translate: x+=m.tx y+=m.ty @@ -552,13 +552,13 @@ proc `len=`*(v:var TVector3d,newlen:float) {.noInit.} = ## an arbitrary vector of the requested length is returned. let fac=newlen/v.len - + if newlen==0.0: v.x=0.0 v.y=0.0 v.z=0.0 return - + if fac==Inf or fac==NegInf: #to short for float accuracy #do as good as possible: @@ -588,7 +588,7 @@ proc `&` *(v:TVector3d,m:TMatrix3d):TVector3d {.noInit.} = ## Concatenate vector `v` with a transformation matrix. ## Transforming a vector ignores the translational part ## of the matrix. - + # | AX AY AZ AW | # | X Y Z 1 | * | BX BY BZ BW | # | CX CY CZ CW | @@ -605,12 +605,12 @@ proc `&=` *(v:var TVector3d,m:TMatrix3d) {.noInit.} = ## Applies transformation `m` onto `v` in place. ## Transforming a vector ignores the translational part ## of the matrix. - + # | AX AY AZ AW | # | X Y Z 1 | * | BX BY BZ BW | # | CX CY CZ CW | # | 0 0 0 1 | - + let newx=m.cx*v.z+m.bx*v.y+m.ax*v.x newy=m.cy*v.z+m.by*v.y+m.ay*v.x @@ -620,38 +620,38 @@ proc `&=` *(v:var TVector3d,m:TMatrix3d) {.noInit.} = proc transformNorm*(v:var TVector3d,m:TMatrix3d)= ## Applies a normal direction transformation `m` onto `v` in place. - ## The resulting vector is *not* normalized. Transforming a vector ignores the - ## translational part of the matrix. If the matrix is not invertible + ## The resulting vector is *not* normalized. Transforming a vector ignores the + ## translational part of the matrix. If the matrix is not invertible ## (determinant=0), an EDivByZero will be raised. # transforming a normal is done by transforming # by the transpose of the inverse of the original matrix - + # Major reason this simple function is here is that this function can be optimized in the future, # (possibly by hardware) as well as having a consistent API with the 2d version. v&=transpose(inverse(m)) - + proc transformInv*(v:var TVector3d,m:TMatrix3d)= - ## Applies the inverse of `m` on vector `v`. Transforming a vector ignores - ## the translational part of the matrix. Transforming a vector ignores the + ## Applies the inverse of `m` on vector `v`. Transforming a vector ignores + ## the translational part of the matrix. Transforming a vector ignores the ## translational part of the matrix. ## If the matrix is not invertible (determinant=0), an EDivByZero ## will be raised. - + # Major reason this simple function is here is that this function can be optimized in the future, # (possibly by hardware) as well as having a consistent API with the 2d version. v&=m.inverse - + proc transformNormInv*(vec:var TVector3d,m:TMatrix3d)= ## Applies an inverse normal direction transformation `m` onto `v` in place. - ## This is faster than creating an inverse - ## matrix and transformNorm(...) it. Transforming a vector ignores the + ## This is faster than creating an inverse + ## matrix and transformNorm(...) it. Transforming a vector ignores the ## translational part of the matrix. - + # see vector2d:s equivalent for a deeper look how/why this works vec&=m.transpose -proc tryNormalize*(v:var TVector3d):bool= +proc tryNormalize*(v:var TVector3d):bool= ## Modifies `v` to have a length of 1.0, keeping its angle. ## If `v` has zero length (and thus no angle), it is left unmodified and false is ## returned, otherwise true is returned. @@ -663,26 +663,26 @@ proc tryNormalize*(v:var TVector3d):bool= v.x/=mag v.y/=mag v.z/=mag - + return true -proc normalize*(v:var TVector3d) {.inline.}= +proc normalize*(v:var TVector3d) {.inline.}= ## Modifies `v` to have a length of 1.0, keeping its angle. ## If `v` has zero length, an EDivByZero will be raised. if not tryNormalize(v): raise newException(DivByZeroError,"Cannot normalize zero length vector") proc rotate*(vec:var TVector3d,angle:float,axis:TVector3d)= - ## Rotates `vec` in place, with `angle` radians over `axis`, which passes + ## Rotates `vec` in place, with `angle` radians over `axis`, which passes ## through origo. # see PDF document http://inside.mines.edu/~gmurray/ArbitraryAxisRotation/ArbitraryAxisRotation.pdf # for how this is computed - + var normax=axis if not normax.tryNormalize: raise newException(DivByZeroError,"Cannot rotate around zero length axis") - + let cs=cos(angle) si=sin(angle) @@ -694,11 +694,11 @@ proc rotate*(vec:var TVector3d,angle:float,axis:TVector3d)= y=vec.y z=vec.z uxyzomc=(u*x+v*y+w*z)*omc - + vec.x=u*uxyzomc+x*cs+(v*z-w*y)*si vec.y=v*uxyzomc+y*cs+(w*x-u*z)*si vec.z=w*uxyzomc+z*cs+(u*y-v*x)*si - + proc scale*(v:var TVector3d,s:float)= ## Scales the vector in place with factor `s` v.x*=s @@ -713,12 +713,12 @@ proc stretch*(v:var TVector3d,sx,sy,sz:float)= proc mirror*(v:var TVector3d,planeperp:TVector3d)= ## Computes the mirrored vector of `v` over the plane - ## that has `planeperp` as normal direction. + ## that has `planeperp` as normal direction. ## `planeperp` does not need to be normalized. - + var n=planeperp n.normalize - + let x=v.x y=v.y @@ -729,7 +729,7 @@ proc mirror*(v:var TVector3d,planeperp:TVector3d)= ac=a*c ab=a*b bc=b*c - + v.x= -2*(ac*z+ab*y+a*a*x)+x v.y= -2*(bc*z+b*b*y+ab*x)+y v.z= -2*(c*c*z+bc*y+ac*x)+z @@ -740,7 +740,7 @@ proc `-` *(v:TVector3d):TVector3d= result.x= -v.x result.y= -v.y result.z= -v.z - + # declare templated binary operators makeBinOpVector(`+`) makeBinOpVector(`-`) @@ -752,7 +752,7 @@ makeBinOpAssignVector(`*=`) makeBinOpAssignVector(`/=`) proc dot*(v1,v2:TVector3d):float {.inline.}= - ## Computes the dot product of two vectors. + ## Computes the dot product of two vectors. ## Returns 0.0 if the vectors are perpendicular. return v1.x*v2.x+v1.y*v2.y+v1.z*v2.z @@ -769,12 +769,12 @@ proc cross*(v1,v2:TVector3d):TVector3d {.inline.}= proc equals*(v1,v2:TVector3d,tol=1.0e-6):bool= ## Checks if two vectors approximately equals with a tolerance. return abs(v2.x-v1.x)<=tol and abs(v2.y-v1.y)<=tol and abs(v2.z-v1.z)<=tol - + proc `=~` *(v1,v2:TVector3d):bool= - ## Checks if two vectors approximately equals with a + ## Checks if two vectors approximately equals with a ## hardcoded tolerance 1e-6 equals(v1,v2) - + proc angleTo*(v1,v2:TVector3d):float= ## Returns the smallest angle between v1 and v2, ## which is in range 0-PI @@ -788,7 +788,7 @@ proc angleTo*(v1,v2:TVector3d):float= proc arbitraryAxis*(norm:TVector3d):TMatrix3d {.noInit.}= ## Computes the rotation matrix that would transform ## world z vector into `norm`. The inverse of this matrix - ## is useful to tranform a planar 3d object to 2d space. + ## is useful to transform a planar 3d object to 2d space. ## This is the same algorithm used to interpret DXF and DWG files. const lim=1.0/64.0 var ax,ay,az:TVector3d @@ -801,7 +801,7 @@ proc arbitraryAxis*(norm:TVector3d):TMatrix3d {.noInit.}= ay=cross(norm,ax) ay.normalize() az=cross(ax,ay) - + result.setElements( ax.x,ax.y,ax.z,0.0, ay.x,ay.y,ay.z,0.0, @@ -811,20 +811,20 @@ proc arbitraryAxis*(norm:TVector3d):TMatrix3d {.noInit.}= proc bisect*(v1,v2:TVector3d):TVector3d {.noInit.}= ## Computes the bisector between v1 and v2 as a normalized vector. ## If one of the input vectors has zero length, a normalized version - ## of the other is returned. If both input vectors has zero length, - ## an arbitrary normalized vector `v1`is returned. + ## of the other is returned. If both input vectors has zero length, + ## an arbitrary normalized vector `v1` is returned. var vmag1=v1.len vmag2=v2.len - - # zero length vector equals arbitrary vector, just change + + # zero length vector equals arbitrary vector, just change # magnitude to one to avoid zero division - if vmag1==0.0: + if vmag1==0.0: if vmag2==0: #both are zero length return any normalized vector return XAXIS vmag1=1.0 - if vmag2==0.0: vmag2=1.0 - + if vmag2==0.0: vmag2=1.0 + let x1=v1.x/vmag1 y1=v1.y/vmag1 @@ -832,14 +832,14 @@ proc bisect*(v1,v2:TVector3d):TVector3d {.noInit.}= x2=v2.x/vmag2 y2=v2.y/vmag2 z2=v2.z/vmag2 - + result.x=(x1 + x2) * 0.5 result.y=(y1 + y2) * 0.5 result.z=(z1 + z2) * 0.5 - + if not result.tryNormalize(): # This can happen if vectors are colinear. In this special case - # there are actually inifinitely many bisectors, we select just + # there are actually inifinitely many bisectors, we select just # one of them. result=v1.cross(XAXIS) if result.sqrLen<1.0e-9: @@ -857,14 +857,14 @@ proc point3d*(x,y,z:float):TPoint3d= result.x=x result.y=y result.z=z - + proc sqrDist*(a,b:TPoint3d):float= ## Computes the squared distance between `a`and `b` let dx=b.x-a.x let dy=b.y-a.y let dz=b.z-a.z result=dx*dx+dy*dy+dz*dz - + proc dist*(a,b:TPoint3d):float {.inline.}= ## Computes the absolute distance between `a`and `b` result=sqrt(sqrDist(a,b)) @@ -876,7 +876,7 @@ proc `$` *(p:TPoint3d):string= result.add(rtos(p.y)) result.add(",") result.add(rtos(p.z)) - + proc `&`*(p:TPoint3d,m:TMatrix3d):TPoint3d= ## Concatenates a point `p` with a transform `m`, ## resulting in a new, transformed point. @@ -893,18 +893,18 @@ proc `&=` *(p:var TPoint3d,m:TMatrix3d)= p.x=m.cx*z+m.bx*y+m.ax*x+m.tx p.y=m.cy*z+m.by*y+m.ay*x+m.ty p.z=m.cz*z+m.bz*y+m.az*x+m.tz - + proc transformInv*(p:var TPoint3d,m:TMatrix3d)= ## Applies the inverse of transformation `m` onto `p` in place. ## If the matrix is not invertable (determinant=0) , EDivByZero will ## be raised. - + # can possibly be more optimized in the future so use this function when possible p&=inverse(m) proc `+`*(p:TPoint3d,v:TVector3d):TPoint3d {.noInit,inline.} = - ## Adds a vector `v` to a point `p`, resulting + ## Adds a vector `v` to a point `p`, resulting ## in a new point. result.x=p.x+v.x result.y=p.y+v.y @@ -917,7 +917,7 @@ proc `+=`*(p:var TPoint3d,v:TVector3d) {.noInit,inline.} = p.z+=v.z proc `-`*(p:TPoint3d,v:TVector3d):TPoint3d {.noInit,inline.} = - ## Subtracts a vector `v` from a point `p`, resulting + ## Subtracts a vector `v` from a point `p`, resulting ## in a new point. result.x=p.x-v.x result.y=p.y-v.y @@ -933,37 +933,37 @@ proc `-=`*(p:var TPoint3d,v:TVector3d) {.noInit,inline.} = ## Subtracts a vector `v` from a point `p` in place. p.x-=v.x p.y-=v.y - p.z-=v.z + p.z-=v.z proc equals(p1,p2:TPoint3d,tol=1.0e-6):bool {.inline.}= ## Checks if two points approximately equals with a tolerance. return abs(p2.x-p1.x)<=tol and abs(p2.y-p1.y)<=tol and abs(p2.z-p1.z)<=tol proc `=~`*(p1,p2:TPoint3d):bool {.inline.}= - ## Checks if two vectors approximately equals with a + ## Checks if two vectors approximately equals with a ## hardcoded tolerance 1e-6 equals(p1,p2) proc rotate*(p:var TPoint3d,rad:float,axis:TVector3d)= - ## Rotates point `p` in place `rad` radians about an axis + ## Rotates point `p` in place `rad` radians about an axis ## passing through origo. - + var v=vector3d(p.x,p.y,p.z) v.rotate(rad,axis) # reuse this code here since doing the same thing and quite complicated p.x=v.x p.y=v.y p.z=v.z - + proc rotate*(p:var TPoint3d,angle:float,org:TPoint3d,axis:TVector3d)= - ## Rotates point `p` in place `rad` radians about an axis + ## Rotates point `p` in place `rad` radians about an axis ## passing through `org` - + # see PDF document http://inside.mines.edu/~gmurray/ArbitraryAxisRotation/ArbitraryAxisRotation.pdf # for how this is computed - + var normax=axis normax.normalize - + let cs=cos(angle) omc=1.0-cs @@ -987,17 +987,17 @@ proc rotate*(p:var TPoint3d,angle:float,org:TPoint3d,axis:TVector3d)= bv=b*v cw=c*w uxmvymwz=ux-vy-wz - + p.x=(a*(vv+ww)-u*(bv+cw-uxmvymwz))*omc + x*cs + (b*w+v*z-c*v-w*y)*si p.y=(b*(uu+ww)-v*(au+cw-uxmvymwz))*omc + y*cs + (c*u-a*w+w*x-u*z)*si p.z=(c*(uu+vv)-w*(au+bv-uxmvymwz))*omc + z*cs + (a*v+u*y-b*u-v*x)*si - + proc scale*(p:var TPoint3d,fac:float) {.inline.}= ## Scales a point in place `fac` times with world origo as origin. p.x*=fac p.y*=fac p.z*=fac - + proc scale*(p:var TPoint3d,fac:float,org:TPoint3d){.inline.}= ## Scales the point in place `fac` times with `org` as origin. p.x=(p.x - org.x) * fac + org.x @@ -1005,7 +1005,7 @@ proc scale*(p:var TPoint3d,fac:float,org:TPoint3d){.inline.}= p.z=(p.z - org.z) * fac + org.z proc stretch*(p:var TPoint3d,facx,facy,facz:float){.inline.}= - ## Scales a point in place non uniformly `facx` , `facy` , `facz` times + ## Scales a point in place non uniformly `facx` , `facy` , `facz` times ## with world origo as origin. p.x*=facx p.y*=facy @@ -1017,7 +1017,7 @@ proc stretch*(p:var TPoint3d,facx,facy,facz:float,org:TPoint3d){.inline.}= p.x=(p.x - org.x) * facx + org.x p.y=(p.y - org.y) * facy + org.y p.z=(p.z - org.z) * facz + org.z - + proc move*(p:var TPoint3d,dx,dy,dz:float){.inline.}= ## Translates a point `dx` , `dy` , `dz` in place. @@ -1033,7 +1033,7 @@ proc move*(p:var TPoint3d,v:TVector3d){.inline.}= proc area*(a,b,c:TPoint3d):float {.inline.}= ## Computes the area of the triangle thru points `a` , `b` and `c` - + # The area of a planar 3d quadliteral is the magnitude of the cross # product of two edge vectors. Taking this time 0.5 gives the triangle area. return cross(b-a,c-a).len*0.5 diff --git a/lib/pure/browsers.nim b/lib/pure/browsers.nim index 52035ee48..c6a603318 100644 --- a/lib/pure/browsers.nim +++ b/lib/pure/browsers.nim @@ -42,7 +42,7 @@ proc openDefaultBrowser*(url: string) = for b in getEnv("BROWSER").string.split(PathSep): try: # we use ``startProcess`` here because we don't want to block! - discard startProcess(command=b, args=[url], options={poUseShell}) + discard startProcess(command=b, args=[url], options={poUsePath}) return except OSError: discard diff --git a/lib/pure/collections/LockFreeHash.nim b/lib/pure/collections/LockFreeHash.nim index 5640838b1..0df97c685 100644 --- a/lib/pure/collections/LockFreeHash.nim +++ b/lib/pure/collections/LockFreeHash.nim @@ -404,7 +404,7 @@ proc setVal[K,V](table: var PConcTable[K,V], key: int, val: int, #echo("tomb old slot then set in new table") nextTable = copySlotAndCheck(table,idx) return setVal(nextTable, key, val, expVal, match) - # Finaly ready to add new val to table + # Finally ready to add new val to table while true: if match and oldVal != expVal: #echo("set failed, no match oldVal= " & $oldVal & " expVal= " & $expVal) @@ -525,7 +525,7 @@ proc get*[K,V](table: var PConcTable[K,V], key: var K): V = #Tests ---------------------------- -when isMainModule: +when not defined(testing) and isMainModule: import locks, times, mersenne const diff --git a/lib/pure/collections/critbits.nim b/lib/pure/collections/critbits.nim index 8f506409b..7e3f23851 100644 --- a/lib/pure/collections/critbits.nim +++ b/lib/pure/collections/critbits.nim @@ -12,7 +12,7 @@ ## by Adam Langley. type - NodeObj[T] = object {.pure, final, acyclic.} + NodeObj[T] = object {.acyclic.} byte: int ## byte index of the difference otherbits: char case isLeaf: bool @@ -23,11 +23,10 @@ type val: T Node[T] = ref NodeObj[T] - CritBitTree*[T] = object {. - pure, final.} ## The crit bit tree can either be used - ## as a mapping from strings to - ## some type ``T`` or as a set of - ## strings if ``T`` is void. + CritBitTree*[T] = object ## The crit bit tree can either be used + ## as a mapping from strings to + ## some type ``T`` or as a set of + ## strings if ``T`` is void. root: Node[T] count: int @@ -175,7 +174,7 @@ proc excl*[T](c: var CritBitTree[T], key: string) = iterator leaves[T](n: Node[T]): Node[T] = if n != nil: # XXX actually we could compute the necessary stack size in advance: - # it's rougly log2(c.count). + # it's roughly log2(c.count). var stack = @[n] while stack.len > 0: var it = stack.pop @@ -287,18 +286,19 @@ proc `$`*[T](c: CritBitTree[T]): string = result.add("}") when isMainModule: + import sequtils + var r: CritBitTree[void] r.incl "abc" r.incl "xyz" r.incl "def" r.incl "definition" r.incl "prefix" + doAssert r.contains"def" - #r.del "def" - for w in r.items: - echo w - - for w in r.itemsWithPrefix("de"): - echo w + r.excl "def" + + assert toSeq(r.items) == @["abc", "definition", "prefix", "xyz"] + assert toSeq(r.itemsWithPrefix("de")) == @["definition"] diff --git a/lib/pure/collections/intsets.nim b/lib/pure/collections/intsets.nim index 5d9c3f445..25f6616a6 100644 --- a/lib/pure/collections/intsets.nim +++ b/lib/pure/collections/intsets.nim @@ -8,18 +8,18 @@ # ## The ``intsets`` module implements an efficient int set implemented as a -## sparse bit set. +## `sparse bit set`:idx:. ## **Note**: Since Nim currently does not allow the assignment operator to ## be overloaded, ``=`` for int sets performs some rather meaningless shallow ## copy; use ``assign`` to get a deep copy. import - os, hashes, math + hashes, math type BitScalar = int -const +const InitIntSetSize = 8 # must be a power of two! TrunkShift = 9 BitsPerTrunk = 1 shl TrunkShift # needs to be a power of 2 and @@ -31,11 +31,11 @@ const type PTrunk = ref TTrunk - TTrunk {.final.} = object + TTrunk {.final.} = object next: PTrunk # all nodes are connected with this pointer key: int # start address at bit 0 bits: array[0..IntsPerTrunk - 1, BitScalar] # a bit vector - + TTrunkSeq = seq[PTrunk] IntSet* = object ## an efficient set of 'int' implemented as a sparse bit set counter, max: int @@ -44,42 +44,42 @@ type {.deprecated: [TIntSet: IntSet].} -proc mustRehash(length, counter: int): bool {.inline.} = +proc mustRehash(length, counter: int): bool {.inline.} = assert(length > counter) result = (length * 2 < counter * 3) or (length - counter < 4) -proc nextTry(h, maxHash: THash): THash {.inline.} = - result = ((5 * h) + 1) and maxHash +proc nextTry(h, maxHash: THash): THash {.inline.} = + result = ((5 * h) + 1) and maxHash -proc intSetGet(t: IntSet, key: int): PTrunk = +proc intSetGet(t: IntSet, key: int): PTrunk = var h = key and t.max - while t.data[h] != nil: - if t.data[h].key == key: + while t.data[h] != nil: + if t.data[h].key == key: return t.data[h] h = nextTry(h, t.max) result = nil -proc intSetRawInsert(t: IntSet, data: var TTrunkSeq, desc: PTrunk) = +proc intSetRawInsert(t: IntSet, data: var TTrunkSeq, desc: PTrunk) = var h = desc.key and t.max - while data[h] != nil: + while data[h] != nil: assert(data[h] != desc) h = nextTry(h, t.max) assert(data[h] == nil) data[h] = desc -proc intSetEnlarge(t: var IntSet) = +proc intSetEnlarge(t: var IntSet) = var n: TTrunkSeq var oldMax = t.max t.max = ((t.max + 1) * 2) - 1 newSeq(n, t.max + 1) - for i in countup(0, oldMax): + for i in countup(0, oldMax): if t.data[i] != nil: intSetRawInsert(t, n, t.data[i]) swap(t.data, n) -proc intSetPut(t: var IntSet, key: int): PTrunk = +proc intSetPut(t: var IntSet, key: int): PTrunk = var h = key and t.max - while t.data[h] != nil: - if t.data[h].key == key: + while t.data[h] != nil: + if t.data[h].key == key: return t.data[h] h = nextTry(h, t.max) if mustRehash(t.max + 1, t.counter): intSetEnlarge(t) @@ -94,43 +94,43 @@ proc intSetPut(t: var IntSet, key: int): PTrunk = t.data[h] = result proc contains*(s: IntSet, key: int): bool = - ## returns true iff `key` is in `s`. + ## returns true iff `key` is in `s`. var t = intSetGet(s, `shr`(key, TrunkShift)) - if t != nil: + if t != nil: var u = key and TrunkMask result = (t.bits[`shr`(u, IntShift)] and `shl`(1, u and IntMask)) != 0 - else: + else: result = false - -proc incl*(s: var IntSet, key: int) = + +proc incl*(s: var IntSet, key: int) = ## includes an element `key` in `s`. var t = intSetPut(s, `shr`(key, TrunkShift)) var u = key and TrunkMask t.bits[`shr`(u, IntShift)] = t.bits[`shr`(u, IntShift)] or `shl`(1, u and IntMask) -proc excl*(s: var IntSet, key: int) = +proc excl*(s: var IntSet, key: int) = ## excludes `key` from the set `s`. var t = intSetGet(s, `shr`(key, TrunkShift)) - if t != nil: + if t != nil: var u = key and TrunkMask t.bits[`shr`(u, IntShift)] = t.bits[`shr`(u, IntShift)] and not `shl`(1, u and IntMask) -proc containsOrIncl*(s: var IntSet, key: int): bool = +proc containsOrIncl*(s: var IntSet, key: int): bool = ## returns true if `s` contains `key`, otherwise `key` is included in `s` ## and false is returned. var t = intSetGet(s, `shr`(key, TrunkShift)) - if t != nil: + if t != nil: var u = key and TrunkMask result = (t.bits[`shr`(u, IntShift)] and `shl`(1, u and IntMask)) != 0 - if not result: + if not result: t.bits[`shr`(u, IntShift)] = t.bits[`shr`(u, IntShift)] or `shl`(1, u and IntMask) - else: + else: incl(s, key) result = false - + proc initIntSet*: IntSet = ## creates a new int set that is empty. newSeq(result.data, InitIntSetSize) @@ -140,14 +140,14 @@ proc initIntSet*: IntSet = proc assign*(dest: var IntSet, src: IntSet) = ## copies `src` to `dest`. `dest` does not need to be initialized by - ## `initIntSet`. + ## `initIntSet`. dest.counter = src.counter dest.max = src.max newSeq(dest.data, src.data.len) - + var it = src.head while it != nil: - + var h = it.key and dest.max while dest.data[h] != nil: h = nextTry(h, dest.max) assert(dest.data[h] == nil) @@ -168,7 +168,7 @@ iterator items*(s: IntSet): int {.inline.} = while r != nil: var i = 0 while i <= high(r.bits): - var w = r.bits[i] + var w = r.bits[i] # taking a copy of r.bits[i] here is correct, because # modifying operations are not allowed during traversation var j = 0 @@ -191,20 +191,28 @@ proc `$`*(s: IntSet): string = ## The `$` operator for int sets. dollarImpl() -proc empty*(s: IntSet): bool {.inline.} = +proc empty*(s: IntSet): bool {.inline, deprecated.} = ## returns true if `s` is empty. This is safe to call even before - ## the set has been initialized with `initIntSet`. + ## the set has been initialized with `initIntSet`. Note this never + ## worked reliably and so is deprecated. result = s.counter == 0 when isMainModule: + import sequtils, algorithm + var x = initIntSet() x.incl(1) x.incl(2) x.incl(7) x.incl(1056) - for e in items(x): echo e - var y: TIntSet + var xs = toSeq(items(x)) + xs.sort(cmp[int]) + assert xs == @[1, 2, 7, 1056] + + var y: IntSet assign(y, x) - for e in items(y): echo e + var ys = toSeq(items(y)) + ys.sort(cmp[int]) + assert ys == @[1, 2, 7, 1056] diff --git a/lib/pure/collections/lists.nim b/lib/pure/collections/lists.nim index 929de5973..535d5e21d 100644 --- a/lib/pure/collections/lists.nim +++ b/lib/pure/collections/lists.nim @@ -32,7 +32,7 @@ type head*, tail*: DoublyLinkedNode[T] SinglyLinkedRing*[T] = object ## a singly linked ring - head*: SinglyLinkedNode[T] + head*, tail*: SinglyLinkedNode[T] DoublyLinkedRing*[T] = object ## a doubly linked ring head*: DoublyLinkedNode[T] @@ -122,6 +122,22 @@ iterator items*[T](L: DoublyLinkedRing[T]): T = ## yields every value of `L`. itemsRingImpl() +iterator mitems*[T](L: var DoublyLinkedList[T]): var T = + ## yields every value of `L` so that you can modify it. + itemsListImpl() + +iterator mitems*[T](L: var SinglyLinkedList[T]): var T = + ## yields every value of `L` so that you can modify it. + itemsListImpl() + +iterator mitems*[T](L: var SinglyLinkedRing[T]): var T = + ## yields every value of `L` so that you can modify it. + itemsRingImpl() + +iterator mitems*[T](L: var DoublyLinkedRing[T]): var T = + ## yields every value of `L` so that you can modify it. + itemsRingImpl() + iterator nodes*[T](L: SinglyLinkedList[T]): SinglyLinkedNode[T] = ## iterates over every node of `x`. Removing the current node from the ## list during traversal is supported. @@ -251,13 +267,31 @@ proc remove*[T](L: var DoublyLinkedList[T], n: DoublyLinkedNode[T]) = if n.prev != nil: n.prev.next = n.next +proc append*[T](L: var SinglyLinkedRing[T], n: SinglyLinkedNode[T]) = + ## appends a node `n` to `L`. Efficiency: O(1). + if L.head != nil: + n.next = L.head + assert(L.tail != nil) + L.tail.next = n + L.tail = n + else: + n.next = n + L.head = n + L.tail = n + +proc append*[T](L: var SinglyLinkedRing[T], value: T) = + ## appends a value to `L`. Efficiency: O(1). + append(L, newSinglyLinkedNode(value)) + proc prepend*[T](L: var SinglyLinkedRing[T], n: SinglyLinkedNode[T]) = ## prepends a node `n` to `L`. Efficiency: O(1). - if L.head != nil: - n.next = L.head - L.head.next = n - else: + if L.head != nil: + n.next = L.head + assert(L.tail != nil) + L.tail.next = n + else: n.next = n + L.tail = n L.head = n proc prepend*[T](L: var SinglyLinkedRing[T], value: T) = diff --git a/lib/pure/collections/queues.nim b/lib/pure/collections/queues.nim index defce657f..af5e7b6cd 100644 --- a/lib/pure/collections/queues.nim +++ b/lib/pure/collections/queues.nim @@ -7,7 +7,7 @@ # distribution, for details about the copyright. # -## Implementation of a queue. The underlying implementation uses a ``seq``. +## Implementation of a `queue`:idx:. The underlying implementation uses a ``seq``. ## Note: For inter thread communication use ## a `TChannel <channels.html>`_ instead. @@ -39,6 +39,15 @@ iterator items*[T](q: Queue[T]): T = yield q.data[i] i = (i + 1) and q.mask +iterator mitems*[T](q: var Queue[T]): var T = + ## yields every element of `q`. + var i = q.rd + var c = q.count + while c > 0: + dec c + yield q.data[i] + i = (i + 1) and q.mask + proc add*[T](q: var Queue[T], item: T) = ## adds an `item` to the end of the queue `q`. var cap = q.mask+1 diff --git a/lib/pure/collections/rtarrays.nim b/lib/pure/collections/rtarrays.nim index e5d0b0d8a..9d8085643 100644 --- a/lib/pure/collections/rtarrays.nim +++ b/lib/pure/collections/rtarrays.nim @@ -1,7 +1,7 @@ # # # Nim's Runtime Library -# (c) Copyright 2014 Andreas Rumpf +# (c) Copyright 2015 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. diff --git a/lib/pure/collections/sequtils.nim b/lib/pure/collections/sequtils.nim index befc9bacb..e9cd2cb3c 100644 --- a/lib/pure/collections/sequtils.nim +++ b/lib/pure/collections/sequtils.nim @@ -47,6 +47,24 @@ proc concat*[T](seqs: varargs[seq[T]]): seq[T] = result[i] = itm inc(i) +proc repeat*[T](s: seq[T], n: Natural): seq[T] = + ## Returns a new sequence with the items of `s` repeated `n` times. + ## + ## Example: + ## + ## .. code-block: + ## + ## let + ## s = @[1, 2, 3] + ## total = s.repeat(3) + ## assert total == @[1, 2, 3, 1, 2, 3, 1, 2, 3] + result = newSeq[T](n * s.len) + var o = 0 + for x in 1..n: + for e in s: + result[o] = e + inc o + proc deduplicate*[T](seq1: seq[T]): seq[T] = ## Returns a new sequence without duplicates. ## @@ -63,7 +81,7 @@ proc deduplicate*[T](seq1: seq[T]): seq[T] = if not result.contains(itm): result.add(itm) {.deprecated: [distnct: deduplicate].} - + proc zip*[S, T](seq1: seq[S], seq2: seq[T]): seq[tuple[a: S, b: T]] = ## Returns a new sequence with a combination of the two input sequences. ## @@ -86,7 +104,7 @@ proc zip*[S, T](seq1: seq[S], seq2: seq[T]): seq[tuple[a: S, b: T]] = newSeq(result, m) for i in 0 .. m-1: result[i] = (seq1[i], seq2[i]) -proc distribute*[T](s: seq[T], num: int, spread = true): seq[seq[T]] = +proc distribute*[T](s: seq[T], num: Positive, spread = true): seq[seq[T]] = ## Splits and distributes a sequence `s` into `num` sub sequences. ## ## Returns a sequence of `num` sequences. For some input values this is the @@ -113,11 +131,12 @@ proc distribute*[T](s: seq[T], num: int, spread = true): seq[seq[T]] = ## assert numbers.distribute(6)[0] == @[1, 2] ## assert numbers.distribute(6)[5] == @[7] assert(not s.isNil, "`s` can't be nil") - assert(num > 0, "`num` has to be greater than zero") if num < 2: result = @[s] return + let num = int(num) # XXX probably only needed because of .. bug + # Create the result and calculate the stride size and the remainder if any. result = newSeq[seq[T]](num) var @@ -162,7 +181,7 @@ iterator filter*[T](seq1: seq[T], pred: proc(item: T): bool {.closure.}): T = ## for n in filter(numbers, proc (x: int): bool = x mod 2 == 0): ## echo($n) ## # echoes 4, 8, 4 in separate lines - for i in countup(0, len(seq1) -1): + for i in countup(0, len(seq1)-1): var item = seq1[i] if pred(item): yield seq1[i] @@ -209,7 +228,7 @@ proc delete*[T](s: var seq[T], first=0, last=0) = ## var dest = @[1,1,1,2,2,2,2,2,2,1,1,1,1,1] ## dest.delete(3, 8) ## assert outcome == dest - + var i = first var j = last+1 var newLen = len(s)-j+i @@ -227,12 +246,12 @@ proc insert*[T](dest: var seq[T], src: openArray[T], pos=0) = ## ##.. code-block:: ## var dest = @[1,1,1,1,1,1,1,1] - ## let + ## let ## src = @[2,2,2,2,2,2] ## outcome = @[1,1,1,2,2,2,2,2,2,1,1,1,1,1] ## dest.insert(src, 3) ## assert dest == outcome - + var j = len(dest) - 1 var i = len(dest) + len(src) - 1 dest.setLen(i + 1) @@ -320,7 +339,7 @@ template foldl*(sequence, operation: expr): expr = ## ## The ``operation`` parameter should be an expression which uses the ## variables ``a`` and ``b`` for each step of the fold. Since this is a left - ## fold, for non associative binary operations like substraction think that + ## fold, for non associative binary operations like subtraction think that ## the sequence of numbers 1, 2 and 3 will be parenthesized as (((1) - 2) - ## 3). Example: ## @@ -328,12 +347,12 @@ template foldl*(sequence, operation: expr): expr = ## let ## numbers = @[5, 9, 11] ## addition = foldl(numbers, a + b) - ## substraction = foldl(numbers, a - b) + ## subtraction = foldl(numbers, a - b) ## multiplication = foldl(numbers, a * b) ## words = @["nim", "is", "cool"] ## concatenation = foldl(words, a & b) ## assert addition == 25, "Addition is (((5)+9)+11)" - ## assert substraction == -15, "Substraction is (((5)-9)-11)" + ## assert subtraction == -15, "Subtraction is (((5)-9)-11)" ## assert multiplication == 495, "Multiplication is (((5)*9)*11)" ## assert concatenation == "nimiscool" assert sequence.len > 0, "Can't fold empty sequences" @@ -356,7 +375,7 @@ template foldr*(sequence, operation: expr): expr = ## ## The ``operation`` parameter should be an expression which uses the ## variables ``a`` and ``b`` for each step of the fold. Since this is a right - ## fold, for non associative binary operations like substraction think that + ## fold, for non associative binary operations like subtraction think that ## the sequence of numbers 1, 2 and 3 will be parenthesized as (1 - (2 - ## (3))). Example: ## @@ -364,12 +383,12 @@ template foldr*(sequence, operation: expr): expr = ## let ## numbers = @[5, 9, 11] ## addition = foldr(numbers, a + b) - ## substraction = foldr(numbers, a - b) + ## subtraction = foldr(numbers, a - b) ## multiplication = foldr(numbers, a * b) ## words = @["nim", "is", "cool"] ## concatenation = foldr(words, a & b) ## assert addition == 25, "Addition is (5+(9+(11)))" - ## assert substraction == 7, "Substraction is (5-(9-(11)))" + ## assert subtraction == 7, "Subtraction is (5-(9-(11)))" ## assert multiplication == 495, "Multiplication is (5*(9*(11)))" ## assert concatenation == "nimiscool" assert sequence.len > 0, "Can't fold empty sequences" @@ -382,7 +401,7 @@ template foldr*(sequence, operation: expr): expr = result = operation result -template mapIt*(seq1, typ, pred: expr): expr = +template mapIt*(seq1, typ, op: expr): expr = ## Convenience template around the ``map`` proc to reduce typing. ## ## The template injects the ``it`` variable which you can use directly in an @@ -397,10 +416,10 @@ template mapIt*(seq1, typ, pred: expr): expr = ## assert strings == @["4", "8", "12", "16"] var result {.gensym.}: seq[typ] = @[] for it {.inject.} in items(seq1): - result.add(pred) + result.add(op) result -template mapIt*(varSeq, pred: expr) = +template mapIt*(varSeq, op: expr) = ## Convenience template around the mutable ``map`` proc to reduce typing. ## ## The template injects the ``it`` variable which you can use directly in an @@ -413,7 +432,7 @@ template mapIt*(varSeq, pred: expr) = ## assert nums[0] + nums[3] == 15 for i in 0 .. <len(varSeq): let it {.inject.} = varSeq[i] - varSeq[i] = pred + varSeq[i] = op template newSeqWith*(len: int, init: expr): expr = ## creates a new sequence, calling `init` to initialize each value. Example: @@ -473,9 +492,8 @@ when isMainModule: block: # filter iterator test let numbers = @[1, 4, 5, 8, 9, 7, 4] - for n in filter(numbers, proc (x: int): bool = x mod 2 == 0): - echo($n) - # echoes 4, 8, 4 in separate lines + assert toSeq(filter(numbers, proc (x: int): bool = x mod 2 == 0)) == + @[4, 8, 4] block: # keepIf test var floats = @[13.0, 12.5, 5.8, 2.0, 6.1, 9.9, 10.1] @@ -507,12 +525,12 @@ when isMainModule: let numbers = @[5, 9, 11] addition = foldl(numbers, a + b) - substraction = foldl(numbers, a - b) + subtraction = foldl(numbers, a - b) multiplication = foldl(numbers, a * b) words = @["nim", "is", "cool"] concatenation = foldl(words, a & b) assert addition == 25, "Addition is (((5)+9)+11)" - assert substraction == -15, "Substraction is (((5)-9)-11)" + assert subtraction == -15, "Subtraction is (((5)-9)-11)" assert multiplication == 495, "Multiplication is (((5)*9)*11)" assert concatenation == "nimiscool" @@ -520,12 +538,12 @@ when isMainModule: let numbers = @[5, 9, 11] addition = foldr(numbers, a + b) - substraction = foldr(numbers, a - b) + subtraction = foldr(numbers, a - b) multiplication = foldr(numbers, a * b) words = @["nim", "is", "cool"] concatenation = foldr(words, a & b) assert addition == 25, "Addition is (5+(9+(11)))" - assert substraction == 7, "Substraction is (5-(9-(11)))" + assert subtraction == 7, "Subtraction is (5-(9-(11)))" assert multiplication == 495, "Multiplication is (5*(9*(11)))" assert concatenation == "nimiscool" @@ -534,12 +552,12 @@ when isMainModule: var dest = @[1,1,1,2,2,2,2,2,2,1,1,1,1,1] dest.delete(3, 8) assert outcome == dest, """\ - Deleting range 3-9 from [1,1,1,2,2,2,2,2,2,1,1,1,1,1] + Deleting range 3-9 from [1,1,1,2,2,2,2,2,2,1,1,1,1,1] is [1,1,1,1,1,1,1,1]""" block: # insert tests var dest = @[1,1,1,1,1,1,1,1] - let + let src = @[2,2,2,2,2,2] outcome = @[1,1,1,2,2,2,2,2,2,1,1,1,1,1] dest.insert(src, 3) @@ -587,4 +605,15 @@ when isMainModule: seq2D[0][1] = true doAssert seq2D == @[@[true, true], @[true, false], @[false, false], @[false, false]] - echo "Finished doc tests" + block: # repeat tests + let + a = @[1, 2, 3] + b: seq[int] = @[] + + doAssert a.repeat(3) == @[1, 2, 3, 1, 2, 3, 1, 2, 3] + doAssert a.repeat(0) == @[] + #doAssert a.repeat(-1) == @[] # will not compile! + doAssert b.repeat(3) == @[] + + when not defined(testing): + echo "Finished doc tests" diff --git a/lib/pure/collections/sets.nim b/lib/pure/collections/sets.nim index 92ef3152d..280e0eeba 100644 --- a/lib/pure/collections/sets.nim +++ b/lib/pure/collections/sets.nim @@ -7,7 +7,8 @@ # distribution, for details about the copyright. # -## The ``sets`` module implements an efficient hash set and ordered hash set. +## The ``sets`` module implements an efficient `hash set`:idx: and +## ordered hash set. ## ## Hash sets are different from the `built in set type ## <manual.html#set-type>`_. Sets allow you to store any value that can be @@ -23,9 +24,12 @@ import when not defined(nimhygiene): {.pragma: dirty.} +# For "integer-like A" that are too big for intsets/bit-vectors to be practical, +# it would be best to shrink hcode to the same size as the integer. Larger +# codes should never be needed, and this can pack more entries per cache-line. +# Losing hcode entirely is also possible - if some element value is forbidden. type - SlotEnum = enum seEmpty, seFilled, seDeleted - KeyValuePair[A] = tuple[slot: SlotEnum, key: A] + KeyValuePair[A] = tuple[hcode: THash, key: A] KeyValuePairSeq[A] = seq[KeyValuePair[A]] HashSet* {.myShallow.}[A] = object ## \ ## A generic hash set. @@ -37,6 +41,14 @@ type {.deprecated: [TSet: HashSet].} +# hcode for real keys cannot be zero. hcode==0 signifies an empty slot. These +# two procs retain clarity of that encoding without the space cost of an enum. +proc isEmpty(hcode: THash): bool {.inline.} = + result = hcode == 0 + +proc isFilled(hcode: THash): bool {.inline.} = + result = hcode != 0 + proc isValid*[A](s: HashSet[A]): bool = ## Returns `true` if the set has been initialized with `initSet <#initSet>`_. ## @@ -93,7 +105,7 @@ iterator items*[A](s: HashSet[A]): A = ## # --> {(a: 1, b: 3), (a: 0, b: 4)} assert s.isValid, "The set needs to be initialized." for h in 0..high(s.data): - if s.data[h].slot == seFilled: yield s.data[h].key + if isFilled(s.data[h].hcode): yield s.data[h].key const growthFactor = 2 @@ -102,25 +114,44 @@ proc mustRehash(length, counter: int): bool {.inline.} = assert(length > counter) result = (length * 2 < counter * 3) or (length - counter < 4) -proc nextTry(h, maxHash: THash): THash {.inline.} = - result = ((5 * h) + 1) and maxHash +proc rightSize*(count: Natural): int {.inline.} = + ## Return the value of `initialSize` to support `count` items. + ## + ## If more items are expected to be added, simply add that + ## expected extra amount to the parameter before calling this. + ## + ## Internally, we want mustRehash(rightSize(x), x) == false. + result = nextPowerOfTwo(count * 3 div 2 + 4) -template rawGetImpl() {.dirty.} = - var h: THash = hash(key) and high(s.data) # start with real hash value - while s.data[h].slot != seEmpty: - if s.data[h].key == key and s.data[h].slot == seFilled: +proc nextTry(h, maxHash: THash): THash {.inline.} = + result = (h + 1) and maxHash + +template rawGetKnownHCImpl() {.dirty.} = + var h: THash = hc and high(s.data) # start with real hash value + while isFilled(s.data[h].hcode): + # Compare hc THEN key with boolean short circuit. This makes the common case + # zero ==key's for missing (e.g.inserts) and exactly one ==key for present. + # It does slow down succeeding lookups by one extra THash cmp&and..usually + # just a few clock cycles, generally worth it for any non-integer-like A. + if s.data[h].hcode == hc and s.data[h].key == key: # compare hc THEN key return h h = nextTry(h, high(s.data)) - result = -1 + result = -1 - h # < 0 => MISSING; insert idx = -1 - result + +template rawGetImpl() {.dirty.} = + hc = hash(key) + if hc == 0: # This almost never taken branch should be very predictable. + hc = 314159265 # Value doesn't matter; Any non-zero favorite is fine. + rawGetKnownHCImpl() template rawInsertImpl() {.dirty.} = - var h: THash = hash(key) and high(data) - while data[h].slot == seFilled: - h = nextTry(h, high(data)) data[h].key = key - data[h].slot = seFilled + data[h].hcode = hc -proc rawGet[A](s: HashSet[A], key: A): int = +proc rawGetKnownHC[A](s: HashSet[A], key: A, hc: THash): int {.inline.} = + rawGetKnownHCImpl() + +proc rawGet[A](s: HashSet[A], key: A, hc: var THash): int {.inline.} = rawGetImpl() proc mget*[A](s: var HashSet[A], key: A): var A = @@ -129,7 +160,8 @@ proc mget*[A](s: var HashSet[A], key: A): var A = ## when one overloaded 'hash' and '==' but still needs reference semantics ## for sharing. assert s.isValid, "The set needs to be initialized." - var index = rawGet(s, key) + var hc: THash + var index = rawGet(s, key, hc) if index >= 0: result = s.data[index].key else: raise newException(KeyError, "key not found: " & $key) @@ -146,33 +178,43 @@ proc contains*[A](s: HashSet[A], key: A): bool = ## values.excl(2) ## assert(not values.contains(2)) assert s.isValid, "The set needs to be initialized." - var index = rawGet(s, key) + var hc: THash + var index = rawGet(s, key, hc) result = index >= 0 -proc rawInsert[A](s: var HashSet[A], data: var KeyValuePairSeq[A], key: A) = +proc rawInsert[A](s: var HashSet[A], data: var KeyValuePairSeq[A], key: A, + hc: THash, h: THash) = rawInsertImpl() proc enlarge[A](s: var HashSet[A]) = var n: KeyValuePairSeq[A] newSeq(n, len(s.data) * growthFactor) - for i in countup(0, high(s.data)): - if s.data[i].slot == seFilled: rawInsert(s, n, s.data[i].key) - swap(s.data, n) + swap(s.data, n) # n is now old seq + for i in countup(0, high(n)): + if isFilled(n[i].hcode): + var j = -1 - rawGetKnownHC(s, n[i].key, n[i].hcode) + rawInsert(s, s.data, n[i].key, n[i].hcode, j) template inclImpl() {.dirty.} = - var index = rawGet(s, key) + var hc: THash + var index = rawGet(s, key, hc) if index < 0: - if mustRehash(len(s.data), s.counter): enlarge(s) - rawInsert(s, s.data, key) + if mustRehash(len(s.data), s.counter): + enlarge(s) + index = rawGetKnownHC(s, key, hc) + rawInsert(s, s.data, key, hc, -1 - index) inc(s.counter) template containsOrInclImpl() {.dirty.} = - var index = rawGet(s, key) + var hc: THash + var index = rawGet(s, key, hc) if index >= 0: result = true else: - if mustRehash(len(s.data), s.counter): enlarge(s) - rawInsert(s, s.data, key) + if mustRehash(len(s.data), s.counter): + enlarge(s) + index = rawGetKnownHC(s, key, hc) + rawInsert(s, s.data, key, hc, -1 - index) inc(s.counter) proc incl*[A](s: var HashSet[A], key: A) = @@ -203,6 +245,11 @@ proc incl*[A](s: var HashSet[A], other: HashSet[A]) = assert other.isValid, "The set `other` needs to be initialized." for item in other: incl(s, item) +template doWhile(a: expr, b: stmt): stmt = + while true: + b + if not a: break + proc excl*[A](s: var HashSet[A], key: A) = ## Excludes `key` from the set `s`. ## @@ -214,10 +261,22 @@ proc excl*[A](s: var HashSet[A], key: A) = ## s.excl(2) ## assert s.len == 3 assert s.isValid, "The set needs to be initialized." - var index = rawGet(s, key) - if index >= 0: - s.data[index].slot = seDeleted + var hc: THash + var i = rawGet(s, key, hc) + var msk = high(s.data) + if i >= 0: + s.data[i].hcode = 0 dec(s.counter) + while true: # KnuthV3 Algo6.4R adapted for i=i+1 instead of i=i-1 + var j = i # The correctness of this depends on (h+1) in nextTry, + var r = j # though may be adaptable to other simple sequences. + s.data[i].hcode = 0 # mark current EMPTY + doWhile ((i >= r and r > j) or (r > j and j > i) or (j > i and i >= r)): + i = (i + 1) and msk # increment mod table size + if isEmpty(s.data[i].hcode): # end of collision cluster; So all done + return + r = s.data[i].hcode and msk # "home" location of key@i + shallowCopy(s.data[j], s.data[i]) # data[j] will be marked EMPTY next loop proc excl*[A](s: var HashSet[A], other: HashSet[A]) = ## Excludes everything in `other` from `s`. @@ -253,10 +312,10 @@ proc containsOrIncl*[A](s: var HashSet[A], key: A): bool = proc init*[A](s: var HashSet[A], initialSize=64) = ## Initializes a hash set. ## - ## The `initialSize` parameter needs to be a power of too. You can use - ## `math.nextPowerOfTwo() <math.html#nextPowerOfTwo>`_ to guarantee that at - ## runtime. All set variables have to be initialized before you can use them - ## with other procs from this module with the exception of `isValid() + ## The `initialSize` parameter needs to be a power of two. You can use + ## `math.nextPowerOfTwo() <math.html#nextPowerOfTwo>`_ or `rightSize` to + ## guarantee that at runtime. All set variables must be initialized before + ## use with other procs from this module with the exception of `isValid() ## <#isValid,TSet[A]>`_ and `len() <#len,TSet[A]>`_. ## ## You can call this proc on a previously initialized hash set, which will @@ -294,7 +353,7 @@ proc toSet*[A](keys: openArray[A]): HashSet[A] = ## var numbers = toSet([1, 2, 3, 4, 5]) ## assert numbers.contains(2) ## assert numbers.contains(4) - result = initSet[A](nextPowerOfTwo(keys.len+10)) + result = initSet[A](rightSize(keys.len)) for key in items(keys): result.incl(key) template dollarImpl(): stmt {.dirty.} = @@ -493,7 +552,7 @@ proc map*[A, B](data: HashSet[A], op: proc (x: A): B {.closure.}): HashSet[B] = type OrderedKeyValuePair[A] = tuple[ - slot: SlotEnum, next: int, key: A] + hcode: THash, next: int, key: A] OrderedKeyValuePairSeq[A] = seq[OrderedKeyValuePair[A]] OrderedSet* {.myShallow.}[A] = object ## \ ## A generic hash set that remembers insertion order. @@ -545,7 +604,7 @@ template forAllOrderedPairs(yieldStmt: stmt) {.dirty, immediate.} = var h = s.first while h >= 0: var nxt = s.data[h].next - if s.data[h].slot == seFilled: yieldStmt + if isFilled(s.data[h].hcode): yieldStmt h = nxt iterator items*[A](s: OrderedSet[A]): A = @@ -570,7 +629,10 @@ iterator items*[A](s: OrderedSet[A]): A = forAllOrderedPairs: yield s.data[h].key -proc rawGet[A](s: OrderedSet[A], key: A): int = +proc rawGetKnownHC[A](s: OrderedSet[A], key: A, hc: THash): int {.inline.} = + rawGetKnownHCImpl() + +proc rawGet[A](s: OrderedSet[A], key: A, hc: var THash): int {.inline.} = rawGetImpl() proc contains*[A](s: OrderedSet[A], key: A): bool = @@ -584,11 +646,12 @@ proc contains*[A](s: OrderedSet[A], key: A): bool = ## values.incl(2) ## assert values.contains(2) assert s.isValid, "The set needs to be initialized." - var index = rawGet(s, key) + var hc: THash + var index = rawGet(s, key, hc) result = index >= 0 -proc rawInsert[A](s: var OrderedSet[A], - data: var OrderedKeyValuePairSeq[A], key: A) = +proc rawInsert[A](s: var OrderedSet[A], data: var OrderedKeyValuePairSeq[A], + key: A, hc: THash, h: THash) = rawInsertImpl() data[h].next = -1 if s.first < 0: s.first = h @@ -601,12 +664,13 @@ proc enlarge[A](s: var OrderedSet[A]) = var h = s.first s.first = -1 s.last = -1 + swap(s.data, n) while h >= 0: - var nxt = s.data[h].next - if s.data[h].slot == seFilled: - rawInsert(s, n, s.data[h].key) + var nxt = n[h].next + if isFilled(n[h].hcode): + var j = -1 - rawGetKnownHC(s, n[h].key, n[h].hcode) + rawInsert(s, s.data, n[h].key, n[h].hcode, j) h = nxt - swap(s.data, n) proc incl*[A](s: var OrderedSet[A], key: A) = ## Includes an element `key` in `s`. @@ -654,10 +718,10 @@ proc containsOrIncl*[A](s: var OrderedSet[A], key: A): bool = proc init*[A](s: var OrderedSet[A], initialSize=64) = ## Initializes an ordered hash set. ## - ## The `initialSize` parameter needs to be a power of too. You can use - ## `math.nextPowerOfTwo() <math.html#nextPowerOfTwo>`_ to guarantee that at - ## runtime. All set variables have to be initialized before you can use them - ## with other procs from this module with the exception of `isValid() + ## The `initialSize` parameter needs to be a power of two. You can use + ## `math.nextPowerOfTwo() <math.html#nextPowerOfTwo>`_ or `rightSize` to + ## guarantee that at runtime. All set variables must be initialized before + ## use with other procs from this module with the exception of `isValid() ## <#isValid,TOrderedSet[A]>`_ and `len() <#len,TOrderedSet[A]>`_. ## ## You can call this proc on a previously initialized ordered hash set to @@ -697,7 +761,7 @@ proc toOrderedSet*[A](keys: openArray[A]): OrderedSet[A] = ## var numbers = toOrderedSet([1, 2, 3, 4, 5]) ## assert numbers.contains(2) ## assert numbers.contains(4) - result = initOrderedSet[A](nextPowerOfTwo(keys.len+10)) + result = initOrderedSet[A](rightSize(keys.len)) for key in items(keys): result.incl(key) proc `$`*[A](s: OrderedSet[A]): string = @@ -725,7 +789,7 @@ proc `==`*[A](s, t: OrderedSet[A]): bool = while h >= 0 and g >= 0: var nxh = s.data[h].next var nxg = t.data[g].next - if s.data[h].slot == seFilled and s.data[g].slot == seFilled: + if isFilled(s.data[h].hcode) and isFilled(s.data[g].hcode): if s.data[h].key == s.data[g].key: inc compared else: @@ -734,172 +798,179 @@ proc `==`*[A](s, t: OrderedSet[A]): bool = g = nxg result = compared == s.counter -proc testModule() = - ## Internal micro test to validate docstrings and such. - block isValidTest: - var options: HashSet[string] - proc savePreferences(options: HashSet[string]) = - assert options.isValid, "Pass an initialized set!" - options = initSet[string]() - options.savePreferences - - block lenTest: - var values: HashSet[int] - assert(not values.isValid) - assert values.len == 0 - assert values.card == 0 - - block setIterator: - type pair = tuple[a, b: int] - var a, b = initSet[pair]() - a.incl((2, 3)) - a.incl((3, 2)) - a.incl((2, 3)) - for x, y in a.items: - b.incl((x - 2, y + 1)) - assert a.len == b.card - assert a.len == 2 - #echo b - - block setContains: - var values = initSet[int]() - assert(not values.contains(2)) - values.incl(2) - assert values.contains(2) - values.excl(2) - assert(not values.contains(2)) - - values.incl(4) - var others = toSet([6, 7]) - values.incl(others) - assert values.len == 3 - - values.init - assert values.containsOrIncl(2) == false - assert values.containsOrIncl(2) == true - var - a = toSet([1, 2]) - b = toSet([1]) - b.incl(2) - assert a == b - - block exclusions: - var s = toSet([2, 3, 6, 7]) - s.excl(2) - s.excl(2) - assert s.len == 3 - - var - numbers = toSet([1, 2, 3, 4, 5]) - even = toSet([2, 4, 6, 8]) - numbers.excl(even) - #echo numbers - # --> {1, 3, 5} - - block toSeqAndString: - var a = toSet([2, 4, 5]) - var b = initSet[int]() - for x in [2, 4, 5]: b.incl(x) - assert($a == $b) - #echo a - #echo toSet(["no", "esc'aping", "is \" provided"]) - - #block orderedToSeqAndString: - # echo toOrderedSet([2, 4, 5]) - # echo toOrderedSet(["no", "esc'aping", "is \" provided"]) - - block setOperations: - var - a = toSet(["a", "b"]) - b = toSet(["b", "c"]) - c = union(a, b) - assert c == toSet(["a", "b", "c"]) - var d = intersection(a, b) - assert d == toSet(["b"]) - var e = difference(a, b) - assert e == toSet(["a"]) - var f = symmetricDifference(a, b) - assert f == toSet(["a", "c"]) - assert d < a and d < b - assert((a < a) == false) - assert d <= a and d <= b - assert((a <= a)) - # Alias test. - assert a + b == toSet(["a", "b", "c"]) - assert a * b == toSet(["b"]) - assert a - b == toSet(["a"]) - assert a -+- b == toSet(["a", "c"]) - assert disjoint(a, b) == false - assert disjoint(a, b - a) == true - - block mapSet: - var a = toSet([1, 2, 3]) - var b = a.map(proc (x: int): string = $x) - assert b == toSet(["1", "2", "3"]) - - block isValidTest: - var cards: OrderedSet[string] - proc saveTarotCards(cards: OrderedSet[string]) = - assert cards.isValid, "Pass an initialized set!" - cards = initOrderedSet[string]() - cards.saveTarotCards - - block lenTest: - var values: OrderedSet[int] - assert(not values.isValid) - assert values.len == 0 - assert values.card == 0 - - block setIterator: - type pair = tuple[a, b: int] - var a, b = initOrderedSet[pair]() - a.incl((2, 3)) - a.incl((3, 2)) - a.incl((2, 3)) - for x, y in a.items: - b.incl((x - 2, y + 1)) - assert a.len == b.card - assert a.len == 2 - - #block orderedSetIterator: - # var a = initOrderedSet[int]() - # for value in [9, 2, 1, 5, 1, 8, 4, 2]: - # a.incl(value) - # for value in a.items: - # echo "Got ", value - - block setContains: - var values = initOrderedSet[int]() - assert(not values.contains(2)) - values.incl(2) - assert values.contains(2) - - block toSeqAndString: - var a = toOrderedSet([2, 4, 5]) - var b = initOrderedSet[int]() - for x in [2, 4, 5]: b.incl(x) - assert($a == $b) - assert(a == b) # https://github.com/Araq/Nimrod/issues/1413 - - block initBlocks: - var a: OrderedSet[int] - a.init(4) - a.incl(2) - a.init - assert a.len == 0 and a.isValid - a = initOrderedSet[int](4) - a.incl(2) - assert a.len == 1 - - var b: HashSet[int] - b.init(4) - b.incl(2) - b.init - assert b.len == 0 and b.isValid - b = initSet[int](4) - b.incl(2) - assert b.len == 1 - - echo "Micro tests run successfully." - -when isMainModule and not defined(release): testModule() +when isMainModule and not defined(release): + proc testModule() = + ## Internal micro test to validate docstrings and such. + block isValidTest: + var options: HashSet[string] + proc savePreferences(options: HashSet[string]) = + assert options.isValid, "Pass an initialized set!" + options = initSet[string]() + options.savePreferences + + block lenTest: + var values: HashSet[int] + assert(not values.isValid) + assert values.len == 0 + assert values.card == 0 + + block setIterator: + type pair = tuple[a, b: int] + var a, b = initSet[pair]() + a.incl((2, 3)) + a.incl((3, 2)) + a.incl((2, 3)) + for x, y in a.items: + b.incl((x - 2, y + 1)) + assert a.len == b.card + assert a.len == 2 + #echo b + + block setContains: + var values = initSet[int]() + assert(not values.contains(2)) + values.incl(2) + assert values.contains(2) + values.excl(2) + assert(not values.contains(2)) + + values.incl(4) + var others = toSet([6, 7]) + values.incl(others) + assert values.len == 3 + + values.init + assert values.containsOrIncl(2) == false + assert values.containsOrIncl(2) == true + var + a = toSet([1, 2]) + b = toSet([1]) + b.incl(2) + assert a == b + + block exclusions: + var s = toSet([2, 3, 6, 7]) + s.excl(2) + s.excl(2) + assert s.len == 3 + + var + numbers = toSet([1, 2, 3, 4, 5]) + even = toSet([2, 4, 6, 8]) + numbers.excl(even) + #echo numbers + # --> {1, 3, 5} + + block toSeqAndString: + var a = toSet([2, 4, 5]) + var b = initSet[int]() + for x in [2, 4, 5]: b.incl(x) + assert($a == $b) + #echo a + #echo toSet(["no", "esc'aping", "is \" provided"]) + + #block orderedToSeqAndString: + # echo toOrderedSet([2, 4, 5]) + # echo toOrderedSet(["no", "esc'aping", "is \" provided"]) + + block setOperations: + var + a = toSet(["a", "b"]) + b = toSet(["b", "c"]) + c = union(a, b) + assert c == toSet(["a", "b", "c"]) + var d = intersection(a, b) + assert d == toSet(["b"]) + var e = difference(a, b) + assert e == toSet(["a"]) + var f = symmetricDifference(a, b) + assert f == toSet(["a", "c"]) + assert d < a and d < b + assert((a < a) == false) + assert d <= a and d <= b + assert((a <= a)) + # Alias test. + assert a + b == toSet(["a", "b", "c"]) + assert a * b == toSet(["b"]) + assert a - b == toSet(["a"]) + assert a -+- b == toSet(["a", "c"]) + assert disjoint(a, b) == false + assert disjoint(a, b - a) == true + + block mapSet: + var a = toSet([1, 2, 3]) + var b = a.map(proc (x: int): string = $x) + assert b == toSet(["1", "2", "3"]) + + block isValidTest: + var cards: OrderedSet[string] + proc saveTarotCards(cards: OrderedSet[string]) = + assert cards.isValid, "Pass an initialized set!" + cards = initOrderedSet[string]() + cards.saveTarotCards + + block lenTest: + var values: OrderedSet[int] + assert(not values.isValid) + assert values.len == 0 + assert values.card == 0 + + block setIterator: + type pair = tuple[a, b: int] + var a, b = initOrderedSet[pair]() + a.incl((2, 3)) + a.incl((3, 2)) + a.incl((2, 3)) + for x, y in a.items: + b.incl((x - 2, y + 1)) + assert a.len == b.card + assert a.len == 2 + + #block orderedSetIterator: + # var a = initOrderedSet[int]() + # for value in [9, 2, 1, 5, 1, 8, 4, 2]: + # a.incl(value) + # for value in a.items: + # echo "Got ", value + + block setContains: + var values = initOrderedSet[int]() + assert(not values.contains(2)) + values.incl(2) + assert values.contains(2) + + block toSeqAndString: + var a = toOrderedSet([2, 4, 5]) + var b = initOrderedSet[int]() + for x in [2, 4, 5]: b.incl(x) + assert($a == $b) + assert(a == b) # https://github.com/Araq/Nimrod/issues/1413 + + block initBlocks: + var a: OrderedSet[int] + a.init(4) + a.incl(2) + a.init + assert a.len == 0 and a.isValid + a = initOrderedSet[int](4) + a.incl(2) + assert a.len == 1 + + var b: HashSet[int] + b.init(4) + b.incl(2) + b.init + assert b.len == 0 and b.isValid + b = initSet[int](4) + b.incl(2) + assert b.len == 1 + + for i in 0 .. 32: + var s = rightSize(i) + if s <= i or mustRehash(s, i): + echo "performance issue: rightSize() will not elide enlarge() at ", i + + when not defined(testing): + echo "Micro tests run successfully." + + testModule() diff --git a/lib/pure/collections/tables.nim b/lib/pure/collections/tables.nim index 28b7fde92..232e52c89 100644 --- a/lib/pure/collections/tables.nim +++ b/lib/pure/collections/tables.nim @@ -1,16 +1,17 @@ # # # Nim's Runtime Library -# (c) Copyright 2014 Andreas Rumpf +# (c) Copyright 2015 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. # -## The ``tables`` module implements variants of an efficient hash table that is +## The ``tables`` module implements variants of an efficient `hash table`:idx: +## (also often named `dictionary`:idx: in other programming languages) that is ## a mapping from keys to values. ``Table`` is the usual hash table, ## ``OrderedTable`` is like ``Table`` but remembers insertion order -## and ``CountTable`` is a mapping from a key to its number of occurances. +## and ``CountTable`` is a mapping from a key to its number of occurrences. ## For consistency with every other data type in Nim these have **value** ## semantics, this means that ``=`` performs a copy of the hash table. ## For **reference** semantics use the ``Ref`` variant: ``TableRef``, @@ -70,8 +71,7 @@ import {.pragma: myShallow.} type - SlotEnum = enum seEmpty, seFilled, seDeleted - KeyValuePair[A, B] = tuple[slot: SlotEnum, key: A, val: B] + KeyValuePair[A, B] = tuple[hcode: THash, key: A, val: B] KeyValuePairSeq[A, B] = seq[KeyValuePair[A, B]] Table* {.myShallow.}[A, B] = object ## generic hash table data: KeyValuePairSeq[A, B] @@ -83,35 +83,43 @@ type when not defined(nimhygiene): {.pragma: dirty.} +# hcode for real keys cannot be zero. hcode==0 signifies an empty slot. These +# two procs retain clarity of that encoding without the space cost of an enum. +proc isEmpty(hcode: THash): bool {.inline.} = + result = hcode == 0 + +proc isFilled(hcode: THash): bool {.inline.} = + result = hcode != 0 + proc len*[A, B](t: Table[A, B]): int = ## returns the number of keys in `t`. result = t.counter -iterator pairs*[A, B](t: Table[A, B]): tuple[key: A, val: B] = +iterator pairs*[A, B](t: Table[A, B]): (A, B) = ## iterates over any (key, value) pair in the table `t`. for h in 0..high(t.data): - if t.data[h].slot == seFilled: yield (t.data[h].key, t.data[h].val) + if isFilled(t.data[h].hcode): yield (t.data[h].key, t.data[h].val) -iterator mpairs*[A, B](t: var Table[A, B]): tuple[key: A, val: var B] = +iterator mpairs*[A, B](t: var Table[A, B]): (A, var B) = ## iterates over any (key, value) pair in the table `t`. The values ## can be modified. for h in 0..high(t.data): - if t.data[h].slot == seFilled: yield (t.data[h].key, t.data[h].val) + if isFilled(t.data[h].hcode): yield (t.data[h].key, t.data[h].val) iterator keys*[A, B](t: Table[A, B]): A = ## iterates over any key in the table `t`. for h in 0..high(t.data): - if t.data[h].slot == seFilled: yield t.data[h].key + if isFilled(t.data[h].hcode): yield t.data[h].key iterator values*[A, B](t: Table[A, B]): B = ## iterates over any value in the table `t`. for h in 0..high(t.data): - if t.data[h].slot == seFilled: yield t.data[h].val + if isFilled(t.data[h].hcode): yield t.data[h].val iterator mvalues*[A, B](t: var Table[A, B]): var B = ## iterates over any value in the table `t`. The values can be modified. for h in 0..high(t.data): - if t.data[h].slot == seFilled: yield t.data[h].val + if isFilled(t.data[h].hcode): yield t.data[h].val const growthFactor = 2 @@ -120,26 +128,57 @@ proc mustRehash(length, counter: int): bool {.inline.} = assert(length > counter) result = (length * 2 < counter * 3) or (length - counter < 4) +proc rightSize*(count: Natural): int {.inline.} = + ## Return the value of `initialSize` to support `count` items. + ## + ## If more items are expected to be added, simply add that + ## expected extra amount to the parameter before calling this. + ## + ## Internally, we want mustRehash(rightSize(x), x) == false. + result = nextPowerOfTwo(count * 3 div 2 + 4) + proc nextTry(h, maxHash: THash): THash {.inline.} = - result = ((5 * h) + 1) and maxHash + result = (h + 1) and maxHash + +template rawGetKnownHCImpl() {.dirty.} = + var h: THash = hc and high(t.data) # start with real hash value + while isFilled(t.data[h].hcode): + # Compare hc THEN key with boolean short circuit. This makes the common case + # zero ==key's for missing (e.g.inserts) and exactly one ==key for present. + # It does slow down succeeding lookups by one extra THash cmp&and..usually + # just a few clock cycles, generally worth it for any non-integer-like A. + if t.data[h].hcode == hc and t.data[h].key == key: + return h + h = nextTry(h, high(t.data)) + result = -1 - h # < 0 => MISSING; insert idx = -1 - result template rawGetImpl() {.dirty.} = - var h: THash = hash(key) and high(t.data) # start with real hash value - while t.data[h].slot != seEmpty: - if t.data[h].key == key and t.data[h].slot == seFilled: - return h + hc = hash(key) + if hc == 0: # This almost never taken branch should be very predictable. + hc = 314159265 # Value doesn't matter; Any non-zero favorite is fine. + rawGetKnownHCImpl() + +template rawGetDeepImpl() {.dirty.} = # Search algo for unconditional add + hc = hash(key) + if hc == 0: + hc = 314159265 + var h: THash = hc and high(t.data) + while isFilled(t.data[h].hcode): h = nextTry(h, high(t.data)) - result = -1 + result = h template rawInsertImpl() {.dirty.} = - var h: THash = hash(key) and high(data) - while data[h].slot == seFilled: - h = nextTry(h, high(data)) data[h].key = key data[h].val = val - data[h].slot = seFilled + data[h].hcode = hc + +proc rawGetKnownHC[A, B](t: Table[A, B], key: A, hc: THash): int {.inline.} = + rawGetKnownHCImpl() + +proc rawGetDeep[A, B](t: Table[A, B], key: A, hc: var THash): int {.inline.} = + rawGetDeepImpl() -proc rawGet[A, B](t: Table[A, B], key: A): int = +proc rawGet[A, B](t: Table[A, B], key: A, hc: var THash): int {.inline.} = rawGetImpl() proc `[]`*[A, B](t: Table[A, B], key: A): B = @@ -147,63 +186,91 @@ proc `[]`*[A, B](t: Table[A, B], key: A): B = ## default empty value for the type `B` is returned ## and no exception is raised. One can check with ``hasKey`` whether the key ## exists. - var index = rawGet(t, key) + var hc: THash + var index = rawGet(t, key, hc) if index >= 0: result = t.data[index].val proc mget*[A, B](t: var Table[A, B], key: A): var B = ## retrieves the value at ``t[key]``. The value can be modified. - ## If `key` is not in `t`, the ``EInvalidKey`` exception is raised. - var index = rawGet(t, key) + ## If `key` is not in `t`, the ``KeyError`` exception is raised. + var hc: THash + var index = rawGet(t, key, hc) if index >= 0: result = t.data[index].val - else: raise newException(KeyError, "key not found: " & $key) + else: + when compiles($key): + raise newException(KeyError, "key not found: " & $key) + else: + raise newException(KeyError, "key not found") iterator allValues*[A, B](t: Table[A, B]; key: A): B = ## iterates over any value in the table `t` that belongs to the given `key`. var h: THash = hash(key) and high(t.data) - while t.data[h].slot != seEmpty: - if t.data[h].key == key and t.data[h].slot == seFilled: + while isFilled(t.data[h].hcode): + if t.data[h].key == key: yield t.data[h].val h = nextTry(h, high(t.data)) proc hasKey*[A, B](t: Table[A, B], key: A): bool = ## returns true iff `key` is in the table `t`. - result = rawGet(t, key) >= 0 + var hc: THash + result = rawGet(t, key, hc) >= 0 proc rawInsert[A, B](t: var Table[A, B], data: var KeyValuePairSeq[A, B], - key: A, val: B) = + key: A, val: B, hc: THash, h: THash) = rawInsertImpl() proc enlarge[A, B](t: var Table[A, B]) = var n: KeyValuePairSeq[A, B] newSeq(n, len(t.data) * growthFactor) - for i in countup(0, high(t.data)): - if t.data[i].slot == seFilled: rawInsert(t, n, t.data[i].key, t.data[i].val) swap(t.data, n) + for i in countup(0, high(n)): + if isFilled(n[i].hcode): + var j = -1 - rawGetKnownHC(t, n[i].key, n[i].hcode) + rawInsert(t, t.data, n[i].key, n[i].val, n[i].hcode, j) template addImpl() {.dirty.} = if mustRehash(len(t.data), t.counter): enlarge(t) - rawInsert(t, t.data, key, val) + var hc: THash + var j = rawGetDeep(t, key, hc) + rawInsert(t, t.data, key, val, hc, j) + inc(t.counter) + +template maybeRehashPutImpl() {.dirty.} = + if mustRehash(len(t.data), t.counter): + enlarge(t) + index = rawGetKnownHC(t, key, hc) + index = -1 - index # important to transform for mgetOrPutImpl + rawInsert(t, t.data, key, val, hc, index) inc(t.counter) template putImpl() {.dirty.} = - var index = rawGet(t, key) - if index >= 0: - t.data[index].val = val - else: - addImpl() - -when false: - # not yet used: - template hasKeyOrPutImpl() {.dirty.} = - var index = rawGet(t, key) - if index >= 0: - t.data[index].val = val - result = true - else: - if mustRehash(len(t.data), t.counter): enlarge(t) - rawInsert(t, t.data, key, val) - inc(t.counter) - result = false + var hc: THash + var index = rawGet(t, key, hc) + if index >= 0: t.data[index].val = val + else: maybeRehashPutImpl() + +template mgetOrPutImpl() {.dirty.} = + var hc: THash + var index = rawGet(t, key, hc) + if index < 0: maybeRehashPutImpl() # not present: insert (flipping index) + result = t.data[index].val # either way return modifiable val + +template hasKeyOrPutImpl() {.dirty.} = + var hc: THash + var index = rawGet(t, key, hc) + if index < 0: + result = false + maybeRehashPutImpl() + else: result = true + +proc mgetOrPut*[A, B](t: var Table[A, B], key: A, val: B): var B = + ## retrieves value at ``t[key]`` or puts ``val`` if not present, either way + ## returning a value which can be modified. + mgetOrPutImpl() + +proc hasKeyOrPut*[A, B](t: var Table[A, B], key: A, val: B): bool = + ## returns true iff `key` is in the table, otherwise inserts `value`. + hasKeyOrPutImpl() proc `[]=`*[A, B](t: var Table[A, B], key: A, val: B) = ## puts a (key, value)-pair into `t`. @@ -212,28 +279,45 @@ proc `[]=`*[A, B](t: var Table[A, B], key: A, val: B) = proc add*[A, B](t: var Table[A, B], key: A, val: B) = ## puts a new (key, value)-pair into `t` even if ``t[key]`` already exists. addImpl() - + +template doWhile(a: expr, b: stmt): stmt = + while true: + b + if not a: break + proc del*[A, B](t: var Table[A, B], key: A) = ## deletes `key` from hash table `t`. - let index = rawGet(t, key) - if index >= 0: - t.data[index].slot = seDeleted + var hc: THash + var i = rawGet(t, key, hc) + let msk = high(t.data) + if i >= 0: + t.data[i].hcode = 0 dec(t.counter) + while true: # KnuthV3 Algo6.4R adapted for i=i+1 instead of i=i-1 + var j = i # The correctness of this depends on (h+1) in nextTry, + var r = j # though may be adaptable to other simple sequences. + t.data[i].hcode = 0 # mark current EMPTY + doWhile ((i >= r and r > j) or (r > j and j > i) or (j > i and i >= r)): + i = (i + 1) and msk # increment mod table size + if isEmpty(t.data[i].hcode): # end of collision cluster; So all done + return + r = t.data[i].hcode and msk # "home" location of key@i + shallowCopy(t.data[j], t.data[i]) # data[j] will be marked EMPTY next loop proc initTable*[A, B](initialSize=64): Table[A, B] = ## creates a new hash table that is empty. ## ## `initialSize` needs to be a power of two. If you need to accept runtime ## values for this you could use the ``nextPowerOfTwo`` proc from the - ## `math <math.html>`_ module. + ## `math <math.html>`_ module or the ``rightSize`` proc from this module. assert isPowerOfTwo(initialSize) result.counter = 0 newSeq(result.data, initialSize) -proc toTable*[A, B](pairs: openArray[tuple[key: A, - val: B]]): Table[A, B] = +proc toTable*[A, B](pairs: openArray[(A, + B)]): Table[A, B] = ## creates a new hash table that contains the given `pairs`. - result = initTable[A, B](nextPowerOfTwo(pairs.len+10)) + result = initTable[A, B](rightSize(pairs.len)) for key, val in items(pairs): result[key] = val template dollarImpl(): stmt {.dirty.} = @@ -251,7 +335,7 @@ template dollarImpl(): stmt {.dirty.} = proc `$`*[A, B](t: Table[A, B]): string = ## The `$` operator for hash tables. dollarImpl() - + template equalsImpl() = if s.counter == t.counter: # different insertion orders mean different 'data' seqs, so we have @@ -261,10 +345,10 @@ template equalsImpl() = if not t.hasKey(key): return false if t[key] != val: return false return true - + proc `==`*[A, B](s, t: Table[A, B]): bool = equalsImpl() - + proc indexBy*[A, B, C](collection: A, index: proc(x: B): C): Table[C, B] = ## Index the collection with the proc provided. # TODO: As soon as supported, change collection: A to collection: A[B] @@ -276,31 +360,31 @@ proc len*[A, B](t: TableRef[A, B]): int = ## returns the number of keys in `t`. result = t.counter -iterator pairs*[A, B](t: TableRef[A, B]): tuple[key: A, val: B] = +iterator pairs*[A, B](t: TableRef[A, B]): (A, B) = ## iterates over any (key, value) pair in the table `t`. for h in 0..high(t.data): - if t.data[h].slot == seFilled: yield (t.data[h].key, t.data[h].val) + if isFilled(t.data[h].hcode): yield (t.data[h].key, t.data[h].val) -iterator mpairs*[A, B](t: TableRef[A, B]): tuple[key: A, val: var B] = +iterator mpairs*[A, B](t: TableRef[A, B]): (A, var B) = ## iterates over any (key, value) pair in the table `t`. The values ## can be modified. for h in 0..high(t.data): - if t.data[h].slot == seFilled: yield (t.data[h].key, t.data[h].val) + if isFilled(t.data[h].hcode): yield (t.data[h].key, t.data[h].val) iterator keys*[A, B](t: TableRef[A, B]): A = ## iterates over any key in the table `t`. for h in 0..high(t.data): - if t.data[h].slot == seFilled: yield t.data[h].key + if isFilled(t.data[h].hcode): yield t.data[h].key iterator values*[A, B](t: TableRef[A, B]): B = ## iterates over any value in the table `t`. for h in 0..high(t.data): - if t.data[h].slot == seFilled: yield t.data[h].val + if isFilled(t.data[h].hcode): yield t.data[h].val iterator mvalues*[A, B](t: TableRef[A, B]): var B = ## iterates over any value in the table `t`. The values can be modified. for h in 0..high(t.data): - if t.data[h].slot == seFilled: yield t.data[h].val + if isFilled(t.data[h].hcode): yield t.data[h].val proc `[]`*[A, B](t: TableRef[A, B], key: A): B = ## retrieves the value at ``t[key]``. If `key` is not in `t`, @@ -314,6 +398,15 @@ proc mget*[A, B](t: TableRef[A, B], key: A): var B = ## If `key` is not in `t`, the ``EInvalidKey`` exception is raised. t[].mget(key) +proc mgetOrPut*[A, B](t: TableRef[A, B], key: A, val: B): var B = + ## retrieves value at ``t[key]`` or puts ``val`` if not present, either way + ## returning a value which can be modified. + t[].mgetOrPut(key, val) + +proc hasKeyOrPut*[A, B](t: var TableRef[A, B], key: A, val: B): bool = + ## returns true iff `key` is in the table, otherwise inserts `value`. + t[].hasKeyOrPut(key, val) + proc hasKey*[A, B](t: TableRef[A, B], key: A): bool = ## returns true iff `key` is in the table `t`. result = t[].hasKey(key) @@ -325,7 +418,7 @@ proc `[]=`*[A, B](t: TableRef[A, B], key: A, val: B) = proc add*[A, B](t: TableRef[A, B], key: A, val: B) = ## puts a new (key, value)-pair into `t` even if ``t[key]`` already exists. t[].add(key, val) - + proc del*[A, B](t: TableRef[A, B], key: A) = ## deletes `key` from hash table `t`. t[].del(key) @@ -334,7 +427,7 @@ proc newTable*[A, B](initialSize=64): TableRef[A, B] = new(result) result[] = initTable[A, B](initialSize) -proc newTable*[A, B](pairs: openArray[tuple[key: A, val: B]]): TableRef[A, B] = +proc newTable*[A, B](pairs: openArray[(A, B)]): TableRef[A, B] = ## creates a new hash table that contains the given `pairs`. new(result) result[] = toTable[A, B](pairs) @@ -346,7 +439,7 @@ proc `$`*[A, B](t: TableRef[A, B]): string = proc `==`*[A, B](s, t: TableRef[A, B]): bool = if isNil(s): result = isNil(t) elif isNil(t): result = false - else: result = equalsImpl() + else: equalsImpl() proc newTableFrom*[A, B, C](collection: A, index: proc(x: B): C): TableRef[C, B] = ## Index the collection with the proc provided. @@ -359,7 +452,7 @@ proc newTableFrom*[A, B, C](collection: A, index: proc(x: B): C): TableRef[C, B] type OrderedKeyValuePair[A, B] = tuple[ - slot: SlotEnum, next: int, key: A, val: B] + hcode: THash, next: int, key: A, val: B] OrderedKeyValuePairSeq[A, B] = seq[OrderedKeyValuePair[A, B]] OrderedTable* {. myShallow.}[A, B] = object ## table that remembers insertion order @@ -377,16 +470,16 @@ template forAllOrderedPairs(yieldStmt: stmt) {.dirty, immediate.} = var h = t.first while h >= 0: var nxt = t.data[h].next - if t.data[h].slot == seFilled: yieldStmt + if isFilled(t.data[h].hcode): yieldStmt h = nxt -iterator pairs*[A, B](t: OrderedTable[A, B]): tuple[key: A, val: B] = +iterator pairs*[A, B](t: OrderedTable[A, B]): (A, B) = ## iterates over any (key, value) pair in the table `t` in insertion ## order. forAllOrderedPairs: yield (t.data[h].key, t.data[h].val) -iterator mpairs*[A, B](t: var OrderedTable[A, B]): tuple[key: A, val: var B] = +iterator mpairs*[A, B](t: var OrderedTable[A, B]): (A, var B) = ## iterates over any (key, value) pair in the table `t` in insertion ## order. The values can be modified. forAllOrderedPairs: @@ -408,7 +501,13 @@ iterator mvalues*[A, B](t: var OrderedTable[A, B]): var B = forAllOrderedPairs: yield t.data[h].val -proc rawGet[A, B](t: OrderedTable[A, B], key: A): int = +proc rawGetKnownHC[A, B](t: OrderedTable[A, B], key: A, hc: THash): int = + rawGetKnownHCImpl() + +proc rawGetDeep[A, B](t: OrderedTable[A, B], key: A, hc: var THash): int {.inline.} = + rawGetDeepImpl() + +proc rawGet[A, B](t: OrderedTable[A, B], key: A, hc: var THash): int = rawGetImpl() proc `[]`*[A, B](t: OrderedTable[A, B], key: A): B = @@ -416,23 +515,26 @@ proc `[]`*[A, B](t: OrderedTable[A, B], key: A): B = ## default empty value for the type `B` is returned ## and no exception is raised. One can check with ``hasKey`` whether the key ## exists. - var index = rawGet(t, key) + var hc: THash + var index = rawGet(t, key, hc) if index >= 0: result = t.data[index].val proc mget*[A, B](t: var OrderedTable[A, B], key: A): var B = ## retrieves the value at ``t[key]``. The value can be modified. ## If `key` is not in `t`, the ``EInvalidKey`` exception is raised. - var index = rawGet(t, key) + var hc: THash + var index = rawGet(t, key, hc) if index >= 0: result = t.data[index].val else: raise newException(KeyError, "key not found: " & $key) proc hasKey*[A, B](t: OrderedTable[A, B], key: A): bool = ## returns true iff `key` is in the table `t`. - result = rawGet(t, key) >= 0 + var hc: THash + result = rawGet(t, key, hc) >= 0 -proc rawInsert[A, B](t: var OrderedTable[A, B], +proc rawInsert[A, B](t: var OrderedTable[A, B], data: var OrderedKeyValuePairSeq[A, B], - key: A, val: B) = + key: A, val: B, hc: THash, h: THash) = rawInsertImpl() data[h].next = -1 if t.first < 0: t.first = h @@ -445,12 +547,13 @@ proc enlarge[A, B](t: var OrderedTable[A, B]) = var h = t.first t.first = -1 t.last = -1 + swap(t.data, n) while h >= 0: - var nxt = t.data[h].next - if t.data[h].slot == seFilled: - rawInsert(t, n, t.data[h].key, t.data[h].val) + var nxt = n[h].next + if isFilled(n[h].hcode): + var j = -1 - rawGetKnownHC(t, n[h].key, n[h].hcode) + rawInsert(t, t.data, n[h].key, n[h].val, n[h].hcode, j) h = nxt - swap(t.data, n) proc `[]=`*[A, B](t: var OrderedTable[A, B], key: A, val: B) = ## puts a (key, value)-pair into `t`. @@ -460,30 +563,39 @@ proc add*[A, B](t: var OrderedTable[A, B], key: A, val: B) = ## puts a new (key, value)-pair into `t` even if ``t[key]`` already exists. addImpl() +proc mgetOrPut*[A, B](t: var OrderedTable[A, B], key: A, val: B): var B = + ## retrieves value at ``t[key]`` or puts ``value`` if not present, either way + ## returning a value which can be modified. + mgetOrPutImpl() + +proc hasKeyOrPut*[A, B](t: var OrderedTable[A, B], key: A, val: B): bool = + ## returns true iff `key` is in the table, otherwise inserts `value`. + hasKeyOrPutImpl() + proc initOrderedTable*[A, B](initialSize=64): OrderedTable[A, B] = ## creates a new ordered hash table that is empty. ## ## `initialSize` needs to be a power of two. If you need to accept runtime ## values for this you could use the ``nextPowerOfTwo`` proc from the - ## `math <math.html>`_ module. + ## `math <math.html>`_ module or the ``rightSize`` proc from this module. assert isPowerOfTwo(initialSize) result.counter = 0 result.first = -1 result.last = -1 newSeq(result.data, initialSize) -proc toOrderedTable*[A, B](pairs: openArray[tuple[key: A, - val: B]]): OrderedTable[A, B] = +proc toOrderedTable*[A, B](pairs: openArray[(A, + B)]): OrderedTable[A, B] = ## creates a new ordered hash table that contains the given `pairs`. - result = initOrderedTable[A, B](nextPowerOfTwo(pairs.len+10)) + result = initOrderedTable[A, B](rightSize(pairs.len)) for key, val in items(pairs): result[key] = val proc `$`*[A, B](t: OrderedTable[A, B]): string = ## The `$` operator for ordered hash tables. dollarImpl() -proc sort*[A, B](t: var OrderedTable[A, B], - cmp: proc (x,y: tuple[key: A, val: B]): int) = +proc sort*[A, B](t: var OrderedTable[A, B], + cmp: proc (x,y: (A, B)): int) = ## sorts `t` according to `cmp`. This modifies the internal list ## that kept the insertion order, so insertion order is lost after this ## call but key lookup and insertions remain possible after `sort` (in @@ -505,7 +617,7 @@ proc sort*[A, B](t: var OrderedTable[A, B], while i < insize: inc(psize) q = t.data[q].next - if q < 0: break + if q < 0: break inc(i) qsize = insize while psize > 0 or (qsize > 0 and q >= 0): @@ -513,7 +625,7 @@ proc sort*[A, B](t: var OrderedTable[A, B], e = q; q = t.data[q].next; dec(qsize) elif qsize == 0 or q < 0: e = p; p = t.data[p].next; dec(psize) - elif cmp((t.data[p].key, t.data[p].val), + elif cmp((t.data[p].key, t.data[p].val), (t.data[q].key, t.data[q].val)) <= 0: e = p; p = t.data[p].next; dec(psize) else: @@ -536,16 +648,16 @@ template forAllOrderedPairs(yieldStmt: stmt) {.dirty, immediate.} = var h = t.first while h >= 0: var nxt = t.data[h].next - if t.data[h].slot == seFilled: yieldStmt + if isFilled(t.data[h].hcode): yieldStmt h = nxt -iterator pairs*[A, B](t: OrderedTableRef[A, B]): tuple[key: A, val: B] = +iterator pairs*[A, B](t: OrderedTableRef[A, B]): (A, B) = ## iterates over any (key, value) pair in the table `t` in insertion ## order. forAllOrderedPairs: yield (t.data[h].key, t.data[h].val) -iterator mpairs*[A, B](t: OrderedTableRef[A, B]): tuple[key: A, val: var B] = +iterator mpairs*[A, B](t: OrderedTableRef[A, B]): (A, var B) = ## iterates over any (key, value) pair in the table `t` in insertion ## order. The values can be modified. forAllOrderedPairs: @@ -579,6 +691,15 @@ proc mget*[A, B](t: OrderedTableRef[A, B], key: A): var B = ## If `key` is not in `t`, the ``EInvalidKey`` exception is raised. result = t[].mget(key) +proc mgetOrPut*[A, B](t: OrderedTableRef[A, B], key: A, val: B): var B = + ## retrieves value at ``t[key]`` or puts ``val`` if not present, either way + ## returning a value which can be modified. + result = t[].mgetOrPut(key, val) + +proc hasKeyOrPut*[A, B](t: var OrderedTableRef[A, B], key: A, val: B): bool = + ## returns true iff `key` is in the table, otherwise inserts `val`. + result = t[].hasKeyOrPut(key, val) + proc hasKey*[A, B](t: OrderedTableRef[A, B], key: A): bool = ## returns true iff `key` is in the table `t`. result = t[].hasKey(key) @@ -596,22 +717,21 @@ proc newOrderedTable*[A, B](initialSize=64): OrderedTableRef[A, B] = ## ## `initialSize` needs to be a power of two. If you need to accept runtime ## values for this you could use the ``nextPowerOfTwo`` proc from the - ## `math <math.html>`_ module. + ## `math <math.html>`_ module or the ``rightSize`` proc from this module. new(result) result[] = initOrderedTable[A, B]() -proc newOrderedTable*[A, B](pairs: openArray[tuple[key: A, - val: B]]): OrderedTableRef[A, B] = +proc newOrderedTable*[A, B](pairs: openArray[(A, B)]): OrderedTableRef[A, B] = ## creates a new ordered hash table that contains the given `pairs`. - result = newOrderedTable[A, B](nextPowerOfTwo(pairs.len+10)) + result = newOrderedTable[A, B](rightSize(pairs.len)) for key, val in items(pairs): result[key] = val proc `$`*[A, B](t: OrderedTableRef[A, B]): string = ## The `$` operator for ordered hash tables. dollarImpl() -proc sort*[A, B](t: OrderedTableRef[A, B], - cmp: proc (x,y: tuple[key: A, val: B]): int) = +proc sort*[A, B](t: OrderedTableRef[A, B], + cmp: proc (x,y: (A, B)): int) = ## sorts `t` according to `cmp`. This modifies the internal list ## that kept the insertion order, so insertion order is lost after this ## call but key lookup and insertions remain possible after `sort` (in @@ -633,12 +753,12 @@ proc len*[A](t: CountTable[A]): int = ## returns the number of keys in `t`. result = t.counter -iterator pairs*[A](t: CountTable[A]): tuple[key: A, val: int] = +iterator pairs*[A](t: CountTable[A]): (A, int) = ## iterates over any (key, value) pair in the table `t`. for h in 0..high(t.data): if t.data[h].val != 0: yield (t.data[h].key, t.data[h].val) -iterator mpairs*[A](t: var CountTable[A]): tuple[key: A, val: var int] = +iterator mpairs*[A](t: var CountTable[A]): (A, var int) = ## iterates over any (key, value) pair in the table `t`. The values can ## be modified. for h in 0..high(t.data): @@ -664,7 +784,7 @@ proc rawGet[A](t: CountTable[A], key: A): int = while t.data[h].val != 0: if t.data[h].key == key: return h h = nextTry(h, high(t.data)) - result = -1 + result = -1 - h # < 0 => MISSING; insert idx = -1 - result proc `[]`*[A](t: CountTable[A], key: A): int = ## retrieves the value at ``t[key]``. If `key` is not in `t`, @@ -701,28 +821,34 @@ proc enlarge[A](t: var CountTable[A]) = proc `[]=`*[A](t: var CountTable[A], key: A, val: int) = ## puts a (key, value)-pair into `t`. `val` has to be positive. assert val > 0 - putImpl() + var h = rawGet(t, key) + if h >= 0: + t.data[h].val = val + else: + h = -1 - h + t.data[h].key = key + t.data[h].val = val proc initCountTable*[A](initialSize=64): CountTable[A] = ## creates a new count table that is empty. ## ## `initialSize` needs to be a power of two. If you need to accept runtime ## values for this you could use the ``nextPowerOfTwo`` proc from the - ## `math <math.html>`_ module. + ## `math <math.html>`_ module or the ``rightSize`` proc in this module. assert isPowerOfTwo(initialSize) result.counter = 0 newSeq(result.data, initialSize) proc toCountTable*[A](keys: openArray[A]): CountTable[A] = ## creates a new count table with every key in `keys` having a count of 1. - result = initCountTable[A](nextPowerOfTwo(keys.len+10)) + result = initCountTable[A](rightSize(keys.len)) for key in items(keys): result[key] = 1 proc `$`*[A](t: CountTable[A]): string = ## The `$` operator for count tables. dollarImpl() -proc inc*[A](t: var CountTable[A], key: A, val = 1) = +proc inc*[A](t: var CountTable[A], key: A, val = 1) = ## increments `t[key]` by `val`. var index = rawGet(t, key) if index >= 0: @@ -775,12 +901,12 @@ proc len*[A](t: CountTableRef[A]): int = ## returns the number of keys in `t`. result = t.counter -iterator pairs*[A](t: CountTableRef[A]): tuple[key: A, val: int] = +iterator pairs*[A](t: CountTableRef[A]): (A, int) = ## iterates over any (key, value) pair in the table `t`. for h in 0..high(t.data): if t.data[h].val != 0: yield (t.data[h].key, t.data[h].val) -iterator mpairs*[A](t: CountTableRef[A]): tuple[key: A, val: var int] = +iterator mpairs*[A](t: CountTableRef[A]): (A, var int) = ## iterates over any (key, value) pair in the table `t`. The values can ## be modified. for h in 0..high(t.data): @@ -826,28 +952,28 @@ proc newCountTable*[A](initialSize=64): CountTableRef[A] = ## ## `initialSize` needs to be a power of two. If you need to accept runtime ## values for this you could use the ``nextPowerOfTwo`` proc from the - ## `math <math.html>`_ module. + ## `math <math.html>`_ module or the ``rightSize`` method in this module. new(result) result[] = initCountTable[A](initialSize) proc newCountTable*[A](keys: openArray[A]): CountTableRef[A] = ## creates a new count table with every key in `keys` having a count of 1. - result = newCountTable[A](nextPowerOfTwo(keys.len+10)) + result = newCountTable[A](rightSize(keys.len)) for key in items(keys): result[key] = 1 proc `$`*[A](t: CountTableRef[A]): string = ## The `$` operator for count tables. dollarImpl() -proc inc*[A](t: CountTableRef[A], key: A, val = 1) = +proc inc*[A](t: CountTableRef[A], key: A, val = 1) = ## increments `t[key]` by `val`. t[].inc(key, val) -proc smallest*[A](t: CountTableRef[A]): tuple[key: A, val: int] = +proc smallest*[A](t: CountTableRef[A]): (A, int) = ## returns the largest (key,val)-pair. Efficiency: O(n) t[].smallest -proc largest*[A](t: CountTableRef[A]): tuple[key: A, val: int] = +proc largest*[A](t: CountTableRef[A]): (A, int) = ## returns the (key,val)-pair with the largest `val`. Efficiency: O(n) t[].largest @@ -858,6 +984,22 @@ proc sort*[A](t: CountTableRef[A]) = ## `t` in the sorted order. t[].sort +proc merge*[A](s: var CountTable[A], t: CountTable[A]) = + ## merges the second table into the first one + for key, value in t: + s.inc(key, value) + +proc merge*[A](s, t: CountTable[A]): CountTable[A] = + ## merges the two tables into a new one + result = initCountTable[A](nextPowerOfTwo(max(s.len, t.len))) + for table in @[s, t]: + for key, value in table: + result.inc(key, value) + +proc merge*[A](s, t: CountTableRef[A]) = + ## merges the second table into the first one + s[].merge(t[]) + when isMainModule: type Person = object @@ -886,3 +1028,48 @@ when isMainModule: s2[p2] = 45_000 s3[p1] = 30_000 s3[p2] = 45_000 + + var + t1 = initCountTable[string]() + t2 = initCountTable[string]() + t1.inc("foo") + t1.inc("bar", 2) + t1.inc("baz", 3) + t2.inc("foo", 4) + t2.inc("bar") + t2.inc("baz", 11) + merge(t1, t2) + assert(t1["foo"] == 5) + assert(t1["bar"] == 3) + assert(t1["baz"] == 14) + + let + t1r = newCountTable[string]() + t2r = newCountTable[string]() + t1r.inc("foo") + t1r.inc("bar", 2) + t1r.inc("baz", 3) + t2r.inc("foo", 4) + t2r.inc("bar") + t2r.inc("baz", 11) + merge(t1r, t2r) + assert(t1r["foo"] == 5) + assert(t1r["bar"] == 3) + assert(t1r["baz"] == 14) + + var + t1l = initCountTable[string]() + t2l = initCountTable[string]() + t1l.inc("foo") + t1l.inc("bar", 2) + t1l.inc("baz", 3) + t2l.inc("foo", 4) + t2l.inc("bar") + t2l.inc("baz", 11) + let + t1merging = t1l + t2merging = t2l + let merged = merge(t1merging, t2merging) + assert(merged["foo"] == 5) + assert(merged["bar"] == 3) + assert(merged["baz"] == 14) diff --git a/lib/pure/colors.nim b/lib/pure/colors.nim index 7942255cb..f24cc0072 100644 --- a/lib/pure/colors.nim +++ b/lib/pure/colors.nim @@ -46,7 +46,7 @@ proc `+`*(a, b: Color): Color = colorOp(satPlus) proc `-`*(a, b: Color): Color = - ## substracts two colors: This uses saturated artithmetic, so that each color + ## subtracts two colors: This uses saturated artithmetic, so that each color ## component cannot overflow (255 is used as a maximum). colorOp(satMinus) @@ -392,7 +392,7 @@ proc parseColor*(name: string): Color = result = Color(parseHexInt(name)) else: var idx = binaryStrSearch(colorNames, name) - if idx < 0: raise newException(ValueError, "unkown color: " & name) + if idx < 0: raise newException(ValueError, "unknown color: " & name) result = colorNames[idx][1] proc isColor*(name: string): bool = diff --git a/lib/pure/complex.nim b/lib/pure/complex.nim index 0c5704f69..8577bf7a1 100644 --- a/lib/pure/complex.nim +++ b/lib/pure/complex.nim @@ -27,6 +27,11 @@ type {.deprecated: [TComplex: Complex].} +proc toComplex*(x: SomeInteger): Complex = + ## Convert some integer ``x`` to a complex number. + result.re = x + result.im = 0 + proc `==` *(x, y: Complex): bool = ## Compare two complex numbers `x` and `y` for equality. result = x.re == y.re and x.im == y.im @@ -171,6 +176,12 @@ proc abs*(z: Complex): float = result = y * sqrt(1.0 + temp * temp) +proc conjugate*(z: Complex): Complex = + ## Conjugate of complex number `z`. + result.re = z.re + result.im = -z.im + + proc sqrt*(z: Complex): Complex = ## Square root for a complex number `z`. var x, y, w, r: float @@ -263,27 +274,101 @@ proc tan*(z: Complex): Complex = ## Returns the tangent of `z`. result = sin(z)/cos(z) +proc arctan*(z: Complex): Complex = + ## Returns the inverse tangent of `z`. + var i: Complex = (0.0,1.0) + result = 0.5*i*(ln(1-i*z)-ln(1+i*z)) + proc cot*(z: Complex): Complex = ## Returns the cotangent of `z`. result = cos(z)/sin(z) +proc arccot*(z: Complex): Complex = + ## Returns the inverse cotangent of `z`. + var i: Complex = (0.0,1.0) + result = 0.5*i*(ln(1-i/z)-ln(1+i/z)) + proc sec*(z: Complex): Complex = ## Returns the secant of `z`. result = 1.0/cos(z) +proc arcsec*(z: Complex): Complex = + ## Returns the inverse secant of `z`. + var i: Complex = (0.0,1.0) + result = -i*ln(i*sqrt(1-1/(z*z))+1/z) + proc csc*(z: Complex): Complex = ## Returns the cosecant of `z`. result = 1.0/sin(z) +proc arccsc*(z: Complex): Complex = + ## Returns the inverse cosecant of `z`. + var i: Complex = (0.0,1.0) + result = -i*ln(sqrt(1-1/(z*z))+i/z) + proc sinh*(z: Complex): Complex = ## Returns the hyperbolic sine of `z`. result = 0.5*(exp(z)-exp(-z)) +proc arcsinh*(z: Complex): Complex = + ## Returns the inverse hyperbolic sine of `z`. + result = ln(z+sqrt(z*z+1)) + proc cosh*(z: Complex): Complex = ## Returns the hyperbolic cosine of `z`. result = 0.5*(exp(z)+exp(-z)) +proc arccosh*(z: Complex): Complex = + ## Returns the inverse hyperbolic cosine of `z`. + result = ln(z+sqrt(z*z-1)) + +proc tanh*(z: Complex): Complex = + ## Returns the hyperbolic tangent of `z`. + result = sinh(z)/cosh(z) + +proc arctanh*(z: Complex): Complex = + ## Returns the inverse hyperbolic tangent of `z`. + result = 0.5*(ln((1+z)/(1-z))) + +proc sech*(z: Complex): Complex = + ## Returns the hyperbolic secant of `z`. + result = 2/(exp(z)+exp(-z)) + +proc arcsech*(z: Complex): Complex = + ## Returns the inverse hyperbolic secant of `z`. + result = ln(1/z+sqrt(1/z+1)*sqrt(1/z-1)) + +proc csch*(z: Complex): Complex = + ## Returns the hyperbolic cosecant of `z`. + result = 2/(exp(z)-exp(-z)) + +proc arccsch*(z: Complex): Complex = + ## Returns the inverse hyperbolic cosecant of `z`. + result = ln(1/z+sqrt(1/(z*z)+1)) + +proc coth*(z: Complex): Complex = + ## Returns the hyperbolic cotangent of `z`. + result = cosh(z)/sinh(z) + +proc arccoth*(z: Complex): Complex = + ## Returns the inverse hyperbolic cotangent of `z`. + result = 0.5*(ln(1+1/z)-ln(1-1/z)) + +proc phase*(z: Complex): float = + ## Returns the phase of `z`. + arctan2(z.im, z.re) + +proc polar*(z: Complex): tuple[r, phi: float] = + ## Returns `z` in polar coordinates. + result.r = abs(z) + result.phi = phase(z) + +proc rect*(r: float, phi: float): Complex = + ## Returns the complex number with polar coordinates `r` and `phi`. + result.re = r * cos(phi) + result.im = r * sin(phi) + proc `$`*(z: Complex): string = ## Returns `z`'s string representation as ``"(re, im)"``. @@ -303,41 +388,56 @@ when isMainModule: var tt = (10.0, 20.0) var ipi = (0.0, -PI) - assert( a == a ) - assert( (a-a) == z ) - assert( (a+b) == z ) - assert( (a/b) == m1 ) - assert( (1.0/a) == (0.2, -0.4) ) - assert( (a*b) == (3.0, -4.0) ) - assert( 10.0*a == tt ) - assert( a*10.0 == tt ) - assert( tt/10.0 == a ) + assert( a == a ) + assert( (a-a) == z ) + assert( (a+b) == z ) + assert( (a/b) == m1 ) + assert( (1.0/a) == (0.2, -0.4) ) + assert( (a*b) == (3.0, -4.0) ) + assert( 10.0*a == tt ) + assert( a*10.0 == tt ) + assert( tt/10.0 == a ) assert( oo+(-1.0) == i ) assert( (-1.0)+oo == i ) - assert( abs(oo) == sqrt(2.0) ) - assert( sqrt(m1) == i ) - assert( exp(ipi) =~ m1 ) + assert( abs(oo) == sqrt(2.0) ) + assert( conjugate(a) == (1.0, -2.0) ) + assert( sqrt(m1) == i ) + assert( exp(ipi) =~ m1 ) - assert( pow(a,b) =~ (-3.72999124927876, -1.68815826725068) ) - assert( pow(z,a) =~ (0.0, 0.0) ) - assert( pow(z,z) =~ (1.0, 0.0) ) + assert( pow(a,b) =~ (-3.72999124927876, -1.68815826725068) ) + assert( pow(z,a) =~ (0.0, 0.0) ) + assert( pow(z,z) =~ (1.0, 0.0) ) assert( pow(a,one) =~ a ) - assert( pow(a,m1) =~ (0.2, -0.4) ) + assert( pow(a,m1) =~ (0.2, -0.4) ) - assert( ln(a) =~ (0.804718956217050, 1.107148717794090) ) + assert( ln(a) =~ (0.804718956217050, 1.107148717794090) ) assert( log10(a) =~ (0.349485002168009, 0.480828578784234) ) - assert( log2(a) =~ (1.16096404744368, 1.59727796468811) ) - - assert( sin(a) =~ (3.16577851321617, 1.95960104142161) ) - assert( cos(a) =~ (2.03272300701967, -3.05189779915180) ) - assert( tan(a) =~ (0.0338128260798967, 1.0147936161466335) ) - assert( cot(a) =~ 1.0/tan(a) ) - assert( sec(a) =~ 1.0/cos(a) ) - assert( csc(a) =~ 1.0/sin(a) ) + assert( log2(a) =~ (1.16096404744368, 1.59727796468811) ) + + assert( sin(a) =~ (3.16577851321617, 1.95960104142161) ) + assert( cos(a) =~ (2.03272300701967, -3.05189779915180) ) + assert( tan(a) =~ (0.0338128260798967, 1.0147936161466335) ) + assert( cot(a) =~ 1.0/tan(a) ) + assert( sec(a) =~ 1.0/cos(a) ) + assert( csc(a) =~ 1.0/sin(a) ) assert( arcsin(a) =~ (0.427078586392476, 1.528570919480998) ) assert( arccos(a) =~ (1.14371774040242, -1.52857091948100) ) + assert( arctan(a) =~ (1.338972522294494, 0.402359478108525) ) - assert( cosh(a) =~ (-0.642148124715520, 1.068607421382778) ) + assert( cosh(a) =~ (-0.642148124715520, 1.068607421382778) ) assert( sinh(a) =~ (-0.489056259041294, 1.403119250622040) ) - - + assert( tanh(a) =~ (1.1667362572409199,-0.243458201185725) ) + assert( sech(a) =~ 1/cosh(a) ) + assert( csch(a) =~ 1/sinh(a) ) + assert( coth(a) =~ 1/tanh(a) ) + assert( arccosh(a) =~ (1.528570919480998, 1.14371774040242) ) + assert( arcsinh(a) =~ (1.469351744368185, 1.06344002357775) ) + assert( arctanh(a) =~ (0.173286795139986, 1.17809724509617) ) + assert( arcsech(a) =~ arccosh(1/a) ) + assert( arccsch(a) =~ arcsinh(1/a) ) + assert( arccoth(a) =~ arctanh(1/a) ) + + assert( phase(a) == 1.1071487177940904 ) + var t = polar(a) + assert( rect(t.r, t.phi) =~ a ) + assert( rect(1.0, 2.0) =~ (-0.4161468365471424, 0.9092974268256817) ) diff --git a/lib/pure/concurrency/cpuinfo.nim b/lib/pure/concurrency/cpuinfo.nim index ac5fa5dd9..6f2bc4491 100644 --- a/lib/pure/concurrency/cpuinfo.nim +++ b/lib/pure/concurrency/cpuinfo.nim @@ -1,7 +1,7 @@ # # # Nim's Runtime Library -# (c) Copyright 2014 Andreas Rumpf +# (c) Copyright 2015 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. diff --git a/lib/pure/concurrency/cpuload.nim b/lib/pure/concurrency/cpuload.nim index 88fb4a064..7ce5e01b7 100644 --- a/lib/pure/concurrency/cpuload.nim +++ b/lib/pure/concurrency/cpuload.nim @@ -1,7 +1,7 @@ # # # Nim's Runtime Library -# (c) Copyright 2014 Andreas Rumpf +# (c) Copyright 2015 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -78,7 +78,7 @@ proc advice*(s: var ThreadPoolState): ThreadPoolAdvice = result = doNothing inc s.calls -when isMainModule: +when not defined(testing) and isMainModule: proc busyLoop() = while true: discard random(80) diff --git a/lib/pure/concurrency/threadpool.nim b/lib/pure/concurrency/threadpool.nim index 0bd97043a..9f1e53fb8 100644 --- a/lib/pure/concurrency/threadpool.nim +++ b/lib/pure/concurrency/threadpool.nim @@ -1,7 +1,7 @@ # # # Nim's Runtime Library -# (c) Copyright 2014 Andreas Rumpf +# (c) Copyright 2015 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -204,7 +204,7 @@ proc nimFlowVarSignal(fv: FlowVarBase) {.compilerProc.} = inc fv.ai.cv.counter release(fv.ai.cv.L) signal(fv.ai.cv.c) - if fv.usesSemaphore: + if fv.usesSemaphore: signal(fv.cv) proc awaitAndThen*[T](fv: FlowVar[T]; action: proc (x: T) {.closure.}) = diff --git a/lib/pure/cookies.nim b/lib/pure/cookies.nim index 6247efed2..9983c4a04 100644 --- a/lib/pure/cookies.nim +++ b/lib/pure/cookies.nim @@ -56,6 +56,12 @@ proc setCookie*(key, value: string, expires: TimeInfo, when isMainModule: var tim = Time(int(getTime()) + 76 * (60 * 60 * 24)) - echo(setCookie("test", "value", tim.getGMTime())) + let cookie = setCookie("test", "value", tim.getGMTime()) + when not defined(testing): + echo cookie + let start = "Set-Cookie: test=value; Expires=" + assert cookie[0..start.high] == start - echo parseCookies("uid=1; kp=2") + let table = parseCookies("uid=1; kp=2") + assert table["uid"] == "1" + assert table["kp"] == "2" diff --git a/lib/pure/encodings.nim b/lib/pure/encodings.nim index 958a4133b..2a6134615 100644 --- a/lib/pure/encodings.nim +++ b/lib/pure/encodings.nim @@ -1,7 +1,7 @@ # # # Nim's Runtime Library -# (c) Copyright 2014 Andreas Rumpf +# (c) Copyright 2015 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -301,7 +301,7 @@ proc getCurrentEncoding*(): string = proc open*(destEncoding = "UTF-8", srcEncoding = "CP1252"): EncodingConverter = ## opens a converter that can convert from `srcEncoding` to `destEncoding`. - ## Raises `EIO` if it cannot fullfill the request. + ## Raises `EIO` if it cannot fulfill the request. when not defined(windows): result = iconvOpen(destEncoding, srcEncoding) if result == nil: @@ -451,7 +451,7 @@ proc convert*(s: string, destEncoding = "UTF-8", finally: close(c) -when isMainModule: +when not defined(testing) and isMainModule: let orig = "öäüß" cp1252 = convert(orig, "CP1252", "UTF-8") diff --git a/lib/pure/events.nim b/lib/pure/events.nim index 77faa6a66..44e9ed286 100644 --- a/lib/pure/events.nim +++ b/lib/pure/events.nim @@ -9,7 +9,7 @@ ## :Author: Alex Mitchell ## -## This module implements an event system that is not dependant on external +## This module implements an event system that is not dependent on external ## graphical toolkits. It was originally called ``NimEE`` because ## it was inspired by Python's PyEE module. There are two ways you can use ## events: one is a python-inspired way; the other is more of a C-style way. diff --git a/lib/pure/fenv.nim b/lib/pure/fenv.nim index 415ef455e..f8f115ecc 100644 --- a/lib/pure/fenv.nim +++ b/lib/pure/fenv.nim @@ -1,7 +1,7 @@ # # # Nim's Runtime Library -# (c) Copyright 2014 Andreas Rumpf +# (c) Copyright 2015 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -101,3 +101,81 @@ proc feupdateenv*(envp: ptr Tfenv): cint {.importc, header: "<fenv.h>".} ## Save current exceptions in temporary storage, install environment ## represented by object pointed to by `envp` and raise exceptions ## according to saved exceptions. + +var FP_RADIX_INTERNAL {. importc: "FLT_RADIX" header: "<float.h>" .} : int + +template fpRadix* : int = FP_RADIX_INTERNAL + ## The (integer) value of the radix used to represent any floating + ## point type on the architecture used to build the program. + +var FLT_MANT_DIG {. importc: "FLT_MANT_DIG" header: "<float.h>" .} : int +var FLT_DIG {. importc: "FLT_DIG" header: "<float.h>" .} : int +var FLT_MIN_EXP {. importc: "FLT_MIN_EXP" header: "<float.h>" .} : int +var FLT_MAX_EXP {. importc: "FLT_MAX_EXP" header: "<float.h>" .} : int +var FLT_MIN_10_EXP {. importc: "FLT_MIN_10_EXP" header: "<float.h>" .} : int +var FLT_MAX_10_EXP {. importc: "FLT_MAX_10_EXP" header: "<float.h>" .} : int +var FLT_MIN {. importc: "FLT_MIN" header: "<float.h>" .} : cfloat +var FLT_MAX {. importc: "FLT_MAX" header: "<float.h>" .} : cfloat +var FLT_EPSILON {. importc: "FLT_EPSILON" header: "<float.h>" .} : cfloat + +var DBL_MANT_DIG {. importc: "DBL_MANT_DIG" header: "<float.h>" .} : int +var DBL_DIG {. importc: "DBL_DIG" header: "<float.h>" .} : int +var DBL_MIN_EXP {. importc: "DBL_MIN_EXP" header: "<float.h>" .} : int +var DBL_MAX_EXP {. importc: "DBL_MAX_EXP" header: "<float.h>" .} : int +var DBL_MIN_10_EXP {. importc: "DBL_MIN_10_EXP" header: "<float.h>" .} : int +var DBL_MAX_10_EXP {. importc: "DBL_MAX_10_EXP" header: "<float.h>" .} : int +var DBL_MIN {. importc: "DBL_MIN" header: "<float.h>" .} : cdouble +var DBL_MAX {. importc: "DBL_MAX" header: "<float.h>" .} : cdouble +var DBL_EPSILON {. importc: "DBL_EPSILON" header: "<float.h>" .} : cdouble + +template mantissaDigits*(T : typedesc[float32]) : int = FLT_MANT_DIG + ## Number of digits (in base ``floatingPointRadix``) in the mantissa + ## of 32-bit floating-point numbers. +template digits*(T : typedesc[float32]) : int = FLT_DIG + ## Number of decimal digits that can be represented in a + ## 32-bit floating-point type without losing precision. +template minExponent*(T : typedesc[float32]) : int = FLT_MIN_EXP + ## Minimum (negative) exponent for 32-bit floating-point numbers. +template maxExponent*(T : typedesc[float32]) : int = FLT_MAX_EXP + ## Maximum (positive) exponent for 32-bit floating-point numbers. +template min10Exponent*(T : typedesc[float32]) : int = FLT_MIN_10_EXP + ## Minimum (negative) exponent in base 10 for 32-bit floating-point + ## numbers. +template max10Exponent*(T : typedesc[float32]) : int = FLT_MAX_10_EXP + ## Maximum (positive) exponent in base 10 for 32-bit floating-point + ## numbers. +template minimumPositiveValue*(T : typedesc[float32]) : float32 = FLT_MIN + ## The smallest positive (nonzero) number that can be represented in a + ## 32-bit floating-point type. +template maximumPositiveValue*(T : typedesc[float32]) : float32 = FLT_MAX + ## The largest positive number that can be represented in a 32-bit + ## floating-point type. +template epsilon*(T : typedesc[float32]): float32 = FLT_EPSILON + ## The difference between 1.0 and the smallest number greater than + ## 1.0 that can be represented in a 32-bit floating-point type. + +template mantissaDigits*(T : typedesc[float64]) : int = DBL_MANT_DIG + ## Number of digits (in base ``floatingPointRadix``) in the mantissa + ## of 64-bit floating-point numbers. +template digits*(T : typedesc[float64]) : int = DBL_DIG + ## Number of decimal digits that can be represented in a + ## 64-bit floating-point type without losing precision. +template minExponent*(T : typedesc[float64]) : int = DBL_MIN_EXP + ## Minimum (negative) exponent for 64-bit floating-point numbers. +template maxExponent*(T : typedesc[float64]) : int = DBL_MAX_EXP + ## Maximum (positive) exponent for 64-bit floating-point numbers. +template min10Exponent*(T : typedesc[float64]) : int = DBL_MIN_10_EXP + ## Minimum (negative) exponent in base 10 for 64-bit floating-point + ## numbers. +template max10Exponent*(T : typedesc[float64]) : int = DBL_MAX_10_EXP + ## Maximum (positive) exponent in base 10 for 64-bit floating-point + ## numbers. +template minimumPositiveValue*(T : typedesc[float64]) : float64 = DBL_MIN + ## The smallest positive (nonzero) number that can be represented in a + ## 64-bit floating-point type. +template maximumPositiveValue*(T : typedesc[float64]) : float64 = DBL_MAX + ## The largest positive number that can be represented in a 64-bit + ## floating-point type. +template epsilon*(T : typedesc[float64]): float64 = DBL_EPSILON + ## The difference between 1.0 and the smallest number greater than + ## 1.0 that can be represented in a 64-bit floating-point type. diff --git a/lib/pure/fsmonitor.nim b/lib/pure/fsmonitor.nim index bf4aef61c..83779eb9c 100644 --- a/lib/pure/fsmonitor.nim +++ b/lib/pure/fsmonitor.nim @@ -29,7 +29,7 @@ type FSMonitorObj = object of RootObj fd: cint handleEvent: proc (m: FSMonitor, ev: MonitorEvent) {.closure.} - targets: TTable[cint, string] + targets: Table[cint, string] MonitorEventType* = enum ## Monitor event type MonitorAccess, ## File was accessed. @@ -64,7 +64,7 @@ type const MaxEvents = 100 -proc newMonitor*(): PFSMonitor = +proc newMonitor*(): FSMonitor = ## Creates a new file system monitor. new(result) result.targets = initTable[cint, string]() @@ -72,7 +72,7 @@ proc newMonitor*(): PFSMonitor = if result.fd < 0: raiseOSError(osLastError()) -proc add*(monitor: PFSMonitor, target: string, +proc add*(monitor: FSMonitor, target: string, filters = {MonitorAll}): cint {.discardable.} = ## Adds ``target`` which may be a directory or a file to the list of ## watched paths of ``monitor``. @@ -99,14 +99,14 @@ proc add*(monitor: PFSMonitor, target: string, raiseOSError(osLastError()) monitor.targets.add(result, target) -proc del*(monitor: PFSMonitor, wd: cint) = +proc del*(monitor: FSMonitor, wd: cint) = ## Removes watched directory or file as specified by ``wd`` from ``monitor``. ## ## If ``wd`` is not a part of ``monitor`` an EOS error is raised. if inotifyRmWatch(monitor.fd, wd) < 0: raiseOSError(osLastError()) -proc getEvent(m: PFSMonitor, fd: cint): seq[TMonitorEvent] = +proc getEvent(m: FSMonitor, fd: cint): seq[MonitorEvent] = result = @[] let size = (sizeof(TINotifyEvent)+2000)*MaxEvents var buffer = newString(size) @@ -118,7 +118,7 @@ proc getEvent(m: PFSMonitor, fd: cint): seq[TMonitorEvent] = var i = 0 while i < le: var event = cast[ptr TINotifyEvent](addr(buffer[i])) - var mev: TMonitorEvent + var mev: MonitorEvent mev.wd = event.wd if event.len.int != 0: let cstr = event.name.addr.cstring @@ -137,7 +137,7 @@ proc getEvent(m: PFSMonitor, fd: cint): seq[TMonitorEvent] = # Find the MovedFrom event. mev.oldPath = movedFrom[event.cookie.cint].old mev.newPath = "" # Set later - # Delete it from the TTable + # Delete it from the Table movedFrom.del(event.cookie.cint) elif (event.mask.int and IN_ACCESS) != 0: mev.kind = MonitorAccess elif (event.mask.int and IN_ATTRIB) != 0: mev.kind = MonitorAttrib @@ -164,26 +164,26 @@ proc getEvent(m: PFSMonitor, fd: cint): seq[TMonitorEvent] = # If movedFrom events have not been matched with a moveTo. File has # been moved to an unwatched location, emit a MonitorDelete. for cookie, t in pairs(movedFrom): - var mev: TMonitorEvent + var mev: MonitorEvent mev.kind = MonitorDelete mev.wd = t.wd mev.name = t.old result.add(mev) -proc FSMonitorRead(h: PObject) = - var events = PFSMonitor(h).getEvent(PFSMonitor(h).fd) - #var newEv: TMonitorEvent +proc FSMonitorRead(h: RootRef) = + var events = FSMonitor(h).getEvent(FSMonitor(h).fd) + #var newEv: MonitorEvent for ev in events: - var target = PFSMonitor(h).targets[ev.wd] + var target = FSMonitor(h).targets[ev.wd] var newEv = ev if newEv.kind == MonitorMoved: newEv.oldPath = target / newEv.oldPath newEv.newPath = target / newEv.name else: newEv.fullName = target / newEv.name - PFSMonitor(h).handleEvent(PFSMonitor(h), newEv) + FSMonitor(h).handleEvent(FSMonitor(h), newEv) -proc toDelegate(m: PFSMonitor): PDelegate = +proc toDelegate(m: FSMonitor): Delegate = result = newDelegate() result.deleVal = m result.fd = (type(result.fd))(m.fd) @@ -191,20 +191,20 @@ proc toDelegate(m: PFSMonitor): PDelegate = result.handleRead = FSMonitorRead result.open = true -proc register*(d: PDispatcher, monitor: PFSMonitor, - handleEvent: proc (m: PFSMonitor, ev: TMonitorEvent) {.closure.}) = +proc register*(d: Dispatcher, monitor: FSMonitor, + handleEvent: proc (m: FSMonitor, ev: MonitorEvent) {.closure.}) = ## Registers ``monitor`` with dispatcher ``d``. monitor.handleEvent = handleEvent var deleg = toDelegate(monitor) d.register(deleg) -when isMainModule: +when not defined(testing) and isMainModule: proc main = var disp = newDispatcher() var monitor = newMonitor() echo monitor.add("/home/dom/inotifytests/") disp.register(monitor, - proc (m: PFSMonitor, ev: TMonitorEvent) = + proc (m: FSMonitor, ev: MonitorEvent) = echo("Got event: ", ev.kind) if ev.kind == MonitorMoved: echo("From ", ev.oldPath, " to ", ev.newPath) diff --git a/lib/pure/ftpclient.nim b/lib/pure/ftpclient.nim index 975eae7e2..dd141eb01 100644 --- a/lib/pure/ftpclient.nim +++ b/lib/pure/ftpclient.nim @@ -1,7 +1,7 @@ # # # Nim's Runtime Library -# (c) Copyright 2014 Dominik Picheta +# (c) Copyright 2015 Dominik Picheta # See the file "copying.txt", included in this # distribution, for details about the copyright. # @@ -15,8 +15,8 @@ from rawsockets import nil from asyncdispatch import PFuture ## This module **partially** implements an FTP client as specified -## by `RFC 959 <http://tools.ietf.org/html/rfc959>`_. -## +## by `RFC 959 <http://tools.ietf.org/html/rfc959>`_. +## ## This module provides both a synchronous and asynchronous implementation. ## The asynchronous implementation requires you to use the ``asyncFTPClient`` ## function. You are then required to register the ``AsyncFTPClient`` with a @@ -27,7 +27,7 @@ from asyncdispatch import PFuture ## file transfers, calls to functions which use the command socket will block. ## ## Here is some example usage of this module: -## +## ## .. code-block:: Nim ## var ftp = ftpClient("example.org", user = "user", pass = "pass") ## ftp.connect() @@ -51,7 +51,7 @@ type port*: rawsockets.Port else: port*: Port - + jobInProgress*: bool job*: FTPJob[SockType] @@ -91,7 +91,7 @@ type of EvLines: lines*: string ## Lines that have been transferred. of EvRetr, EvStore: ## Retr/Store operation finished. - nil + nil of EvTransferProgress: bytesTotal*: BiggestInt ## Bytes total. bytesFinished*: BiggestInt ## Bytes transferred. @@ -107,7 +107,7 @@ type EInvalidReply: ReplyError, EFTP: FTPError ].} -proc ftpClient*(address: string, port = TPort(21), +proc ftpClient*(address: string, port = Port(21), user, pass = ""): FtpClient = ## Create a ``FtpClient`` object. new(result) @@ -120,10 +120,10 @@ proc ftpClient*(address: string, port = TPort(21), result.csock = socket() if result.csock == invalidSocket: raiseOSError(osLastError()) -template blockingOperation(sock: TSocket, body: stmt) {.immediate.} = +template blockingOperation(sock: Socket, body: stmt) {.immediate.} = body -template blockingOperation(sock: asyncio.PAsyncSocket, body: stmt) {.immediate.} = +template blockingOperation(sock: asyncio.AsyncSocket, body: stmt) {.immediate.} = sock.setBlocking(true) body sock.setBlocking(false) @@ -145,14 +145,14 @@ proc send*[T](ftp: FtpBase[T], m: string): TaintedString = proc assertReply(received: TaintedString, expected: string) = if not received.string.startsWith(expected): - raise newException(EInvalidReply, + raise newException(ReplyError, "Expected reply '$1' got: $2" % [ expected, received.string]) proc assertReply(received: TaintedString, expected: varargs[string]) = for i in items(expected): if received.string.startsWith(i): return - raise newException(EInvalidReply, + raise newException(ReplyError, "Expected reply '$1' got: $2" % [expected.join("' or '"), received.string]) @@ -161,7 +161,7 @@ proc createJob[T](ftp: FtpBase[T], nimcall,gcsafe.}, cmd: FTPJobType) = if ftp.jobInProgress: - raise newException(EFTP, "Unable to do two jobs at once.") + raise newException(FTPError, "Unable to do two jobs at once.") ftp.jobInProgress = true new(ftp.job) ftp.job.prc = prc @@ -182,11 +182,11 @@ proc deleteJob[T](ftp: FtpBase[T]) = ftp.job.file.close() ftp.dsock.close() -proc handleTask(s: PAsyncSocket, ftp: PAsyncFTPClient) = +proc handleTask(s: AsyncSocket, ftp: AsyncFTPClient) = if ftp.jobInProgress: if ftp.job.typ in {JRetr, JStore}: if epochTime() - ftp.job.lastProgressReport >= 1.0: - var r: TFTPEvent + var r: FTPEvent ftp.job.lastProgressReport = epochTime() r.typ = EvTransferProgress r.bytesTotal = ftp.job.total @@ -195,57 +195,57 @@ proc handleTask(s: PAsyncSocket, ftp: PAsyncFTPClient) = r.filename = ftp.job.filename r.currentJob = ftp.job.typ ftp.job.oneSecond = 0 - ftp.handleEvent(PAsyncFTPClient(ftp), r) + ftp.handleEvent(ftp, r) -proc handleWrite(s: PAsyncSocket, ftp: PAsyncFTPClient) = +proc handleWrite(s: AsyncSocket, ftp: AsyncFTPClient) = if ftp.jobInProgress: if ftp.job.typ == JStore: assert (not ftp.job.prc(ftp, true)) -proc handleConnect(s: PAsyncSocket, ftp: PAsyncFTPClient) = +proc handleConnect(s: AsyncSocket, ftp: AsyncFTPClient) = ftp.dsockConnected = true assert(ftp.jobInProgress) if ftp.job.typ == JStore: - s.setHandleWrite(proc (s: PAsyncSocket) = handleWrite(s, ftp)) + s.setHandleWrite(proc (s: AsyncSocket) = handleWrite(s, ftp)) else: s.delHandleWrite() -proc handleRead(s: PAsyncSocket, ftp: PAsyncFTPClient) = +proc handleRead(s: AsyncSocket, ftp: AsyncFTPClient) = assert ftp.jobInProgress assert ftp.job.typ != JStore - # This can never return true, because it shouldn't check for code + # This can never return true, because it shouldn't check for code # 226 from csock. assert(not ftp.job.prc(ftp, true)) proc pasv[T](ftp: FtpBase[T]) = ## Negotiate a data connection. - when T is TSocket: + when T is Socket: ftp.dsock = socket() if ftp.dsock == invalidSocket: raiseOSError(osLastError()) - elif T is PAsyncSocket: + elif T is AsyncSocket: ftp.dsock = asyncSocket() ftp.dsock.handleRead = - proc (s: PAsyncSocket) = + proc (s: AsyncSocket) = handleRead(s, ftp) ftp.dsock.handleConnect = - proc (s: PAsyncSocket) = + proc (s: AsyncSocket) = handleConnect(s, ftp) ftp.dsock.handleTask = - proc (s: PAsyncSocket) = + proc (s: AsyncSocket) = handleTask(s, ftp) ftp.disp.register(ftp.dsock) else: {.fatal: "Incorrect socket instantiation".} - + var pasvMsg = ftp.send("PASV").string.strip.TaintedString assertReply(pasvMsg, "227") var betweenParens = captureBetween(pasvMsg.string, '(', ')') var nums = betweenParens.split(',') - var ip = nums[0.. -3] - var port = nums[-2.. -1] + var ip = nums[0.. ^3] + var port = nums[^2.. ^1] var properPort = port[0].parseInt()*256+port[1].parseInt() - ftp.dsock.connect(ip.join("."), TPort(properPort.toU16)) - when T is PAsyncSocket: + ftp.dsock.connect(ip.join("."), Port(properPort.toU16)) + when T is AsyncSocket: ftp.dsockConnected = false else: ftp.dsockConnected = true @@ -255,10 +255,10 @@ proc normalizePathSep(path: string): string = proc connect*[T](ftp: FtpBase[T]) = ## Connect to the FTP server specified by ``ftp``. - when T is PAsyncSocket: + when T is AsyncSocket: blockingOperation(ftp.csock): ftp.csock.connect(ftp.address, ftp.port) - elif T is TSocket: + elif T is Socket: ftp.csock.connect(ftp.address, ftp.port) else: {.fatal: "Incorrect socket instantiation".} @@ -292,13 +292,13 @@ proc getLines[T](ftp: FtpBase[T], async: bool = false): bool = ## It doesn't if `async` is true, because it doesn't check for 226 then. if ftp.dsockConnected: var r = TaintedString"" - when T is PAsyncSocket: + when T is AsyncSocket: if ftp.asyncDSock.readLine(r): if r.string == "": ftp.dsockConnected = false else: ftp.job.lines.add(r.string & "\n") - elif T is TSocket: + elif T is Socket: assert(not async) ftp.dsock.readLine(r) if r.string == "": @@ -307,9 +307,9 @@ proc getLines[T](ftp: FtpBase[T], async: bool = false): bool = ftp.job.lines.add(r.string & "\n") else: {.fatal: "Incorrect socket instantiation".} - + if not async: - var readSocks: seq[TSocket] = @[ftp.csock] + var readSocks: seq[Socket] = @[ftp.csock] # This is only needed here. Asyncio gets this socket... blockingOperation(ftp.csock): if readSocks.select(1) != 0 and ftp.csock in readSocks: @@ -372,7 +372,7 @@ proc createDir*[T](ftp: FtpBase[T], dir: string, recursive: bool = false) = assertReply reply, "257" proc chmod*[T](ftp: FtpBase[T], path: string, - permissions: set[TFilePermission]) = + permissions: set[FilePermission]) = ## Changes permission of ``path`` to ``permissions``. var userOctal = 0 var groupOctal = 0 @@ -396,7 +396,7 @@ proc chmod*[T](ftp: FtpBase[T], path: string, proc list*[T](ftp: FtpBase[T], dir: string = "", async = false): string = ## Lists all files in ``dir``. If ``dir`` is ``""``, uses the current ## working directory. If ``async`` is true, this function will return - ## immediately and it will be your job to call asyncio's + ## immediately and it will be your job to call asyncio's ## ``poll`` to progress this operation. ftp.createJob(getLines[T], JRetrText) ftp.pasv() @@ -417,7 +417,7 @@ proc retrText*[T](ftp: FtpBase[T], file: string, async = false): string = ftp.createJob(getLines[T], JRetrText) ftp.pasv() assertReply ftp.send("RETR " & file.normalizePathSep), ["125", "150"] - + if not async: while not ftp.job.prc(ftp, false): discard result = ftp.job.lines @@ -431,12 +431,12 @@ proc getFile[T](ftp: FtpBase[T], async = false): bool = var bytesRead = 0 var returned = false if async: - when T is TSocket: - raise newException(EFTP, "FTPClient must be async.") + when T is Socket: + raise newException(FTPError, "FTPClient must be async.") else: bytesRead = ftp.dsock.recvAsync(r, BufferSize) returned = bytesRead != -1 - else: + else: bytesRead = ftp.dsock.recv(r, BufferSize) returned = true let r2 = r.string @@ -447,9 +447,9 @@ proc getFile[T](ftp: FtpBase[T], async = false): bool = elif returned and r2 == "": ftp.dsockConnected = false - when T is TSocket: + when T is Socket: if not async: - var readSocks: seq[TSocket] = @[ftp.csock] + var readSocks: seq[Socket] = @[ftp.csock] blockingOperation(ftp.csock): if readSocks.select(1) != 0 and ftp.csock in readSocks: assertReply ftp.expectReply(), "226" @@ -458,7 +458,7 @@ proc getFile[T](ftp: FtpBase[T], async = false): bool = proc retrFile*[T](ftp: FtpBase[T], file, dest: string, async = false) = ## Downloads ``file`` and saves it to ``dest``. Usage of this function ## asynchronously is recommended to view the progress of the download. - ## The ``EvRetr`` event is passed to the specified ``handleEvent`` function + ## The ``EvRetr`` event is passed to the specified ``handleEvent`` function ## when the download is finished, and the ``filename`` field will be equal ## to ``file``. ftp.createJob(getFile[T], JRetr) @@ -467,11 +467,11 @@ proc retrFile*[T](ftp: FtpBase[T], file, dest: string, async = false) = var reply = ftp.send("RETR " & file.normalizePathSep) assertReply reply, ["125", "150"] if {'(', ')'} notin reply.string: - raise newException(EInvalidReply, "Reply has no file size.") + raise newException(ReplyError, "Reply has no file size.") var fileSize: BiggestInt if reply.string.captureBetween('(', ')').parseBiggestInt(fileSize) == 0: - raise newException(EInvalidReply, "Reply has no file size.") - + raise newException(ReplyError, "Reply has no file size.") + ftp.job.total = fileSize ftp.job.lastProgressReport = epochTime() ftp.job.filename = file.normalizePathSep @@ -488,7 +488,7 @@ proc doUpload[T](ftp: FtpBase[T], async = false): bool = if bytesSent == ftp.job.toStore.len: ftp.job.toStore = "" elif bytesSent != ftp.job.toStore.len and bytesSent != 0: - ftp.job.toStore = ftp.job.toStore[bytesSent .. -1] + ftp.job.toStore = ftp.job.toStore[bytesSent .. ^1] ftp.job.progress.inc(bytesSent) ftp.job.oneSecond.inc(bytesSent) else: @@ -499,12 +499,12 @@ proc doUpload[T](ftp: FtpBase[T], async = false): bool = # File finished uploading. ftp.dsock.close() ftp.dsockConnected = false - + if not async: assertReply ftp.expectReply(), "226" return true return false - + if not async: ftp.dsock.send(s) else: @@ -512,9 +512,9 @@ proc doUpload[T](ftp: FtpBase[T], async = false): bool = if bytesSent == 0: ftp.job.toStore.add(s) elif bytesSent != s.len: - ftp.job.toStore.add(s[bytesSent .. -1]) + ftp.job.toStore.add(s[bytesSent .. ^1]) len = bytesSent - + ftp.job.progress.inc(len) ftp.job.oneSecond.inc(len) @@ -522,8 +522,8 @@ proc store*[T](ftp: FtpBase[T], file, dest: string, async = false) = ## Uploads ``file`` to ``dest`` on the remote FTP server. Usage of this ## function asynchronously is recommended to view the progress of ## the download. - ## The ``EvStore`` event is passed to the specified ``handleEvent`` function - ## when the upload is finished, and the ``filename`` field will be + ## The ``EvStore`` event is passed to the specified ``handleEvent`` function + ## when the upload is finished, and the ``filename`` field will be ## equal to ``file``. ftp.createJob(doUpload[T], JStore) ftp.job.file = open(file) @@ -531,7 +531,7 @@ proc store*[T](ftp: FtpBase[T], file, dest: string, async = false) = ftp.job.lastProgressReport = epochTime() ftp.job.filename = file ftp.pasv() - + assertReply ftp.send("STOR " & dest.normalizePathSep), ["125", "150"] if not async: @@ -545,10 +545,10 @@ proc close*[T](ftp: FtpBase[T]) = ftp.csock.close() ftp.dsock.close() -proc csockHandleRead(s: PAsyncSocket, ftp: PAsyncFTPClient) = +proc csockHandleRead(s: AsyncSocket, ftp: AsyncFTPClient) = if ftp.jobInProgress: assertReply ftp.expectReply(), "226" # Make sure the transfer completed. - var r: TFTPEvent + var r: FTPEvent case ftp.job.typ of JRetrText: r.typ = EvLines @@ -557,21 +557,21 @@ proc csockHandleRead(s: PAsyncSocket, ftp: PAsyncFTPClient) = r.typ = EvRetr r.filename = ftp.job.filename if ftp.job.progress != ftp.job.total: - raise newException(EFTP, "Didn't download full file.") + raise newException(FTPError, "Didn't download full file.") of JStore: r.typ = EvStore r.filename = ftp.job.filename if ftp.job.progress != ftp.job.total: - raise newException(EFTP, "Didn't upload full file.") + raise newException(FTPError, "Didn't upload full file.") ftp.deleteJob() - + ftp.handleEvent(ftp, r) -proc asyncFTPClient*(address: string, port = TPort(21), +proc asyncFTPClient*(address: string, port = Port(21), user, pass = "", - handleEvent: proc (ftp: PAsyncFTPClient, ev: TFTPEvent) {.closure,gcsafe.} = - (proc (ftp: PAsyncFTPClient, ev: TFTPEvent) = discard)): PAsyncFTPClient = - ## Create a ``PAsyncFTPClient`` object. + handleEvent: proc (ftp: AsyncFTPClient, ev: FTPEvent) {.closure,gcsafe.} = + (proc (ftp: AsyncFTPClient, ev: FTPEvent) = discard)): AsyncFTPClient = + ## Create a ``AsyncFTPClient`` object. ## ## Use this if you want to use asyncio's dispatcher. var dres: AsyncFtpClient @@ -588,12 +588,12 @@ proc asyncFTPClient*(address: string, port = TPort(21), csockHandleRead(s, dres) result = dres -proc register*(d: PDispatcher, ftp: PAsyncFTPClient): PDelegate {.discardable.} = +proc register*(d: Dispatcher, ftp: AsyncFTPClient): Delegate {.discardable.} = ## Registers ``ftp`` with dispatcher ``d``. ftp.disp = d return ftp.disp.register(ftp.csock) -when isMainModule: +when not defined(testing) and isMainModule: proc main = var d = newDispatcher() let hev = @@ -617,7 +617,7 @@ when isMainModule: echo d.len else: assert(false) var ftp = asyncFTPClient("example.com", user = "foo", pass = "bar", handleEvent = hev) - + d.register(ftp) d.len.echo() ftp.connect() @@ -629,7 +629,7 @@ when isMainModule: if not d.poll(): break main() -when isMainModule and false: +when not defined(testing) and isMainModule: var ftp = ftpClient("example.com", user = "foo", pass = "bar") ftp.connect() echo ftp.pwd() diff --git a/lib/pure/future.nim b/lib/pure/future.nim index a0c1b5c2d..661afd7b3 100644 --- a/lib/pure/future.nim +++ b/lib/pure/future.nim @@ -1,7 +1,7 @@ # # # Nim's Runtime Library -# (c) Copyright 2014 Dominik Picheta +# (c) Copyright 2015 Dominik Picheta # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -12,7 +12,7 @@ import macros -proc createProcType(p, b: PNimrodNode): PNimrodNode {.compileTime.} = +proc createProcType(p, b: NimNode): NimNode {.compileTime.} = #echo treeRepr(p) #echo treeRepr(b) result = newNimNode(nnkProcTy) @@ -44,7 +44,7 @@ proc createProcType(p, b: PNimrodNode): PNimrodNode {.compileTime.} = formalParams.add identDefs else: error("Incorrect type list in proc type declaration.") - + result.add formalParams result.add newEmptyNode() #echo(treeRepr(result)) @@ -59,10 +59,10 @@ macro `=>`*(p, b: expr): expr {.immediate.} = ## f(2, 2) ## ## passTwoAndTwo((x, y) => x + y) # 4 - + #echo treeRepr(p) #echo(treeRepr(b)) - var params: seq[PNimrodNode] = @[newIdentNode("auto")] + var params: seq[NimNode] = @[newIdentNode("auto")] case p.kind of nnkPar: @@ -118,7 +118,7 @@ macro `->`*(p, b: expr): expr {.immediate.} = ## ## proc pass2(f: (float, float) -> float): float = ## f(2, 2) - ## + ## ## # is the same as: ## ## proc pass2(f: proc (x, y: float): float): float = @@ -155,7 +155,6 @@ macro `[]`*(lc: ListComprehension, comp, typ: expr): expr = for i in countdown(comp[2].len-1, 0): let x = comp[2][i] - expectKind(x, nnkInfix) expectMinLen(x, 1) if x[0].kind == nnkIdent and $x[0].ident == "<-": expectLen(x, 3) diff --git a/lib/pure/gentabs.nim b/lib/pure/gentabs.nim index a6128efc9..84d0a44de 100644 --- a/lib/pure/gentabs.nim +++ b/lib/pure/gentabs.nim @@ -9,7 +9,7 @@ ## The ``gentabs`` module implements an efficient hash table that is a ## key-value mapping. The keys are required to be strings, but the values -## may be any Nim or user defined type. This module supports matching +## may be any Nim or user defined type. This module supports matching ## of keys in case-sensitive, case-insensitive and style-insensitive modes. {.deprecated.} @@ -22,7 +22,7 @@ type modeCaseSensitive, ## case sensitive matching of keys modeCaseInsensitive, ## case insensitive matching of keys modeStyleInsensitive ## style sensitive matching of keys - + TGenKeyValuePair[T] = tuple[key: string, val: T] TGenKeyValuePairSeq[T] = seq[TGenKeyValuePair[T]] TGenTable*[T] = object of RootObj @@ -83,7 +83,7 @@ proc rawGet[T](tbl: PGenTable[T], key: string): int = h = nextTry(h, high(tbl.data)) result = - 1 -proc rawInsert[T](tbl: PGenTable[T], data: var TGenKeyValuePairSeq[T], +proc rawInsert[T](tbl: PGenTable[T], data: var TGenKeyValuePairSeq[T], key: string, val: T) = var h: THash h = myhash(tbl, key) and high(data) @@ -96,7 +96,7 @@ proc enlarge[T](tbl: PGenTable[T]) = var n: TGenKeyValuePairSeq[T] newSeq(n, len(tbl.data) * growthFactor) for i in countup(0, high(tbl.data)): - if not isNil(tbl.data[i].key): + if not isNil(tbl.data[i].key): rawInsert[T](tbl, n, tbl.data[i].key, tbl.data[i].val) swap(tbl.data, n) @@ -141,20 +141,20 @@ when isMainModule: assert(not x.hasKey("NOPE")) # ...but key "NOPE" is not in the table. for k,v in pairs(x): # make sure the 'pairs' iterator works assert(x[k]==v) - + # # Verify a table of user-defined types # type TMyType = tuple[first, second: string] # a pair of strings - + var y = newGenTable[TMyType](modeCaseInsensitive) # hash table where each # value is TMyType tuple - + #var junk: TMyType = ("OK", "Here") - + #echo junk.first, " ", junk.second - + y["Hello"] = ("Hello", "World") y["Goodbye"] = ("Goodbye", "Everyone") #y["Hello"] = TMyType( ("Hello", "World") ) @@ -163,31 +163,44 @@ when isMainModule: assert( not isNil(y["Hello"].first) ) assert( y["Hello"].first == "Hello" ) assert( y["Hello"].second == "World" ) - + # # Verify table of tables # - var z: PGenTable[ PGenTable[int] ] # hash table where each value is + var z: PGenTable[ PGenTable[int] ] # hash table where each value is # a hash table of ints - + z = newGenTable[PGenTable[int]](modeCaseInsensitive) z["first"] = newGenTable[int](modeCaseInsensitive) z["first"]["one"] = 1 z["first"]["two"] = 2 z["first"]["three"] = 3 - + z["second"] = newGenTable[int](modeCaseInsensitive) z["second"]["red"] = 10 z["second"]["blue"] = 20 - + assert(len(z) == 2) # length of outer table assert(len(z["first"]) == 3) # length of "first" table assert(len(z["second"]) == 2) # length of "second" table assert( z["first"]["one"] == 1) # retrieve from first inner table assert( z["second"]["red"] == 10) # retrieve from second inner table - - for k,v in pairs(z): - echo( "$# ($#) ->" % [k,$len(v)] ) - #for k2,v2 in pairs(v): - # echo( " $# <-> $#" % [k2,$v2] ) - echo() + + when false: + # disabled: depends on hash order: + var output = "" + for k, v in pairs(z): + output.add( "$# ($#) ->\L" % [k,$len(v)] ) + for k2,v2 in pairs(v): + output.add( " $# <-> $#\L" % [k2,$v2] ) + + let expected = unindent """ + first (3) -> + two <-> 2 + three <-> 3 + one <-> 1 + second (2) -> + red <-> 10 + blue <-> 20 + """ + assert output == expected diff --git a/lib/pure/hashes.nim b/lib/pure/hashes.nim index 30daaf2dc..a16342d44 100644 --- a/lib/pure/hashes.nim +++ b/lib/pure/hashes.nim @@ -33,8 +33,8 @@ ## proc hash(x: Something): THash = ## ## Computes a THash from `x`. ## var h: THash = 0 -## h = h &! hash(x.foo) -## h = h &! hash(x.bar) +## h = h !& hash(x.foo) +## h = h !& hash(x.bar) ## result = !$h import diff --git a/lib/pure/htmlgen.nim b/lib/pure/htmlgen.nim index 4852ed50d..e6c15371e 100644 --- a/lib/pure/htmlgen.nim +++ b/lib/pure/htmlgen.nim @@ -1,7 +1,7 @@ # # # Nim's Runtime Library -# (c) Copyright 2014 Andreas Rumpf +# (c) Copyright 2015 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -12,7 +12,7 @@ ## as ``from htmlgen import nil`` and then fully qualify the macros. ## ## -## This module implements a simple `XML`:idx: and `HTML`:idx: code +## This module implements a simple `XML`:idx: and `HTML`:idx: code ## generator. Each commonly used HTML tag has a corresponding macro ## that generates a string with its HTML representation. ## @@ -21,9 +21,9 @@ ## .. code-block:: Nim ## var nim = "Nim" ## echo h1(a(href="http://nim-lang.org", nim)) -## +## ## Writes the string:: -## +## ## <h1><a href="http://nim-lang.org">Nim</a></h1> ## @@ -36,16 +36,16 @@ const "onmouseover onmousemove onmouseout onkeypress onkeydown onkeyup " commonAttr* = coreAttr & eventAttr -proc getIdent(e: PNimrodNode): string {.compileTime.} = +proc getIdent(e: NimNode): string {.compileTime.} = case e.kind of nnkIdent: result = normalize($e.ident) - of nnkAccQuoted: + of nnkAccQuoted: result = getIdent(e[0]) for i in 1 .. e.len-1: result.add getIdent(e[i]) else: error("cannot extract identifier from node: " & toStrLit(e).strVal) -proc delete[T](s: var seq[T], attr: T): bool = +proc delete[T](s: var seq[T], attr: T): bool = var idx = find(s, attr) if idx >= 0: var L = s.len @@ -53,10 +53,10 @@ proc delete[T](s: var seq[T], attr: T): bool = setLen(s, L-1) result = true -proc xmlCheckedTag*(e: PNimrodNode, tag: string, optAttr = "", reqAttr = "", - isLeaf = false): PNimrodNode {.compileTime.} = +proc xmlCheckedTag*(e: NimNode, tag: string, optAttr = "", reqAttr = "", + isLeaf = false): NimNode {.compileTime.} = ## use this procedure to define a new XML tag - + # copy the attributes; when iterating over them these lists # will be modified, so that each attribute is only given one value var req = split(reqAttr) @@ -66,7 +66,7 @@ proc xmlCheckedTag*(e: PNimrodNode, tag: string, optAttr = "", reqAttr = "", result.add(newStrLitNode(tag)) # first pass over attributes: for i in 1..e.len-1: - if e[i].kind == nnkExprEqExpr: + if e[i].kind == nnkExprEqExpr: var name = getIdent(e[i][0]) if delete(req, name) or delete(opt, name): result.add(newStrLitNode(" ")) @@ -81,7 +81,7 @@ proc xmlCheckedTag*(e: PNimrodNode, tag: string, optAttr = "", reqAttr = "", error(req[0] & " attribute for '" & tag & "' element expected") if isLeaf: for i in 1..e.len-1: - if e[i].kind != nnkExprEqExpr: + if e[i].kind != nnkExprEqExpr: error("element " & tag & " cannot be nested") result.add(newStrLitNode(" />")) else: @@ -95,395 +95,396 @@ proc xmlCheckedTag*(e: PNimrodNode, tag: string, optAttr = "", reqAttr = "", result = nestList(!"&", result) -macro a*(e: expr): expr {.immediate.} = +macro a*(e: expr): expr {.immediate.} = ## generates the HTML ``a`` element. let e = callsite() result = xmlCheckedTag(e, "a", "href charset type hreflang rel rev " & "accesskey tabindex" & commonAttr) -macro acronym*(e: expr): expr {.immediate.} = +macro acronym*(e: expr): expr {.immediate.} = ## generates the HTML ``acronym`` element. let e = callsite() result = xmlCheckedTag(e, "acronym", commonAttr) -macro address*(e: expr): expr {.immediate.} = +macro address*(e: expr): expr {.immediate.} = ## generates the HTML ``address`` element. let e = callsite() result = xmlCheckedTag(e, "address", commonAttr) -macro area*(e: expr): expr {.immediate.} = +macro area*(e: expr): expr {.immediate.} = ## generates the HTML ``area`` element. let e = callsite() result = xmlCheckedTag(e, "area", "shape coords href nohref" & " accesskey tabindex" & commonAttr, "alt", true) -macro b*(e: expr): expr {.immediate.} = +macro b*(e: expr): expr {.immediate.} = ## generates the HTML ``b`` element. let e = callsite() result = xmlCheckedTag(e, "b", commonAttr) -macro base*(e: expr): expr {.immediate.} = +macro base*(e: expr): expr {.immediate.} = ## generates the HTML ``base`` element. let e = callsite() result = xmlCheckedTag(e, "base", "", "href", true) -macro big*(e: expr): expr {.immediate.} = +macro big*(e: expr): expr {.immediate.} = ## generates the HTML ``big`` element. let e = callsite() result = xmlCheckedTag(e, "big", commonAttr) -macro blockquote*(e: expr): expr {.immediate.} = +macro blockquote*(e: expr): expr {.immediate.} = ## generates the HTML ``blockquote`` element. let e = callsite() result = xmlCheckedTag(e, "blockquote", " cite" & commonAttr) -macro body*(e: expr): expr {.immediate.} = +macro body*(e: expr): expr {.immediate.} = ## generates the HTML ``body`` element. let e = callsite() result = xmlCheckedTag(e, "body", commonAttr) -macro br*(e: expr): expr {.immediate.} = +macro br*(e: expr): expr {.immediate.} = ## generates the HTML ``br`` element. let e = callsite() result = xmlCheckedTag(e, "br", "", "", true) -macro button*(e: expr): expr {.immediate.} = +macro button*(e: expr): expr {.immediate.} = ## generates the HTML ``button`` element. let e = callsite() result = xmlCheckedTag(e, "button", "accesskey tabindex " & "disabled name type value" & commonAttr) -macro caption*(e: expr): expr {.immediate.} = +macro caption*(e: expr): expr {.immediate.} = ## generates the HTML ``caption`` element. let e = callsite() result = xmlCheckedTag(e, "caption", commonAttr) -macro cite*(e: expr): expr {.immediate.} = +macro cite*(e: expr): expr {.immediate.} = ## generates the HTML ``cite`` element. let e = callsite() result = xmlCheckedTag(e, "cite", commonAttr) -macro code*(e: expr): expr {.immediate.} = +macro code*(e: expr): expr {.immediate.} = ## generates the HTML ``code`` element. let e = callsite() result = xmlCheckedTag(e, "code", commonAttr) -macro col*(e: expr): expr {.immediate.} = +macro col*(e: expr): expr {.immediate.} = ## generates the HTML ``col`` element. let e = callsite() result = xmlCheckedTag(e, "col", "span align valign" & commonAttr, "", true) -macro colgroup*(e: expr): expr {.immediate.} = +macro colgroup*(e: expr): expr {.immediate.} = ## generates the HTML ``colgroup`` element. let e = callsite() result = xmlCheckedTag(e, "colgroup", "span align valign" & commonAttr) -macro dd*(e: expr): expr {.immediate.} = +macro dd*(e: expr): expr {.immediate.} = ## generates the HTML ``dd`` element. let e = callsite() result = xmlCheckedTag(e, "dd", commonAttr) -macro del*(e: expr): expr {.immediate.} = +macro del*(e: expr): expr {.immediate.} = ## generates the HTML ``del`` element. let e = callsite() result = xmlCheckedTag(e, "del", "cite datetime" & commonAttr) -macro dfn*(e: expr): expr {.immediate.} = +macro dfn*(e: expr): expr {.immediate.} = ## generates the HTML ``dfn`` element. let e = callsite() result = xmlCheckedTag(e, "dfn", commonAttr) -macro `div`*(e: expr): expr {.immediate.} = +macro `div`*(e: expr): expr {.immediate.} = ## generates the HTML ``div`` element. let e = callsite() result = xmlCheckedTag(e, "div", commonAttr) -macro dl*(e: expr): expr {.immediate.} = +macro dl*(e: expr): expr {.immediate.} = ## generates the HTML ``dl`` element. let e = callsite() result = xmlCheckedTag(e, "dl", commonAttr) -macro dt*(e: expr): expr {.immediate.} = +macro dt*(e: expr): expr {.immediate.} = ## generates the HTML ``dt`` element. let e = callsite() result = xmlCheckedTag(e, "dt", commonAttr) -macro em*(e: expr): expr {.immediate.} = +macro em*(e: expr): expr {.immediate.} = ## generates the HTML ``em`` element. let e = callsite() result = xmlCheckedTag(e, "em", commonAttr) -macro fieldset*(e: expr): expr {.immediate.} = +macro fieldset*(e: expr): expr {.immediate.} = ## generates the HTML ``fieldset`` element. let e = callsite() result = xmlCheckedTag(e, "fieldset", commonAttr) -macro form*(e: expr): expr {.immediate.} = +macro form*(e: expr): expr {.immediate.} = ## generates the HTML ``form`` element. let e = callsite() - result = xmlCheckedTag(e, "form", "method encype accept accept-charset" & + result = xmlCheckedTag(e, "form", "method encype accept accept-charset" & commonAttr, "action") -macro h1*(e: expr): expr {.immediate.} = +macro h1*(e: expr): expr {.immediate.} = ## generates the HTML ``h1`` element. let e = callsite() result = xmlCheckedTag(e, "h1", commonAttr) -macro h2*(e: expr): expr {.immediate.} = +macro h2*(e: expr): expr {.immediate.} = ## generates the HTML ``h2`` element. let e = callsite() result = xmlCheckedTag(e, "h2", commonAttr) -macro h3*(e: expr): expr {.immediate.} = +macro h3*(e: expr): expr {.immediate.} = ## generates the HTML ``h3`` element. let e = callsite() result = xmlCheckedTag(e, "h3", commonAttr) -macro h4*(e: expr): expr {.immediate.} = +macro h4*(e: expr): expr {.immediate.} = ## generates the HTML ``h4`` element. let e = callsite() result = xmlCheckedTag(e, "h4", commonAttr) -macro h5*(e: expr): expr {.immediate.} = +macro h5*(e: expr): expr {.immediate.} = ## generates the HTML ``h5`` element. let e = callsite() result = xmlCheckedTag(e, "h5", commonAttr) -macro h6*(e: expr): expr {.immediate.} = +macro h6*(e: expr): expr {.immediate.} = ## generates the HTML ``h6`` element. let e = callsite() result = xmlCheckedTag(e, "h6", commonAttr) -macro head*(e: expr): expr {.immediate.} = +macro head*(e: expr): expr {.immediate.} = ## generates the HTML ``head`` element. let e = callsite() result = xmlCheckedTag(e, "head", "profile") -macro html*(e: expr): expr {.immediate.} = +macro html*(e: expr): expr {.immediate.} = ## generates the HTML ``html`` element. let e = callsite() result = xmlCheckedTag(e, "html", "xmlns", "") -macro hr*(): expr {.immediate.} = +macro hr*(): expr {.immediate.} = ## generates the HTML ``hr`` element. let e = callsite() result = xmlCheckedTag(e, "hr", commonAttr, "", true) -macro i*(e: expr): expr {.immediate.} = +macro i*(e: expr): expr {.immediate.} = ## generates the HTML ``i`` element. let e = callsite() result = xmlCheckedTag(e, "i", commonAttr) -macro img*(e: expr): expr {.immediate.} = +macro img*(e: expr): expr {.immediate.} = ## generates the HTML ``img`` element. let e = callsite() result = xmlCheckedTag(e, "img", "longdesc height width", "src alt", true) -macro input*(e: expr): expr {.immediate.} = +macro input*(e: expr): expr {.immediate.} = ## generates the HTML ``input`` element. let e = callsite() result = xmlCheckedTag(e, "input", "name type value checked maxlength src" & " alt accept disabled readonly accesskey tabindex" & commonAttr, "", true) -macro ins*(e: expr): expr {.immediate.} = +macro ins*(e: expr): expr {.immediate.} = ## generates the HTML ``ins`` element. let e = callsite() result = xmlCheckedTag(e, "ins", "cite datetime" & commonAttr) -macro kbd*(e: expr): expr {.immediate.} = +macro kbd*(e: expr): expr {.immediate.} = ## generates the HTML ``kbd`` element. let e = callsite() result = xmlCheckedTag(e, "kbd", commonAttr) -macro label*(e: expr): expr {.immediate.} = +macro label*(e: expr): expr {.immediate.} = ## generates the HTML ``label`` element. let e = callsite() result = xmlCheckedTag(e, "label", "for accesskey" & commonAttr) -macro legend*(e: expr): expr {.immediate.} = +macro legend*(e: expr): expr {.immediate.} = ## generates the HTML ``legend`` element. let e = callsite() result = xmlCheckedTag(e, "legend", "accesskey" & commonAttr) -macro li*(e: expr): expr {.immediate.} = +macro li*(e: expr): expr {.immediate.} = ## generates the HTML ``li`` element. let e = callsite() result = xmlCheckedTag(e, "li", commonAttr) -macro link*(e: expr): expr {.immediate.} = +macro link*(e: expr): expr {.immediate.} = ## generates the HTML ``link`` element. let e = callsite() - result = xmlCheckedTag(e, "link", "href charset hreflang type rel rev media" & + result = xmlCheckedTag(e, "link", "href charset hreflang type rel rev media" & commonAttr, "", true) -macro map*(e: expr): expr {.immediate.} = +macro map*(e: expr): expr {.immediate.} = ## generates the HTML ``map`` element. let e = callsite() result = xmlCheckedTag(e, "map", "class title" & eventAttr, "id", false) -macro meta*(e: expr): expr {.immediate.} = +macro meta*(e: expr): expr {.immediate.} = ## generates the HTML ``meta`` element. let e = callsite() result = xmlCheckedTag(e, "meta", "name http-equiv scheme", "content", true) -macro noscript*(e: expr): expr {.immediate.} = +macro noscript*(e: expr): expr {.immediate.} = ## generates the HTML ``noscript`` element. let e = callsite() result = xmlCheckedTag(e, "noscript", commonAttr) -macro `object`*(e: expr): expr {.immediate.} = +macro `object`*(e: expr): expr {.immediate.} = ## generates the HTML ``object`` element. let e = callsite() result = xmlCheckedTag(e, "object", "classid data codebase declare type " & "codetype archive standby width height name tabindex" & commonAttr) -macro ol*(e: expr): expr {.immediate.} = +macro ol*(e: expr): expr {.immediate.} = ## generates the HTML ``ol`` element. let e = callsite() result = xmlCheckedTag(e, "ol", commonAttr) -macro optgroup*(e: expr): expr {.immediate.} = +macro optgroup*(e: expr): expr {.immediate.} = ## generates the HTML ``optgroup`` element. let e = callsite() result = xmlCheckedTag(e, "optgroup", "disabled" & commonAttr, "label", false) -macro option*(e: expr): expr {.immediate.} = +macro option*(e: expr): expr {.immediate.} = ## generates the HTML ``option`` element. let e = callsite() result = xmlCheckedTag(e, "option", "selected value" & commonAttr) -macro p*(e: expr): expr {.immediate.} = +macro p*(e: expr): expr {.immediate.} = ## generates the HTML ``p`` element. let e = callsite() result = xmlCheckedTag(e, "p", commonAttr) -macro param*(e: expr): expr {.immediate.} = +macro param*(e: expr): expr {.immediate.} = ## generates the HTML ``param`` element. let e = callsite() result = xmlCheckedTag(e, "param", "value id type valuetype", "name", true) -macro pre*(e: expr): expr {.immediate.} = +macro pre*(e: expr): expr {.immediate.} = ## generates the HTML ``pre`` element. let e = callsite() result = xmlCheckedTag(e, "pre", commonAttr) -macro q*(e: expr): expr {.immediate.} = +macro q*(e: expr): expr {.immediate.} = ## generates the HTML ``q`` element. let e = callsite() result = xmlCheckedTag(e, "q", "cite" & commonAttr) -macro samp*(e: expr): expr {.immediate.} = +macro samp*(e: expr): expr {.immediate.} = ## generates the HTML ``samp`` element. let e = callsite() result = xmlCheckedTag(e, "samp", commonAttr) -macro script*(e: expr): expr {.immediate.} = +macro script*(e: expr): expr {.immediate.} = ## generates the HTML ``script`` element. let e = callsite() result = xmlCheckedTag(e, "script", "src charset defer", "type", false) -macro select*(e: expr): expr {.immediate.} = +macro select*(e: expr): expr {.immediate.} = ## generates the HTML ``select`` element. let e = callsite() - result = xmlCheckedTag(e, "select", "name size multiple disabled tabindex" & + result = xmlCheckedTag(e, "select", "name size multiple disabled tabindex" & commonAttr) -macro small*(e: expr): expr {.immediate.} = +macro small*(e: expr): expr {.immediate.} = ## generates the HTML ``small`` element. let e = callsite() result = xmlCheckedTag(e, "small", commonAttr) -macro span*(e: expr): expr {.immediate.} = +macro span*(e: expr): expr {.immediate.} = ## generates the HTML ``span`` element. let e = callsite() result = xmlCheckedTag(e, "span", commonAttr) -macro strong*(e: expr): expr {.immediate.} = +macro strong*(e: expr): expr {.immediate.} = ## generates the HTML ``strong`` element. let e = callsite() result = xmlCheckedTag(e, "strong", commonAttr) -macro style*(e: expr): expr {.immediate.} = +macro style*(e: expr): expr {.immediate.} = ## generates the HTML ``style`` element. let e = callsite() result = xmlCheckedTag(e, "style", "media title", "type") -macro sub*(e: expr): expr {.immediate.} = +macro sub*(e: expr): expr {.immediate.} = ## generates the HTML ``sub`` element. let e = callsite() result = xmlCheckedTag(e, "sub", commonAttr) -macro sup*(e: expr): expr {.immediate.} = +macro sup*(e: expr): expr {.immediate.} = ## generates the HTML ``sup`` element. let e = callsite() result = xmlCheckedTag(e, "sup", commonAttr) -macro table*(e: expr): expr {.immediate.} = +macro table*(e: expr): expr {.immediate.} = ## generates the HTML ``table`` element. let e = callsite() result = xmlCheckedTag(e, "table", "summary border cellpadding cellspacing" & " frame rules width" & commonAttr) -macro tbody*(e: expr): expr {.immediate.} = +macro tbody*(e: expr): expr {.immediate.} = ## generates the HTML ``tbody`` element. let e = callsite() result = xmlCheckedTag(e, "tbody", "align valign" & commonAttr) -macro td*(e: expr): expr {.immediate.} = +macro td*(e: expr): expr {.immediate.} = ## generates the HTML ``td`` element. let e = callsite() result = xmlCheckedTag(e, "td", "colspan rowspan abbr axis headers scope" & " align valign" & commonAttr) -macro textarea*(e: expr): expr {.immediate.} = +macro textarea*(e: expr): expr {.immediate.} = ## generates the HTML ``textarea`` element. let e = callsite() result = xmlCheckedTag(e, "textarea", " name disabled readonly accesskey" & " tabindex" & commonAttr, "rows cols", false) -macro tfoot*(e: expr): expr {.immediate.} = +macro tfoot*(e: expr): expr {.immediate.} = ## generates the HTML ``tfoot`` element. let e = callsite() result = xmlCheckedTag(e, "tfoot", "align valign" & commonAttr) -macro th*(e: expr): expr {.immediate.} = +macro th*(e: expr): expr {.immediate.} = ## generates the HTML ``th`` element. let e = callsite() result = xmlCheckedTag(e, "th", "colspan rowspan abbr axis headers scope" & " align valign" & commonAttr) -macro thead*(e: expr): expr {.immediate.} = +macro thead*(e: expr): expr {.immediate.} = ## generates the HTML ``thead`` element. let e = callsite() result = xmlCheckedTag(e, "thead", "align valign" & commonAttr) -macro title*(e: expr): expr {.immediate.} = +macro title*(e: expr): expr {.immediate.} = ## generates the HTML ``title`` element. let e = callsite() result = xmlCheckedTag(e, "title") -macro tr*(e: expr): expr {.immediate.} = +macro tr*(e: expr): expr {.immediate.} = ## generates the HTML ``tr`` element. let e = callsite() result = xmlCheckedTag(e, "tr", "align valign" & commonAttr) -macro tt*(e: expr): expr {.immediate.} = +macro tt*(e: expr): expr {.immediate.} = ## generates the HTML ``tt`` element. let e = callsite() result = xmlCheckedTag(e, "tt", commonAttr) -macro ul*(e: expr): expr {.immediate.} = +macro ul*(e: expr): expr {.immediate.} = ## generates the HTML ``ul`` element. let e = callsite() result = xmlCheckedTag(e, "ul", commonAttr) -macro `var`*(e: expr): expr {.immediate.} = +macro `var`*(e: expr): expr {.immediate.} = ## generates the HTML ``var`` element. let e = callsite() result = xmlCheckedTag(e, "var", commonAttr) when isMainModule: - var nim = "Nim" - echo h1(a(href="http://nim-lang.org", nim)) - echo form(action="test", `accept-charset` = "Content-Type") - + let nim = "Nim" + assert h1(a(href="http://nim-lang.org", nim)) == + """<h1><a href="http://nim-lang.org">Nim</a></h1>""" + assert form(action="test", `accept-charset` = "Content-Type") == + """<form action="test" accept-charset="Content-Type"></form>""" diff --git a/lib/pure/htmlparser.nim b/lib/pure/htmlparser.nim index e2cbb4949..9719181b8 100644 --- a/lib/pure/htmlparser.nim +++ b/lib/pure/htmlparser.nim @@ -552,7 +552,7 @@ proc parse(x: var XmlParser, errors: var seq[string]): XmlNode = proc parseHtml*(s: Stream, filename: string, errors: var seq[string]): XmlNode = ## parses the XML from stream `s` and returns a ``PXmlNode``. Every - ## occured parsing error is added to the `errors` sequence. + ## occurred parsing error is added to the `errors` sequence. var x: XmlParser open(x, s, filename, {reportComments, reportWhitespace}) next(x) @@ -581,7 +581,7 @@ proc parseHtml*(s: Stream): XmlNode = proc loadHtml*(path: string, errors: var seq[string]): XmlNode = ## Loads and parses HTML from file specified by ``path``, and returns - ## a ``PXmlNode``. Every occured parsing error is added to + ## a ``PXmlNode``. Every occurred parsing error is added to ## the `errors` sequence. var s = newFileStream(path, fmRead) if s == nil: raise newException(IOError, "Unable to read file: " & path) @@ -593,7 +593,7 @@ proc loadHtml*(path: string): XmlNode = var errors: seq[string] = @[] result = loadHtml(path, errors) -when isMainModule: +when not defined(testing) and isMainModule: import os var errors: seq[string] = @[] diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index 3afb625ee..9c27ecdab 100644 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -32,21 +32,12 @@ ## the server. ## ## .. code-block:: Nim -## var headers: string = "Content-Type: multipart/form-data; boundary=xyz\c\L" -## var body: string = "--xyz\c\L" -## # soap 1.2 output -## body.add("Content-Disposition: form-data; name=\"output\"\c\L") -## body.add("\c\Lsoap12\c\L") +## var data = newMultipartData() +## data["output"] = "soap12" +## data["uploaded_file"] = ("test.html", "text/html", +## "<html><head></head><body><p>test</p></body></html>") ## -## # html -## body.add("--xyz\c\L") -## body.add("Content-Disposition: form-data; name=\"uploaded_file\";" & -## " filename=\"test.html\"\c\L") -## body.add("Content-Type: text/html\c\L") -## body.add("\c\L<html><head></head><body><p>test</p></body></html>\c\L") -## body.add("--xyz--") -## -## echo(postContent("http://validator.w3.org/check", headers, body)) +## echo postContent("http://validator.w3.org/check", multipart=data) ## ## Asynchronous HTTP requests ## ========================== @@ -88,7 +79,7 @@ ## constructor should be used for this purpose. However, ## currently only basic authentication is supported. -import net, strutils, uri, parseutils, strtabs, base64, os +import net, strutils, uri, parseutils, strtabs, base64, os, mimetypes, math import asyncnet, asyncdispatch import rawsockets @@ -103,6 +94,10 @@ type url*: Uri auth*: string + MultipartEntries* = openarray[tuple[name, content: string]] + MultipartData* = ref object + content: seq[string] + ProtocolError* = object of IOError ## exception that is raised when server ## does not conform to the implemented ## protocol @@ -232,7 +227,7 @@ proc parseResponse(s: Socket, getBody: bool, timeout: int): Response = inc(linei, le) # Status code linei.inc skipWhitespace(line, linei) - result.status = line[linei .. -1] + result.status = line[linei .. ^1] parsedStatus = true else: # Parse headers @@ -243,7 +238,7 @@ proc parseResponse(s: Socket, getBody: bool, timeout: int): Response = if line[linei] != ':': httpError("invalid headers") inc(linei) # Skip : - result.headers[name] = line[linei.. -1].strip() + result.headers[name] = line[linei.. ^1].strip() if not fullyRead: httpError("Connection was closed before full request has been made") if getBody: @@ -282,19 +277,126 @@ proc newProxy*(url: string, auth = ""): Proxy = ## Constructs a new ``TProxy`` object. result = Proxy(url: parseUri(url), auth: auth) -proc request*(url: string, httpMethod = httpGET, extraHeaders = "", - body = "", - sslContext: SSLContext = defaultSSLContext, - timeout = -1, userAgent = defUserAgent, - proxy: Proxy = nil): Response = - ## | Requests ``url`` with the specified ``httpMethod``. - ## | Extra headers can be specified and must be seperated by ``\c\L`` +proc newMultipartData*: MultipartData = + ## Constructs a new ``MultipartData`` object. + MultipartData(content: @[]) + +proc add*(p: var MultipartData, name, content: string, filename: string = nil, + contentType: string = nil) = + ## Add a value to the multipart data. Raises a `ValueError` exception if + ## `name`, `filename` or `contentType` contain newline characters. + + if {'\c','\L'} in name: + raise newException(ValueError, "name contains a newline character") + if filename != nil and {'\c','\L'} in filename: + raise newException(ValueError, "filename contains a newline character") + if contentType != nil and {'\c','\L'} in contentType: + raise newException(ValueError, "contentType contains a newline character") + + var str = "Content-Disposition: form-data; name=\"" & name & "\"" + if filename != nil: + str.add("; filename=\"" & filename & "\"") + str.add("\c\L") + if contentType != nil: + str.add("Content-Type: " & contentType & "\c\L") + str.add("\c\L" & content & "\c\L") + + p.content.add(str) + +proc add*(p: var MultipartData, xs: MultipartEntries): MultipartData + {.discardable.} = + ## Add a list of multipart entries to the multipart data `p`. All values are + ## added without a filename and without a content type. + ## + ## .. code-block:: Nim + ## data.add({"action": "login", "format": "json"}) + for name, content in xs.items: + p.add(name, content) + result = p + +proc newMultipartData*(xs: MultipartEntries): MultipartData = + ## Create a new multipart data object and fill it with the entries `xs` + ## directly. + ## + ## .. code-block:: Nim + ## var data = newMultipartData({"action": "login", "format": "json"}) + result = MultipartData(content: @[]) + result.add(xs) + +proc addFiles*(p: var MultipartData, xs: openarray[tuple[name, file: string]]): + MultipartData {.discardable.} = + ## Add files to a multipart data object. The file will be opened from your + ## disk, read and sent with the automatically determined MIME type. Raises an + ## `IOError` if the file cannot be opened or reading fails. To manually + ## specify file content, filename and MIME type, use `[]=` instead. + ## + ## .. code-block:: Nim + ## data.addFiles({"uploaded_file": "public/test.html"}) + var m = newMimetypes() + for name, file in xs.items: + var contentType: string + let (dir, fName, ext) = splitFile(file) + if ext.len > 0: + contentType = m.getMimetype(ext[1..ext.high], nil) + p.add(name, readFile(file), fName & ext, contentType) + result = p + +proc `[]=`*(p: var MultipartData, name, content: string) = + ## Add a multipart entry to the multipart data `p`. The value is added + ## without a filename and without a content type. + ## + ## .. code-block:: Nim + ## data["username"] = "NimUser" + p.add(name, content) + +proc `[]=`*(p: var MultipartData, name: string, + file: tuple[name, contentType, content: string]) = + ## Add a file to the multipart data `p`, specifying filename, contentType and + ## content manually. + ## + ## .. code-block:: Nim + ## data["uploaded_file"] = ("test.html", "text/html", + ## "<html><head></head><body><p>test</p></body></html>") + p.add(name, file.content, file.name, file.contentType) + +proc format(p: MultipartData): tuple[header, body: string] = + if p == nil or p.content == nil or p.content.len == 0: + return ("", "") + + # Create boundary that is not in the data to be formatted + var bound: string + while true: + bound = $random(int.high) + var found = false + for s in p.content: + if bound in s: + found = true + if not found: + break + + result.header = "Content-Type: multipart/form-data; boundary=" & bound & "\c\L" + result.body = "" + for s in p.content: + result.body.add("--" & bound & "\c\L" & s) + result.body.add("--" & bound & "--\c\L") + +proc request*(url: string, httpMethod: string, extraHeaders = "", + body = "", sslContext = defaultSSLContext, timeout = -1, + userAgent = defUserAgent, proxy: Proxy = nil): Response = + ## | Requests ``url`` with the custom method string specified by the + ## | ``httpMethod`` parameter. + ## | Extra headers can be specified and must be separated by ``\c\L`` ## | An optional timeout can be specified in miliseconds, if reading from the ## server takes longer than specified an ETimeout exception will be raised. var r = if proxy == nil: parseUri(url) else: proxy.url - var headers = substr($httpMethod, len("http")) + var headers = substr(httpMethod, len("http")) + # TODO: Use generateHeaders further down once it supports proxies. if proxy == nil: - headers.add(" /" & r.path & r.query) + headers.add ' ' + if r.path[0] != '/': headers.add '/' + headers.add(r.path) + if r.query.len > 0: + headers.add("?" & r.query) else: headers.add(" " & url) @@ -330,9 +432,19 @@ proc request*(url: string, httpMethod = httpGET, extraHeaders = "", if body != "": s.send(body) - result = parseResponse(s, httpMethod != httpHEAD, timeout) + result = parseResponse(s, httpMethod != "httpHEAD", timeout) s.close() +proc request*(url: string, httpMethod = httpGET, extraHeaders = "", + body = "", sslContext = defaultSSLContext, timeout = -1, + userAgent = defUserAgent, proxy: Proxy = nil): Response = + ## | Requests ``url`` with the specified ``httpMethod``. + ## | Extra headers can be specified and must be separated by ``\c\L`` + ## | An optional timeout can be specified in miliseconds, if reading from the + ## server takes longer than specified an ETimeout exception will be raised. + result = request(url, $httpMethod, extraHeaders, body, sslContext, timeout, + userAgent, proxy) + proc redirection(status: string): bool = const redirectionNRs = ["301", "302", "303", "307"] for i in items(redirectionNRs): @@ -387,22 +499,37 @@ proc post*(url: string, extraHeaders = "", body = "", maxRedirects = 5, sslContext: SSLContext = defaultSSLContext, timeout = -1, userAgent = defUserAgent, - proxy: Proxy = nil): Response = + proxy: Proxy = nil, + multipart: MultipartData = nil): Response = ## | POSTs ``body`` to the ``url`` and returns a ``Response`` object. ## | This proc adds the necessary Content-Length header. ## | This proc also handles redirection. ## | Extra headers can be specified and must be separated by ``\c\L``. ## | An optional timeout can be specified in miliseconds, if reading from the ## server takes longer than specified an ETimeout exception will be raised. - var xh = extraHeaders & "Content-Length: " & $len(body) & "\c\L" - result = request(url, httpPOST, xh, body, sslContext, timeout, userAgent, + ## | The optional ``multipart`` parameter can be used to create + ## ``multipart/form-data`` POSTs comfortably. + let (mpHeaders, mpBody) = format(multipart) + + template withNewLine(x): expr = + if x.len > 0 and not x.endsWith("\c\L"): + x & "\c\L" + else: + x + + var xb = mpBody.withNewLine() & body + + var xh = extraHeaders.withNewLine() & mpHeaders.withNewLine() & + withNewLine("Content-Length: " & $len(xb)) + + result = request(url, httpPOST, xh, xb, sslContext, timeout, userAgent, proxy) var lastUrl = "" for i in 1..maxRedirects: if result.status.redirection(): let redirectTo = getNewLocation(lastURL, result.headers) var meth = if result.status != "307": httpGet else: httpPost - result = request(redirectTo, meth, xh, body, sslContext, timeout, + result = request(redirectTo, meth, xh, xb, sslContext, timeout, userAgent, proxy) lastUrl = redirectTo @@ -410,14 +537,17 @@ proc postContent*(url: string, extraHeaders = "", body = "", maxRedirects = 5, sslContext: SSLContext = defaultSSLContext, timeout = -1, userAgent = defUserAgent, - proxy: Proxy = nil): string = + proxy: Proxy = nil, + multipart: MultipartData = nil): string = ## | POSTs ``body`` to ``url`` and returns the response's body as a string ## | Raises exceptions for the status codes ``4xx`` and ``5xx`` ## | Extra headers can be specified and must be separated by ``\c\L``. ## | An optional timeout can be specified in miliseconds, if reading from the ## server takes longer than specified an ETimeout exception will be raised. + ## | The optional ``multipart`` parameter can be used to create + ## ``multipart/form-data`` POSTs comfortably. var r = post(url, extraHeaders, body, maxRedirects, sslContext, timeout, - userAgent, proxy) + userAgent, proxy, multipart) if r.status[0] in {'4','5'}: raise newException(HttpRequestError, r.status) else: @@ -438,11 +568,16 @@ proc downloadFile*(url: string, outputFilename: string, else: fileError("Unable to open file") -proc generateHeaders(r: Uri, httpMethod: HttpMethod, +proc generateHeaders(r: Uri, httpMethod: string, headers: StringTableRef): string = - result = substr($httpMethod, len("http")) + # TODO: Use this in the blocking HttpClient once it supports proxies. + result = substr(httpMethod, len("http")) # TODO: Proxies - result.add(" /" & r.path & r.query) + result.add ' ' + if r.path[0] != '/': result.add '/' + result.add(r.path) + if r.query.len > 0: + result.add("?" & r.query) result.add(" HTTP/1.1\c\L") add(result, "Host: " & r.hostname & "\c\L") @@ -590,7 +725,7 @@ proc parseResponse(client: AsyncHttpClient, inc(linei, le) # Status code linei.inc skipWhitespace(line, linei) - result.status = line[linei .. -1] + result.status = line[linei .. ^1] parsedStatus = true else: # Parse headers @@ -601,7 +736,7 @@ proc parseResponse(client: AsyncHttpClient, if line[linei] != ':': httpError("invalid headers") inc(linei) # Skip : - result.headers[name] = line[linei.. -1].strip() + result.headers[name] = line[linei.. ^1].strip() if not fullyRead: httpError("Connection was closed before full request has been made") if getBody: @@ -635,10 +770,10 @@ proc newConnection(client: AsyncHttpClient, url: Uri) {.async.} = client.currentURL = url client.connected = true -proc request*(client: AsyncHttpClient, url: string, httpMethod = httpGET, +proc request*(client: AsyncHttpClient, url: string, httpMethod: string, body = ""): Future[Response] {.async.} = ## Connects to the hostname specified by the URL and performs a request - ## using the method specified. + ## using the custom method string specified by ``httpMethod``. ## ## Connection will kept alive. Further requests on the same ``client`` to ## the same hostname will not require a new connection to be made. The @@ -651,13 +786,25 @@ proc request*(client: AsyncHttpClient, url: string, httpMethod = httpGET, if not client.headers.hasKey("user-agent") and client.userAgent != "": client.headers["User-Agent"] = client.userAgent - var headers = generateHeaders(r, httpMethod, client.headers) + var headers = generateHeaders(r, $httpMethod, client.headers) await client.socket.send(headers) if body != "": await client.socket.send(body) - result = await parseResponse(client, httpMethod != httpHEAD) + result = await parseResponse(client, httpMethod != "httpHEAD") + +proc request*(client: AsyncHttpClient, url: string, httpMethod = httpGET, + body = ""): Future[Response] = + ## Connects to the hostname specified by the URL and performs a request + ## using the method specified. + ## + ## Connection will kept alive. Further requests on the same ``client`` to + ## the same hostname will not require a new connection to be made. The + ## connection can be closed by using the ``close`` procedure. + ## + ## The returned future will complete once the request is completed. + result = request(client, url, $httpMethod, body) proc get*(client: AsyncHttpClient, url: string): Future[Response] {.async.} = ## Connects to the hostname specified by the URL and performs a GET request. @@ -672,7 +819,7 @@ proc get*(client: AsyncHttpClient, url: string): Future[Response] {.async.} = result = await client.request(redirectTo, httpGET) lastUrl = redirectTo -when isMainModule: +when not defined(testing) and isMainModule: when true: # Async proc main() {.async.} = @@ -706,18 +853,9 @@ when isMainModule: #var r = get("http://validator.w3.org/check?uri=http%3A%2F%2Fgoogle.com& # charset=%28detect+automatically%29&doctype=Inline&group=0") - var headers: string = "Content-Type: multipart/form-data; boundary=xyz\c\L" - var body: string = "--xyz\c\L" - # soap 1.2 output - body.add("Content-Disposition: form-data; name=\"output\"\c\L") - body.add("\c\Lsoap12\c\L") - - # html - body.add("--xyz\c\L") - body.add("Content-Disposition: form-data; name=\"uploaded_file\";" & - " filename=\"test.html\"\c\L") - body.add("Content-Type: text/html\c\L") - body.add("\c\L<html><head></head><body><p>test</p></body></html>\c\L") - body.add("--xyz--") - - echo(postContent("http://validator.w3.org/check", headers, body)) + var data = newMultipartData() + data["output"] = "soap12" + data["uploaded_file"] = ("test.html", "text/html", + "<html><head></head><body><p>test</p></body></html>") + + echo postContent("http://validator.w3.org/check", multipart=data) diff --git a/lib/pure/httpserver.nim b/lib/pure/httpserver.nim index 38a068ea1..dc76c9228 100644 --- a/lib/pure/httpserver.nim +++ b/lib/pure/httpserver.nim @@ -363,7 +363,7 @@ proc run*(handleRequest: proc (client: Socket, port = Port(80)) = ## encapsulates the server object and main loop var s: TServer - open(s, port) + open(s, port, reuseAddr = true) #echo("httpserver running on port ", s.port) while true: next(s) @@ -514,7 +514,7 @@ proc close*(h: PAsyncHTTPServer) = ## Closes the ``PAsyncHTTPServer``. h.asyncSocket.close() -when isMainModule: +when not defined(testing) and isMainModule: var counter = 0 var s: TServer diff --git a/lib/pure/json.nim b/lib/pure/json.nim index 385787d6c..5d824d6f8 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -1,15 +1,15 @@ # # # Nim's Runtime Library -# (c) Copyright 2014 Andreas Rumpf, Dominik Picheta +# (c) Copyright 2015 Andreas Rumpf, Dominik Picheta # # See the file "copying.txt", included in this # distribution, for details about the copyright. # ## This module implements a simple high performance `JSON`:idx: -## parser. JSON (JavaScript Object Notation) is a lightweight -## data-interchange format that is easy for humans to read and write +## parser. JSON (JavaScript Object Notation) is a lightweight +## data-interchange format that is easy for humans to read and write ## (unlike XML). It is easy for machines to parse and generate. ## JSON is based on a subset of the JavaScript Programming Language, ## Standard ECMA-262 3rd Edition - December 1999. @@ -30,13 +30,32 @@ ## ## 1.3000000000000000e+00 ## true +## +## This module can also be used to comfortably create JSON using the `%*` +## operator: +## +## .. code-block:: nim +## +## var hisName = "John" +## let herAge = 31 +## var j = %* +## [ +## { +## "name": hisName, +## "age": 30 +## }, +## { +## "name": "Susan", +## "age": herAge +## } +## ] + +import + hashes, strutils, lexbase, streams, unicode, macros -import - hashes, strutils, lexbase, streams, unicode - -type +type JsonEventKind* = enum ## enumeration of all events that may occur when parsing - jsonError, ## an error ocurred during parsing + jsonError, ## an error occurred during parsing jsonEof, ## end of file reached jsonString, ## a string literal jsonInt, ## an integer literal @@ -48,7 +67,7 @@ type jsonObjectEnd, ## end of an object: the ``}`` token jsonArrayStart, ## start of an array: the ``[`` token jsonArrayEnd ## start of an array: the ``]`` token - + TTokKind = enum # must be synchronized with TJsonEventKind! tkError, tkEof, @@ -64,7 +83,7 @@ type tkBracketRi, tkColon, tkComma - + JsonError* = enum ## enumeration that lists all errors that can occur errNone, ## no error errInvalidToken, ## invalid token @@ -77,8 +96,8 @@ type errEOC_Expected, ## ``*/`` expected errEofExpected, ## EOF expected errExprExpected ## expr expected - - ParserState = enum + + ParserState = enum stateEof, stateStart, stateObject, stateArray, stateExpectArrayComma, stateExpectObjectComma, stateExpectColon, stateExpectValue @@ -92,7 +111,7 @@ type {.deprecated: [TJsonEventKind: JsonEventKind, TJsonError: JsonError, TJsonParser: JsonParser].} - + const errorMessages: array [JsonError, string] = [ "no error", @@ -127,56 +146,56 @@ proc open*(my: var JsonParser, input: Stream, filename: string) = my.state = @[stateStart] my.kind = jsonError my.a = "" - -proc close*(my: var JsonParser) {.inline.} = + +proc close*(my: var JsonParser) {.inline.} = ## closes the parser `my` and its associated input stream. lexbase.close(my) -proc str*(my: JsonParser): string {.inline.} = - ## returns the character data for the events: ``jsonInt``, ``jsonFloat``, +proc str*(my: JsonParser): string {.inline.} = + ## returns the character data for the events: ``jsonInt``, ``jsonFloat``, ## ``jsonString`` assert(my.kind in {jsonInt, jsonFloat, jsonString}) return my.a -proc getInt*(my: JsonParser): BiggestInt {.inline.} = +proc getInt*(my: JsonParser): BiggestInt {.inline.} = ## returns the number for the event: ``jsonInt`` assert(my.kind == jsonInt) return parseBiggestInt(my.a) -proc getFloat*(my: JsonParser): float {.inline.} = +proc getFloat*(my: JsonParser): float {.inline.} = ## returns the number for the event: ``jsonFloat`` assert(my.kind == jsonFloat) return parseFloat(my.a) -proc kind*(my: JsonParser): JsonEventKind {.inline.} = +proc kind*(my: JsonParser): JsonEventKind {.inline.} = ## returns the current event type for the JSON parser return my.kind - -proc getColumn*(my: JsonParser): int {.inline.} = + +proc getColumn*(my: JsonParser): int {.inline.} = ## get the current column the parser has arrived at. result = getColNumber(my, my.bufpos) -proc getLine*(my: JsonParser): int {.inline.} = +proc getLine*(my: JsonParser): int {.inline.} = ## get the current line the parser has arrived at. result = my.lineNumber -proc getFilename*(my: JsonParser): string {.inline.} = +proc getFilename*(my: JsonParser): string {.inline.} = ## get the filename of the file that the parser processes. result = my.filename - -proc errorMsg*(my: JsonParser): string = + +proc errorMsg*(my: JsonParser): string = ## returns a helpful error message for the event ``jsonError`` assert(my.kind == jsonError) result = "$1($2, $3) Error: $4" % [ my.filename, $getLine(my), $getColumn(my), errorMessages[my.err]] -proc errorMsgExpected*(my: JsonParser, e: string): string = +proc errorMsgExpected*(my: JsonParser, e: string): string = ## returns an error message "`e` expected" in the same format as the - ## other error messages + ## other error messages result = "$1($2, $3) Error: $4" % [ my.filename, $getLine(my), $getColumn(my), e & " expected"] -proc handleHexChar(c: char, x: var int): bool = +proc handleHexChar(c: char, x: var int): bool = result = true # Success case c of '0'..'9': x = (x shl 4) or (ord(c) - ord('0')) @@ -189,8 +208,8 @@ proc parseString(my: var JsonParser): TTokKind = var pos = my.bufpos + 1 var buf = my.buf while true: - case buf[pos] - of '\0': + case buf[pos] + of '\0': my.err = errQuoteExpected result = tkError break @@ -199,21 +218,21 @@ proc parseString(my: var JsonParser): TTokKind = break of '\\': case buf[pos+1] - of '\\', '"', '\'', '/': + of '\\', '"', '\'', '/': add(my.a, buf[pos+1]) inc(pos, 2) of 'b': add(my.a, '\b') - inc(pos, 2) + inc(pos, 2) of 'f': add(my.a, '\f') - inc(pos, 2) + inc(pos, 2) of 'n': add(my.a, '\L') - inc(pos, 2) + inc(pos, 2) of 'r': add(my.a, '\C') - inc(pos, 2) + inc(pos, 2) of 't': add(my.a, '\t') inc(pos, 2) @@ -225,15 +244,15 @@ proc parseString(my: var JsonParser): TTokKind = if handleHexChar(buf[pos], r): inc(pos) if handleHexChar(buf[pos], r): inc(pos) add(my.a, toUTF8(Rune(r))) - else: + else: # don't bother with the error add(my.a, buf[pos]) inc(pos) - of '\c': + of '\c': pos = lexbase.handleCR(my, pos) buf = my.buf add(my.a, '\c') - of '\L': + of '\L': pos = lexbase.handleLF(my, pos) buf = my.buf add(my.a, '\L') @@ -241,25 +260,25 @@ proc parseString(my: var JsonParser): TTokKind = add(my.a, buf[pos]) inc(pos) my.bufpos = pos # store back - -proc skip(my: var JsonParser) = + +proc skip(my: var JsonParser) = var pos = my.bufpos var buf = my.buf - while true: + while true: case buf[pos] - of '/': - if buf[pos+1] == '/': + of '/': + if buf[pos+1] == '/': # skip line comment: inc(pos, 2) while true: - case buf[pos] - of '\0': + case buf[pos] + of '\0': break - of '\c': + of '\c': pos = lexbase.handleCR(my, pos) buf = my.buf break - of '\L': + of '\L': pos = lexbase.handleLF(my, pos) buf = my.buf break @@ -269,44 +288,44 @@ proc skip(my: var JsonParser) = # skip long comment: inc(pos, 2) while true: - case buf[pos] - of '\0': + case buf[pos] + of '\0': my.err = errEOC_Expected break - of '\c': + of '\c': pos = lexbase.handleCR(my, pos) buf = my.buf - of '\L': + of '\L': pos = lexbase.handleLF(my, pos) buf = my.buf of '*': inc(pos) - if buf[pos] == '/': + if buf[pos] == '/': inc(pos) break else: inc(pos) - else: + else: break - of ' ', '\t': + of ' ', '\t': inc(pos) - of '\c': + of '\c': pos = lexbase.handleCR(my, pos) buf = my.buf - of '\L': + of '\L': pos = lexbase.handleLF(my, pos) buf = my.buf else: break my.bufpos = pos -proc parseNumber(my: var JsonParser) = +proc parseNumber(my: var JsonParser) = var pos = my.bufpos var buf = my.buf - if buf[pos] == '-': + if buf[pos] == '-': add(my.a, '-') inc(pos) - if buf[pos] == '.': + if buf[pos] == '.': add(my.a, "0.") inc(pos) else: @@ -331,7 +350,7 @@ proc parseNumber(my: var JsonParser) = inc(pos) my.bufpos = pos -proc parseName(my: var JsonParser) = +proc parseName(my: var JsonParser) = var pos = my.bufpos var buf = my.buf if buf[pos] in IdentStartChars: @@ -340,11 +359,11 @@ proc parseName(my: var JsonParser) = inc(pos) my.bufpos = pos -proc getTok(my: var JsonParser): TTokKind = +proc getTok(my: var JsonParser): TTokKind = setLen(my.a, 0) skip(my) # skip whitespace, comments case my.buf[my.bufpos] - of '-', '.', '0'..'9': + of '-', '.', '0'..'9': parseNumber(my) if {'.', 'e', 'E'} in my.a: result = tkFloat @@ -374,17 +393,17 @@ proc getTok(my: var JsonParser): TTokKind = result = tkEof of 'a'..'z', 'A'..'Z', '_': parseName(my) - case my.a + case my.a of "null": result = tkNull of "true": result = tkTrue of "false": result = tkFalse else: result = tkError - else: + else: inc(my.bufpos) result = tkError my.tok = result -proc next*(my: var JsonParser) = +proc next*(my: var JsonParser) = ## retrieves the first/next event. This controls the parser. var tk = getTok(my) var i = my.state.len-1 @@ -397,13 +416,13 @@ proc next*(my: var JsonParser) = else: my.kind = jsonError my.err = errEofExpected - of stateStart: - # tokens allowed? + of stateStart: + # tokens allowed? case tk of tkString, tkInt, tkFloat, tkTrue, tkFalse, tkNull: my.state[i] = stateEof # expect EOF next! my.kind = JsonEventKind(ord(tk)) - of tkBracketLe: + of tkBracketLe: my.state.add(stateArray) # we expect any my.kind = jsonArrayStart of tkCurlyLe: @@ -414,12 +433,12 @@ proc next*(my: var JsonParser) = else: my.kind = jsonError my.err = errEofExpected - of stateObject: + of stateObject: case tk of tkString, tkInt, tkFloat, tkTrue, tkFalse, tkNull: my.state.add(stateExpectColon) my.kind = JsonEventKind(ord(tk)) - of tkBracketLe: + of tkBracketLe: my.state.add(stateExpectColon) my.state.add(stateArray) my.kind = jsonArrayStart @@ -438,7 +457,7 @@ proc next*(my: var JsonParser) = of tkString, tkInt, tkFloat, tkTrue, tkFalse, tkNull: my.state.add(stateExpectArrayComma) # expect value next! my.kind = JsonEventKind(ord(tk)) - of tkBracketLe: + of tkBracketLe: my.state.add(stateExpectArrayComma) my.state.add(stateArray) my.kind = jsonArrayStart @@ -453,8 +472,8 @@ proc next*(my: var JsonParser) = my.kind = jsonError my.err = errBracketRiExpected of stateExpectArrayComma: - case tk - of tkComma: + case tk + of tkComma: discard my.state.pop() next(my) of tkBracketRi: @@ -465,8 +484,8 @@ proc next*(my: var JsonParser) = my.kind = jsonError my.err = errBracketRiExpected of stateExpectObjectComma: - case tk - of tkComma: + case tk + of tkComma: discard my.state.pop() next(my) of tkCurlyRi: @@ -476,9 +495,9 @@ proc next*(my: var JsonParser) = else: my.kind = jsonError my.err = errCurlyRiExpected - of stateExpectColon: - case tk - of tkColon: + of stateExpectColon: + case tk + of tkColon: my.state[i] = stateExpectValue next(my) else: @@ -489,7 +508,7 @@ proc next*(my: var JsonParser) = of tkString, tkInt, tkFloat, tkTrue, tkFalse, tkNull: my.state[i] = stateExpectObjectComma my.kind = JsonEventKind(ord(tk)) - of tkBracketLe: + of tkBracketLe: my.state[i] = stateExpectObjectComma my.state.add(stateArray) my.kind = jsonArrayStart @@ -513,8 +532,8 @@ type JString, JObject, JArray - - JsonNode* = ref JsonNodeObj ## JSON node + + JsonNode* = ref JsonNodeObj ## JSON node JsonNodeObj* {.acyclic.} = object case kind*: JsonNodeKind of JString: @@ -586,6 +605,49 @@ proc newJArray*(): JsonNode = result.kind = JArray result.elems = @[] +proc getStr*(n: JsonNode, default: string = ""): string = + ## Retrieves the string value of a `JString JsonNode`. + ## + ## Returns ``default`` if ``n`` is not a ``JString``. + if n.kind != JString: return default + else: return n.str + +proc getNum*(n: JsonNode, default: BiggestInt = 0): BiggestInt = + ## Retrieves the int value of a `JInt JsonNode`. + ## + ## Returns ``default`` if ``n`` is not a ``JInt``. + if n.kind != JInt: return default + else: return n.num + +proc getFNum*(n: JsonNode, default: float = 0.0): float = + ## Retrieves the float value of a `JFloat JsonNode`. + ## + ## Returns ``default`` if ``n`` is not a ``JFloat``. + if n.kind != JFloat: return default + else: return n.fnum + +proc getBVal*(n: JsonNode, default: bool = false): bool = + ## Retrieves the bool value of a `JBool JsonNode`. + ## + ## Returns ``default`` if ``n`` is not a ``JBool``. + if n.kind != JBool: return default + else: return n.bval + +proc getFields*(n: JsonNode, + default: seq[tuple[key: string, val: JsonNode]] = @[]): + seq[tuple[key: string, val: JsonNode]] = + ## Retrieves the key, value pairs of a `JObject JsonNode`. + ## + ## Returns ``default`` if ``n`` is not a ``JObject``. + if n.kind != JObject: return default + else: return n.fields + +proc getElems*(n: JsonNode, default: seq[JsonNode] = @[]): seq[JsonNode] = + ## Retrieves the int value of a `JArray JsonNode`. + ## + ## Returns ``default`` if ``n`` is not a ``JArray``. + if n.kind != JArray: return default + else: return n.elems proc `%`*(s: string): JsonNode = ## Generic constructor for JSON data. Creates a new `JString JsonNode`. @@ -625,12 +687,35 @@ proc `%`*(elements: openArray[JsonNode]): JsonNode = newSeq(result.elems, elements.len) for i, p in pairs(elements): result.elems[i] = p +proc toJson(x: NimNode): NimNode {.compiletime.} = + case x.kind + of nnkBracket: + result = newNimNode(nnkBracket) + for i in 0 .. <x.len: + result.add(toJson(x[i])) + + of nnkTableConstr: + result = newNimNode(nnkTableConstr) + for i in 0 .. <x.len: + assert x[i].kind == nnkExprColonExpr + result.add(newNimNode(nnkExprColonExpr).add(x[i][0]).add(toJson(x[i][1]))) + + else: + result = x + + result = prefix(result, "%") + +macro `%*`*(x: expr): expr = + ## Convert an expression to a JsonNode directly, without having to specify + ## `%` for every element. + result = toJson(x) + proc `==`* (a,b: JsonNode): bool = ## Check two nodes for equality if a.isNil: if b.isNil: return true return false - elif b.isNil or a.kind != b.kind: + elif b.isNil or a.kind != b.kind: return false else: return case a.kind @@ -667,7 +752,7 @@ proc hash* (n:JsonNode): THash = of JNull: result = hash(0) -proc len*(n: JsonNode): int = +proc len*(n: JsonNode): int = ## If `n` is a `JArray`, it returns the number of elements. ## If `n` is a `JObject`, it returns the number of pairs. ## Else it returns 0. @@ -685,7 +770,7 @@ proc `[]`*(node: JsonNode, name: string): JsonNode = if key == name: return item return nil - + proc `[]`*(node: JsonNode, index: int): JsonNode = ## Gets the node at `index` in an Array. Result is undefined if `index` ## is out of bounds @@ -702,12 +787,12 @@ proc hasKey*(node: JsonNode, key: string): bool = proc existsKey*(node: JsonNode, key: string): bool {.deprecated.} = node.hasKey(key) ## Deprecated for `hasKey` -proc add*(father, child: JsonNode) = - ## Adds `child` to a JArray node `father`. +proc add*(father, child: JsonNode) = + ## Adds `child` to a JArray node `father`. assert father.kind == JArray father.elems.add(child) -proc add*(obj: JsonNode, key: string, val: JsonNode) = +proc add*(obj: JsonNode, key: string, val: JsonNode) = ## Adds ``(key, val)`` pair to the JObject node `obj`. For speed ## reasons no check for duplicate keys is performed! ## But ``[]=`` performs the check. @@ -718,27 +803,30 @@ proc `[]=`*(obj: JsonNode, key: string, val: JsonNode) = ## Sets a field from a `JObject`. Performs a check for duplicate keys. assert(obj.kind == JObject) for i in 0..obj.fields.len-1: - if obj.fields[i].key == key: + if obj.fields[i].key == key: obj.fields[i].val = val return obj.fields.add((key, val)) -proc `{}`*(node: JsonNode, key: string): JsonNode = - ## Transverses the node and gets the given value. If any of the - ## names does not exist, returns nil +proc `{}`*(node: JsonNode, keys: varargs[string]): JsonNode = + ## Traverses the node and gets the given value. If any of the + ## keys do not exist, returns nil. Also returns nil if one of the + ## intermediate data structures is not an object result = node - if isNil(node): return nil - result = result[key] - -proc `{}=`*(node: JsonNode, names: varargs[string], value: JsonNode) = - ## Transverses the node and tries to set the value at the given location - ## to `value` If any of the names are missing, they are added + for key in keys: + if isNil(result) or result.kind!=JObject: + return nil + result=result[key] + +proc `{}=`*(node: JsonNode, keys: varargs[string], value: JsonNode) = + ## Traverses the node and tries to set the value at the given location + ## to `value` If any of the keys are missing, they are added var node = node - for i in 0..(names.len-2): - if isNil(node[names[i]]): - node[names[i]] = newJObject() - node = node[names[i]] - node[names[names.len-1]] = value + for i in 0..(keys.len-2): + if isNil(node[keys[i]]): + node[keys[i]] = newJObject() + node = node[keys[i]] + node[keys[keys.len-1]] = value proc delete*(obj: JsonNode, key: string) = ## Deletes ``obj[key]`` preserving the order of the other (key, value)-pairs. @@ -773,17 +861,17 @@ proc copy*(p: JsonNode): JsonNode = # ------------- pretty printing ---------------------------------------------- -proc indent(s: var string, i: int) = - s.add(repeatChar(i)) +proc indent(s: var string, i: int) = + s.add(spaces(i)) proc newIndent(curr, indent: int, ml: bool): int = if ml: return curr + indent else: return indent -proc nl(s: var string, ml: bool) = +proc nl(s: var string, ml: bool) = if ml: s.add("\n") -proc escapeJson*(s: string): string = +proc escapeJson*(s: string): string = ## Converts a string `s` to its JSON representation. result = newStringOfCap(s.len + s.len shr 3) result.add("\"") @@ -800,13 +888,13 @@ proc escapeJson*(s: string): string = result.add(toHex(r, 4)) result.add("\"") -proc toPretty(result: var string, node: JsonNode, indent = 2, ml = true, +proc toPretty(result: var string, node: JsonNode, indent = 2, ml = true, lstArr = false, currIndent = 0) = case node.kind of JObject: if currIndent != 0 and not lstArr: result.nl(ml) result.indent(currIndent) # Indentation - if node.fields.len > 0: + if node.fields.len > 0: result.add("{") result.nl(ml) # New line for i in 0..len(node.fields)-1: @@ -814,17 +902,17 @@ proc toPretty(result: var string, node: JsonNode, indent = 2, ml = true, result.add(", ") result.nl(ml) # New Line # Need to indent more than { - result.indent(newIndent(currIndent, indent, ml)) + result.indent(newIndent(currIndent, indent, ml)) result.add(escapeJson(node.fields[i].key)) result.add(": ") - toPretty(result, node.fields[i].val, indent, ml, false, + toPretty(result, node.fields[i].val, indent, ml, false, newIndent(currIndent, indent, ml)) result.nl(ml) result.indent(currIndent) # indent the same as { result.add("}") else: result.add("{}") - of JString: + of JString: if lstArr: result.indent(currIndent) result.add(escapeJson(node.str)) of JInt: @@ -864,7 +952,7 @@ proc pretty*(node: JsonNode, indent = 2): string = proc `$`*(node: JsonNode): string = ## Converts `node` to its JSON Representation on one line. result = "" - toPretty(result, node, 1, false) + toPretty(result, node, 0, false) iterator items*(node: JsonNode): JsonNode = ## Iterator for the items of `node`. `node` has to be a JArray. @@ -872,17 +960,31 @@ iterator items*(node: JsonNode): JsonNode = for i in items(node.elems): yield i +iterator mitems*(node: var JsonNode): var JsonNode = + ## Iterator for the items of `node`. `node` has to be a JArray. Items can be + ## modified. + assert node.kind == JArray + for i in mitems(node.elems): + yield i + iterator pairs*(node: JsonNode): tuple[key: string, val: JsonNode] = ## Iterator for the child elements of `node`. `node` has to be a JObject. assert node.kind == JObject for key, val in items(node.fields): yield (key, val) -proc eat(p: var JsonParser, tok: TTokKind) = +iterator mpairs*(node: var JsonNode): var tuple[key: string, val: JsonNode] = + ## Iterator for the child elements of `node`. `node` has to be a JObject. + ## Items can be modified + assert node.kind == JObject + for keyVal in mitems(node.fields): + yield keyVal + +proc eat(p: var JsonParser, tok: TTokKind) = if p.tok == tok: discard getTok(p) else: raiseParseErr(p, tokToStr[tok]) -proc parseJson(p: var JsonParser): JsonNode = +proc parseJson(p: var JsonParser): JsonNode = ## Parses JSON from a JSON Parser `p`. case p.tok of tkString: @@ -899,17 +1001,17 @@ proc parseJson(p: var JsonParser): JsonNode = of tkTrue: result = newJBool(true) discard getTok(p) - of tkFalse: + of tkFalse: result = newJBool(false) discard getTok(p) - of tkNull: + of tkNull: result = newJNull() discard getTok(p) - of tkCurlyLe: + of tkCurlyLe: result = newJObject() discard getTok(p) - while p.tok != tkCurlyRi: - if p.tok != tkString: + while p.tok != tkCurlyRi: + if p.tok != tkString: raiseParseErr(p, "string literal as key expected") var key = p.a discard getTok(p) @@ -922,7 +1024,7 @@ proc parseJson(p: var JsonParser): JsonNode = of tkBracketLe: result = newJArray() discard getTok(p) - while p.tok != tkBracketRi: + while p.tok != tkBracketRi: result.add(parseJson(p)) if p.tok != tkComma: break discard getTok(p) @@ -1042,28 +1144,31 @@ when false: of jsonObjectEnd: echo("}") of jsonArrayStart: echo("[") of jsonArrayEnd: echo("]") - + close(x) -# { "json": 5 } +# { "json": 5 } # To get that we shall use, obj["json"] when isMainModule: #var node = parse("{ \"test\": null }") #echo(node.existsKey("test56")) + var parsed = parseFile("tests/testdata/jsontest.json") var parsed2 = parseFile("tests/testdata/jsontest2.json") - echo(parsed) - echo() - echo(pretty(parsed, 2)) - echo() - echo(parsed["keyÄÖöoßß"]) - echo() - echo(pretty(parsed2)) - try: - echo(parsed["key2"][12123]) - raise newException(ValueError, "That line was expected to fail") - except IndexError: echo() + + when not defined(testing): + echo(parsed) + echo() + echo(pretty(parsed, 2)) + echo() + echo(parsed["keyÄÖöoßß"]) + echo() + echo(pretty(parsed2)) + try: + echo(parsed["key2"][12123]) + raise newException(ValueError, "That line was expected to fail") + except IndexError: echo() let testJson = parseJson"""{ "a": [1, 2, 3, 4], "b": "asd" }""" # nil passthrough @@ -1087,11 +1192,51 @@ when isMainModule: except: assert(false, "EInvalidIndex thrown for valid index") - discard """ - while true: - var json = stdin.readLine() - var node = parse(json) - echo(node) - echo() - echo() - """ + assert(testJson{"b"}.str=="asd", "Couldn't fetch a singly nested key with {}") + assert(isNil(testJson{"nonexistent"}), "Non-existent keys should return nil") + assert(parsed2{"repository", "description"}.str=="IRC Library for Haskell", "Couldn't fetch via multiply nested key using {}") + assert(isNil(testJson{"a", "b"}), "Indexing through a list should return nil") + assert(isNil(testJson{"a", "b"}), "Indexing through a list should return nil") + assert(testJson{"a"}==parseJson"[1, 2, 3, 4]", "Didn't return a non-JObject when there was one to be found") + assert(isNil(parseJson("[1, 2, 3]"){"foo"}), "Indexing directly into a list should return nil") + + # Generator: + var j = %* [{"name": "John", "age": 30}, {"name": "Susan", "age": 31}] + assert j == %[%{"name": %"John", "age": %30}, %{"name": %"Susan", "age": %31}] + + var j2 = %* + [ + { + "name": "John", + "age": 30 + }, + { + "name": "Susan", + "age": 31 + } + ] + assert j2 == %[%{"name": %"John", "age": %30}, %{"name": %"Susan", "age": %31}] + + var name = "John" + let herAge = 30 + const hisAge = 31 + + var j3 = %* + [ { "name": "John" + , "age": herAge + } + , { "name": "Susan" + , "age": hisAge + } + ] + assert j3 == %[%{"name": %"John", "age": %30}, %{"name": %"Susan", "age": %31}] + + when not defined(testing): + discard """ + while true: + var json = stdin.readLine() + var node = parse(json) + echo(node) + echo() + echo() + """ diff --git a/lib/pure/lexbase.nim b/lib/pure/lexbase.nim index a3a3d7b5c..23a87d9f8 100644 --- a/lib/pure/lexbase.nim +++ b/lib/pure/lexbase.nim @@ -165,5 +165,5 @@ proc getCurrentLine(L: BaseLexer, marker: bool = true): string = inc(i) add(result, "\n") if marker: - add(result, repeatChar(getColNumber(L, L.bufpos)) & "^\n") + add(result, spaces(getColNumber(L, L.bufpos)) & "^\n") diff --git a/lib/pure/logging.nim b/lib/pure/logging.nim index 9247f69c5..a2ea53472 100644 --- a/lib/pure/logging.nim +++ b/lib/pure/logging.nim @@ -1,16 +1,16 @@ # # # Nim's Runtime Library -# (c) Copyright 2014 Andreas Rumpf, Dominik Picheta +# (c) Copyright 2015 Andreas Rumpf, Dominik Picheta # # See the file "copying.txt", included in this # distribution, for details about the copyright. # ## This module implements a simple logger. It has been designed to be as simple -## as possible to avoid bloat, if this library does not fullfill your needs, +## as possible to avoid bloat, if this library does not fulfill your needs, ## write your own. -## +## ## Format strings support the following variables which must be prefixed with ## the dollar operator (``$``): ## @@ -21,23 +21,26 @@ ## $time Current time ## $app ``os.getAppFilename()`` ## ============ ======================= -## +## ## ## The following example demonstrates logging to three different handlers ## simultaneously: ## ## .. code-block:: nim -## +## ## var L = newConsoleLogger() ## var fL = newFileLogger("test.log", fmtStr = verboseFmtStr) ## var rL = newRollingFileLogger("rolling.log", fmtStr = verboseFmtStr) -## handlers.add(L) -## handlers.add(fL) -## handlers.add(rL) +## addHandler(L) +## addHandler(fL) +## addHandler(rL) ## info("920410:52 accepted") ## warn("4 8 15 16 23 4-- Error") ## error("922044:16 SYSTEM FAILURE") ## fatal("SYSTEM FAILURE SYSTEM FAILURE") +## +## **Warning:** The global list of handlers is a thread var, this means that +## the handlers must be re-added in each thread. import strutils, os, times @@ -61,20 +64,20 @@ const type Logger* = ref object of RootObj ## abstract logger; the base type of all loggers - levelThreshold*: Level ## only messages of level >= levelThreshold + levelThreshold*: Level ## only messages of level >= levelThreshold ## should be processed fmtStr: string ## = defaultFmtStr by default, see substituteLog for $date etc. - + ConsoleLogger* = ref object of Logger ## logger that writes the messages to the ## console - + FileLogger* = ref object of Logger ## logger that writes the messages to a file f: File - - RollingFileLogger* = ref object of FileLogger ## logger that writes the + + RollingFileLogger* = ref object of FileLogger ## logger that writes the ## messages to a file and ## performs log rotation - maxLines: int # maximum number of lines + maxLines: int # maximum number of lines curLine : int baseName: string # initial filename baseMode: FileMode # initial file mode @@ -83,22 +86,22 @@ type {.deprecated: [TLevel: Level, PLogger: Logger, PConsoleLogger: ConsoleLogger, PFileLogger: FileLogger, PRollingFileLogger: RollingFileLogger].} -proc substituteLog(frmt: string): string = +proc substituteLog(frmt: string): string = ## converts $date to the current date ## converts $time to the current time ## converts $app to getAppFilename() - ## converts + ## converts result = newStringOfCap(frmt.len + 20) var i = 0 - while i < frmt.len: - if frmt[i] != '$': + while i < frmt.len: + if frmt[i] != '$': result.add(frmt[i]) inc(i) else: inc(i) var v = "" var app = getAppFilename() - while frmt[i] in IdentChars: + while frmt[i] in IdentChars: v.add(toLower(frmt[i])) inc(i) case v @@ -111,12 +114,12 @@ proc substituteLog(frmt: string): string = method log*(logger: Logger, level: Level, frmt: string, args: varargs[string, `$`]) {. - raises: [Exception], + raises: [Exception], tags: [TimeEffect, WriteIOEffect, ReadIOEffect].} = ## Override this method in custom loggers. Default implementation does ## nothing. discard - + method log*(logger: ConsoleLogger, level: Level, frmt: string, args: varargs[string, `$`]) = ## Logs to the console using ``logger`` only. @@ -124,14 +127,14 @@ method log*(logger: ConsoleLogger, level: Level, writeln(stdout, LevelNames[level], " ", substituteLog(logger.fmtStr), frmt % args) -method log*(logger: FileLogger, level: Level, +method log*(logger: FileLogger, level: Level, frmt: string, args: varargs[string, `$`]) = ## Logs to a file using ``logger`` only. if level >= logger.levelThreshold: writeln(logger.f, LevelNames[level], " ", substituteLog(logger.fmtStr), frmt % args) -proc defaultFilename*(): string = +proc defaultFilename*(): string = ## Returns the default filename for a logger. var (path, name, ext) = splitFile(getAppFilename()) result = changeFileExt(path / name, "log") @@ -142,10 +145,10 @@ proc newConsoleLogger*(levelThreshold = lvlAll, fmtStr = defaultFmtStr): Console result.fmtStr = fmtStr result.levelThreshold = levelThreshold -proc newFileLogger*(filename = defaultFilename(), +proc newFileLogger*(filename = defaultFilename(), mode: FileMode = fmAppend, levelThreshold = lvlAll, - fmtStr = defaultFmtStr): FileLogger = + fmtStr = defaultFmtStr): FileLogger = ## Creates a new file logger. This logger logs to a file. new(result) result.levelThreshold = levelThreshold @@ -167,14 +170,14 @@ proc countFiles(filename: string): int = if kind == pcFile: let llfn = name & ext & ExtSep if path.extractFilename.startsWith(llfn): - let numS = path.extractFilename[llfn.len .. -1] + let numS = path.extractFilename[llfn.len .. ^1] try: let num = parseInt(numS) if num > result: result = num except ValueError: discard -proc newRollingFileLogger*(filename = defaultFilename(), +proc newRollingFileLogger*(filename = defaultFilename(), mode: FileMode = fmReadWrite, levelThreshold = lvlAll, fmtStr = defaultFmtStr, @@ -183,15 +186,15 @@ proc newRollingFileLogger*(filename = defaultFilename(), ## a new log file will be started and the old will be renamed. new(result) result.levelThreshold = levelThreshold - result.fmtStr = defaultFmtStr + result.fmtStr = fmtStr result.maxLines = maxLines result.f = open(filename, mode) result.curLine = 0 result.baseName = filename result.baseMode = mode - + result.logFiles = countFiles(filename) - + if mode == fmAppend: # We need to get a line count because we will be appending to the file. result.curLine = countLogLines(result) @@ -203,7 +206,7 @@ proc rotate(logger: RollingFileLogger) = moveFile(dir / (name & ext & srcSuff), dir / (name & ext & ExtSep & $(i+1))) -method log*(logger: RollingFileLogger, level: Level, +method log*(logger: RollingFileLogger, level: Level, frmt: string, args: varargs[string, `$`]) = ## Logs to a file using rolling ``logger`` only. if level >= logger.levelThreshold: @@ -213,18 +216,17 @@ method log*(logger: RollingFileLogger, level: Level, logger.logFiles.inc logger.curLine = 0 logger.f = open(logger.baseName, logger.baseMode) - - writeln(logger.f, LevelNames[level], " ", frmt % args) + + writeln(logger.f, LevelNames[level], " ",substituteLog(logger.fmtStr), frmt % args) logger.curLine.inc # -------- -var - level* = lvlAll ## global log filter - handlers*: seq[Logger] = @[] ## handlers with their own log levels +var level {.threadvar.}: Level ## global log filter +var handlers {.threadvar.}: seq[Logger] ## handlers with their own log levels proc logLoop(level: Level, frmt: string, args: varargs[string, `$`]) = - for logger in items(handlers): + for logger in items(handlers): if level >= logger.levelThreshold: log(logger, level, frmt, args) @@ -233,7 +235,7 @@ template log*(level: Level, frmt: string, args: varargs[string, `$`]) = bind logLoop bind `%` bind logging.level - + if level >= logging.level: logLoop(level, frmt, args) @@ -241,33 +243,49 @@ template debug*(frmt: string, args: varargs[string, `$`]) = ## Logs a debug message to all registered handlers. log(lvlDebug, frmt, args) -template info*(frmt: string, args: varargs[string, `$`]) = +template info*(frmt: string, args: varargs[string, `$`]) = ## Logs an info message to all registered handlers. log(lvlInfo, frmt, args) -template warn*(frmt: string, args: varargs[string, `$`]) = +template warn*(frmt: string, args: varargs[string, `$`]) = ## Logs a warning message to all registered handlers. log(lvlWarn, frmt, args) -template error*(frmt: string, args: varargs[string, `$`]) = +template error*(frmt: string, args: varargs[string, `$`]) = ## Logs an error message to all registered handlers. log(lvlError, frmt, args) - -template fatal*(frmt: string, args: varargs[string, `$`]) = + +template fatal*(frmt: string, args: varargs[string, `$`]) = ## Logs a fatal error message to all registered handlers. log(lvlFatal, frmt, args) +proc addHandler*(handler: Logger) = + ## Adds ``handler`` to the list of handlers. + if handlers.isNil: handlers = @[] + handlers.add(handler) + +proc getHandlers*(): seq[Logger] = + ## Returns a list of all the registered handlers. + return handlers + +proc setLogFilter*(lvl: Level) = + ## Sets the global log filter. + level = lvl + +proc getLogFilter*(): Level = + ## Gets the global log filter. + return level # -------------- -when isMainModule: +when not defined(testing) and isMainModule: var L = newConsoleLogger() var fL = newFileLogger("test.log", fmtStr = verboseFmtStr) var rL = newRollingFileLogger("rolling.log", fmtStr = verboseFmtStr) - handlers.add(L) - handlers.add(fL) - handlers.add(rL) + addHandler(L) + addHandler(fL) + addHandler(rL) for i in 0 .. 25: info("hello" & $i, []) - + diff --git a/lib/pure/marshal.nim b/lib/pure/marshal.nim index 02abda722..e0092f314 100644 --- a/lib/pure/marshal.nim +++ b/lib/pure/marshal.nim @@ -1,7 +1,7 @@ # # # Nim's Runtime Library -# (c) Copyright 2014 Andreas Rumpf +# (c) Copyright 2015 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -15,8 +15,8 @@ ## type than its compiletime type: ## ## .. code-block:: nim -## -## type +## +## type ## TA = object ## TB = object of TA ## f: int @@ -28,6 +28,8 @@ ## new(b) ## a = b ## echo($$a[]) # produces "{}", not "{f: 0}" +## +## **Note**: The ``to`` and ``$$`` operations are available at compile-time! import streams, typeinfo, json, intsets, tables @@ -38,7 +40,12 @@ proc storeAny(s: Stream, a: TAny, stored: var IntSet) = case a.kind of akNone: assert false of akBool: s.write($getBool(a)) - of akChar: s.write(escapeJson($getChar(a))) + of akChar: + let ch = getChar(a) + if ch < '\128': + s.write(escapeJson($ch)) + else: + s.write($int(ch)) of akArray, akSequence: if a.kind == akSequence and isNil(a): s.write("null") else: @@ -92,7 +99,7 @@ proc storeAny(s: Stream, a: TAny, stored: var IntSet) = proc loadAny(p: var JsonParser, a: TAny, t: var Table[BiggestInt, pointer]) = case a.kind of akNone: assert false - of akBool: + of akBool: case p.kind of jsonFalse: setBiggestInt(a, 0) of jsonTrue: setBiggestInt(a, 1) @@ -105,8 +112,12 @@ proc loadAny(p: var JsonParser, a: TAny, t: var Table[BiggestInt, pointer]) = setBiggestInt(a, ord(x[0])) next(p) return + elif p.kind == jsonInt: + setBiggestInt(a, getInt(p)) + next(p) + return raiseParseErr(p, "string of length 1 expected for a char") - of akEnum: + of akEnum: if p.kind == jsonString: setBiggestInt(a, getEnumOrdinal(a, p.str)) next(p) @@ -122,7 +133,7 @@ proc loadAny(p: var JsonParser, a: TAny, t: var Table[BiggestInt, pointer]) = if p.kind == jsonArrayEnd: next(p) else: raiseParseErr(p, "']' end of array expected") of akSequence: - case p.kind + case p.kind of jsonNull: setPointer(a, nil) next(p) @@ -143,7 +154,7 @@ proc loadAny(p: var JsonParser, a: TAny, t: var Table[BiggestInt, pointer]) = if p.kind != jsonObjectStart: raiseParseErr(p, "'{' expected for an object") next(p) while p.kind != jsonObjectEnd and p.kind != jsonEof: - if p.kind != jsonString: + if p.kind != jsonString: raiseParseErr(p, "string expected for a field name") var fieldName = p.str next(p) @@ -160,7 +171,7 @@ proc loadAny(p: var JsonParser, a: TAny, t: var Table[BiggestInt, pointer]) = if p.kind == jsonArrayEnd: next(p) else: raiseParseErr(p, "']' end of array expected") of akPtr, akRef: - case p.kind + case p.kind of jsonNull: setPointer(a, nil) next(p) @@ -170,7 +181,7 @@ proc loadAny(p: var JsonParser, a: TAny, t: var Table[BiggestInt, pointer]) = of jsonArrayStart: next(p) if a.kind == akRef: invokeNew(a) - else: setPointer(a, alloc0(a.baseTypeSize)) + else: setPointer(a, alloc0(a.baseTypeSize)) if p.kind == jsonInt: t[p.getInt] = getPointer(a) next(p) @@ -179,8 +190,8 @@ proc loadAny(p: var JsonParser, a: TAny, t: var Table[BiggestInt, pointer]) = if p.kind == jsonArrayEnd: next(p) else: raiseParseErr(p, "']' end of ref-address pair expected") else: raiseParseErr(p, "int for pointer type expected") - of akProc, akPointer, akCString: - case p.kind + of akProc, akPointer, akCString: + case p.kind of jsonNull: setPointer(a, nil) next(p) @@ -189,7 +200,7 @@ proc loadAny(p: var JsonParser, a: TAny, t: var Table[BiggestInt, pointer]) = next(p) else: raiseParseErr(p, "int for pointer type expected") of akString: - case p.kind + case p.kind of jsonNull: setPointer(a, nil) next(p) @@ -197,7 +208,7 @@ proc loadAny(p: var JsonParser, a: TAny, t: var Table[BiggestInt, pointer]) = setString(a, p.str) next(p) else: raiseParseErr(p, "string expected") - of akInt..akInt64, akUInt..akUInt64: + of akInt..akInt64, akUInt..akUInt64: if p.kind == jsonInt: setBiggestInt(a, getInt(p)) next(p) @@ -243,22 +254,22 @@ proc to*[T](data: string): T = ## reads data and transforms it to a ``T``. var tab = initTable[BiggestInt, pointer]() loadAny(newStringStream(data), toAny(result), tab) - -when isMainModule: + +when not defined(testing) and isMainModule: template testit(x: expr) = echo($$to[type(x)]($$x)) var x: array[0..4, array[0..4, string]] = [ - ["test", "1", "2", "3", "4"], ["test", "1", "2", "3", "4"], - ["test", "1", "2", "3", "4"], ["test", "1", "2", "3", "4"], + ["test", "1", "2", "3", "4"], ["test", "1", "2", "3", "4"], + ["test", "1", "2", "3", "4"], ["test", "1", "2", "3", "4"], ["test", "1", "2", "3", "4"]] testit(x) var test2: tuple[name: string, s: uint] = ("tuple test", 56u) testit(test2) - + type TE = enum blah, blah2 - + TestObj = object test, asd: int case test2: TE @@ -266,7 +277,7 @@ when isMainModule: help: string else: nil - + PNode = ref TNode TNode = object next, prev: PNode @@ -294,7 +305,7 @@ when isMainModule: test4.a = "ref string test: A" test4.b = "ref string test: B" testit(test4) - + var test5 = @[(0,1),(2,3),(4,5)] testit(test5) @@ -305,7 +316,7 @@ when isMainModule: echo($$test7) testit(test7) - type + type TA {.inheritable.} = object TB = object of TA f: int diff --git a/lib/pure/matchers.nim b/lib/pure/matchers.nim index 46fc9985c..d55963c15 100644 --- a/lib/pure/matchers.nim +++ b/lib/pure/matchers.nim @@ -1,7 +1,7 @@ # # # Nim's Runtime Library -# (c) Copyright 2014 Andreas Rumpf +# (c) Copyright 2015 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -42,7 +42,7 @@ proc validEmailAddress*(s: string): bool {.noSideEffect, case toLower(x) of "com", "org", "net", "gov", "mil", "biz", "info", "mobi", "name", "aero", "jobs", "museum": return true - return false + else: return false proc parseInt*(s: string, value: var int, validRange: Slice[int]) {. noSideEffect, rtl, extern: "nmatchParseInt".} = diff --git a/lib/pure/math.nim b/lib/pure/math.nim index 987c35d1c..daa108460 100644 --- a/lib/pure/math.nim +++ b/lib/pure/math.nim @@ -1,7 +1,7 @@ # # # Nim's Runtime Library -# (c) Copyright 2014 Andreas Rumpf +# (c) Copyright 2015 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -93,9 +93,9 @@ proc nextPowerOfTwo*(x: int): int {.noSideEffect.} = result = x - 1 when defined(cpu64): result = result or (result shr 32) - when sizeof(int) > 16: + when sizeof(int) > 2: result = result or (result shr 16) - when sizeof(int) > 8: + when sizeof(int) > 1: result = result or (result shr 8) result = result or (result shr 4) result = result or (result shr 2) @@ -129,29 +129,30 @@ proc variance*(x: openArray[float]): float {.noSideEffect.} = result = result + diff*diff result = result / toFloat(len(x)) -proc random*(max: int): int {.gcsafe.} +proc random*(max: int): int {.benign.} ## returns a random number in the range 0..max-1. The sequence of ## random number is always the same, unless `randomize` is called ## which initializes the random number generator with a "random" ## number, i.e. a tickcount. -proc random*(max: float): float {.gcsafe.} +proc random*(max: float): float {.benign.} ## returns a random number in the range 0..<max. The sequence of ## random number is always the same, unless `randomize` is called ## which initializes the random number generator with a "random" ## number, i.e. a tickcount. This has a 16-bit resolution on windows ## and a 48-bit resolution on other platforms. -proc randomize*() {.gcsafe.} +proc randomize*() {.benign.} ## initializes the random number generator with a "random" ## number, i.e. a tickcount. Note: Does nothing for the JavaScript target, ## as JavaScript does not support this. -proc randomize*(seed: int) {.gcsafe.} +proc randomize*(seed: int) {.benign.} ## initializes the random number generator with a specific seed. ## Note: Does nothing for the JavaScript target, ## as JavaScript does not support this. +{.push noSideEffect.} when not defined(JS): proc sqrt*(x: float): float {.importc: "sqrt", header: "<math.h>".} ## computes the square root of `x`. @@ -273,6 +274,8 @@ else: var y = exp(2.0*x) return (y-1.0)/(y+1.0) +{.pop.} + proc `mod`*(x, y: float): float = result = if y == 0.0: x else: x - y * (x/y).floor @@ -280,7 +283,7 @@ proc random*[T](x: Slice[T]): T = ## For a slice `a .. b` returns a value in the range `a .. b-1`. result = random(x.b - x.a) + x.a -proc random[T](a: openArray[T]): T = +proc random*[T](a: openArray[T]): T = ## returns a random element from the openarray `a`. result = a[random(a.low..a.len)] @@ -329,6 +332,31 @@ proc standardDeviation*(s: RunningStat): float = {.pop.} {.pop.} +proc `^`*[T](x, y: T): T = + ## Computes ``x`` to the power ``y`. ``x`` must be non-negative, use + ## `pow <#pow,float,float>` for negative exponents. + assert y >= 0 + var (x, y) = (x, y) + result = 1 + + while y != 0: + if (y and 1) != 0: + result *= x + y = y shr 1 + x *= x + +proc gcd*[T](x, y: T): T = + ## Computes the greatest common divisor of ``x`` and ``y``. + var (x,y) = (x,y) + while y != 0: + x = x mod y + swap x, y + abs x + +proc lcm*[T](x, y: T): T = + ## Computes the least common multiple of ``x`` and ``y``. + x div gcd(x, y) * y + when isMainModule and not defined(JS): proc gettime(dummy: ptr cint): cint {.importc: "time", header: "<time.h>".} @@ -344,4 +372,10 @@ when isMainModule and not defined(JS): randomize(seed) for i in 0..SIZE-1: assert buf[i] == random(high(int)), "non deterministic random seeding" - echo "random values equal after reseeding" + + when not defined(testing): + echo "random values equal after reseeding" + + # Check for no side effect annotation + proc mySqrt(num: float): float {.noSideEffect.} = + return sqrt(num) diff --git a/lib/pure/md5.nim b/lib/pure/md5.nim index 3b5453957..5ee301b15 100644 --- a/lib/pure/md5.nim +++ b/lib/pure/md5.nim @@ -9,18 +9,20 @@ ## Module for computing MD5 checksums. -type - MD5State = array[0..3, int32] - MD5Block = array[0..15, int32] - MD5CBits = array[0..7, int8] - MD5Digest* = array[0..15, int8] - MD5Buffer = array[0..63, int8] - MD5Context* {.final.} = object +import unsigned + +type + MD5State = array[0..3, uint32] + MD5Block = array[0..15, uint32] + MD5CBits = array[0..7, uint8] + MD5Digest* = array[0..15, uint8] + MD5Buffer = array[0..63, uint8] + MD5Context* {.final.} = object state: MD5State - count: array[0..1, int32] + count: array[0..1, uint32] buffer: MD5Buffer -const +const padding: cstring = "\x80\0\0\0" & "\0\0\0\0\0\0\0\0" & "\0\0\0\0\0\0\0\0" & @@ -31,60 +33,60 @@ const "\0\0\0\0\0\0\0\0" & "\0\0\0\0" -proc F(x, y, z: int32): int32 {.inline.} = +proc F(x, y, z: uint32): uint32 {.inline.} = result = (x and y) or ((not x) and z) -proc G(x, y, z: int32): int32 {.inline.} = +proc G(x, y, z: uint32): uint32 {.inline.} = result = (x and z) or (y and (not z)) -proc H(x, y, z: int32): int32 {.inline.} = +proc H(x, y, z: uint32): uint32 {.inline.} = result = x xor y xor z -proc I(x, y, z: int32): int32 {.inline.} = +proc I(x, y, z: uint32): uint32 {.inline.} = result = y xor (x or (not z)) -proc rot(x: var int32, n: int8) {.inline.} = - x = toU32(x shl ze(n)) or (x shr toU32(32 -% ze(n))) +proc rot(x: var uint32, n: uint8) {.inline.} = + x = (x shl n) or (x shr (32'u32 - n)) -proc FF(a: var int32, b, c, d, x: int32, s: int8, ac: int32) = - a = a +% F(b, c, d) +% x +% ac +proc FF(a: var uint32, b, c, d, x: uint32, s: uint8, ac: uint32) = + a = a + F(b, c, d) + x + ac rot(a, s) - a = a +% b + a = a + b -proc GG(a: var int32, b, c, d, x: int32, s: int8, ac: int32) = - a = a +% G(b, c, d) +% x +% ac +proc GG(a: var uint32, b, c, d, x: uint32, s: uint8, ac: uint32) = + a = a + G(b, c, d) + x + ac rot(a, s) - a = a +% b + a = a + b -proc HH(a: var int32, b, c, d, x: int32, s: int8, ac: int32) = - a = a +% H(b, c, d) +% x +% ac +proc HH(a: var uint32, b, c, d, x: uint32, s: uint8, ac: uint32) = + a = a + H(b, c, d) + x + ac rot(a, s) - a = a +% b + a = a + b -proc II(a: var int32, b, c, d, x: int32, s: int8, ac: int32) = - a = a +% I(b, c, d) +% x +% ac +proc II(a: var uint32, b, c, d, x: uint32, s: uint8, ac: uint32) = + a = a + I(b, c, d) + x + ac rot(a, s) - a = a +% b + a = a + b -proc encode(dest: var MD5Block, src: cstring) = +proc encode(dest: var MD5Block, src: cstring) = var j = 0 for i in 0..high(dest): - dest[i] = toU32(ord(src[j]) or - ord(src[j+1]) shl 8 or - ord(src[j+2]) shl 16 or - ord(src[j+3]) shl 24) + dest[i] = uint32(ord(src[j])) or + uint32(ord(src[j+1])) shl 8 or + uint32(ord(src[j+2])) shl 16 or + uint32(ord(src[j+3])) shl 24 inc(j, 4) -proc decode(dest: var openArray[int8], src: openArray[int32]) = +proc decode(dest: var openArray[uint8], src: openArray[uint32]) = var i = 0 for j in 0..high(src): - dest[i] = toU8(src[j] and 0xff'i32) - dest[i+1] = toU8(src[j] shr 8'i32 and 0xff'i32) - dest[i+2] = toU8(src[j] shr 16'i32 and 0xff'i32) - dest[i+3] = toU8(src[j] shr 24'i32 and 0xff'i32) + dest[i] = src[j] and 0xff'u32 + dest[i+1] = src[j] shr 8 and 0xff'u32 + dest[i+2] = src[j] shr 16 and 0xff'u32 + dest[i+3] = src[j] shr 24 and 0xff'u32 inc(i, 4) -proc transform(buffer: pointer, state: var MD5State) = +proc transform(buffer: pointer, state: var MD5State) = var myBlock: MD5Block encode(myBlock, cast[cstring](buffer)) @@ -92,111 +94,111 @@ proc transform(buffer: pointer, state: var MD5State) = var b = state[1] var c = state[2] var d = state[3] - FF(a, b, c, d, myBlock[0], 7'i8, 0xD76AA478'i32) - FF(d, a, b, c, myBlock[1], 12'i8, 0xE8C7B756'i32) - FF(c, d, a, b, myBlock[2], 17'i8, 0x242070DB'i32) - FF(b, c, d, a, myBlock[3], 22'i8, 0xC1BDCEEE'i32) - FF(a, b, c, d, myBlock[4], 7'i8, 0xF57C0FAF'i32) - FF(d, a, b, c, myBlock[5], 12'i8, 0x4787C62A'i32) - FF(c, d, a, b, myBlock[6], 17'i8, 0xA8304613'i32) - FF(b, c, d, a, myBlock[7], 22'i8, 0xFD469501'i32) - FF(a, b, c, d, myBlock[8], 7'i8, 0x698098D8'i32) - FF(d, a, b, c, myBlock[9], 12'i8, 0x8B44F7AF'i32) - FF(c, d, a, b, myBlock[10], 17'i8, 0xFFFF5BB1'i32) - FF(b, c, d, a, myBlock[11], 22'i8, 0x895CD7BE'i32) - FF(a, b, c, d, myBlock[12], 7'i8, 0x6B901122'i32) - FF(d, a, b, c, myBlock[13], 12'i8, 0xFD987193'i32) - FF(c, d, a, b, myBlock[14], 17'i8, 0xA679438E'i32) - FF(b, c, d, a, myBlock[15], 22'i8, 0x49B40821'i32) - GG(a, b, c, d, myBlock[1], 5'i8, 0xF61E2562'i32) - GG(d, a, b, c, myBlock[6], 9'i8, 0xC040B340'i32) - GG(c, d, a, b, myBlock[11], 14'i8, 0x265E5A51'i32) - GG(b, c, d, a, myBlock[0], 20'i8, 0xE9B6C7AA'i32) - GG(a, b, c, d, myBlock[5], 5'i8, 0xD62F105D'i32) - GG(d, a, b, c, myBlock[10], 9'i8, 0x02441453'i32) - GG(c, d, a, b, myBlock[15], 14'i8, 0xD8A1E681'i32) - GG(b, c, d, a, myBlock[4], 20'i8, 0xE7D3FBC8'i32) - GG(a, b, c, d, myBlock[9], 5'i8, 0x21E1CDE6'i32) - GG(d, a, b, c, myBlock[14], 9'i8, 0xC33707D6'i32) - GG(c, d, a, b, myBlock[3], 14'i8, 0xF4D50D87'i32) - GG(b, c, d, a, myBlock[8], 20'i8, 0x455A14ED'i32) - GG(a, b, c, d, myBlock[13], 5'i8, 0xA9E3E905'i32) - GG(d, a, b, c, myBlock[2], 9'i8, 0xFCEFA3F8'i32) - GG(c, d, a, b, myBlock[7], 14'i8, 0x676F02D9'i32) - GG(b, c, d, a, myBlock[12], 20'i8, 0x8D2A4C8A'i32) - HH(a, b, c, d, myBlock[5], 4'i8, 0xFFFA3942'i32) - HH(d, a, b, c, myBlock[8], 11'i8, 0x8771F681'i32) - HH(c, d, a, b, myBlock[11], 16'i8, 0x6D9D6122'i32) - HH(b, c, d, a, myBlock[14], 23'i8, 0xFDE5380C'i32) - HH(a, b, c, d, myBlock[1], 4'i8, 0xA4BEEA44'i32) - HH(d, a, b, c, myBlock[4], 11'i8, 0x4BDECFA9'i32) - HH(c, d, a, b, myBlock[7], 16'i8, 0xF6BB4B60'i32) - HH(b, c, d, a, myBlock[10], 23'i8, 0xBEBFBC70'i32) - HH(a, b, c, d, myBlock[13], 4'i8, 0x289B7EC6'i32) - HH(d, a, b, c, myBlock[0], 11'i8, 0xEAA127FA'i32) - HH(c, d, a, b, myBlock[3], 16'i8, 0xD4EF3085'i32) - HH(b, c, d, a, myBlock[6], 23'i8, 0x04881D05'i32) - HH(a, b, c, d, myBlock[9], 4'i8, 0xD9D4D039'i32) - HH(d, a, b, c, myBlock[12], 11'i8, 0xE6DB99E5'i32) - HH(c, d, a, b, myBlock[15], 16'i8, 0x1FA27CF8'i32) - HH(b, c, d, a, myBlock[2], 23'i8, 0xC4AC5665'i32) - II(a, b, c, d, myBlock[0], 6'i8, 0xF4292244'i32) - II(d, a, b, c, myBlock[7], 10'i8, 0x432AFF97'i32) - II(c, d, a, b, myBlock[14], 15'i8, 0xAB9423A7'i32) - II(b, c, d, a, myBlock[5], 21'i8, 0xFC93A039'i32) - II(a, b, c, d, myBlock[12], 6'i8, 0x655B59C3'i32) - II(d, a, b, c, myBlock[3], 10'i8, 0x8F0CCC92'i32) - II(c, d, a, b, myBlock[10], 15'i8, 0xFFEFF47D'i32) - II(b, c, d, a, myBlock[1], 21'i8, 0x85845DD1'i32) - II(a, b, c, d, myBlock[8], 6'i8, 0x6FA87E4F'i32) - II(d, a, b, c, myBlock[15], 10'i8, 0xFE2CE6E0'i32) - II(c, d, a, b, myBlock[6], 15'i8, 0xA3014314'i32) - II(b, c, d, a, myBlock[13], 21'i8, 0x4E0811A1'i32) - II(a, b, c, d, myBlock[4], 6'i8, 0xF7537E82'i32) - II(d, a, b, c, myBlock[11], 10'i8, 0xBD3AF235'i32) - II(c, d, a, b, myBlock[2], 15'i8, 0x2AD7D2BB'i32) - II(b, c, d, a, myBlock[9], 21'i8, 0xEB86D391'i32) - state[0] = state[0] +% a - state[1] = state[1] +% b - state[2] = state[2] +% c - state[3] = state[3] +% d - -proc md5Init*(c: var MD5Context) = - ## initializes a MD5Context - c.state[0] = 0x67452301'i32 - c.state[1] = 0xEFCDAB89'i32 - c.state[2] = 0x98BADCFE'i32 - c.state[3] = 0x10325476'i32 - c.count[0] = 0'i32 - c.count[1] = 0'i32 + FF(a, b, c, d, myBlock[0], 7'u8, 0xD76AA478'u32) + FF(d, a, b, c, myBlock[1], 12'u8, 0xE8C7B756'u32) + FF(c, d, a, b, myBlock[2], 17'u8, 0x242070DB'u32) + FF(b, c, d, a, myBlock[3], 22'u8, 0xC1BDCEEE'u32) + FF(a, b, c, d, myBlock[4], 7'u8, 0xF57C0FAF'u32) + FF(d, a, b, c, myBlock[5], 12'u8, 0x4787C62A'u32) + FF(c, d, a, b, myBlock[6], 17'u8, 0xA8304613'u32) + FF(b, c, d, a, myBlock[7], 22'u8, 0xFD469501'u32) + FF(a, b, c, d, myBlock[8], 7'u8, 0x698098D8'u32) + FF(d, a, b, c, myBlock[9], 12'u8, 0x8B44F7AF'u32) + FF(c, d, a, b, myBlock[10], 17'u8, 0xFFFF5BB1'u32) + FF(b, c, d, a, myBlock[11], 22'u8, 0x895CD7BE'u32) + FF(a, b, c, d, myBlock[12], 7'u8, 0x6B901122'u32) + FF(d, a, b, c, myBlock[13], 12'u8, 0xFD987193'u32) + FF(c, d, a, b, myBlock[14], 17'u8, 0xA679438E'u32) + FF(b, c, d, a, myBlock[15], 22'u8, 0x49B40821'u32) + GG(a, b, c, d, myBlock[1], 5'u8, 0xF61E2562'u32) + GG(d, a, b, c, myBlock[6], 9'u8, 0xC040B340'u32) + GG(c, d, a, b, myBlock[11], 14'u8, 0x265E5A51'u32) + GG(b, c, d, a, myBlock[0], 20'u8, 0xE9B6C7AA'u32) + GG(a, b, c, d, myBlock[5], 5'u8, 0xD62F105D'u32) + GG(d, a, b, c, myBlock[10], 9'u8, 0x02441453'u32) + GG(c, d, a, b, myBlock[15], 14'u8, 0xD8A1E681'u32) + GG(b, c, d, a, myBlock[4], 20'u8, 0xE7D3FBC8'u32) + GG(a, b, c, d, myBlock[9], 5'u8, 0x21E1CDE6'u32) + GG(d, a, b, c, myBlock[14], 9'u8, 0xC33707D6'u32) + GG(c, d, a, b, myBlock[3], 14'u8, 0xF4D50D87'u32) + GG(b, c, d, a, myBlock[8], 20'u8, 0x455A14ED'u32) + GG(a, b, c, d, myBlock[13], 5'u8, 0xA9E3E905'u32) + GG(d, a, b, c, myBlock[2], 9'u8, 0xFCEFA3F8'u32) + GG(c, d, a, b, myBlock[7], 14'u8, 0x676F02D9'u32) + GG(b, c, d, a, myBlock[12], 20'u8, 0x8D2A4C8A'u32) + HH(a, b, c, d, myBlock[5], 4'u8, 0xFFFA3942'u32) + HH(d, a, b, c, myBlock[8], 11'u8, 0x8771F681'u32) + HH(c, d, a, b, myBlock[11], 16'u8, 0x6D9D6122'u32) + HH(b, c, d, a, myBlock[14], 23'u8, 0xFDE5380C'u32) + HH(a, b, c, d, myBlock[1], 4'u8, 0xA4BEEA44'u32) + HH(d, a, b, c, myBlock[4], 11'u8, 0x4BDECFA9'u32) + HH(c, d, a, b, myBlock[7], 16'u8, 0xF6BB4B60'u32) + HH(b, c, d, a, myBlock[10], 23'u8, 0xBEBFBC70'u32) + HH(a, b, c, d, myBlock[13], 4'u8, 0x289B7EC6'u32) + HH(d, a, b, c, myBlock[0], 11'u8, 0xEAA127FA'u32) + HH(c, d, a, b, myBlock[3], 16'u8, 0xD4EF3085'u32) + HH(b, c, d, a, myBlock[6], 23'u8, 0x04881D05'u32) + HH(a, b, c, d, myBlock[9], 4'u8, 0xD9D4D039'u32) + HH(d, a, b, c, myBlock[12], 11'u8, 0xE6DB99E5'u32) + HH(c, d, a, b, myBlock[15], 16'u8, 0x1FA27CF8'u32) + HH(b, c, d, a, myBlock[2], 23'u8, 0xC4AC5665'u32) + II(a, b, c, d, myBlock[0], 6'u8, 0xF4292244'u32) + II(d, a, b, c, myBlock[7], 10'u8, 0x432AFF97'u32) + II(c, d, a, b, myBlock[14], 15'u8, 0xAB9423A7'u32) + II(b, c, d, a, myBlock[5], 21'u8, 0xFC93A039'u32) + II(a, b, c, d, myBlock[12], 6'u8, 0x655B59C3'u32) + II(d, a, b, c, myBlock[3], 10'u8, 0x8F0CCC92'u32) + II(c, d, a, b, myBlock[10], 15'u8, 0xFFEFF47D'u32) + II(b, c, d, a, myBlock[1], 21'u8, 0x85845DD1'u32) + II(a, b, c, d, myBlock[8], 6'u8, 0x6FA87E4F'u32) + II(d, a, b, c, myBlock[15], 10'u8, 0xFE2CE6E0'u32) + II(c, d, a, b, myBlock[6], 15'u8, 0xA3014314'u32) + II(b, c, d, a, myBlock[13], 21'u8, 0x4E0811A1'u32) + II(a, b, c, d, myBlock[4], 6'u8, 0xF7537E82'u32) + II(d, a, b, c, myBlock[11], 10'u8, 0xBD3AF235'u32) + II(c, d, a, b, myBlock[2], 15'u8, 0x2AD7D2BB'u32) + II(b, c, d, a, myBlock[9], 21'u8, 0xEB86D391'u32) + state[0] = state[0] + a + state[1] = state[1] + b + state[2] = state[2] + c + state[3] = state[3] + d + +proc md5Init*(c: var MD5Context) = + ## initializes a MD5Context + c.state[0] = 0x67452301'u32 + c.state[1] = 0xEFCDAB89'u32 + c.state[2] = 0x98BADCFE'u32 + c.state[3] = 0x10325476'u32 + c.count[0] = 0'u32 + c.count[1] = 0'u32 zeroMem(addr(c.buffer), sizeof(MD5buffer)) -proc md5Update*(c: var MD5Context, input: cstring, len: int) = +proc md5Update*(c: var MD5Context, input: cstring, len: int) = ## updates the MD5Context with the `input` data of length `len` var input = input - var Index = (c.count[0] shr 3) and 0x3F - c.count[0] = c.count[0] +% toU32(len shl 3) - if c.count[0] < (len shl 3): c.count[1] = c.count[1] +% 1'i32 - c.count[1] = c.count[1] +% toU32(len shr 29) + var Index = int((c.count[0] shr 3) and 0x3F) + c.count[0] = c.count[0] + (uint32(len) shl 3) + if c.count[0] < (uint32(len) shl 3): c.count[1] = c.count[1] + 1'u32 + c.count[1] = c.count[1] + (uint32(len) shr 29) var PartLen = 64 - Index - if len >= PartLen: + if len >= PartLen: copyMem(addr(c.buffer[Index]), input, PartLen) transform(addr(c.buffer), c.state) var i = PartLen - while i + 63 < len: + while i + 63 < len: transform(addr(input[i]), c.state) inc(i, 64) copyMem(addr(c.buffer[0]), addr(input[i]), len-i) else: copyMem(addr(c.buffer[Index]), addr(input[0]), len) -proc md5Final*(c: var MD5Context, digest: var MD5Digest) = +proc md5Final*(c: var MD5Context, digest: var MD5Digest) = ## finishes the MD5Context and stores the result in `digest` var Bits: MD5CBits PadLen: int decode(Bits, c.count) - var Index = (c.count[0] shr 3) and 0x3F + var Index = int((c.count[0] shr 3) and 0x3F) if Index < 56: PadLen = 56 - Index else: PadLen = 120 - Index md5Update(c, padding, PadLen) @@ -204,34 +206,34 @@ proc md5Final*(c: var MD5Context, digest: var MD5Digest) = decode(digest, c.state) zeroMem(addr(c), sizeof(MD5Context)) -proc toMD5*(s: string): MD5Digest = +proc toMD5*(s: string): MD5Digest = ## computes the MD5Digest value for a string `s` var c: MD5Context md5Init(c) md5Update(c, cstring(s), len(s)) md5Final(c, result) - -proc `$`*(D: MD5Digest): string = + +proc `$`*(d: MD5Digest): string = ## converts a MD5Digest value into its string representation const digits = "0123456789abcdef" result = "" - for i in 0..15: - add(result, digits[(D[i] shr 4) and 0xF]) - add(result, digits[D[i] and 0xF]) + for i in 0..15: + add(result, digits[(d[i] shr 4) and 0xF]) + add(result, digits[d[i] and 0xF]) -proc getMD5*(s: string): string = +proc getMD5*(s: string): string = ## computes an MD5 value of `s` and returns its string representation - var + var c: MD5Context d: MD5Digest md5Init(c) md5Update(c, cstring(s), len(s)) md5Final(c, d) result = $d - -proc `==`*(D1, D2: MD5Digest): bool = + +proc `==`*(D1, D2: MD5Digest): bool = ## checks if two MD5Digest values are identical - for i in 0..15: + for i in 0..15: if D1[i] != D2[i]: return false return true @@ -241,5 +243,3 @@ when isMainModule: assert(getMD5("Frank jagt im komplett verwahrlosten Taxi quer durch Bayern") == "7e716d0e702df0505fc72e2b89467910") assert($toMD5("") == "d41d8cd98f00b204e9800998ecf8427e") - - diff --git a/lib/pure/memfiles.nim b/lib/pure/memfiles.nim index 24dbfb6d3..d49dfae9f 100644 --- a/lib/pure/memfiles.nim +++ b/lib/pure/memfiles.nim @@ -1,7 +1,7 @@ # # # Nim's Runtime Library -# (c) Copyright 2014 Nim Contributors +# (c) Copyright 2015 Nim Contributors # # See the file "copying.txt", included in this # distribution, for details about the copyright. diff --git a/lib/pure/mersenne.nim b/lib/pure/mersenne.nim index a6a781cb8..74112e304 100644 --- a/lib/pure/mersenne.nim +++ b/lib/pure/mersenne.nim @@ -32,7 +32,7 @@ proc getNum*(m: var MersenneTwister): int = return int(y) # Test -when isMainModule: +when not defined(testing) and isMainModule: var mt = newMersenneTwister(2525) for i in 0..99: diff --git a/lib/pure/mimetypes.nim b/lib/pure/mimetypes.nim index a52ba4ebe..642419e64 100644 --- a/lib/pure/mimetypes.nim +++ b/lib/pure/mimetypes.nim @@ -518,5 +518,5 @@ proc register*(mimedb: var MimeDB, ext: string, mimetype: string) = when isMainModule: var m = newMimetypes() - echo m.getMimetype("mp4") - echo m.getExt("text/html") + assert m.getMimetype("mp4") == "video/mp4" + assert m.getExt("text/html") == "html" diff --git a/lib/pure/net.nim b/lib/pure/net.nim index e6fe79740..ffbc6e320 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -1,7 +1,7 @@ # # # Nim's Runtime Library -# (c) Copyright 2014 Dominik Picheta +# (c) Copyright 2015 Dominik Picheta # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -60,7 +60,8 @@ type sslHasPeekChar: bool sslPeekChar: char of false: nil - + lastError: OSErrorCode ## stores the last error on this socket + Socket* = ref SocketImpl SOBool* = enum ## Boolean socket options. @@ -80,6 +81,23 @@ type TReadLineResult: ReadLineResult, TSOBool: SOBool, PSocket: Socket, TSocketImpl: SocketImpl].} +type + IpAddressFamily* {.pure.} = enum ## Describes the type of an IP address + IPv6, ## IPv6 address + IPv4 ## IPv4 address + + TIpAddress* = object ## stores an arbitrary IP address + case family*: IpAddressFamily ## the type of the IP address (IPv4 or IPv6) + of IpAddressFamily.IPv6: + address_v6*: array[0..15, uint8] ## Contains the IP address in bytes in + ## case of IPv6 + of IpAddressFamily.IPv4: + address_v4*: array[0..3, uint8] ## Contains the IP address in bytes in + ## case of IPv4 + +proc isIpAddress*(address_str: string): bool {.tags: [].} +proc parseIpAddress*(address_str: string): TIpAddress + proc isDisconnectionError*(flags: set[SocketFlag], lastError: OSErrorCode): bool = ## Determines whether ``lastError`` is a disconnection error. Only does this @@ -119,7 +137,7 @@ proc newSocket*(domain, typ, protocol: cint, buffered = true): Socket = result = newSocket(fd, buffered) proc newSocket*(domain: Domain = AF_INET, typ: SockType = SOCK_STREAM, - protocol: Protocol = IPPROTO_TCP, buffered = true): Socket = + protocol: Protocol = IPPROTO_TCP, buffered = true): Socket = ## Creates a new socket. ## ## If an error occurs EOS will be raised. @@ -231,6 +249,15 @@ when defined(ssl): if SSLSetFd(socket.sslHandle, socket.fd) != 1: raiseSSLError() +proc getSocketError*(socket: Socket): OSErrorCode = + ## Checks ``osLastError`` for a valid error. If it has been reset it uses + ## the last error stored in the socket object. + result = osLastError() + if result == 0.OSErrorCode: + result = socket.lastError + if result == 0.OSErrorCode: + raise newException(OSError, "No valid socket error code available") + proc socketError*(socket: Socket, err: int = -1, async = false, lastError = (-1).OSErrorCode) = ## Raises an OSError based on the error code returned by ``SSLGetError`` @@ -258,7 +285,7 @@ proc socketError*(socket: Socket, err: int = -1, async = false, of SSL_ERROR_WANT_X509_LOOKUP: raiseSSLError("Function for x509 lookup has been called.") of SSL_ERROR_SYSCALL: - var errStr = "IO error has occured " + var errStr = "IO error has occurred " let sslErr = ErrPeekLastError() if sslErr == 0 and err == 0: errStr.add "because an EOF was observed that violates the protocol" @@ -276,7 +303,7 @@ proc socketError*(socket: Socket, err: int = -1, async = false, else: raiseSSLError("Unknown Error") if err == -1 and not (when defined(ssl): socket.isSSL else: false): - let lastE = if lastError.int == -1: osLastError() else: lastError + var lastE = if lastError.int == -1: getSocketError(socket) else: lastError if async: when useWinVersion: if lastE.int32 == WSAEWOULDBLOCK: @@ -294,7 +321,8 @@ proc listen*(socket: Socket, backlog = SOMAXCONN) {.tags: [ReadIOEffect].} = ## queue of pending connections. ## ## Raises an EOS error upon failure. - if listen(socket.fd, backlog) < 0'i32: raiseOSError(osLastError()) + if rawsockets.listen(socket.fd, backlog) < 0'i32: + raiseOSError(osLastError()) proc bindAddr*(socket: Socket, port = Port(0), address = "") {. tags: [ReadIOEffect].} = @@ -321,7 +349,8 @@ proc bindAddr*(socket: Socket, port = Port(0), address = "") {. dealloc(aiList) proc acceptAddr*(server: Socket, client: var Socket, address: var string, - flags = {SocketFlag.SafeDisconn}) {.tags: [ReadIOEffect].} = + flags = {SocketFlag.SafeDisconn}) {. + tags: [ReadIOEffect], gcsafe, locks: 0.} = ## Blocks until a connection is being made from a client. When a connection ## is made sets ``client`` to the client socket and ``address`` to the address ## of the connecting client. @@ -442,6 +471,8 @@ proc close*(socket: Socket) = # shutdown i.e not wait for the peers "close notify" alert with a second # call to SSLShutdown let res = SSLShutdown(socket.sslHandle) + SSLFree(socket.sslHandle) + socket.sslHandle = nil if res == 0: discard elif res != 1: @@ -497,6 +528,12 @@ proc connect*(socket: Socket, address: string, port = Port(0), when defined(ssl): if socket.isSSL: + # RFC3546 for SNI specifies that IP addresses are not allowed. + if not isIpAddress(address): + # Discard result in case OpenSSL version doesn't support SNI, or we're + # not using TLSv1+ + discard SSL_set_tlsext_host_name(socket.sslHandle, address) + let ret = SSLConnect(socket.sslHandle) socketError(socket, ret) @@ -568,6 +605,10 @@ proc readIntoBuf(socket: Socket, flags: int32): int = result = recv(socket.fd, addr(socket.buffer), cint(socket.buffer.high), flags) else: result = recv(socket.fd, addr(socket.buffer), cint(socket.buffer.high), flags) + if result < 0: + # Save it in case it gets reset (the Nim codegen occassionally may call + # Win API functions which reset it). + socket.lastError = osLastError() if result <= 0: socket.bufLen = 0 socket.currPos = 0 @@ -623,6 +664,9 @@ proc recv*(socket: Socket, data: pointer, size: int): int {.tags: [ReadIOEffect] result = recv(socket.fd, data, size.cint, 0'i32) else: result = recv(socket.fd, data, size.cint, 0'i32) + if result < 0: + # Save the error in case it gets reset. + socket.lastError = osLastError() proc waitFor(socket: Socket, waited: var float, timeout, size: int, funcName: string): int {.tags: [TimeEffect].} = @@ -695,7 +739,7 @@ proc recv*(socket: Socket, data: var string, size: int, timeout = -1, result = recv(socket, cstring(data), size, timeout) if result < 0: data.setLen(0) - let lastError = osLastError() + let lastError = getSocketError(socket) if flags.isDisconnectionError(lastError): return socket.socketError(result, lastError = lastError) data.setLen(result) @@ -743,7 +787,7 @@ proc readLine*(socket: Socket, line: var TaintedString, timeout = -1, line.add("\c\L") template raiseSockError(): stmt {.dirty, immediate.} = - let lastError = osLastError() + let lastError = getSocketError(socket) if flags.isDisconnectionError(lastError): setLen(line.string, 0); return socket.socketError(n, lastError = lastError) @@ -886,7 +930,7 @@ proc connectAsync(socket: Socket, name: string, port = Port(0), af: Domain = AF_INET) {.tags: [ReadIOEffect].} = ## A variant of ``connect`` for non-blocking sockets. ## - ## This procedure will immediatelly return, it will not block until a connection + ## This procedure will immediately return, it will not block until a connection ## is made. It is up to the caller to make sure the connection has been established ## by checking (using ``select``) whether the socket is writeable. ## @@ -938,26 +982,16 @@ proc connect*(socket: Socket, address: string, port = Port(0), timeout: int, doAssert socket.handshake() socket.fd.setBlocking(true) -proc isSsl*(socket: Socket): bool = return socket.isSSL +proc isSsl*(socket: Socket): bool = ## Determines whether ``socket`` is a SSL socket. + when defined(ssl): + result = socket.isSSL + else: + result = false proc getFd*(socket: Socket): SocketHandle = return socket.fd ## Returns the socket's file descriptor -type - IpAddressFamily* {.pure.} = enum ## Describes the type of an IP address - IPv6, ## IPv6 address - IPv4 ## IPv4 address - - TIpAddress* = object ## stores an arbitrary IP address - case family*: IpAddressFamily ## the type of the IP address (IPv4 or IPv6) - of IpAddressFamily.IPv6: - address_v6*: array[0..15, uint8] ## Contains the IP address in bytes in - ## case of IPv6 - of IpAddressFamily.IPv4: - address_v4*: array[0..3, uint8] ## Contains the IP address in bytes in - ## case of IPv4 - proc IPv4_any*(): TIpAddress = ## Returns the IPv4 any address, which can be used to listen on all available ## network adapters @@ -1216,7 +1250,7 @@ proc parseIPv6Address(address_str: string): TIpAddress = raise newException(ValueError, "Invalid IP Address. The address consists of too many groups") -proc parseIpAddress*(address_str: string): TIpAddress = +proc parseIpAddress(address_str: string): TIpAddress = ## Parses an IP address ## Raises EInvalidValue on error if address_str == nil: @@ -1225,3 +1259,13 @@ proc parseIpAddress*(address_str: string): TIpAddress = return parseIPv6Address(address_str) else: return parseIPv4Address(address_str) + + +proc isIpAddress(address_str: string): bool = + ## Checks if a string is an IP address + ## Returns true if it is, false otherwise + try: + discard parseIpAddress(address_str) + except ValueError: + return false + return true diff --git a/lib/pure/nimprof.nim b/lib/pure/nimprof.nim index 3598cdd3a..cce2a20ae 100644 --- a/lib/pure/nimprof.nim +++ b/lib/pure/nimprof.nim @@ -132,7 +132,7 @@ else: proc hook(st: TStackTrace) {.nimcall.} = if interval == 0: hookAux(st, 1) - elif getTicks() - t0 > interval: + elif int64(t0) == 0 or getTicks() - t0 > interval: hookAux(st, 1) t0 = getTicks() diff --git a/lib/pure/nimprof.nimrod.cfg b/lib/pure/nimprof.nim.cfg index 1589e7394..1589e7394 100644 --- a/lib/pure/nimprof.nimrod.cfg +++ b/lib/pure/nimprof.nim.cfg diff --git a/lib/pure/oids.nim b/lib/pure/oids.nim index 0dc8e3c15..ac90dd16b 100644 --- a/lib/pure/oids.nim +++ b/lib/pure/oids.nim @@ -88,6 +88,6 @@ proc generatedTime*(oid: Oid): Time = bigEndian32(addr(tmp), addr(dummy)) result = Time(tmp) -when isMainModule: +when not defined(testing) and isMainModule: let xo = genOid() echo xo.generatedTime diff --git a/lib/pure/os.nim b/lib/pure/os.nim index ad82dc682..f53abe81d 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -1,7 +1,7 @@ # # # Nim's Runtime Library -# (c) Copyright 2014 Andreas Rumpf +# (c) Copyright 2015 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -185,7 +185,7 @@ const proc osErrorMsg*(): string {.rtl, extern: "nos$1", deprecated.} = ## Retrieves the operating system's error flag, ``errno``. ## On Windows ``GetLastError`` is checked before ``errno``. - ## Returns "" if no error occured. + ## Returns "" if no error occurred. ## ## **Deprecated since version 0.9.4**: use the other ``osErrorMsg`` proc. @@ -599,7 +599,7 @@ proc splitPath*(path: string): tuple[head, tail: string] {. proc parentDirPos(path: string): int = var q = 1 - if path[len(path)-1] in {DirSep, AltSep}: q = 2 + if len(path) >= 1 and path[len(path)-1] in {DirSep, AltSep}: q = 2 for i in countdown(len(path)-q, 0): if path[i] in {DirSep, AltSep}: return i result = -1 @@ -1010,8 +1010,16 @@ proc copyFile*(source, dest: string) {.rtl, extern: "nos$1", proc moveFile*(source, dest: string) {.rtl, extern: "nos$1", tags: [ReadIOEffect, WriteIOEffect].} = ## Moves a file from `source` to `dest`. If this fails, `OSError` is raised. - if c_rename(source, dest) != 0'i32: - raise newException(OSError, $strerror(errno)) + when defined(Windows): + when useWinUnicode: + let s = newWideCString(source) + let d = newWideCString(dest) + if moveFileW(s, d, 0'i32) == 0'i32: raiseOSError(osLastError()) + else: + if moveFileA(source, dest, 0'i32) == 0'i32: raiseOSError(osLastError()) + else: + if c_rename(source, dest) != 0'i32: + raise newException(OSError, $strerror(errno)) when not declared(ENOENT) and not defined(Windows): when NoFakeVars: @@ -1074,8 +1082,12 @@ when defined(windows): # because we support Windows GUI applications, things get really # messy here... when useWinUnicode: - proc strEnd(cstr: WideCString, c = 0'i32): WideCString {. - importc: "wcschr", header: "<string.h>".} + when defined(cpp): + proc strEnd(cstr: WideCString, c = 0'i32): WideCString {. + importcpp: "(NI16*)wcschr((const wchar_t *)#, #)", header: "<string.h>".} + else: + proc strEnd(cstr: WideCString, c = 0'i32): WideCString {. + importc: "wcschr", header: "<string.h>".} else: proc strEnd(cstr: cstring, c = 0'i32): cstring {. importc: "strchr", header: "<string.h>".} @@ -1087,7 +1099,7 @@ when defined(windows): var env = getEnvironmentStringsW() e = env - if e == nil: return # an error occured + if e == nil: return # an error occurred while true: var eend = strEnd(e) add(environment, $e) @@ -1098,7 +1110,7 @@ when defined(windows): var env = getEnvironmentStringsA() e = env - if e == nil: return # an error occured + if e == nil: return # an error occurred while true: var eend = strEnd(e) add(environment, $e) @@ -1109,8 +1121,8 @@ when defined(windows): else: const - useNSGetEnviron = defined(macosx) and - (defined(createNimRtl) or defined(useNimRtl)) + useNSGetEnviron = defined(macosx) + when useNSGetEnviron: # From the manual: # Shared libraries and bundles don't have direct access to environ, @@ -1170,7 +1182,7 @@ proc putEnv*(key, val: string) {.tags: [WriteEnvEffect].} = ## If an error occurs, `EInvalidEnvVar` is raised. # Note: by storing the string in the environment sequence, - # we gurantee that we don't free the memory before the program + # we guarantee that we don't free the memory before the program # ends (this is needed for POSIX compliance). It is also needed so that # the process itself may access its modified environment variables! var indx = findEnvVar(key) @@ -1287,8 +1299,16 @@ iterator walkDir*(dir: string): tuple[kind: PathComponent, path: string] {. if y != "." and y != "..": var s: TStat y = dir / y - if lstat(y, s) < 0'i32: break var k = pcFile + + when defined(linux) or defined(macosx) or defined(bsd): + if x.d_type != DT_UNKNOWN: + if x.d_type == DT_DIR: k = pcDir + if x.d_type == DT_LNK: k = succ(k) + yield (k, y) + continue + + if lstat(y, s) < 0'i32: break if S_ISDIR(s.st_mode): k = pcDir if S_ISLNK(s.st_mode): k = succ(k) yield (k, y) @@ -1335,7 +1355,7 @@ proc rawRemoveDir(dir: string) = if rmdir(dir) != 0'i32 and errno != ENOENT: raiseOSError(osLastError()) proc removeDir*(dir: string) {.rtl, extern: "nos$1", tags: [ - WriteDirEffect, ReadDirEffect].} = + WriteDirEffect, ReadDirEffect], benign.} = ## Removes the directory `dir` including all subdirectories and files ## in `dir` (recursively). ## @@ -1381,7 +1401,7 @@ proc createDir*(dir: string) {.rtl, extern: "nos$1", tags: [WriteDirEffect].} = rawCreateDir(dir) proc copyDir*(source, dest: string) {.rtl, extern: "nos$1", - tags: [WriteIOEffect, ReadIOEffect].} = + tags: [WriteIOEffect, ReadIOEffect], benign.} = ## Copies a directory from `source` to `dest`. ## ## If this fails, `OSError` is raised. On the Windows platform this proc will @@ -1442,7 +1462,7 @@ proc createHardlink*(src, dest: string) = proc parseCmdLine*(c: string): seq[string] {. noSideEffect, rtl, extern: "nos$1".} = ## Splits a command line into several components; - ## This proc is only occassionally useful, better use the `parseopt` module. + ## This proc is only occasionally useful, better use the `parseopt` module. ## ## On Windows, it uses the following parsing rules ## (see http://msdn.microsoft.com/en-us/library/17w5ykft.aspx ): @@ -1554,7 +1574,7 @@ proc copyFileWithPermissions*(source, dest: string, proc copyDirWithPermissions*(source, dest: string, ignorePermissionErrors = true) {.rtl, extern: "nos$1", - tags: [WriteIOEffect, ReadIOEffect].} = + tags: [WriteIOEffect, ReadIOEffect], benign.} = ## Copies a directory from `source` to `dest` preserving file permissions. ## ## If this fails, `OSError` is raised. This is a wrapper proc around `copyDir() @@ -1843,16 +1863,21 @@ proc getFileSize*(file: string): BiggestInt {.rtl, extern: "nos$1", close(f) else: raiseOSError(osLastError()) +proc expandTilde*(path: string): string {.tags: [ReadEnvEffect].} + proc findExe*(exe: string): string {.tags: [ReadDirEffect, ReadEnvEffect].} = ## Searches for `exe` in the current working directory and then ## in directories listed in the ``PATH`` environment variable. ## Returns "" if the `exe` cannot be found. On DOS-like platforms, `exe` - ## is added an ``.exe`` file extension if it has no extension. + ## is added the `ExeExt <#ExeExt>`_ file extension if it has none. result = addFileExt(exe, os.ExeExt) if existsFile(result): return var path = string(os.getEnv("PATH")) for candidate in split(path, PathSep): - var x = candidate / result + when defined(windows): + var x = candidate / result + else: + var x = expandTilde(candidate) / result if existsFile(x): return x result = "" @@ -1977,6 +2002,8 @@ proc getFileInfo*(handle: FileHandle): FileInfo = rawToFormalFileInfo(rawInfo, result) proc getFileInfo*(file: File): FileInfo = + if file.isNil: + raise newException(IOError, "File is nil") result = getFileInfo(file.getFileHandle()) proc getFileInfo*(path: string, followSymlink = true): FileInfo = diff --git a/lib/pure/osproc.nim b/lib/pure/osproc.nim index bfdb0efed..dce0673ba 100644 --- a/lib/pure/osproc.nim +++ b/lib/pure/osproc.nim @@ -1,7 +1,7 @@ # # # Nim's Runtime Library -# (c) Copyright 2014 Andreas Rumpf +# (c) Copyright 2015 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -137,7 +137,7 @@ proc startProcess*(command: string, ## `env` is the environment that will be passed to the process. ## If ``env == nil`` the environment is inherited of ## the parent process. `options` are additional flags that may be passed - ## to `startProcess`. See the documentation of ``TProcessOption`` for the + ## to `startProcess`. See the documentation of ``ProcessOption`` for the ## meaning of these flags. You need to `close` the process when done. ## ## Note that you can't pass any `args` if you use the option @@ -146,7 +146,7 @@ proc startProcess*(command: string, ## of `args` to `command` carefully escaping/quoting any special characters, ## since it will be passed *as is* to the system shell. Each system/shell may ## feature different escaping rules, so try to avoid this kind of shell - ## invokation if possible as it leads to non portable software. + ## invocation if possible as it leads to non portable software. ## ## Return value: The newly created process object. Nil is never returned, ## but ``EOS`` is raised in case of an error. @@ -174,7 +174,7 @@ proc terminate*(p: Process) {.rtl, extern: "nosp$1", tags: [].} proc kill*(p: Process) {.rtl, extern: "nosp$1", tags: [].} ## Kill the process `p`. On Posix OSes the procedure sends ``SIGKILL`` to ## the process. On Windows ``kill()`` is simply an alias for ``terminate()``. - + proc running*(p: Process): bool {.rtl, extern: "nosp$1", tags: [].} ## Returns true iff the process `p` is still running. Returns immediately. @@ -260,7 +260,7 @@ proc execProcesses*(cmds: openArray[string], for i in 0..m-1: if beforeRunEvent != nil: beforeRunEvent(i) - q[i] = startCmd(cmds[i], options=options) + q[i] = startProcess(cmds[i], options=options + {poEvalCommand}) when defined(noBusyWaiting): var r = 0 for i in m..high(cmds): @@ -275,7 +275,7 @@ proc execProcesses*(cmds: openArray[string], if q[r] != nil: close(q[r]) if beforeRunEvent != nil: beforeRunEvent(i) - q[r] = startCmd(cmds[i], options=options) + q[r] = startProcess(cmds[i], options=options + {poEvalCommand}) r = (r + 1) mod n else: var i = m @@ -288,7 +288,7 @@ proc execProcesses*(cmds: openArray[string], if q[r] != nil: close(q[r]) if beforeRunEvent != nil: beforeRunEvent(i) - q[r] = startCmd(cmds[i], options=options) + q[r] = startProcess(cmds[i], options=options + {poEvalCommand}) inc(i) if i > high(cmds): break for j in 0..m-1: @@ -298,7 +298,7 @@ proc execProcesses*(cmds: openArray[string], for i in 0..high(cmds): if beforeRunEvent != nil: beforeRunEvent(i) - var p = startCmd(cmds[i], options=options) + var p = startProcess(cmds[i], options=options + {poEvalCommand}) result = max(waitForExit(p), result) close(p) @@ -644,14 +644,14 @@ elif not defined(useNimRtl): var pid: TPid var sysArgs = allocCStringArray(sysArgsRaw) - finally: deallocCStringArray(sysArgs) + defer: deallocCStringArray(sysArgs) var sysEnv = if env == nil: envToCStringArray() else: envToCStringArray(env) - finally: deallocCStringArray(sysEnv) + defer: deallocCStringArray(sysEnv) var data: TStartProcessData data.sysCommand = sysCommand @@ -666,7 +666,7 @@ elif not defined(useNimRtl): data.workingDir = workingDir - when declared(posix_spawn) and not defined(useFork) and + when declared(posix_spawn) and not defined(useFork) and not defined(useClone) and not defined(linux): pid = startProcessAuxSpawn(data) else: @@ -748,7 +748,7 @@ elif not defined(useNimRtl): if pipe(data.pErrorPipe) != 0: raiseOSError(osLastError()) - finally: + defer: discard close(data.pErrorPipe[readIdx]) var pid: TPid @@ -823,7 +823,7 @@ elif not defined(useNimRtl): discard execvp(data.sysCommand, data.sysArgs) else: when defined(uClibc): - # uClibc environment (OpenWrt included) doesn't have the full execvpe + # uClibc environment (OpenWrt included) doesn't have the full execvpe discard execve(data.sysCommand, data.sysArgs, data.sysEnv) else: discard execvpe(data.sysCommand, data.sysArgs, data.sysEnv) @@ -848,7 +848,14 @@ elif not defined(useNimRtl): if kill(p.id, SIGCONT) != 0'i32: raiseOsError(osLastError()) proc running(p: Process): bool = - var ret = waitpid(p.id, p.exitCode, WNOHANG) + var ret : int + when not defined(freebsd): + ret = waitpid(p.id, p.exitCode, WNOHANG) + else: + var status : cint = 1 + ret = waitpid(p.id, status, WNOHANG) + if WIFEXITED(status): + p.exitCode = status if ret == 0: return true # Can't establish status. Assume running. result = ret == int(p.id) @@ -857,9 +864,9 @@ elif not defined(useNimRtl): raiseOsError(osLastError()) proc kill(p: Process) = - if kill(p.id, SIGKILL) != 0'i32: + if kill(p.id, SIGKILL) != 0'i32: raiseOsError(osLastError()) - + proc waitForExit(p: Process, timeout: int = -1): int = #if waitPid(p.id, p.exitCode, 0) == int(p.id): # ``waitPid`` fails if the process is not running anymore. But then @@ -876,7 +883,7 @@ elif not defined(useNimRtl): var ret = waitpid(p.id, p.exitCode, WNOHANG) var b = ret == int(p.id) if b: result = -1 - if p.exitCode == -3: result = -1 + if not WIFEXITED(p.exitCode): result = -1 else: result = p.exitCode.int shr 8 proc createStream(stream: var Stream, handle: var FileHandle, @@ -900,7 +907,7 @@ elif not defined(useNimRtl): createStream(p.errStream, p.errHandle, fmRead) return p.errStream - proc csystem(cmd: cstring): cint {.nodecl, importc: "system", + proc csystem(cmd: cstring): cint {.nodecl, importc: "system", header: "<stdlib.h>".} proc execCmd(command: string): int = @@ -949,7 +956,7 @@ proc execCmdEx*(command: string, options: set[ProcessOption] = { exitCode: int] {.tags: [ExecIOEffect, ReadIOEffect], gcsafe.} = ## a convenience proc that runs the `command`, grabs all its output and ## exit code and returns both. - var p = startCmd(command, options) + var p = startProcess(command, options=options + {poEvalCommand}) var outp = outputStream(p) result = (TaintedString"", -1) var line = newStringOfCap(120).TaintedString diff --git a/lib/pure/parsecfg.nim b/lib/pure/parsecfg.nim index bb9d2aed2..bb64c8134 100644 --- a/lib/pure/parsecfg.nim +++ b/lib/pure/parsecfg.nim @@ -35,7 +35,7 @@ type cfgSectionStart, ## a ``[section]`` has been parsed cfgKeyValuePair, ## a ``key=value`` pair has been detected cfgOption, ## a ``--key=value`` command line option - cfgError ## an error ocurred during parsing + cfgError ## an error occurred during parsing CfgEvent* = object of RootObj ## describes a parsing event case kind*: CfgEventKind ## the kind of the event diff --git a/lib/pure/parsecsv.nim b/lib/pure/parsecsv.nim index f4943ed89..117d75cfa 100644 --- a/lib/pure/parsecsv.nim +++ b/lib/pure/parsecsv.nim @@ -166,7 +166,7 @@ proc close*(my: var CsvParser) {.inline.} = ## closes the parser `my` and its associated input stream. lexbase.close(my) -when isMainModule: +when not defined(testing) and isMainModule: import os var s = newFileStream(paramStr(1), fmRead) if s == nil: quit("cannot open the file" & paramStr(1)) diff --git a/lib/pure/parseopt.nim b/lib/pure/parseopt.nim index 8af6920c1..4c92a7cdf 100644 --- a/lib/pure/parseopt.nim +++ b/lib/pure/parseopt.nim @@ -1,7 +1,7 @@ # # # Nim's Runtime Library -# (c) Copyright 2014 Andreas Rumpf +# (c) Copyright 2015 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -11,9 +11,12 @@ ## It supports one convenience iterator over all command line options and some ## lower-level features. ## -## **Deprecated since version 0.9.3:** Use the `parseopt2 <parseopt2.html>`_ -## module instead as this version has issues with spaces in arguments. -{.deprecated.} +## Supported syntax: +## +## 1. short options - ``-abcd``, where a, b, c, d are names +## 2. long option - ``--foo:bar``, ``--foo=bar`` or ``--foo`` +## 3. argument - everything else + {.push debugger: off.} include "system/inclrtl" diff --git a/lib/pure/parseopt2.nim b/lib/pure/parseopt2.nim index 5b1f50958..73b498fe0 100644 --- a/lib/pure/parseopt2.nim +++ b/lib/pure/parseopt2.nim @@ -1,7 +1,7 @@ # # # Nim's Runtime Library -# (c) Copyright 2014 Andreas Rumpf +# (c) Copyright 2015 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -60,7 +60,7 @@ proc initOptParser*(cmdline: string): OptParser {.rtl, deprecated.} = ## Initalizes option parses with cmdline. Splits cmdline in on spaces ## and calls initOptParser(openarray[string]) ## Do not use. - if cmdline == "": # backward compatibilty + if cmdline == "": # backward compatibility return initOptParser(seq[string](nil)) else: return initOptParser(cmdline.split) diff --git a/lib/pure/parsesql.nim b/lib/pure/parsesql.nim index bb4ede779..91917b1c5 100644 --- a/lib/pure/parsesql.nim +++ b/lib/pure/parsesql.nim @@ -1330,7 +1330,7 @@ proc renderSQL*(n: SqlNode): string = result = "" ra(n, result, 0) -when isMainModule: +when not defined(testing) and isMainModule: echo(renderSQL(parseSQL(newStringStream(""" CREATE TYPE happiness AS ENUM ('happy', 'very happy', 'ecstatic'); CREATE TABLE holidays ( diff --git a/lib/pure/parseurl.nim b/lib/pure/parseurl.nim index f27cd8c12..56bf10768 100644 --- a/lib/pure/parseurl.nim +++ b/lib/pure/parseurl.nim @@ -1,7 +1,7 @@ # # # Nim's Runtime Library -# (c) Copyright 2014 Dominik Picheta +# (c) Copyright 2015 Dominik Picheta # # See the file "copying.txt", included in this # distribution, for details about the copyright. diff --git a/lib/pure/parseutils.nim b/lib/pure/parseutils.nim index 1efb141fc..c07b713de 100644 --- a/lib/pure/parseutils.nim +++ b/lib/pure/parseutils.nim @@ -181,7 +181,7 @@ proc parseWhile*(s: string, token: var string, validChars: set[char], token = substr(s, start, i-1) proc captureBetween*(s: string, first: char, second = '\0', start = 0): string = - ## Finds the first occurence of ``first``, then returns everything from there + ## Finds the first occurrence of ``first``, then returns everything from there ## up to ``second``(if ``second`` is '\0', then ``first`` is used). var i = skipUntil(s, first, start)+1+start result = "" @@ -228,7 +228,7 @@ proc parseInt*(s: string, number: var int, start = 0): int {. if (sizeof(int) <= 4) and ((res < low(int)) or (res > high(int))): raise newException(OverflowError, "overflow") - else: + elif result != 0: number = int(res) proc parseBiggestFloat*(s: string, number: var BiggestFloat, start = 0): int {. @@ -240,11 +240,12 @@ proc parseBiggestFloat*(s: string, number: var BiggestFloat, start = 0): int {. proc parseFloat*(s: string, number: var float, start = 0): int {. rtl, extern: "npuParseFloat", noSideEffect.} = ## parses a float starting at `start` and stores the value into `number`. - ## Result is the number of processed chars or 0 if there occured a parsing + ## Result is the number of processed chars or 0 if there occurred a parsing ## error. var bf: BiggestFloat result = parseBiggestFloat(s, bf, start) - number = bf + if result != 0: + number = bf type InterpolatedKind* = enum ## describes for `interpolatedFragments` @@ -294,7 +295,7 @@ iterator interpolatedFragments*(s: string): tuple[kind: InterpolatedKind, dec nesting of '\0': raise newException(ValueError, - "Expected closing '}': " & s[i..s.len]) + "Expected closing '}': " & substr(s, i, s.high)) else: discard inc j inc i, 2 # skip ${ @@ -310,7 +311,7 @@ iterator interpolatedFragments*(s: string): tuple[kind: InterpolatedKind, kind = ikDollar else: raise newException(ValueError, - "Unable to parse a varible name at " & s[i..s.len]) + "Unable to parse a varible name at " & substr(s, i, s.high)) else: while j < s.len and s[j] != '$': inc j kind = ikStr @@ -322,8 +323,12 @@ iterator interpolatedFragments*(s: string): tuple[kind: InterpolatedKind, i = j when isMainModule: - for k, v in interpolatedFragments("$test{} $this is ${an{ example}} "): - echo "(", k, ", \"", v, "\")" + import sequtils + let input = "$test{} $this is ${an{ example}} " + let expected = @[(ikVar, "test"), (ikStr, "{} "), (ikVar, "this"), + (ikStr, " is "), (ikExpr, "an{ example}"), (ikStr, " ")] + assert toSeq(interpolatedFragments(input)) == expected + var value = 0 discard parseHex("0x38", value) assert value == 56 diff --git a/lib/pure/parsexml.nim b/lib/pure/parsexml.nim index 39dead3c0..eb792f086 100644 --- a/lib/pure/parsexml.nim +++ b/lib/pure/parsexml.nim @@ -57,7 +57,7 @@ import type XmlEventKind* = enum ## enumation of all events that may occur when parsing - xmlError, ## an error ocurred during parsing + xmlError, ## an error occurred during parsing xmlEof, ## end of file reached xmlCharData, ## character data xmlWhitespace, ## whitespace has been parsed @@ -128,6 +128,7 @@ proc open*(my: var XmlParser, input: Stream, filename: string, my.kind = xmlError my.a = "" my.b = "" + my.c = nil my.options = options proc close*(my: var XmlParser) {.inline.} = @@ -138,43 +139,43 @@ proc kind*(my: XmlParser): XmlEventKind {.inline.} = ## returns the current event type for the XML parser return my.kind -proc charData*(my: XmlParser): string {.inline.} = +template charData*(my: XmlParser): string = ## returns the character data for the events: ``xmlCharData``, ## ``xmlWhitespace``, ``xmlComment``, ``xmlCData``, ``xmlSpecial`` assert(my.kind in {xmlCharData, xmlWhitespace, xmlComment, xmlCData, xmlSpecial}) - return my.a + my.a -proc elementName*(my: XmlParser): string {.inline.} = +template elementName*(my: XmlParser): string = ## returns the element name for the events: ``xmlElementStart``, ## ``xmlElementEnd``, ``xmlElementOpen`` assert(my.kind in {xmlElementStart, xmlElementEnd, xmlElementOpen}) - return my.a + my.a -proc entityName*(my: XmlParser): string {.inline.} = +template entityName*(my: XmlParser): string = ## returns the entity name for the event: ``xmlEntity`` assert(my.kind == xmlEntity) - return my.a + my.a -proc attrKey*(my: XmlParser): string {.inline.} = +template attrKey*(my: XmlParser): string = ## returns the attribute key for the event ``xmlAttribute`` assert(my.kind == xmlAttribute) - return my.a + my.a -proc attrValue*(my: XmlParser): string {.inline.} = +template attrValue*(my: XmlParser): string = ## returns the attribute value for the event ``xmlAttribute`` assert(my.kind == xmlAttribute) - return my.b + my.b -proc piName*(my: XmlParser): string {.inline.} = +template piName*(my: XmlParser): string = ## returns the processing instruction name for the event ``xmlPI`` assert(my.kind == xmlPI) - return my.a + my.a -proc piRest*(my: XmlParser): string {.inline.} = +template piRest*(my: XmlParser): string = ## returns the rest of the processing instruction for the event ``xmlPI`` assert(my.kind == xmlPI) - return my.b + my.b proc rawData*(my: XmlParser): string {.inline.} = ## returns the underlying 'data' string by reference. @@ -621,13 +622,13 @@ proc next*(my: var XmlParser) = of stateEmptyElementTag: my.state = stateNormal my.kind = xmlElementEnd - if not isNil(my.c): + if not my.c.isNil: my.a = my.c of stateError: my.kind = xmlError my.state = stateNormal -when isMainModule: +when not defined(testing) and isMainModule: import os var s = newFileStream(paramStr(1), fmRead) if s == nil: quit("cannot open the file" & paramStr(1)) diff --git a/lib/pure/pegs.nim b/lib/pure/pegs.nim index 7cef0a00d..39f0bfa95 100644 --- a/lib/pure/pegs.nim +++ b/lib/pure/pegs.nim @@ -29,7 +29,7 @@ when useUnicode: const InlineThreshold = 5 ## number of leaves; -1 to disable inlining MaxSubpatterns* = 20 ## defines the maximum number of subpatterns that - ## can be captured. More subpatterns cannot be captured! + ## can be captured. More subpatterns cannot be captured! type PegKind = enum @@ -85,14 +85,14 @@ type of pkBackRef..pkBackRefIgnoreStyle: index: range[0..MaxSubpatterns] else: sons: seq[TNode] NonTerminal* = ref NonTerminalObj - + Peg* = TNode ## type that represents a PEG {.deprecated: [TPeg: Peg].} proc term*(t: string): Peg {.nosideEffect, rtl, extern: "npegs$1Str".} = ## constructs a PEG from a terminal string - if t.len != 1: + if t.len != 1: result.kind = pkTerminal result.term = t else: @@ -116,7 +116,7 @@ proc term*(t: char): Peg {.nosideEffect, rtl, extern: "npegs$1Char".} = assert t != '\0' result.kind = pkChar result.ch = t - + proc charSet*(s: set[char]): Peg {.nosideEffect, rtl, extern: "npegs$1".} = ## constructs a PEG from a character set `s` assert '\0' notin s @@ -129,12 +129,12 @@ proc add(d: var Peg, s: Peg) {.inline.} = add(d.sons, s) proc addChoice(dest: var Peg, elem: Peg) = var L = dest.len-1 - if L >= 0 and dest.sons[L].kind == pkCharChoice: + if L >= 0 and dest.sons[L].kind == pkCharChoice: # caution! Do not introduce false aliasing here! case elem.kind of pkCharChoice: dest.sons[L] = charSet(dest.sons[L].charChoice[] + elem.charChoice[]) - of pkChar: + of pkChar: dest.sons[L] = charSet(dest.sons[L].charChoice[] + {elem.ch}) else: add(dest, elem) else: add(dest, elem) @@ -158,12 +158,12 @@ proc `/`*(a: varargs[Peg]): Peg {. proc addSequence(dest: var Peg, elem: Peg) = var L = dest.len-1 - if L >= 0 and dest.sons[L].kind == pkTerminal: + if L >= 0 and dest.sons[L].kind == pkTerminal: # caution! Do not introduce false aliasing here! case elem.kind - of pkTerminal: + of pkTerminal: dest.sons[L] = term(dest.sons[L].term & elem.term) - of pkChar: + of pkChar: dest.sons[L] = term(dest.sons[L].term & elem.ch) else: add(dest, elem) else: add(dest, elem) @@ -172,7 +172,7 @@ proc sequence*(a: varargs[Peg]): Peg {. nosideEffect, rtl, extern: "npegs$1".} = ## constructs a sequence with all the PEGs from `a` multipleOp(pkSequence, addSequence) - + proc `?`*(a: Peg): Peg {.nosideEffect, rtl, extern: "npegsOptional".} = ## constructs an optional for the PEG `a` if a.kind in {pkOption, pkGreedyRep, pkGreedyAny, pkGreedyRepChar, @@ -207,7 +207,7 @@ proc `!*`*(a: Peg): Peg {.nosideEffect, rtl, extern: "npegsSearch".} = result.kind = pkSearch result.sons = @[a] -proc `!*\`*(a: Peg): Peg {.noSideEffect, rtl, +proc `!*\`*(a: Peg): Peg {.noSideEffect, rtl, extern: "npgegsCapturedSearch".} = ## constructs a "captured search" for the PEG `a` result.kind = pkCapturedSearch @@ -216,7 +216,7 @@ proc `!*\`*(a: Peg): Peg {.noSideEffect, rtl, proc `+`*(a: Peg): Peg {.nosideEffect, rtl, extern: "npegsGreedyPosRep".} = ## constructs a "greedy positive repetition" with the PEG `a` return sequence(a, *a) - + proc `&`*(a: Peg): Peg {.nosideEffect, rtl, extern: "npegsAndPredicate".} = ## constructs an "and predicate" with the PEG `a` result.kind = pkAndPredicate @@ -239,33 +239,33 @@ proc newLine*: Peg {.inline.} = ## constructs the PEG `newline`:idx: (``\n``) result.kind = pkNewLine -proc unicodeLetter*: Peg {.inline.} = +proc unicodeLetter*: Peg {.inline.} = ## constructs the PEG ``\letter`` which matches any Unicode letter. result.kind = pkLetter - -proc unicodeLower*: Peg {.inline.} = + +proc unicodeLower*: Peg {.inline.} = ## constructs the PEG ``\lower`` which matches any Unicode lowercase letter. - result.kind = pkLower + result.kind = pkLower -proc unicodeUpper*: Peg {.inline.} = +proc unicodeUpper*: Peg {.inline.} = ## constructs the PEG ``\upper`` which matches any Unicode uppercase letter. result.kind = pkUpper - -proc unicodeTitle*: Peg {.inline.} = + +proc unicodeTitle*: Peg {.inline.} = ## constructs the PEG ``\title`` which matches any Unicode title letter. result.kind = pkTitle -proc unicodeWhitespace*: Peg {.inline.} = - ## constructs the PEG ``\white`` which matches any Unicode +proc unicodeWhitespace*: Peg {.inline.} = + ## constructs the PEG ``\white`` which matches any Unicode ## whitespace character. result.kind = pkWhitespace -proc startAnchor*: Peg {.inline.} = - ## constructs the PEG ``^`` which matches the start of the input. +proc startAnchor*: Peg {.inline.} = + ## constructs the PEG ``^`` which matches the start of the input. result.kind = pkStartAnchor -proc endAnchor*: Peg {.inline.} = - ## constructs the PEG ``$`` which matches the end of the input. +proc endAnchor*: Peg {.inline.} = + ## constructs the PEG ``$`` which matches the end of the input. result = !any() proc capture*(a: Peg): Peg {.nosideEffect, rtl, extern: "npegsCapture".} = @@ -274,21 +274,21 @@ proc capture*(a: Peg): Peg {.nosideEffect, rtl, extern: "npegsCapture".} = result.sons = @[a] proc backref*(index: range[1..MaxSubpatterns]): Peg {. - nosideEffect, rtl, extern: "npegs$1".} = + nosideEffect, rtl, extern: "npegs$1".} = ## constructs a back reference of the given `index`. `index` starts counting ## from 1. result.kind = pkBackRef result.index = index-1 proc backrefIgnoreCase*(index: range[1..MaxSubpatterns]): Peg {. - nosideEffect, rtl, extern: "npegs$1".} = + nosideEffect, rtl, extern: "npegs$1".} = ## constructs a back reference of the given `index`. `index` starts counting ## from 1. Ignores case for matching. result.kind = pkBackRefIgnoreCase result.index = index-1 proc backrefIgnoreStyle*(index: range[1..MaxSubpatterns]): Peg {. - nosideEffect, rtl, extern: "npegs$1".}= + nosideEffect, rtl, extern: "npegs$1".}= ## constructs a back reference of the given `index`. `index` starts counting ## from 1. Ignores style for matching. result.kind = pkBackRefIgnoreStyle @@ -298,7 +298,7 @@ proc spaceCost(n: Peg): int = case n.kind of pkEmpty: discard of pkTerminal, pkTerminalIgnoreCase, pkTerminalIgnoreStyle, pkChar, - pkGreedyRepChar, pkCharChoice, pkGreedyRepSet, + pkGreedyRepChar, pkCharChoice, pkGreedyRepSet, pkAny..pkWhitespace, pkGreedyAny: result = 1 of pkNonTerminal: @@ -310,7 +310,7 @@ proc spaceCost(n: Peg): int = if result >= InlineThreshold: break proc nonterminal*(n: NonTerminal): Peg {. - nosideEffect, rtl, extern: "npegs$1".} = + nosideEffect, rtl, extern: "npegs$1".} = ## constructs a PEG that consists of the nonterminal symbol assert n != nil if ntDeclared in n.flags and spaceCost(n.rule) < InlineThreshold: @@ -331,7 +331,7 @@ proc newNonTerminal*(name: string, line, column: int): NonTerminal {. template letters*: expr = ## expands to ``charset({'A'..'Z', 'a'..'z'})`` charSet({'A'..'Z', 'a'..'z'}) - + template digits*: expr = ## expands to ``charset({'0'..'9'})`` charSet({'0'..'9'}) @@ -339,11 +339,11 @@ template digits*: expr = template whitespace*: expr = ## expands to ``charset({' ', '\9'..'\13'})`` charSet({' ', '\9'..'\13'}) - + template identChars*: expr = ## expands to ``charset({'a'..'z', 'A'..'Z', '0'..'9', '_'})`` charSet({'a'..'z', 'A'..'Z', '0'..'9', '_'}) - + template identStartChars*: expr = ## expands to ``charset({'A'..'Z', 'a'..'z', '_'})`` charSet({'a'..'z', 'A'..'Z', '_'}) @@ -352,14 +352,14 @@ template ident*: expr = ## same as ``[a-zA-Z_][a-zA-z_0-9]*``; standard identifier sequence(charSet({'a'..'z', 'A'..'Z', '_'}), *charSet({'a'..'z', 'A'..'Z', '0'..'9', '_'})) - + template natural*: expr = ## same as ``\d+`` +digits # ------------------------- debugging ----------------------------------------- -proc esc(c: char, reserved = {'\0'..'\255'}): string = +proc esc(c: char, reserved = {'\0'..'\255'}): string = case c of '\b': result = "\\b" of '\t': result = "\\t" @@ -374,38 +374,38 @@ proc esc(c: char, reserved = {'\0'..'\255'}): string = elif c < ' ' or c >= '\128': result = '\\' & $ord(c) elif c in reserved: result = '\\' & c else: result = $c - + proc singleQuoteEsc(c: char): string = return "'" & esc(c, {'\''}) & "'" -proc singleQuoteEsc(str: string): string = +proc singleQuoteEsc(str: string): string = result = "'" for c in items(str): add result, esc(c, {'\''}) add result, '\'' - -proc charSetEscAux(cc: set[char]): string = + +proc charSetEscAux(cc: set[char]): string = const reserved = {'^', '-', ']'} result = "" var c1 = 0 - while c1 <= 0xff: - if chr(c1) in cc: + while c1 <= 0xff: + if chr(c1) in cc: var c2 = c1 while c2 < 0xff and chr(succ(c2)) in cc: inc(c2) - if c1 == c2: + if c1 == c2: add result, esc(chr(c1), reserved) - elif c2 == succ(c1): + elif c2 == succ(c1): add result, esc(chr(c1), reserved) & esc(chr(c2), reserved) - else: + else: add result, esc(chr(c1), reserved) & '-' & esc(chr(c2), reserved) c1 = c2 inc(c1) - + proc charSetEsc(cc: set[char]): string = - if card(cc) >= 128+64: + if card(cc) >= 128+64: result = "[^" & charSetEscAux({'\1'..'\xFF'} - cc) & ']' - else: + else: result = '[' & charSetEscAux(cc) & ']' - -proc toStrAux(r: Peg, res: var string) = + +proc toStrAux(r: Peg, res: var string) = case r.kind of pkEmpty: add(res, "()") of pkAny: add(res, '.') @@ -469,25 +469,25 @@ proc toStrAux(r: Peg, res: var string) = toStrAux(r.sons[0], res) of pkCapture: add(res, '{') - toStrAux(r.sons[0], res) + toStrAux(r.sons[0], res) add(res, '}') - of pkBackRef: + of pkBackRef: add(res, '$') add(res, $r.index) - of pkBackRefIgnoreCase: + of pkBackRefIgnoreCase: add(res, "i$") add(res, $r.index) - of pkBackRefIgnoreStyle: + of pkBackRefIgnoreStyle: add(res, "y$") add(res, $r.index) of pkRule: - toStrAux(r.sons[0], res) + toStrAux(r.sons[0], res) add(res, " <- ") toStrAux(r.sons[1], res) of pkList: for i in 0 .. high(r.sons): toStrAux(r.sons[i], res) - add(res, "\n") + add(res, "\n") of pkStartAnchor: add(res, '^') @@ -506,14 +506,14 @@ type {.deprecated: [TCaptures: Captures].} -proc bounds*(c: Captures, - i: range[0..MaxSubpatterns-1]): tuple[first, last: int] = +proc bounds*(c: Captures, + i: range[0..MaxSubpatterns-1]): tuple[first, last: int] = ## returns the bounds ``[first..last]`` of the `i`'th capture. result = c.matches[i] when not useUnicode: type - TRune = char + Rune = char template fastRuneAt(s, i, ch: expr) = ch = s[i] inc(i) @@ -527,7 +527,7 @@ when not useUnicode: proc rawMatch*(s: string, p: Peg, start: int, c: var Captures): int {. nosideEffect, rtl, extern: "npegs$1".} = - ## low-level matching proc that implements the PEG interpreter. Use this + ## low-level matching proc that implements the PEG interpreter. Use this ## for maximum efficiency (every other PEG operation ends up calling this ## proc). ## Returns -1 if it does not match, else the length of the match @@ -541,7 +541,7 @@ proc rawMatch*(s: string, p: Peg, start: int, c: var Captures): int {. result = runeLenAt(s, start) else: result = -1 - of pkLetter: + of pkLetter: if s[start] != '\0': var a: Rune result = start @@ -550,7 +550,7 @@ proc rawMatch*(s: string, p: Peg, start: int, c: var Captures): int {. else: result = -1 else: result = -1 - of pkLower: + of pkLower: if s[start] != '\0': var a: Rune result = start @@ -559,7 +559,7 @@ proc rawMatch*(s: string, p: Peg, start: int, c: var Captures): int {. else: result = -1 else: result = -1 - of pkUpper: + of pkUpper: if s[start] != '\0': var a: Rune result = start @@ -568,16 +568,16 @@ proc rawMatch*(s: string, p: Peg, start: int, c: var Captures): int {. else: result = -1 else: result = -1 - of pkTitle: + of pkTitle: if s[start] != '\0': var a: Rune result = start fastRuneAt(s, result, a) - if isTitle(a): dec(result, start) + if isTitle(a): dec(result, start) else: result = -1 else: result = -1 - of pkWhitespace: + of pkWhitespace: if s[start] != '\0': var a: Rune result = start @@ -641,7 +641,7 @@ proc rawMatch*(s: string, p: Peg, start: int, c: var Captures): int {. when false: echo "leave: ", p.nt.name if result < 0: c.ml = oldMl of pkSequence: - var oldMl = c.ml + var oldMl = c.ml result = 0 for i in 0..high(p.sons): var x = rawMatch(s, p.sons[i], start+result, c) @@ -723,11 +723,11 @@ proc rawMatch*(s: string, p: Peg, start: int, c: var Captures): int {. #else: silently ignore the capture else: c.ml = idx - of pkBackRef..pkBackRefIgnoreStyle: + of pkBackRef..pkBackRefIgnoreStyle: if p.index >= c.ml: return -1 var (a, b) = c.matches[p.index] var n: Peg - n.kind = succ(pkTerminal, ord(p.kind)-ord(pkBackRef)) + n.kind = succ(pkTerminal, ord(p.kind)-ord(pkBackRef)) n.term = s.substr(a, b) result = rawMatch(s, n, start, c) of pkStartAnchor: @@ -744,24 +744,6 @@ template fillMatches(s, caps, c: expr) = else: caps[k] = nil -proc match*(s: string, pattern: Peg, matches: var openArray[string], - start = 0): bool {.nosideEffect, rtl, extern: "npegs$1Capture".} = - ## returns ``true`` if ``s[start..]`` matches the ``pattern`` and - ## the captured substrings in the array ``matches``. If it does not - ## match, nothing is written into ``matches`` and ``false`` is - ## returned. - var c: Captures - c.origStart = start - result = rawMatch(s, pattern, start, c) == len(s) - start - if result: fillMatches(s, matches, c) - -proc match*(s: string, pattern: Peg, - start = 0): bool {.nosideEffect, rtl, extern: "npegs$1".} = - ## returns ``true`` if ``s`` matches the ``pattern`` beginning from ``start``. - var c: Captures - c.origStart = start - result = rawMatch(s, pattern, start, c) == len(s)-start - proc matchLen*(s: string, pattern: Peg, matches: var openArray[string], start = 0): int {.nosideEffect, rtl, extern: "npegs$1Capture".} = ## the same as ``match``, but it returns the length of the match, @@ -773,7 +755,7 @@ proc matchLen*(s: string, pattern: Peg, matches: var openArray[string], result = rawMatch(s, pattern, start, c) if result >= 0: fillMatches(s, matches, c) -proc matchLen*(s: string, pattern: Peg, +proc matchLen*(s: string, pattern: Peg, start = 0): int {.nosideEffect, rtl, extern: "npegs$1".} = ## the same as ``match``, but it returns the length of the match, ## if there is no match, -1 is returned. Note that a match length @@ -783,6 +765,20 @@ proc matchLen*(s: string, pattern: Peg, c.origStart = start result = rawMatch(s, pattern, start, c) +proc match*(s: string, pattern: Peg, matches: var openArray[string], + start = 0): bool {.nosideEffect, rtl, extern: "npegs$1Capture".} = + ## returns ``true`` if ``s[start..]`` matches the ``pattern`` and + ## the captured substrings in the array ``matches``. If it does not + ## match, nothing is written into ``matches`` and ``false`` is + ## returned. + result = matchLen(s, pattern, matches, start) != -1 + +proc match*(s: string, pattern: Peg, + start = 0): bool {.nosideEffect, rtl, extern: "npegs$1".} = + ## returns ``true`` if ``s`` matches the ``pattern`` beginning from ``start``. + result = matchLen(s, pattern, start) != -1 + + proc find*(s: string, pattern: Peg, matches: var openArray[string], start = 0): int {.nosideEffect, rtl, extern: "npegs$1Capture".} = ## returns the starting position of ``pattern`` in ``s`` and the captured @@ -797,11 +793,11 @@ proc find*(s: string, pattern: Peg, matches: var openArray[string], return i return -1 # could also use the pattern here: (!P .)* P - + proc findBounds*(s: string, pattern: Peg, matches: var openArray[string], start = 0): tuple[first, last: int] {. nosideEffect, rtl, extern: "npegs$1Capture".} = - ## returns the starting position and end position of ``pattern`` in ``s`` + ## returns the starting position and end position of ``pattern`` in ``s`` ## and the captured ## substrings in the array ``matches``. If it does not match, nothing ## is written into ``matches`` and (-1,0) is returned. @@ -814,8 +810,8 @@ proc findBounds*(s: string, pattern: Peg, matches: var openArray[string], fillMatches(s, matches, c) return (i, i+L-1) return (-1, 0) - -proc find*(s: string, pattern: Peg, + +proc find*(s: string, pattern: Peg, start = 0): int {.nosideEffect, rtl, extern: "npegs$1".} = ## returns the starting position of ``pattern`` in ``s``. If it does not ## match, -1 is returned. @@ -824,8 +820,8 @@ proc find*(s: string, pattern: Peg, for i in start .. s.len-1: if rawMatch(s, pattern, i, c) >= 0: return i return -1 - -iterator findAll*(s: string, pattern: Peg, start = 0): string = + +iterator findAll*(s: string, pattern: Peg, start = 0): string = ## yields all matching *substrings* of `s` that match `pattern`. var c: Captures c.origStart = start @@ -838,23 +834,23 @@ iterator findAll*(s: string, pattern: Peg, start = 0): string = else: yield substr(s, i, i+L-1) inc(i, L) - + proc findAll*(s: string, pattern: Peg, start = 0): seq[string] {. - nosideEffect, rtl, extern: "npegs$1".} = + nosideEffect, rtl, extern: "npegs$1".} = ## returns all matching *substrings* of `s` that match `pattern`. ## If it does not match, @[] is returned. accumulateResult(findAll(s, pattern, start)) when not defined(nimhygiene): {.pragma: inject.} - + template `=~`*(s: string, pattern: Peg): bool = - ## This calls ``match`` with an implicit declared ``matches`` array that - ## can be used in the scope of the ``=~`` call: - ## + ## This calls ``match`` with an implicit declared ``matches`` array that + ## can be used in the scope of the ``=~`` call: + ## ## .. code-block:: nim ## - ## if line =~ peg"\s* {\w+} \s* '=' \s* {\w+}": + ## if line =~ peg"\s* {\w+} \s* '=' \s* {\w+}": ## # matches a key=value pair: ## echo("Key: ", matches[0]) ## echo("Value: ", matches[1]) @@ -865,7 +861,7 @@ template `=~`*(s: string, pattern: Peg): bool = ## echo("comment: ", matches[0]) ## else: ## echo("syntax error") - ## + ## bind MaxSubpatterns when not declaredInScope(matches): var matches {.inject.}: array[0..MaxSubpatterns-1, string] @@ -902,7 +898,7 @@ proc replacef*(s: string, sub: Peg, by: string): string {. ## with the notation ``$i`` and ``$#`` (see strutils.`%`). Examples: ## ## .. code-block:: nim - ## "var1=key; var2=key2".replace(peg"{\ident}'='{\ident}", "$1<-$2$2") + ## "var1=key; var2=key2".replacef(peg"{\ident}'='{\ident}", "$1<-$2$2") ## ## Results in: ## @@ -941,10 +937,10 @@ proc replace*(s: string, sub: Peg, by = ""): string {. add(result, by) inc(i, x) add(result, substr(s, i)) - + proc parallelReplace*(s: string, subs: varargs[ tuple[pattern: Peg, repl: string]]): string {. - nosideEffect, rtl, extern: "npegs$1".} = + nosideEffect, rtl, extern: "npegs$1".} = ## Returns a modified copy of `s` with the substitutions in `subs` ## applied in parallel. result = "" @@ -964,8 +960,8 @@ proc parallelReplace*(s: string, subs: varargs[ add(result, s[i]) inc(i) # copy the rest: - add(result, substr(s, i)) - + add(result, substr(s, i)) + proc transformFile*(infile, outfile: string, subs: varargs[tuple[pattern: Peg, repl: string]]) {. rtl, extern: "npegs$1".} = @@ -974,7 +970,7 @@ proc transformFile*(infile, outfile: string, ## error occurs. This is supposed to be used for quick scripting. var x = readFile(infile).string writeFile(outfile, x.parallelReplace(subs)) - + iterator split*(s: string, sep: Peg): string = ## Splits the string `s` into substrings. ## @@ -1049,14 +1045,14 @@ type tkBackref, ## '$' tkDollar, ## '$' tkHat ## '^' - + TToken {.final.} = object ## a token kind: TTokKind ## the type of the token modifier: TModifier literal: string ## the parsed (string) literal charset: set[char] ## if kind == tkCharSet index: int ## if kind == tkBackref - + PegLexer {.inheritable.} = object ## the lexer object. bufpos: int ## the current position within the buffer buf: cstring ## the buffer itself @@ -1086,7 +1082,7 @@ proc handleLF(L: var PegLexer, pos: int): int = result = pos+1 L.lineStart = result -proc init(L: var PegLexer, input, filename: string, line = 1, col = 0) = +proc init(L: var PegLexer, input, filename: string, line = 1, col = 0) = L.buf = input L.bufpos = 0 L.lineNumber = line @@ -1094,69 +1090,69 @@ proc init(L: var PegLexer, input, filename: string, line = 1, col = 0) = L.lineStart = 0 L.filename = filename -proc getColumn(L: PegLexer): int {.inline.} = +proc getColumn(L: PegLexer): int {.inline.} = result = abs(L.bufpos - L.lineStart) + L.colOffset -proc getLine(L: PegLexer): int {.inline.} = +proc getLine(L: PegLexer): int {.inline.} = result = L.lineNumber - + proc errorStr(L: PegLexer, msg: string, line = -1, col = -1): string = var line = if line < 0: getLine(L) else: line var col = if col < 0: getColumn(L) else: col result = "$1($2, $3) Error: $4" % [L.filename, $line, $col, msg] -proc handleHexChar(c: var PegLexer, xi: var int) = +proc handleHexChar(c: var PegLexer, xi: var int) = case c.buf[c.bufpos] - of '0'..'9': + of '0'..'9': xi = (xi shl 4) or (ord(c.buf[c.bufpos]) - ord('0')) inc(c.bufpos) - of 'a'..'f': + of 'a'..'f': xi = (xi shl 4) or (ord(c.buf[c.bufpos]) - ord('a') + 10) inc(c.bufpos) - of 'A'..'F': + of 'A'..'F': xi = (xi shl 4) or (ord(c.buf[c.bufpos]) - ord('A') + 10) inc(c.bufpos) else: discard -proc getEscapedChar(c: var PegLexer, tok: var TToken) = +proc getEscapedChar(c: var PegLexer, tok: var TToken) = inc(c.bufpos) case c.buf[c.bufpos] - of 'r', 'R', 'c', 'C': + of 'r', 'R', 'c', 'C': add(tok.literal, '\c') inc(c.bufpos) - of 'l', 'L': + of 'l', 'L': add(tok.literal, '\L') inc(c.bufpos) - of 'f', 'F': + of 'f', 'F': add(tok.literal, '\f') inc(c.bufpos) - of 'e', 'E': + of 'e', 'E': add(tok.literal, '\e') inc(c.bufpos) - of 'a', 'A': + of 'a', 'A': add(tok.literal, '\a') inc(c.bufpos) - of 'b', 'B': + of 'b', 'B': add(tok.literal, '\b') inc(c.bufpos) - of 'v', 'V': + of 'v', 'V': add(tok.literal, '\v') inc(c.bufpos) - of 't', 'T': + of 't', 'T': add(tok.literal, '\t') inc(c.bufpos) - of 'x', 'X': + of 'x', 'X': inc(c.bufpos) var xi = 0 handleHexChar(c, xi) handleHexChar(c, xi) if xi == 0: tok.kind = tkInvalid else: add(tok.literal, chr(xi)) - of '0'..'9': + of '0'..'9': var val = ord(c.buf[c.bufpos]) - ord('0') inc(c.bufpos) var i = 1 - while (i <= 3) and (c.buf[c.bufpos] in {'0'..'9'}): + while (i <= 3) and (c.buf[c.bufpos] in {'0'..'9'}): val = val * 10 + ord(c.buf[c.bufpos]) - ord('0') inc(c.bufpos) inc(i) @@ -1169,32 +1165,32 @@ proc getEscapedChar(c: var PegLexer, tok: var TToken) = else: add(tok.literal, c.buf[c.bufpos]) inc(c.bufpos) - -proc skip(c: var PegLexer) = + +proc skip(c: var PegLexer) = var pos = c.bufpos var buf = c.buf - while true: + while true: case buf[pos] - of ' ', '\t': + of ' ', '\t': inc(pos) of '#': while not (buf[pos] in {'\c', '\L', '\0'}): inc(pos) of '\c': pos = handleCR(c, pos) buf = c.buf - of '\L': + of '\L': pos = handleLF(c, pos) buf = c.buf - else: + else: break # EndOfFile also leaves the loop c.bufpos = pos - -proc getString(c: var PegLexer, tok: var TToken) = + +proc getString(c: var PegLexer, tok: var TToken) = tok.kind = tkStringLit var pos = c.bufpos + 1 var buf = c.buf var quote = buf[pos-1] - while true: + while true: case buf[pos] of '\\': c.bufpos = pos @@ -1205,13 +1201,13 @@ proc getString(c: var PegLexer, tok: var TToken) = break elif buf[pos] == quote: inc(pos) - break + break else: add(tok.literal, buf[pos]) inc(pos) c.bufpos = pos - -proc getDollar(c: var PegLexer, tok: var TToken) = + +proc getDollar(c: var PegLexer, tok: var TToken) = var pos = c.bufpos + 1 var buf = c.buf if buf[pos] in {'0'..'9'}: @@ -1223,8 +1219,8 @@ proc getDollar(c: var PegLexer, tok: var TToken) = else: tok.kind = tkDollar c.bufpos = pos - -proc getCharSet(c: var PegLexer, tok: var TToken) = + +proc getCharSet(c: var PegLexer, tok: var TToken) = tok.kind = tkCharSet tok.charset = {} var pos = c.bufpos + 1 @@ -1247,7 +1243,7 @@ proc getCharSet(c: var PegLexer, tok: var TToken) = of '\C', '\L', '\0': tok.kind = tkInvalid break - else: + else: ch = buf[pos] inc(pos) incl(tok.charset, ch) @@ -1267,18 +1263,18 @@ proc getCharSet(c: var PegLexer, tok: var TToken) = of '\C', '\L', '\0': tok.kind = tkInvalid break - else: + else: ch2 = buf[pos] inc(pos) for i in ord(ch)+1 .. ord(ch2): incl(tok.charset, chr(i)) c.bufpos = pos if caret: tok.charset = {'\1'..'\xFF'} - tok.charset - -proc getSymbol(c: var PegLexer, tok: var TToken) = + +proc getSymbol(c: var PegLexer, tok: var TToken) = var pos = c.bufpos var buf = c.buf - while true: + while true: add(tok.literal, buf[pos]) inc(pos) if buf[pos] notin strutils.IdentChars: break @@ -1294,7 +1290,7 @@ proc getBuiltin(c: var PegLexer, tok: var TToken) = tok.kind = tkEscaped getEscapedChar(c, tok) # may set tok.kind to tkInvalid -proc getTok(c: var PegLexer, tok: var TToken) = +proc getTok(c: var PegLexer, tok: var TToken) = tok.kind = tkInvalid tok.modifier = modNone setLen(tok.literal, 0) @@ -1309,11 +1305,11 @@ proc getTok(c: var PegLexer, tok: var TToken) = else: tok.kind = tkCurlyLe add(tok.literal, '{') - of '}': + of '}': tok.kind = tkCurlyRi inc(c.bufpos) add(tok.literal, '}') - of '[': + of '[': getCharSet(c, tok) of '(': tok.kind = tkParLe @@ -1323,7 +1319,7 @@ proc getTok(c: var PegLexer, tok: var TToken) = tok.kind = tkParRi inc(c.bufpos) add(tok.literal, ')') - of '.': + of '.': tok.kind = tkAny inc(c.bufpos) add(tok.literal, '.') @@ -1331,16 +1327,16 @@ proc getTok(c: var PegLexer, tok: var TToken) = tok.kind = tkAnyRune inc(c.bufpos) add(tok.literal, '_') - of '\\': + of '\\': getBuiltin(c, tok) of '\'', '"': getString(c, tok) of '$': getDollar(c, tok) - of '\0': + of '\0': tok.kind = tkEof tok.literal = "[EOF]" of 'a'..'z', 'A'..'Z', '\128'..'\255': getSymbol(c, tok) - if c.buf[c.bufpos] in {'\'', '"'} or + if c.buf[c.bufpos] in {'\'', '"'} or c.buf[c.bufpos] == '$' and c.buf[c.bufpos+1] in {'0'..'9'}: case tok.literal of "i": tok.modifier = modIgnoreCase @@ -1388,7 +1384,7 @@ proc getTok(c: var PegLexer, tok: var TToken) = tok.kind = tkAt inc(c.bufpos) add(tok.literal, '@') - if c.buf[c.bufpos] == '@': + if c.buf[c.bufpos] == '@': tok.kind = tkCurlyAt inc(c.bufpos) add(tok.literal, '@') @@ -1407,7 +1403,7 @@ proc arrowIsNextTok(c: PegLexer): bool = result = c.buf[pos] == '<' and c.buf[pos+1] == '-' # ----------------------------- parser ---------------------------------------- - + type EInvalidPeg* = object of ValueError ## raised if an invalid ## PEG has been detected @@ -1425,7 +1421,7 @@ proc pegError(p: PegParser, msg: string, line = -1, col = -1) = e.msg = errorStr(p, msg, line, col) raise e -proc getTok(p: var PegParser) = +proc getTok(p: var PegParser) = getTok(p, p.tok) if p.tok.kind == tkInvalid: pegError(p, "invalid token") @@ -1475,7 +1471,7 @@ proc builtin(p: var PegParser): Peg = of "white": result = unicodeWhitespace() else: pegError(p, "unknown built-in: " & p.tok.literal) -proc token(terminal: Peg, p: PegParser): Peg = +proc token(terminal: Peg, p: PegParser): Peg = if p.skip.kind == pkEmpty: result = terminal else: result = sequence(p.skip, terminal) @@ -1496,7 +1492,7 @@ proc primary(p: var PegParser): Peg = else: discard case p.tok.kind of tkIdentifier: - if p.identIsVerbatim: + if p.identIsVerbatim: var m = p.tok.modifier if m == modNone: m = p.modifier result = modifiedTerm(p.tok.literal, m).token(p) @@ -1539,17 +1535,17 @@ proc primary(p: var PegParser): Peg = of tkEscaped: result = term(p.tok.literal[0]).token(p) getTok(p) - of tkDollar: + of tkDollar: result = endAnchor() getTok(p) - of tkHat: + of tkHat: result = startAnchor() getTok(p) of tkBackref: var m = p.tok.modifier if m == modNone: m = p.modifier result = modifiedBackref(p.tok.index, m).token(p) - if p.tok.index < 0 or p.tok.index > p.captures: + if p.tok.index < 0 or p.tok.index > p.captures: pegError(p, "invalid back reference index: " & $p.tok.index) getTok(p) else: @@ -1573,7 +1569,7 @@ proc seqExpr(p: var PegParser): Peg = while true: case p.tok.kind of tkAmp, tkNot, tkAt, tkStringLit, tkCharSet, tkParLe, tkCurlyLe, - tkAny, tkAnyRune, tkBuiltin, tkEscaped, tkDollar, tkBackref, + tkAny, tkAnyRune, tkBuiltin, tkEscaped, tkDollar, tkBackref, tkHat, tkCurlyAt: result = sequence(result, primary(p)) of tkIdentifier: @@ -1587,7 +1583,7 @@ proc parseExpr(p: var PegParser): Peg = while p.tok.kind == tkBar: getTok(p) result = result / seqExpr(p) - + proc parseRule(p: var PegParser): NonTerminal = if p.tok.kind == tkIdentifier and arrowIsNextTok(p): result = getNonTerminal(p, p.tok.literal) @@ -1601,7 +1597,7 @@ proc parseRule(p: var PegParser): NonTerminal = incl(result.flags, ntDeclared) # NOW inlining may be attempted else: pegError(p, "rule expected, but found: " & p.tok.literal) - + proc rawParse(p: var PegParser): Peg = ## parses a rule or a PEG expression while p.tok.kind == tkBuiltin: @@ -1680,27 +1676,29 @@ when isMainModule: assert(not match("W_HI_L", peg"\y 'while'")) assert(not match("W_HI_Le", peg"\y v'while'")) assert match("W_HI_Le", peg"y'while'") - + assert($ +digits == $peg"\d+") assert "0158787".match(peg"\d+") assert "ABC 0232".match(peg"\w+\s+\d+") assert "ABC".match(peg"\d+ / \w+") + var accum: seq[string] = @[] for word in split("00232this02939is39an22example111", peg"\d+"): - writeln(stdout, word) + accum.add(word) + assert(accum == @["this", "is", "an", "example"]) assert matchLen("key", ident) == 3 var pattern = sequence(ident, *whitespace, term('='), *whitespace, ident) assert matchLen("key1= cal9", pattern) == 11 - + var ws = newNonTerminal("ws", 1, 1) ws.rule = *whitespace - + var expr = newNonTerminal("expr", 1, 1) expr.rule = sequence(capture(ident), *sequence( nonterminal(ws), term('+'), nonterminal(ws), nonterminal(expr))) - + var c: Captures var s = "a+b + c +d+e+f" assert rawMatch(s, expr.rule, 0, c) == len(s) @@ -1722,7 +1720,7 @@ when isMainModule: assert matches[0] == "abc" else: assert false - + var g2 = peg"""S <- A B / C D A <- 'a'+ B <- 'b'+ @@ -1748,18 +1746,20 @@ when isMainModule: else: assert false + accum = @[] for x in findAll("abcdef", peg".", 3): - echo x + accum.add(x) + assert(accum == @["d", "e", "f"]) for x in findAll("abcdef", peg"^{.}", 3): assert x == "d" - + if "f(a, b)" =~ peg"{[0-9]+} / ({\ident} '(' {@} ')')": assert matches[0] == "f" assert matches[1] == "a, b" else: assert false - + assert match("eine übersicht und außerdem", peg"(\letter \white*)+") # ß is not a lower cased letter?! assert match("eine übersicht und auerdem", peg"(\lower \white*)+") @@ -1783,3 +1783,9 @@ when isMainModule: if "foo" =~ peg"{'foo'}": assert matches[0] == "foo" else: assert false + + let empty_test = peg"^\d*" + let str = "XYZ" + + assert(str.find(empty_test) == 0) + assert(str.match(empty_test)) diff --git a/lib/pure/pegs.nimfix b/lib/pure/pegs.nimfix deleted file mode 100644 index 8dd172a48..000000000 --- a/lib/pure/pegs.nimfix +++ /dev/null @@ -1,1770 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2012 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## Simple PEG (Parsing expression grammar) matching. Uses no memorization, but -## uses superoperators and symbol inlining to improve performance. Note: -## Matching performance is hopefully competitive with optimized regular -## expression engines. -## -## .. include:: ../doc/pegdocs.txt -## - -include "system/inclrtl" - -const - useUnicode = true ## change this to deactivate proper UTF-8 support - -import - strutils - -when useUnicode: - import unicode - -const - InlineThreshold = 5 ## number of leaves; -1 to disable inlining - MaxSubpatterns* = 10 ## defines the maximum number of subpatterns that - ## can be captured. More subpatterns cannot be captured! - -type - PegKind = enum - pkEmpty, - pkAny, ## any character (.) - pkAnyRune, ## any Unicode character (_) - pkNewLine, ## CR-LF, LF, CR - pkLetter, ## Unicode letter - pkLower, ## Unicode lower case letter - pkUpper, ## Unicode upper case letter - pkTitle, ## Unicode title character - pkWhitespace, ## Unicode whitespace character - pkTerminal, - pkTerminalIgnoreCase, - pkTerminalIgnoreStyle, - pkChar, ## single character to match - pkCharChoice, - pkNonTerminal, - pkSequence, ## a b c ... --> Internal DSL: peg(a, b, c) - pkOrderedChoice, ## a / b / ... --> Internal DSL: a / b or /[a, b, c] - pkGreedyRep, ## a* --> Internal DSL: *a - ## a+ --> (a a*) - pkGreedyRepChar, ## x* where x is a single character (superop) - pkGreedyRepSet, ## [set]* (superop) - pkGreedyAny, ## .* or _* (superop) - pkOption, ## a? --> Internal DSL: ?a - pkAndPredicate, ## &a --> Internal DSL: &a - pkNotPredicate, ## !a --> Internal DSL: !a - pkCapture, ## {a} --> Internal DSL: capture(a) - pkBackRef, ## $i --> Internal DSL: backref(i) - pkBackRefIgnoreCase, - pkBackRefIgnoreStyle, - pkSearch, ## @a --> Internal DSL: !*a - pkCapturedSearch, ## {@} a --> Internal DSL: !*\a - pkRule, ## a <- b - pkList, ## a, b - pkStartAnchor ## ^ --> Internal DSL: startAnchor() - NonTerminalFlag = enum - ntDeclared, ntUsed - NonTerminalObj = object ## represents a non terminal symbol - name: string ## the name of the symbol - line: int ## line the symbol has been declared/used in - col: int ## column the symbol has been declared/used in - flags: set[NonTerminalFlag] ## the nonterminal's flags - rule: TNode ## the rule that the symbol refers to - TNode {.shallow.} = object - case kind: PegKind - of pkEmpty..pkWhitespace: nil - of pkTerminal, pkTerminalIgnoreCase, pkTerminalIgnoreStyle: term: string - of pkChar, pkGreedyRepChar: ch: char - of pkCharChoice, pkGreedyRepSet: charChoice: ref set[char] - of pkNonTerminal: nt: PNonTerminal - of pkBackRef..pkBackRefIgnoreStyle: index: range[0..MaxSubpatterns] - else: sons: seq[TNode] - PNonTerminal* = ref NonTerminalObj - TPeg* = TNode - -block: - type - Peg = TNode - NonTerminal = PNonTerminal - {.deprecated: [TPeg: Peg, PNonTerminal: NonTerminal].} - -proc term*(t: string): TPeg {.nosideEffect, rtl, extern: "npegs$1Str".} = - ## constructs a PEG from a terminal string - if t.len != 1: - result.kind = pkTerminal - result.term = t - else: - result.kind = pkChar - result.ch = t[0] - -proc termIgnoreCase*(t: string): TPeg {.nosideEffect, rtl, extern: "npegs$1".} = - ## constructs a PEG from a terminal string; ignore case for matching - result.kind = pkTerminalIgnoreCase - result.term = t - -proc termIgnoreStyle*(t: string): TPeg {.nosideEffect, rtl, extern: "npegs$1".} = - ## constructs a PEG from a terminal string; ignore style for matching - result.kind = pkTerminalIgnoreStyle - result.term = t - -proc term*(t: char): TPeg {.nosideEffect, rtl, extern: "npegs$1Char".} = - ## constructs a PEG from a terminal char - assert t != '\0' - result.kind = pkChar - result.ch = t - -proc charSet*(s: set[char]): TPeg {.nosideEffect, rtl, extern: "npegs$1".} = - ## constructs a PEG from a character set `s` - assert '\0' notin s - result.kind = pkCharChoice - new(result.charChoice) - result.charChoice[] = s - -proc len(a: TPeg): int {.inline.} = return a.sons.len -proc add(d: var TPeg, s: TPeg) {.inline.} = add(d.sons, s) - -proc addChoice(dest: var TPeg, elem: TPeg) = - var L = dest.len-1 - if L >= 0 and dest.sons[L].kind == pkCharChoice: - # caution! Do not introduce false aliasing here! - case elem.kind - of pkCharChoice: - dest.sons[L] = charSet(dest.sons[L].charChoice[] + elem.charChoice[]) - of pkChar: - dest.sons[L] = charSet(dest.sons[L].charChoice[] + {elem.ch}) - else: add(dest, elem) - else: add(dest, elem) - -template multipleOp(k: PegKind, localOpt: expr) = - result.kind = k - result.sons = @[] - for x in items(a): - if x.kind == k: - for y in items(x.sons): - localOpt(result, y) - else: - localOpt(result, x) - if result.len == 1: - result = result.sons[0] - -proc `/`*(a: varargs[TPeg]): TPeg {. - nosideEffect, rtl, extern: "npegsOrderedChoice".} = - ## constructs an ordered choice with the PEGs in `a` - multipleOp(pkOrderedChoice, addChoice) - -proc addSequence(dest: var TPeg, elem: TPeg) = - var L = dest.len-1 - if L >= 0 and dest.sons[L].kind == pkTerminal: - # caution! Do not introduce false aliasing here! - case elem.kind - of pkTerminal: - dest.sons[L] = term(dest.sons[L].term & elem.term) - of pkChar: - dest.sons[L] = term(dest.sons[L].term & elem.ch) - else: add(dest, elem) - else: add(dest, elem) - -proc sequence*(a: varargs[TPeg]): TPeg {. - nosideEffect, rtl, extern: "npegs$1".} = - ## constructs a sequence with all the PEGs from `a` - multipleOp(pkSequence, addSequence) - -proc `?`*(a: TPeg): TPeg {.nosideEffect, rtl, extern: "npegsOptional".} = - ## constructs an optional for the PEG `a` - if a.kind in {pkOption, pkGreedyRep, pkGreedyAny, pkGreedyRepChar, - pkGreedyRepSet}: - # a* ? --> a* - # a? ? --> a? - result = a - else: - result.kind = pkOption - result.sons = @[a] - -proc `*`*(a: TPeg): TPeg {.nosideEffect, rtl, extern: "npegsGreedyRep".} = - ## constructs a "greedy repetition" for the PEG `a` - case a.kind - of pkGreedyRep, pkGreedyRepChar, pkGreedyRepSet, pkGreedyAny, pkOption: - assert false - # produces endless loop! - of pkChar: - result.kind = pkGreedyRepChar - result.ch = a.ch - of pkCharChoice: - result.kind = pkGreedyRepSet - result.charChoice = a.charChoice # copying a reference suffices! - of pkAny, pkAnyRune: - result.kind = pkGreedyAny - else: - result.kind = pkGreedyRep - result.sons = @[a] - -proc `!*`*(a: TPeg): TPeg {.nosideEffect, rtl, extern: "npegsSearch".} = - ## constructs a "search" for the PEG `a` - result.kind = pkSearch - result.sons = @[a] - -proc `!*\`*(a: TPeg): TPeg {.noSideEffect, rtl, - extern: "npgegsCapturedSearch".} = - ## constructs a "captured search" for the PEG `a` - result.kind = pkCapturedSearch - result.sons = @[a] - -proc `+`*(a: TPeg): TPeg {.nosideEffect, rtl, extern: "npegsGreedyPosRep".} = - ## constructs a "greedy positive repetition" with the PEG `a` - return sequence(a, *a) - -proc `&`*(a: TPeg): TPeg {.nosideEffect, rtl, extern: "npegsAndPredicate".} = - ## constructs an "and predicate" with the PEG `a` - result.kind = pkAndPredicate - result.sons = @[a] - -proc `!`*(a: TPeg): TPeg {.nosideEffect, rtl, extern: "npegsNotPredicate".} = - ## constructs a "not predicate" with the PEG `a` - result.kind = pkNotPredicate - result.sons = @[a] - -proc any*: TPeg {.inline.} = - ## constructs the PEG `any character`:idx: (``.``) - result.kind = pkAny - -proc anyRune*: TPeg {.inline.} = - ## constructs the PEG `any rune`:idx: (``_``) - result.kind = pkAnyRune - -proc newLine*: TPeg {.inline.} = - ## constructs the PEG `newline`:idx: (``\n``) - result.kind = pkNewLine - -proc unicodeLetter*: TPeg {.inline.} = - ## constructs the PEG ``\letter`` which matches any Unicode letter. - result.kind = pkLetter - -proc unicodeLower*: TPeg {.inline.} = - ## constructs the PEG ``\lower`` which matches any Unicode lowercase letter. - result.kind = pkLower - -proc unicodeUpper*: TPeg {.inline.} = - ## constructs the PEG ``\upper`` which matches any Unicode uppercase letter. - result.kind = pkUpper - -proc unicodeTitle*: TPeg {.inline.} = - ## constructs the PEG ``\title`` which matches any Unicode title letter. - result.kind = pkTitle - -proc unicodeWhitespace*: TPeg {.inline.} = - ## constructs the PEG ``\white`` which matches any Unicode - ## whitespace character. - result.kind = pkWhitespace - -proc startAnchor*: TPeg {.inline.} = - ## constructs the PEG ``^`` which matches the start of the input. - result.kind = pkStartAnchor - -proc endAnchor*: TPeg {.inline.} = - ## constructs the PEG ``$`` which matches the end of the input. - result = !any() - -proc capture*(a: TPeg): TPeg {.nosideEffect, rtl, extern: "npegsCapture".} = - ## constructs a capture with the PEG `a` - result.kind = pkCapture - result.sons = @[a] - -proc backref*(index: range[1..MaxSubpatterns]): TPeg {. - nosideEffect, rtl, extern: "npegs$1".} = - ## constructs a back reference of the given `index`. `index` starts counting - ## from 1. - result.kind = pkBackRef - result.index = index-1 - -proc backrefIgnoreCase*(index: range[1..MaxSubpatterns]): TPeg {. - nosideEffect, rtl, extern: "npegs$1".} = - ## constructs a back reference of the given `index`. `index` starts counting - ## from 1. Ignores case for matching. - result.kind = pkBackRefIgnoreCase - result.index = index-1 - -proc backrefIgnoreStyle*(index: range[1..MaxSubpatterns]): TPeg {. - nosideEffect, rtl, extern: "npegs$1".}= - ## constructs a back reference of the given `index`. `index` starts counting - ## from 1. Ignores style for matching. - result.kind = pkBackRefIgnoreStyle - result.index = index-1 - -proc spaceCost(n: TPeg): int = - case n.kind - of pkEmpty: discard - of pkTerminal, pkTerminalIgnoreCase, pkTerminalIgnoreStyle, pkChar, - pkGreedyRepChar, pkCharChoice, pkGreedyRepSet, - pkAny..pkWhitespace, pkGreedyAny: - result = 1 - of pkNonTerminal: - # we cannot inline a rule with a non-terminal - result = InlineThreshold+1 - else: - for i in 0..n.len-1: - inc(result, spaceCost(n.sons[i])) - if result >= InlineThreshold: break - -proc nonterminal*(n: PNonTerminal): TPeg {. - nosideEffect, rtl, extern: "npegs$1".} = - ## constructs a PEG that consists of the nonterminal symbol - assert n != nil - if ntDeclared in n.flags and spaceCost(n.rule) < InlineThreshold: - when false: echo "inlining symbol: ", n.name - result = n.rule # inlining of rule enables better optimizations - else: - result.kind = pkNonTerminal - result.nt = n - -proc newNonTerminal*(name: string, line, column: int): PNonTerminal {. - nosideEffect, rtl, extern: "npegs$1".} = - ## constructs a nonterminal symbol - new(result) - result.name = name - result.line = line - result.col = column - -template letters*: expr = - ## expands to ``charset({'A'..'Z', 'a'..'z'})`` - charSet({'A'..'Z', 'a'..'z'}) - -template digits*: expr = - ## expands to ``charset({'0'..'9'})`` - charSet({'0'..'9'}) - -template whitespace*: expr = - ## expands to ``charset({' ', '\9'..'\13'})`` - charSet({' ', '\9'..'\13'}) - -template identChars*: expr = - ## expands to ``charset({'a'..'z', 'A'..'Z', '0'..'9', '_'})`` - charSet({'a'..'z', 'A'..'Z', '0'..'9', '_'}) - -template identStartChars*: expr = - ## expands to ``charset({'A'..'Z', 'a'..'z', '_'})`` - charSet({'a'..'z', 'A'..'Z', '_'}) - -template ident*: expr = - ## same as ``[a-zA-Z_][a-zA-z_0-9]*``; standard identifier - sequence(charSet({'a'..'z', 'A'..'Z', '_'}), - *charSet({'a'..'z', 'A'..'Z', '0'..'9', '_'})) - -template natural*: expr = - ## same as ``\d+`` - +digits - -# ------------------------- debugging ----------------------------------------- - -proc esc(c: char, reserved = {'\0'..'\255'}): string = - case c - of '\b': result = "\\b" - of '\t': result = "\\t" - of '\c': result = "\\c" - of '\L': result = "\\l" - of '\v': result = "\\v" - of '\f': result = "\\f" - of '\e': result = "\\e" - of '\a': result = "\\a" - of '\\': result = "\\\\" - of 'a'..'z', 'A'..'Z', '0'..'9', '_': result = $c - elif c < ' ' or c >= '\128': result = '\\' & $ord(c) - elif c in reserved: result = '\\' & c - else: result = $c - -proc singleQuoteEsc(c: char): string = return "'" & esc(c, {'\''}) & "'" - -proc singleQuoteEsc(str: string): string = - result = "'" - for c in items(str): add result, esc(c, {'\''}) - add result, '\'' - -proc charSetEscAux(cc: set[char]): string = - const reserved = {'^', '-', ']'} - result = "" - var c1 = 0 - while c1 <= 0xff: - if chr(c1) in cc: - var c2 = c1 - while c2 < 0xff and chr(succ(c2)) in cc: inc(c2) - if c1 == c2: - add result, esc(chr(c1), reserved) - elif c2 == succ(c1): - add result, esc(chr(c1), reserved) & esc(chr(c2), reserved) - else: - add result, esc(chr(c1), reserved) & '-' & esc(chr(c2), reserved) - c1 = c2 - inc(c1) - -proc charSetEsc(cc: set[char]): string = - if card(cc) >= 128+64: - result = "[^" & charSetEscAux({'\1'..'\xFF'} - cc) & ']' - else: - result = '[' & charSetEscAux(cc) & ']' - -proc toStrAux(r: TPeg, res: var string) = - case r.kind - of pkEmpty: add(res, "()") - of pkAny: add(res, '.') - of pkAnyRune: add(res, '_') - of pkLetter: add(res, "\\letter") - of pkLower: add(res, "\\lower") - of pkUpper: add(res, "\\upper") - of pkTitle: add(res, "\\title") - of pkWhitespace: add(res, "\\white") - - of pkNewLine: add(res, "\\n") - of pkTerminal: add(res, singleQuoteEsc(r.term)) - of pkTerminalIgnoreCase: - add(res, 'i') - add(res, singleQuoteEsc(r.term)) - of pkTerminalIgnoreStyle: - add(res, 'y') - add(res, singleQuoteEsc(r.term)) - of pkChar: add(res, singleQuoteEsc(r.ch)) - of pkCharChoice: add(res, charSetEsc(r.charChoice[])) - of pkNonTerminal: add(res, r.nt.name) - of pkSequence: - add(res, '(') - toStrAux(r.sons[0], res) - for i in 1 .. high(r.sons): - add(res, ' ') - toStrAux(r.sons[i], res) - add(res, ')') - of pkOrderedChoice: - add(res, '(') - toStrAux(r.sons[0], res) - for i in 1 .. high(r.sons): - add(res, " / ") - toStrAux(r.sons[i], res) - add(res, ')') - of pkGreedyRep: - toStrAux(r.sons[0], res) - add(res, '*') - of pkGreedyRepChar: - add(res, singleQuoteEsc(r.ch)) - add(res, '*') - of pkGreedyRepSet: - add(res, charSetEsc(r.charChoice[])) - add(res, '*') - of pkGreedyAny: - add(res, ".*") - of pkOption: - toStrAux(r.sons[0], res) - add(res, '?') - of pkAndPredicate: - add(res, '&') - toStrAux(r.sons[0], res) - of pkNotPredicate: - add(res, '!') - toStrAux(r.sons[0], res) - of pkSearch: - add(res, '@') - toStrAux(r.sons[0], res) - of pkCapturedSearch: - add(res, "{@}") - toStrAux(r.sons[0], res) - of pkCapture: - add(res, '{') - toStrAux(r.sons[0], res) - add(res, '}') - of pkBackRef: - add(res, '$') - add(res, $r.index) - of pkBackRefIgnoreCase: - add(res, "i$") - add(res, $r.index) - of pkBackRefIgnoreStyle: - add(res, "y$") - add(res, $r.index) - of pkRule: - toStrAux(r.sons[0], res) - add(res, " <- ") - toStrAux(r.sons[1], res) - of pkList: - for i in 0 .. high(r.sons): - toStrAux(r.sons[i], res) - add(res, "\n") - of pkStartAnchor: - add(res, '^') - -proc `$` *(r: TPeg): string {.nosideEffect, rtl, extern: "npegsToString".} = - ## converts a PEG to its string representation - result = "" - toStrAux(r, result) - -# --------------------- core engine ------------------------------------------- - -type - Captures* = object ## contains the captured substrings. - matches: array[0..MaxSubpatterns-1, tuple[first, last: int]] - ml: int - origStart: int - -{.deprecated: [TCaptures: Captures].} - -proc bounds*(c: Captures, - i: range[0..MaxSubpatterns-1]): tuple[first, last: int] = - ## returns the bounds ``[first..last]`` of the `i`'th capture. - result = c.matches[i] - -when not useUnicode: - type - TRune = char - template fastRuneAt(s, i, ch: expr) = - ch = s[i] - inc(i) - template runeLenAt(s, i: expr): expr = 1 - - proc isAlpha(a: char): bool {.inline.} = return a in {'a'..'z','A'..'Z'} - proc isUpper(a: char): bool {.inline.} = return a in {'A'..'Z'} - proc isLower(a: char): bool {.inline.} = return a in {'a'..'z'} - proc isTitle(a: char): bool {.inline.} = return false - proc isWhiteSpace(a: char): bool {.inline.} = return a in {' ', '\9'..'\13'} - -proc rawMatch*(s: string, p: TPeg, start: int, c: var Captures): int {. - nosideEffect, rtl, extern: "npegs$1".} = - ## low-level matching proc that implements the PEG interpreter. Use this - ## for maximum efficiency (every other PEG operation ends up calling this - ## proc). - ## Returns -1 if it does not match, else the length of the match - case p.kind - of pkEmpty: result = 0 # match of length 0 - of pkAny: - if s[start] != '\0': result = 1 - else: result = -1 - of pkAnyRune: - if s[start] != '\0': - result = runeLenAt(s, start) - else: - result = -1 - of pkLetter: - if s[start] != '\0': - var a: Rune - result = start - fastRuneAt(s, result, a) - if isAlpha(a): dec(result, start) - else: result = -1 - else: - result = -1 - of pkLower: - if s[start] != '\0': - var a: Rune - result = start - fastRuneAt(s, result, a) - if isLower(a): dec(result, start) - else: result = -1 - else: - result = -1 - of pkUpper: - if s[start] != '\0': - var a: Rune - result = start - fastRuneAt(s, result, a) - if isUpper(a): dec(result, start) - else: result = -1 - else: - result = -1 - of pkTitle: - if s[start] != '\0': - var a: Rune - result = start - fastRuneAt(s, result, a) - if isTitle(a): dec(result, start) - else: result = -1 - else: - result = -1 - of pkWhitespace: - if s[start] != '\0': - var a: Rune - result = start - fastRuneAt(s, result, a) - if isWhiteSpace(a): dec(result, start) - else: result = -1 - else: - result = -1 - of pkGreedyAny: - result = len(s) - start - of pkNewLine: - if s[start] == '\L': result = 1 - elif s[start] == '\C': - if s[start+1] == '\L': result = 2 - else: result = 1 - else: result = -1 - of pkTerminal: - result = len(p.term) - for i in 0..result-1: - if p.term[i] != s[start+i]: - result = -1 - break - of pkTerminalIgnoreCase: - var - i = 0 - a, b: Rune - result = start - while i < len(p.term): - fastRuneAt(p.term, i, a) - fastRuneAt(s, result, b) - if toLower(a) != toLower(b): - result = -1 - break - dec(result, start) - of pkTerminalIgnoreStyle: - var - i = 0 - a, b: Rune - result = start - while i < len(p.term): - while true: - fastRuneAt(p.term, i, a) - if a != Rune('_'): break - while true: - fastRuneAt(s, result, b) - if b != Rune('_'): break - if toLower(a) != toLower(b): - result = -1 - break - dec(result, start) - of pkChar: - if p.ch == s[start]: result = 1 - else: result = -1 - of pkCharChoice: - if contains(p.charChoice[], s[start]): result = 1 - else: result = -1 - of pkNonTerminal: - var oldMl = c.ml - when false: echo "enter: ", p.nt.name - result = rawMatch(s, p.nt.rule, start, c) - when false: echo "leave: ", p.nt.name - if result < 0: c.ml = oldMl - of pkSequence: - var oldMl = c.ml - result = 0 - for i in 0..high(p.sons): - var x = rawMatch(s, p.sons[i], start+result, c) - if x < 0: - c.ml = oldMl - result = -1 - break - else: inc(result, x) - of pkOrderedChoice: - var oldMl = c.ml - for i in 0..high(p.sons): - result = rawMatch(s, p.sons[i], start, c) - if result >= 0: break - c.ml = oldMl - of pkSearch: - var oldMl = c.ml - result = 0 - while start+result < s.len: - var x = rawMatch(s, p.sons[0], start+result, c) - if x >= 0: - inc(result, x) - return - inc(result) - result = -1 - c.ml = oldMl - of pkCapturedSearch: - var idx = c.ml # reserve a slot for the subpattern - inc(c.ml) - result = 0 - while start+result < s.len: - var x = rawMatch(s, p.sons[0], start+result, c) - if x >= 0: - if idx < MaxSubpatterns: - c.matches[idx] = (start, start+result-1) - #else: silently ignore the capture - inc(result, x) - return - inc(result) - result = -1 - c.ml = idx - of pkGreedyRep: - result = 0 - while true: - var x = rawMatch(s, p.sons[0], start+result, c) - # if x == 0, we have an endless loop; so the correct behaviour would be - # not to break. But endless loops can be easily introduced: - # ``(comment / \w*)*`` is such an example. Breaking for x == 0 does the - # expected thing in this case. - if x <= 0: break - inc(result, x) - of pkGreedyRepChar: - result = 0 - var ch = p.ch - while ch == s[start+result]: inc(result) - of pkGreedyRepSet: - result = 0 - while contains(p.charChoice[], s[start+result]): inc(result) - of pkOption: - result = max(0, rawMatch(s, p.sons[0], start, c)) - of pkAndPredicate: - var oldMl = c.ml - result = rawMatch(s, p.sons[0], start, c) - if result >= 0: result = 0 # do not consume anything - else: c.ml = oldMl - of pkNotPredicate: - var oldMl = c.ml - result = rawMatch(s, p.sons[0], start, c) - if result < 0: result = 0 - else: - c.ml = oldMl - result = -1 - of pkCapture: - var idx = c.ml # reserve a slot for the subpattern - inc(c.ml) - result = rawMatch(s, p.sons[0], start, c) - if result >= 0: - if idx < MaxSubpatterns: - c.matches[idx] = (start, start+result-1) - #else: silently ignore the capture - else: - c.ml = idx - of pkBackRef..pkBackRefIgnoreStyle: - if p.index >= c.ml: return -1 - var (a, b) = c.matches[p.index] - var n: TPeg - n.kind = succ(pkTerminal, ord(p.kind)-ord(pkBackRef)) - n.term = s.substr(a, b) - result = rawMatch(s, n, start, c) - of pkStartAnchor: - if c.origStart == start: result = 0 - else: result = -1 - of pkRule, pkList: assert false - -template fillMatches(s, caps, c: expr) = - for k in 0..c.ml-1: - caps[k] = substr(s, c.matches[k][0], c.matches[k][1]) - -proc match*(s: string, pattern: TPeg, matches: var openArray[string], - start = 0): bool {.nosideEffect, rtl, extern: "npegs$1Capture".} = - ## returns ``true`` if ``s[start..]`` matches the ``pattern`` and - ## the captured substrings in the array ``matches``. If it does not - ## match, nothing is written into ``matches`` and ``false`` is - ## returned. - var c: Captures - c.origStart = start - result = rawMatch(s, pattern, start, c) == len(s) - start - if result: fillMatches(s, matches, c) - -proc match*(s: string, pattern: TPeg, - start = 0): bool {.nosideEffect, rtl, extern: "npegs$1".} = - ## returns ``true`` if ``s`` matches the ``pattern`` beginning from ``start``. - var c: Captures - c.origStart = start - result = rawMatch(s, pattern, start, c) == len(s)-start - -proc matchLen*(s: string, pattern: TPeg, matches: var openArray[string], - start = 0): int {.nosideEffect, rtl, extern: "npegs$1Capture".} = - ## the same as ``match``, but it returns the length of the match, - ## if there is no match, -1 is returned. Note that a match length - ## of zero can happen. It's possible that a suffix of `s` remains - ## that does not belong to the match. - var c: Captures - c.origStart = start - result = rawMatch(s, pattern, start, c) - if result >= 0: fillMatches(s, matches, c) - -proc matchLen*(s: string, pattern: TPeg, - start = 0): int {.nosideEffect, rtl, extern: "npegs$1".} = - ## the same as ``match``, but it returns the length of the match, - ## if there is no match, -1 is returned. Note that a match length - ## of zero can happen. It's possible that a suffix of `s` remains - ## that does not belong to the match. - var c: Captures - c.origStart = start - result = rawMatch(s, pattern, start, c) - -proc find*(s: string, pattern: TPeg, matches: var openArray[string], - start = 0): int {.nosideEffect, rtl, extern: "npegs$1Capture".} = - ## returns the starting position of ``pattern`` in ``s`` and the captured - ## substrings in the array ``matches``. If it does not match, nothing - ## is written into ``matches`` and -1 is returned. - var c: Captures - c.origStart = start - for i in start .. s.len-1: - c.ml = 0 - if rawMatch(s, pattern, i, c) >= 0: - fillMatches(s, matches, c) - return i - return -1 - # could also use the pattern here: (!P .)* P - -proc findBounds*(s: string, pattern: TPeg, matches: var openArray[string], - start = 0): tuple[first, last: int] {. - nosideEffect, rtl, extern: "npegs$1Capture".} = - ## returns the starting position and end position of ``pattern`` in ``s`` - ## and the captured - ## substrings in the array ``matches``. If it does not match, nothing - ## is written into ``matches`` and (-1,0) is returned. - var c: Captures - c.origStart = start - for i in start .. s.len-1: - c.ml = 0 - var L = rawMatch(s, pattern, i, c) - if L >= 0: - fillMatches(s, matches, c) - return (i, i+L-1) - return (-1, 0) - -proc find*(s: string, pattern: TPeg, - start = 0): int {.nosideEffect, rtl, extern: "npegs$1".} = - ## returns the starting position of ``pattern`` in ``s``. If it does not - ## match, -1 is returned. - var c: Captures - c.origStart = start - for i in start .. s.len-1: - if rawMatch(s, pattern, i, c) >= 0: return i - return -1 - -iterator findAll*(s: string, pattern: TPeg, start = 0): string = - ## yields all matching *substrings* of `s` that match `pattern`. - var c: Captures - c.origStart = start - var i = start - while i < s.len: - c.ml = 0 - var L = rawMatch(s, pattern, i, c) - if L < 0: - inc(i, 1) - else: - yield substr(s, i, i+L-1) - inc(i, L) - -proc findAll*(s: string, pattern: TPeg, start = 0): seq[string] {. - nosideEffect, rtl, extern: "npegs$1".} = - ## returns all matching *substrings* of `s` that match `pattern`. - ## If it does not match, @[] is returned. - accumulateResult(findAll(s, pattern, start)) - -when not defined(nimhygiene): - {.pragma: inject.} - -template `=~`*(s: string, pattern: TPeg): bool = - ## This calls ``match`` with an implicit declared ``matches`` array that - ## can be used in the scope of the ``=~`` call: - ## - ## .. code-block:: nim - ## - ## if line =~ peg"\s* {\w+} \s* '=' \s* {\w+}": - ## # matches a key=value pair: - ## echo("Key: ", matches[0]) - ## echo("Value: ", matches[1]) - ## elif line =~ peg"\s*{'#'.*}": - ## # matches a comment - ## # note that the implicit ``matches`` array is different from the - ## # ``matches`` array of the first branch - ## echo("comment: ", matches[0]) - ## else: - ## echo("syntax error") - ## - bind MaxSubpatterns - when not declaredInScope(matches): - var matches {.inject.}: array[0..MaxSubpatterns-1, string] - match(s, pattern, matches) - -# ------------------------- more string handling ------------------------------ - -proc contains*(s: string, pattern: TPeg, start = 0): bool {. - nosideEffect, rtl, extern: "npegs$1".} = - ## same as ``find(s, pattern, start) >= 0`` - return find(s, pattern, start) >= 0 - -proc contains*(s: string, pattern: TPeg, matches: var openArray[string], - start = 0): bool {.nosideEffect, rtl, extern: "npegs$1Capture".} = - ## same as ``find(s, pattern, matches, start) >= 0`` - return find(s, pattern, matches, start) >= 0 - -proc startsWith*(s: string, prefix: TPeg, start = 0): bool {. - nosideEffect, rtl, extern: "npegs$1".} = - ## returns true if `s` starts with the pattern `prefix` - result = matchLen(s, prefix, start) >= 0 - -proc endsWith*(s: string, suffix: TPeg, start = 0): bool {. - nosideEffect, rtl, extern: "npegs$1".} = - ## returns true if `s` ends with the pattern `prefix` - var c: Captures - c.origStart = start - for i in start .. s.len-1: - if rawMatch(s, suffix, i, c) == s.len - i: return true - -proc replacef*(s: string, sub: TPeg, by: string): string {. - nosideEffect, rtl, extern: "npegs$1".} = - ## Replaces `sub` in `s` by the string `by`. Captures can be accessed in `by` - ## with the notation ``$i`` and ``$#`` (see strutils.`%`). Examples: - ## - ## .. code-block:: nim - ## "var1=key; var2=key2".replace(peg"{\ident}'='{\ident}", "$1<-$2$2") - ## - ## Results in: - ## - ## .. code-block:: nim - ## - ## "var1<-keykey; val2<-key2key2" - result = "" - var i = 0 - var caps: array[0..MaxSubpatterns-1, string] - var c: Captures - while i < s.len: - c.ml = 0 - var x = rawMatch(s, sub, i, c) - if x <= 0: - add(result, s[i]) - inc(i) - else: - fillMatches(s, caps, c) - addf(result, by, caps) - inc(i, x) - add(result, substr(s, i)) - -proc replace*(s: string, sub: TPeg, by = ""): string {. - nosideEffect, rtl, extern: "npegs$1".} = - ## Replaces `sub` in `s` by the string `by`. Captures cannot be accessed - ## in `by`. - result = "" - var i = 0 - var c: Captures - while i < s.len: - var x = rawMatch(s, sub, i, c) - if x <= 0: - add(result, s[i]) - inc(i) - else: - add(result, by) - inc(i, x) - add(result, substr(s, i)) - -proc parallelReplace*(s: string, subs: varargs[ - tuple[pattern: TPeg, repl: string]]): string {. - nosideEffect, rtl, extern: "npegs$1".} = - ## Returns a modified copy of `s` with the substitutions in `subs` - ## applied in parallel. - result = "" - var i = 0 - var c: Captures - var caps: array[0..MaxSubpatterns-1, string] - while i < s.len: - block searchSubs: - for j in 0..high(subs): - c.ml = 0 - var x = rawMatch(s, subs[j][0], i, c) - if x > 0: - fillMatches(s, caps, c) - addf(result, subs[j][1], caps) - inc(i, x) - break searchSubs - add(result, s[i]) - inc(i) - # copy the rest: - add(result, substr(s, i)) - -proc transformFile*(infile, outfile: string, - subs: varargs[tuple[pattern: TPeg, repl: string]]) {. - rtl, extern: "npegs$1".} = - ## reads in the file `infile`, performs a parallel replacement (calls - ## `parallelReplace`) and writes back to `outfile`. Raises ``EIO`` if an - ## error occurs. This is supposed to be used for quick scripting. - var x = readFile(infile).string - writeFile(outfile, x.parallelReplace(subs)) - -iterator split*(s: string, sep: TPeg): string = - ## Splits the string `s` into substrings. - ## - ## Substrings are separated by the PEG `sep`. - ## Examples: - ## - ## .. code-block:: nim - ## for word in split("00232this02939is39an22example111", peg"\d+"): - ## writeln(stdout, word) - ## - ## Results in: - ## - ## .. code-block:: nim - ## "this" - ## "is" - ## "an" - ## "example" - ## - var c: Captures - var - first = 0 - last = 0 - while last < len(s): - c.ml = 0 - var x = rawMatch(s, sep, last, c) - if x > 0: inc(last, x) - first = last - while last < len(s): - inc(last) - c.ml = 0 - x = rawMatch(s, sep, last, c) - if x > 0: break - if first < last: - yield substr(s, first, last-1) - -proc split*(s: string, sep: TPeg): seq[string] {. - nosideEffect, rtl, extern: "npegs$1".} = - ## Splits the string `s` into substrings. - accumulateResult(split(s, sep)) - -# ------------------- scanner ------------------------------------------------- - -type - TModifier = enum - modNone, - modVerbatim, - modIgnoreCase, - modIgnoreStyle - TTokKind = enum ## enumeration of all tokens - tkInvalid, ## invalid token - tkEof, ## end of file reached - tkAny, ## . - tkAnyRune, ## _ - tkIdentifier, ## abc - tkStringLit, ## "abc" or 'abc' - tkCharSet, ## [^A-Z] - tkParLe, ## '(' - tkParRi, ## ')' - tkCurlyLe, ## '{' - tkCurlyRi, ## '}' - tkCurlyAt, ## '{@}' - tkArrow, ## '<-' - tkBar, ## '/' - tkStar, ## '*' - tkPlus, ## '+' - tkAmp, ## '&' - tkNot, ## '!' - tkOption, ## '?' - tkAt, ## '@' - tkBuiltin, ## \identifier - tkEscaped, ## \\ - tkBackref, ## '$' - tkDollar, ## '$' - tkHat ## '^' - - TToken {.final.} = object ## a token - kind: TTokKind ## the type of the token - modifier: TModifier - literal: string ## the parsed (string) literal - charset: set[char] ## if kind == tkCharSet - index: int ## if kind == tkBackref - - PegLexer {.inheritable.} = object ## the lexer object. - bufpos: int ## the current position within the buffer - buf: cstring ## the buffer itself - lineNumber: int ## the current line number - lineStart: int ## index of last line start in buffer - colOffset: int ## column to add - filename: string - -const - tokKindToStr: array[TTokKind, string] = [ - "invalid", "[EOF]", ".", "_", "identifier", "string literal", - "character set", "(", ")", "{", "}", "{@}", - "<-", "/", "*", "+", "&", "!", "?", - "@", "built-in", "escaped", "$", "$", "^" - ] - -proc handleCR(L: var PegLexer, pos: int): int = - assert(L.buf[pos] == '\c') - inc(L.lineNumber) - result = pos+1 - if L.buf[result] == '\L': inc(result) - L.lineStart = result - -proc handleLF(L: var PegLexer, pos: int): int = - assert(L.buf[pos] == '\L') - inc(L.lineNumber) - result = pos+1 - L.lineStart = result - -proc init(L: var PegLexer, input, filename: string, line = 1, col = 0) = - L.buf = input - L.bufpos = 0 - L.lineNumber = line - L.colOffset = col - L.lineStart = 0 - L.filename = filename - -proc getColumn(L: PegLexer): int {.inline.} = - result = abs(L.bufpos - L.lineStart) + L.colOffset - -proc getLine(L: PegLexer): int {.inline.} = - result = L.lineNumber - -proc errorStr(L: PegLexer, msg: string, line = -1, col = -1): string = - var line = if line < 0: getLine(L) else: line - var col = if col < 0: getColumn(L) else: col - result = "$1($2, $3) Error: $4" % [L.filename, $line, $col, msg] - -proc handleHexChar(c: var PegLexer, xi: var int) = - case c.buf[c.bufpos] - of '0'..'9': - xi = (xi shl 4) or (ord(c.buf[c.bufpos]) - ord('0')) - inc(c.bufpos) - of 'a'..'f': - xi = (xi shl 4) or (ord(c.buf[c.bufpos]) - ord('a') + 10) - inc(c.bufpos) - of 'A'..'F': - xi = (xi shl 4) or (ord(c.buf[c.bufpos]) - ord('A') + 10) - inc(c.bufpos) - else: discard - -proc getEscapedChar(c: var PegLexer, tok: var TToken) = - inc(c.bufpos) - case c.buf[c.bufpos] - of 'r', 'R', 'c', 'C': - add(tok.literal, '\c') - inc(c.bufpos) - of 'l', 'L': - add(tok.literal, '\L') - inc(c.bufpos) - of 'f', 'F': - add(tok.literal, '\f') - inc(c.bufpos) - of 'e', 'E': - add(tok.literal, '\e') - inc(c.bufpos) - of 'a', 'A': - add(tok.literal, '\a') - inc(c.bufpos) - of 'b', 'B': - add(tok.literal, '\b') - inc(c.bufpos) - of 'v', 'V': - add(tok.literal, '\v') - inc(c.bufpos) - of 't', 'T': - add(tok.literal, '\t') - inc(c.bufpos) - of 'x', 'X': - inc(c.bufpos) - var xi = 0 - handleHexChar(c, xi) - handleHexChar(c, xi) - if xi == 0: tok.kind = tkInvalid - else: add(tok.literal, chr(xi)) - of '0'..'9': - var val = ord(c.buf[c.bufpos]) - ord('0') - inc(c.bufpos) - var i = 1 - while (i <= 3) and (c.buf[c.bufpos] in {'0'..'9'}): - val = val * 10 + ord(c.buf[c.bufpos]) - ord('0') - inc(c.bufpos) - inc(i) - if val > 0 and val <= 255: add(tok.literal, chr(val)) - else: tok.kind = tkInvalid - of '\0'..'\31': - tok.kind = tkInvalid - elif c.buf[c.bufpos] in strutils.Letters: - tok.kind = tkInvalid - else: - add(tok.literal, c.buf[c.bufpos]) - inc(c.bufpos) - -proc skip(c: var PegLexer) = - var pos = c.bufpos - var buf = c.buf - while true: - case buf[pos] - of ' ', '\t': - inc(pos) - of '#': - while not (buf[pos] in {'\c', '\L', '\0'}): inc(pos) - of '\c': - pos = handleCR(c, pos) - buf = c.buf - of '\L': - pos = handleLF(c, pos) - buf = c.buf - else: - break # EndOfFile also leaves the loop - c.bufpos = pos - -proc getString(c: var PegLexer, tok: var TToken) = - tok.kind = tkStringLit - var pos = c.bufpos + 1 - var buf = c.buf - var quote = buf[pos-1] - while true: - case buf[pos] - of '\\': - c.bufpos = pos - getEscapedChar(c, tok) - pos = c.bufpos - of '\c', '\L', '\0': - tok.kind = tkInvalid - break - elif buf[pos] == quote: - inc(pos) - break - else: - add(tok.literal, buf[pos]) - inc(pos) - c.bufpos = pos - -proc getDollar(c: var PegLexer, tok: var TToken) = - var pos = c.bufpos + 1 - var buf = c.buf - if buf[pos] in {'0'..'9'}: - tok.kind = tkBackref - tok.index = 0 - while buf[pos] in {'0'..'9'}: - tok.index = tok.index * 10 + ord(buf[pos]) - ord('0') - inc(pos) - else: - tok.kind = tkDollar - c.bufpos = pos - -proc getCharSet(c: var PegLexer, tok: var TToken) = - tok.kind = tkCharSet - tok.charset = {} - var pos = c.bufpos + 1 - var buf = c.buf - var caret = false - if buf[pos] == '^': - inc(pos) - caret = true - while true: - var ch: char - case buf[pos] - of ']': - inc(pos) - break - of '\\': - c.bufpos = pos - getEscapedChar(c, tok) - pos = c.bufpos - ch = tok.literal[tok.literal.len-1] - of '\C', '\L', '\0': - tok.kind = tkInvalid - break - else: - ch = buf[pos] - inc(pos) - incl(tok.charset, ch) - if buf[pos] == '-': - if buf[pos+1] == ']': - incl(tok.charset, '-') - inc(pos) - else: - inc(pos) - var ch2: char - case buf[pos] - of '\\': - c.bufpos = pos - getEscapedChar(c, tok) - pos = c.bufpos - ch2 = tok.literal[tok.literal.len-1] - of '\C', '\L', '\0': - tok.kind = tkInvalid - break - else: - ch2 = buf[pos] - inc(pos) - for i in ord(ch)+1 .. ord(ch2): - incl(tok.charset, chr(i)) - c.bufpos = pos - if caret: tok.charset = {'\1'..'\xFF'} - tok.charset - -proc getSymbol(c: var PegLexer, tok: var TToken) = - var pos = c.bufpos - var buf = c.buf - while true: - add(tok.literal, buf[pos]) - inc(pos) - if buf[pos] notin strutils.IdentChars: break - c.bufpos = pos - tok.kind = tkIdentifier - -proc getBuiltin(c: var PegLexer, tok: var TToken) = - if c.buf[c.bufpos+1] in strutils.Letters: - inc(c.bufpos) - getSymbol(c, tok) - tok.kind = tkBuiltin - else: - tok.kind = tkEscaped - getEscapedChar(c, tok) # may set tok.kind to tkInvalid - -proc getTok(c: var PegLexer, tok: var TToken) = - tok.kind = tkInvalid - tok.modifier = modNone - setLen(tok.literal, 0) - skip(c) - case c.buf[c.bufpos] - of '{': - inc(c.bufpos) - if c.buf[c.bufpos] == '@' and c.buf[c.bufpos+1] == '}': - tok.kind = tkCurlyAt - inc(c.bufpos, 2) - add(tok.literal, "{@}") - else: - tok.kind = tkCurlyLe - add(tok.literal, '{') - of '}': - tok.kind = tkCurlyRi - inc(c.bufpos) - add(tok.literal, '}') - of '[': - getCharSet(c, tok) - of '(': - tok.kind = tkParLe - inc(c.bufpos) - add(tok.literal, '(') - of ')': - tok.kind = tkParRi - inc(c.bufpos) - add(tok.literal, ')') - of '.': - tok.kind = tkAny - inc(c.bufpos) - add(tok.literal, '.') - of '_': - tok.kind = tkAnyRune - inc(c.bufpos) - add(tok.literal, '_') - of '\\': - getBuiltin(c, tok) - of '\'', '"': getString(c, tok) - of '$': getDollar(c, tok) - of '\0': - tok.kind = tkEof - tok.literal = "[EOF]" - of 'a'..'z', 'A'..'Z', '\128'..'\255': - getSymbol(c, tok) - if c.buf[c.bufpos] in {'\'', '"'} or - c.buf[c.bufpos] == '$' and c.buf[c.bufpos+1] in {'0'..'9'}: - case tok.literal - of "i": tok.modifier = modIgnoreCase - of "y": tok.modifier = modIgnoreStyle - of "v": tok.modifier = modVerbatim - else: discard - setLen(tok.literal, 0) - if c.buf[c.bufpos] == '$': - getDollar(c, tok) - else: - getString(c, tok) - if tok.modifier == modNone: tok.kind = tkInvalid - of '+': - tok.kind = tkPlus - inc(c.bufpos) - add(tok.literal, '+') - of '*': - tok.kind = tkStar - inc(c.bufpos) - add(tok.literal, '+') - of '<': - if c.buf[c.bufpos+1] == '-': - inc(c.bufpos, 2) - tok.kind = tkArrow - add(tok.literal, "<-") - else: - add(tok.literal, '<') - of '/': - tok.kind = tkBar - inc(c.bufpos) - add(tok.literal, '/') - of '?': - tok.kind = tkOption - inc(c.bufpos) - add(tok.literal, '?') - of '!': - tok.kind = tkNot - inc(c.bufpos) - add(tok.literal, '!') - of '&': - tok.kind = tkAmp - inc(c.bufpos) - add(tok.literal, '!') - of '@': - tok.kind = tkAt - inc(c.bufpos) - add(tok.literal, '@') - if c.buf[c.bufpos] == '@': - tok.kind = tkCurlyAt - inc(c.bufpos) - add(tok.literal, '@') - of '^': - tok.kind = tkHat - inc(c.bufpos) - add(tok.literal, '^') - else: - add(tok.literal, c.buf[c.bufpos]) - inc(c.bufpos) - -proc arrowIsNextTok(c: PegLexer): bool = - # the only look ahead we need - var pos = c.bufpos - while c.buf[pos] in {'\t', ' '}: inc(pos) - result = c.buf[pos] == '<' and c.buf[pos+1] == '-' - -# ----------------------------- parser ---------------------------------------- - -type - EInvalidPeg* = object of ValueError ## raised if an invalid - ## PEG has been detected - PegParser = object of PegLexer ## the PEG parser object - tok: TToken - nonterms: seq[PNonTerminal] - modifier: TModifier - captures: int - identIsVerbatim: bool - skip: TPeg - -proc pegError(p: PegParser, msg: string, line = -1, col = -1) = - var e: ref EInvalidPeg - new(e) - e.msg = errorStr(p, msg, line, col) - raise e - -proc getTok(p: var PegParser) = - getTok(p, p.tok) - if p.tok.kind == tkInvalid: pegError(p, "invalid token") - -proc eat(p: var PegParser, kind: TTokKind) = - if p.tok.kind == kind: getTok(p) - else: pegError(p, tokKindToStr[kind] & " expected") - -proc parseExpr(p: var PegParser): TPeg - -proc getNonTerminal(p: var PegParser, name: string): PNonTerminal = - for i in 0..high(p.nonterms): - result = p.nonterms[i] - if cmpIgnoreStyle(result.name, name) == 0: return - # forward reference: - result = newNonTerminal(name, getLine(p), getColumn(p)) - add(p.nonterms, result) - -proc modifiedTerm(s: string, m: TModifier): TPeg = - case m - of modNone, modVerbatim: result = term(s) - of modIgnoreCase: result = termIgnoreCase(s) - of modIgnoreStyle: result = termIgnoreStyle(s) - -proc modifiedBackref(s: int, m: TModifier): TPeg = - case m - of modNone, modVerbatim: result = backref(s) - of modIgnoreCase: result = backrefIgnoreCase(s) - of modIgnoreStyle: result = backrefIgnoreStyle(s) - -proc builtin(p: var PegParser): TPeg = - # do not use "y", "skip" or "i" as these would be ambiguous - case p.tok.literal - of "n": result = newLine() - of "d": result = charSet({'0'..'9'}) - of "D": result = charSet({'\1'..'\xff'} - {'0'..'9'}) - of "s": result = charSet({' ', '\9'..'\13'}) - of "S": result = charSet({'\1'..'\xff'} - {' ', '\9'..'\13'}) - of "w": result = charSet({'a'..'z', 'A'..'Z', '_', '0'..'9'}) - of "W": result = charSet({'\1'..'\xff'} - {'a'..'z','A'..'Z','_','0'..'9'}) - of "a": result = charSet({'a'..'z', 'A'..'Z'}) - of "A": result = charSet({'\1'..'\xff'} - {'a'..'z', 'A'..'Z'}) - of "ident": result = pegs.ident - of "letter": result = unicodeLetter() - of "upper": result = unicodeUpper() - of "lower": result = unicodeLower() - of "title": result = unicodeTitle() - of "white": result = unicodeWhitespace() - else: pegError(p, "unknown built-in: " & p.tok.literal) - -proc token(terminal: TPeg, p: PegParser): TPeg = - if p.skip.kind == pkEmpty: result = terminal - else: result = sequence(p.skip, terminal) - -proc primary(p: var PegParser): TPeg = - case p.tok.kind - of tkAmp: - getTok(p) - return &primary(p) - of tkNot: - getTok(p) - return !primary(p) - of tkAt: - getTok(p) - return !*primary(p) - of tkCurlyAt: - getTok(p) - return !*\primary(p).token(p) - else: discard - case p.tok.kind - of tkIdentifier: - if p.identIsVerbatim: - var m = p.tok.modifier - if m == modNone: m = p.modifier - result = modifiedTerm(p.tok.literal, m).token(p) - getTok(p) - elif not arrowIsNextTok(p): - var nt = getNonTerminal(p, p.tok.literal) - incl(nt.flags, ntUsed) - result = nonterminal(nt).token(p) - getTok(p) - else: - pegError(p, "expression expected, but found: " & p.tok.literal) - of tkStringLit: - var m = p.tok.modifier - if m == modNone: m = p.modifier - result = modifiedTerm(p.tok.literal, m).token(p) - getTok(p) - of tkCharSet: - if '\0' in p.tok.charset: - pegError(p, "binary zero ('\\0') not allowed in character class") - result = charSet(p.tok.charset).token(p) - getTok(p) - of tkParLe: - getTok(p) - result = parseExpr(p) - eat(p, tkParRi) - of tkCurlyLe: - getTok(p) - result = capture(parseExpr(p)).token(p) - eat(p, tkCurlyRi) - inc(p.captures) - of tkAny: - result = any().token(p) - getTok(p) - of tkAnyRune: - result = anyRune().token(p) - getTok(p) - of tkBuiltin: - result = builtin(p).token(p) - getTok(p) - of tkEscaped: - result = term(p.tok.literal[0]).token(p) - getTok(p) - of tkDollar: - result = endAnchor() - getTok(p) - of tkHat: - result = startAnchor() - getTok(p) - of tkBackref: - var m = p.tok.modifier - if m == modNone: m = p.modifier - result = modifiedBackref(p.tok.index, m).token(p) - if p.tok.index < 0 or p.tok.index > p.captures: - pegError(p, "invalid back reference index: " & $p.tok.index) - getTok(p) - else: - pegError(p, "expression expected, but found: " & p.tok.literal) - getTok(p) # we must consume a token here to prevent endless loops! - while true: - case p.tok.kind - of tkOption: - result = ?result - getTok(p) - of tkStar: - result = *result - getTok(p) - of tkPlus: - result = +result - getTok(p) - else: break - -proc seqExpr(p: var PegParser): TPeg = - result = primary(p) - while true: - case p.tok.kind - of tkAmp, tkNot, tkAt, tkStringLit, tkCharSet, tkParLe, tkCurlyLe, - tkAny, tkAnyRune, tkBuiltin, tkEscaped, tkDollar, tkBackref, - tkHat, tkCurlyAt: - result = sequence(result, primary(p)) - of tkIdentifier: - if not arrowIsNextTok(p): - result = sequence(result, primary(p)) - else: break - else: break - -proc parseExpr(p: var PegParser): TPeg = - result = seqExpr(p) - while p.tok.kind == tkBar: - getTok(p) - result = result / seqExpr(p) - -proc parseRule(p: var PegParser): PNonTerminal = - if p.tok.kind == tkIdentifier and arrowIsNextTok(p): - result = getNonTerminal(p, p.tok.literal) - if ntDeclared in result.flags: - pegError(p, "attempt to redefine: " & result.name) - result.line = getLine(p) - result.col = getColumn(p) - getTok(p) - eat(p, tkArrow) - result.rule = parseExpr(p) - incl(result.flags, ntDeclared) # NOW inlining may be attempted - else: - pegError(p, "rule expected, but found: " & p.tok.literal) - -proc rawParse(p: var PegParser): TPeg = - ## parses a rule or a PEG expression - while p.tok.kind == tkBuiltin: - case p.tok.literal - of "i": - p.modifier = modIgnoreCase - getTok(p) - of "y": - p.modifier = modIgnoreStyle - getTok(p) - of "skip": - getTok(p) - p.skip = ?primary(p) - else: break - if p.tok.kind == tkIdentifier and arrowIsNextTok(p): - result = parseRule(p).rule - while p.tok.kind != tkEof: - discard parseRule(p) - else: - p.identIsVerbatim = true - result = parseExpr(p) - if p.tok.kind != tkEof: - pegError(p, "EOF expected, but found: " & p.tok.literal) - for i in 0..high(p.nonterms): - var nt = p.nonterms[i] - if ntDeclared notin nt.flags: - pegError(p, "undeclared identifier: " & nt.name, nt.line, nt.col) - elif ntUsed notin nt.flags and i > 0: - pegError(p, "unused rule: " & nt.name, nt.line, nt.col) - -proc parsePeg*(pattern: string, filename = "pattern", line = 1, col = 0): TPeg = - ## constructs a Peg object from `pattern`. `filename`, `line`, `col` are - ## used for error messages, but they only provide start offsets. `parsePeg` - ## keeps track of line and column numbers within `pattern`. - var p: PegParser - init(PegLexer(p), pattern, filename, line, col) - p.tok.kind = tkInvalid - p.tok.modifier = modNone - p.tok.literal = "" - p.tok.charset = {} - p.nonterms = @[] - p.identIsVerbatim = false - getTok(p) - result = rawParse(p) - -proc peg*(pattern: string): TPeg = - ## constructs a Peg object from the `pattern`. The short name has been - ## chosen to encourage its use as a raw string modifier:: - ## - ## peg"{\ident} \s* '=' \s* {.*}" - result = parsePeg(pattern, "pattern") - -proc escapePeg*(s: string): string = - ## escapes `s` so that it is matched verbatim when used as a peg. - result = "" - var inQuote = false - for c in items(s): - case c - of '\0'..'\31', '\'', '"', '\\': - if inQuote: - result.add('\'') - inQuote = false - result.add("\\x") - result.add(toHex(ord(c), 2)) - else: - if not inQuote: - result.add('\'') - inQuote = true - result.add(c) - if inQuote: result.add('\'') - -when isMainModule: - assert escapePeg("abc''def'") == r"'abc'\x27\x27'def'\x27" - assert match("(a b c)", peg"'(' @ ')'") - assert match("W_HI_Le", peg"\y 'while'") - assert(not match("W_HI_L", peg"\y 'while'")) - assert(not match("W_HI_Le", peg"\y v'while'")) - assert match("W_HI_Le", peg"y'while'") - - assert($ +digits == $peg"\d+") - assert "0158787".match(peg"\d+") - assert "ABC 0232".match(peg"\w+\s+\d+") - assert "ABC".match(peg"\d+ / \w+") - - for word in split("00232this02939is39an22example111", peg"\d+"): - writeln(stdout, word) - - assert matchLen("key", ident) == 3 - - var pattern = sequence(ident, *whitespace, term('='), *whitespace, ident) - assert matchLen("key1= cal9", pattern) == 11 - - var ws = newNonTerminal("ws", 1, 1) - ws.rule = *whitespace - - var expr = newNonTerminal("expr", 1, 1) - expr.rule = sequence(capture(ident), *sequence( - nonterminal(ws), term('+'), nonterminal(ws), nonterminal(expr))) - - var c: Captures - var s = "a+b + c +d+e+f" - assert rawMatch(s, expr.rule, 0, c) == len(s) - var a = "" - for i in 0..c.ml-1: - a.add(substr(s, c.matches[i][0], c.matches[i][1])) - assert a == "abcdef" - #echo expr.rule - - #const filename = "lib/devel/peg/grammar.txt" - #var grammar = parsePeg(newFileStream(filename, fmRead), filename) - #echo "a <- [abc]*?".match(grammar) - assert find("_____abc_______", term("abc"), 2) == 5 - assert match("_______ana", peg"A <- 'ana' / . A") - assert match("abcs%%%", peg"A <- ..A / .A / '%'") - - var matches: array[0..MaxSubpatterns-1, string] - if "abc" =~ peg"{'a'}'bc' 'xyz' / {\ident}": - assert matches[0] == "abc" - else: - assert false - - var g2 = peg"""S <- A B / C D - A <- 'a'+ - B <- 'b'+ - C <- 'c'+ - D <- 'd'+ - """ - assert($g2 == "((A B) / (C D))") - assert match("cccccdddddd", g2) - assert("var1=key; var2=key2".replacef(peg"{\ident}'='{\ident}", "$1<-$2$2") == - "var1<-keykey; var2<-key2key2") - assert("var1=key; var2=key2".replace(peg"{\ident}'='{\ident}", "$1<-$2$2") == - "$1<-$2$2; $1<-$2$2") - assert "var1=key; var2=key2".endsWith(peg"{\ident}'='{\ident}") - - if "aaaaaa" =~ peg"'aa' !. / ({'a'})+": - assert matches[0] == "a" - else: - assert false - - if match("abcdefg", peg"c {d} ef {g}", matches, 2): - assert matches[0] == "d" - assert matches[1] == "g" - else: - assert false - - for x in findAll("abcdef", peg".", 3): - echo x - - for x in findAll("abcdef", peg"^{.}", 3): - assert x == "d" - - if "f(a, b)" =~ peg"{[0-9]+} / ({\ident} '(' {@} ')')": - assert matches[0] == "f" - assert matches[1] == "a, b" - else: - assert false - - assert match("eine übersicht und außerdem", peg"(\letter \white*)+") - # ß is not a lower cased letter?! - assert match("eine übersicht und auerdem", peg"(\lower \white*)+") - assert match("EINE ÜBERSICHT UND AUSSERDEM", peg"(\upper \white*)+") - assert(not match("456678", peg"(\letter)+")) - - assert("var1 = key; var2 = key2".replacef( - peg"\skip(\s*) {\ident}'='{\ident}", "$1<-$2$2") == - "var1<-keykey;var2<-key2key2") - - assert match("prefix/start", peg"^start$", 7) - diff --git a/lib/pure/poly.nim b/lib/pure/poly.nim index 286e5a8fd..58dcdc1ad 100644 --- a/lib/pure/poly.nim +++ b/lib/pure/poly.nim @@ -58,7 +58,7 @@ proc `[]=` *(p:var Poly;idx:int,v:float)= iterator items*(p:Poly):float= - ## Iterates through the corfficients of the polynomial. + ## Iterates through the coefficients of the polynomial. var i=p.degree while i>=0: yield p[i] diff --git a/lib/pure/rationals.nim b/lib/pure/rationals.nim new file mode 100644 index 000000000..3b68a2381 --- /dev/null +++ b/lib/pure/rationals.nim @@ -0,0 +1,289 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2015 Dennis Felsing +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + + +## This module implements rational numbers, consisting of a numerator `num` and +## a denominator `den`, both of type int. The denominator can not be 0. + +import math +import hashes + +type Rational*[T] = object + ## a rational number, consisting of a numerator and denominator + num*, den*: T + +proc initRational*[T](num, den: T): Rational[T] = + ## Create a new rational number. + result.num = num + result.den = den + +proc `//`*[T](num, den: T): Rational[T] = initRational[T](num, den) + ## A friendlier version of `initRational`. Example usage: + ## + ## .. code-block:: nim + ## var x = 1//3 + 1//5 + +proc `$`*[T](x: Rational[T]): string = + ## Turn a rational number into a string. + result = $x.num & "/" & $x.den + +proc toRational*[T](x: T): Rational[T] = + ## Convert some integer `x` to a rational number. + result.num = x + result.den = 1 + +proc toFloat*[T](x: Rational[T]): float = + ## Convert a rational number `x` to a float. + x.num / x.den + +proc toInt*[T](x: Rational[T]): int = + ## Convert a rational number `x` to an int. Conversion rounds towards 0 if + ## `x` does not contain an integer value. + x.num div x.den + +proc reduce*[T](x: var Rational[T]) = + ## Reduce rational `x`. + let common = gcd(x.num, x.den) + if x.den > 0: + x.num = x.num div common + x.den = x.den div common + elif x.den < 0: + x.num = -x.num div common + x.den = -x.den div common + else: + raise newException(DivByZeroError, "division by zero") + +proc `+` *[T](x, y: Rational[T]): Rational[T] = + ## Add two rational numbers. + let common = lcm(x.den, y.den) + result.num = common div x.den * x.num + common div y.den * y.num + result.den = common + reduce(result) + +proc `+` *[T](x: Rational[T], y: T): Rational[T] = + ## Add rational `x` to int `y`. + result.num = x.num + y * x.den + result.den = x.den + +proc `+` *[T](x: T, y: Rational[T]): Rational[T] = + ## Add int `x` to rational `y`. + result.num = x * y.den + y.num + result.den = y.den + +proc `+=` *[T](x: var Rational[T], y: Rational[T]) = + ## Add rational `y` to rational `x`. + let common = lcm(x.den, y.den) + x.num = common div x.den * x.num + common div y.den * y.num + x.den = common + reduce(x) + +proc `+=` *[T](x: var Rational[T], y: T) = + ## Add int `y` to rational `x`. + x.num += y * x.den + +proc `-` *[T](x: Rational[T]): Rational[T] = + ## Unary minus for rational numbers. + result.num = -x.num + result.den = x.den + +proc `-` *[T](x, y: Rational[T]): Rational[T] = + ## Subtract two rational numbers. + let common = lcm(x.den, y.den) + result.num = common div x.den * x.num - common div y.den * y.num + result.den = common + reduce(result) + +proc `-` *[T](x: Rational[T], y: T): Rational[T] = + ## Subtract int `y` from rational `x`. + result.num = x.num - y * x.den + result.den = x.den + +proc `-` *[T](x: T, y: Rational[T]): Rational[T] = + ## Subtract rational `y` from int `x`. + result.num = - x * y.den + y.num + result.den = y.den + +proc `-=` *[T](x: var Rational[T], y: Rational[T]) = + ## Subtract rational `y` from rational `x`. + let common = lcm(x.den, y.den) + x.num = common div x.den * x.num - common div y.den * y.num + x.den = common + reduce(x) + +proc `-=` *[T](x: var Rational[T], y: T) = + ## Subtract int `y` from rational `x`. + x.num -= y * x.den + +proc `*` *[T](x, y: Rational[T]): Rational[T] = + ## Multiply two rational numbers. + result.num = x.num * y.num + result.den = x.den * y.den + reduce(result) + +proc `*` *[T](x: Rational[T], y: T): Rational[T] = + ## Multiply rational `x` with int `y`. + result.num = x.num * y + result.den = x.den + reduce(result) + +proc `*` *[T](x: T, y: Rational[T]): Rational[T] = + ## Multiply int `x` with rational `y`. + result.num = x * y.num + result.den = y.den + reduce(result) + +proc `*=` *[T](x: var Rational[T], y: Rational[T]) = + ## Multiply rationals `y` to `x`. + x.num *= y.num + x.den *= y.den + reduce(x) + +proc `*=` *[T](x: var Rational[T], y: T) = + ## Multiply int `y` to rational `x`. + x.num *= y + reduce(x) + +proc reciprocal*[T](x: Rational[T]): Rational[T] = + ## Calculate the reciprocal of `x`. (1/x) + if x.num > 0: + result.num = x.den + result.den = x.num + elif x.num < 0: + result.num = -x.den + result.den = -x.num + else: + raise newException(DivByZeroError, "division by zero") + +proc `/`*[T](x, y: Rational[T]): Rational[T] = + ## Divide rationals `x` by `y`. + result.num = x.num * y.den + result.den = x.den * y.num + reduce(result) + +proc `/`*[T](x: Rational[T], y: T): Rational[T] = + ## Divide rational `x` by int `y`. + result.num = x.num + result.den = x.den * y + reduce(result) + +proc `/`*[T](x: T, y: Rational[T]): Rational[T] = + ## Divide int `x` by Rational `y`. + result.num = x * y.den + result.den = y.num + reduce(result) + +proc `/=`*[T](x: var Rational[T], y: Rational[T]) = + ## Divide rationals `x` by `y` in place. + x.num *= y.den + x.den *= y.num + reduce(x) + +proc `/=`*[T](x: var Rational[T], y: T) = + ## Divide rational `x` by int `y` in place. + x.den *= y + reduce(x) + +proc cmp*(x, y: Rational): int {.procvar.} = + ## Compares two rationals. + (x - y).num + +proc `<` *(x, y: Rational): bool = + (x - y).num < 0 + +proc `<=` *(x, y: Rational): bool = + (x - y).num <= 0 + +proc `==` *(x, y: Rational): bool = + (x - y).num == 0 + +proc abs*[T](x: Rational[T]): Rational[T] = + result.num = abs x.num + result.den = abs x.den + +proc hash*[T](x: Rational[T]): THash = + ## Computes hash for rational `x` + # reduce first so that hash(x) == hash(y) for x == y + var copy = x + reduce(copy) + + var h: THash = 0 + h = h !& hash(copy.num) + h = h !& hash(copy.den) + result = !$h + +when isMainModule: + var + z = Rational[int](num: 0, den: 1) + o = initRational(num=1, den=1) + a = initRational(1, 2) + b = -1 // -2 + m1 = -1 // 1 + tt = 10 // 2 + + assert( a == a ) + assert( (a-a) == z ) + assert( (a+b) == o ) + assert( (a/b) == o ) + assert( (a*b) == 1 // 4 ) + assert( (3/a) == 6 // 1 ) + assert( (a/3) == 1 // 6 ) + assert( a*b == 1 // 4 ) + assert( tt*z == z ) + assert( 10*a == tt ) + assert( a*10 == tt ) + assert( tt/10 == a ) + assert( a-m1 == 3 // 2 ) + assert( a+m1 == -1 // 2 ) + assert( m1+tt == 16 // 4 ) + assert( m1-tt == 6 // -1 ) + + assert( z < o ) + assert( z <= o ) + assert( z == z ) + assert( cmp(z, o) < 0 ) + assert( cmp(o, z) > 0 ) + + assert( o == o ) + assert( o >= o ) + assert( not(o > o) ) + assert( cmp(o, o) == 0 ) + assert( cmp(z, z) == 0 ) + assert( hash(o) == hash(o) ) + + assert( a == b ) + assert( a >= b ) + assert( not(b > a) ) + assert( cmp(a, b) == 0 ) + assert( hash(a) == hash(b) ) + + var x = 1//3 + + x *= 5//1 + assert( x == 5//3 ) + x += 2 // 9 + assert( x == 17//9 ) + x -= 9//18 + assert( x == 25//18 ) + x /= 1//2 + assert( x == 50//18 ) + + var y = 1//3 + + y *= 4 + assert( y == 4//3 ) + y += 5 + assert( y == 19//3 ) + y -= 2 + assert( y == 13//3 ) + y /= 9 + assert( y == 13//27 ) + + assert toRational(5) == 5//1 + assert abs(toFloat(y) - 0.4814814814814815) < 1.0e-7 + assert toInt(z) == 0 diff --git a/lib/pure/rawsockets.nim b/lib/pure/rawsockets.nim index 62a011999..a30c23ada 100644 --- a/lib/pure/rawsockets.nim +++ b/lib/pure/rawsockets.nim @@ -1,7 +1,7 @@ # # # Nim's Runtime Library -# (c) Copyright 2014 Dominik Picheta +# (c) Copyright 2015 Dominik Picheta # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -39,6 +39,9 @@ export SO_KEEPALIVE, SO_OOBINLINE, SO_REUSEADDR, MSG_PEEK +when defined(macosx): + export SO_NOSIGPIPE + type Port* = distinct uint16 ## port type diff --git a/lib/pure/rawsockets.pretty.nim b/lib/pure/rawsockets.pretty.nim deleted file mode 100644 index 46bfba9e7..000000000 --- a/lib/pure/rawsockets.pretty.nim +++ /dev/null @@ -1,426 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2014 Dominik Picheta -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## This module implements a low-level cross-platform sockets interface. Look -## at the ``net`` module for the higher-level version. - -# TODO: Clean up the exports a bit and everything else in general. - -import unsigned, os - -when hostOS == "solaris": - {.passl: "-lsocket -lnsl".} - -const useWinVersion = defined(Windows) or defined(nimdoc) - -when useWinVersion: - import winlean - export WSAEWOULDBLOCK, WSAECONNRESET, WSAECONNABORTED, WSAENETRESET, - WSAEDISCON, ERROR_NETNAME_DELETED -else: - import posix - export fcntl, F_GETFL, O_NONBLOCK, F_SETFL, EAGAIN, EWOULDBLOCK, MSG_NOSIGNAL, - EINTR, EINPROGRESS, ECONNRESET, EPIPE, ENETRESET - -export SocketHandle, Sockaddr_in, Addrinfo, INADDR_ANY, SockAddr, SockLen, - inet_ntoa, recv, `==`, connect, send, accept, recvfrom, sendto - -export - SO_ERROR, - SOL_SOCKET, - SOMAXCONN, - SO_ACCEPTCONN, SO_BROADCAST, SO_DEBUG, SO_DONTROUTE, - SO_KEEPALIVE, SO_OOBINLINE, SO_REUSEADDR, - MSG_PEEK - -type - Port* = distinct uint16 ## port type - - Domain* = enum ## domain, which specifies the protocol family of the - ## created socket. Other domains than those that are listed - ## here are unsupported. - AF_UNIX, ## for local socket (using a file). Unsupported on Windows. - AF_INET = 2, ## for network protocol IPv4 or - AF_INET6 = 23 ## for network protocol IPv6. - - SockType* = enum ## second argument to `socket` proc - SOCK_STREAM = 1, ## reliable stream-oriented service or Stream Sockets - SOCK_DGRAM = 2, ## datagram service or Datagram Sockets - SOCK_RAW = 3, ## raw protocols atop the network layer. - SOCK_SEQPACKET = 5 ## reliable sequenced packet service - - Protocol* = enum ## third argument to `socket` proc - IPPROTO_TCP = 6, ## Transmission control protocol. - IPPROTO_UDP = 17, ## User datagram protocol. - IPPROTO_IP, ## Internet protocol. Unsupported on Windows. - IPPROTO_IPV6, ## Internet Protocol Version 6. Unsupported on Windows. - IPPROTO_RAW, ## Raw IP Packets Protocol. Unsupported on Windows. - IPPROTO_ICMP ## Control message protocol. Unsupported on Windows. - - Servent* = object ## information about a service - name*: string - aliases*: seq[string] - port*: Port - proto*: string - - Hostent* = object ## information about a given host - name*: string - aliases*: seq[string] - addrtype*: Domain - length*: int - addrList*: seq[string] - -{.deprecated: [TPort: Port, TDomain: Domain, TType: SockType, - TProtocol: Protocol, TServent: Servent, THostent: Hostent].} - -when useWinVersion: - let - osInvalidSocket* = winlean.INVALID_SOCKET - - const - IOCPARM_MASK* = 127 - IOC_IN* = int(-2147483648) - FIONBIO* = IOC_IN.int32 or ((sizeof(int32) and IOCPARM_MASK) shl 16) or - (102 shl 8) or 126 - - proc ioctlsocket*(s: SocketHandle, cmd: clong, - argptr: ptr clong): cint {. - stdcall, importc: "ioctlsocket", dynlib: "ws2_32.dll".} -else: - let - osInvalidSocket* = posix.INVALID_SOCKET - -proc `==`*(a, b: Port): bool {.borrow.} - ## ``==`` for ports. - -proc `$`*(p: Port): string {.borrow.} - ## returns the port number as a string - -proc toInt*(domain: Domain): cint - ## Converts the TDomain enum to a platform-dependent ``cint``. - -proc toInt*(typ: SockType): cint - ## Converts the TType enum to a platform-dependent ``cint``. - -proc toInt*(p: Protocol): cint - ## Converts the TProtocol enum to a platform-dependent ``cint``. - -when not useWinVersion: - proc toInt(domain: Domain): cint = - case domain - of AF_UNIX: result = posix.AF_UNIX - of AF_INET: result = posix.AF_INET - of AF_INET6: result = posix.AF_INET6 - else: discard - - proc toInt(typ: SockType): cint = - case typ - of SOCK_STREAM: result = posix.SOCK_STREAM - of SOCK_DGRAM: result = posix.SOCK_DGRAM - of SOCK_SEQPACKET: result = posix.SOCK_SEQPACKET - of SOCK_RAW: result = posix.SOCK_RAW - else: discard - - proc toInt(p: Protocol): cint = - case p - of IPPROTO_TCP: result = posix.IPPROTO_TCP - of IPPROTO_UDP: result = posix.IPPROTO_UDP - of IPPROTO_IP: result = posix.IPPROTO_IP - of IPPROTO_IPV6: result = posix.IPPROTO_IPV6 - of IPPROTO_RAW: result = posix.IPPROTO_RAW - of IPPROTO_ICMP: result = posix.IPPROTO_ICMP - else: discard - -else: - proc toInt(domain: Domain): cint = - result = toU16(ord(domain)) - - proc toInt(typ: SockType): cint = - result = cint(ord(typ)) - - proc toInt(p: Protocol): cint = - result = cint(ord(p)) - - -proc newRawSocket*(domain: Domain = AF_INET, typ: SockType = SOCK_STREAM, - protocol: Protocol = IPPROTO_TCP): SocketHandle = - ## Creates a new socket; returns `InvalidSocket` if an error occurs. - socket(toInt(domain), toInt(typ), toInt(protocol)) - -proc close*(socket: SocketHandle) = - ## closes a socket. - when useWinVersion: - discard winlean.closesocket(socket) - else: - discard posix.close(socket) - # TODO: These values should not be discarded. An EOS should be raised. - # http://stackoverflow.com/questions/12463473/what-happens-if-you-call-close-on-a-bsd-socket-multiple-times - -proc bindAddr*(socket: SocketHandle, name: ptr SockAddr, namelen: SockLen): cint = - result = bindSocket(socket, name, namelen) - -proc listen*(socket: SocketHandle, backlog = SOMAXCONN): cint {.tags: [ReadIOEffect].} = - ## Marks ``socket`` as accepting connections. - ## ``Backlog`` specifies the maximum length of the - ## queue of pending connections. - when useWinVersion: - result = winlean.listen(socket, cint(backlog)) - else: - result = posix.listen(socket, cint(backlog)) - -proc getAddrInfo*(address: string, port: Port, af: Domain = AF_INET, typ: SockType = SOCK_STREAM, - prot: Protocol = IPPROTO_TCP): ptr AddrInfo = - ## - ## - ## **Warning**: The resulting ``ptr TAddrInfo`` must be freed using ``dealloc``! - var hints: AddrInfo - result = nil - hints.ai_family = toInt(af) - hints.ai_socktype = toInt(typ) - hints.ai_protocol = toInt(prot) - var gaiResult = getaddrinfo(address, $port, addr(hints), result) - if gaiResult != 0'i32: - when useWinVersion: - raiseOSError(osLastError()) - else: - raise newException(OSError, $gai_strerror(gaiResult)) - -proc dealloc*(ai: ptr AddrInfo) = - freeaddrinfo(ai) - -proc ntohl*(x: int32): int32 = - ## Converts 32-bit integers from network to host byte order. - ## On machines where the host byte order is the same as network byte order, - ## this is a no-op; otherwise, it performs a 4-byte swap operation. - when cpuEndian == bigEndian: result = x - else: result = (x shr 24'i32) or - (x shr 8'i32 and 0xff00'i32) or - (x shl 8'i32 and 0xff0000'i32) or - (x shl 24'i32) - -proc ntohs*(x: int16): int16 = - ## Converts 16-bit integers from network to host byte order. On machines - ## where the host byte order is the same as network byte order, this is - ## a no-op; otherwise, it performs a 2-byte swap operation. - when cpuEndian == bigEndian: result = x - else: result = (x shr 8'i16) or (x shl 8'i16) - -proc htonl*(x: int32): int32 = - ## Converts 32-bit integers from host to network byte order. On machines - ## where the host byte order is the same as network byte order, this is - ## a no-op; otherwise, it performs a 4-byte swap operation. - result = rawsockets.ntohl(x) - -proc htons*(x: int16): int16 = - ## Converts 16-bit positive integers from host to network byte order. - ## On machines where the host byte order is the same as network byte - ## order, this is a no-op; otherwise, it performs a 2-byte swap operation. - result = rawsockets.ntohs(x) - -proc getServByName*(name, proto: string): Servent {.tags: [ReadIOEffect].} = - ## Searches the database from the beginning and finds the first entry for - ## which the service name specified by ``name`` matches the s_name member - ## and the protocol name specified by ``proto`` matches the s_proto member. - ## - ## On posix this will search through the ``/etc/services`` file. - when useWinVersion: - var s = winlean.getservbyname(name, proto) - else: - var s = posix.getservbyname(name, proto) - if s == nil: raise newException(OSError, "Service not found.") - result.name = $s.s_name - result.aliases = cstringArrayToSeq(s.s_aliases) - result.port = Port(s.s_port) - result.proto = $s.s_proto - -proc getServByPort*(port: Port, proto: string): Servent {.tags: [ReadIOEffect].} = - ## Searches the database from the beginning and finds the first entry for - ## which the port specified by ``port`` matches the s_port member and the - ## protocol name specified by ``proto`` matches the s_proto member. - ## - ## On posix this will search through the ``/etc/services`` file. - when useWinVersion: - var s = winlean.getservbyport(ze(int16(port)).cint, proto) - else: - var s = posix.getservbyport(ze(int16(port)).cint, proto) - if s == nil: raise newException(OSError, "Service not found.") - result.name = $s.s_name - result.aliases = cstringArrayToSeq(s.s_aliases) - result.port = Port(s.s_port) - result.proto = $s.s_proto - -proc getHostByAddr*(ip: string): Hostent {.tags: [ReadIOEffect].} = - ## This function will lookup the hostname of an IP Address. - var myaddr: InAddr - myaddr.s_addr = inet_addr(ip) - - when useWinVersion: - var s = winlean.gethostbyaddr(addr(myaddr), sizeof(myaddr).cuint, - cint(rawsockets.AF_INET)) - if s == nil: raiseOSError(osLastError()) - else: - var s = posix.gethostbyaddr(addr(myaddr), sizeof(myaddr).Socklen, - cint(posix.AF_INET)) - if s == nil: - raise newException(OSError, $hstrerror(h_errno)) - - result.name = $s.h_name - result.aliases = cstringArrayToSeq(s.h_aliases) - when useWinVersion: - result.addrtype = Domain(s.h_addrtype) - else: - if s.h_addrtype == posix.AF_INET: - result.addrtype = AF_INET - elif s.h_addrtype == posix.AF_INET6: - result.addrtype = AF_INET6 - else: - raise newException(OSError, "unknown h_addrtype") - result.addrList = cstringArrayToSeq(s.h_addr_list) - result.length = int(s.h_length) - -proc getHostByName*(name: string): Hostent {.tags: [ReadIOEffect].} = - ## This function will lookup the IP address of a hostname. - when useWinVersion: - var s = winlean.gethostbyname(name) - else: - var s = posix.gethostbyname(name) - if s == nil: raiseOSError(osLastError()) - result.name = $s.h_name - result.aliases = cstringArrayToSeq(s.h_aliases) - when useWinVersion: - result.addrtype = Domain(s.h_addrtype) - else: - if s.h_addrtype == posix.AF_INET: - result.addrtype = AF_INET - elif s.h_addrtype == posix.AF_INET6: - result.addrtype = AF_INET6 - else: - raise newException(OSError, "unknown h_addrtype") - result.addrList = cstringArrayToSeq(s.h_addr_list) - result.length = int(s.h_length) - -proc getSockName*(socket: SocketHandle): Port = - ## returns the socket's associated port number. - var name: Sockaddr_in - when useWinVersion: - name.sin_family = int16(ord(AF_INET)) - else: - name.sin_family = posix.AF_INET - #name.sin_port = htons(cint16(port)) - #name.sin_addr.s_addr = htonl(INADDR_ANY) - var namelen = sizeof(name).SockLen - if getsockname(socket, cast[ptr SockAddr](addr(name)), - addr(namelen)) == -1'i32: - raiseOSError(osLastError()) - result = Port(rawsockets.ntohs(name.sin_port)) - -proc getSockOptInt*(socket: SocketHandle, level, optname: int): int {. - tags: [ReadIOEffect].} = - ## getsockopt for integer options. - var res: cint - var size = sizeof(res).SockLen - if getsockopt(socket, cint(level), cint(optname), - addr(res), addr(size)) < 0'i32: - raiseOSError(osLastError()) - result = int(res) - -proc setSockOptInt*(socket: SocketHandle, level, optname, optval: int) {. - tags: [WriteIOEffect].} = - ## setsockopt for integer options. - var value = cint(optval) - if setsockopt(socket, cint(level), cint(optname), addr(value), - sizeof(value).SockLen) < 0'i32: - raiseOSError(osLastError()) - -proc setBlocking*(s: SocketHandle, blocking: bool) = - ## Sets blocking mode on socket. - ## - ## Raises EOS on error. - when useWinVersion: - var mode = clong(ord(not blocking)) # 1 for non-blocking, 0 for blocking - if ioctlsocket(s, FIONBIO, addr(mode)) == -1: - raiseOSError(osLastError()) - else: # BSD sockets - var x: int = fcntl(s, F_GETFL, 0) - if x == -1: - raiseOSError(osLastError()) - else: - var mode = if blocking: x and not O_NONBLOCK else: x or O_NONBLOCK - if fcntl(s, F_SETFL, mode) == -1: - raiseOSError(osLastError()) - -proc timeValFromMilliseconds(timeout = 500): Timeval = - if timeout != -1: - var seconds = timeout div 1000 - result.tv_sec = seconds.int32 - result.tv_usec = ((timeout - seconds * 1000) * 1000).int32 - -proc createFdSet(fd: var FdSet, s: seq[SocketHandle], m: var int) = - FD_ZERO(fd) - for i in items(s): - m = max(m, int(i)) - fdSet(i, fd) - -proc pruneSocketSet(s: var seq[SocketHandle], fd: var FdSet) = - var i = 0 - var L = s.len - while i < L: - if FD_ISSET(s[i], fd) == 0'i32: - s[i] = s[L-1] - dec(L) - else: - inc(i) - setLen(s, L) - -proc select*(readfds: var seq[SocketHandle], timeout = 500): int = - ## Traditional select function. This function will return the number of - ## sockets that are ready to be read from, written to, or which have errors. - ## If there are none; 0 is returned. - ## ``Timeout`` is in miliseconds and -1 can be specified for no timeout. - ## - ## A socket is removed from the specific ``seq`` when it has data waiting to - ## be read/written to or has errors (``exceptfds``). - var tv {.noInit.}: Timeval = timeValFromMilliseconds(timeout) - - var rd: FdSet - var m = 0 - createFdSet((rd), readfds, m) - - if timeout != -1: - result = int(select(cint(m+1), addr(rd), nil, nil, addr(tv))) - else: - result = int(select(cint(m+1), addr(rd), nil, nil, nil)) - - pruneSocketSet(readfds, (rd)) - -proc selectWrite*(writefds: var seq[SocketHandle], - timeout = 500): int {.tags: [ReadIOEffect].} = - ## When a socket in ``writefds`` is ready to be written to then a non-zero - ## value will be returned specifying the count of the sockets which can be - ## written to. The sockets which can be written to will also be removed - ## from ``writefds``. - ## - ## ``timeout`` is specified in miliseconds and ``-1`` can be specified for - ## an unlimited time. - var tv {.noInit.}: Timeval = timeValFromMilliseconds(timeout) - - var wr: FdSet - var m = 0 - createFdSet((wr), writefds, m) - - if timeout != -1: - result = int(select(cint(m+1), nil, addr(wr), nil, addr(tv))) - else: - result = int(select(cint(m+1), nil, addr(wr), nil, nil)) - - pruneSocketSet(writefds, (wr)) - -when defined(Windows): - var wsa: WSAData - if wsaStartup(0x0101'i16, addr wsa) != 0: raiseOSError(osLastError()) diff --git a/lib/pure/redis.nim b/lib/pure/redis.nim index 52d81b3a4..aa2e0f9bd 100644 --- a/lib/pure/redis.nim +++ b/lib/pure/redis.nim @@ -285,6 +285,30 @@ proc keys*(r: Redis, pattern: string): RedisList = r.sendCommand("KEYS", pattern) return r.readArray() +proc scan*(r: Redis, cursor: var BiggestInt): RedisList = + ## Find all keys matching the given pattern and yield it to client in portions + ## using default Redis values for MATCH and COUNT parameters + r.sendCommand("SCAN", $cursor) + let reply = r.readArray() + cursor = strutils.parseBiggestInt(reply[0]) + return reply[1..high(reply)] + +proc scan*(r: Redis, cursor: var BiggestInt, pattern: string): RedisList = + ## Find all keys matching the given pattern and yield it to client in portions + ## using cursor as a client query identifier. Using default Redis value for COUNT argument + r.sendCommand("SCAN", $cursor, ["MATCH", pattern]) + let reply = r.readArray() + cursor = strutils.parseBiggestInt(reply[0]) + return reply[1..high(reply)] + +proc scan*(r: Redis, cursor: var BiggestInt, pattern: string, count: int): RedisList = + ## Find all keys matching the given pattern and yield it to client in portions + ## using cursor as a client query identifier. + r.sendCommand("SCAN", $cursor, ["MATCH", pattern, "COUNT", $count]) + let reply = r.readArray() + cursor = strutils.parseBiggestInt(reply[0]) + return reply[1..high(reply)] + proc move*(r: Redis, key: string, db: int): bool = ## Move a key to another database. Returns `true` on a successful move. r.sendCommand("MOVE", key, $db) @@ -798,6 +822,22 @@ proc zunionstore*(r: Redis, destination: string, numkeys: string, return r.readInteger() +# HyperLogLog + +proc pfadd*(r: Redis, key: string, elements: varargs[string]): RedisInteger = + ## Add variable number of elements into special 'HyperLogLog' set type + r.sendCommand("PFADD", key, elements) + return r.readInteger() + +proc pfcount*(r: Redis, key: string): RedisInteger = + ## Count approximate number of elements in 'HyperLogLog' + r.sendCommand("PFCOUNT", key) + return r.readInteger() + +proc pfmerge*(r: Redis, destination: string, sources: varargs[string]) = + ## Merge several source HyperLogLog's into one specified by destKey + r.sendCommand("PFMERGE", destination, sources) + raiseNoOK(r.readStatus(), r.pipeline.enabled) # Pub/Sub @@ -1040,7 +1080,7 @@ proc assertListsIdentical(listA, listB: seq[string]) = assert(item == listB[i]) i = i + 1 -when isMainModule: +when not defined(testing) and isMainModule: when false: var r = open() diff --git a/lib/pure/romans.nim b/lib/pure/romans.nim index 79fb75526..0c182843a 100644 --- a/lib/pure/romans.nim +++ b/lib/pure/romans.nim @@ -44,16 +44,13 @@ proc decimalToRoman*(number: range[1..3_999]): string = ("XC", 90), ("L", 50), ("XL", 40), ("X", 10), ("IX", 9), ("V", 5), ("IV", 4), ("I", 1)] result = "" - var decVal = number + var decVal: int = number for key, val in items(romanComposites): while decVal >= val: dec(decVal, val) result.add(key) when isMainModule: - import math - randomize() - for i in 1 .. 100: - var rnd = 1 + random(3990) - assert rnd == rnd.decimalToRoman.romanToDecimal + for i in 1 .. 3_999: + assert i == i.decimalToRoman.romanToDecimal diff --git a/lib/pure/ropes.nim b/lib/pure/ropes.nim index 995dff2aa..5c7fedfe3 100644 --- a/lib/pure/ropes.nim +++ b/lib/pure/ropes.nim @@ -43,7 +43,7 @@ proc isConc(r: Rope): bool {.inline.} = return isNil(r.data) # Note that the left and right pointers are not needed for leafs. # Leaves have relatively high memory overhead (~30 bytes on a 32 # bit machine) and we produce many of them. This is why we cache and -# share leafs accross different rope trees. +# share leafs across different rope trees. # To cache them they are inserted in another tree, a splay tree for best # performance. But for the caching tree we use the leaf's left and right # pointers. @@ -52,9 +52,9 @@ proc len*(a: Rope): int {.rtl, extern: "nro$1".} = ## the rope's length if a == nil: result = 0 else: result = a.length - + proc newRope(): Rope = new(result) -proc newRope(data: string): Rope = +proc newRope(data: string): Rope = new(result) result.length = len(data) result.data = data @@ -65,18 +65,18 @@ var when countCacheMisses: var misses, hits: int - -proc splay(s: string, tree: Rope, cmpres: var int): Rope = + +proc splay(s: string, tree: Rope, cmpres: var int): Rope = var c: int var t = tree N.left = nil N.right = nil # reset to nil var le = N var r = N - while true: + while true: c = cmp(s, t.data) - if c < 0: - if (t.left != nil) and (s < t.left.data): + if c < 0: + if (t.left != nil) and (s < t.left.data): var y = t.left t.left = y.right y.right = t @@ -85,8 +85,8 @@ proc splay(s: string, tree: Rope, cmpres: var int): Rope = r.left = t r = t t = t.left - elif c > 0: - if (t.right != nil) and (s > t.right.data): + elif c > 0: + if (t.right != nil) and (s > t.right.data): var y = t.right t.right = y.left y.left = t @@ -95,8 +95,8 @@ proc splay(s: string, tree: Rope, cmpres: var int): Rope = le.right = t le = t t = t.right - else: - break + else: + break cmpres = c le.right = t.left r.left = t.right @@ -104,50 +104,50 @@ proc splay(s: string, tree: Rope, cmpres: var int): Rope = t.right = N.left result = t -proc insertInCache(s: string, tree: Rope): Rope = +proc insertInCache(s: string, tree: Rope): Rope = var t = tree - if t == nil: + if t == nil: result = newRope(s) when countCacheMisses: inc(misses) return var cmp: int t = splay(s, t, cmp) - if cmp == 0: + if cmp == 0: # We get here if it's already in the Tree # Don't add it again result = t when countCacheMisses: inc(hits) - else: + else: when countCacheMisses: inc(misses) result = newRope(s) - if cmp < 0: + if cmp < 0: result.left = t.left result.right = t t.left = nil - else: + else: # i > t.item: result.right = t.right result.left = t t.right = nil proc rope*(s: string): Rope {.rtl, extern: "nro$1Str".} = - ## Converts a string to a rope. - if s.len == 0: + ## Converts a string to a rope. + if s.len == 0: result = nil - elif cacheEnabled: + elif cacheEnabled: result = insertInCache(s, cache) cache = result - else: + else: result = newRope(s) - -proc rope*(i: BiggestInt): Rope {.rtl, extern: "nro$1BiggestInt".} = - ## Converts an int to a rope. + +proc rope*(i: BiggestInt): Rope {.rtl, extern: "nro$1BiggestInt".} = + ## Converts an int to a rope. result = rope($i) proc rope*(f: BiggestFloat): Rope {.rtl, extern: "nro$1BiggestFloat".} = - ## Converts a float to a rope. + ## Converts a float to a rope. result = rope($f) - + proc enableCache*() {.rtl, extern: "nro$1".} = ## Enables the caching of leaves. This reduces the memory footprint at ## the cost of runtime efficiency. @@ -160,9 +160,9 @@ proc disableCache*() {.rtl, extern: "nro$1".} = proc `&`*(a, b: Rope): Rope {.rtl, extern: "nroConcRopeRope".} = ## the concatenation operator for ropes. - if a == nil: + if a == nil: result = b - elif b == nil: + elif b == nil: result = a else: result = newRope() @@ -177,16 +177,16 @@ proc `&`*(a, b: Rope): Rope {.rtl, extern: "nroConcRopeRope".} = else: result.left = a result.right = b - -proc `&`*(a: Rope, b: string): Rope {.rtl, extern: "nroConcRopeStr".} = + +proc `&`*(a: Rope, b: string): Rope {.rtl, extern: "nroConcRopeStr".} = ## the concatenation operator for ropes. result = a & rope(b) - -proc `&`*(a: string, b: Rope): Rope {.rtl, extern: "nroConcStrRope".} = + +proc `&`*(a: string, b: Rope): Rope {.rtl, extern: "nroConcStrRope".} = ## the concatenation operator for ropes. result = rope(a) & b - -proc `&`*(a: openArray[Rope]): Rope {.rtl, extern: "nroConcOpenArray".} = + +proc `&`*(a: openArray[Rope]): Rope {.rtl, extern: "nroConcOpenArray".} = ## the concatenation operator for an openarray of ropes. for i in countup(0, high(a)): result = result & a[i] @@ -219,7 +219,7 @@ iterator leaves*(r: Rope): string = ## iterates over any leaf string in the rope `r`. if r != nil: var stack = @[r] - while stack.len > 0: + while stack.len > 0: var it = stack.pop while isConc(it): stack.add(it.right) @@ -227,7 +227,7 @@ iterator leaves*(r: Rope): string = assert(it != nil) assert(it.data != nil) yield it.data - + iterator items*(r: Rope): char = ## iterates over any character in the rope `r`. for s in leaves(r): @@ -237,7 +237,7 @@ proc write*(f: File, r: Rope) {.rtl, extern: "nro$1".} = ## writes a rope to a file. for s in leaves(r): write(f, s) -proc `$`*(r: Rope): string {.rtl, extern: "nroToString".}= +proc `$`*(r: Rope): string {.rtl, extern: "nroToString".}= ## converts a rope back to a string. result = newString(r.len) setLen(result, 0) @@ -251,25 +251,25 @@ when false: new(result) result.length = -idx - proc compileFrmt(frmt: string): Rope = + proc compileFrmt(frmt: string): Rope = var i = 0 var length = len(frmt) result = nil var num = 0 - while i < length: - if frmt[i] == '$': + while i < length: + if frmt[i] == '$': inc(i) case frmt[i] - of '$': + of '$': add(result, "$") inc(i) - of '#': + of '#': inc(i) add(result, compiledArg(num+1)) inc(num) - of '0'..'9': + of '0'..'9': var j = 0 - while true: + while true: j = j * 10 + ord(frmt[i]) - ord('0') inc(i) if frmt[i] notin {'0'..'9'}: break @@ -285,13 +285,13 @@ when false: add(s, compiledArg(j)) else: raise newException(EInvalidValue, "invalid format string") var start = i - while i < length: + while i < length: if frmt[i] != '$': inc(i) - else: break - if i - 1 >= start: + else: break + if i - 1 >= start: add(result, substr(frmt, start, i-1)) - -proc `%`*(frmt: string, args: openArray[Rope]): Rope {. + +proc `%`*(frmt: string, args: openArray[Rope]): Rope {. rtl, extern: "nroFormat".} = ## `%` substitution operator for ropes. Does not support the ``$identifier`` ## nor ``${identifier}`` notations. @@ -299,23 +299,23 @@ proc `%`*(frmt: string, args: openArray[Rope]): Rope {. var length = len(frmt) result = nil var num = 0 - while i < length: - if frmt[i] == '$': + while i < length: + if frmt[i] == '$': inc(i) case frmt[i] - of '$': + of '$': add(result, "$") inc(i) - of '#': + of '#': inc(i) add(result, args[num]) inc(num) - of '0'..'9': + of '0'..'9': var j = 0 - while true: + while true: j = j * 10 + ord(frmt[i]) - ord('0') inc(i) - if frmt[i] notin {'0'..'9'}: break + if frmt[i] notin {'0'..'9'}: break add(result, args[j-1]) of '{': inc(i) @@ -325,13 +325,14 @@ proc `%`*(frmt: string, args: openArray[Rope]): Rope {. inc(i) if frmt[i] == '}': inc(i) else: raise newException(ValueError, "invalid format string") + add(result, args[j-1]) else: raise newException(ValueError, "invalid format string") var start = i - while i < length: + while i < length: if frmt[i] != '$': inc(i) - else: break - if i - 1 >= start: + else: break + if i - 1 >= start: add(result, substr(frmt, start, i - 1)) proc addf*(c: var Rope, frmt: string, args: openArray[Rope]) {. @@ -339,29 +340,46 @@ proc addf*(c: var Rope, frmt: string, args: openArray[Rope]) {. ## shortcut for ``add(c, frmt % args)``. add(c, frmt % args) +const + bufSize = 1024 # 1 KB is reasonable + proc equalsFile*(r: Rope, f: File): bool {.rtl, extern: "nro$1File".} = ## returns true if the contents of the file `f` equal `r`. - var bufSize = 1024 # reasonable start value - var buf = alloc(bufSize) + var + buf: array[bufSize, char] + bpos = buf.len + blen = buf.len + for s in leaves(r): - if s.len > bufSize: - bufSize = max(bufSize * 2, s.len) - buf = realloc(buf, bufSize) - var readBytes = readBuffer(f, buf, s.len) - result = readBytes == s.len and equalMem(buf, cstring(s), s.len) - if not result: break - if result: - result = readBuffer(f, buf, 1) == 0 # really at the end of file? - dealloc(buf) + var spos = 0 + let slen = s.len + while spos < slen: + if bpos == blen: + # Read more data + bpos = 0 + blen = readBuffer(f, addr(buf[0]), buf.len) + if blen == 0: # no more data in file + result = false + return + let n = min(blen - bpos, slen - spos) + # TODO There's gotta be a better way of comparing here... + if not equalMem(addr(buf[bpos]), + cast[pointer](cast[int](cstring(s))+spos), n): + result = false + return + spos += n + bpos += n + + result = readBuffer(f, addr(buf[0]), 1) == 0 # check that we've read all -proc equalsFile*(r: Rope, f: string): bool {.rtl, extern: "nro$1Str".} = +proc equalsFile*(r: Rope, filename: string): bool {.rtl, extern: "nro$1Str".} = ## returns true if the contents of the file `f` equal `r`. If `f` does not ## exist, false is returned. - var bin: File - result = open(bin, f) + var f: File + result = open(f, filename) if result: - result = equalsFile(r, bin) - close(bin) + result = equalsFile(r, f) + close(f) new(N) # init dummy node for splay algorithm diff --git a/lib/pure/selectors.nim b/lib/pure/selectors.nim index 1c988c609..6901ecf58 100644 --- a/lib/pure/selectors.nim +++ b/lib/pure/selectors.nim @@ -1,7 +1,7 @@ # # # Nim's Runtime Library -# (c) Copyright 2014 Dominik Picheta +# (c) Copyright 2015 Dominik Picheta # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -23,7 +23,7 @@ proc `$`*(x: SocketHandle): string {.borrow.} type Event* = enum - EvRead, EvWrite + EvRead, EvWrite, EvError SelectorKey* = ref object fd*: SocketHandle @@ -35,7 +35,7 @@ type when defined(nimdoc): type Selector* = ref object - ## An object which holds file descripters to be checked for read/write + ## An object which holds file descriptors to be checked for read/write ## status. fds: Table[SocketHandle, SelectorKey] @@ -48,12 +48,21 @@ when defined(nimdoc): events: set[Event]): SelectorKey {.discardable.} = ## Updates the events which ``fd`` wants notifications for. + proc unregister*(s: Selector, fd: SocketHandle): SelectorKey {.discardable.} = + ## Unregisters file descriptor ``fd`` from selector ``s``. + + proc close*(s: Selector) = + ## Closes the selector + proc select*(s: Selector, timeout: int): seq[ReadyInfo] = ## The ``events`` field of the returned ``key`` contains the original events ## for which the ``fd`` was bound. This is contrary to the ``events`` field ## of the ``TReadyInfo`` tuple which determines which events are ready ## on the ``fd``. + proc newSelector*(): Selector = + ## Creates a new selector + proc contains*(s: Selector, fd: SocketHandle): bool = ## Determines whether selector contains a file descriptor. @@ -78,8 +87,6 @@ elif defined(linux): proc register*(s: Selector, fd: SocketHandle, events: set[Event], data: RootRef): SelectorKey {.discardable.} = - ## Registers file descriptor ``fd`` to selector ``s`` with a set of TEvent - ## ``events``. var event = createEventStruct(events, fd) if events != {}: if epoll_ctl(s.epollFD, EPOLL_CTL_ADD, fd, addr(event)) != 0: @@ -92,7 +99,6 @@ elif defined(linux): proc update*(s: Selector, fd: SocketHandle, events: set[Event]): SelectorKey {.discardable.} = - ## Updates the events which ``fd`` wants notifications for. if s.fds[fd].events != events: if events == {}: # This fd is idle -- it should not be registered to epoll. @@ -146,12 +152,17 @@ elif defined(linux): ## on the ``fd``. result = @[] let evNum = epoll_wait(s.epollFD, addr s.events[0], 64.cint, timeout.cint) - if evNum < 0: raiseOSError(osLastError()) + if evNum < 0: + let err = osLastError() + if err.cint == EINTR: + return @[] + raiseOSError(osLastError()) if evNum == 0: return @[] for i in 0 .. <evNum: let fd = s.events[i].data.fd.SocketHandle var evSet: set[Event] = {} + if (s.events[i].events and EPOLLERR) != 0 or (s.events[i].events and EPOLLHUP) != 0: evSet = evSet + {EvError} if (s.events[i].events and EPOLLIN) != 0: evSet = evSet + {EvRead} if (s.events[i].events and EPOLLOUT) != 0: evSet = evSet + {EvWrite} let selectorKey = s.fds[fd] @@ -199,7 +210,6 @@ elif not defined(nimdoc): proc update*(s: Selector, fd: SocketHandle, events: set[Event]): SelectorKey {.discardable.} = - ## Updates the events which ``fd`` wants notifications for. if not s.fds.hasKey(fd): raise newException(ValueError, "File descriptor not found.") @@ -286,7 +296,7 @@ proc contains*(s: Selector, key: SelectorKey): bool = TReadyInfo: ReadyInfo, PSelector: Selector].} -when isMainModule and not defined(nimdoc): +when not defined(testing) and isMainModule and not defined(nimdoc): # Select() import sockets type @@ -294,7 +304,7 @@ when isMainModule and not defined(nimdoc): sock: Socket var sock = socket() - if sock == sockets.InvalidSocket: raiseOSError(osLastError()) + if sock == sockets.invalidSocket: raiseOSError(osLastError()) #sock.setBlocking(false) sock.connect("irc.freenode.net", Port(6667)) diff --git a/lib/pure/smtp.nim b/lib/pure/smtp.nim index 26f0c9591..c1bc259a5 100644 --- a/lib/pure/smtp.nim +++ b/lib/pure/smtp.nim @@ -238,7 +238,7 @@ proc sendMail*(smtp: AsyncSmtp, fromAddr: string, await smtp.sock.send("MAIL FROM:<" & fromAddr & ">\c\L") await smtp.checkReply("250") for address in items(toAddrs): - await smtp.sock.send("RCPT TO:<" & smtp.address & ">\c\L") + await smtp.sock.send("RCPT TO:<" & address & ">\c\L") await smtp.checkReply("250") # Send the message @@ -253,7 +253,7 @@ proc close*(smtp: AsyncSmtp) {.async.} = await smtp.sock.send("QUIT\c\L") smtp.sock.close() -when isMainModule: +when not defined(testing) and isMainModule: #var msg = createMessage("Test subject!", # "Hello, my name is dom96.\n What\'s yours?", @["dominik@localhost"]) #echo(msg) diff --git a/lib/pure/smtp.nimrod.cfg b/lib/pure/smtp.nim.cfg index 521e21de4..521e21de4 100644 --- a/lib/pure/smtp.nimrod.cfg +++ b/lib/pure/smtp.nim.cfg diff --git a/lib/pure/sockets.nim b/lib/pure/sockets.nim index 11eeefcb9..3afb545c8 100644 --- a/lib/pure/sockets.nim +++ b/lib/pure/sockets.nim @@ -657,6 +657,8 @@ proc close*(socket: Socket) = when defined(ssl): if socket.isSSL: discard SSLShutdown(socket.sslHandle) + SSLFree(socket.sslHandle) + socket.sslHandle = nil proc getServByName*(name, proto: string): Servent {.tags: [ReadIOEffect].} = ## Searches the database from the beginning and finds the first entry for @@ -851,7 +853,7 @@ proc connectAsync*(socket: Socket, name: string, port = Port(0), af: Domain = AF_INET) {.tags: [ReadIOEffect].} = ## A variant of ``connect`` for non-blocking sockets. ## - ## This procedure will immediatelly return, it will not block until a connection + ## This procedure will immediately return, it will not block until a connection ## is made. It is up to the caller to make sure the connection has been established ## by checking (using ``select``) whether the socket is writeable. ## @@ -1467,7 +1469,7 @@ proc recvAsync*(socket: Socket, s: var TaintedString): bool {. of SSL_ERROR_ZERO_RETURN: raiseSslError("TLS/SSL connection failed to initiate, socket closed prematurely.") of SSL_ERROR_WANT_CONNECT, SSL_ERROR_WANT_ACCEPT: - raiseSslError("Unexpected error occured.") # This should just not happen. + raiseSslError("Unexpected error occurred.") # This should just not happen. of SSL_ERROR_WANT_WRITE, SSL_ERROR_WANT_READ: return false of SSL_ERROR_WANT_X509_LOOKUP: @@ -1610,7 +1612,7 @@ proc sendAsync*(socket: Socket, data: string): int {.tags: [WriteIOEffect].} = of SSL_ERROR_ZERO_RETURN: raiseSslError("TLS/SSL connection failed to initiate, socket closed prematurely.") of SSL_ERROR_WANT_CONNECT, SSL_ERROR_WANT_ACCEPT: - raiseSslError("Unexpected error occured.") # This should just not happen. + raiseSslError("Unexpected error occurred.") # This should just not happen. of SSL_ERROR_WANT_WRITE, SSL_ERROR_WANT_READ: return 0 of SSL_ERROR_WANT_X509_LOOKUP: diff --git a/lib/pure/streams.nim b/lib/pure/streams.nim index 31aa7497d..e706f2016 100644 --- a/lib/pure/streams.nim +++ b/lib/pure/streams.nim @@ -122,41 +122,41 @@ proc read[T](s: Stream, result: var T) = raise newEIO("cannot read from stream") proc readChar*(s: Stream): char = - ## reads a char from the stream `s`. Raises `EIO` if an error occured. + ## reads a char from the stream `s`. Raises `EIO` if an error occurred. ## Returns '\0' as an EOF marker. if readData(s, addr(result), sizeof(result)) != 1: result = '\0' proc readBool*(s: Stream): bool = - ## reads a bool from the stream `s`. Raises `EIO` if an error occured. + ## reads a bool from the stream `s`. Raises `EIO` if an error occurred. read(s, result) proc readInt8*(s: Stream): int8 = - ## reads an int8 from the stream `s`. Raises `EIO` if an error occured. + ## reads an int8 from the stream `s`. Raises `EIO` if an error occurred. read(s, result) proc readInt16*(s: Stream): int16 = - ## reads an int16 from the stream `s`. Raises `EIO` if an error occured. + ## reads an int16 from the stream `s`. Raises `EIO` if an error occurred. read(s, result) proc readInt32*(s: Stream): int32 = - ## reads an int32 from the stream `s`. Raises `EIO` if an error occured. + ## reads an int32 from the stream `s`. Raises `EIO` if an error occurred. read(s, result) proc readInt64*(s: Stream): int64 = - ## reads an int64 from the stream `s`. Raises `EIO` if an error occured. + ## reads an int64 from the stream `s`. Raises `EIO` if an error occurred. read(s, result) proc readFloat32*(s: Stream): float32 = - ## reads a float32 from the stream `s`. Raises `EIO` if an error occured. + ## reads a float32 from the stream `s`. Raises `EIO` if an error occurred. read(s, result) proc readFloat64*(s: Stream): float64 = - ## reads a float64 from the stream `s`. Raises `EIO` if an error occured. + ## reads a float64 from the stream `s`. Raises `EIO` if an error occurred. read(s, result) proc readStr*(s: Stream, length: int): TaintedString = ## reads a string of length `length` from the stream `s`. Raises `EIO` if - ## an error occured. + ## an error occurred. result = newString(length).TaintedString var L = readData(s, addr(string(result)[0]), length) if L != length: setLen(result.string, L) @@ -183,7 +183,7 @@ proc readLine*(s: Stream, line: var TaintedString): bool = proc readLine*(s: Stream): TaintedString = ## Reads a line from a stream `s`. Note: This is not very efficient. Raises - ## `EIO` if an error occured. + ## `EIO` if an error occurred. result = TaintedString"" while true: var c = readChar(s) @@ -224,10 +224,12 @@ proc ssReadData(s: Stream, buffer: pointer, bufLen: int): int = proc ssWriteData(s: Stream, buffer: pointer, bufLen: int) = var s = StringStream(s) - if bufLen > 0: - setLen(s.data, s.data.len + bufLen) - copyMem(addr(s.data[s.pos]), buffer, bufLen) - inc(s.pos, bufLen) + if bufLen <= 0: + return + if s.pos + bufLen > s.data.len: + setLen(s.data, s.pos + bufLen) + copyMem(addr(s.data[s.pos]), buffer, bufLen) + inc(s.pos, bufLen) proc ssClose(s: Stream) = var s = StringStream(s) @@ -284,7 +286,7 @@ when not defined(js): proc newFileStream*(filename: string, mode: FileMode): FileStream = ## creates a new stream from the file named `filename` with the mode `mode`. ## If the file cannot be opened, nil is returned. See the `system - ## <system.html>`_ module for a list of available TFileMode enums. + ## <system.html>`_ module for a list of available FileMode enums. var f: File if open(f, filename, mode): result = newFileStream(f) diff --git a/lib/pure/strtabs.nim b/lib/pure/strtabs.nim index 5b7149d8e..7fdd994f2 100644 --- a/lib/pure/strtabs.nim +++ b/lib/pure/strtabs.nim @@ -112,7 +112,7 @@ proc `[]`*(t: StringTableRef, key: string): string {.rtl, extern: "nstGet".} = proc mget*(t: StringTableRef, key: string): var string {. rtl, extern: "nstTake".} = ## retrieves the location at ``t[key]``. If `key` is not in `t`, the - ## ``EInvalidKey`` exception is raised. + ## ``KeyError`` exception is raised. var index = rawGet(t, key) if index >= 0: result = t.data[index].val else: raise newException(KeyError, "key does not exist: " & key) @@ -158,7 +158,7 @@ proc getValue(t: StringTableRef, flags: set[FormatFlag], key: string): string = else: result = "" if result.len == 0: if useKey in flags: result = '$' & key - elif not (useEmpty in flags): raiseFormatException(key) + elif useEmpty notin flags: raiseFormatException(key) proc newStringTable*(mode: StringTableMode): StringTableRef {. rtl, extern: "nst$1".} = @@ -168,6 +168,12 @@ proc newStringTable*(mode: StringTableMode): StringTableRef {. result.counter = 0 newSeq(result.data, startSize) +proc clear*(s: StringTableRef, mode: StringTableMode) = + ## resets a string table to be empty again. + s.mode = mode + s.counter = 0 + s.data.setLen(startSize) + proc newStringTable*(keyValuePairs: varargs[string], mode: StringTableMode): StringTableRef {. rtl, extern: "nst$1WithPairs".} = @@ -227,7 +233,7 @@ proc `$`*(t: StringTableRef): string {.rtl, extern: "nstDollar".} = result = "{:}" else: result = "{" - for key, val in pairs(t): + for key, val in pairs(t): if result.len > 1: result.add(", ") result.add(key) result.add(": ") diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index 4f449cb0e..59cebf7fa 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -386,7 +386,7 @@ proc split*(s: string, sep: string): seq[string] {.noSideEffect, ## `split iterator <#split.i,string,string>`_. accumulateResult(split(s, sep)) -proc toHex*(x: BiggestInt, len: int): string {.noSideEffect, +proc toHex*(x: BiggestInt, len: Positive): string {.noSideEffect, rtl, extern: "nsuToHex".} = ## Converts `x` to its hexadecimal representation. ## @@ -395,13 +395,15 @@ proc toHex*(x: BiggestInt, len: int): string {.noSideEffect, const HexChars = "0123456789ABCDEF" var - shift: BiggestInt + n = x result = newString(len) for j in countdown(len-1, 0): - result[j] = HexChars[toU32(x shr shift) and 0xF'i32] - shift = shift + 4 + result[j] = HexChars[n and 0xF] + n = n shr 4 + # handle negative overflow + if n == 0 and x < 0: n = -1 -proc intToStr*(x: int, minchars: int = 1): string {.noSideEffect, +proc intToStr*(x: int, minchars: Positive = 1): string {.noSideEffect, rtl, extern: "nsuIntToStr".} = ## Converts `x` to its decimal representation. ## @@ -497,30 +499,51 @@ proc parseEnum*[T: enum](s: string, default: T): T = return e result = default -proc repeatChar*(count: int, c: char = ' '): string {.noSideEffect, +proc repeat*(c: char, count: Natural): string {.noSideEffect, rtl, extern: "nsuRepeatChar".} = ## Returns a string of length `count` consisting only of ## the character `c`. You can use this proc to left align strings. Example: ## ## .. code-block:: nim + ## proc tabexpand(indent: int, text: string, tabsize: int = 4) = + ## echo '\t'.repeat(indent div tabsize), ' '.repeat(indent mod tabsize), text + ## + ## tabexpand(4, "At four") + ## tabexpand(5, "At five") + ## tabexpand(6, "At six") + result = newString(count) + for i in 0..count-1: result[i] = c + +proc repeat*(s: string, n: Natural): string {.noSideEffect, + rtl, extern: "nsuRepeatStr".} = + ## Returns String `s` concatenated `n` times. Example: + ## + ## .. code-block:: nim + ## echo "+++ STOP ".repeat(4), "+++" + result = newStringOfCap(n * s.len) + for i in 1..n: result.add(s) + +template spaces*(n: Natural): string = repeat(' ',n) + ## Returns a String with `n` space characters. You can use this proc + ## to left align strings. Example: + ## + ## .. code-block:: nim ## let ## width = 15 ## text1 = "Hello user!" ## text2 = "This is a very long string" - ## echo text1 & repeatChar(max(0, width - text1.len)) & "|" - ## echo text2 & repeatChar(max(0, width - text2.len)) & "|" - result = newString(count) - for i in 0..count-1: result[i] = c + ## echo text1 & spaces(max(0, width - text1.len)) & "|" + ## echo text2 & spaces(max(0, width - text2.len)) & "|" -proc repeatStr*(count: int, s: string): string {.noSideEffect, - rtl, extern: "nsuRepeatStr".} = - ## Returns `s` concatenated `count` times. - result = newStringOfCap(count*s.len) - for i in 0..count-1: result.add(s) +proc repeatChar*(count: Natural, c: char = ' '): string {.deprecated.} = repeat(c, count) + ## deprecated: use repeat() or spaces() -proc align*(s: string, count: int, padding = ' '): string {. +proc repeatStr*(count: Natural, s: string): string {.deprecated.} = repeat(s, count) + ## deprecated: use repeat(string, count) or string.repeat(count) + +proc align*(s: string, count: Natural, padding = ' '): string {. noSideEffect, rtl, extern: "nsuAlignString".} = - ## Aligns a string `s` with `padding`, so that is of length `count`. + ## Aligns a string `s` with `padding`, so that it is of length `count`. ## ## `padding` characters (by default spaces) are added before `s` resulting in ## right alignment. If ``s.len >= count``, no spaces are added and `s` is @@ -659,7 +682,7 @@ proc endsWith*(s, suffix: string): bool {.noSideEffect, inc(i) if suffix[i] == '\0': return true -proc continuesWith*(s, substr: string, start: int): bool {.noSideEffect, +proc continuesWith*(s, substr: string, start: Natural): bool {.noSideEffect, rtl, extern: "nsuContinuesWith".} = ## Returns true iff ``s`` continues with ``substr`` at position ``start``. ## @@ -670,8 +693,8 @@ proc continuesWith*(s, substr: string, start: int): bool {.noSideEffect, if s[i+start] != substr[i]: return false inc(i) -proc addSep*(dest: var string, sep = ", ", startLen = 0) {.noSideEffect, - inline.} = +proc addSep*(dest: var string, sep = ", ", startLen: Natural = 0) + {.noSideEffect, inline.} = ## Adds a separator to `dest` only if its length is bigger than `startLen`. ## ## A shorthand for: @@ -761,7 +784,7 @@ proc findAux(s, sub: string, start: int, a: SkipTable): int = inc(j, a[s[j+m]]) return -1 -proc find*(s, sub: string, start: int = 0): int {.noSideEffect, +proc find*(s, sub: string, start: Natural = 0): int {.noSideEffect, rtl, extern: "nsuFindStr".} = ## Searches for `sub` in `s` starting at position `start`. ## @@ -770,7 +793,7 @@ proc find*(s, sub: string, start: int = 0): int {.noSideEffect, preprocessSub(sub, a) result = findAux(s, sub, start, a) -proc find*(s: string, sub: char, start: int = 0): int {.noSideEffect, +proc find*(s: string, sub: char, start: Natural = 0): int {.noSideEffect, rtl, extern: "nsuFindChar".} = ## Searches for `sub` in `s` starting at position `start`. ## @@ -779,7 +802,7 @@ proc find*(s: string, sub: char, start: int = 0): int {.noSideEffect, if sub == s[i]: return i return -1 -proc find*(s: string, chars: set[char], start: int = 0): int {.noSideEffect, +proc find*(s: string, chars: set[char], start: Natural = 0): int {.noSideEffect, rtl, extern: "nsuFindCharSet".} = ## Searches for `chars` in `s` starting at position `start`. ## @@ -815,8 +838,8 @@ proc rfind*(s: string, sub: char, start: int = -1): int {.noSideEffect, proc count*(s: string, sub: string, overlapping: bool = false): int {.noSideEffect, rtl, extern: "nsuCountString".} = - ## Count the occurences of a substring `sub` in the string `s`. - ## Overlapping occurences of `sub` only count when `overlapping` + ## Count the occurrences of a substring `sub` in the string `s`. + ## Overlapping occurrences of `sub` only count when `overlapping` ## is set to true. var i = 0 while true: @@ -831,14 +854,14 @@ proc count*(s: string, sub: string, overlapping: bool = false): int {.noSideEffe proc count*(s: string, sub: char): int {.noSideEffect, rtl, extern: "nsuCountChar".} = - ## Count the occurences of the character `sub` in the string `s`. + ## Count the occurrences of the character `sub` in the string `s`. for c in s: if c == sub: inc result proc count*(s: string, subs: set[char]): int {.noSideEffect, rtl, extern: "nsuCountCharSet".} = - ## Count the occurences of the group of character `subs` in the string `s`. + ## Count the occurrences of the group of character `subs` in the string `s`. for c in s: if c in subs: inc result @@ -898,7 +921,7 @@ proc replaceWord*(s, sub: string, by = ""): string {.noSideEffect, rtl, extern: "nsuReplaceWord".} = ## Replaces `sub` in `s` by the string `by`. ## - ## Each occurance of `sub` has to be surrounded by word boundaries + ## Each occurrence of `sub` has to be surrounded by word boundaries ## (comparable to ``\\w`` in regular expressions), otherwise it is not ## replaced. const wordChars = {'a'..'z', 'A'..'Z', '0'..'9', '_', '\128'..'\255'} @@ -910,7 +933,7 @@ proc replaceWord*(s, sub: string, by = ""): string {.noSideEffect, var j = findAux(s, sub, i, a) if j < 0: break # word boundary? - if (j == 0 or s[j-1] notin wordChars) and + if (j == 0 or s[j-1] notin wordChars) and (j+sub.len >= s.len or s[j+sub.len] notin wordChars): add result, substr(s, i, j - 1) add result, by @@ -953,7 +976,7 @@ proc parseOctInt*(s: string): int {.noSideEffect, of '\0': break else: raise newException(ValueError, "invalid integer: " & s) -proc toOct*(x: BiggestInt, len: int): string {.noSideEffect, +proc toOct*(x: BiggestInt, len: Positive): string {.noSideEffect, rtl, extern: "nsuToOct".} = ## Converts `x` into its octal representation. ## @@ -969,7 +992,7 @@ proc toOct*(x: BiggestInt, len: int): string {.noSideEffect, shift = shift + 3 mask = mask shl 3 -proc toBin*(x: BiggestInt, len: int): string {.noSideEffect, +proc toBin*(x: BiggestInt, len: Positive): string {.noSideEffect, rtl, extern: "nsuToBin".} = ## Converts `x` into its binary representation. ## @@ -1042,18 +1065,20 @@ proc unescape*(s: string, prefix = "\"", suffix = "\""): string {.noSideEffect, ## ValueError exception will be raised. result = newStringOfCap(s.len) var i = 0 - if s[0 .. prefix.len-1] != prefix: + if not s.startsWith(prefix): raise newException(ValueError, "String does not start with a prefix of: " & prefix) - i.inc() + inc(i) while true: if i == s.len-suffix.len: break case s[i] of '\\': case s[i+1]: of 'x': - let j = parseHexInt(s[i+2 .. i+3]) - result.add(chr(j)) + inc i + var c: int + i += parseutils.parseHex(s, c, i) + result.add(chr(c)) inc(i, 2) of '\\': result.add('\\') @@ -1066,8 +1091,8 @@ proc unescape*(s: string, prefix = "\"", suffix = "\""): string {.noSideEffect, of '\0': break else: result.add(s[i]) - i.inc() - if s[i .. -1] != suffix: + inc(i) + if not s.endsWith(suffix): raise newException(ValueError, "String does not end with a suffix of: " & suffix) @@ -1196,7 +1221,7 @@ proc formatBiggestFloat*(f: BiggestFloat, format: FloatFormatMode = ffDefault, ## of significant digits to be printed. ## `precision`'s default value is the maximum number of meaningful digits ## after the decimal point for Nim's ``biggestFloat`` type. - ## + ## ## If ``precision == 0``, it tries to format it nicely. const floatFormatToChar: array[FloatFormatMode, char] = ['g', 'f', 'e'] var @@ -1261,7 +1286,7 @@ proc findNormalized(x: string, inArray: openArray[string]): int = return -1 proc invalidFormatString() {.noinline.} = - raise newException(ValueError, "invalid format string") + raise newException(ValueError, "invalid format string") proc addf*(s: var string, formatstr: string, a: varargs[string, `$`]) {. noSideEffect, rtl, extern: "nsuAddf".} = @@ -1377,21 +1402,28 @@ when isMainModule: doAssert align("a", 0) == "a" doAssert align("1232", 6) == " 1232" doAssert align("1232", 6, '#') == "##1232" - echo wordWrap(""" this is a long text -- muchlongerthan10chars and here - it goes""", 10, false) + + let + inp = """ this is a long text -- muchlongerthan10chars and here + it goes""" + outp = " this is a\nlong text\n--\nmuchlongerthan10chars\nand here\nit goes" + doAssert wordWrap(inp, 10, false) == outp + doAssert formatBiggestFloat(0.00000000001, ffDecimal, 11) == "0.00000000001" - doAssert formatBiggestFloat(0.00000000001, ffScientific, 1) == "1.0e-11" + doAssert formatBiggestFloat(0.00000000001, ffScientific, 1) in + ["1.0e-11", "1.0e-011"] doAssert "$# $3 $# $#" % ["a", "b", "c"] == "a c b c" - echo formatSize(1'i64 shl 31 + 300'i64) # == "4,GB" - echo formatSize(1'i64 shl 31) + when not defined(testing): + echo formatSize(1'i64 shl 31 + 300'i64) # == "4,GB" + echo formatSize(1'i64 shl 31) doAssert "$animal eats $food." % ["animal", "The cat", "food", "fish"] == "The cat eats fish." doAssert "-ld a-ldz -ld".replaceWord("-ld") == " a-ldz " doAssert "-lda-ldz -ld abc".replaceWord("-ld") == "-lda-ldz abc" - + type MyEnum = enum enA, enB, enC, enuD, enE doAssert parseEnum[MyEnum]("enu_D") == enuD diff --git a/lib/pure/subexes.nim b/lib/pure/subexes.nim index adcfdd288..d213c99e6 100644 --- a/lib/pure/subexes.nim +++ b/lib/pure/subexes.nim @@ -353,11 +353,11 @@ when isMainModule: proc `%`(formatstr: string, a: openarray[string]): string = result = newStringOfCap(formatstr.len + a.len shl 4) - addf(result, formatstr.TSubex, a) + addf(result, formatstr.Subex, a) proc `%`(formatstr: string, a: string): string = result = newStringOfCap(formatstr.len + a.len) - addf(result, formatstr.TSubex, [a]) + addf(result, formatstr.Subex, [a]) doAssert "$# $3 $# $#" % ["a", "b", "c"] == "a c b c" @@ -386,8 +386,13 @@ when isMainModule: longishA, longish)""" - echo "type TMyEnum* = enum\n $', '2i'\n '{..}" % ["fieldA", - "fieldB", "FiledClkad", "fieldD", "fieldE", "longishFieldName"] + assert "type TMyEnum* = enum\n $', '2i'\n '{..}" % ["fieldA", + "fieldB", "FiledClkad", "fieldD", "fieldE", "longishFieldName"] == + strutils.unindent """ + type TMyEnum* = enum + fieldA, fieldB, + FiledClkad, fieldD, + fieldE, longishFieldName""" doAssert subex"$1($', '{2..})" % ["f", "a", "b", "c"] == "f(a, b, c)" @@ -395,7 +400,12 @@ when isMainModule: doAssert subex"$['''|'|''''|']']#" % "0" == "'|" - echo subex("type\n TEnum = enum\n $', '40c'\n '{..}") % [ - "fieldNameA", "fieldNameB", "fieldNameC", "fieldNameD"] + assert subex("type\n TEnum = enum\n $', '40c'\n '{..}") % [ + "fieldNameA", "fieldNameB", "fieldNameC", "fieldNameD"] == + strutils.unindent """ + type + TEnum = enum + fieldNameA, fieldNameB, fieldNameC, + fieldNameD""" diff --git a/lib/pure/terminal.nim b/lib/pure/terminal.nim index 1c1d973ee..29f700db5 100644 --- a/lib/pure/terminal.nim +++ b/lib/pure/terminal.nim @@ -45,6 +45,21 @@ when defined(windows): var oldAttr = getAttributes() +else: + import termios, unsigned + + proc setRaw(fd: FileHandle, time: cint = TCSAFLUSH) = + var mode: Termios + discard fd.tcgetattr(addr mode) + mode.c_iflag = mode.c_iflag and not Tcflag(BRKINT or ICRNL or INPCK or + ISTRIP or IXON) + mode.c_oflag = mode.c_oflag and not Tcflag(OPOST) + mode.c_cflag = (mode.c_cflag and not Tcflag(CSIZE or PARENB)) or CS8 + mode.c_lflag = mode.c_lflag and not Tcflag(ECHO or ICANON or IEXTEN or ISIG) + mode.c_cc[VMIN] = 1.cuchar + mode.c_cc[VTIME] = 0.cuchar + discard fd.tcsetattr(time, addr mode) + proc setCursorPos*(x, y: int) = ## sets the terminal's cursor to the (x,y) position. (0,0) is the ## upper left of the screen. @@ -185,13 +200,15 @@ proc eraseScreen* = var numwrote: DWORD var origin: TCOORD # is inititalized to 0, 0 var hStdout = conHandle + if GetConsoleScreenBufferInfo(hStdout, addr(scrbuf)) == 0: raiseOSError(osLastError()) - if FillConsoleOutputCharacter(hStdout, ' ', scrbuf.dwSize.X*scrbuf.dwSize.Y, + let numChars = int32(scrbuf.dwSize.X)*int32(scrbuf.dwSize.Y) + + if FillConsoleOutputCharacter(hStdout, ' ', numChars, origin, addr(numwrote)) == 0: raiseOSError(osLastError()) - if FillConsoleOutputAttribute(hStdout, scrbuf.wAttributes, - scrbuf.dwSize.X * scrbuf.dwSize.Y, + if FillConsoleOutputAttribute(hStdout, scrbuf.wAttributes, numChars, origin, addr(numwrote)) == 0: raiseOSError(osLastError()) setCursorXPos(0) @@ -327,7 +344,7 @@ proc isatty*(f: File): bool = else: proc isatty(fildes: FileHandle): cint {. importc: "_isatty", header: "<io.h>".} - + result = isatty(getFileHandle(f)) != 0'i32 proc styledEchoProcessArg(s: string) = write stdout, s @@ -347,7 +364,19 @@ macro styledEcho*(m: varargs[expr]): stmt = result.add(newCall(bindSym"write", bindSym"stdout", newStrLitNode("\n"))) result.add(newCall(bindSym"resetAttributes")) -when isMainModule: +when not defined(windows): + proc getch*(): char = + ## Read a single character from the terminal, blocking until it is entered. + ## The character is not printed to the terminal. This is not available for + ## Windows. + let fd = getFileHandle(stdin) + var oldMode: Termios + discard fd.tcgetattr(addr oldMode) + fd.setRaw() + result = stdin.readChar() + discard fd.tcsetattr(TCSADRAIN, addr oldMode) + +when not defined(testing) and isMainModule: system.addQuitProc(resetAttributes) write(stdout, "never mind") eraseLine() @@ -357,5 +386,5 @@ when isMainModule: setForeGroundColor(fgBlue) writeln(stdout, "ordinary text") - styledEcho("styled text ", {styleBright, styleBlink, styleUnderscore}) - + styledEcho("styled text ", {styleBright, styleBlink, styleUnderscore}) + diff --git a/lib/pure/times.nim b/lib/pure/times.nim index 1cabd381b..5f8835c6a 100644 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -16,7 +16,7 @@ # of the standard library! import - strutils + strutils, parseutils include "system/inclrtl" @@ -26,81 +26,87 @@ type WeekDay* = enum ## represents a weekday dMon, dTue, dWed, dThu, dFri, dSat, dSun -var - timezone {.importc, header: "<time.h>".}: int - tzname {.importc, header: "<time.h>" .}: array[0..1, cstring] +when not defined(JS): + var + timezone {.importc, header: "<time.h>".}: int + tzname {.importc, header: "<time.h>" .}: array[0..1, cstring] when defined(posix) and not defined(JS): type - TimeImpl {.importc: "time_t", header: "<sys/time.h>".} = int + TimeImpl {.importc: "time_t", header: "<time.h>".} = int Time* = distinct TimeImpl ## distinct type that represents a time ## measured as number of seconds since the epoch - - Timeval {.importc: "struct timeval", + + Timeval {.importc: "struct timeval", header: "<sys/select.h>".} = object ## struct timeval - tv_sec: int ## Seconds. - tv_usec: int ## Microseconds. - + tv_sec: int ## Seconds. + tv_usec: int ## Microseconds. + # we cannot import posix.nim here, because posix.nim depends on times.nim. - # Ok, we could, but I don't want circular dependencies. + # Ok, we could, but I don't want circular dependencies. # And gettimeofday() is not defined in the posix module anyway. Sigh. - + proc posix_gettimeofday(tp: var Timeval, unused: pointer = nil) {. importc: "gettimeofday", header: "<sys/time.h>".} + # we also need tzset() to make sure that tzname is initialized + proc tzset() {.importc, header: "<time.h>".} + # calling tzset() implicitly to initialize tzname data. + tzset() + elif defined(windows): import winlean - + when defined(vcc): # newest version of Visual C++ defines time_t to be of 64 bits type TimeImpl {.importc: "time_t", header: "<time.h>".} = int64 else: type TimeImpl {.importc: "time_t", header: "<time.h>".} = int32 - + type Time* = distinct TimeImpl elif defined(JS): type Time* {.importc.} = object - getDay: proc (): int {.tags: [], raises: [], gcsafe.} - getFullYear: proc (): int {.tags: [], raises: [], gcsafe.} - getHours: proc (): int {.tags: [], raises: [], gcsafe.} - getMilliseconds: proc (): int {.tags: [], raises: [], gcsafe.} - getMinutes: proc (): int {.tags: [], raises: [], gcsafe.} - getMonth: proc (): int {.tags: [], raises: [], gcsafe.} - getSeconds: proc (): int {.tags: [], raises: [], gcsafe.} - getTime: proc (): int {.tags: [], raises: [], gcsafe.} - getTimezoneOffset: proc (): int {.tags: [], raises: [], gcsafe.} - getDate: proc (): int {.tags: [], raises: [], gcsafe.} - getUTCDate: proc (): int {.tags: [], raises: [], gcsafe.} - getUTCFullYear: proc (): int {.tags: [], raises: [], gcsafe.} - getUTCHours: proc (): int {.tags: [], raises: [], gcsafe.} - getUTCMilliseconds: proc (): int {.tags: [], raises: [], gcsafe.} - getUTCMinutes: proc (): int {.tags: [], raises: [], gcsafe.} - getUTCMonth: proc (): int {.tags: [], raises: [], gcsafe.} - getUTCSeconds: proc (): int {.tags: [], raises: [], gcsafe.} - getUTCDay: proc (): int {.tags: [], raises: [], gcsafe.} - getYear: proc (): int {.tags: [], raises: [], gcsafe.} - parse: proc (s: cstring): Time {.tags: [], raises: [], gcsafe.} - setDate: proc (x: int) {.tags: [], raises: [], gcsafe.} - setFullYear: proc (x: int) {.tags: [], raises: [], gcsafe.} - setHours: proc (x: int) {.tags: [], raises: [], gcsafe.} - setMilliseconds: proc (x: int) {.tags: [], raises: [], gcsafe.} - setMinutes: proc (x: int) {.tags: [], raises: [], gcsafe.} - setMonth: proc (x: int) {.tags: [], raises: [], gcsafe.} - setSeconds: proc (x: int) {.tags: [], raises: [], gcsafe.} - setTime: proc (x: int) {.tags: [], raises: [], gcsafe.} - setUTCDate: proc (x: int) {.tags: [], raises: [], gcsafe.} - setUTCFullYear: proc (x: int) {.tags: [], raises: [], gcsafe.} - setUTCHours: proc (x: int) {.tags: [], raises: [], gcsafe.} - setUTCMilliseconds: proc (x: int) {.tags: [], raises: [], gcsafe.} - setUTCMinutes: proc (x: int) {.tags: [], raises: [], gcsafe.} - setUTCMonth: proc (x: int) {.tags: [], raises: [], gcsafe.} - setUTCSeconds: proc (x: int) {.tags: [], raises: [], gcsafe.} - setYear: proc (x: int) {.tags: [], raises: [], gcsafe.} - toGMTString: proc (): cstring {.tags: [], raises: [], gcsafe.} - toLocaleString: proc (): cstring {.tags: [], raises: [], gcsafe.} + getDay: proc (): int {.tags: [], raises: [], benign.} + getFullYear: proc (): int {.tags: [], raises: [], benign.} + getHours: proc (): int {.tags: [], raises: [], benign.} + getMilliseconds: proc (): int {.tags: [], raises: [], benign.} + getMinutes: proc (): int {.tags: [], raises: [], benign.} + getMonth: proc (): int {.tags: [], raises: [], benign.} + getSeconds: proc (): int {.tags: [], raises: [], benign.} + getTime: proc (): int {.tags: [], raises: [], benign.} + getTimezoneOffset: proc (): int {.tags: [], raises: [], benign.} + getDate: proc (): int {.tags: [], raises: [], benign.} + getUTCDate: proc (): int {.tags: [], raises: [], benign.} + getUTCFullYear: proc (): int {.tags: [], raises: [], benign.} + getUTCHours: proc (): int {.tags: [], raises: [], benign.} + getUTCMilliseconds: proc (): int {.tags: [], raises: [], benign.} + getUTCMinutes: proc (): int {.tags: [], raises: [], benign.} + getUTCMonth: proc (): int {.tags: [], raises: [], benign.} + getUTCSeconds: proc (): int {.tags: [], raises: [], benign.} + getUTCDay: proc (): int {.tags: [], raises: [], benign.} + getYear: proc (): int {.tags: [], raises: [], benign.} + parse: proc (s: cstring): Time {.tags: [], raises: [], benign.} + setDate: proc (x: int) {.tags: [], raises: [], benign.} + setFullYear: proc (x: int) {.tags: [], raises: [], benign.} + setHours: proc (x: int) {.tags: [], raises: [], benign.} + setMilliseconds: proc (x: int) {.tags: [], raises: [], benign.} + setMinutes: proc (x: int) {.tags: [], raises: [], benign.} + setMonth: proc (x: int) {.tags: [], raises: [], benign.} + setSeconds: proc (x: int) {.tags: [], raises: [], benign.} + setTime: proc (x: int) {.tags: [], raises: [], benign.} + setUTCDate: proc (x: int) {.tags: [], raises: [], benign.} + setUTCFullYear: proc (x: int) {.tags: [], raises: [], benign.} + setUTCHours: proc (x: int) {.tags: [], raises: [], benign.} + setUTCMilliseconds: proc (x: int) {.tags: [], raises: [], benign.} + setUTCMinutes: proc (x: int) {.tags: [], raises: [], benign.} + setUTCMonth: proc (x: int) {.tags: [], raises: [], benign.} + setUTCSeconds: proc (x: int) {.tags: [], raises: [], benign.} + setYear: proc (x: int) {.tags: [], raises: [], benign.} + toGMTString: proc (): cstring {.tags: [], raises: [], benign.} + toLocaleString: proc (): cstring {.tags: [], raises: [], benign.} type TimeInfo* = object of RootObj ## represents a time in different parts @@ -139,51 +145,51 @@ type {.deprecated: [TMonth: Month, TWeekDay: WeekDay, TTime: Time, TTimeInterval: TimeInterval, TTimeInfo: TimeInfo].} -proc getTime*(): Time {.tags: [TimeEffect], gcsafe.} +proc getTime*(): Time {.tags: [TimeEffect], benign.} ## gets the current calendar time as a UNIX epoch value (number of seconds ## elapsed since 1970) with integer precission. Use epochTime for higher ## resolution. -proc getLocalTime*(t: Time): TimeInfo {.tags: [TimeEffect], raises: [], gcsafe.} +proc getLocalTime*(t: Time): TimeInfo {.tags: [TimeEffect], raises: [], benign.} ## converts the calendar time `t` to broken-time representation, ## expressed relative to the user's specified time zone. -proc getGMTime*(t: Time): TimeInfo {.tags: [TimeEffect], raises: [], gcsafe.} +proc getGMTime*(t: Time): TimeInfo {.tags: [TimeEffect], raises: [], benign.} ## converts the calendar time `t` to broken-down time representation, ## expressed in Coordinated Universal Time (UTC). -proc timeInfoToTime*(timeInfo: TimeInfo): Time {.tags: [], gcsafe.} +proc timeInfoToTime*(timeInfo: TimeInfo): Time {.tags: [], benign.} ## converts a broken-down time structure to ## calendar time representation. The function ignores the specified ## contents of the structure members `weekday` and `yearday` and recomputes ## them from the other information in the broken-down time structure. -proc fromSeconds*(since1970: float): Time {.tags: [], raises: [], gcsafe.} +proc fromSeconds*(since1970: float): Time {.tags: [], raises: [], benign.} ## Takes a float which contains the number of seconds since the unix epoch and ## returns a time object. -proc fromSeconds*(since1970: int64): Time {.tags: [], raises: [], gcsafe.} = +proc fromSeconds*(since1970: int64): Time {.tags: [], raises: [], benign.} = ## Takes an int which contains the number of seconds since the unix epoch and ## returns a time object. fromSeconds(float(since1970)) -proc toSeconds*(time: Time): float {.tags: [], raises: [], gcsafe.} +proc toSeconds*(time: Time): float {.tags: [], raises: [], benign.} ## Returns the time in seconds since the unix epoch. -proc `$` *(timeInfo: TimeInfo): string {.tags: [], raises: [], gcsafe.} +proc `$` *(timeInfo: TimeInfo): string {.tags: [], raises: [], benign.} ## converts a `TimeInfo` object to a string representation. -proc `$` *(time: Time): string {.tags: [], raises: [], gcsafe.} +proc `$` *(time: Time): string {.tags: [], raises: [], benign.} ## converts a calendar time to a string representation. proc `-`*(a, b: Time): int64 {. - rtl, extern: "ntDiffTime", tags: [], raises: [].} + rtl, extern: "ntDiffTime", tags: [], raises: [], benign.} ## computes the difference of two calendar times. Result is in seconds. proc `<`*(a, b: Time): bool {. - rtl, extern: "ntLtTime", tags: [], raises: [].} = + rtl, extern: "ntLtTime", tags: [], raises: [].} = ## returns true iff ``a < b``, that is iff a happened before b. result = a - b < 0 - + proc `<=` * (a, b: Time): bool {. - rtl, extern: "ntLeTime", tags: [], raises: [].}= + rtl, extern: "ntLeTime", tags: [], raises: [].}= ## returns true iff ``a <= b``. result = a - b <= 0 @@ -194,18 +200,18 @@ proc `==`*(a, b: Time): bool {. when not defined(JS): proc getTzname*(): tuple[nonDST, DST: string] {.tags: [TimeEffect], raises: [], - gcsafe.} + benign.} ## returns the local timezone; ``nonDST`` is the name of the local non-DST ## timezone, ``DST`` is the name of the local DST timezone. -proc getTimezone*(): int {.tags: [TimeEffect], raises: [], gcsafe.} +proc getTimezone*(): int {.tags: [TimeEffect], raises: [], benign.} ## returns the offset of the local (non-DST) timezone in seconds west of UTC. -proc getStartMilsecs*(): int {.deprecated, tags: [TimeEffect], gcsafe.} +proc getStartMilsecs*(): int {.deprecated, tags: [TimeEffect], benign.} ## get the miliseconds from the start of the program. **Deprecated since ## version 0.8.10.** Use ``epochTime`` or ``cpuTime`` instead. -proc initInterval*(miliseconds, seconds, minutes, hours, days, months, +proc initInterval*(miliseconds, seconds, minutes, hours, days, months, years: int = 0): TimeInterval = ## creates a new ``TimeInterval``. result.miliseconds = miliseconds @@ -221,9 +227,9 @@ proc isLeapYear*(year: int): bool = if year mod 400 == 0: return true - elif year mod 100 == 0: + elif year mod 100 == 0: return false - elif year mod 4 == 0: + elif year mod 4 == 0: return true else: return false @@ -232,7 +238,7 @@ proc getDaysInMonth*(month: Month, year: int): int = ## gets the amount of days in a ``month`` of a ``year`` # http://www.dispersiondesign.com/articles/time/number_of_days_in_a_month - case month + case month of mFeb: result = if isLeapYear(year): 29 else: 28 of mApr, mJun, mSep, mNov: result = 30 else: result = 31 @@ -244,7 +250,7 @@ proc toSeconds(a: TimeInfo, interval: TimeInterval): float = var anew = a var newinterv = interval result = 0 - + newinterv.months += interval.years * 12 var curMonth = anew.month for mth in 1 .. newinterv.months: @@ -284,18 +290,18 @@ proc `-`*(a: TimeInfo, interval: TimeInterval): TimeInfo = else: result = getLocalTime(fromSeconds(t - secs)) -when not defined(JS): +when not defined(JS): proc epochTime*(): float {.rtl, extern: "nt$1", tags: [TimeEffect].} ## gets time after the UNIX epoch (1970) in seconds. It is a float - ## because sub-second resolution is likely to be supported (depending + ## because sub-second resolution is likely to be supported (depending ## on the hardware/OS). proc cpuTime*(): float {.rtl, extern: "nt$1", tags: [TimeEffect].} ## gets time spent that the CPU spent to run the current process in ## seconds. This may be more useful for benchmarking than ``epochTime``. ## However, it may measure the real time instead (depending on the OS). - ## The value of the result has no meaning. - ## To generate useful timing values, take the difference between + ## The value of the result has no meaning. + ## To generate useful timing values, take the difference between ## the results of two ``cpuTime`` calls: ## ## .. code-block:: nim @@ -316,10 +322,10 @@ when not defined(JS): weekday {.importc: "tm_wday".}, yearday {.importc: "tm_yday".}, isdst {.importc: "tm_isdst".}: cint - + TimeInfoPtr = ptr StructTM Clock {.importc: "clock_t".} = distinct int - + proc localtime(timer: ptr Time): TimeInfoPtr {. importc: "localtime", header: "<time.h>", tags: [].} proc gmtime(timer: ptr Time): TimeInfoPtr {. @@ -335,12 +341,12 @@ when not defined(JS): # strftime(s: CString, maxsize: int, fmt: CString, t: tm): int {. # importc: "strftime", header: "<time.h>".} proc getClock(): Clock {.importc: "clock", header: "<time.h>", tags: [TimeEffect].} - proc difftime(a, b: Time): float {.importc: "difftime", header: "<time.h>", + proc difftime(a, b: Time): float {.importc: "difftime", header: "<time.h>", tags: [].} - + var clocksPerSec {.importc: "CLOCKS_PER_SEC", nodecl.}: int - + # our own procs on top of that: proc tmToTimeInfo(tm: StructTM, local: bool): TimeInfo = const @@ -364,7 +370,7 @@ when not defined(JS): "UTC", timezone: if local: getTimezone() else: 0 ) - + proc timeInfoToTM(t: TimeInfo): StructTM = const weekDays: array [WeekDay, int8] = [1'i8,2'i8,3'i8,4'i8,5'i8,6'i8,0'i8] @@ -377,11 +383,11 @@ when not defined(JS): result.weekday = weekDays[t.weekday] result.yearday = t.yearday result.isdst = if t.isDST: 1 else: 0 - + when not defined(useNimRtl): proc `-` (a, b: Time): int64 = return toBiggestInt(difftime(a, b)) - + proc getStartMilsecs(): int = #echo "clocks per sec: ", clocksPerSec, "clock: ", int(getClock()) #return getClock() div (clocksPerSec div 1000) @@ -394,37 +400,37 @@ when not defined(JS): posix_gettimeofday(a) result = a.tv_sec * 1000'i64 + a.tv_usec div 1000'i64 #echo "result: ", result - + proc getTime(): Time = return timec(nil) proc getLocalTime(t: Time): TimeInfo = var a = t result = tmToTimeInfo(localtime(addr(a))[], true) # copying is needed anyway to provide reentrancity; thus # the conversion is not expensive - + proc getGMTime(t: Time): TimeInfo = var a = t result = tmToTimeInfo(gmtime(addr(a))[], false) # copying is needed anyway to provide reentrancity; thus # the conversion is not expensive - + proc timeInfoToTime(timeInfo: TimeInfo): Time = var cTimeInfo = timeInfo # for C++ we have to make a copy, # because the header of mktime is broken in my version of libc return mktime(timeInfoToTM(cTimeInfo)) - proc toStringTillNL(p: cstring): string = + proc toStringTillNL(p: cstring): string = result = "" var i = 0 - while p[i] != '\0' and p[i] != '\10' and p[i] != '\13': + while p[i] != '\0' and p[i] != '\10' and p[i] != '\13': add(result, p[i]) inc(i) - + proc `$`(timeInfo: TimeInfo): string = # BUGFIX: asctime returns a newline at the end! var p = asctime(timeInfoToTM(timeInfo)) result = toStringTillNL(p) - + proc `$`(time: Time): string = # BUGFIX: ctime returns a newline at the end! var a = time @@ -434,17 +440,17 @@ when not defined(JS): epochDiff = 116444736000000000'i64 rateDiff = 10000000'i64 # 100 nsecs - proc unixTimeToWinTime*(t: Time): int64 = + proc unixTimeToWinTime*(t: Time): int64 = ## converts a UNIX `Time` (``time_t``) to a Windows file time result = int64(t) * rateDiff + epochDiff - - proc winTimeToUnixTime*(t: int64): Time = + + proc winTimeToUnixTime*(t: int64): Time = ## converts a Windows time to a UNIX `Time` (``time_t``) result = Time((t - epochDiff) div rateDiff) - + proc getTzname(): tuple[nonDST, DST: string] = return ($tzname[0], $tzname[1]) - + proc getTimezone(): int = return timezone @@ -453,7 +459,7 @@ when not defined(JS): proc toSeconds(time: Time): float = float(time) when not defined(useNimRtl): - proc epochTime(): float = + proc epochTime(): float = when defined(posix): var a: Timeval posix_gettimeofday(a) @@ -467,14 +473,14 @@ when not defined(JS): result = toFloat(int(secs)) + toFloat(int(subsecs)) * 0.0000001 else: {.error: "unknown OS".} - - proc cpuTime(): float = + + proc cpuTime(): float = result = toFloat(int(getClock())) / toFloat(clocksPerSec) - + elif defined(JS): proc newDate(): Time {.importc: "new Date".} proc internGetTime(): Time {.importc: "new Date", tags: [].} - + proc newDate(value: float): Time {.importc: "new Date".} proc newDate(value: string): Time {.importc: "new Date".} proc getTime(): Time = @@ -482,15 +488,15 @@ elif defined(JS): return newDate() const - weekDays: array [0..6, TWeekDay] = [ + weekDays: array [0..6, WeekDay] = [ dSun, dMon, dTue, dWed, dThu, dFri, dSat] - + proc getLocalTime(t: Time): TimeInfo = result.second = t.getSeconds() result.minute = t.getMinutes() result.hour = t.getHours() result.monthday = t.getDate() - result.month = TMonth(t.getMonth()) + result.month = Month(t.getMonth()) result.year = t.getFullYear() result.weekday = weekDays[t.getDay()] result.yearday = 0 @@ -500,11 +506,11 @@ elif defined(JS): result.minute = t.getUTCMinutes() result.hour = t.getUTCHours() result.monthday = t.getUTCDate() - result.month = TMonth(t.getUTCMonth()) + result.month = Month(t.getUTCMonth()) result.year = t.getUTCFullYear() result.weekday = weekDays[t.getUTCDay()] result.yearday = 0 - + proc timeInfoToTime*(timeInfo: TimeInfo): Time = result = internGetTime() result.setSeconds(timeInfo.second) @@ -513,16 +519,16 @@ elif defined(JS): result.setMonth(ord(timeInfo.month)) result.setFullYear(timeInfo.year) result.setDate(timeInfo.monthday) - + proc `$`(timeInfo: TimeInfo): string = return $(timeInfoToTime(timeInfo)) proc `$`(time: Time): string = return $time.toLocaleString() - - proc `-` (a, b: Time): int64 = + + proc `-` (a, b: Time): int64 = return a.getTime() - b.getTime() - + var startMilsecs = getTime() - + proc getStartMilsecs(): int = ## get the miliseconds from the start of the program return int(getTime() - startMilsecs) @@ -535,6 +541,8 @@ elif defined(JS): proc getTimezone(): int = result = newDate().getTimezoneOffset() + proc epochTime*(): float {.tags: [TimeEffect].} = newDate().toSeconds() + proc getDateStr*(): string {.rtl, extern: "nt$1", tags: [TimeEffect].} = ## gets the current date as a string of the format ``YYYY-MM-DD``. var ti = getLocalTime(getTime()) @@ -554,8 +562,8 @@ proc `$`*(day: WeekDay): string = return lookup[day] proc `$`*(m: Month): string = - ## stingify operator for ``TMonth``. - const lookup: array[Month, string] = ["January", "February", "March", + ## stingify operator for ``Month``. + const lookup: array[Month, string] = ["January", "February", "March", "April", "May", "June", "July", "August", "September", "October", "November", "December"] return lookup[m] @@ -628,25 +636,25 @@ proc formatToken(info: TimeInfo, token: string, buf: var string) = var fr = ($info.year).len()-2 if fr < 0: fr = 0 var fyear = ($info.year)[fr .. ($info.year).len()-1] - if fyear.len != 2: fyear = repeatChar(2-fyear.len(), '0') & fyear + if fyear.len != 2: fyear = repeat('0', 2-fyear.len()) & fyear buf.add(fyear) of "yyy": var fr = ($info.year).len()-3 if fr < 0: fr = 0 var fyear = ($info.year)[fr .. ($info.year).len()-1] - if fyear.len != 3: fyear = repeatChar(3-fyear.len(), '0') & fyear + if fyear.len != 3: fyear = repeat('0', 3-fyear.len()) & fyear buf.add(fyear) of "yyyy": var fr = ($info.year).len()-4 if fr < 0: fr = 0 var fyear = ($info.year)[fr .. ($info.year).len()-1] - if fyear.len != 4: fyear = repeatChar(4-fyear.len(), '0') & fyear + if fyear.len != 4: fyear = repeat('0', 4-fyear.len()) & fyear buf.add(fyear) of "yyyyy": var fr = ($info.year).len()-5 if fr < 0: fr = 0 var fyear = ($info.year)[fr .. ($info.year).len()-1] - if fyear.len != 5: fyear = repeatChar(5-fyear.len(), '0') & fyear + if fyear.len != 5: fyear = repeat('0', 5-fyear.len()) & fyear buf.add(fyear) of "z": let hrs = (info.timezone div 60) div 60 @@ -676,7 +684,7 @@ proc formatToken(info: TimeInfo, token: string, buf: var string) = proc format*(info: TimeInfo, f: string): string = ## This function formats `info` as specified by `f`. The following format ## specifiers are available: - ## + ## ## ========== ================================================================================= ================================================ ## Specifier Description Example ## ========== ================================================================================= ================================================ @@ -723,14 +731,14 @@ proc format*(info: TimeInfo, f: string): string = currentF = "" if f[i] == '\0': break - + if f[i] == '\'': inc(i) # Skip ' while f[i] != '\'' and f.len-1 > i: result.add(f[i]) inc(i) else: result.add(f[i]) - + else: # Check if the letter being added matches previous accumulated buffer. if currentF.len < 1 or currentF[high(currentF)] == f[i]: @@ -744,37 +752,364 @@ proc format*(info: TimeInfo, f: string): string = {.pop.} +proc parseToken(info: var TimeInfo; token, value: string; j: var int) = + ## Helper of the parse proc to parse individual tokens. + var sv: int + case token + of "d": + var pd = parseInt(value[j..j+1], sv) + info.monthday = sv + j += pd + of "dd": + info.monthday = value[j..j+1].parseInt() + j += 2 + of "ddd": + case value[j..j+2].toLower() + of "sun": + info.weekday = dSun + of "mon": + info.weekday = dMon + of "tue": + info.weekday = dTue + of "wed": + info.weekday = dWed + of "thu": + info.weekday = dThu + of "fri": + info.weekday = dFri + of "sat": + info.weekday = dSat + else: + raise newException(ValueError, "invalid day of week ") + j += 3 + of "dddd": + if value.len >= j+6 and value[j..j+5].cmpIgnoreCase("sunday") == 0: + info.weekday = dSun + j += 6 + elif value.len >= j+6 and value[j..j+5].cmpIgnoreCase("monday") == 0: + info.weekday = dMon + j += 6 + elif value.len >= j+7 and value[j..j+6].cmpIgnoreCase("tuesday") == 0: + info.weekday = dTue + j += 7 + elif value.len >= j+9 and value[j..j+8].cmpIgnoreCase("wednesday") == 0: + info.weekday = dWed + j += 9 + elif value.len >= j+8 and value[j..j+7].cmpIgnoreCase("thursday") == 0: + info.weekday = dThu + j += 8 + elif value.len >= j+6 and value[j..j+5].cmpIgnoreCase("friday") == 0: + info.weekday = dFri + j += 6 + elif value.len >= j+8 and value[j..j+7].cmpIgnoreCase("saturday") == 0: + info.weekday = dSat + j += 8 + else: + raise newException(ValueError, "invalid day of week ") + of "h", "H": + var pd = parseInt(value[j..j+1], sv) + info.hour = sv + j += pd + of "hh", "HH": + info.hour = value[j..j+1].parseInt() + j += 2 + of "m": + var pd = parseInt(value[j..j+1], sv) + info.minute = sv + j += pd + of "mm": + info.minute = value[j..j+1].parseInt() + j += 2 + of "M": + var pd = parseInt(value[j..j+1], sv) + info.month = Month(sv-1) + info.monthday = sv + j += pd + of "MM": + var month = value[j..j+1].parseInt() + j += 2 + info.month = Month(month-1) + of "MMM": + case value[j..j+2].toLower(): + of "jan": + info.month = mJan + of "feb": + info.month = mFeb + of "mar": + info.month = mMar + of "apr": + info.month = mApr + of "may": + info.month = mMay + of "jun": + info.month = mJun + of "jul": + info.month = mJul + of "aug": + info.month = mAug + of "sep": + info.month = mSep + of "oct": + info.month = mOct + of "nov": + info.month = mNov + of "dec": + info.month = mDec + else: + raise newException(ValueError, "invalid month") + j += 3 + of "MMMM": + if value.len >= j+7 and value[j..j+6].cmpIgnoreCase("january") == 0: + info.month = mJan + j += 7 + elif value.len >= j+8 and value[j..j+7].cmpIgnoreCase("february") == 0: + info.month = mFeb + j += 8 + elif value.len >= j+5 and value[j..j+4].cmpIgnoreCase("march") == 0: + info.month = mMar + j += 5 + elif value.len >= j+5 and value[j..j+4].cmpIgnoreCase("april") == 0: + info.month = mApr + j += 5 + elif value.len >= j+3 and value[j..j+2].cmpIgnoreCase("may") == 0: + info.month = mMay + j += 3 + elif value.len >= j+4 and value[j..j+3].cmpIgnoreCase("june") == 0: + info.month = mJun + j += 4 + elif value.len >= j+4 and value[j..j+3].cmpIgnoreCase("july") == 0: + info.month = mJul + j += 4 + elif value.len >= j+6 and value[j..j+5].cmpIgnoreCase("august") == 0: + info.month = mAug + j += 6 + elif value.len >= j+9 and value[j..j+8].cmpIgnoreCase("september") == 0: + info.month = mSep + j += 9 + elif value.len >= j+7 and value[j..j+6].cmpIgnoreCase("october") == 0: + info.month = mOct + j += 7 + elif value.len >= j+8 and value[j..j+7].cmpIgnoreCase("november") == 0: + info.month = mNov + j += 8 + elif value.len >= j+8 and value[j..j+7].cmpIgnoreCase("december") == 0: + info.month = mDec + j += 8 + else: + raise newException(ValueError, "invalid month") + of "s": + var pd = parseInt(value[j..j+1], sv) + info.second = sv + j += pd + of "ss": + info.second = value[j..j+1].parseInt() + j += 2 + of "t": + if value[j] == 'P' and info.hour > 0 and info.hour < 12: + info.hour += 12 + j += 1 + of "tt": + if value[j..j+1] == "PM" and info.hour > 0 and info.hour < 12: + info.hour += 12 + j += 2 + of "yy": + # Assumes current century + var year = value[j..j+1].parseInt() + var thisCen = getLocalTime(getTime()).year div 100 + info.year = thisCen*100 + year + j += 2 + of "yyyy": + info.year = value[j..j+3].parseInt() + j += 4 + of "z": + if value[j] == '+': + info.timezone = parseInt($value[j+1]) + elif value[j] == '-': + info.timezone = 0-parseInt($value[j+1]) + else: + raise newException(ValueError, "Sign for timezone " & value[j]) + j += 2 + of "zz": + if value[j] == '+': + info.timezone = value[j+1..j+2].parseInt() + elif value[j] == '-': + info.timezone = 0-value[j+1..j+2].parseInt() + else: + raise newException(ValueError, "Sign for timezone " & value[j]) + j += 3 + of "zzz": + if value[j] == '+': + info.timezone = value[j+1..j+2].parseInt() + elif value[j] == '-': + info.timezone = 0-value[j+1..j+2].parseInt() + else: + raise newException(ValueError, "Sign for timezone " & value[j]) + j += 6 + of "ZZZ": + info.tzname = value[j..j+2].toUpper() + j += 3 + else: + # Ignore the token and move forward in the value string by the same length + j += token.len + +proc parse*(value, layout: string): TimeInfo = + ## This function parses a date/time string using the standard format identifiers (below) + ## The function defaults information not provided in the format string from the running program (timezone, month, year, etc) + ## + ## ========== ================================================================================= ================================================ + ## Specifier Description Example + ## ========== ================================================================================= ================================================ + ## d Numeric value of the day of the month, it will be one or two digits long. ``1/04/2012 -> 1``, ``21/04/2012 -> 21`` + ## dd Same as above, but always two digits. ``1/04/2012 -> 01``, ``21/04/2012 -> 21`` + ## ddd Three letter string which indicates the day of the week. ``Saturday -> Sat``, ``Monday -> Mon`` + ## dddd Full string for the day of the week. ``Saturday -> Saturday``, ``Monday -> Monday`` + ## h The hours in one digit if possible. Ranging from 0-12. ``5pm -> 5``, ``2am -> 2`` + ## hh The hours in two digits always. If the hour is one digit 0 is prepended. ``5pm -> 05``, ``11am -> 11`` + ## H The hours in one digit if possible, randing from 0-24. ``5pm -> 17``, ``2am -> 2`` + ## HH The hours in two digits always. 0 is prepended if the hour is one digit. ``5pm -> 17``, ``2am -> 02`` + ## m The minutes in 1 digit if possible. ``5:30 -> 30``, ``2:01 -> 1`` + ## mm Same as above but always 2 digits, 0 is prepended if the minute is one digit. ``5:30 -> 30``, ``2:01 -> 01`` + ## M The month in one digit if possible. ``September -> 9``, ``December -> 12`` + ## MM The month in two digits always. 0 is prepended. ``September -> 09``, ``December -> 12`` + ## MMM Abbreviated three-letter form of the month. ``September -> Sep``, ``December -> Dec`` + ## MMMM Full month string, properly capitalized. ``September -> September`` + ## s Seconds as one digit if possible. ``00:00:06 -> 6`` + ## ss Same as above but always two digits. 0 is prepended. ``00:00:06 -> 06`` + ## t ``A`` when time is in the AM. ``P`` when time is in the PM. + ## tt Same as above, but ``AM`` and ``PM`` instead of ``A`` and ``P`` respectively. + ## yy Displays the year to two digits. ``2012 -> 12`` + ## yyyy Displays the year to four digits. ``2012 -> 2012`` + ## z Displays the timezone offset from UTC. ``GMT+7 -> +7``, ``GMT-5 -> -5`` + ## zz Same as above but with leading 0. ``GMT+7 -> +07``, ``GMT-5 -> -05`` + ## zzz Same as above but with ``:00``. ``GMT+7 -> +07:00``, ``GMT-5 -> -05:00`` + ## ZZZ Displays the name of the timezone. ``GMT -> GMT``, ``EST -> EST`` + ## ========== ================================================================================= ================================================ + ## + ## Other strings can be inserted by putting them in ``''``. For example + ## ``hh'->'mm`` will give ``01->56``. The following characters can be + ## inserted without quoting them: ``:`` ``-`` ``(`` ``)`` ``/`` ``[`` ``]`` + ## ``,``. However you don't need to necessarily separate format specifiers, a + ## unambiguous format string like ``yyyyMMddhhmmss`` is valid too. + var i = 0 # pointer for format string + var j = 0 # pointer for value string + var token = "" + # Assumes current day of month, month and year, but time is reset to 00:00:00. Weekday will be reset after parsing. + var info = getLocalTime(getTime()) + info.hour = 0 + info.minute = 0 + info.second = 0 + while true: + case layout[i] + of ' ', '-', '/', ':', '\'', '\0', '(', ')', '[', ']', ',': + if token.len > 0: + parseToken(info, token, value, j) + # Reset token + token = "" + # Break if at end of line + if layout[i] == '\0': break + # Skip separator and everything between single quotes + # These are literals in both the layout and the value string + if layout[i] == '\'': + inc(i) + inc(j) + while layout[i] != '\'' and layout.len-1 > i: + inc(i) + inc(j) + else: + inc(i) + inc(j) + else: + # Check if the letter being added matches previous accumulated buffer. + if token.len < 1 or token[high(token)] == layout[i]: + token.add(layout[i]) + inc(i) + else: + parseToken(info, token, value, j) + token = "" + # Reset weekday as it might not have been provided and the default may be wrong + info.weekday = getLocalTime(timeInfoToTime(info)).weekday + return info + + when isMainModule: # $ date --date='@2147483647' # Tue 19 Jan 03:14:07 GMT 2038 var t = getGMTime(fromSeconds(2147483647)) - echo t.format("ddd dd MMM hh:mm:ss ZZZ yyyy") - echo t.format("ddd ddMMMhhmmssZZZyyyy") assert t.format("ddd dd MMM hh:mm:ss ZZZ yyyy") == "Tue 19 Jan 03:14:07 UTC 2038" assert t.format("ddd ddMMMhh:mm:ssZZZyyyy") == "Tue 19Jan03:14:07UTC2038" - + assert t.format("d dd ddd dddd h hh H HH m mm M MM MMM MMMM s" & - " ss t tt y yy yyy yyyy yyyyy z zz zzz ZZZ") == + " ss t tt y yy yyy yyyy yyyyy z zz zzz ZZZ") == "19 19 Tue Tuesday 3 03 3 03 14 14 1 01 Jan January 7 07 A AM 8 38 038 2038 02038 0 00 00:00 UTC" assert t.format("yyyyMMddhhmmss") == "20380119031407" - + var t2 = getGMTime(fromSeconds(160070789)) # Mon 27 Jan 16:06:29 GMT 1975 assert t2.format("d dd ddd dddd h hh H HH m mm M MM MMM MMMM s" & " ss t tt y yy yyy yyyy yyyyy z zz zzz ZZZ") == "27 27 Mon Monday 4 04 16 16 6 06 1 01 Jan January 29 29 P PM 5 75 975 1975 01975 0 00 00:00 UTC" - + when not defined(JS) and sizeof(Time) == 8: var t3 = getGMTime(fromSeconds(889067643645)) # Fri 7 Jun 19:20:45 BST 30143 assert t3.format("d dd ddd dddd h hh H HH m mm M MM MMM MMMM s" & - " ss t tt y yy yyy yyyy yyyyy z zz zzz ZZZ") == + " ss t tt y yy yyy yyyy yyyyy z zz zzz ZZZ") == "7 07 Fri Friday 6 06 18 18 20 20 6 06 Jun June 45 45 P PM 3 43 143 0143 30143 0 00 00:00 UTC" - assert t3.format(":,[]()-/") == ":,[]()-/" - + assert t3.format(":,[]()-/") == ":,[]()-/" + var t4 = getGMTime(fromSeconds(876124714)) # Mon 6 Oct 08:58:34 BST 1997 assert t4.format("M MM MMM MMMM") == "10 10 Oct October" - + # Interval tests assert((t4 - initInterval(years = 2)).format("yyyy") == "1995") assert((t4 - initInterval(years = 7, minutes = 34, seconds = 24)).format("yyyy mm ss") == "1990 24 10") + + var s = "Tuesday at 09:04am on Dec 15, 2015" + var f = "dddd at hh:mmtt on MMM d, yyyy" + assert($s.parse(f) == "Tue Dec 15 09:04:00 2015") + # ANSIC = "Mon Jan _2 15:04:05 2006" + s = "Thu Jan 12 15:04:05 2006" + f = "ddd MMM dd HH:mm:ss yyyy" + assert($s.parse(f) == "Thu Jan 12 15:04:05 2006") + # UnixDate = "Mon Jan _2 15:04:05 MST 2006" + s = "Thu Jan 12 15:04:05 MST 2006" + f = "ddd MMM dd HH:mm:ss ZZZ yyyy" + assert($s.parse(f) == "Thu Jan 12 15:04:05 2006") + # RubyDate = "Mon Jan 02 15:04:05 -0700 2006" + s = "Thu Jan 12 15:04:05 -07:00 2006" + f = "ddd MMM dd HH:mm:ss zzz yyyy" + assert($s.parse(f) == "Thu Jan 12 15:04:05 2006") + # RFC822 = "02 Jan 06 15:04 MST" + s = "12 Jan 16 15:04 MST" + f = "dd MMM yy HH:mm ZZZ" + assert($s.parse(f) == "Tue Jan 12 15:04:00 2016") + # RFC822Z = "02 Jan 06 15:04 -0700" # RFC822 with numeric zone + s = "12 Jan 16 15:04 -07:00" + f = "dd MMM yy HH:mm zzz" + assert($s.parse(f) == "Tue Jan 12 15:04:00 2016") + # RFC850 = "Monday, 02-Jan-06 15:04:05 MST" + s = "Monday, 12-Jan-06 15:04:05 MST" + f = "dddd, dd-MMM-yy HH:mm:ss ZZZ" + assert($s.parse(f) == "Thu Jan 12 15:04:05 2006") + # RFC1123 = "Mon, 02 Jan 2006 15:04:05 MST" + s = "Thu, 12 Jan 2006 15:04:05 MST" + f = "ddd, dd MMM yyyy HH:mm:ss ZZZ" + assert($s.parse(f) == "Thu Jan 12 15:04:05 2006") + # RFC1123Z = "Mon, 02 Jan 2006 15:04:05 -0700" # RFC1123 with numeric zone + s = "Thu, 12 Jan 2006 15:04:05 -07:00" + f = "ddd, dd MMM yyyy HH:mm:ss zzz" + assert($s.parse(f) == "Thu Jan 12 15:04:05 2006") + # RFC3339 = "2006-01-02T15:04:05Z07:00" + s = "2006-01-12T15:04:05Z-07:00" + f = "yyyy-MM-ddTHH:mm:ssZzzz" + assert($s.parse(f) == "Thu Jan 12 15:04:05 2006") + # RFC3339Nano = "2006-01-02T15:04:05.999999999Z07:00" + s = "2006-01-12T15:04:05.999999999Z-07:00" + f = "yyyy-MM-ddTHH:mm:ss.999999999Zzzz" + assert($s.parse(f) == "Thu Jan 12 15:04:05 2006") + # Kitchen = "3:04PM" + s = "3:04PM" + f = "h:mmtt" + assert "15:04:00" in $s.parse(f) + when not defined(testing): + echo "Kitchen: " & $s.parse(f) diff --git a/lib/pure/unicode.nim b/lib/pure/unicode.nim index c2eb001f6..5fd3c2418 100644 --- a/lib/pure/unicode.nim +++ b/lib/pure/unicode.nim @@ -19,7 +19,7 @@ type Rune16* = distinct int16 ## 16 bit Unicode character {.deprecated: [TRune: Rune, TRune16: Rune16].} - + proc `<=%`*(a, b: Rune): bool = return int(a) <=% int(b) proc `<%`*(a, b: Rune): bool = return int(a) <% int(b) proc `==`*(a, b: Rune): bool = return int(a) == int(b) @@ -39,7 +39,7 @@ proc runeLen*(s: string): int {.rtl, extern: "nuc$1".} = else: inc i inc(result) -proc runeLenAt*(s: string, i: int): int = +proc runeLenAt*(s: string, i: Natural): int = ## returns the number of bytes the rune starting at ``s[i]`` takes. if ord(s[i]) <=% 127: result = 1 elif ord(s[i]) shr 5 == 0b110: result = 2 @@ -58,7 +58,7 @@ template fastRuneAt*(s: string, i: int, result: expr, doInc = true) = when doInc: inc(i) elif ord(s[i]) shr 5 == 0b110: # assert(ord(s[i+1]) shr 6 == 0b10) - result = Rune((ord(s[i]) and (ones(5))) shl 6 or + result = Rune((ord(s[i]) and (ones(5))) shl 6 or (ord(s[i+1]) and ones(6))) when doInc: inc(i, 2) elif ord(s[i]) shr 4 == 0b1110: @@ -77,7 +77,7 @@ template fastRuneAt*(s: string, i: int, result: expr, doInc = true) = (ord(s[i+2]) and ones(6)) shl 6 or (ord(s[i+3]) and ones(6))) when doInc: inc(i, 4) - elif ord(s[i]) shr 2 == 0b111110: + elif ord(s[i]) shr 2 == 0b111110: # assert(ord(s[i+1]) shr 6 == 0b10) # assert(ord(s[i+2]) shr 6 == 0b10) # assert(ord(s[i+3]) shr 6 == 0b10) @@ -88,7 +88,7 @@ template fastRuneAt*(s: string, i: int, result: expr, doInc = true) = (ord(s[i+3]) and ones(6)) shl 6 or (ord(s[i+4]) and ones(6))) when doInc: inc(i, 5) - elif ord(s[i]) shr 1 == 0b1111110: + elif ord(s[i]) shr 1 == 0b1111110: # assert(ord(s[i+1]) shr 6 == 0b10) # assert(ord(s[i+2]) shr 6 == 0b10) # assert(ord(s[i+3]) shr 6 == 0b10) @@ -105,11 +105,36 @@ template fastRuneAt*(s: string, i: int, result: expr, doInc = true) = result = Rune(ord(s[i])) when doInc: inc(i) -proc runeAt*(s: string, i: int): Rune = +proc validateUtf8*(s: string): int = + ## returns the position of the invalid byte in ``s`` if the string ``s`` does + ## not hold valid UTF-8 data. Otherwise -1 is returned. + var i = 0 + let L = s.len + while i < L: + if ord(s[i]) <=% 127: + inc(i) + elif ord(s[i]) shr 5 == 0b110: + if i+1 < L and ord(s[i+1]) shr 6 == 0b10: inc(i, 2) + else: return i + elif ord(s[i]) shr 4 == 0b1110: + if i+2 < L and ord(s[i+1]) shr 6 == 0b10 and ord(s[i+2]) shr 6 == 0b10: + inc i, 3 + else: return i + elif ord(s[i]) shr 3 == 0b11110: + if i+3 < L and ord(s[i+1]) shr 6 == 0b10 and + ord(s[i+2]) shr 6 == 0b10 and + ord(s[i+3]) shr 6 == 0b10: + inc i, 4 + else: return i + else: + return i + return -1 + +proc runeAt*(s: string, i: Natural): Rune = ## returns the unicode character in `s` at byte index `i` fastRuneAt(s, i, result, false) -proc toUTF8*(c: Rune): string {.rtl, extern: "nuc$1".} = +proc toUTF8*(c: Rune): string {.rtl, extern: "nuc$1".} = ## converts a rune into its UTF8 representation var i = RuneImpl(c) if i <=% 127: @@ -118,21 +143,35 @@ proc toUTF8*(c: Rune): string {.rtl, extern: "nuc$1".} = elif i <=% 0x07FF: result = newString(2) result[0] = chr((i shr 6) or 0b110_00000) - result[1] = chr((i and ones(6)) or 0b10_000000) + result[1] = chr((i and ones(6)) or 0b10_0000_00) elif i <=% 0xFFFF: result = newString(3) result[0] = chr(i shr 12 or 0b1110_0000) result[1] = chr(i shr 6 and ones(6) or 0b10_0000_00) result[2] = chr(i and ones(6) or 0b10_0000_00) - elif i <=% 0x0010FFFF: + elif i <=% 0x001FFFFF: result = newString(4) result[0] = chr(i shr 18 or 0b1111_0000) result[1] = chr(i shr 12 and ones(6) or 0b10_0000_00) result[2] = chr(i shr 6 and ones(6) or 0b10_0000_00) result[3] = chr(i and ones(6) or 0b10_0000_00) + elif i <=% 0x03FFFFFF: + result = newString(5) + result[0] = chr(i shr 24 or 0b111110_00) + result[1] = chr(i shr 18 and ones(6) or 0b10_0000_00) + result[2] = chr(i shr 12 and ones(6) or 0b10_0000_00) + result[3] = chr(i shr 6 and ones(6) or 0b10_0000_00) + result[4] = chr(i and ones(6) or 0b10_0000_00) + elif i <=% 0x7FFFFFFF: + result = newString(6) + result[0] = chr(i shr 30 or 0b1111110_0) + result[1] = chr(i shr 24 and ones(6) or 0b10_0000_00) + result[2] = chr(i shr 18 and ones(6) or 0b10_0000_00) + result[3] = chr(i shr 12 and ones(6) or 0b10_0000_00) + result[4] = chr(i shr 6 and ones(6) or 0b10_0000_00) + result[5] = chr(i and ones(6) or 0b10_0000_00) else: - result = newString(1) - result[0] = chr(i) + discard # error, exception? proc `$`*(rune: Rune): string = ## converts a rune to a string @@ -145,967 +184,967 @@ proc `$`*(runes: seq[Rune]): string = const alphaRanges = [ - 0x00d8, 0x00f6, # - - 0x00f8, 0x01f5, # - - 0x0250, 0x02a8, # - - 0x038e, 0x03a1, # - - 0x03a3, 0x03ce, # - - 0x03d0, 0x03d6, # - - 0x03e2, 0x03f3, # - - 0x0490, 0x04c4, # - - 0x0561, 0x0587, # - - 0x05d0, 0x05ea, # - - 0x05f0, 0x05f2, # - - 0x0621, 0x063a, # - - 0x0640, 0x064a, # - - 0x0671, 0x06b7, # - - 0x06ba, 0x06be, # - - 0x06c0, 0x06ce, # - - 0x06d0, 0x06d3, # - - 0x0905, 0x0939, # - - 0x0958, 0x0961, # - - 0x0985, 0x098c, # - - 0x098f, 0x0990, # - - 0x0993, 0x09a8, # - - 0x09aa, 0x09b0, # - - 0x09b6, 0x09b9, # - - 0x09dc, 0x09dd, # - - 0x09df, 0x09e1, # - - 0x09f0, 0x09f1, # - - 0x0a05, 0x0a0a, # - - 0x0a0f, 0x0a10, # - - 0x0a13, 0x0a28, # - - 0x0a2a, 0x0a30, # - - 0x0a32, 0x0a33, # - - 0x0a35, 0x0a36, # - - 0x0a38, 0x0a39, # - - 0x0a59, 0x0a5c, # - - 0x0a85, 0x0a8b, # - - 0x0a8f, 0x0a91, # - - 0x0a93, 0x0aa8, # - - 0x0aaa, 0x0ab0, # - - 0x0ab2, 0x0ab3, # - - 0x0ab5, 0x0ab9, # - - 0x0b05, 0x0b0c, # - - 0x0b0f, 0x0b10, # - - 0x0b13, 0x0b28, # - - 0x0b2a, 0x0b30, # - - 0x0b32, 0x0b33, # - - 0x0b36, 0x0b39, # - - 0x0b5c, 0x0b5d, # - - 0x0b5f, 0x0b61, # - - 0x0b85, 0x0b8a, # - - 0x0b8e, 0x0b90, # - - 0x0b92, 0x0b95, # - - 0x0b99, 0x0b9a, # - - 0x0b9e, 0x0b9f, # - - 0x0ba3, 0x0ba4, # - - 0x0ba8, 0x0baa, # - - 0x0bae, 0x0bb5, # - - 0x0bb7, 0x0bb9, # - - 0x0c05, 0x0c0c, # - - 0x0c0e, 0x0c10, # - - 0x0c12, 0x0c28, # - - 0x0c2a, 0x0c33, # - - 0x0c35, 0x0c39, # - - 0x0c60, 0x0c61, # - - 0x0c85, 0x0c8c, # - - 0x0c8e, 0x0c90, # - - 0x0c92, 0x0ca8, # - - 0x0caa, 0x0cb3, # - - 0x0cb5, 0x0cb9, # - - 0x0ce0, 0x0ce1, # - - 0x0d05, 0x0d0c, # - - 0x0d0e, 0x0d10, # - - 0x0d12, 0x0d28, # - - 0x0d2a, 0x0d39, # - - 0x0d60, 0x0d61, # - - 0x0e01, 0x0e30, # - - 0x0e32, 0x0e33, # - - 0x0e40, 0x0e46, # - - 0x0e5a, 0x0e5b, # - - 0x0e81, 0x0e82, # - - 0x0e87, 0x0e88, # - - 0x0e94, 0x0e97, # - - 0x0e99, 0x0e9f, # - - 0x0ea1, 0x0ea3, # - - 0x0eaa, 0x0eab, # - - 0x0ead, 0x0eae, # - - 0x0eb2, 0x0eb3, # - - 0x0ec0, 0x0ec4, # - - 0x0edc, 0x0edd, # - - 0x0f18, 0x0f19, # - - 0x0f40, 0x0f47, # - - 0x0f49, 0x0f69, # - - 0x10d0, 0x10f6, # - - 0x1100, 0x1159, # - - 0x115f, 0x11a2, # - - 0x11a8, 0x11f9, # - - 0x1e00, 0x1e9b, # - - 0x1f50, 0x1f57, # - - 0x1f80, 0x1fb4, # - - 0x1fb6, 0x1fbc, # - - 0x1fc2, 0x1fc4, # - - 0x1fc6, 0x1fcc, # - - 0x1fd0, 0x1fd3, # - - 0x1fd6, 0x1fdb, # - - 0x1fe0, 0x1fec, # - - 0x1ff2, 0x1ff4, # - - 0x1ff6, 0x1ffc, # - - 0x210a, 0x2113, # - - 0x2115, 0x211d, # - - 0x2120, 0x2122, # - - 0x212a, 0x2131, # - - 0x2133, 0x2138, # - - 0x3041, 0x3094, # - - 0x30a1, 0x30fa, # - - 0x3105, 0x312c, # - - 0x3131, 0x318e, # - - 0x3192, 0x319f, # - - 0x3260, 0x327b, # - - 0x328a, 0x32b0, # - - 0x32d0, 0x32fe, # - - 0x3300, 0x3357, # - - 0x3371, 0x3376, # - - 0x337b, 0x3394, # - - 0x3399, 0x339e, # - - 0x33a9, 0x33ad, # - - 0x33b0, 0x33c1, # - - 0x33c3, 0x33c5, # - - 0x33c7, 0x33d7, # - - 0x33d9, 0x33dd, # - - 0x4e00, 0x9fff, # - - 0xac00, 0xd7a3, # - - 0xf900, 0xfb06, # - - 0xfb13, 0xfb17, # - - 0xfb1f, 0xfb28, # - - 0xfb2a, 0xfb36, # - - 0xfb38, 0xfb3c, # - - 0xfb40, 0xfb41, # - - 0xfb43, 0xfb44, # - - 0xfb46, 0xfbb1, # - - 0xfbd3, 0xfd3d, # - - 0xfd50, 0xfd8f, # - - 0xfd92, 0xfdc7, # - - 0xfdf0, 0xfdf9, # - - 0xfe70, 0xfe72, # - - 0xfe76, 0xfefc, # - - 0xff66, 0xff6f, # - - 0xff71, 0xff9d, # - - 0xffa0, 0xffbe, # - - 0xffc2, 0xffc7, # - - 0xffca, 0xffcf, # - - 0xffd2, 0xffd7, # - - 0xffda, 0xffdc] # - + 0x00d8, 0x00f6, # - + 0x00f8, 0x01f5, # - + 0x0250, 0x02a8, # - + 0x038e, 0x03a1, # - + 0x03a3, 0x03ce, # - + 0x03d0, 0x03d6, # - + 0x03e2, 0x03f3, # - + 0x0490, 0x04c4, # - + 0x0561, 0x0587, # - + 0x05d0, 0x05ea, # - + 0x05f0, 0x05f2, # - + 0x0621, 0x063a, # - + 0x0640, 0x064a, # - + 0x0671, 0x06b7, # - + 0x06ba, 0x06be, # - + 0x06c0, 0x06ce, # - + 0x06d0, 0x06d3, # - + 0x0905, 0x0939, # - + 0x0958, 0x0961, # - + 0x0985, 0x098c, # - + 0x098f, 0x0990, # - + 0x0993, 0x09a8, # - + 0x09aa, 0x09b0, # - + 0x09b6, 0x09b9, # - + 0x09dc, 0x09dd, # - + 0x09df, 0x09e1, # - + 0x09f0, 0x09f1, # - + 0x0a05, 0x0a0a, # - + 0x0a0f, 0x0a10, # - + 0x0a13, 0x0a28, # - + 0x0a2a, 0x0a30, # - + 0x0a32, 0x0a33, # - + 0x0a35, 0x0a36, # - + 0x0a38, 0x0a39, # - + 0x0a59, 0x0a5c, # - + 0x0a85, 0x0a8b, # - + 0x0a8f, 0x0a91, # - + 0x0a93, 0x0aa8, # - + 0x0aaa, 0x0ab0, # - + 0x0ab2, 0x0ab3, # - + 0x0ab5, 0x0ab9, # - + 0x0b05, 0x0b0c, # - + 0x0b0f, 0x0b10, # - + 0x0b13, 0x0b28, # - + 0x0b2a, 0x0b30, # - + 0x0b32, 0x0b33, # - + 0x0b36, 0x0b39, # - + 0x0b5c, 0x0b5d, # - + 0x0b5f, 0x0b61, # - + 0x0b85, 0x0b8a, # - + 0x0b8e, 0x0b90, # - + 0x0b92, 0x0b95, # - + 0x0b99, 0x0b9a, # - + 0x0b9e, 0x0b9f, # - + 0x0ba3, 0x0ba4, # - + 0x0ba8, 0x0baa, # - + 0x0bae, 0x0bb5, # - + 0x0bb7, 0x0bb9, # - + 0x0c05, 0x0c0c, # - + 0x0c0e, 0x0c10, # - + 0x0c12, 0x0c28, # - + 0x0c2a, 0x0c33, # - + 0x0c35, 0x0c39, # - + 0x0c60, 0x0c61, # - + 0x0c85, 0x0c8c, # - + 0x0c8e, 0x0c90, # - + 0x0c92, 0x0ca8, # - + 0x0caa, 0x0cb3, # - + 0x0cb5, 0x0cb9, # - + 0x0ce0, 0x0ce1, # - + 0x0d05, 0x0d0c, # - + 0x0d0e, 0x0d10, # - + 0x0d12, 0x0d28, # - + 0x0d2a, 0x0d39, # - + 0x0d60, 0x0d61, # - + 0x0e01, 0x0e30, # - + 0x0e32, 0x0e33, # - + 0x0e40, 0x0e46, # - + 0x0e5a, 0x0e5b, # - + 0x0e81, 0x0e82, # - + 0x0e87, 0x0e88, # - + 0x0e94, 0x0e97, # - + 0x0e99, 0x0e9f, # - + 0x0ea1, 0x0ea3, # - + 0x0eaa, 0x0eab, # - + 0x0ead, 0x0eae, # - + 0x0eb2, 0x0eb3, # - + 0x0ec0, 0x0ec4, # - + 0x0edc, 0x0edd, # - + 0x0f18, 0x0f19, # - + 0x0f40, 0x0f47, # - + 0x0f49, 0x0f69, # - + 0x10d0, 0x10f6, # - + 0x1100, 0x1159, # - + 0x115f, 0x11a2, # - + 0x11a8, 0x11f9, # - + 0x1e00, 0x1e9b, # - + 0x1f50, 0x1f57, # - + 0x1f80, 0x1fb4, # - + 0x1fb6, 0x1fbc, # - + 0x1fc2, 0x1fc4, # - + 0x1fc6, 0x1fcc, # - + 0x1fd0, 0x1fd3, # - + 0x1fd6, 0x1fdb, # - + 0x1fe0, 0x1fec, # - + 0x1ff2, 0x1ff4, # - + 0x1ff6, 0x1ffc, # - + 0x210a, 0x2113, # - + 0x2115, 0x211d, # - + 0x2120, 0x2122, # - + 0x212a, 0x2131, # - + 0x2133, 0x2138, # - + 0x3041, 0x3094, # - + 0x30a1, 0x30fa, # - + 0x3105, 0x312c, # - + 0x3131, 0x318e, # - + 0x3192, 0x319f, # - + 0x3260, 0x327b, # - + 0x328a, 0x32b0, # - + 0x32d0, 0x32fe, # - + 0x3300, 0x3357, # - + 0x3371, 0x3376, # - + 0x337b, 0x3394, # - + 0x3399, 0x339e, # - + 0x33a9, 0x33ad, # - + 0x33b0, 0x33c1, # - + 0x33c3, 0x33c5, # - + 0x33c7, 0x33d7, # - + 0x33d9, 0x33dd, # - + 0x4e00, 0x9fff, # - + 0xac00, 0xd7a3, # - + 0xf900, 0xfb06, # - + 0xfb13, 0xfb17, # - + 0xfb1f, 0xfb28, # - + 0xfb2a, 0xfb36, # - + 0xfb38, 0xfb3c, # - + 0xfb40, 0xfb41, # - + 0xfb43, 0xfb44, # - + 0xfb46, 0xfbb1, # - + 0xfbd3, 0xfd3d, # - + 0xfd50, 0xfd8f, # - + 0xfd92, 0xfdc7, # - + 0xfdf0, 0xfdf9, # - + 0xfe70, 0xfe72, # - + 0xfe76, 0xfefc, # - + 0xff66, 0xff6f, # - + 0xff71, 0xff9d, # - + 0xffa0, 0xffbe, # - + 0xffc2, 0xffc7, # - + 0xffca, 0xffcf, # - + 0xffd2, 0xffd7, # - + 0xffda, 0xffdc] # - alphaSinglets = [ - 0x00aa, # - 0x00b5, # - 0x00ba, # - 0x03da, # - 0x03dc, # - 0x03de, # - 0x03e0, # - 0x06d5, # - 0x09b2, # - 0x0a5e, # - 0x0a8d, # - 0x0ae0, # - 0x0b9c, # - 0x0cde, # - 0x0e4f, # - 0x0e84, # - 0x0e8a, # - 0x0e8d, # - 0x0ea5, # - 0x0ea7, # - 0x0eb0, # - 0x0ebd, # - 0x1fbe, # - 0x207f, # - 0x20a8, # - 0x2102, # - 0x2107, # - 0x2124, # - 0x2126, # - 0x2128, # - 0xfb3e, # - 0xfe74] # + 0x00aa, # + 0x00b5, # + 0x00ba, # + 0x03da, # + 0x03dc, # + 0x03de, # + 0x03e0, # + 0x06d5, # + 0x09b2, # + 0x0a5e, # + 0x0a8d, # + 0x0ae0, # + 0x0b9c, # + 0x0cde, # + 0x0e4f, # + 0x0e84, # + 0x0e8a, # + 0x0e8d, # + 0x0ea5, # + 0x0ea7, # + 0x0eb0, # + 0x0ebd, # + 0x1fbe, # + 0x207f, # + 0x20a8, # + 0x2102, # + 0x2107, # + 0x2124, # + 0x2126, # + 0x2128, # + 0xfb3e, # + 0xfe74] # spaceRanges = [ - 0x0009, 0x000a, # tab and newline - 0x0020, 0x0020, # space - 0x00a0, 0x00a0, # - 0x2000, 0x200b, # - - 0x2028, 0x2029, # - 0x3000, 0x3000, # - 0xfeff, 0xfeff] # + 0x0009, 0x000a, # tab and newline + 0x0020, 0x0020, # space + 0x00a0, 0x00a0, # + 0x2000, 0x200b, # - + 0x2028, 0x2029, # - 0x3000, 0x3000, # + 0xfeff, 0xfeff] # toupperRanges = [ - 0x0061, 0x007a, 468, # a-z A-Z - 0x00e0, 0x00f6, 468, # - - - 0x00f8, 0x00fe, 468, # - - - 0x0256, 0x0257, 295, # - - - 0x0258, 0x0259, 298, # - - - 0x028a, 0x028b, 283, # - - - 0x03ad, 0x03af, 463, # - - - 0x03b1, 0x03c1, 468, # - - - 0x03c3, 0x03cb, 468, # - - - 0x03cd, 0x03ce, 437, # - - - 0x0430, 0x044f, 468, # - - - 0x0451, 0x045c, 420, # - - - 0x045e, 0x045f, 420, # - - - 0x0561, 0x0586, 452, # - - - 0x1f00, 0x1f07, 508, # - - - 0x1f10, 0x1f15, 508, # - - - 0x1f20, 0x1f27, 508, # - - - 0x1f30, 0x1f37, 508, # - - - 0x1f40, 0x1f45, 508, # - - - 0x1f60, 0x1f67, 508, # - - - 0x1f70, 0x1f71, 574, # - - - 0x1f72, 0x1f75, 586, # - - - 0x1f76, 0x1f77, 600, # - - - 0x1f78, 0x1f79, 628, # - - - 0x1f7a, 0x1f7b, 612, # - - - 0x1f7c, 0x1f7d, 626, # - - - 0x1f80, 0x1f87, 508, # - - - 0x1f90, 0x1f97, 508, # - - - 0x1fa0, 0x1fa7, 508, # - - - 0x1fb0, 0x1fb1, 508, # - - - 0x1fd0, 0x1fd1, 508, # - - - 0x1fe0, 0x1fe1, 508, # - - - 0x2170, 0x217f, 484, # - - - 0x24d0, 0x24e9, 474, # - - - 0xff41, 0xff5a, 468] # - - + 0x0061, 0x007a, 468, # a-z A-Z + 0x00e0, 0x00f6, 468, # - - + 0x00f8, 0x00fe, 468, # - - + 0x0256, 0x0257, 295, # - - + 0x0258, 0x0259, 298, # - - + 0x028a, 0x028b, 283, # - - + 0x03ad, 0x03af, 463, # - - + 0x03b1, 0x03c1, 468, # - - + 0x03c3, 0x03cb, 468, # - - + 0x03cd, 0x03ce, 437, # - - + 0x0430, 0x044f, 468, # - - + 0x0451, 0x045c, 420, # - - + 0x045e, 0x045f, 420, # - - + 0x0561, 0x0586, 452, # - - + 0x1f00, 0x1f07, 508, # - - + 0x1f10, 0x1f15, 508, # - - + 0x1f20, 0x1f27, 508, # - - + 0x1f30, 0x1f37, 508, # - - + 0x1f40, 0x1f45, 508, # - - + 0x1f60, 0x1f67, 508, # - - + 0x1f70, 0x1f71, 574, # - - + 0x1f72, 0x1f75, 586, # - - + 0x1f76, 0x1f77, 600, # - - + 0x1f78, 0x1f79, 628, # - - + 0x1f7a, 0x1f7b, 612, # - - + 0x1f7c, 0x1f7d, 626, # - - + 0x1f80, 0x1f87, 508, # - - + 0x1f90, 0x1f97, 508, # - - + 0x1fa0, 0x1fa7, 508, # - - + 0x1fb0, 0x1fb1, 508, # - - + 0x1fd0, 0x1fd1, 508, # - - + 0x1fe0, 0x1fe1, 508, # - - + 0x2170, 0x217f, 484, # - - + 0x24d0, 0x24e9, 474, # - - + 0xff41, 0xff5a, 468] # - - toupperSinglets = [ - 0x00ff, 621, # - 0x0101, 499, # - 0x0103, 499, # - 0x0105, 499, # - 0x0107, 499, # - 0x0109, 499, # - 0x010b, 499, # - 0x010d, 499, # - 0x010f, 499, # - 0x0111, 499, # - 0x0113, 499, # - 0x0115, 499, # - 0x0117, 499, # - 0x0119, 499, # - 0x011b, 499, # - 0x011d, 499, # - 0x011f, 499, # - 0x0121, 499, # - 0x0123, 499, # - 0x0125, 499, # - 0x0127, 499, # - 0x0129, 499, # - 0x012b, 499, # - 0x012d, 499, # - 0x012f, 499, # - 0x0131, 268, # I - 0x0133, 499, # - 0x0135, 499, # - 0x0137, 499, # - 0x013a, 499, # - 0x013c, 499, # - 0x013e, 499, # - 0x0140, 499, # - 0x0142, 499, # - 0x0144, 499, # - 0x0146, 499, # - 0x0148, 499, # - 0x014b, 499, # - 0x014d, 499, # - 0x014f, 499, # - 0x0151, 499, # - 0x0153, 499, # - 0x0155, 499, # - 0x0157, 499, # - 0x0159, 499, # - 0x015b, 499, # - 0x015d, 499, # - 0x015f, 499, # - 0x0161, 499, # - 0x0163, 499, # - 0x0165, 499, # - 0x0167, 499, # - 0x0169, 499, # - 0x016b, 499, # - 0x016d, 499, # - 0x016f, 499, # - 0x0171, 499, # - 0x0173, 499, # - 0x0175, 499, # - 0x0177, 499, # - 0x017a, 499, # - 0x017c, 499, # - 0x017e, 499, # - 0x017f, 200, # S - 0x0183, 499, # - 0x0185, 499, # - 0x0188, 499, # - 0x018c, 499, # - 0x0192, 499, # - 0x0199, 499, # - 0x01a1, 499, # - 0x01a3, 499, # - 0x01a5, 499, # - 0x01a8, 499, # - 0x01ad, 499, # - 0x01b0, 499, # - 0x01b4, 499, # - 0x01b6, 499, # - 0x01b9, 499, # - 0x01bd, 499, # - 0x01c5, 499, # - 0x01c6, 498, # - 0x01c8, 499, # - 0x01c9, 498, # - 0x01cb, 499, # - 0x01cc, 498, # - 0x01ce, 499, # - 0x01d0, 499, # - 0x01d2, 499, # - 0x01d4, 499, # - 0x01d6, 499, # - 0x01d8, 499, # - 0x01da, 499, # - 0x01dc, 499, # - 0x01df, 499, # - 0x01e1, 499, # - 0x01e3, 499, # - 0x01e5, 499, # - 0x01e7, 499, # - 0x01e9, 499, # - 0x01eb, 499, # - 0x01ed, 499, # - 0x01ef, 499, # - 0x01f2, 499, # - 0x01f3, 498, # - 0x01f5, 499, # - 0x01fb, 499, # - 0x01fd, 499, # - 0x01ff, 499, # - 0x0201, 499, # - 0x0203, 499, # - 0x0205, 499, # - 0x0207, 499, # - 0x0209, 499, # - 0x020b, 499, # - 0x020d, 499, # - 0x020f, 499, # - 0x0211, 499, # - 0x0213, 499, # - 0x0215, 499, # - 0x0217, 499, # - 0x0253, 290, # - 0x0254, 294, # - 0x025b, 297, # - 0x0260, 295, # - 0x0263, 293, # - 0x0268, 291, # - 0x0269, 289, # - 0x026f, 289, # - 0x0272, 287, # - 0x0283, 282, # - 0x0288, 282, # - 0x0292, 281, # - 0x03ac, 462, # - 0x03cc, 436, # - 0x03d0, 438, # - 0x03d1, 443, # - 0x03d5, 453, # - 0x03d6, 446, # - 0x03e3, 499, # - 0x03e5, 499, # - 0x03e7, 499, # - 0x03e9, 499, # - 0x03eb, 499, # - 0x03ed, 499, # - 0x03ef, 499, # - 0x03f0, 414, # - 0x03f1, 420, # - 0x0461, 499, # - 0x0463, 499, # - 0x0465, 499, # - 0x0467, 499, # - 0x0469, 499, # - 0x046b, 499, # - 0x046d, 499, # - 0x046f, 499, # - 0x0471, 499, # - 0x0473, 499, # - 0x0475, 499, # - 0x0477, 499, # - 0x0479, 499, # - 0x047b, 499, # - 0x047d, 499, # - 0x047f, 499, # - 0x0481, 499, # - 0x0491, 499, # - 0x0493, 499, # - 0x0495, 499, # - 0x0497, 499, # - 0x0499, 499, # - 0x049b, 499, # - 0x049d, 499, # - 0x049f, 499, # - 0x04a1, 499, # - 0x04a3, 499, # - 0x04a5, 499, # - 0x04a7, 499, # - 0x04a9, 499, # - 0x04ab, 499, # - 0x04ad, 499, # - 0x04af, 499, # - 0x04b1, 499, # - 0x04b3, 499, # - 0x04b5, 499, # - 0x04b7, 499, # - 0x04b9, 499, # - 0x04bb, 499, # - 0x04bd, 499, # - 0x04bf, 499, # - 0x04c2, 499, # - 0x04c4, 499, # - 0x04c8, 499, # - 0x04cc, 499, # - 0x04d1, 499, # - 0x04d3, 499, # - 0x04d5, 499, # - 0x04d7, 499, # - 0x04d9, 499, # - 0x04db, 499, # - 0x04dd, 499, # - 0x04df, 499, # - 0x04e1, 499, # - 0x04e3, 499, # - 0x04e5, 499, # - 0x04e7, 499, # - 0x04e9, 499, # - 0x04eb, 499, # - 0x04ef, 499, # - 0x04f1, 499, # - 0x04f3, 499, # - 0x04f5, 499, # - 0x04f9, 499, # - 0x1e01, 499, # - 0x1e03, 499, # - 0x1e05, 499, # - 0x1e07, 499, # - 0x1e09, 499, # - 0x1e0b, 499, # - 0x1e0d, 499, # - 0x1e0f, 499, # - 0x1e11, 499, # - 0x1e13, 499, # - 0x1e15, 499, # - 0x1e17, 499, # - 0x1e19, 499, # - 0x1e1b, 499, # - 0x1e1d, 499, # - 0x1e1f, 499, # - 0x1e21, 499, # - 0x1e23, 499, # - 0x1e25, 499, # - 0x1e27, 499, # - 0x1e29, 499, # - 0x1e2b, 499, # - 0x1e2d, 499, # - 0x1e2f, 499, # - 0x1e31, 499, # - 0x1e33, 499, # - 0x1e35, 499, # - 0x1e37, 499, # - 0x1e39, 499, # - 0x1e3b, 499, # - 0x1e3d, 499, # - 0x1e3f, 499, # - 0x1e41, 499, # - 0x1e43, 499, # - 0x1e45, 499, # - 0x1e47, 499, # - 0x1e49, 499, # - 0x1e4b, 499, # - 0x1e4d, 499, # - 0x1e4f, 499, # - 0x1e51, 499, # - 0x1e53, 499, # - 0x1e55, 499, # - 0x1e57, 499, # - 0x1e59, 499, # - 0x1e5b, 499, # - 0x1e5d, 499, # - 0x1e5f, 499, # - 0x1e61, 499, # - 0x1e63, 499, # - 0x1e65, 499, # - 0x1e67, 499, # - 0x1e69, 499, # - 0x1e6b, 499, # - 0x1e6d, 499, # - 0x1e6f, 499, # - 0x1e71, 499, # - 0x1e73, 499, # - 0x1e75, 499, # - 0x1e77, 499, # - 0x1e79, 499, # - 0x1e7b, 499, # - 0x1e7d, 499, # - 0x1e7f, 499, # - 0x1e81, 499, # - 0x1e83, 499, # - 0x1e85, 499, # - 0x1e87, 499, # - 0x1e89, 499, # - 0x1e8b, 499, # - 0x1e8d, 499, # - 0x1e8f, 499, # - 0x1e91, 499, # - 0x1e93, 499, # - 0x1e95, 499, # - 0x1ea1, 499, # - 0x1ea3, 499, # - 0x1ea5, 499, # - 0x1ea7, 499, # - 0x1ea9, 499, # - 0x1eab, 499, # - 0x1ead, 499, # - 0x1eaf, 499, # - 0x1eb1, 499, # - 0x1eb3, 499, # - 0x1eb5, 499, # - 0x1eb7, 499, # - 0x1eb9, 499, # - 0x1ebb, 499, # - 0x1ebd, 499, # - 0x1ebf, 499, # - 0x1ec1, 499, # - 0x1ec3, 499, # - 0x1ec5, 499, # - 0x1ec7, 499, # - 0x1ec9, 499, # - 0x1ecb, 499, # - 0x1ecd, 499, # - 0x1ecf, 499, # - 0x1ed1, 499, # - 0x1ed3, 499, # - 0x1ed5, 499, # - 0x1ed7, 499, # - 0x1ed9, 499, # - 0x1edb, 499, # - 0x1edd, 499, # - 0x1edf, 499, # - 0x1ee1, 499, # - 0x1ee3, 499, # - 0x1ee5, 499, # - 0x1ee7, 499, # - 0x1ee9, 499, # - 0x1eeb, 499, # - 0x1eed, 499, # - 0x1eef, 499, # - 0x1ef1, 499, # - 0x1ef3, 499, # - 0x1ef5, 499, # - 0x1ef7, 499, # - 0x1ef9, 499, # - 0x1f51, 508, # - 0x1f53, 508, # - 0x1f55, 508, # - 0x1f57, 508, # - 0x1fb3, 509, # - 0x1fc3, 509, # - 0x1fe5, 507, # - 0x1ff3, 509] # + 0x00ff, 621, # + 0x0101, 499, # + 0x0103, 499, # + 0x0105, 499, # + 0x0107, 499, # + 0x0109, 499, # + 0x010b, 499, # + 0x010d, 499, # + 0x010f, 499, # + 0x0111, 499, # + 0x0113, 499, # + 0x0115, 499, # + 0x0117, 499, # + 0x0119, 499, # + 0x011b, 499, # + 0x011d, 499, # + 0x011f, 499, # + 0x0121, 499, # + 0x0123, 499, # + 0x0125, 499, # + 0x0127, 499, # + 0x0129, 499, # + 0x012b, 499, # + 0x012d, 499, # + 0x012f, 499, # + 0x0131, 268, # I + 0x0133, 499, # + 0x0135, 499, # + 0x0137, 499, # + 0x013a, 499, # + 0x013c, 499, # + 0x013e, 499, # + 0x0140, 499, # + 0x0142, 499, # + 0x0144, 499, # + 0x0146, 499, # + 0x0148, 499, # + 0x014b, 499, # + 0x014d, 499, # + 0x014f, 499, # + 0x0151, 499, # + 0x0153, 499, # + 0x0155, 499, # + 0x0157, 499, # + 0x0159, 499, # + 0x015b, 499, # + 0x015d, 499, # + 0x015f, 499, # + 0x0161, 499, # + 0x0163, 499, # + 0x0165, 499, # + 0x0167, 499, # + 0x0169, 499, # + 0x016b, 499, # + 0x016d, 499, # + 0x016f, 499, # + 0x0171, 499, # + 0x0173, 499, # + 0x0175, 499, # + 0x0177, 499, # + 0x017a, 499, # + 0x017c, 499, # + 0x017e, 499, # + 0x017f, 200, # S + 0x0183, 499, # + 0x0185, 499, # + 0x0188, 499, # + 0x018c, 499, # + 0x0192, 499, # + 0x0199, 499, # + 0x01a1, 499, # + 0x01a3, 499, # + 0x01a5, 499, # + 0x01a8, 499, # + 0x01ad, 499, # + 0x01b0, 499, # + 0x01b4, 499, # + 0x01b6, 499, # + 0x01b9, 499, # + 0x01bd, 499, # + 0x01c5, 499, # + 0x01c6, 498, # + 0x01c8, 499, # + 0x01c9, 498, # + 0x01cb, 499, # + 0x01cc, 498, # + 0x01ce, 499, # + 0x01d0, 499, # + 0x01d2, 499, # + 0x01d4, 499, # + 0x01d6, 499, # + 0x01d8, 499, # + 0x01da, 499, # + 0x01dc, 499, # + 0x01df, 499, # + 0x01e1, 499, # + 0x01e3, 499, # + 0x01e5, 499, # + 0x01e7, 499, # + 0x01e9, 499, # + 0x01eb, 499, # + 0x01ed, 499, # + 0x01ef, 499, # + 0x01f2, 499, # + 0x01f3, 498, # + 0x01f5, 499, # + 0x01fb, 499, # + 0x01fd, 499, # + 0x01ff, 499, # + 0x0201, 499, # + 0x0203, 499, # + 0x0205, 499, # + 0x0207, 499, # + 0x0209, 499, # + 0x020b, 499, # + 0x020d, 499, # + 0x020f, 499, # + 0x0211, 499, # + 0x0213, 499, # + 0x0215, 499, # + 0x0217, 499, # + 0x0253, 290, # + 0x0254, 294, # + 0x025b, 297, # + 0x0260, 295, # + 0x0263, 293, # + 0x0268, 291, # + 0x0269, 289, # + 0x026f, 289, # + 0x0272, 287, # + 0x0283, 282, # + 0x0288, 282, # + 0x0292, 281, # + 0x03ac, 462, # + 0x03cc, 436, # + 0x03d0, 438, # + 0x03d1, 443, # + 0x03d5, 453, # + 0x03d6, 446, # + 0x03e3, 499, # + 0x03e5, 499, # + 0x03e7, 499, # + 0x03e9, 499, # + 0x03eb, 499, # + 0x03ed, 499, # + 0x03ef, 499, # + 0x03f0, 414, # + 0x03f1, 420, # + 0x0461, 499, # + 0x0463, 499, # + 0x0465, 499, # + 0x0467, 499, # + 0x0469, 499, # + 0x046b, 499, # + 0x046d, 499, # + 0x046f, 499, # + 0x0471, 499, # + 0x0473, 499, # + 0x0475, 499, # + 0x0477, 499, # + 0x0479, 499, # + 0x047b, 499, # + 0x047d, 499, # + 0x047f, 499, # + 0x0481, 499, # + 0x0491, 499, # + 0x0493, 499, # + 0x0495, 499, # + 0x0497, 499, # + 0x0499, 499, # + 0x049b, 499, # + 0x049d, 499, # + 0x049f, 499, # + 0x04a1, 499, # + 0x04a3, 499, # + 0x04a5, 499, # + 0x04a7, 499, # + 0x04a9, 499, # + 0x04ab, 499, # + 0x04ad, 499, # + 0x04af, 499, # + 0x04b1, 499, # + 0x04b3, 499, # + 0x04b5, 499, # + 0x04b7, 499, # + 0x04b9, 499, # + 0x04bb, 499, # + 0x04bd, 499, # + 0x04bf, 499, # + 0x04c2, 499, # + 0x04c4, 499, # + 0x04c8, 499, # + 0x04cc, 499, # + 0x04d1, 499, # + 0x04d3, 499, # + 0x04d5, 499, # + 0x04d7, 499, # + 0x04d9, 499, # + 0x04db, 499, # + 0x04dd, 499, # + 0x04df, 499, # + 0x04e1, 499, # + 0x04e3, 499, # + 0x04e5, 499, # + 0x04e7, 499, # + 0x04e9, 499, # + 0x04eb, 499, # + 0x04ef, 499, # + 0x04f1, 499, # + 0x04f3, 499, # + 0x04f5, 499, # + 0x04f9, 499, # + 0x1e01, 499, # + 0x1e03, 499, # + 0x1e05, 499, # + 0x1e07, 499, # + 0x1e09, 499, # + 0x1e0b, 499, # + 0x1e0d, 499, # + 0x1e0f, 499, # + 0x1e11, 499, # + 0x1e13, 499, # + 0x1e15, 499, # + 0x1e17, 499, # + 0x1e19, 499, # + 0x1e1b, 499, # + 0x1e1d, 499, # + 0x1e1f, 499, # + 0x1e21, 499, # + 0x1e23, 499, # + 0x1e25, 499, # + 0x1e27, 499, # + 0x1e29, 499, # + 0x1e2b, 499, # + 0x1e2d, 499, # + 0x1e2f, 499, # + 0x1e31, 499, # + 0x1e33, 499, # + 0x1e35, 499, # + 0x1e37, 499, # + 0x1e39, 499, # + 0x1e3b, 499, # + 0x1e3d, 499, # + 0x1e3f, 499, # + 0x1e41, 499, # + 0x1e43, 499, # + 0x1e45, 499, # + 0x1e47, 499, # + 0x1e49, 499, # + 0x1e4b, 499, # + 0x1e4d, 499, # + 0x1e4f, 499, # + 0x1e51, 499, # + 0x1e53, 499, # + 0x1e55, 499, # + 0x1e57, 499, # + 0x1e59, 499, # + 0x1e5b, 499, # + 0x1e5d, 499, # + 0x1e5f, 499, # + 0x1e61, 499, # + 0x1e63, 499, # + 0x1e65, 499, # + 0x1e67, 499, # + 0x1e69, 499, # + 0x1e6b, 499, # + 0x1e6d, 499, # + 0x1e6f, 499, # + 0x1e71, 499, # + 0x1e73, 499, # + 0x1e75, 499, # + 0x1e77, 499, # + 0x1e79, 499, # + 0x1e7b, 499, # + 0x1e7d, 499, # + 0x1e7f, 499, # + 0x1e81, 499, # + 0x1e83, 499, # + 0x1e85, 499, # + 0x1e87, 499, # + 0x1e89, 499, # + 0x1e8b, 499, # + 0x1e8d, 499, # + 0x1e8f, 499, # + 0x1e91, 499, # + 0x1e93, 499, # + 0x1e95, 499, # + 0x1ea1, 499, # + 0x1ea3, 499, # + 0x1ea5, 499, # + 0x1ea7, 499, # + 0x1ea9, 499, # + 0x1eab, 499, # + 0x1ead, 499, # + 0x1eaf, 499, # + 0x1eb1, 499, # + 0x1eb3, 499, # + 0x1eb5, 499, # + 0x1eb7, 499, # + 0x1eb9, 499, # + 0x1ebb, 499, # + 0x1ebd, 499, # + 0x1ebf, 499, # + 0x1ec1, 499, # + 0x1ec3, 499, # + 0x1ec5, 499, # + 0x1ec7, 499, # + 0x1ec9, 499, # + 0x1ecb, 499, # + 0x1ecd, 499, # + 0x1ecf, 499, # + 0x1ed1, 499, # + 0x1ed3, 499, # + 0x1ed5, 499, # + 0x1ed7, 499, # + 0x1ed9, 499, # + 0x1edb, 499, # + 0x1edd, 499, # + 0x1edf, 499, # + 0x1ee1, 499, # + 0x1ee3, 499, # + 0x1ee5, 499, # + 0x1ee7, 499, # + 0x1ee9, 499, # + 0x1eeb, 499, # + 0x1eed, 499, # + 0x1eef, 499, # + 0x1ef1, 499, # + 0x1ef3, 499, # + 0x1ef5, 499, # + 0x1ef7, 499, # + 0x1ef9, 499, # + 0x1f51, 508, # + 0x1f53, 508, # + 0x1f55, 508, # + 0x1f57, 508, # + 0x1fb3, 509, # + 0x1fc3, 509, # + 0x1fe5, 507, # + 0x1ff3, 509] # tolowerRanges = [ 0x0041, 0x005a, 532, # A-Z a-z - 0x00c0, 0x00d6, 532, # - - - 0x00d8, 0x00de, 532, # - - - 0x0189, 0x018a, 705, # - - - 0x018e, 0x018f, 702, # - - - 0x01b1, 0x01b2, 717, # - - - 0x0388, 0x038a, 537, # - - - 0x038e, 0x038f, 563, # - - - 0x0391, 0x03a1, 532, # - - - 0x03a3, 0x03ab, 532, # - - - 0x0401, 0x040c, 580, # - - - 0x040e, 0x040f, 580, # - - - 0x0410, 0x042f, 532, # - - - 0x0531, 0x0556, 548, # - - - 0x10a0, 0x10c5, 548, # - - - 0x1f08, 0x1f0f, 492, # - - - 0x1f18, 0x1f1d, 492, # - - - 0x1f28, 0x1f2f, 492, # - - - 0x1f38, 0x1f3f, 492, # - - - 0x1f48, 0x1f4d, 492, # - - - 0x1f68, 0x1f6f, 492, # - - - 0x1f88, 0x1f8f, 492, # - - - 0x1f98, 0x1f9f, 492, # - - - 0x1fa8, 0x1faf, 492, # - - - 0x1fb8, 0x1fb9, 492, # - - - 0x1fba, 0x1fbb, 426, # - - - 0x1fc8, 0x1fcb, 414, # - - - 0x1fd8, 0x1fd9, 492, # - - - 0x1fda, 0x1fdb, 400, # - - - 0x1fe8, 0x1fe9, 492, # - - - 0x1fea, 0x1feb, 388, # - - - 0x1ff8, 0x1ff9, 372, # - - - 0x1ffa, 0x1ffb, 374, # - - - 0x2160, 0x216f, 516, # - - - 0x24b6, 0x24cf, 526, # - - - 0xff21, 0xff3a, 532] # - - + 0x00c0, 0x00d6, 532, # - - + 0x00d8, 0x00de, 532, # - - + 0x0189, 0x018a, 705, # - - + 0x018e, 0x018f, 702, # - - + 0x01b1, 0x01b2, 717, # - - + 0x0388, 0x038a, 537, # - - + 0x038e, 0x038f, 563, # - - + 0x0391, 0x03a1, 532, # - - + 0x03a3, 0x03ab, 532, # - - + 0x0401, 0x040c, 580, # - - + 0x040e, 0x040f, 580, # - - + 0x0410, 0x042f, 532, # - - + 0x0531, 0x0556, 548, # - - + 0x10a0, 0x10c5, 548, # - - + 0x1f08, 0x1f0f, 492, # - - + 0x1f18, 0x1f1d, 492, # - - + 0x1f28, 0x1f2f, 492, # - - + 0x1f38, 0x1f3f, 492, # - - + 0x1f48, 0x1f4d, 492, # - - + 0x1f68, 0x1f6f, 492, # - - + 0x1f88, 0x1f8f, 492, # - - + 0x1f98, 0x1f9f, 492, # - - + 0x1fa8, 0x1faf, 492, # - - + 0x1fb8, 0x1fb9, 492, # - - + 0x1fba, 0x1fbb, 426, # - - + 0x1fc8, 0x1fcb, 414, # - - + 0x1fd8, 0x1fd9, 492, # - - + 0x1fda, 0x1fdb, 400, # - - + 0x1fe8, 0x1fe9, 492, # - - + 0x1fea, 0x1feb, 388, # - - + 0x1ff8, 0x1ff9, 372, # - - + 0x1ffa, 0x1ffb, 374, # - - + 0x2160, 0x216f, 516, # - - + 0x24b6, 0x24cf, 526, # - - + 0xff21, 0xff3a, 532] # - - tolowerSinglets = [ - 0x0100, 501, # - 0x0102, 501, # - 0x0104, 501, # - 0x0106, 501, # - 0x0108, 501, # - 0x010a, 501, # - 0x010c, 501, # - 0x010e, 501, # - 0x0110, 501, # - 0x0112, 501, # - 0x0114, 501, # - 0x0116, 501, # - 0x0118, 501, # - 0x011a, 501, # - 0x011c, 501, # - 0x011e, 501, # - 0x0120, 501, # - 0x0122, 501, # - 0x0124, 501, # - 0x0126, 501, # - 0x0128, 501, # - 0x012a, 501, # - 0x012c, 501, # - 0x012e, 501, # - 0x0130, 301, # i - 0x0132, 501, # - 0x0134, 501, # - 0x0136, 501, # - 0x0139, 501, # - 0x013b, 501, # - 0x013d, 501, # - 0x013f, 501, # - 0x0141, 501, # - 0x0143, 501, # - 0x0145, 501, # - 0x0147, 501, # - 0x014a, 501, # - 0x014c, 501, # - 0x014e, 501, # - 0x0150, 501, # - 0x0152, 501, # - 0x0154, 501, # - 0x0156, 501, # - 0x0158, 501, # - 0x015a, 501, # - 0x015c, 501, # - 0x015e, 501, # - 0x0160, 501, # - 0x0162, 501, # - 0x0164, 501, # - 0x0166, 501, # - 0x0168, 501, # - 0x016a, 501, # - 0x016c, 501, # - 0x016e, 501, # - 0x0170, 501, # - 0x0172, 501, # - 0x0174, 501, # - 0x0176, 501, # - 0x0178, 379, # - 0x0179, 501, # - 0x017b, 501, # - 0x017d, 501, # - 0x0181, 710, # - 0x0182, 501, # - 0x0184, 501, # - 0x0186, 706, # - 0x0187, 501, # - 0x018b, 501, # - 0x0190, 703, # - 0x0191, 501, # - 0x0193, 705, # - 0x0194, 707, # - 0x0196, 711, # - 0x0197, 709, # - 0x0198, 501, # - 0x019c, 711, # - 0x019d, 713, # - 0x01a0, 501, # - 0x01a2, 501, # - 0x01a4, 501, # - 0x01a7, 501, # - 0x01a9, 718, # - 0x01ac, 501, # - 0x01ae, 718, # - 0x01af, 501, # - 0x01b3, 501, # - 0x01b5, 501, # - 0x01b7, 719, # - 0x01b8, 501, # - 0x01bc, 501, # - 0x01c4, 502, # - 0x01c5, 501, # - 0x01c7, 502, # - 0x01c8, 501, # - 0x01ca, 502, # - 0x01cb, 501, # - 0x01cd, 501, # - 0x01cf, 501, # - 0x01d1, 501, # - 0x01d3, 501, # - 0x01d5, 501, # - 0x01d7, 501, # - 0x01d9, 501, # - 0x01db, 501, # - 0x01de, 501, # - 0x01e0, 501, # - 0x01e2, 501, # - 0x01e4, 501, # - 0x01e6, 501, # - 0x01e8, 501, # - 0x01ea, 501, # - 0x01ec, 501, # - 0x01ee, 501, # - 0x01f1, 502, # - 0x01f2, 501, # - 0x01f4, 501, # - 0x01fa, 501, # - 0x01fc, 501, # - 0x01fe, 501, # - 0x0200, 501, # - 0x0202, 501, # - 0x0204, 501, # - 0x0206, 501, # - 0x0208, 501, # - 0x020a, 501, # - 0x020c, 501, # - 0x020e, 501, # - 0x0210, 501, # - 0x0212, 501, # - 0x0214, 501, # - 0x0216, 501, # - 0x0386, 538, # - 0x038c, 564, # - 0x03e2, 501, # - 0x03e4, 501, # - 0x03e6, 501, # - 0x03e8, 501, # - 0x03ea, 501, # - 0x03ec, 501, # - 0x03ee, 501, # - 0x0460, 501, # - 0x0462, 501, # - 0x0464, 501, # - 0x0466, 501, # - 0x0468, 501, # - 0x046a, 501, # - 0x046c, 501, # - 0x046e, 501, # - 0x0470, 501, # - 0x0472, 501, # - 0x0474, 501, # - 0x0476, 501, # - 0x0478, 501, # - 0x047a, 501, # - 0x047c, 501, # - 0x047e, 501, # - 0x0480, 501, # - 0x0490, 501, # - 0x0492, 501, # - 0x0494, 501, # - 0x0496, 501, # - 0x0498, 501, # - 0x049a, 501, # - 0x049c, 501, # - 0x049e, 501, # - 0x04a0, 501, # - 0x04a2, 501, # - 0x04a4, 501, # - 0x04a6, 501, # - 0x04a8, 501, # - 0x04aa, 501, # - 0x04ac, 501, # - 0x04ae, 501, # - 0x04b0, 501, # - 0x04b2, 501, # - 0x04b4, 501, # - 0x04b6, 501, # - 0x04b8, 501, # - 0x04ba, 501, # - 0x04bc, 501, # - 0x04be, 501, # - 0x04c1, 501, # - 0x04c3, 501, # - 0x04c7, 501, # - 0x04cb, 501, # - 0x04d0, 501, # - 0x04d2, 501, # - 0x04d4, 501, # - 0x04d6, 501, # - 0x04d8, 501, # - 0x04da, 501, # - 0x04dc, 501, # - 0x04de, 501, # - 0x04e0, 501, # - 0x04e2, 501, # - 0x04e4, 501, # - 0x04e6, 501, # - 0x04e8, 501, # - 0x04ea, 501, # - 0x04ee, 501, # - 0x04f0, 501, # - 0x04f2, 501, # - 0x04f4, 501, # - 0x04f8, 501, # - 0x1e00, 501, # - 0x1e02, 501, # - 0x1e04, 501, # - 0x1e06, 501, # - 0x1e08, 501, # - 0x1e0a, 501, # - 0x1e0c, 501, # - 0x1e0e, 501, # - 0x1e10, 501, # - 0x1e12, 501, # - 0x1e14, 501, # - 0x1e16, 501, # - 0x1e18, 501, # - 0x1e1a, 501, # - 0x1e1c, 501, # - 0x1e1e, 501, # - 0x1e20, 501, # - 0x1e22, 501, # - 0x1e24, 501, # - 0x1e26, 501, # - 0x1e28, 501, # - 0x1e2a, 501, # - 0x1e2c, 501, # - 0x1e2e, 501, # - 0x1e30, 501, # - 0x1e32, 501, # - 0x1e34, 501, # - 0x1e36, 501, # - 0x1e38, 501, # - 0x1e3a, 501, # - 0x1e3c, 501, # - 0x1e3e, 501, # - 0x1e40, 501, # - 0x1e42, 501, # - 0x1e44, 501, # - 0x1e46, 501, # - 0x1e48, 501, # - 0x1e4a, 501, # - 0x1e4c, 501, # - 0x1e4e, 501, # - 0x1e50, 501, # - 0x1e52, 501, # - 0x1e54, 501, # - 0x1e56, 501, # - 0x1e58, 501, # - 0x1e5a, 501, # - 0x1e5c, 501, # - 0x1e5e, 501, # - 0x1e60, 501, # - 0x1e62, 501, # - 0x1e64, 501, # - 0x1e66, 501, # - 0x1e68, 501, # - 0x1e6a, 501, # - 0x1e6c, 501, # - 0x1e6e, 501, # - 0x1e70, 501, # - 0x1e72, 501, # - 0x1e74, 501, # - 0x1e76, 501, # - 0x1e78, 501, # - 0x1e7a, 501, # - 0x1e7c, 501, # - 0x1e7e, 501, # - 0x1e80, 501, # - 0x1e82, 501, # - 0x1e84, 501, # - 0x1e86, 501, # - 0x1e88, 501, # - 0x1e8a, 501, # - 0x1e8c, 501, # - 0x1e8e, 501, # - 0x1e90, 501, # - 0x1e92, 501, # - 0x1e94, 501, # - 0x1ea0, 501, # - 0x1ea2, 501, # - 0x1ea4, 501, # - 0x1ea6, 501, # - 0x1ea8, 501, # - 0x1eaa, 501, # - 0x1eac, 501, # - 0x1eae, 501, # - 0x1eb0, 501, # - 0x1eb2, 501, # - 0x1eb4, 501, # - 0x1eb6, 501, # - 0x1eb8, 501, # - 0x1eba, 501, # - 0x1ebc, 501, # - 0x1ebe, 501, # - 0x1ec0, 501, # - 0x1ec2, 501, # - 0x1ec4, 501, # - 0x1ec6, 501, # - 0x1ec8, 501, # - 0x1eca, 501, # - 0x1ecc, 501, # - 0x1ece, 501, # - 0x1ed0, 501, # - 0x1ed2, 501, # - 0x1ed4, 501, # - 0x1ed6, 501, # - 0x1ed8, 501, # - 0x1eda, 501, # - 0x1edc, 501, # - 0x1ede, 501, # - 0x1ee0, 501, # - 0x1ee2, 501, # - 0x1ee4, 501, # - 0x1ee6, 501, # - 0x1ee8, 501, # - 0x1eea, 501, # - 0x1eec, 501, # - 0x1eee, 501, # - 0x1ef0, 501, # - 0x1ef2, 501, # - 0x1ef4, 501, # - 0x1ef6, 501, # - 0x1ef8, 501, # - 0x1f59, 492, # - 0x1f5b, 492, # - 0x1f5d, 492, # - 0x1f5f, 492, # - 0x1fbc, 491, # - 0x1fcc, 491, # - 0x1fec, 493, # - 0x1ffc, 491] # + 0x0100, 501, # + 0x0102, 501, # + 0x0104, 501, # + 0x0106, 501, # + 0x0108, 501, # + 0x010a, 501, # + 0x010c, 501, # + 0x010e, 501, # + 0x0110, 501, # + 0x0112, 501, # + 0x0114, 501, # + 0x0116, 501, # + 0x0118, 501, # + 0x011a, 501, # + 0x011c, 501, # + 0x011e, 501, # + 0x0120, 501, # + 0x0122, 501, # + 0x0124, 501, # + 0x0126, 501, # + 0x0128, 501, # + 0x012a, 501, # + 0x012c, 501, # + 0x012e, 501, # + 0x0130, 301, # i + 0x0132, 501, # + 0x0134, 501, # + 0x0136, 501, # + 0x0139, 501, # + 0x013b, 501, # + 0x013d, 501, # + 0x013f, 501, # + 0x0141, 501, # + 0x0143, 501, # + 0x0145, 501, # + 0x0147, 501, # + 0x014a, 501, # + 0x014c, 501, # + 0x014e, 501, # + 0x0150, 501, # + 0x0152, 501, # + 0x0154, 501, # + 0x0156, 501, # + 0x0158, 501, # + 0x015a, 501, # + 0x015c, 501, # + 0x015e, 501, # + 0x0160, 501, # + 0x0162, 501, # + 0x0164, 501, # + 0x0166, 501, # + 0x0168, 501, # + 0x016a, 501, # + 0x016c, 501, # + 0x016e, 501, # + 0x0170, 501, # + 0x0172, 501, # + 0x0174, 501, # + 0x0176, 501, # + 0x0178, 379, # + 0x0179, 501, # + 0x017b, 501, # + 0x017d, 501, # + 0x0181, 710, # + 0x0182, 501, # + 0x0184, 501, # + 0x0186, 706, # + 0x0187, 501, # + 0x018b, 501, # + 0x0190, 703, # + 0x0191, 501, # + 0x0193, 705, # + 0x0194, 707, # + 0x0196, 711, # + 0x0197, 709, # + 0x0198, 501, # + 0x019c, 711, # + 0x019d, 713, # + 0x01a0, 501, # + 0x01a2, 501, # + 0x01a4, 501, # + 0x01a7, 501, # + 0x01a9, 718, # + 0x01ac, 501, # + 0x01ae, 718, # + 0x01af, 501, # + 0x01b3, 501, # + 0x01b5, 501, # + 0x01b7, 719, # + 0x01b8, 501, # + 0x01bc, 501, # + 0x01c4, 502, # + 0x01c5, 501, # + 0x01c7, 502, # + 0x01c8, 501, # + 0x01ca, 502, # + 0x01cb, 501, # + 0x01cd, 501, # + 0x01cf, 501, # + 0x01d1, 501, # + 0x01d3, 501, # + 0x01d5, 501, # + 0x01d7, 501, # + 0x01d9, 501, # + 0x01db, 501, # + 0x01de, 501, # + 0x01e0, 501, # + 0x01e2, 501, # + 0x01e4, 501, # + 0x01e6, 501, # + 0x01e8, 501, # + 0x01ea, 501, # + 0x01ec, 501, # + 0x01ee, 501, # + 0x01f1, 502, # + 0x01f2, 501, # + 0x01f4, 501, # + 0x01fa, 501, # + 0x01fc, 501, # + 0x01fe, 501, # + 0x0200, 501, # + 0x0202, 501, # + 0x0204, 501, # + 0x0206, 501, # + 0x0208, 501, # + 0x020a, 501, # + 0x020c, 501, # + 0x020e, 501, # + 0x0210, 501, # + 0x0212, 501, # + 0x0214, 501, # + 0x0216, 501, # + 0x0386, 538, # + 0x038c, 564, # + 0x03e2, 501, # + 0x03e4, 501, # + 0x03e6, 501, # + 0x03e8, 501, # + 0x03ea, 501, # + 0x03ec, 501, # + 0x03ee, 501, # + 0x0460, 501, # + 0x0462, 501, # + 0x0464, 501, # + 0x0466, 501, # + 0x0468, 501, # + 0x046a, 501, # + 0x046c, 501, # + 0x046e, 501, # + 0x0470, 501, # + 0x0472, 501, # + 0x0474, 501, # + 0x0476, 501, # + 0x0478, 501, # + 0x047a, 501, # + 0x047c, 501, # + 0x047e, 501, # + 0x0480, 501, # + 0x0490, 501, # + 0x0492, 501, # + 0x0494, 501, # + 0x0496, 501, # + 0x0498, 501, # + 0x049a, 501, # + 0x049c, 501, # + 0x049e, 501, # + 0x04a0, 501, # + 0x04a2, 501, # + 0x04a4, 501, # + 0x04a6, 501, # + 0x04a8, 501, # + 0x04aa, 501, # + 0x04ac, 501, # + 0x04ae, 501, # + 0x04b0, 501, # + 0x04b2, 501, # + 0x04b4, 501, # + 0x04b6, 501, # + 0x04b8, 501, # + 0x04ba, 501, # + 0x04bc, 501, # + 0x04be, 501, # + 0x04c1, 501, # + 0x04c3, 501, # + 0x04c7, 501, # + 0x04cb, 501, # + 0x04d0, 501, # + 0x04d2, 501, # + 0x04d4, 501, # + 0x04d6, 501, # + 0x04d8, 501, # + 0x04da, 501, # + 0x04dc, 501, # + 0x04de, 501, # + 0x04e0, 501, # + 0x04e2, 501, # + 0x04e4, 501, # + 0x04e6, 501, # + 0x04e8, 501, # + 0x04ea, 501, # + 0x04ee, 501, # + 0x04f0, 501, # + 0x04f2, 501, # + 0x04f4, 501, # + 0x04f8, 501, # + 0x1e00, 501, # + 0x1e02, 501, # + 0x1e04, 501, # + 0x1e06, 501, # + 0x1e08, 501, # + 0x1e0a, 501, # + 0x1e0c, 501, # + 0x1e0e, 501, # + 0x1e10, 501, # + 0x1e12, 501, # + 0x1e14, 501, # + 0x1e16, 501, # + 0x1e18, 501, # + 0x1e1a, 501, # + 0x1e1c, 501, # + 0x1e1e, 501, # + 0x1e20, 501, # + 0x1e22, 501, # + 0x1e24, 501, # + 0x1e26, 501, # + 0x1e28, 501, # + 0x1e2a, 501, # + 0x1e2c, 501, # + 0x1e2e, 501, # + 0x1e30, 501, # + 0x1e32, 501, # + 0x1e34, 501, # + 0x1e36, 501, # + 0x1e38, 501, # + 0x1e3a, 501, # + 0x1e3c, 501, # + 0x1e3e, 501, # + 0x1e40, 501, # + 0x1e42, 501, # + 0x1e44, 501, # + 0x1e46, 501, # + 0x1e48, 501, # + 0x1e4a, 501, # + 0x1e4c, 501, # + 0x1e4e, 501, # + 0x1e50, 501, # + 0x1e52, 501, # + 0x1e54, 501, # + 0x1e56, 501, # + 0x1e58, 501, # + 0x1e5a, 501, # + 0x1e5c, 501, # + 0x1e5e, 501, # + 0x1e60, 501, # + 0x1e62, 501, # + 0x1e64, 501, # + 0x1e66, 501, # + 0x1e68, 501, # + 0x1e6a, 501, # + 0x1e6c, 501, # + 0x1e6e, 501, # + 0x1e70, 501, # + 0x1e72, 501, # + 0x1e74, 501, # + 0x1e76, 501, # + 0x1e78, 501, # + 0x1e7a, 501, # + 0x1e7c, 501, # + 0x1e7e, 501, # + 0x1e80, 501, # + 0x1e82, 501, # + 0x1e84, 501, # + 0x1e86, 501, # + 0x1e88, 501, # + 0x1e8a, 501, # + 0x1e8c, 501, # + 0x1e8e, 501, # + 0x1e90, 501, # + 0x1e92, 501, # + 0x1e94, 501, # + 0x1ea0, 501, # + 0x1ea2, 501, # + 0x1ea4, 501, # + 0x1ea6, 501, # + 0x1ea8, 501, # + 0x1eaa, 501, # + 0x1eac, 501, # + 0x1eae, 501, # + 0x1eb0, 501, # + 0x1eb2, 501, # + 0x1eb4, 501, # + 0x1eb6, 501, # + 0x1eb8, 501, # + 0x1eba, 501, # + 0x1ebc, 501, # + 0x1ebe, 501, # + 0x1ec0, 501, # + 0x1ec2, 501, # + 0x1ec4, 501, # + 0x1ec6, 501, # + 0x1ec8, 501, # + 0x1eca, 501, # + 0x1ecc, 501, # + 0x1ece, 501, # + 0x1ed0, 501, # + 0x1ed2, 501, # + 0x1ed4, 501, # + 0x1ed6, 501, # + 0x1ed8, 501, # + 0x1eda, 501, # + 0x1edc, 501, # + 0x1ede, 501, # + 0x1ee0, 501, # + 0x1ee2, 501, # + 0x1ee4, 501, # + 0x1ee6, 501, # + 0x1ee8, 501, # + 0x1eea, 501, # + 0x1eec, 501, # + 0x1eee, 501, # + 0x1ef0, 501, # + 0x1ef2, 501, # + 0x1ef4, 501, # + 0x1ef6, 501, # + 0x1ef8, 501, # + 0x1f59, 492, # + 0x1f5b, 492, # + 0x1f5d, 492, # + 0x1f5f, 492, # + 0x1fbc, 491, # + 0x1fcc, 491, # + 0x1fec, 493, # + 0x1ffc, 491] # toTitleSinglets = [ - 0x01c4, 501, # - 0x01c6, 499, # - 0x01c7, 501, # - 0x01c9, 499, # - 0x01ca, 501, # - 0x01cc, 499, # - 0x01f1, 501, # - 0x01f3, 499] # + 0x01c4, 501, # + 0x01c6, 499, # + 0x01c7, 501, # + 0x01c9, 499, # + 0x01ca, 501, # + 0x01cc, 499, # + 0x01f1, 501, # + 0x01f3, 499] # -proc binarySearch(c: RuneImpl, tab: openArray[RuneImpl], len, stride: int): int = +proc binarySearch(c: RuneImpl, tab: openArray[RuneImpl], len, stride: int): int = var n = len var t = 0 - while n > 1: + while n > 1: var m = n div 2 var p = t + m*stride if c >= tab[p]: @@ -1117,9 +1156,9 @@ proc binarySearch(c: RuneImpl, tab: openArray[RuneImpl], len, stride: int): int return t return -1 -proc toLower*(c: Rune): Rune {.rtl, extern: "nuc$1", procvar.} = +proc toLower*(c: Rune): Rune {.rtl, extern: "nuc$1", procvar.} = ## Converts `c` into lower case. This works for any Unicode character. - ## If possible, prefer `toLower` over `toUpper`. + ## If possible, prefer `toLower` over `toUpper`. var c = RuneImpl(c) var p = binarySearch(c, tolowerRanges, len(tolowerRanges) div 3, 3) if p >= 0 and c >= tolowerRanges[p] and c <= tolowerRanges[p+1]: @@ -1129,9 +1168,9 @@ proc toLower*(c: Rune): Rune {.rtl, extern: "nuc$1", procvar.} = return Rune(c + tolowerSinglets[p+1] - 500) return Rune(c) -proc toUpper*(c: Rune): Rune {.rtl, extern: "nuc$1", procvar.} = +proc toUpper*(c: Rune): Rune {.rtl, extern: "nuc$1", procvar.} = ## Converts `c` into upper case. This works for any Unicode character. - ## If possible, prefer `toLower` over `toUpper`. + ## If possible, prefer `toLower` over `toUpper`. var c = RuneImpl(c) var p = binarySearch(c, toupperRanges, len(toupperRanges) div 3, 3) if p >= 0 and c >= toupperRanges[p] and c <= toupperRanges[p+1]: @@ -1141,16 +1180,16 @@ proc toUpper*(c: Rune): Rune {.rtl, extern: "nuc$1", procvar.} = return Rune(c + toupperSinglets[p+1] - 500) return Rune(c) -proc toTitle*(c: Rune): Rune {.rtl, extern: "nuc$1", procvar.} = +proc toTitle*(c: Rune): Rune {.rtl, extern: "nuc$1", procvar.} = var c = RuneImpl(c) var p = binarySearch(c, toTitleSinglets, len(toTitleSinglets) div 2, 2) if p >= 0 and c == toTitleSinglets[p]: return Rune(c + toTitleSinglets[p+1] - 500) return Rune(c) -proc isLower*(c: Rune): bool {.rtl, extern: "nuc$1", procvar.} = +proc isLower*(c: Rune): bool {.rtl, extern: "nuc$1", procvar.} = ## returns true iff `c` is a lower case Unicode character - ## If possible, prefer `isLower` over `isUpper`. + ## If possible, prefer `isLower` over `isUpper`. var c = RuneImpl(c) # Note: toUpperRanges is correct here! var p = binarySearch(c, toupperRanges, len(toupperRanges) div 3, 3) @@ -1160,9 +1199,9 @@ proc isLower*(c: Rune): bool {.rtl, extern: "nuc$1", procvar.} = if p >= 0 and c == toupperSinglets[p]: return true -proc isUpper*(c: Rune): bool {.rtl, extern: "nuc$1", procvar.} = +proc isUpper*(c: Rune): bool {.rtl, extern: "nuc$1", procvar.} = ## returns true iff `c` is a upper case Unicode character - ## If possible, prefer `isLower` over `isUpper`. + ## If possible, prefer `isLower` over `isUpper`. var c = RuneImpl(c) # Note: toLowerRanges is correct here! var p = binarySearch(c, tolowerRanges, len(tolowerRanges) div 3, 3) @@ -1172,9 +1211,9 @@ proc isUpper*(c: Rune): bool {.rtl, extern: "nuc$1", procvar.} = if p >= 0 and c == tolowerSinglets[p]: return true -proc isAlpha*(c: Rune): bool {.rtl, extern: "nuc$1", procvar.} = +proc isAlpha*(c: Rune): bool {.rtl, extern: "nuc$1", procvar.} = ## returns true iff `c` is an *alpha* Unicode character (i.e. a letter) - if isUpper(c) or isLower(c): + if isUpper(c) or isLower(c): return true var c = RuneImpl(c) var p = binarySearch(c, alphaRanges, len(alphaRanges) div 2, 2) @@ -1183,17 +1222,28 @@ proc isAlpha*(c: Rune): bool {.rtl, extern: "nuc$1", procvar.} = p = binarySearch(c, alphaSinglets, len(alphaSinglets), 1) if p >= 0 and c == alphaSinglets[p]: return true - -proc isTitle*(c: Rune): bool {.rtl, extern: "nuc$1", procvar.} = + +proc isTitle*(c: Rune): bool {.rtl, extern: "nuc$1", procvar.} = return isUpper(c) and isLower(c) -proc isWhiteSpace*(c: Rune): bool {.rtl, extern: "nuc$1", procvar.} = +proc isWhiteSpace*(c: Rune): bool {.rtl, extern: "nuc$1", procvar.} = ## returns true iff `c` is a Unicode whitespace character var c = RuneImpl(c) var p = binarySearch(c, spaceRanges, len(spaceRanges) div 2, 2) if p >= 0 and c >= spaceRanges[p] and c <= spaceRanges[p+1]: return true +proc isCombining*(c: Rune): bool {.rtl, extern: "nuc$1", procvar.} = + ## returns true iff `c` is a Unicode combining character + var c = RuneImpl(c) + + # Optimized to return false immediately for ASCII + return c >= 0x0300 and (c <= 0x036f or + (c >= 0x1ab0 and c <= 0x1aff) or + (c >= 0x1dc0 and c <= 0x1dff) or + (c >= 0x20d0 and c <= 0x20ff) or + (c >= 0xfe20 and c <= 0xfe2f)) + iterator runes*(s: string): Rune = ## iterates over any unicode character of the string `s`. var @@ -1203,7 +1253,7 @@ iterator runes*(s: string): Rune = fastRuneAt(s, i, result, true) yield result -proc cmpRunesIgnoreCase*(a, b: string): int {.rtl, extern: "nuc$1", procvar.} = +proc cmpRunesIgnoreCase*(a, b: string): int {.rtl, extern: "nuc$1", procvar.} = ## compares two UTF8 strings and ignores the case. Returns: ## ## | 0 iff a == b @@ -1220,9 +1270,49 @@ proc cmpRunesIgnoreCase*(a, b: string): int {.rtl, extern: "nuc$1", procvar.} = if result != 0: return result = a.len - b.len +proc reversed*(s: string): string = + ## returns the reverse of `s`, interpreting it as unicode characters. Unicode + ## combining characters are correctly interpreted as well: + ## + ## .. code-block:: nim + ## + ## assert reversed("Reverse this!") == "!siht esreveR" + ## assert reversed("先秦兩漢") == "漢兩秦先" + ## assert reversed("as⃝df̅") == "f̅ds⃝a" + ## assert reversed("a⃞b⃞c⃞") == "c⃞b⃞a⃞" + var + i = 0 + lastI = 0 + newPos = len(s) - 1 + blockPos = 0 + r: Rune + + template reverseUntil(pos): stmt = + var j = pos - 1 + while j > blockPos: + result[newPos] = s[j] + dec j + dec newPos + blockPos = pos - 1 + + result = newString(len(s)) + + while i < len(s): + lastI = i + fastRuneAt(s, i, r, true) + if not isCombining(r): + reverseUntil(lastI) + + reverseUntil(len(s)) + when isMainModule: let someString = "öÑ" someRunes = @[runeAt(someString, 0), runeAt(someString, 2)] compared = (someString == $someRunes) assert compared == true + + assert reversed("Reverse this!") == "!siht esreveR" + assert reversed("先秦兩漢") == "漢兩秦先" + assert reversed("as⃝df̅") == "f̅ds⃝a" + assert reversed("a⃞b⃞c⃞") == "c⃞b⃞a⃞" diff --git a/lib/pure/unidecode/unidecode.nim b/lib/pure/unidecode/unidecode.nim index 798eef5d0..a83b9be0f 100644 --- a/lib/pure/unidecode/unidecode.nim +++ b/lib/pure/unidecode/unidecode.nim @@ -70,5 +70,5 @@ proc unidecode*(s: string): string = when isMainModule: loadUnidecodeTable("lib/pure/unidecode/unidecode.dat") - echo unidecode("Äußerst") + assert unidecode("Äußerst") == "Ausserst" diff --git a/lib/pure/unittest.nim b/lib/pure/unittest.nim index 21efea3bc..3bf4724b9 100644 --- a/lib/pure/unittest.nim +++ b/lib/pure/unittest.nim @@ -1,7 +1,7 @@ # # # Nim's Runtime Library -# (c) Copyright 2012 Nim Contributors +# (c) Copyright 2015 Nim Contributors # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -9,12 +9,27 @@ ## :Author: Zahary Karadjov ## -## This module implements the standard unit testing facilities such as -## suites, fixtures and test cases as well as facilities for combinatorial -## and randomzied test case generation (not yet available) -## and object mocking (not yet available) +## This module implements boilerplate to make testing easy. ## -## It is loosely based on C++'s boost.test and Haskell's QuickTest +## Example: +## +## .. code:: nim +## +## suite "description for this stuff": +## test "essential truths": +## # give up and stop if this fails +## require(true) +## +## test "slightly less obvious stuff": +## # print a nasty message and move on, skipping +## # the remainder of this block +## check(1 != 1) +## check("asd"[2] == 'd') +## +## test "out of bounds error is thrown on bad access": +## let v = @[1, 2, 3] # you can do initialization here +## expect(IndexError): +## discard v[4] import macros @@ -24,6 +39,7 @@ when declared(stdout): when not defined(ECMAScript): import terminal + system.addQuitProc(resetAttributes) type TestStatus* = enum OK, FAILED @@ -31,7 +47,7 @@ type {.deprecated: [TTestStatus: TestStatus, TOutputLevel: OutputLevel]} -var +var abortOnError* {.threadvar.}: bool outputLevel* {.threadvar.}: OutputLevel colorOutput* {.threadvar.}: bool @@ -84,6 +100,7 @@ template test*(name: expr, body: stmt): stmt {.immediate, dirty.} = except: checkpoint("Unhandled exception: " & getCurrentExceptionMsg()) + echo getCurrentException().getStackTrace() fail() finally: @@ -127,16 +144,25 @@ macro check*(conditions: stmt): stmt {.immediate.} = when compiles(string($value)): checkpoint(name & " was " & $value) - proc inspectArgs(exp: PNimrodNode) = + proc inspectArgs(exp: NimNode) = for i in 1 .. <exp.len: if exp[i].kind notin nnkLiterals: inc counter var arg = newIdentNode(":p" & $counter) var argStr = exp[i].toStrLit + var paramAst = exp[i] if exp[i].kind in nnkCallKinds: inspectArgs(exp[i]) - argsAsgns.add getAst(asgn(arg, exp[i])) + if exp[i].kind == nnkExprEqExpr: + # ExprEqExpr + # Ident !"v" + # IntLit 2 + paramAst = exp[i][1] + argsAsgns.add getAst(asgn(arg, paramAst)) argsPrintOuts.add getAst(print(argStr, arg)) - exp[i] = arg + if exp[i].kind != nnkExprEqExpr: + exp[i] = arg + else: + exp[i][1] = arg case checked.kind of nnkCallKinds: @@ -176,7 +202,7 @@ template require*(conditions: stmt): stmt {.immediate, dirty.} = macro expect*(exceptions: varargs[expr], body: stmt): stmt {.immediate.} = let exp = callsite() template expectBody(errorTypes, lineInfoLit: expr, - body: stmt): PNimrodNode {.dirty.} = + body: stmt): NimNode {.dirty.} = try: body checkpoint(lineInfoLit & ": Expect Failed, no exception was thrown.") diff --git a/lib/pure/uri.nim b/lib/pure/uri.nim index 368802dc2..1890a9bf4 100644 --- a/lib/pure/uri.nim +++ b/lib/pure/uri.nim @@ -1,7 +1,7 @@ # # # Nim's Runtime Library -# (c) Copyright 2014 Dominik Picheta +# (c) Copyright 2015 Dominik Picheta # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -16,9 +16,11 @@ type Uri* = object scheme*, username*, password*: string hostname*, port*, path*, query*, anchor*: string + opaque*: bool {.deprecated: [TUrl: Url, TUri: Uri].} +{.push warning[deprecated]: off.} proc `$`*(url: Url): string {.deprecated.} = ## **Deprecated since 0.9.6**: Use ``Uri`` instead. return string(url) @@ -43,6 +45,7 @@ proc add*(url: var Url, a: Url) {.deprecated.} = ## ## **Deprecated since 0.9.6**: Use ``Uri`` instead. url = url / a +{.pop.} proc parseAuthority(authority: string, result: var Uri) = var i = 0 @@ -50,10 +53,10 @@ proc parseAuthority(authority: string, result: var Uri) = while true: case authority[i] of '@': - result.password = result.port - result.port = "" - result.username = result.hostname - result.hostname = "" + swap result.password, result.port + result.port.setLen(0) + swap result.username, result.hostname + result.hostname.setLen(0) inPort = false of ':': inPort = true @@ -72,7 +75,7 @@ proc parsePath(uri: string, i: var int, result: var Uri) = # The 'mailto' scheme's PATH actually contains the hostname/username if result.scheme.toLower == "mailto": parseAuthority(result.path, result) - result.path = "" + result.path.setLen(0) if uri[i] == '?': i.inc # Skip '?' @@ -82,13 +85,21 @@ proc parsePath(uri: string, i: var int, result: var Uri) = i.inc # Skip '#' i.inc parseUntil(uri, result.anchor, {}, i) -proc initUri(): Uri = +proc initUri*(): Uri = + ## Initializes a URI. result = Uri(scheme: "", username: "", password: "", hostname: "", port: "", path: "", query: "", anchor: "") -proc parseUri*(uri: string): Uri = - ## Parses a URI. - result = initUri() +proc resetUri(uri: var Uri) = + for f in uri.fields: + when f is string: + f.setLen(0) + else: + f = false + +proc parseUri*(uri: string, result: var Uri) = + ## Parses a URI. The `result` variable will be cleared before. + resetUri(result) var i = 0 @@ -102,7 +113,7 @@ proc parseUri*(uri: string): Uri = if uri[i] != ':': # Assume this is a reference URI (relative URI) i = 0 - result.scheme = "" + result.scheme.setLen(0) parsePath(uri, i, result) return i.inc # Skip ':' @@ -115,10 +126,17 @@ proc parseUri*(uri: string): Uri = if authority == "": raise newException(ValueError, "Expected authority got nothing.") parseAuthority(authority, result) + else: + result.opaque = true # Path parsePath(uri, i, result) +proc parseUri*(uri: string): Uri = + ## Parses a URI and returns it. + result = initUri() + parseUri(uri, result) + proc removeDotSegments(path: string): string = var collection: seq[string] = @[] let endsWithSlash = path[path.len-1] == '/' @@ -179,10 +197,10 @@ proc combine*(base: Uri, reference: Uri): Uri = ## assert foo.path == "/baz" ## ## let bar = combine(parseUri("http://example.com/foo/bar"), parseUri("baz")) - ## assert foo.path == "/foo/baz" + ## assert bar.path == "/foo/baz" ## ## let bar = combine(parseUri("http://example.com/foo/bar/"), parseUri("baz")) - ## assert foo.path == "/foo/bar/baz" + ## assert bar.path == "/foo/bar/baz" template setAuthority(dest, src: expr): stmt = dest.hostname = src.hostname @@ -236,10 +254,10 @@ proc `/`*(x: Uri, path: string): Uri = ## assert foo.path == "/foo/bar/baz" ## ## let bar = parseUri("http://example.com/foo/bar") / parseUri("baz") - ## assert foo.path == "/foo/bar/baz" + ## assert bar.path == "/foo/bar/baz" ## ## let bar = parseUri("http://example.com/foo/bar/") / parseUri("baz") - ## assert foo.path == "/foo/bar/baz" + ## assert bar.path == "/foo/bar/baz" result = x if result.path[result.path.len-1] == '/': if path[0] == '/': @@ -256,7 +274,10 @@ proc `$`*(u: Uri): string = result = "" if u.scheme.len > 0: result.add(u.scheme) - result.add("://") + if u.opaque: + result.add(":") + else: + result.add("://") if u.username.len > 0: result.add(u.username) if u.password.len > 0: @@ -268,22 +289,38 @@ proc `$`*(u: Uri): string = result.add(":") result.add(u.port) if u.path.len > 0: - if u.path[0] != '/': result.add("/") result.add(u.path) - result.add(u.query) - result.add(u.anchor) + if u.query.len > 0: + result.add("?") + result.add(u.query) + if u.anchor.len > 0: + result.add("#") + result.add(u.anchor) when isMainModule: block: - let test = parseUri("http://localhost:8080/test") + let str = "http://localhost" + let test = parseUri(str) + doAssert test.path == "" + + block: + let str = "http://localhost/" + let test = parseUri(str) + doAssert test.path == "/" + + block: + let str = "http://localhost:8080/test" + let test = parseUri(str) doAssert test.scheme == "http" doAssert test.port == "8080" doAssert test.path == "/test" doAssert test.hostname == "localhost" + doAssert($test == str) block: - let test = parseUri("foo://username:password@example.com:8042/over/there" & - "/index.dtb?type=animal&name=narwhal#nose") + let str = "foo://username:password@example.com:8042/over/there" & + "/index.dtb?type=animal&name=narwhal#nose" + let test = parseUri(str) doAssert test.scheme == "foo" doAssert test.username == "username" doAssert test.password == "password" @@ -292,34 +329,45 @@ when isMainModule: doAssert test.path == "/over/there/index.dtb" doAssert test.query == "type=animal&name=narwhal" doAssert test.anchor == "nose" + doAssert($test == str) block: - let test = parseUri("urn:example:animal:ferret:nose") + let str = "urn:example:animal:ferret:nose" + let test = parseUri(str) doAssert test.scheme == "urn" doAssert test.path == "example:animal:ferret:nose" + doAssert($test == str) block: - let test = parseUri("mailto:username@example.com?subject=Topic") + let str = "mailto:username@example.com?subject=Topic" + let test = parseUri(str) doAssert test.scheme == "mailto" doAssert test.username == "username" doAssert test.hostname == "example.com" doAssert test.query == "subject=Topic" + doAssert($test == str) block: - let test = parseUri("magnet:?xt=urn:sha1:72hsga62ba515sbd62&dn=foobar") + let str = "magnet:?xt=urn:sha1:72hsga62ba515sbd62&dn=foobar" + let test = parseUri(str) doAssert test.scheme == "magnet" doAssert test.query == "xt=urn:sha1:72hsga62ba515sbd62&dn=foobar" + doAssert($test == str) block: - let test = parseUri("/test/foo/bar?q=2#asdf") + let str = "/test/foo/bar?q=2#asdf" + let test = parseUri(str) doAssert test.scheme == "" doAssert test.path == "/test/foo/bar" doAssert test.query == "q=2" doAssert test.anchor == "asdf" + doAssert($test == str) block: - let test = parseUri("test/no/slash") + let str = "test/no/slash" + let test = parseUri(str) doAssert test.path == "test/no/slash" + doAssert($test == str) # Remove dot segments tests block: @@ -371,5 +419,3 @@ when isMainModule: block: let test = parseUri("http://example.com/foo/") / "/bar/asd" doAssert test.path == "/foo/bar/asd" - - diff --git a/lib/pure/xmldom.nim b/lib/pure/xmldom.nim index 660932d92..6cf837f25 100644 --- a/lib/pure/xmldom.nim +++ b/lib/pure/xmldom.nim @@ -365,23 +365,21 @@ discard """proc getElementById*(doc: PDocument, elementId: string): PElement = proc getElementsByTagName*(doc: PDocument, tagName: string): seq[PNode] = ## Returns a NodeList of all the Elements with a given tag name in ## the order in which they are encountered in a preorder traversal of the Document tree. - var result: seq[PNode] = @[] + result = @[] if doc.fDocumentElement.fNodeName == tagName or tagName == "*": result.add(doc.fDocumentElement) result.add(doc.fDocumentElement.findNodes(tagName)) - return result proc getElementsByTagNameNS*(doc: PDocument, namespaceURI: string, localName: string): seq[PNode] = ## Returns a NodeList of all the Elements with a given localName and namespaceURI ## in the order in which they are encountered in a preorder traversal of the Document tree. - var result: seq[PNode] = @[] + result = @[] if doc.fDocumentElement.fLocalName == localName or localName == "*": if doc.fDocumentElement.fNamespaceURI == namespaceURI or namespaceURI == "*": result.add(doc.fDocumentElement) result.add(doc.fDocumentElement.findNodesNS(namespaceURI, localName)) - return result proc importNode*(doc: PDocument, importedNode: PNode, deep: bool): PNode = ## Imports a node from another document to this document @@ -642,7 +640,7 @@ proc isEmpty(s: string): bool = return true proc normalize*(n: PNode) = - ## Merges all seperated TextNodes together, and removes any empty TextNodes + ## Merges all separated TextNodes together, and removes any empty TextNodes var curTextNode: PNode = nil var i: int = 0 @@ -677,7 +675,7 @@ proc removeChild*(n: PNode, oldChild: PNode): PNode = if n.childNodes[i] == oldChild: result = n.childNodes[i] n.childNodes.delete(i) - return result + return raise newException(ENotFoundErr, "Node not found") @@ -693,7 +691,7 @@ proc replaceChild*(n: PNode, newChild: PNode, oldChild: PNode): PNode = if n.childNodes[i] == oldChild: result = n.childNodes[i] n.childNodes[i] = newChild - return result + return raise newException(ENotFoundErr, "Node not found") @@ -740,7 +738,7 @@ proc removeNamedItem*(nList: var seq[PNode], name: string): PNode = if nList[i].fNodeName == name: result = nList[i] nList.delete(i) - return result + return raise newException(ENotFoundErr, "Node not found") @@ -750,7 +748,7 @@ proc removeNamedItemNS*(nList: var seq[PNode], namespaceURI: string, localName: if nList[i].fLocalName == localName and nList[i].fNamespaceURI == namespaceURI: result = nList[i] nList.delete(i) - return result + return raise newException(ENotFoundErr, "Node not found") @@ -965,7 +963,7 @@ proc removeAttributeNode*(el: PElement, oldAttr: PAttr): PAttr = if el.attributes[i] == oldAttr: result = el.attributes[i] el.attributes.delete(i) - return result + return raise newException(ENotFoundErr, "oldAttr is not a member of el's Attributes") @@ -1083,7 +1081,7 @@ proc addEscaped(s: string): string = else: result.add(c) proc nodeToXml(n: PNode, indent: int = 0): string = - result = repeatChar(indent, ' ') & "<" & n.nodeName + result = spaces(indent) & "<" & n.nodeName if not isNil(n.attributes): for i in items(n.attributes): result.add(" " & i.name & "=\"" & addEscaped(i.value) & "\"") @@ -1098,23 +1096,23 @@ proc nodeToXml(n: PNode, indent: int = 0): string = of ElementNode: result.add(nodeToXml(i, indent + 2)) of TextNode: - result.add(repeatChar(indent * 2, ' ')) + result.add(spaces(indent * 2)) result.add(addEscaped(i.nodeValue)) of CDataSectionNode: - result.add(repeatChar(indent * 2, ' ')) + result.add(spaces(indent * 2)) result.add("<![CDATA[" & i.nodeValue & "]]>") of ProcessingInstructionNode: - result.add(repeatChar(indent * 2, ' ')) + result.add(spaces(indent * 2)) result.add("<?" & PProcessingInstruction(i).target & " " & PProcessingInstruction(i).data & " ?>") of CommentNode: - result.add(repeatChar(indent * 2, ' ')) + result.add(spaces(indent * 2)) result.add("<!-- " & i.nodeValue & " -->") else: continue result.add("\n") # Add the ending tag - </tag> - result.add(repeatChar(indent, ' ') & "</" & n.nodeName & ">") + result.add(spaces(indent) & "</" & n.nodeName & ">") proc `$`*(doc: PDocument): string = ## Converts a PDocument object into a string representation of it's XML diff --git a/lib/pure/xmldomparser.nim b/lib/pure/xmldomparser.nim index 7f34d72a8..050362435 100644 --- a/lib/pure/xmldomparser.nim +++ b/lib/pure/xmldomparser.nim @@ -155,7 +155,7 @@ proc loadXMLFile*(path: string): PDocument = return loadXMLStream(s) -when isMainModule: +when not defined(testing) and isMainModule: var xml = loadXMLFile("nim/xmldom/test.xml") #echo(xml.getElementsByTagName("m:test2")[0].namespaceURI) #echo(xml.getElementsByTagName("bla:test")[0].namespaceURI) diff --git a/lib/pure/xmlparser.nim b/lib/pure/xmlparser.nim index 8591e894c..840cae734 100644 --- a/lib/pure/xmlparser.nim +++ b/lib/pure/xmlparser.nim @@ -103,7 +103,7 @@ proc parse(x: var XmlParser, errors: var seq[string]): XmlNode = proc parseXml*(s: Stream, filename: string, errors: var seq[string]): XmlNode = ## parses the XML from stream `s` and returns a ``PXmlNode``. Every - ## occured parsing error is added to the `errors` sequence. + ## occurred parsing error is added to the `errors` sequence. var x: XmlParser open(x, s, filename, {reportComments}) while true: @@ -129,7 +129,7 @@ proc parseXml*(s: Stream): XmlNode = proc loadXml*(path: string, errors: var seq[string]): XmlNode = ## Loads and parses XML from file specified by ``path``, and returns - ## a ``PXmlNode``. Every occured parsing error is added to the `errors` + ## a ``PXmlNode``. Every occurred parsing error is added to the `errors` ## sequence. var s = newFileStream(path, fmRead) if s == nil: raise newException(IOError, "Unable to read file: " & path) @@ -143,7 +143,7 @@ proc loadXml*(path: string): XmlNode = result = loadXml(path, errors) if errors.len > 0: raiseInvalidXml(errors) -when isMainModule: +when not defined(testing) and isMainModule: import os var errors: seq[string] = @[] diff --git a/lib/pure/xmltree.nim b/lib/pure/xmltree.nim index b84da9586..7526a989a 100644 --- a/lib/pure/xmltree.nim +++ b/lib/pure/xmltree.nim @@ -12,20 +12,20 @@ import macros, strtabs type - XmlNode* = ref XmlNodeObj ## an XML tree consists of ``PXmlNode``'s. - + XmlNode* = ref XmlNodeObj ## an XML tree consists of ``PXmlNode``'s. + XmlNodeKind* = enum ## different kinds of ``PXmlNode``'s xnText, ## a text element xnElement, ## an element with 0 or more children xnCData, ## a CDATA node xnEntity, ## an entity (like ``&thing;``) xnComment ## an XML comment - + XmlAttributes* = StringTableRef ## an alias for a string to string mapping - - XmlNodeObj {.acyclic.} = object + + XmlNodeObj {.acyclic.} = object case k: XmlNodeKind # private, use the kind() proc to read this field. - of xnText, xnComment, xnCData, xnEntity: + of xnText, xnComment, xnCData, xnEntity: fText: string of xnElement: fTag: string @@ -41,34 +41,34 @@ proc newXmlNode(kind: XmlNodeKind): XmlNode = new(result) result.k = kind -proc newElement*(tag: string): XmlNode = +proc newElement*(tag: string): XmlNode = ## creates a new ``PXmlNode`` of kind ``xnText`` with the given `tag`. result = newXmlNode(xnElement) result.fTag = tag result.s = @[] # init attributes lazily to safe memory -proc newText*(text: string): XmlNode = +proc newText*(text: string): XmlNode = ## creates a new ``PXmlNode`` of kind ``xnText`` with the text `text`. result = newXmlNode(xnText) result.fText = text -proc newComment*(comment: string): XmlNode = +proc newComment*(comment: string): XmlNode = ## creates a new ``PXmlNode`` of kind ``xnComment`` with the text `comment`. result = newXmlNode(xnComment) result.fText = comment -proc newCData*(cdata: string): XmlNode = +proc newCData*(cdata: string): XmlNode = ## creates a new ``PXmlNode`` of kind ``xnComment`` with the text `cdata`. result = newXmlNode(xnCData) result.fText = cdata -proc newEntity*(entity: string): XmlNode = +proc newEntity*(entity: string): XmlNode = ## creates a new ``PXmlNode`` of kind ``xnEntity`` with the text `entity`. result = newXmlNode(xnCData) result.fText = entity -proc text*(n: XmlNode): string {.inline.} = +proc text*(n: XmlNode): string {.inline.} = ## gets the associated text with the node `n`. `n` can be a CDATA, Text, ## comment, or entity node. assert n.k in {xnText, xnComment, xnCData, xnEntity} @@ -93,16 +93,16 @@ proc innerText*(n: XmlNode): string = for i in 0 .. n.s.len-1: if n.s[i].k in {xnText, xnEntity}: result.add(n.s[i].fText) -proc tag*(n: XmlNode): string {.inline.} = +proc tag*(n: XmlNode): string {.inline.} = ## gets the tag name of `n`. `n` has to be an ``xnElement`` node. assert n.k == xnElement result = n.fTag - -proc add*(father, son: XmlNode) {.inline.} = + +proc add*(father, son: XmlNode) {.inline.} = ## adds the child `son` to `father`. add(father.s, son) - -proc len*(n: XmlNode): int {.inline.} = + +proc len*(n: XmlNode): int {.inline.} = ## returns the number `n`'s children. if n.k == xnElement: result = len(n.s) @@ -110,28 +110,38 @@ proc kind*(n: XmlNode): XmlNodeKind {.inline.} = ## returns `n`'s kind. result = n.k -proc `[]`* (n: XmlNode, i: int): XmlNode {.inline.} = +proc `[]`* (n: XmlNode, i: int): XmlNode {.inline.} = ## returns the `i`'th child of `n`. assert n.k == xnElement result = n.s[i] -iterator items*(n: XmlNode): XmlNode {.inline.} = +proc mget* (n: var XmlNode, i: int): var XmlNode {.inline.} = + ## returns the `i`'th child of `n` so that it can be modified + assert n.k == xnElement + result = n.s[i] + +iterator items*(n: XmlNode): XmlNode {.inline.} = ## iterates over any child of `n`. assert n.k == xnElement for i in 0 .. n.len-1: yield n[i] -proc attrs*(n: XmlNode): XmlAttributes {.inline.} = +iterator mitems*(n: var XmlNode): var XmlNode {.inline.} = + ## iterates over any child of `n`. + assert n.k == xnElement + for i in 0 .. n.len-1: yield mget(n, i) + +proc attrs*(n: XmlNode): XmlAttributes {.inline.} = ## gets the attributes belonging to `n`. ## Returns `nil` if attributes have not been initialised for this node. assert n.k == xnElement result = n.fAttr - -proc `attrs=`*(n: XmlNode, attr: XmlAttributes) {.inline.} = + +proc `attrs=`*(n: XmlNode, attr: XmlAttributes) {.inline.} = ## sets the attributes belonging to `n`. assert n.k == xnElement n.fAttr = attr -proc attrsLen*(n: XmlNode): int {.inline.} = +proc attrsLen*(n: XmlNode): int {.inline.} = ## returns the number of `n`'s attributes. assert n.k == xnElement if not isNil(n.fAttr): result = len(n.fAttr) @@ -141,12 +151,12 @@ proc clientData*(n: XmlNode): int {.inline.} = ## parser and generator. result = n.fClientData -proc `clientData=`*(n: XmlNode, data: int) {.inline.} = +proc `clientData=`*(n: XmlNode, data: int) {.inline.} = ## sets the client data of `n`. The client data field is used by the HTML ## parser and generator. n.fClientData = data -proc addEscaped*(result: var string, s: string) = +proc addEscaped*(result: var string, s: string) = ## same as ``result.add(escape(s))``, but more efficient. for c in items(s): case c @@ -158,8 +168,8 @@ proc addEscaped*(result: var string, s: string) = of '/': result.add("/") else: result.add(c) -proc escape*(s: string): string = - ## escapes `s` for inclusion into an XML document. +proc escape*(s: string): string = + ## escapes `s` for inclusion into an XML document. ## Escapes these characters: ## ## ------------ ------------------- @@ -174,26 +184,26 @@ proc escape*(s: string): string = ## ------------ ------------------- result = newStringOfCap(s.len) addEscaped(result, s) - -proc addIndent(result: var string, indent: int) = + +proc addIndent(result: var string, indent: int) = result.add("\n") for i in 1..indent: result.add(' ') - + proc noWhitespace(n: XmlNode): bool = #for i in 1..n.len-1: # if n[i].kind != n[0].kind: return true for i in 0..n.len-1: if n[i].kind in {xnText, xnEntity}: return true - -proc add*(result: var string, n: XmlNode, indent = 0, indWidth = 2) = + +proc add*(result: var string, n: XmlNode, indent = 0, indWidth = 2) = ## adds the textual representation of `n` to `result`. if n == nil: return case n.k of xnElement: result.add('<') result.add(n.fTag) - if not isNil(n.fAttr): - for key, val in pairs(n.fAttr): + if not isNil(n.fAttr): + for key, val in pairs(n.fAttr): result.add(' ') result.add(key) result.add("=\"") @@ -207,7 +217,7 @@ proc add*(result: var string, n: XmlNode, indent = 0, indWidth = 2) = # because this would be wrong. For example: ``a<b>b</b>`` is # different from ``a <b>b</b>``. for i in 0..n.len-1: result.add(n[i], indent+indWidth, indWidth) - else: + else: for i in 0..n.len-1: result.addIndent(indent+indWidth) result.add(n[i], indent+indWidth, indWidth) @@ -217,7 +227,7 @@ proc add*(result: var string, n: XmlNode, indent = 0, indWidth = 2) = result.add("</") result.add(n.fTag) result.add(">") - else: + else: result.add(" />") of xnText: result.addEscaped(n.fText) @@ -235,7 +245,7 @@ proc add*(result: var string, n: XmlNode, indent = 0, indWidth = 2) = result.add(';') const - xmlHeader* = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n" + xmlHeader* = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n" ## header to use for complete XML output proc `$`*(n: XmlNode): string = @@ -245,21 +255,21 @@ proc `$`*(n: XmlNode): string = result.add(n) proc newXmlTree*(tag: string, children: openArray[XmlNode], - attributes: XmlAttributes = nil): XmlNode = + attributes: XmlAttributes = nil): XmlNode = ## creates a new XML tree with `tag`, `children` and `attributes` result = newXmlNode(xnElement) result.fTag = tag newSeq(result.s, children.len) for i in 0..children.len-1: result.s[i] = children[i] result.fAttr = attributes - -proc xmlConstructor(e: PNimrodNode): PNimrodNode {.compileTime.} = + +proc xmlConstructor(e: NimNode): NimNode {.compileTime.} = expectLen(e, 2) var a = e[1] if a.kind == nnkCall: result = newCall("newXmlTree", toStrLit(a[0])) var attrs = newNimNode(nnkBracket, a) - var newStringTabCall = newCall("newStringTable", attrs, + var newStringTabCall = newCall("newStringTable", attrs, newIdentNode("modeCaseSensitive")) var elements = newNimNode(nnkBracket, a) for i in 1..a.len-1: @@ -270,21 +280,21 @@ proc xmlConstructor(e: PNimrodNode): PNimrodNode {.compileTime.} = else: elements.add(a[i]) result.add(elements) - if attrs.len > 1: + if attrs.len > 1: #echo repr(newStringTabCall) result.add(newStringTabCall) else: result = newCall("newXmlTree", toStrLit(a)) -macro `<>`*(x: expr): expr {.immediate.} = +macro `<>`*(x: expr): expr {.immediate.} = ## Constructor macro for XML. Example usage: ## ## .. code-block:: nim - ## <>a(href="http://nim-code.org", newText("Nim rules.")) + ## <>a(href="http://nim-lang.org", newText("Nim rules.")) ## ## Produces an XML tree for:: ## - ## <a href="http://nim-code.org">Nim rules.</a> + ## <a href="http://nim-lang.org">Nim rules.</a> ## let x = callsite() result = xmlConstructor(x) @@ -343,5 +353,6 @@ proc findAll*(n: XmlNode, tag: string): seq[XmlNode] = findAll(n, tag, result) when isMainModule: - assert """<a href="http://nim-code.org">Nim rules.</a>""" == - $(<>a(href="http://nim-code.org", newText("Nim rules."))) + let link = "http://nim-lang.org" + assert """<a href="""" & escape(link) & """">Nim rules.</a>""" == + $(<>a(href="http://nim-lang.org", newText("Nim rules."))) |