diff options
Diffstat (limited to 'lib/pure')
73 files changed, 4928 insertions, 3768 deletions
diff --git a/lib/pure/algorithm.nim b/lib/pure/algorithm.nim index 9eee04404..fdf2d7cbb 100644 --- a/lib/pure/algorithm.nim +++ b/lib/pure/algorithm.nim @@ -371,3 +371,126 @@ when isMainModule: for i in 0 .. high(arr1): assert arr1.reversed(0, i) == arr1.reversed()[high(arr1) - i .. high(arr1)] assert arr1.reversed(i, high(arr1)) == arr1.reversed()[0 .. high(arr1) - i] + + +proc rotateInternal[T](arg: var openarray[T]; first, middle, last: int): int = + ## A port of std::rotate from c++. Ported from `this reference <http://www.cplusplus.com/reference/algorithm/rotate/>`_. + result = first + last - middle + + if first == middle or middle == last: + return + + assert first < middle + assert middle < last + + # m prefix for mutable + var + mFirst = first + mMiddle = middle + next = middle + + swap(arg[mFirst], arg[next]) + mFirst += 1 + next += 1 + if mFirst == mMiddle: + mMiddle = next + + while next != last: + swap(arg[mFirst], arg[next]) + mFirst += 1 + next += 1 + if mFirst == mMiddle: + mMiddle = next + + next = mMiddle + while next != last: + swap(arg[mFirst], arg[next]) + mFirst += 1 + next += 1 + if mFirst == mMiddle: + mMiddle = next + elif next == last: + next = mMiddle + +proc rotatedInternal[T](arg: openarray[T]; first, middle, last: int): seq[T] = + result = newSeq[T](arg.len) + for i in 0 ..< first: + result[i] = arg[i] + let N = last - middle + let M = middle - first + for i in 0 ..< N: + result[first+i] = arg[middle+i] + for i in 0 ..< M: + result[first+N+i] = arg[first+i] + for i in last ..< arg.len: + result[i] = arg[i] + +proc rotateLeft*[T](arg: var openarray[T]; slice: HSlice[int, int]; dist: int): int = + ## Performs a left rotation on a range of elements. If you want to rotate right, use a negative ``dist``. + ## Specifically, ``rotateLeft`` rotates the elements at ``slice`` by ``dist`` positions. + ## The element at index ``slice.a + dist`` will be at index ``slice.a``. + ## The element at index ``slice.b`` will be at ``slice.a + dist -1``. + ## The element at index ``slice.a`` will be at ``slice.b + 1 - dist``. + ## The element at index ``slice.a + dist - 1`` will be at ``slice.b``. + # + ## Elements outsize of ``slice`` will be left unchanged. + ## The time complexity is linear to ``slice.b - slice.a + 1``. + ## + ## ``slice`` + ## the indices of the element range that should be rotated. + ## + ## ``dist`` + ## the distance in amount of elements that the data should be rotated. Can be negative, can be any number. + ## + ## .. code-block:: nim + ## var list = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + ## list.rotateLeft(1 .. 8, 3) + ## doAssert list == [0, 4, 5, 6, 7, 8, 1, 2, 3, 9, 10] + let sliceLen = slice.b + 1 - slice.a + let distLeft = ((dist mod sliceLen) + sliceLen) mod sliceLen + arg.rotateInternal(slice.a, slice.a+distLeft, slice.b + 1) + +proc rotateLeft*[T](arg: var openarray[T]; dist: int): int = + ## default arguments for slice, so that this procedure operates on the entire + ## ``arg``, and not just on a part of it. + let arglen = arg.len + let distLeft = ((dist mod arglen) + arglen) mod arglen + arg.rotateInternal(0, distLeft, arglen) + +proc rotatedLeft*[T](arg: openarray[T]; slice: HSlice[int, int], dist: int): seq[T] = + ## same as ``rotateLeft``, just with the difference that it does + ## not modify the argument. It creates a new ``seq`` instead + let sliceLen = slice.b + 1 - slice.a + let distLeft = ((dist mod sliceLen) + sliceLen) mod sliceLen + arg.rotatedInternal(slice.a, slice.a+distLeft, slice.b+1) + +proc rotatedLeft*[T](arg: openarray[T]; dist: int): seq[T] = + ## same as ``rotateLeft``, just with the difference that it does + ## not modify the argument. It creates a new ``seq`` instead + let arglen = arg.len + let distLeft = ((dist mod arglen) + arglen) mod arglen + arg.rotatedInternal(0, distLeft, arg.len) + +when isMainModule: + var list = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + let list2 = list.rotatedLeft(1 ..< 9, 3) + let expected = [0, 4, 5, 6, 7, 8, 1, 2, 3, 9, 10] + + doAssert list.rotateLeft(1 ..< 9, 3) == 6 + doAssert list == expected + doAssert list2 == @expected + + var s0,s1,s2,s3,s4,s5 = "xxxabcdefgxxx" + + doAssert s0.rotateLeft(3 ..< 10, 3) == 7 + doAssert s0 == "xxxdefgabcxxx" + doAssert s1.rotateLeft(3 ..< 10, 2) == 8 + doAssert s1 == "xxxcdefgabxxx" + doAssert s2.rotateLeft(3 ..< 10, 4) == 6 + doAssert s2 == "xxxefgabcdxxx" + doAssert s3.rotateLeft(3 ..< 10, -3) == 6 + doAssert s3 == "xxxefgabcdxxx" + doAssert s4.rotateLeft(3 ..< 10, -10) == 6 + doAssert s4 == "xxxefgabcdxxx" + doAssert s5.rotateLeft(3 ..< 10, 11) == 6 + doAssert s5 == "xxxefgabcdxxx" diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index 281d5b848..42ffa236c 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -9,8 +9,9 @@ include "system/inclrtl" -import os, tables, strutils, times, heapqueue, options, asyncstreams +import os, tables, strutils, times, heapqueue, lists, options, asyncstreams import asyncfutures except callSoon + import nativesockets, net, deques export Port, SocketFlag @@ -58,9 +59,10 @@ export asyncfutures, asyncstreams ## ## .. code-block::nim ## var future = socket.recv(100) -## future.callback = +## future.addCallback( ## proc () = ## echo(future.read) +## ) ## ## All asynchronous functions returning a ``Future`` will not block. They ## will not however return immediately. An asynchronous function will have @@ -136,6 +138,7 @@ export asyncfutures, asyncstreams ## and occasionally the compilation may fail altogether. ## As such it is better to use the former style when possible. ## +## ## Discarding futures ## ------------------ ## @@ -165,18 +168,20 @@ type timers*: HeapQueue[tuple[finishAt: float, fut: Future[void]]] callbacks*: Deque[proc ()] -proc processTimers(p: PDispatcherBase) {.inline.} = +proc processTimers(p: PDispatcherBase; didSomeWork: var bool) {.inline.} = #Process just part if timers at a step var count = p.timers.len let t = epochTime() while count > 0 and t >= p.timers[0].finishAt: p.timers.pop().fut.complete() dec count + didSomeWork = true -proc processPendingCallbacks(p: PDispatcherBase) = +proc processPendingCallbacks(p: PDispatcherBase; didSomeWork: var bool) = while p.callbacks.len > 0: var cb = p.callbacks.popFirst() cb() + didSomeWork = true proc adjustedTimeout(p: PDispatcherBase, timeout: int): int {.inline.} = # If dispatcher has active timers this proc returns the timeout @@ -226,6 +231,12 @@ when defined(windows) or defined(nimdoc): ovl: PCustomOverlapped PostCallbackDataPtr = ptr PostCallbackData + AsyncEventImpl = object + hEvent: Handle + hWaiter: Handle + pcd: PostCallbackDataPtr + AsyncEvent* = ptr AsyncEventImpl + Callback = proc (fd: AsyncFD): bool {.closure,gcsafe.} {.deprecated: [TCompletionKey: CompletionKey, TAsyncFD: AsyncFD, TCustomOverlapped: CustomOverlapped, TCompletionData: CompletionData].} @@ -275,14 +286,13 @@ when defined(windows) or defined(nimdoc): let p = getGlobalDispatcher() p.handles.len != 0 or p.timers.len != 0 or p.callbacks.len != 0 - proc poll*(timeout = 500) = - ## Waits for completion events and processes them. Raises ``ValueError`` - ## if there are no pending operations. + proc runOnce(timeout = 500): bool = let p = getGlobalDispatcher() if p.handles.len == 0 and p.timers.len == 0 and p.callbacks.len == 0: raise newException(ValueError, "No handles or timers registered in dispatcher.") + result = false if p.handles.len != 0: let at = p.adjustedTimeout(timeout) var llTimeout = @@ -295,6 +305,7 @@ when defined(windows) or defined(nimdoc): let res = getQueuedCompletionStatus(p.ioPort, addr lpNumberOfBytesTransferred, addr lpCompletionKey, cast[ptr POVERLAPPED](addr customOverlapped), llTimeout).bool + result = true # http://stackoverflow.com/a/12277264/492186 # TODO: http://www.serverframework.com/handling-multiple-pending-socket-read-and-write-operations.html @@ -324,17 +335,18 @@ when defined(windows) or defined(nimdoc): else: if errCode.int32 == WAIT_TIMEOUT: # Timed out - discard + result = false else: raiseOSError(errCode) # Timer processing. - processTimers(p) + processTimers(p, result) # Callback queue processing - processPendingCallbacks(p) + processPendingCallbacks(p, result) + - var connectExPtr: pointer = nil - var acceptExPtr: pointer = nil - var getAcceptExSockAddrsPtr: pointer = nil + var acceptEx: WSAPROC_ACCEPTEX + var connectEx: WSAPROC_CONNECTEX + var getAcceptExSockAddrs: WSAPROC_GETACCEPTEXSOCKADDRS proc initPointer(s: SocketHandle, fun: var pointer, guid: var GUID): bool = # Ref: https://github.com/powdahound/twisted/blob/master/twisted/internet/iocpreactor/iocpsupport/winsock_pointers.c @@ -346,56 +358,19 @@ when defined(windows) or defined(nimdoc): proc initAll() = let dummySock = newNativeSocket() - if not initPointer(dummySock, connectExPtr, WSAID_CONNECTEX): + if dummySock == INVALID_SOCKET: raiseOSError(osLastError()) - if not initPointer(dummySock, acceptExPtr, WSAID_ACCEPTEX): + var fun: pointer = nil + if not initPointer(dummySock, fun, WSAID_CONNECTEX): raiseOSError(osLastError()) - if not initPointer(dummySock, getAcceptExSockAddrsPtr, WSAID_GETACCEPTEXSOCKADDRS): + connectEx = cast[WSAPROC_CONNECTEX](fun) + if not initPointer(dummySock, fun, WSAID_ACCEPTEX): raiseOSError(osLastError()) - - proc connectEx(s: SocketHandle, name: ptr SockAddr, namelen: cint, - lpSendBuffer: pointer, dwSendDataLength: Dword, - lpdwBytesSent: PDword, lpOverlapped: POVERLAPPED): bool = - if connectExPtr.isNil: raise newException(ValueError, "Need to initialise ConnectEx().") - let fun = - cast[proc (s: SocketHandle, name: ptr SockAddr, namelen: cint, - lpSendBuffer: pointer, dwSendDataLength: Dword, - lpdwBytesSent: PDword, lpOverlapped: POVERLAPPED): bool {.stdcall,gcsafe.}](connectExPtr) - - result = fun(s, name, namelen, lpSendBuffer, dwSendDataLength, lpdwBytesSent, - lpOverlapped) - - proc acceptEx(listenSock, acceptSock: SocketHandle, lpOutputBuffer: pointer, - dwReceiveDataLength, dwLocalAddressLength, - dwRemoteAddressLength: Dword, lpdwBytesReceived: PDword, - lpOverlapped: POVERLAPPED): bool = - if acceptExPtr.isNil: raise newException(ValueError, "Need to initialise AcceptEx().") - let fun = - cast[proc (listenSock, acceptSock: SocketHandle, lpOutputBuffer: pointer, - dwReceiveDataLength, dwLocalAddressLength, - dwRemoteAddressLength: Dword, lpdwBytesReceived: PDword, - lpOverlapped: POVERLAPPED): bool {.stdcall,gcsafe.}](acceptExPtr) - result = fun(listenSock, acceptSock, lpOutputBuffer, dwReceiveDataLength, - dwLocalAddressLength, dwRemoteAddressLength, lpdwBytesReceived, - lpOverlapped) - - proc getAcceptExSockaddrs(lpOutputBuffer: pointer, - dwReceiveDataLength, dwLocalAddressLength, dwRemoteAddressLength: Dword, - LocalSockaddr: ptr ptr SockAddr, LocalSockaddrLength: LPInt, - RemoteSockaddr: ptr ptr SockAddr, RemoteSockaddrLength: LPInt) = - if getAcceptExSockAddrsPtr.isNil: - raise newException(ValueError, "Need to initialise getAcceptExSockAddrs().") - - let fun = - cast[proc (lpOutputBuffer: pointer, - dwReceiveDataLength, dwLocalAddressLength, - dwRemoteAddressLength: Dword, LocalSockaddr: ptr ptr SockAddr, - LocalSockaddrLength: LPInt, RemoteSockaddr: ptr ptr SockAddr, - RemoteSockaddrLength: LPInt) {.stdcall,gcsafe.}](getAcceptExSockAddrsPtr) - - fun(lpOutputBuffer, dwReceiveDataLength, dwLocalAddressLength, - dwRemoteAddressLength, LocalSockaddr, LocalSockaddrLength, - RemoteSockaddr, RemoteSockaddrLength) + acceptEx = cast[WSAPROC_ACCEPTEX](fun) + if not initPointer(dummySock, fun, WSAID_GETACCEPTEXSOCKADDRS): + raiseOSError(osLastError()) + getAcceptExSockAddrs = cast[WSAPROC_GETACCEPTEXSOCKADDRS](fun) + close(dummySock) proc recv*(socket: AsyncFD, size: int, flags = {SocketFlag.SafeDisconn}): Future[string] = @@ -506,10 +481,7 @@ when defined(windows) or defined(nimdoc): proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = if not retFuture.finished: if errcode == OSErrorCode(-1): - if bytesCount == 0 and dataBuf.buf[0] == '\0': - retFuture.complete(0) - else: - retFuture.complete(bytesCount) + retFuture.complete(bytesCount) else: if flags.isDisconnectionError(errcode): retFuture.complete(0) @@ -543,10 +515,11 @@ when defined(windows) or defined(nimdoc): proc send*(socket: AsyncFD, buf: pointer, size: int, flags = {SocketFlag.SafeDisconn}): Future[void] = - ## Sends ``size`` bytes from ``buf`` to ``socket``. The returned future will complete once all - ## data has been sent. - ## **WARNING**: Use it with caution. If ``buf`` refers to GC'ed object, you must use GC_ref/GC_unref calls - ## to avoid early freeing of the buffer + ## Sends ``size`` bytes from ``buf`` to ``socket``. The returned future + ## will complete once all data has been sent. + ## + ## **WARNING**: Use it with caution. If ``buf`` refers to GC'ed object, + ## you must use GC_ref/GC_unref calls to avoid early freeing of the buffer. verifyPresence(socket) var retFuture = newFuture[void]("send") @@ -793,7 +766,7 @@ when defined(windows) or defined(nimdoc): cast[pointer](p.ovl)) {.pop.} - template registerWaitableEvent(mask) = + proc registerWaitableEvent(fd: AsyncFD, cb: Callback; mask: Dword) = let p = getGlobalDispatcher() var flags = (WT_EXECUTEINWAITTHREAD or WT_EXECUTEONLYONCE).Dword var hEvent = wsaCreateEvent() @@ -843,8 +816,8 @@ when defined(windows) or defined(nimdoc): cast[pointer](pcd), INFINITE, flags): # pcd.ovl will be unrefed in poll() let err = osLastError() - discard wsaCloseEvent(hEvent) deallocShared(cast[pointer](pcd)) + discard wsaCloseEvent(hEvent) raiseOSError(err) else: # we incref `pcd.ovl` and `protect` callback one more time, @@ -883,16 +856,17 @@ when defined(windows) or defined(nimdoc): ## ## This is not ``pure`` mechanism for Windows Completion Ports (IOCP), ## so if you can avoid it, please do it. Use `addRead` only if really - ## need it (main usecase is adaptation of `unix like` libraries to be + ## need it (main usecase is adaptation of unix-like libraries to be ## asynchronous on Windows). - ## If you use this function, you dont need to use asyncdispatch.recv() + ## + ## If you use this function, you don't need to use asyncdispatch.recv() ## or asyncdispatch.accept(), because they are using IOCP, please use ## nativesockets.recv() and nativesockets.accept() instead. ## ## Be sure your callback ``cb`` returns ``true``, if you want to remove ## watch of `read` notifications, and ``false``, if you want to continue - ## receiving notifies. - registerWaitableEvent(FD_READ or FD_ACCEPT or FD_OOB or FD_CLOSE) + ## receiving notifications. + registerWaitableEvent(fd, cb, FD_READ or FD_ACCEPT or FD_OOB or FD_CLOSE) proc addWrite*(fd: AsyncFD, cb: Callback) = ## Start watching the file descriptor for write availability and then call @@ -900,43 +874,213 @@ when defined(windows) or defined(nimdoc): ## ## This is not ``pure`` mechanism for Windows Completion Ports (IOCP), ## so if you can avoid it, please do it. Use `addWrite` only if really - ## need it (main usecase is adaptation of `unix like` libraries to be + ## need it (main usecase is adaptation of unix-like libraries to be ## asynchronous on Windows). - ## If you use this function, you dont need to use asyncdispatch.send() + ## + ## If you use this function, you don't need to use asyncdispatch.send() ## or asyncdispatch.connect(), because they are using IOCP, please use ## nativesockets.send() and nativesockets.connect() instead. ## ## Be sure your callback ``cb`` returns ``true``, if you want to remove ## watch of `write` notifications, and ``false``, if you want to continue - ## receiving notifies. - registerWaitableEvent(FD_WRITE or FD_CONNECT or FD_CLOSE) + ## receiving notifications. + registerWaitableEvent(fd, cb, FD_WRITE or FD_CONNECT or FD_CLOSE) + + template registerWaitableHandle(p, hEvent, flags, pcd, timeout, + handleCallback) = + let handleFD = AsyncFD(hEvent) + pcd.ioPort = p.ioPort + pcd.handleFd = handleFD + var ol = PCustomOverlapped() + GC_ref(ol) + ol.data.fd = handleFD + ol.data.cb = handleCallback + # We need to protect our callback environment value, so GC will not free it + # accidentally. + ol.data.cell = system.protect(rawEnv(ol.data.cb)) + + pcd.ovl = ol + if not registerWaitForSingleObject(addr(pcd.waitFd), hEvent, + cast[WAITORTIMERCALLBACK](waitableCallback), + cast[pointer](pcd), timeout.Dword, flags): + let err = osLastError() + GC_unref(ol) + deallocShared(cast[pointer](pcd)) + discard closeHandle(hEvent) + raiseOSError(err) + p.handles.incl(handleFD) + + template closeWaitable(handle: untyped) = + let waitFd = pcd.waitFd + deallocShared(cast[pointer](pcd)) + p.handles.excl(fd) + if unregisterWait(waitFd) == 0: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + discard closeHandle(handle) + raiseOSError(err) + if closeHandle(handle) == 0: + raiseOSError(osLastError()) + + proc addTimer*(timeout: int, oneshot: bool, cb: Callback) = + ## Registers callback ``cb`` to be called when timer expired. + ## + ## Parameters: + ## + ## * ``timeout`` - timeout value in milliseconds. + ## * ``oneshot`` + ## * `true` - generate only one timeout event + ## * `false` - generate timeout events periodically + + doAssert(timeout > 0) + let p = getGlobalDispatcher() + + var hEvent = createEvent(nil, 1, 0, nil) + if hEvent == INVALID_HANDLE_VALUE: + raiseOSError(osLastError()) + + var pcd = cast[PostCallbackDataPtr](allocShared0(sizeof(PostCallbackData))) + var flags = WT_EXECUTEINWAITTHREAD.Dword + if oneshot: flags = flags or WT_EXECUTEONLYONCE + + proc timercb(fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + let res = cb(fd) + if res or oneshot: + closeWaitable(hEvent) + else: + # if callback returned `false`, then it wants to be called again, so + # we need to ref and protect `pcd.ovl` again, because it will be + # unrefed and disposed in `poll()`. + GC_ref(pcd.ovl) + pcd.ovl.data.cell = system.protect(rawEnv(pcd.ovl.data.cb)) + + registerWaitableHandle(p, hEvent, flags, pcd, timeout, timercb) + + proc addProcess*(pid: int, cb: Callback) = + ## Registers callback ``cb`` to be called when process with process ID + ## ``pid`` exited. + let p = getGlobalDispatcher() + let procFlags = SYNCHRONIZE + var hProcess = openProcess(procFlags, 0, pid.Dword) + if hProcess == INVALID_HANDLE_VALUE: + raiseOSError(osLastError()) + + var pcd = cast[PostCallbackDataPtr](allocShared0(sizeof(PostCallbackData))) + var flags = WT_EXECUTEINWAITTHREAD.Dword + + proc proccb(fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + closeWaitable(hProcess) + discard cb(fd) + + registerWaitableHandle(p, hProcess, flags, pcd, INFINITE, proccb) + + proc newAsyncEvent*(): AsyncEvent = + ## Creates a new thread-safe ``AsyncEvent`` object. + ## + ## New ``AsyncEvent`` object is not automatically registered with # TODO: Why? -- DP + ## dispatcher like ``AsyncSocket``. + var sa = SECURITY_ATTRIBUTES( + nLength: sizeof(SECURITY_ATTRIBUTES).cint, + bInheritHandle: 1 + ) + var event = createEvent(addr(sa), 0'i32, 0'i32, nil) + if event == INVALID_HANDLE_VALUE: + raiseOSError(osLastError()) + result = cast[AsyncEvent](allocShared0(sizeof(AsyncEventImpl))) + result.hEvent = event + + proc trigger*(ev: AsyncEvent) = + ## Set event ``ev`` to signaled state. + if setEvent(ev.hEvent) == 0: + raiseOSError(osLastError()) + + proc unregister*(ev: AsyncEvent) = + ## Unregisters event ``ev``. + doAssert(ev.hWaiter != 0, "Event is not registered in the queue!") + let p = getGlobalDispatcher() + p.handles.excl(AsyncFD(ev.hEvent)) + if unregisterWait(ev.hWaiter) == 0: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + raiseOSError(err) + ev.hWaiter = 0 + + proc close*(ev: AsyncEvent) = + ## Closes event ``ev``. + let res = closeHandle(ev.hEvent) + deallocShared(cast[pointer](ev)) + if res == 0: + raiseOSError(osLastError()) + + proc addEvent*(ev: AsyncEvent, cb: Callback) = + ## Registers callback ``cb`` to be called when ``ev`` will be signaled + doAssert(ev.hWaiter == 0, "Event is already registered in the queue!") + + let p = getGlobalDispatcher() + let hEvent = ev.hEvent + + var pcd = cast[PostCallbackDataPtr](allocShared0(sizeof(PostCallbackData))) + var flags = WT_EXECUTEINWAITTHREAD.Dword + + proc eventcb(fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + if ev.hWaiter != 0: + if cb(fd): + # we need this check to avoid exception, if `unregister(event)` was + # called in callback. + deallocShared(cast[pointer](pcd)) + if ev.hWaiter != 0: + unregister(ev) + else: + # if callback returned `false`, then it wants to be called again, so + # we need to ref and protect `pcd.ovl` again, because it will be + # unrefed and disposed in `poll()`. + GC_ref(pcd.ovl) + pcd.ovl.data.cell = system.protect(rawEnv(pcd.ovl.data.cb)) + else: + # if ev.hWaiter == 0, then event was unregistered before `poll()` call. + deallocShared(cast[pointer](pcd)) + + registerWaitableHandle(p, hEvent, flags, pcd, INFINITE, eventcb) + ev.hWaiter = pcd.waitFd initAll() else: import selectors from posix import EINTR, EAGAIN, EINPROGRESS, EWOULDBLOCK, MSG_PEEK, MSG_NOSIGNAL - + const + InitCallbackListSize = 4 # initial size of callbacks sequence, + # associated with file/socket descriptor. + InitDelayedCallbackListSize = 64 # initial size of delayed callbacks + # queue. type AsyncFD* = distinct cint Callback = proc (fd: AsyncFD): bool {.closure,gcsafe.} - PData* = ref object of RootRef - fd: AsyncFD - readCBs: seq[Callback] - writeCBs: seq[Callback] + AsyncData = object + readList: seq[Callback] + writeList: seq[Callback] + + AsyncEvent* = distinct SelectEvent PDispatcher* = ref object of PDispatcherBase - selector: Selector + selector: Selector[AsyncData] {.deprecated: [TAsyncFD: AsyncFD, TCallback: Callback].} proc `==`*(x, y: AsyncFD): bool {.borrow.} + proc `==`*(x, y: AsyncEvent): bool {.borrow.} + + template newAsyncData(): AsyncData = + AsyncData( + readList: newSeqOfCap[Callback](InitCallbackListSize), + writeList: newSeqOfCap[Callback](InitCallbackListSize) + ) proc newDispatcher*(): PDispatcher = new result - result.selector = newSelector() + result.selector = newSelector[AsyncData]() result.timers.newHeapQueue() - result.callbacks = initDeque[proc ()](64) + result.callbacks = initDeque[proc ()](InitDelayedCallbackListSize) var gDisp{.threadvar.}: PDispatcher ## Global dispatcher @@ -951,15 +1095,10 @@ else: setGlobalDispatcher(newDispatcher()) result = gDisp - proc update(fd: AsyncFD, events: set[Event]) = - let p = getGlobalDispatcher() - assert fd.SocketHandle in p.selector - p.selector.update(fd.SocketHandle, events) - proc register*(fd: AsyncFD) = let p = getGlobalDispatcher() - var data = PData(fd: fd, readCBs: @[], writeCBs: @[]) - p.selector.register(fd.SocketHandle, {}, data.RootRef) + var data = newAsyncData() + p.selector.registerHandle(fd.SocketHandle, {}, data) proc closeSocket*(sock: AsyncFD) = let disp = getGlobalDispatcher() @@ -969,80 +1108,158 @@ else: proc unregister*(fd: AsyncFD) = getGlobalDispatcher().selector.unregister(fd.SocketHandle) + proc unregister*(ev: AsyncEvent) = + getGlobalDispatcher().selector.unregister(SelectEvent(ev)) + proc addRead*(fd: AsyncFD, cb: Callback) = let p = getGlobalDispatcher() - if fd.SocketHandle notin p.selector: + var newEvents = {Event.Read} + withData(p.selector, fd.SocketHandle, adata) do: + adata.readList.add(cb) + newEvents.incl(Event.Read) + if len(adata.writeList) != 0: newEvents.incl(Event.Write) + do: raise newException(ValueError, "File descriptor not registered.") - p.selector[fd.SocketHandle].data.PData.readCBs.add(cb) - update(fd, p.selector[fd.SocketHandle].events + {EvRead}) + p.selector.updateHandle(fd.SocketHandle, newEvents) proc addWrite*(fd: AsyncFD, cb: Callback) = let p = getGlobalDispatcher() - if fd.SocketHandle notin p.selector: + var newEvents = {Event.Write} + withData(p.selector, fd.SocketHandle, adata) do: + adata.writeList.add(cb) + newEvents.incl(Event.Write) + if len(adata.readList) != 0: newEvents.incl(Event.Read) + do: raise newException(ValueError, "File descriptor not registered.") - p.selector[fd.SocketHandle].data.PData.writeCBs.add(cb) - update(fd, p.selector[fd.SocketHandle].events + {EvWrite}) - - template processCallbacks(callbacks: untyped) = - # Callback may add items to ``callbacks`` which causes issues if - # we are iterating over it at the same time. We therefore - # make a copy to iterate over. - let currentCBs = callbacks - callbacks = @[] - # Using another sequence because callbacks themselves can add - # other callbacks. - var newCBs: seq[Callback] = @[] - for cb in currentCBs: - if newCBs.len > 0: - # A callback has already returned with EAGAIN, don't call any - # others until next `poll`. - newCBs.add(cb) - else: - if not cb(data.fd): - # Callback wants to be called again. - newCBs.add(cb) - callbacks = newCBs & callbacks + p.selector.updateHandle(fd.SocketHandle, newEvents) proc hasPendingOperations*(): bool = let p = getGlobalDispatcher() - p.selector.len != 0 or p.timers.len != 0 or p.callbacks.len != 0 - - proc poll*(timeout = 500) = + not p.selector.isEmpty() or p.timers.len != 0 or p.callbacks.len != 0 + + template processBasicCallbacks(ident, rwlist: untyped) = + # Process pending descriptor and AsyncEvent callbacks. + # + # Invoke every callback stored in `rwlist`, until one + # returns `false` (which means callback wants to stay + # alive). In such case all remaining callbacks will be added + # to `rwlist` again, in the order they have been inserted. + # + # `rwlist` associated with file descriptor MUST BE emptied before + # dispatching callback (See https://github.com/nim-lang/Nim/issues/5128), + # or it can be possible to fall into endless cycle. + var curList: seq[Callback] + + withData(p.selector, ident, adata) do: + shallowCopy(curList, adata.rwlist) + adata.rwlist = newSeqOfCap[Callback](InitCallbackListSize) + + let newLength = max(len(curList), InitCallbackListSize) + var newList = newSeqOfCap[Callback](newLength) + + for cb in curList: + if len(newList) > 0: + # A callback has already returned with EAGAIN, don't call any others + # until next `poll`. + newList.add(cb) + else: + if not cb(fd.AsyncFD): + # Callback wants to be called again. + newList.add(cb) + + withData(p.selector, ident, adata) do: + # descriptor still present in queue. + adata.rwlist = newList & adata.rwlist + rLength = len(adata.readList) + wLength = len(adata.writeList) + do: + # descriptor was unregistered in callback via `unregister()`. + rLength = -1 + wLength = -1 + + template processCustomCallbacks(ident: untyped) = + # Process pending custom event callbacks. Custom events are + # {Event.Timer, Event.Signal, Event.Process, Event.Vnode}. + # There can be only one callback registered with one descriptor, + # so there is no need to iterate over list. + var curList: seq[Callback] + + withData(p.selector, ident, adata) do: + shallowCopy(curList, adata.readList) + adata.readList = newSeqOfCap[Callback](InitCallbackListSize) + + let newLength = len(curList) + var newList = newSeqOfCap[Callback](newLength) + + var cb = curList[0] + if not cb(fd.AsyncFD): + newList.add(cb) + + withData(p.selector, ident, adata) do: + # descriptor still present in queue. + adata.readList = newList & adata.readList + if len(adata.readList) == 0: + # if no callbacks registered with descriptor, unregister it. + p.selector.unregister(fd) + do: + # descriptor was unregistered in callback via `unregister()`. + discard + + proc runOnce(timeout = 500): bool = let p = getGlobalDispatcher() - if p.selector.len == 0 and p.timers.len == 0 and p.callbacks.len == 0: + when ioselSupportedPlatform: + let customSet = {Event.Timer, Event.Signal, Event.Process, + Event.Vnode} + + if p.selector.isEmpty() and p.timers.len == 0 and p.callbacks.len == 0: raise newException(ValueError, "No handles or timers registered in dispatcher.") - if p.selector.len > 0: - for info in p.selector.select(p.adjustedTimeout(timeout)): - let data = PData(info.key.data) - assert data.fd == info.key.fd.AsyncFD - #echo("In poll ", data.fd.cint) - # There may be EvError here, but we handle them in callbacks, - # so that exceptions can be raised from `send(...)` and - # `recv(...)` routines. - - if EvRead in info.events or info.events == {EvError}: - processCallbacks(data.readCBs) - - if EvWrite in info.events or info.events == {EvError}: - processCallbacks(data.writeCBs) - - if info.key in p.selector: - var newEvents: set[Event] - if data.readCBs.len != 0: newEvents = {EvRead} - if data.writeCBs.len != 0: newEvents = newEvents + {EvWrite} - if newEvents != info.key.events: - update(data.fd, newEvents) - else: - # FD no longer a part of the selector. Likely been closed - # (e.g. socket disconnected). - discard + result = false + if not p.selector.isEmpty(): + var keys: array[64, ReadyKey] + var count = p.selector.selectInto(p.adjustedTimeout(timeout), keys) + for i in 0..<count: + var custom = false + let fd = keys[i].fd + let events = keys[i].events + var rLength = 0 # len(data.readList) after callback + var wLength = 0 # len(data.writeList) after callback + + if Event.Read in events or events == {Event.Error}: + processBasicCallbacks(fd, readList) + result = true + + if Event.Write in events or events == {Event.Error}: + processBasicCallbacks(fd, writeList) + result = true + + if Event.User in events: + processBasicCallbacks(fd, readList) + custom = true + if rLength == 0: + p.selector.unregister(fd) + result = true + + when ioselSupportedPlatform: + if (customSet * events) != {}: + custom = true + processCustomCallbacks(fd) + result = true + + # because state `data` can be modified in callback we need to update + # descriptor events with currently registered callbacks. + if not custom: + var newEvents: set[Event] = {} + if rLength != -1 and wLength != -1: + if rLength > 0: incl(newEvents, Event.Read) + if wLength > 0: incl(newEvents, Event.Write) + p.selector.updateHandle(SocketHandle(fd), newEvents) # Timer processing. - processTimers(p) + processTimers(p, result) # Callback queue processing - processPendingCallbacks(p) + processPendingCallbacks(p, result) proc recv*(socket: AsyncFD, size: int, flags = {SocketFlag.SafeDisconn}): Future[string] = @@ -1075,7 +1292,7 @@ else: return retFuture proc recvInto*(socket: AsyncFD, buf: pointer, size: int, - flags = {SocketFlag.SafeDisconn}): Future[int] = + flags = {SocketFlag.SafeDisconn}): Future[int] = var retFuture = newFuture[int]("recvInto") proc cb(sock: AsyncFD): bool = @@ -1216,6 +1433,68 @@ else: addRead(socket, cb) return retFuture + when ioselSupportedPlatform: + + proc addTimer*(timeout: int, oneshot: bool, cb: Callback) = + ## Start watching for timeout expiration, and then call the + ## callback ``cb``. + ## ``timeout`` - time in milliseconds, + ## ``oneshot`` - if ``true`` only one event will be dispatched, + ## if ``false`` continuous events every ``timeout`` milliseconds. + let p = getGlobalDispatcher() + var data = newAsyncData() + data.readList.add(cb) + p.selector.registerTimer(timeout, oneshot, data) + + proc addSignal*(signal: int, cb: Callback) = + ## Start watching signal ``signal``, and when signal appears, call the + ## callback ``cb``. + let p = getGlobalDispatcher() + var data = newAsyncData() + data.readList.add(cb) + p.selector.registerSignal(signal, data) + + proc addProcess*(pid: int, cb: Callback) = + ## Start watching for process exit with pid ``pid``, and then call + ## the callback ``cb``. + let p = getGlobalDispatcher() + var data = newAsyncData() + data.readList.add(cb) + p.selector.registerProcess(pid, data) + + proc newAsyncEvent*(): AsyncEvent = + ## Creates new ``AsyncEvent``. + result = AsyncEvent(newSelectEvent()) + + proc trigger*(ev: AsyncEvent) = + ## Sets new ``AsyncEvent`` to signaled state. + trigger(SelectEvent(ev)) + + proc close*(ev: AsyncEvent) = + ## Closes ``AsyncEvent`` + close(SelectEvent(ev)) + + proc addEvent*(ev: AsyncEvent, cb: Callback) = + ## Start watching for event ``ev``, and call callback ``cb``, when + ## ev will be set to signaled state. + let p = getGlobalDispatcher() + var data = newAsyncData() + data.readList.add(cb) + p.selector.registerEvent(SelectEvent(ev), data) + +proc drain*(timeout = 500) = + ## Waits for completion events and processes them. Raises ``ValueError`` + ## if there are no pending operations. In contrast to ``poll`` this + ## processes as many events as are available. + if runOnce(timeout): + while hasPendingOperations() and runOnce(0): discard + +proc poll*(timeout = 500) = + ## Waits for completion events and processes them. Raises ``ValueError`` + ## if there are no pending operations. This runs the underlying OS + ## `epoll`:idx: or `kqueue`:idx: primitive only once. + discard runOnce(timeout) + # Common procedures between current and upcoming asyncdispatch include includes.asynccommon @@ -1269,7 +1548,7 @@ proc send*(socket: AsyncFD, data: string, var copiedData = data GC_ref(copiedData) # we need to protect data until send operation is completed - # or failed. + # or failed. let sendFut = socket.send(addr copiedData[0], data.len, flags) sendFut.callback = @@ -1317,7 +1596,7 @@ proc recvLine*(socket: AsyncFD): Future[string] {.async, deprecated.} = ## ## **Deprecated since version 0.15.0**: Use ``asyncnet.recvLine()`` instead. - template addNLIfEmpty(): untyped = + template addNLIfEmpty(): typed = if result.len == 0: result.add("\c\L") @@ -1353,3 +1632,5 @@ proc waitFor*[T](fut: Future[T]): T = poll() fut.read + +{.deprecated: [setEvent: trigger].} diff --git a/lib/pure/asyncfutures.nim b/lib/pure/asyncfutures.nim index bebd19611..bcc3ab613 100644 --- a/lib/pure/asyncfutures.nim +++ b/lib/pure/asyncfutures.nim @@ -1,4 +1,4 @@ -import os, tables, strutils, times, heapqueue, options, deques +import os, tables, strutils, times, heapqueue, options, deques, cstrutils # TODO: This shouldn't need to be included, but should ideally be exported. type @@ -217,17 +217,78 @@ proc `callback=`*[T](future: Future[T], ## If future has already completed then ``cb`` will be called immediately. future.callback = proc () = cb(future) +proc getHint(entry: StackTraceEntry): string = + ## We try to provide some hints about stack trace entries that the user + ## may not be familiar with, in particular calls inside the stdlib. + result = "" + if entry.procname == "processPendingCallbacks": + if cmpIgnoreStyle(entry.filename, "asyncdispatch.nim") == 0: + return "Executes pending callbacks" + elif entry.procname == "poll": + if cmpIgnoreStyle(entry.filename, "asyncdispatch.nim") == 0: + return "Processes asynchronous completion events" + + if entry.procname.endsWith("_continue"): + if cmpIgnoreStyle(entry.filename, "asyncmacro.nim") == 0: + return "Resumes an async procedure" + +proc `$`*(entries: seq[StackTraceEntry]): string = + result = "" + # Find longest filename & line number combo for alignment purposes. + var longestLeft = 0 + for entry in entries: + if entry.procName.isNil: continue + + let left = $entry.filename & $entry.line + if left.len > longestLeft: + longestLeft = left.len + + var indent = 2 + # Format the entries. + for entry in entries: + if entry.procName.isNil: + if entry.line == -10: + result.add(spaces(indent) & "#[\n") + indent.inc(2) + else: + indent.dec(2) + result.add(spaces(indent)& "]#\n") + continue + + let left = "$#($#)" % [$entry.filename, $entry.line] + result.add((spaces(indent) & "$#$# $#\n") % [ + left, + spaces(longestLeft - left.len + 2), + $entry.procName + ]) + let hint = getHint(entry) + if hint.len > 0: + result.add(spaces(indent+2) & "## " & hint & "\n") + proc injectStacktrace[T](future: Future[T]) = - # TODO: Come up with something better. when not defined(release): - var msg = "" - msg.add("\n " & future.fromProc & "'s lead up to read of failed Future:") + const header = "\nAsync traceback:\n" - if not future.errorStackTrace.isNil and future.errorStackTrace != "": - msg.add("\n" & indent(future.errorStackTrace.strip(), 4)) - else: - msg.add("\n Empty or nil stack trace.") - future.error.msg.add(msg) + var exceptionMsg = future.error.msg + if header in exceptionMsg: + # This is messy: extract the original exception message from the msg + # containing the async traceback. + let start = exceptionMsg.find(header) + exceptionMsg = exceptionMsg[0..<start] + + + var newMsg = exceptionMsg & header + + let entries = getStackTraceEntries(future.error) + newMsg.add($entries) + + newMsg.add("Exception message: " & exceptionMsg & "\n") + newMsg.add("Exception type:") + + # # For debugging purposes + # for entry in getStackTraceEntries(future.error): + # newMsg.add "\n" & $entry + future.error.msg = newMsg proc read*[T](future: Future[T] | FutureVar[T]): T = ## Retrieves the value of ``future``. Future must be finished otherwise @@ -333,7 +394,7 @@ proc all*[T](futs: varargs[Future[T]]): auto = let totalFutures = len(futs) for fut in futs: - fut.callback = proc(f: Future[T]) = + fut.addCallback proc (f: Future[T]) = inc(completedFutures) if not retFuture.finished: if f.failed: @@ -355,7 +416,7 @@ proc all*[T](futs: varargs[Future[T]]): auto = for i, fut in futs: proc setCallback(i: int) = - fut.callback = proc(f: Future[T]) = + fut.addCallback proc (f: Future[T]) = inc(completedFutures) if not retFuture.finished: if f.failed: diff --git a/lib/pure/asynchttpserver.nim b/lib/pure/asynchttpserver.nim index 6d4b85145..ba1615651 100644 --- a/lib/pure/asynchttpserver.nim +++ b/lib/pure/asynchttpserver.nim @@ -58,15 +58,18 @@ type socket: AsyncSocket reuseAddr: bool reusePort: bool + maxBody: int ## The maximum content-length that will be read for the body. {.deprecated: [TRequest: Request, PAsyncHttpServer: AsyncHttpServer, THttpCode: HttpCode, THttpVersion: HttpVersion].} -proc newAsyncHttpServer*(reuseAddr = true, reusePort = false): AsyncHttpServer = +proc newAsyncHttpServer*(reuseAddr = true, reusePort = false, + maxBody = 8388608): AsyncHttpServer = ## Creates a new ``AsyncHttpServer`` instance. new result result.reuseAddr = reuseAddr result.reusePort = reusePort + result.maxBody = maxBody proc addHeaders(msg: var string, headers: HttpHeaders) = for k, v in headers: @@ -122,144 +125,157 @@ proc parseProtocol(protocol: string): tuple[orig: string, major, minor: int] = raise newException(ValueError, "Invalid request protocol. Got: " & protocol) result.orig = protocol - i.inc protocol.parseInt(result.major, i) + i.inc protocol.parseSaturatedNatural(result.major, i) i.inc # Skip . - i.inc protocol.parseInt(result.minor, i) + i.inc protocol.parseSaturatedNatural(result.minor, i) proc sendStatus(client: AsyncSocket, status: string): Future[void] = client.send("HTTP/1.1 " & status & "\c\L\c\L") -proc processClient(client: AsyncSocket, address: string, - callback: proc (request: Request): +proc processRequest(server: AsyncHttpServer, req: FutureVar[Request], + client: AsyncSocket, + address: string, lineFut: FutureVar[string], + callback: proc (request: Request): Future[void] {.closure, gcsafe.}) {.async.} = - var request: Request - request.url = initUri() - request.headers = newHttpHeaders() - var lineFut = newFutureVar[string]("asynchttpserver.processClient") - lineFut.mget() = newStringOfCap(80) - var key, value = "" - while not client.isClosed: - # GET /path HTTP/1.1 - # Header: val - # \n - request.headers.clear() - request.body = "" - request.hostname.shallowCopy(address) - assert client != nil - request.client = client - - # We should skip at least one empty line before the request - # https://tools.ietf.org/html/rfc7230#section-3.5 - for i in 0..1: - lineFut.mget().setLen(0) - lineFut.clean() - await client.recvLineInto(lineFut, maxLength=maxLine) # TODO: Timeouts. - - if lineFut.mget == "": - client.close() - return + # Alias `request` to `req.mget()` so we don't have to write `mget` everywhere. + template request(): Request = + req.mget() + + # GET /path HTTP/1.1 + # Header: val + # \n + request.headers.clear() + request.body = "" + request.hostname.shallowCopy(address) + assert client != nil + request.client = client + + # We should skip at least one empty line before the request + # https://tools.ietf.org/html/rfc7230#section-3.5 + for i in 0..1: + lineFut.mget().setLen(0) + lineFut.clean() + await client.recvLineInto(lineFut, maxLength=maxLine) # TODO: Timeouts. + + if lineFut.mget == "": + client.close() + return - if lineFut.mget.len > maxLine: - await request.respondError(Http413) - client.close() + if lineFut.mget.len > maxLine: + await request.respondError(Http413) + client.close() + return + if lineFut.mget != "\c\L": + break + + # First line - GET /path HTTP/1.1 + var i = 0 + for linePart in lineFut.mget.split(' '): + case i + of 0: + try: + # TODO: this is likely slow. + request.reqMethod = parseEnum[HttpMethod]("http" & linePart) + except ValueError: + asyncCheck request.respondError(Http400) return - if lineFut.mget != "\c\L": - break - - # First line - GET /path HTTP/1.1 - var i = 0 - for linePart in lineFut.mget.split(' '): - case i - of 0: - try: - # TODO: this is likely slow. - request.reqMethod = parseEnum[HttpMethod]("http" & linePart) - except ValueError: - asyncCheck request.respondError(Http400) - continue - of 1: - try: - parseUri(linePart, request.url) - except ValueError: - asyncCheck request.respondError(Http400) - continue - of 2: - try: - request.protocol = parseProtocol(linePart) - except ValueError: - asyncCheck request.respondError(Http400) - continue - else: - await request.respondError(Http400) - continue - inc i - - # Headers - while true: - i = 0 - lineFut.mget.setLen(0) - lineFut.clean() - await client.recvLineInto(lineFut, maxLength=maxLine) - - if lineFut.mget == "": - client.close(); return - if lineFut.mget.len > maxLine: - await request.respondError(Http413) - client.close(); return - if lineFut.mget == "\c\L": break - let (key, value) = parseHeader(lineFut.mget) - request.headers[key] = value - # Ensure the client isn't trying to DoS us. - if request.headers.len > headerLimit: - await client.sendStatus("400 Bad Request") - request.client.close() + of 1: + try: + parseUri(linePart, request.url) + except ValueError: + asyncCheck request.respondError(Http400) return + of 2: + try: + request.protocol = parseProtocol(linePart) + except ValueError: + asyncCheck request.respondError(Http400) + return + else: + await request.respondError(Http400) + return + inc i - if request.reqMethod == HttpPost: - # Check for Expect header - if request.headers.hasKey("Expect"): - if "100-continue" in request.headers["Expect"]: - await client.sendStatus("100 Continue") - else: - await client.sendStatus("417 Expectation Failed") - - # Read the body - # - Check for Content-length header - if request.headers.hasKey("Content-Length"): - var contentLength = 0 - if parseInt(request.headers["Content-Length"], - contentLength) == 0: - await request.respond(Http400, "Bad Request. Invalid Content-Length.") - continue - else: - request.body = await client.recv(contentLength) - if request.body.len != contentLength: - await request.respond(Http400, "Bad Request. Content-Length does not match actual.") - continue - elif request.reqMethod == HttpPost: - await request.respond(Http411, "Content-Length required.") - continue - - # Call the user's callback. - await callback(request) - - if "upgrade" in request.headers.getOrDefault("connection"): + # Headers + while true: + i = 0 + lineFut.mget.setLen(0) + lineFut.clean() + await client.recvLineInto(lineFut, maxLength=maxLine) + + if lineFut.mget == "": + client.close(); return + if lineFut.mget.len > maxLine: + await request.respondError(Http413) + client.close(); return + if lineFut.mget == "\c\L": break + let (key, value) = parseHeader(lineFut.mget) + request.headers[key] = value + # Ensure the client isn't trying to DoS us. + if request.headers.len > headerLimit: + await client.sendStatus("400 Bad Request") + request.client.close() return - # Persistent connections - if (request.protocol == HttpVer11 and - request.headers.getOrDefault("connection").normalize != "close") or - (request.protocol == HttpVer10 and - request.headers.getOrDefault("connection").normalize == "keep-alive"): - # In HTTP 1.1 we assume that connection is persistent. Unless connection - # header states otherwise. - # In HTTP 1.0 we assume that the connection should not be persistent. - # Unless the connection header states otherwise. - discard + if request.reqMethod == HttpPost: + # Check for Expect header + if request.headers.hasKey("Expect"): + if "100-continue" in request.headers["Expect"]: + await client.sendStatus("100 Continue") + else: + await client.sendStatus("417 Expectation Failed") + + # Read the body + # - Check for Content-length header + if request.headers.hasKey("Content-Length"): + var contentLength = 0 + if parseSaturatedNatural(request.headers["Content-Length"], contentLength) == 0: + await request.respond(Http400, "Bad Request. Invalid Content-Length.") + return else: - request.client.close() - break + if contentLength > server.maxBody: + await request.respondError(Http413) + return + request.body = await client.recv(contentLength) + if request.body.len != contentLength: + await request.respond(Http400, "Bad Request. Content-Length does not match actual.") + return + elif request.reqMethod == HttpPost: + await request.respond(Http411, "Content-Length required.") + return + + # Call the user's callback. + await callback(request) + + if "upgrade" in request.headers.getOrDefault("connection"): + return + + # Persistent connections + if (request.protocol == HttpVer11 and + cmpIgnoreCase(request.headers.getOrDefault("connection"), "close") != 0) or + (request.protocol == HttpVer10 and + cmpIgnoreCase(request.headers.getOrDefault("connection"), "keep-alive") == 0): + # In HTTP 1.1 we assume that connection is persistent. Unless connection + # header states otherwise. + # In HTTP 1.0 we assume that the connection should not be persistent. + # Unless the connection header states otherwise. + discard + else: + request.client.close() + return + +proc processClient(server: AsyncHttpServer, client: AsyncSocket, address: string, + callback: proc (request: Request): + Future[void] {.closure, gcsafe.}) {.async.} = + var request = newFutureVar[Request]("asynchttpserver.processClient") + request.mget().url = initUri() + request.mget().headers = newHttpHeaders() + var lineFut = newFutureVar[string]("asynchttpserver.processClient") + lineFut.mget() = newStringOfCap(80) + + while not client.isClosed: + await processRequest(server, request, client, address, lineFut, callback) proc serve*(server: AsyncHttpServer, port: Port, callback: proc (request: Request): Future[void] {.closure,gcsafe.}, @@ -280,7 +296,7 @@ proc serve*(server: AsyncHttpServer, port: Port, # TODO: Causes compiler crash. #var (address, client) = await server.socket.acceptAddr() var fut = await server.socket.acceptAddr() - asyncCheck processClient(fut.client, fut.address, callback) + asyncCheck processClient(server, fut.client, fut.address, callback) #echo(f.isNil) #echo(f.repr) diff --git a/lib/pure/asyncmacro.nim b/lib/pure/asyncmacro.nim index 6e7d7993f..8c679929d 100644 --- a/lib/pure/asyncmacro.nim +++ b/lib/pure/asyncmacro.nim @@ -25,10 +25,10 @@ proc skipStmtList(node: NimNode): NimNode {.compileTime.} = result = node[0] template createCb(retFutureSym, iteratorNameSym, - name, futureVarCompletions: untyped) = + strName, identName, futureVarCompletions: untyped) = var nameIterVar = iteratorNameSym #{.push stackTrace: off.} - proc cb0 {.closure.} = + proc identName {.closure.} = try: if not nameIterVar.finished: var next = nameIterVar() @@ -36,11 +36,11 @@ template createCb(retFutureSym, iteratorNameSym, if not retFutureSym.finished: let msg = "Async procedure ($1) yielded `nil`, are you await'ing a " & "`nil` Future?" - raise newException(AssertionError, msg % name) + raise newException(AssertionError, msg % strName) else: {.gcsafe.}: {.push hint[ConvFromXtoItselfNotNeeded]: off.} - next.callback = (proc() {.closure, gcsafe.})(cb0) + next.callback = (proc() {.closure, gcsafe.})(identName) {.pop.} except: futureVarCompletions @@ -52,7 +52,7 @@ template createCb(retFutureSym, iteratorNameSym, else: retFutureSym.fail(getCurrentException()) - cb0() + identName() #{.pop.} proc generateExceptionCheck(futSym, tryStmt, rootReceiver, fromNode: NimNode): NimNode {.compileTime.} = @@ -61,14 +61,14 @@ proc generateExceptionCheck(futSym, else: var exceptionChecks: seq[tuple[cond, body: NimNode]] = @[] let errorNode = newDotExpr(futSym, newIdentNode("error")) - for i in 1 .. <tryStmt.len: + for i in 1 ..< tryStmt.len: let exceptBranch = tryStmt[i] if exceptBranch[0].kind == nnkStmtList: exceptionChecks.add((newIdentNode("true"), exceptBranch[0])) else: var exceptIdentCount = 0 var ifCond: NimNode - for i in 0 .. <exceptBranch.len: + for i in 0 ..< exceptBranch.len: let child = exceptBranch[i] if child.kind == nnkIdent: let cond = infix(errorNode, "of", child) @@ -270,7 +270,7 @@ proc processBody(node, retFutureSym: NimNode, return else: discard - for i in 0 .. <result.len: + for i in 0 ..< result.len: result[i] = processBody(result[i], retFutureSym, subTypeIsVoid, futureVarIdents, nil) @@ -287,7 +287,7 @@ proc getName(node: NimNode): string {.compileTime.} = proc getFutureVarIdents(params: NimNode): seq[NimNode] {.compileTime.} = result = @[] - for i in 1 .. <len(params): + for i in 1 ..< len(params): expectKind(params[i], nnkIdentDefs) if params[i][1].kind == nnkBracketExpr and ($params[i][1][0].ident).normalize == "futurevar": @@ -389,9 +389,12 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = outerProcBody.add(closureIterator) # -> createCb(retFuture) - #var cbName = newIdentNode("cb") + # NOTE: The "_continue" suffix is checked for in asyncfutures.nim to produce + # friendlier stack traces: + var cbName = genSym(nskProc, prcName & "_continue") var procCb = getAst createCb(retFutureSym, iteratorNameSym, newStrLitNode(prcName), + cbName, createFutureVarCompletions(futureVarIdents, nil)) outerProcBody.add procCb @@ -466,33 +469,19 @@ proc stripAwait(node: NimNode): NimNode = node[0][0] = emptyNoopSym else: discard - for i in 0 .. <result.len: + for i in 0 ..< result.len: result[i] = stripAwait(result[i]) proc splitParamType(paramType: NimNode, async: bool): NimNode = result = paramType if paramType.kind == nnkInfix and $paramType[0].ident in ["|", "or"]: - let firstType = paramType[1] - let firstTypeName = $firstType.ident - let secondType = paramType[2] - let secondTypeName = $secondType.ident - - # Make sure that at least one has the name `async`, otherwise we shouldn't - # touch it. - if not ("async" in firstTypeName.normalize or - "async" in secondTypeName.normalize): - return - - if async: - if firstTypeName.normalize.startsWith("async"): - result = paramType[1] - elif secondTypeName.normalize.startsWith("async"): - result = paramType[2] - else: - if not firstTypeName.normalize.startsWith("async"): - result = paramType[1] - elif not secondTypeName.normalize.startsWith("async"): - result = paramType[2] + let firstAsync = "async" in ($paramType[1].ident).normalize + let secondAsync = "async" in ($paramType[2].ident).normalize + + if firstAsync: + result = paramType[if async: 1 else: 2] + elif secondAsync: + result = paramType[if async: 2 else: 1] proc stripReturnType(returnType: NimNode): NimNode = # Strip out the 'Future' from 'Future[T]'. @@ -512,7 +501,7 @@ proc splitProc(prc: NimNode): (NimNode, NimNode) = # Retrieve the `T` inside `Future[T]`. let returnType = stripReturnType(result[0][3][0]) result[0][3][0] = splitParamType(returnType, async=false) - for i in 1 .. <result[0][3].len: + for i in 1 ..< result[0][3].len: # Sync proc (0) -> FormalParams (3) -> IdentDefs, the parameter (i) -> # parameter type (1). result[0][3][i][1] = splitParamType(result[0][3][i][1], async=false) @@ -521,7 +510,7 @@ proc splitProc(prc: NimNode): (NimNode, NimNode) = result[1] = prc.copyNimTree() if result[1][3][0].kind == nnkBracketExpr: result[1][3][0][1] = splitParamType(result[1][3][0][1], async=true) - for i in 1 .. <result[1][3].len: + for i in 1 ..< result[1][3].len: # Async proc (1) -> FormalParams (3) -> IdentDefs, the parameter (i) -> # parameter type (1). result[1][3][i][1] = splitParamType(result[1][3][i][1], async=true) @@ -535,4 +524,4 @@ macro multisync*(prc: untyped): untyped = let (sync, asyncPrc) = splitProc(prc) result = newStmtList() result.add(asyncSingleProc(asyncPrc)) - result.add(sync) \ No newline at end of file + result.add(sync) diff --git a/lib/pure/bitops.nim b/lib/pure/bitops.nim index d1207603d..3f213c5ea 100644 --- a/lib/pure/bitops.nim +++ b/lib/pure/bitops.nim @@ -181,7 +181,7 @@ elif useICC_builtins: proc countSetBits*(x: SomeInteger): int {.inline, nosideeffect.} = - ## Counts the set bits in integer. (also called Hamming weight.) + ## Counts the set bits in integer. (also called `Hamming weight`:idx:.) # TODO: figure out if ICC support _popcnt32/_popcnt64 on platform without POPCNT. # like GCC and MSVC when nimvm: diff --git a/lib/pure/browsers.nim b/lib/pure/browsers.nim index c6a603318..6912b893c 100644 --- a/lib/pure/browsers.nim +++ b/lib/pure/browsers.nim @@ -21,24 +21,18 @@ proc openDefaultBrowser*(url: string) = ## opens `url` with the user's default browser. This does not block. ## ## Under Windows, ``ShellExecute`` is used. Under Mac OS X the ``open`` - ## command is used. Under Unix, it is checked if ``gnome-open`` exists and - ## used if it does. Next attempt is ``kde-open``, then ``xdg-open``. - ## Otherwise the environment variable ``BROWSER`` is used to determine the - ## default browser to use. + ## command is used. Under Unix, it is checked if ``xdg-open`` exists and + ## used if it does. Otherwise the environment variable ``BROWSER`` is + ## used to determine the default browser to use. when defined(windows): - when useWinUnicode: - var o = newWideCString("open") - var u = newWideCString(url) - discard shellExecuteW(0'i32, o, u, nil, nil, SW_SHOWNORMAL) - else: - discard shellExecuteA(0'i32, "open", url, nil, nil, SW_SHOWNORMAL) + var o = newWideCString("open") + var u = newWideCString(url) + discard shellExecuteW(0'i32, o, u, nil, nil, SW_SHOWNORMAL) elif defined(macosx): discard execShellCmd("open " & quoteShell(url)) else: - const attempts = ["gnome-open ", "kde-open ", "xdg-open "] var u = quoteShell(url) - for a in items(attempts): - if execShellCmd(a & u) == 0: return + if execShellCmd("xdg-open " & u) == 0: return for b in getEnv("BROWSER").string.split(PathSep): try: # we use ``startProcess`` here because we don't want to block! diff --git a/lib/pure/cgi.nim b/lib/pure/cgi.nim index 200a4adf1..5de6aa487 100644 --- a/lib/pure/cgi.nim +++ b/lib/pure/cgi.nim @@ -29,21 +29,8 @@ ## writeLine(stdout, "your password: " & myData["password"]) ## writeLine(stdout, "</body></html>") -import strutils, os, strtabs, cookies - -proc encodeUrl*(s: string): string = - ## Encodes a value to be HTTP safe: This means that characters in the set - ## ``{'A'..'Z', 'a'..'z', '0'..'9', '_'}`` are carried over to the result, - ## a space is converted to ``'+'`` and every other character is encoded as - ## ``'%xx'`` where ``xx`` denotes its hexadecimal value. - result = newStringOfCap(s.len + s.len shr 2) # assume 12% non-alnum-chars - for i in 0..s.len-1: - case s[i] - of 'a'..'z', 'A'..'Z', '0'..'9', '_': add(result, s[i]) - of ' ': add(result, '+') - else: - add(result, '%') - add(result, toHex(ord(s[i]), 2)) +import strutils, os, strtabs, cookies, uri +export uri.encodeUrl, uri.decodeUrl proc handleHexChar(c: char, x: var int) {.inline.} = case c @@ -52,30 +39,6 @@ proc handleHexChar(c: char, x: var int) {.inline.} = of 'A'..'F': x = (x shl 4) or (ord(c) - ord('A') + 10) else: assert(false) -proc decodeUrl*(s: string): string = - ## Decodes a value from its HTTP representation: This means that a ``'+'`` - ## is converted to a space, ``'%xx'`` (where ``xx`` denotes a hexadecimal - ## value) is converted to the character with ordinal number ``xx``, and - ## and every other character is carried over. - result = newString(s.len) - var i = 0 - var j = 0 - while i < s.len: - case s[i] - of '%': - var x = 0 - handleHexChar(s[i+1], x) - handleHexChar(s[i+2], x) - inc(i, 2) - result[j] = chr(x) - of '+': result[j] = ' ' - else: result[j] = s[i] - inc(i) - inc(j) - setLen(result, j) - -{.deprecated: [URLDecode: decodeUrl, URLEncode: encodeUrl].} - proc addXmlChar(dest: var string, c: char) {.inline.} = case c of '&': add(dest, "&") @@ -101,8 +64,7 @@ type methodPost, ## query uses the POST method methodGet ## query uses the GET method -{.deprecated: [TRequestMethod: RequestMethod, ECgi: CgiError, - XMLencode: xmlEncode].} +{.deprecated: [TRequestMethod: RequestMethod, ECgi: CgiError].} proc cgiError*(msg: string) {.noreturn.} = ## raises an ECgi exception with message `msg`. @@ -393,8 +355,3 @@ proc existsCookie*(name: string): bool = ## Checks if a cookie of `name` exists. if gcookies == nil: gcookies = parseCookies(getHttpCookie()) result = hasKey(gcookies, name) - -when isMainModule: - const test1 = "abc\L+def xyz" - assert encodeUrl(test1) == "abc%0A%2Bdef+xyz" - assert decodeUrl(encodeUrl(test1)) == test1 diff --git a/lib/pure/collections/critbits.nim b/lib/pure/collections/critbits.nim index f70a12843..34f5c5470 100644 --- a/lib/pure/collections/critbits.nim +++ b/lib/pure/collections/critbits.nim @@ -141,8 +141,8 @@ proc excl*[T](c: var CritBitTree[T], key: string) = proc missingOrExcl*[T](c: var CritBitTree[T], key: string): bool = ## Returns true iff `c` does not contain the given `key`. If the key - ## does exist, c.excl(key) is performed. - let oldCount = c.count + ## does exist, c.excl(key) is performed. + let oldCount = c.count var n = exclImpl(c, key) result = c.count == oldCount @@ -257,7 +257,7 @@ proc allprefixedAux[T](c: CritBitTree[T], key: string; longestMatch: bool): Node p = p.child[dir] if q.byte < key.len: top = p if not longestMatch: - for i in 0 .. <key.len: + for i in 0 ..< key.len: if p.key[i] != key[i]: return result = top @@ -326,7 +326,7 @@ proc `$`*[T](c: CritBitTree[T]): string = result.add($key) when T isnot void: result.add(": ") - result.add($val) + result.addQuoted(val) result.add("}") when isMainModule: diff --git a/lib/pure/collections/deques.nim b/lib/pure/collections/deques.nim index 1bbe9f1ad..328308a9b 100644 --- a/lib/pure/collections/deques.nim +++ b/lib/pure/collections/deques.nim @@ -185,7 +185,7 @@ proc `$`*[T](deq: Deque[T]): string = result = "[" for x in deq: if result.len > 1: result.add(", ") - result.add($x) + result.addQuoted(x) result.add("]") when isMainModule: @@ -207,9 +207,9 @@ when isMainModule: assert($deq == "[4, 56, 6, 789]") assert deq[0] == deq.peekFirst and deq.peekFirst == 4 - assert deq[^1] == deq.peekLast and deq.peekLast == 789 + #assert deq[^1] == deq.peekLast and deq.peekLast == 789 deq[0] = 42 - deq[^1] = 7 + deq[deq.len - 1] = 7 assert 6 in deq and 789 notin deq assert deq.find(6) >= 0 diff --git a/lib/pure/collections/heapqueue.nim b/lib/pure/collections/heapqueue.nim index f86ba1d3f..60869142e 100644 --- a/lib/pure/collections/heapqueue.nim +++ b/lib/pure/collections/heapqueue.nim @@ -1,3 +1,12 @@ + +# +# +# Nim's Runtime Library +# (c) Copyright 2016 Yuriy Glukhov +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. + ##[ Heap queue algorithm (a.k.a. priority queue). Ported from Python heapq. Heaps are arrays for which a[k] <= a[2*k+1] and a[k] <= a[2*k+2] for diff --git a/lib/pure/collections/lists.nim b/lib/pure/collections/lists.nim index f847ddd58..e69acc8d9 100644 --- a/lib/pure/collections/lists.nim +++ b/lib/pure/collections/lists.nim @@ -37,6 +37,14 @@ type DoublyLinkedRing*[T] = object ## a doubly linked ring head*: DoublyLinkedNode[T] + SomeLinkedList*[T] = SinglyLinkedList[T] | DoublyLinkedList[T] + + SomeLinkedRing*[T] = SinglyLinkedRing[T] | DoublyLinkedRing[T] + + SomeLinkedCollection*[T] = SomeLinkedList[T] | SomeLinkedRing[T] + + SomeLinkedNode*[T] = SinglyLinkedNode[T] | DoublyLinkedNode[T] + {.deprecated: [TDoublyLinkedNode: DoublyLinkedNodeObj, PDoublyLinkedNode: DoublyLinkedNode, TSinglyLinkedNode: SinglyLinkedNodeObj, @@ -86,137 +94,57 @@ template itemsRingImpl() {.dirty.} = it = it.next if it == L.head: break -template nodesListImpl() {.dirty.} = - var it = L.head - while it != nil: - var nxt = it.next - yield it - it = nxt - -template nodesRingImpl() {.dirty.} = - var it = L.head - if it != nil: - while true: - var nxt = it.next - yield it - it = nxt - if it == L.head: break - -template findImpl() {.dirty.} = - for x in nodes(L): - if x.value == value: return x - -iterator items*[T](L: DoublyLinkedList[T]): T = +iterator items*[T](L: SomeLinkedList[T]): T = ## yields every value of `L`. itemsListImpl() -iterator items*[T](L: SinglyLinkedList[T]): T = - ## yields every value of `L`. - itemsListImpl() - -iterator items*[T](L: SinglyLinkedRing[T]): T = - ## yields every value of `L`. - itemsRingImpl() - -iterator items*[T](L: DoublyLinkedRing[T]): T = +iterator items*[T](L: SomeLinkedRing[T]): T = ## yields every value of `L`. itemsRingImpl() -iterator mitems*[T](L: var DoublyLinkedList[T]): var T = +iterator mitems*[T](L: var SomeLinkedList[T]): var T = ## yields every value of `L` so that you can modify it. itemsListImpl() -iterator mitems*[T](L: var SinglyLinkedList[T]): var T = - ## yields every value of `L` so that you can modify it. - itemsListImpl() - -iterator mitems*[T](L: var SinglyLinkedRing[T]): var T = +iterator mitems*[T](L: var SomeLinkedRing[T]): var T = ## yields every value of `L` so that you can modify it. itemsRingImpl() -iterator mitems*[T](L: var DoublyLinkedRing[T]): var T = - ## yields every value of `L` so that you can modify it. - itemsRingImpl() - -iterator nodes*[T](L: SinglyLinkedList[T]): SinglyLinkedNode[T] = - ## iterates over every node of `x`. Removing the current node from the - ## list during traversal is supported. - nodesListImpl() - -iterator nodes*[T](L: DoublyLinkedList[T]): DoublyLinkedNode[T] = - ## iterates over every node of `x`. Removing the current node from the - ## list during traversal is supported. - nodesListImpl() - -iterator nodes*[T](L: SinglyLinkedRing[T]): SinglyLinkedNode[T] = +iterator nodes*[T](L: SomeLinkedList[T]): SomeLinkedNode[T] = ## iterates over every node of `x`. Removing the current node from the ## list during traversal is supported. - nodesRingImpl() + var it = L.head + while it != nil: + var nxt = it.next + yield it + it = nxt -iterator nodes*[T](L: DoublyLinkedRing[T]): DoublyLinkedNode[T] = +iterator nodes*[T](L: SomeLinkedRing[T]): SomeLinkedNode[T] = ## iterates over every node of `x`. Removing the current node from the ## list during traversal is supported. - nodesRingImpl() + var it = L.head + if it != nil: + while true: + var nxt = it.next + yield it + it = nxt + if it == L.head: break -template dollarImpl() {.dirty.} = +proc `$`*[T](L: SomeLinkedCollection[T]): string = + ## turns a list into its string representation. result = "[" for x in nodes(L): if result.len > 1: result.add(", ") - result.add($x.value) + result.addQuoted(x.value) result.add("]") -proc `$`*[T](L: SinglyLinkedList[T]): string = - ## turns a list into its string representation. - dollarImpl() - -proc `$`*[T](L: DoublyLinkedList[T]): string = - ## turns a list into its string representation. - dollarImpl() - -proc `$`*[T](L: SinglyLinkedRing[T]): string = - ## turns a list into its string representation. - dollarImpl() - -proc `$`*[T](L: DoublyLinkedRing[T]): string = - ## turns a list into its string representation. - dollarImpl() - -proc find*[T](L: SinglyLinkedList[T], value: T): SinglyLinkedNode[T] = - ## searches in the list for a value. Returns nil if the value does not - ## exist. - findImpl() - -proc find*[T](L: DoublyLinkedList[T], value: T): DoublyLinkedNode[T] = +proc find*[T](L: SomeLinkedCollection[T], value: T): SomeLinkedNode[T] = ## searches in the list for a value. Returns nil if the value does not ## exist. - findImpl() - -proc find*[T](L: SinglyLinkedRing[T], value: T): SinglyLinkedNode[T] = - ## searches in the list for a value. Returns nil if the value does not - ## exist. - findImpl() - -proc find*[T](L: DoublyLinkedRing[T], value: T): DoublyLinkedNode[T] = - ## searches in the list for a value. Returns nil if the value does not - ## exist. - findImpl() - -proc contains*[T](L: SinglyLinkedList[T], value: T): bool {.inline.} = - ## searches in the list for a value. Returns false if the value does not - ## exist, true otherwise. - result = find(L, value) != nil - -proc contains*[T](L: DoublyLinkedList[T], value: T): bool {.inline.} = - ## searches in the list for a value. Returns false if the value does not - ## exist, true otherwise. - result = find(L, value) != nil - -proc contains*[T](L: SinglyLinkedRing[T], value: T): bool {.inline.} = - ## searches in the list for a value. Returns false if the value does not - ## exist, true otherwise. - result = find(L, value) != nil + for x in nodes(L): + if x.value == value: return x -proc contains*[T](L: DoublyLinkedRing[T], value: T): bool {.inline.} = +proc contains*[T](L: SomeLinkedCollection[T], value: T): bool {.inline.} = ## searches in the list for a value. Returns false if the value does not ## exist, true otherwise. result = find(L, value) != nil @@ -266,7 +194,6 @@ proc remove*[T](L: var DoublyLinkedList[T], n: DoublyLinkedNode[T]) = if n.next != nil: n.next.prev = n.prev if n.prev != nil: n.prev.next = n.next - proc append*[T](L: var SinglyLinkedRing[T], n: SinglyLinkedNode[T]) = ## appends a node `n` to `L`. Efficiency: O(1). if L.head != nil: diff --git a/lib/pure/collections/queues.nim b/lib/pure/collections/queues.nim index 401422162..ce792d6da 100644 --- a/lib/pure/collections/queues.nim +++ b/lib/pure/collections/queues.nim @@ -198,9 +198,8 @@ when isMainModule: assert($q == "[4, 56, 6, 789]") assert q[0] == q.front and q.front == 4 - assert q[^1] == q.back and q.back == 789 q[0] = 42 - q[^1] = 7 + q[q.len - 1] = 7 assert 6 in q and 789 notin q assert q.find(6) >= 0 diff --git a/lib/pure/collections/sequtils.nim b/lib/pure/collections/sequtils.nim index e8e725aa3..06e96ca36 100644 --- a/lib/pure/collections/sequtils.nim +++ b/lib/pure/collections/sequtils.nim @@ -13,12 +13,15 @@ ## were inspired by functional programming languages. ## ## For functional style programming you may want to pass `anonymous procs -## <manual.html#anonymous-procs>`_ to procs like ``filter`` to reduce typing. -## Anonymous procs can use `the special do notation <manual.html#do-notation>`_ +## <manual.html#procedures-anonymous-procs>`_ to procs like ``filter`` to +## reduce typing. Anonymous procs can use `the special do notation +## <manual.html#procedures-do-notation>`_ ## which is more convenient in certain situations. include "system/inclrtl" +import macros + when not defined(nimhygiene): {.pragma: dirty.} @@ -43,12 +46,27 @@ proc concat*[T](seqs: varargs[seq[T]]): seq[T] = result[i] = itm inc(i) -proc cycle*[T](s: seq[T], n: Natural): seq[T] = - ## Returns a new sequence with the items of `s` repeated `n` times. +proc count*[T](s: openArray[T], x: T): int = + ## Returns the number of occurrences of the item `x` in the container `s`. + ## + ## Example: + ## + ## .. code-block:: + ## let + ## s = @[1, 2, 2, 3, 2, 4, 2] + ## c = count(s, 2) + ## assert c == 4 + for itm in items(s): + if itm == x: + inc result + +proc cycle*[T](s: openArray[T], n: Natural): seq[T] = + ## Returns a new sequence with the items of the container `s` repeated + ## `n` times. ## ## Example: ## - ## .. code-block: + ## .. code-block:: ## ## let ## s = @[1, 2, 3] @@ -56,7 +74,7 @@ proc cycle*[T](s: seq[T], n: Natural): seq[T] = ## assert total == @[1, 2, 3, 1, 2, 3, 1, 2, 3] result = newSeq[T](n * s.len) var o = 0 - for x in 0..<n: + for x in 0 ..< n: for e in s: result[o] = e inc o @@ -66,18 +84,20 @@ proc repeat*[T](x: T, n: Natural): seq[T] = ## ## Example: ## - ## .. code-block: + ## .. code-block:: ## ## let ## total = repeat(5, 3) ## assert total == @[5, 5, 5] result = newSeq[T](n) - for i in 0..<n: + for i in 0 ..< n: result[i] = x -proc deduplicate*[T](seq1: seq[T]): seq[T] = +proc deduplicate*[T](s: openArray[T]): seq[T] = ## Returns a new sequence without duplicates. ## + ## Example: + ## ## .. code-block:: ## let ## dup1 = @[1, 1, 3, 4, 2, 2, 8, 1, 4] @@ -87,17 +107,17 @@ proc deduplicate*[T](seq1: seq[T]): seq[T] = ## assert unique1 == @[1, 3, 4, 2, 8] ## assert unique2 == @["a", "c", "d"] result = @[] - for itm in items(seq1): + for itm in items(s): if not result.contains(itm): result.add(itm) -{.deprecated: [distnct: deduplicate].} - -proc zip*[S, T](seq1: seq[S], seq2: seq[T]): seq[tuple[a: S, b: T]] = - ## Returns a new sequence with a combination of the two input sequences. +proc zip*[S, T](s1: openArray[S], s2: openArray[T]): seq[tuple[a: S, b: T]] = + ## Returns a new sequence with a combination of the two input containers. ## ## For convenience you can access the returned tuples through the named - ## fields `a` and `b`. If one sequence is shorter, the remaining items in the - ## longer sequence are discarded. Example: + ## fields `a` and `b`. If one container is shorter, the remaining items in + ## the longer container are discarded. + ## + ## Example: ## ## .. code-block:: ## let @@ -110,15 +130,16 @@ proc zip*[S, T](seq1: seq[S], seq2: seq[T]): seq[tuple[a: S, b: T]] = ## assert zip2 == @[(1, "one"), (2, "two"), (3, "three")] ## assert zip1[2].b == 4 ## assert zip2[2].b == "three" - var m = min(seq1.len, seq2.len) + var m = min(s1.len, s2.len) newSeq(result, m) - for i in 0 .. m-1: result[i] = (seq1[i], seq2[i]) + for i in 0 ..< m: + result[i] = (s1[i], s2[i]) proc distribute*[T](s: seq[T], num: Positive, spread = true): seq[seq[T]] = ## Splits and distributes a sequence `s` into `num` sub sequences. ## ## Returns a sequence of `num` sequences. For some input values this is the - ## inverse of the `concat <#concat>`_ proc. The proc will assert in debug + ## inverse of the `concat <#concat>`_ proc. The proc will assert in debug ## builds if `s` is nil or `num` is less than one, and will likely crash on ## release builds. The input sequence `s` can be empty, which will produce ## `num` empty sequences. @@ -159,48 +180,52 @@ proc distribute*[T](s: seq[T], num: Positive, spread = true): seq[seq[T]] = # Use an algorithm which overcounts the stride and minimizes reading limits. if extra > 0: inc(stride) - for i in 0 .. <num: + for i in 0 ..< num: result[i] = newSeq[T]() - for g in first .. <min(s.len, first + stride): + for g in first ..< min(s.len, first + stride): result[i].add(s[g]) first += stride else: # Use an undercounting algorithm which *adds* the remainder each iteration. - for i in 0 .. <num: + for i in 0 ..< num: last = first + stride if extra > 0: extra -= 1 inc(last) result[i] = newSeq[T]() - for g in first .. <last: + for g in first ..< last: result[i].add(s[g]) first = last - -proc map*[T, S](data: openArray[T], op: proc (x: T): S {.closure.}): +proc map*[T, S](s: openArray[T], op: proc (x: T): S {.closure.}): seq[S]{.inline.} = ## Returns a new sequence with the results of `op` applied to every item in - ## `data`. + ## the container `s`. ## ## Since the input is not modified you can use this version of ``map`` to - ## transform the type of the elements in the input sequence. Example: + ## transform the type of the elements in the input container. + ## + ## Example: ## ## .. code-block:: nim ## let ## a = @[1, 2, 3, 4] ## b = map(a, proc(x: int): string = $x) ## assert b == @["1", "2", "3", "4"] - newSeq(result, data.len) - for i in 0..data.len-1: result[i] = op(data[i]) + newSeq(result, s.len) + for i in 0 ..< s.len: + result[i] = op(s[i]) -proc map*[T](data: var openArray[T], op: proc (x: var T) {.closure.}) +proc map*[T](s: var openArray[T], op: proc (x: var T) {.closure.}) {.deprecated.} = - ## Applies `op` to every item in `data` modifying it directly. + ## Applies `op` to every item in `s` modifying it directly. ## ## Note that this version of ``map`` requires your input and output types to - ## be the same, since they are modified in-place. Example: + ## be the same, since they are modified in-place. + ## + ## Example: ## ## .. code-block:: nim ## var a = @["1", "2", "3", "4"] @@ -210,15 +235,16 @@ proc map*[T](data: var openArray[T], op: proc (x: var T) {.closure.}) ## echo repr(a) ## # --> ["142", "242", "342", "442"] ## **Deprecated since version 0.12.0:** Use the ``apply`` proc instead. - for i in 0..data.len-1: op(data[i]) + for i in 0 ..< s.len: op(s[i]) -proc apply*[T](data: var seq[T], op: proc (x: var T) {.closure.}) +proc apply*[T](s: var openArray[T], op: proc (x: var T) {.closure.}) {.inline.} = - ## Applies `op` to every item in `data` modifying it directly. + ## Applies `op` to every item in `s` modifying it directly. ## ## Note that this requires your input and output types to ## be the same, since they are modified in-place. ## The parameter function takes a ``var T`` type parameter. + ## ## Example: ## ## .. code-block:: nim @@ -229,15 +255,16 @@ proc apply*[T](data: var seq[T], op: proc (x: var T) {.closure.}) ## echo repr(a) ## # --> ["142", "242", "342", "442"] ## - for i in 0..data.len-1: op(data[i]) + for i in 0 ..< s.len: op(s[i]) -proc apply*[T](data: var seq[T], op: proc (x: T): T {.closure.}) +proc apply*[T](s: var openArray[T], op: proc (x: T): T {.closure.}) {.inline.} = - ## Applies `op` to every item in `data` modifying it directly. + ## Applies `op` to every item in `s` modifying it directly. ## ## Note that this requires your input and output types to ## be the same, since they are modified in-place. ## The parameter function takes and returns a ``T`` type variable. + ## ## Example: ## ## .. code-block:: nim @@ -248,11 +275,10 @@ proc apply*[T](data: var seq[T], op: proc (x: T): T {.closure.}) ## echo repr(a) ## # --> ["142", "242", "342", "442"] ## - for i in 0..data.len-1: data[i] = op(data[i]) - + for i in 0 ..< s.len: s[i] = op(s[i]) -iterator filter*[T](seq1: seq[T], pred: proc(item: T): bool {.closure.}): T = - ## Iterates through a sequence and yields every item that fulfills the +iterator filter*[T](s: openArray[T], pred: proc(x: T): bool {.closure.}): T = + ## Iterates through a container and yields every item that fulfills the ## predicate. ## ## Example: @@ -262,11 +288,11 @@ iterator filter*[T](seq1: seq[T], pred: proc(item: T): bool {.closure.}): T = ## for n in filter(numbers, proc (x: int): bool = x mod 2 == 0): ## echo($n) ## # echoes 4, 8, 4 in separate lines - for i in 0..<seq1.len: - if pred(seq1[i]): - yield seq1[i] + for i in 0 ..< s.len: + if pred(s[i]): + yield s[i] -proc filter*[T](seq1: seq[T], pred: proc(item: T): bool {.closure.}): seq[T] +proc filter*[T](s: openArray[T], pred: proc(x: T): bool {.closure.}): seq[T] {.inline.} = ## Returns a new sequence with all the items that fulfilled the predicate. ## @@ -280,11 +306,11 @@ proc filter*[T](seq1: seq[T], pred: proc(item: T): bool {.closure.}): seq[T] ## assert f1 == @["red", "black"] ## assert f2 == @["yellow"] result = newSeq[T]() - for i in 0..<seq1.len: - if pred(seq1[i]): - result.add(seq1[i]) + for i in 0 ..< s.len: + if pred(s[i]): + result.add(s[i]) -proc keepIf*[T](seq1: var seq[T], pred: proc(item: T): bool {.closure.}) +proc keepIf*[T](s: var seq[T], pred: proc(x: T): bool {.closure.}) {.inline.} = ## Keeps the items in the passed sequence if they fulfilled the predicate. ## Same as the ``filter`` proc, but modifies the sequence directly. @@ -296,12 +322,12 @@ proc keepIf*[T](seq1: var seq[T], pred: proc(item: T): bool {.closure.}) ## keepIf(floats, proc(x: float): bool = x > 10) ## assert floats == @[13.0, 12.5, 10.1] var pos = 0 - for i in 0 .. <len(seq1): - if pred(seq1[i]): + for i in 0 ..< len(s): + if pred(s[i]): if pos != i: - shallowCopy(seq1[pos], seq1[i]) + shallowCopy(s[pos], s[i]) inc(pos) - setLen(seq1, pos) + setLen(s, pos) proc delete*[T](s: var seq[T]; first, last: Natural) = ## Deletes in `s` the items at position `first` .. `last`. This modifies @@ -354,11 +380,12 @@ proc insert*[T](dest: var seq[T], src: openArray[T], pos=0) = inc(j) -template filterIt*(seq1, pred: untyped): untyped = +template filterIt*(s, pred: untyped): untyped = ## Returns a new sequence with all the items that fulfilled the predicate. ## ## Unlike the `proc` version, the predicate needs to be an expression using ## the ``it`` variable for testing, like: ``filterIt("abcxyz", it == 'x')``. + ## ## Example: ## ## .. code-block:: @@ -368,8 +395,8 @@ template filterIt*(seq1, pred: untyped): untyped = ## notAcceptable = filterIt(temperatures, it > 50 or it < -10) ## assert acceptable == @[-2.0, 24.5, 44.31] ## assert notAcceptable == @[-272.15, 99.9, -113.44] - var result = newSeq[type(seq1[0])]() - for it {.inject.} in items(seq1): + var result = newSeq[type(s[0])]() + for it {.inject.} in items(s): if pred: result.add(it) result @@ -378,6 +405,7 @@ template keepItIf*(varSeq: seq, pred: untyped) = ## ## Unlike the `proc` version, the predicate needs to be an expression using ## the ``it`` variable for testing, like: ``keepItIf("abcxyz", it == 'x')``. + ## ## Example: ## ## .. code-block:: @@ -385,7 +413,7 @@ template keepItIf*(varSeq: seq, pred: untyped) = ## keepItIf(candidates, it.len == 3 and it[0] == 'b') ## assert candidates == @["bar", "baz"] var pos = 0 - for i in 0 .. <len(varSeq): + for i in 0 ..< len(varSeq): let it {.inject.} = varSeq[i] if pred: if pos != i: @@ -393,8 +421,8 @@ template keepItIf*(varSeq: seq, pred: untyped) = inc(pos) setLen(varSeq, pos) -proc all*[T](seq1: seq[T], pred: proc(item: T): bool {.closure.}): bool = - ## Iterates through a sequence and checks if every item fulfills the +proc all*[T](s: openArray[T], pred: proc(x: T): bool {.closure.}): bool = + ## Iterates through a container and checks if every item fulfills the ## predicate. ## ## Example: @@ -403,12 +431,12 @@ proc all*[T](seq1: seq[T], pred: proc(item: T): bool {.closure.}): bool = ## let numbers = @[1, 4, 5, 8, 9, 7, 4] ## assert all(numbers, proc (x: int): bool = return x < 10) == true ## assert all(numbers, proc (x: int): bool = return x < 9) == false - for i in seq1: + for i in s: if not pred(i): return false return true -template allIt*(seq1, pred: untyped): bool = +template allIt*(s, pred: untyped): bool = ## Checks if every item fulfills the predicate. ## ## Example: @@ -418,14 +446,14 @@ template allIt*(seq1, pred: untyped): bool = ## assert allIt(numbers, it < 10) == true ## assert allIt(numbers, it < 9) == false var result = true - for it {.inject.} in items(seq1): + for it {.inject.} in items(s): if not pred: result = false break result -proc any*[T](seq1: seq[T], pred: proc(item: T): bool {.closure.}): bool = - ## Iterates through a sequence and checks if some item fulfills the +proc any*[T](s: openArray[T], pred: proc(x: T): bool {.closure.}): bool = + ## Iterates through a container and checks if some item fulfills the ## predicate. ## ## Example: @@ -434,12 +462,12 @@ proc any*[T](seq1: seq[T], pred: proc(item: T): bool {.closure.}): bool = ## let numbers = @[1, 4, 5, 8, 9, 7, 4] ## assert any(numbers, proc (x: int): bool = return x > 8) == true ## assert any(numbers, proc (x: int): bool = return x > 9) == false - for i in seq1: + for i in s: if pred(i): return true return false -template anyIt*(seq1, pred: untyped): bool = +template anyIt*(s, pred: untyped): bool = ## Checks if some item fulfills the predicate. ## ## Example: @@ -449,7 +477,7 @@ template anyIt*(seq1, pred: untyped): bool = ## assert anyIt(numbers, it > 8) == true ## assert anyIt(numbers, it > 9) == false var result = false - for it {.inject.} in items(seq1): + for it {.inject.} in items(s): if pred: result = true break @@ -493,7 +521,9 @@ template foldl*(sequence, operation: untyped): untyped = ## variables ``a`` and ``b`` for each step of the fold. Since this is a left ## fold, for non associative binary operations like subtraction think that ## the sequence of numbers 1, 2 and 3 will be parenthesized as (((1) - 2) - - ## 3). Example: + ## 3). + ## + ## Example: ## ## .. code-block:: ## let @@ -527,6 +557,7 @@ template foldl*(sequence, operation, first): untyped = ## The ``operation`` parameter should be an expression which uses the variables ## ``a`` and ``b`` for each step of the fold. The ``first`` parameter is the ## start value (the first ``a``) and therefor defines the type of the result. + ## ## Example: ## ## .. code-block:: @@ -555,7 +586,9 @@ template foldr*(sequence, operation: untyped): untyped = ## variables ``a`` and ``b`` for each step of the fold. Since this is a right ## fold, for non associative binary operations like subtraction think that ## the sequence of numbers 1, 2 and 3 will be parenthesized as (1 - (2 - - ## (3))). Example: + ## (3))). + ## + ## Example: ## ## .. code-block:: ## let @@ -580,13 +613,15 @@ template foldr*(sequence, operation: untyped): untyped = result = operation result -template mapIt*(seq1, typ, op: untyped): untyped = +template mapIt*(s, typ, op: untyped): untyped = ## Convenience template around the ``map`` proc to reduce typing. ## ## The template injects the ``it`` variable which you can use directly in an ## expression. You also need to pass as `typ` the type of the expression, ## since the new returned sequence can have a different type than the - ## original. Example: + ## original. + ## + ## Example: ## ## .. code-block:: ## let @@ -596,16 +631,18 @@ template mapIt*(seq1, typ, op: untyped): untyped = ## **Deprecated since version 0.12.0:** Use the ``mapIt(seq1, op)`` ## template instead. var result: seq[typ] = @[] - for it {.inject.} in items(seq1): + for it {.inject.} in items(s): result.add(op) result -template mapIt*(seq1, op: untyped): untyped = +template mapIt*(s, op: untyped): untyped = ## Convenience template around the ``map`` proc to reduce typing. ## ## The template injects the ``it`` variable which you can use directly in an - ## expression. Example: + ## expression. + ## + ## Example: ## ## .. code-block:: ## let @@ -614,19 +651,19 @@ template mapIt*(seq1, op: untyped): untyped = ## assert strings == @["4", "8", "12", "16"] type outType = type(( block: - var it{.inject.}: type(items(seq1)); + var it{.inject.}: type(items(s)); op)) var result: seq[outType] - when compiles(seq1.len): - let s = seq1 + when compiles(s.len): + let t = s var i = 0 result = newSeq[outType](s.len) - for it {.inject.} in s: + for it {.inject.} in t: result[i] = op i += 1 else: result = @[] - for it {.inject.} in seq1: + for it {.inject.} in s: result.add(op) result @@ -635,20 +672,23 @@ template applyIt*(varSeq, op: untyped) = ## ## The template injects the ``it`` variable which you can use directly in an ## expression. The expression has to return the same type as the sequence you - ## are mutating. Example: + ## are mutating. + ## + ## Example: ## ## .. code-block:: ## var nums = @[1, 2, 3, 4] ## nums.applyIt(it * 3) ## assert nums[0] + nums[3] == 15 - for i in 0 .. <varSeq.len: + for i in 0 ..< varSeq.len: let it {.inject.} = varSeq[i] varSeq[i] = op - template newSeqWith*(len: int, init: untyped): untyped = - ## creates a new sequence, calling `init` to initialize each value. Example: + ## creates a new sequence, calling `init` to initialize each value. + ## + ## Example: ## ## .. code-block:: ## var seq2D = newSeqWith(20, newSeq[bool](10)) @@ -660,10 +700,56 @@ template newSeqWith*(len: int, init: untyped): untyped = ## var seqRand = newSeqWith(20, random(10)) ## echo seqRand var result = newSeq[type(init)](len) - for i in 0 .. <len: + for i in 0 ..< len: result[i] = init result +proc mapLitsImpl(constructor: NimNode; op: NimNode; nested: bool; + filter = nnkLiterals): NimNode = + if constructor.kind in filter: + result = newNimNode(nnkCall, lineInfoFrom=constructor) + result.add op + result.add constructor + else: + result = newNimNode(constructor.kind, lineInfoFrom=constructor) + for v in constructor: + if nested or v.kind in filter: + result.add mapLitsImpl(v, op, nested, filter) + else: + result.add v + +macro mapLiterals*(constructor, op: untyped; + nested = true): untyped = + ## applies ``op`` to each of the **atomic** literals like ``3`` + ## or ``"abc"`` in the specified ``constructor`` AST. This can + ## be used to map every array element to some target type: + ## + ## Example: + ## + ## .. code-block:: + ## let x = mapLiterals([0.1, 1.2, 2.3, 3.4], int) + ## doAssert x is array[4, int] + ## + ## Short notation for: + ## + ## .. code-block:: + ## let x = [int(0.1), int(1.2), int(2.3), int(3.4)] + ## + ## If ``nested`` is true, the literals are replaced everywhere + ## in the ``constructor`` AST, otherwise only the first level + ## is considered: + ## + ## .. code-block:: + ## mapLiterals((1, ("abc"), 2), float, nested=false) + ## + ## Produces:: + ## + ## (float(1), ("abc"), float(2)) + ## + ## There are no constraints for the ``constructor`` AST, it + ## works for nested tuples of arrays of sets etc. + result = mapLitsImpl(constructor, op, nested.boolVal) + when isMainModule: import strutils block: # concat test @@ -674,45 +760,178 @@ when isMainModule: total = concat(s1, s2, s3) assert total == @[1, 2, 3, 4, 5, 6, 7] - block: # duplicates test + block: # count test + let + s1 = @[1, 2, 3, 2] + s2 = @['a', 'b', 'x', 'a'] + a1 = [1, 2, 3, 2] + a2 = ['a', 'b', 'x', 'a'] + r0 = count(s1, 0) + r1 = count(s1, 1) + r2 = count(s1, 2) + r3 = count(s2, 'y') + r4 = count(s2, 'x') + r5 = count(s2, 'a') + ar0 = count(a1, 0) + ar1 = count(a1, 1) + ar2 = count(a1, 2) + ar3 = count(a2, 'y') + ar4 = count(a2, 'x') + ar5 = count(a2, 'a') + assert r0 == 0 + assert r1 == 1 + assert r2 == 2 + assert r3 == 0 + assert r4 == 1 + assert r5 == 2 + assert ar0 == 0 + assert ar1 == 1 + assert ar2 == 2 + assert ar3 == 0 + assert ar4 == 1 + assert ar5 == 2 + + block: # cycle tests + let + a = @[1, 2, 3] + b: seq[int] = @[] + c = [1, 2, 3] + + doAssert a.cycle(3) == @[1, 2, 3, 1, 2, 3, 1, 2, 3] + doAssert a.cycle(0) == @[] + #doAssert a.cycle(-1) == @[] # will not compile! + doAssert b.cycle(3) == @[] + doAssert c.cycle(3) == @[1, 2, 3, 1, 2, 3, 1, 2, 3] + doAssert c.cycle(0) == @[] + + block: # repeat tests + assert repeat(10, 5) == @[10, 10, 10, 10, 10] + assert repeat(@[1,2,3], 2) == @[@[1,2,3], @[1,2,3]] + assert repeat([1,2,3], 2) == @[[1,2,3], [1,2,3]] + + block: # deduplicates test let dup1 = @[1, 1, 3, 4, 2, 2, 8, 1, 4] dup2 = @["a", "a", "c", "d", "d"] + dup3 = [1, 1, 3, 4, 2, 2, 8, 1, 4] + dup4 = ["a", "a", "c", "d", "d"] unique1 = deduplicate(dup1) unique2 = deduplicate(dup2) + unique3 = deduplicate(dup3) + unique4 = deduplicate(dup4) assert unique1 == @[1, 3, 4, 2, 8] assert unique2 == @["a", "c", "d"] + assert unique3 == @[1, 3, 4, 2, 8] + assert unique4 == @["a", "c", "d"] block: # zip test let short = @[1, 2, 3] long = @[6, 5, 4, 3, 2, 1] words = @["one", "two", "three"] + ashort = [1, 2, 3] + along = [6, 5, 4, 3, 2, 1] + awords = ["one", "two", "three"] zip1 = zip(short, long) zip2 = zip(short, words) + zip3 = zip(ashort, along) + zip4 = zip(ashort, awords) + zip5 = zip(ashort, words) assert zip1 == @[(1, 6), (2, 5), (3, 4)] assert zip2 == @[(1, "one"), (2, "two"), (3, "three")] + assert zip3 == @[(1, 6), (2, 5), (3, 4)] + assert zip4 == @[(1, "one"), (2, "two"), (3, "three")] + assert zip5 == @[(1, "one"), (2, "two"), (3, "three")] assert zip1[2].b == 4 assert zip2[2].b == "three" + assert zip3[2].b == 4 + assert zip4[2].b == "three" + assert zip5[2].b == "three" + + block: # distribute tests + let numbers = @[1, 2, 3, 4, 5, 6, 7] + doAssert numbers.distribute(3) == @[@[1, 2, 3], @[4, 5], @[6, 7]] + doAssert numbers.distribute(6)[0] == @[1, 2] + doAssert numbers.distribute(6)[5] == @[7] + let a = @[1, 2, 3, 4, 5, 6, 7] + doAssert a.distribute(1, true) == @[@[1, 2, 3, 4, 5, 6, 7]] + doAssert a.distribute(1, false) == @[@[1, 2, 3, 4, 5, 6, 7]] + doAssert a.distribute(2, true) == @[@[1, 2, 3, 4], @[5, 6, 7]] + doAssert a.distribute(2, false) == @[@[1, 2, 3, 4], @[5, 6, 7]] + doAssert a.distribute(3, true) == @[@[1, 2, 3], @[4, 5], @[6, 7]] + doAssert a.distribute(3, false) == @[@[1, 2, 3], @[4, 5, 6], @[7]] + doAssert a.distribute(4, true) == @[@[1, 2], @[3, 4], @[5, 6], @[7]] + doAssert a.distribute(4, false) == @[@[1, 2], @[3, 4], @[5, 6], @[7]] + doAssert a.distribute(5, true) == @[@[1, 2], @[3, 4], @[5], @[6], @[7]] + doAssert a.distribute(5, false) == @[@[1, 2], @[3, 4], @[5, 6], @[7], @[]] + doAssert a.distribute(6, true) == @[@[1, 2], @[3], @[4], @[5], @[6], @[7]] + doAssert a.distribute(6, false) == @[ + @[1, 2], @[3, 4], @[5, 6], @[7], @[], @[]] + doAssert a.distribute(8, false) == a.distribute(8, true) + doAssert a.distribute(90, false) == a.distribute(90, true) + var b = @[0] + for f in 1 .. 25: b.add(f) + doAssert b.distribute(5, true)[4].len == 5 + doAssert b.distribute(5, false)[4].len == 2 + + block: # map test + let + numbers = @[1, 4, 5, 8, 9, 7, 4] + anumbers = [1, 4, 5, 8, 9, 7, 4] + m1 = map(numbers, proc(x: int): int = 2*x) + m2 = map(anumbers, proc(x: int): int = 2*x) + assert m1 == @[2, 8, 10, 16, 18, 14, 8] + assert m2 == @[2, 8, 10, 16, 18, 14, 8] + + block: # apply test + var a = @["1", "2", "3", "4"] + apply(a, proc(x: var string) = x &= "42") + assert a == @["142", "242", "342", "442"] block: # filter proc test let colors = @["red", "yellow", "black"] + acolors = ["red", "yellow", "black"] f1 = filter(colors, proc(x: string): bool = x.len < 6) f2 = filter(colors) do (x: string) -> bool : x.len > 5 + f3 = filter(acolors, proc(x: string): bool = x.len < 6) + f4 = filter(acolors) do (x: string) -> bool : x.len > 5 assert f1 == @["red", "black"] assert f2 == @["yellow"] + assert f3 == @["red", "black"] + assert f4 == @["yellow"] block: # filter iterator test let numbers = @[1, 4, 5, 8, 9, 7, 4] + let anumbers = [1, 4, 5, 8, 9, 7, 4] assert toSeq(filter(numbers, proc (x: int): bool = x mod 2 == 0)) == @[4, 8, 4] + assert toSeq(filter(anumbers, proc (x: int): bool = x mod 2 == 0)) == + @[4, 8, 4] block: # keepIf test var floats = @[13.0, 12.5, 5.8, 2.0, 6.1, 9.9, 10.1] keepIf(floats, proc(x: float): bool = x > 10) assert floats == @[13.0, 12.5, 10.1] + block: # delete tests + let outcome = @[1,1,1,1,1,1,1,1] + var dest = @[1,1,1,2,2,2,2,2,2,1,1,1,1,1] + dest.delete(3, 8) + assert outcome == dest, """\ + Deleting range 3-9 from [1,1,1,2,2,2,2,2,2,1,1,1,1,1] + is [1,1,1,1,1,1,1,1]""" + + block: # insert tests + var dest = @[1,1,1,1,1,1,1,1] + let + src = @[2,2,2,2,2,2] + outcome = @[1,1,1,2,2,2,2,2,2,1,1,1,1,1] + dest.insert(src, 3) + assert dest == outcome, """\ + Inserting [2,2,2,2,2,2] into [1,1,1,1,1,1,1,1] + at 3 is [1,1,1,2,2,2,2,2,2,1,1,1,1,1]""" + block: # filterIt test let temperatures = @[-272.15, -2.0, 24.5, 44.31, 99.9, -113.44] @@ -726,37 +945,49 @@ when isMainModule: keepItIf(candidates, it.len == 3 and it[0] == 'b') assert candidates == @["bar", "baz"] - block: # any - let - numbers = @[1, 4, 5, 8, 9, 7, 4] - len0seq : seq[int] = @[] - assert any(numbers, proc (x: int): bool = return x > 8) == true - assert any(numbers, proc (x: int): bool = return x > 9) == false - assert any(len0seq, proc (x: int): bool = return true) == false - - block: # anyIt - let - numbers = @[1, 4, 5, 8, 9, 7, 4] - len0seq : seq[int] = @[] - assert anyIt(numbers, it > 8) == true - assert anyIt(numbers, it > 9) == false - assert anyIt(len0seq, true) == false - block: # all let numbers = @[1, 4, 5, 8, 9, 7, 4] + anumbers = [1, 4, 5, 8, 9, 7, 4] len0seq : seq[int] = @[] assert all(numbers, proc (x: int): bool = return x < 10) == true assert all(numbers, proc (x: int): bool = return x < 9) == false assert all(len0seq, proc (x: int): bool = return false) == true + assert all(anumbers, proc (x: int): bool = return x < 10) == true + assert all(anumbers, proc (x: int): bool = return x < 9) == false block: # allIt let numbers = @[1, 4, 5, 8, 9, 7, 4] + anumbers = [1, 4, 5, 8, 9, 7, 4] len0seq : seq[int] = @[] assert allIt(numbers, it < 10) == true assert allIt(numbers, it < 9) == false assert allIt(len0seq, false) == true + assert allIt(anumbers, it < 10) == true + assert allIt(anumbers, it < 9) == false + + block: # any + let + numbers = @[1, 4, 5, 8, 9, 7, 4] + anumbers = [1, 4, 5, 8, 9, 7, 4] + len0seq : seq[int] = @[] + assert any(numbers, proc (x: int): bool = return x > 8) == true + assert any(numbers, proc (x: int): bool = return x > 9) == false + assert any(len0seq, proc (x: int): bool = return true) == false + assert any(anumbers, proc (x: int): bool = return x > 8) == true + assert any(anumbers, proc (x: int): bool = return x > 9) == false + + block: # anyIt + let + numbers = @[1, 4, 5, 8, 9, 7, 4] + anumbers = [1, 4, 5, 8, 9, 7, 4] + len0seq : seq[int] = @[] + assert anyIt(numbers, it > 8) == true + assert anyIt(numbers, it > 9) == false + assert anyIt(len0seq, true) == false + assert anyIt(anumbers, it > 8) == true + assert anyIt(anumbers, it > 9) == false block: # toSeq test let @@ -792,56 +1023,13 @@ when isMainModule: assert multiplication == 495, "Multiplication is (5*(9*(11)))" assert concatenation == "nimiscool" - block: # delete tests - let outcome = @[1,1,1,1,1,1,1,1] - var dest = @[1,1,1,2,2,2,2,2,2,1,1,1,1,1] - dest.delete(3, 8) - assert outcome == dest, """\ - Deleting range 3-9 from [1,1,1,2,2,2,2,2,2,1,1,1,1,1] - is [1,1,1,1,1,1,1,1]""" - - block: # insert tests - var dest = @[1,1,1,1,1,1,1,1] - let - src = @[2,2,2,2,2,2] - outcome = @[1,1,1,2,2,2,2,2,2,1,1,1,1,1] - dest.insert(src, 3) - assert dest == outcome, """\ - Inserting [2,2,2,2,2,2] into [1,1,1,1,1,1,1,1] - at 3 is [1,1,1,2,2,2,2,2,2,1,1,1,1,1]""" - block: # mapIt tests var nums = @[1, 2, 3, 4] strings = nums.mapIt($(4 * it)) nums.applyIt(it * 3) assert nums[0] + nums[3] == 15 - - block: # distribute tests - let numbers = @[1, 2, 3, 4, 5, 6, 7] - doAssert numbers.distribute(3) == @[@[1, 2, 3], @[4, 5], @[6, 7]] - doAssert numbers.distribute(6)[0] == @[1, 2] - doAssert numbers.distribute(6)[5] == @[7] - let a = @[1, 2, 3, 4, 5, 6, 7] - doAssert a.distribute(1, true) == @[@[1, 2, 3, 4, 5, 6, 7]] - doAssert a.distribute(1, false) == @[@[1, 2, 3, 4, 5, 6, 7]] - doAssert a.distribute(2, true) == @[@[1, 2, 3, 4], @[5, 6, 7]] - doAssert a.distribute(2, false) == @[@[1, 2, 3, 4], @[5, 6, 7]] - doAssert a.distribute(3, true) == @[@[1, 2, 3], @[4, 5], @[6, 7]] - doAssert a.distribute(3, false) == @[@[1, 2, 3], @[4, 5, 6], @[7]] - doAssert a.distribute(4, true) == @[@[1, 2], @[3, 4], @[5, 6], @[7]] - doAssert a.distribute(4, false) == @[@[1, 2], @[3, 4], @[5, 6], @[7]] - doAssert a.distribute(5, true) == @[@[1, 2], @[3, 4], @[5], @[6], @[7]] - doAssert a.distribute(5, false) == @[@[1, 2], @[3, 4], @[5, 6], @[7], @[]] - doAssert a.distribute(6, true) == @[@[1, 2], @[3], @[4], @[5], @[6], @[7]] - doAssert a.distribute(6, false) == @[ - @[1, 2], @[3, 4], @[5, 6], @[7], @[], @[]] - doAssert a.distribute(8, false) == a.distribute(8, true) - doAssert a.distribute(90, false) == a.distribute(90, true) - var b = @[0] - for f in 1 .. 25: b.add(f) - doAssert b.distribute(5, true)[4].len == 5 - doAssert b.distribute(5, false)[4].len == 2 + assert strings[2] == "12" block: # newSeqWith tests var seq2D = newSeqWith(4, newSeq[bool](2)) @@ -850,19 +1038,11 @@ when isMainModule: seq2D[0][1] = true doAssert seq2D == @[@[true, true], @[true, false], @[false, false], @[false, false]] - block: # cycle tests - let - a = @[1, 2, 3] - b: seq[int] = @[] - - doAssert a.cycle(3) == @[1, 2, 3, 1, 2, 3, 1, 2, 3] - doAssert a.cycle(0) == @[] - #doAssert a.cycle(-1) == @[] # will not compile! - doAssert b.cycle(3) == @[] - - block: # repeat tests - assert repeat(10, 5) == @[10, 10, 10, 10, 10] - assert repeat(@[1,2,3], 2) == @[@[1,2,3], @[1,2,3]] + block: # mapLiterals tests + let x = mapLiterals([0.1, 1.2, 2.3, 3.4], int) + doAssert x is array[4, int] + doAssert mapLiterals((1, ("abc"), 2), float, nested=false) == (float(1), "abc", float(2)) + doAssert mapLiterals(([1], ("abc"), 2), `$`, nested=true) == (["1"], "abc", "2") when not defined(testing): echo "Finished doc tests" diff --git a/lib/pure/collections/sets.nim b/lib/pure/collections/sets.nim index dbdf17514..9e9152fc8 100644 --- a/lib/pure/collections/sets.nim +++ b/lib/pure/collections/sets.nim @@ -46,7 +46,7 @@ template default[T](t: typedesc[T]): T = var v: T v -proc clear*[A](s: var HashSet[A]) = +proc clear*[A](s: var HashSet[A]) = ## Clears the HashSet back to an empty state, without shrinking ## any of the existing storage. O(n) where n is the size of the hash bucket. s.counter = 0 @@ -406,7 +406,7 @@ template dollarImpl() {.dirty.} = result = "{" for key in items(s): if result.len > 1: result.add(", ") - result.add($key) + result.addQuoted(key) result.add("}") proc `$`*[A](s: HashSet[A]): string = @@ -610,7 +610,7 @@ type {.deprecated: [TOrderedSet: OrderedSet].} -proc clear*[A](s: var OrderedSet[A]) = +proc clear*[A](s: var OrderedSet[A]) = ## Clears the OrderedSet back to an empty state, without shrinking ## any of the existing storage. O(n) where n is the size of the hash bucket. s.counter = 0 @@ -911,13 +911,13 @@ proc `==`*[A](s, t: OrderedSet[A]): bool = ## Equality for ordered sets. if s.counter != t.counter: return false var h = s.first - var g = s.first + var g = t.first var compared = 0 while h >= 0 and g >= 0: var nxh = s.data[h].next var nxg = t.data[g].next - if isFilled(s.data[h].hcode) and isFilled(s.data[g].hcode): - if s.data[h].key == s.data[g].key: + if isFilled(s.data[h].hcode) and isFilled(t.data[g].hcode): + if s.data[h].key == t.data[g].key: inc compared else: return false @@ -1120,6 +1120,22 @@ when isMainModule and not defined(release): assert s.missingOrExcl(4) == true assert s.missingOrExcl(6) == false + block orderedSetEquality: + type pair = tuple[a, b: int] + + var aa = initOrderedSet[pair]() + var bb = initOrderedSet[pair]() + + var x = (a:1,b:2) + var y = (a:3,b:4) + + aa.incl(x) + aa.incl(y) + + bb.incl(x) + bb.incl(y) + assert aa == bb + when not defined(testing): echo "Micro tests run successfully." diff --git a/lib/pure/collections/sharedlist.nim b/lib/pure/collections/sharedlist.nim index e93ceb02f..b3e677b79 100644 --- a/lib/pure/collections/sharedlist.nim +++ b/lib/pure/collections/sharedlist.nim @@ -73,10 +73,10 @@ proc add*[A](x: var SharedList[A]; y: A) = node.d[node.dataLen] = y inc(node.dataLen) -proc initSharedList*[A](): SharedList[A] = - initLock result.lock - result.head = nil - result.tail = nil +proc init*[A](t: var SharedList[A]) = + initLock t.lock + t.head = nil + t.tail = nil proc clear*[A](t: var SharedList[A]) = withLock(t): @@ -92,4 +92,11 @@ proc deinitSharedList*[A](t: var SharedList[A]) = clear(t) deinitLock t.lock +proc initSharedList*[A](): SharedList[A] {.deprecated.} = + ## Deprecated. Use `init` instead. + ## This is not posix compliant, may introduce undefined behavior. + initLock result.lock + result.head = nil + result.tail = nil + {.pop.} diff --git a/lib/pure/collections/sharedstrings.nim b/lib/pure/collections/sharedstrings.nim index a9e194fb4..7e9de4b73 100644 --- a/lib/pure/collections/sharedstrings.nim +++ b/lib/pure/collections/sharedstrings.nim @@ -55,7 +55,7 @@ proc `[]=`*(s: var SharedString; i: Natural; value: char) = if i < s.len: s.buffer.data[i+s.first] = value else: raise newException(IndexError, "index out of bounds") -proc `[]`*(s: SharedString; ab: Slice[int]): SharedString = +proc `[]`*(s: SharedString; ab: HSlice[int, int]): SharedString = #incRef(src.buffer) if ab.a < s.len: result.buffer = s.buffer @@ -87,10 +87,10 @@ proc newSharedString*(s: string): SharedString = result.len = len when declared(atomicLoadN): - template load(x): expr = atomicLoadN(addr x, ATOMIC_SEQ_CST) + template load(x): untyped = atomicLoadN(addr x, ATOMIC_SEQ_CST) else: # XXX Fixme - template load(x): expr = x + template load(x): untyped = x proc add*(s: var SharedString; t: cstring; len: Natural) = if len == 0: return diff --git a/lib/pure/collections/sharedtables.nim b/lib/pure/collections/sharedtables.nim index fc50ea41c..4f311af87 100644 --- a/lib/pure/collections/sharedtables.nim +++ b/lib/pure/collections/sharedtables.nim @@ -183,6 +183,7 @@ proc `[]=`*[A, B](t: var SharedTable[A, B], key: A, val: B) = proc add*[A, B](t: var SharedTable[A, B], key: A, val: B) = ## puts a new (key, value)-pair into `t` even if ``t[key]`` already exists. + ## This can introduce duplicate keys into the table! withLock t: addImpl(enlarge) @@ -191,19 +192,29 @@ proc del*[A, B](t: var SharedTable[A, B], key: A) = withLock t: delImpl() -proc initSharedTable*[A, B](initialSize=64): SharedTable[A, B] = +proc init*[A, B](t: var SharedTable[A, B], initialSize=64) = ## creates a new hash table that is empty. ## ## `initialSize` needs to be a power of two. If you need to accept runtime ## values for this you could use the ``nextPowerOfTwo`` proc from the ## `math <math.html>`_ module or the ``rightSize`` proc from this module. assert isPowerOfTwo(initialSize) - result.counter = 0 - result.dataLen = initialSize - result.data = cast[KeyValuePairSeq[A, B]](allocShared0( + t.counter = 0 + t.dataLen = initialSize + t.data = cast[KeyValuePairSeq[A, B]](allocShared0( sizeof(KeyValuePair[A, B]) * initialSize)) - initLock result.lock + initLock t.lock proc deinitSharedTable*[A, B](t: var SharedTable[A, B]) = deallocShared(t.data) deinitLock t.lock + +proc initSharedTable*[A, B](initialSize=64): SharedTable[A, B] {.deprecated.} = + ## Deprecated. Use `init` instead. + ## This is not posix compliant, may introduce undefined behavior. + assert isPowerOfTwo(initialSize) + result.counter = 0 + result.dataLen = initialSize + result.data = cast[KeyValuePairSeq[A, B]](allocShared0( + sizeof(KeyValuePair[A, B]) * initialSize)) + initLock result.lock diff --git a/lib/pure/collections/tableimpl.nim b/lib/pure/collections/tableimpl.nim index eec98fcaf..9a5bffcef 100644 --- a/lib/pure/collections/tableimpl.nim +++ b/lib/pure/collections/tableimpl.nim @@ -149,7 +149,7 @@ template delImpl() {.dirty.} = delImplIdx(t, i) template clearImpl() {.dirty.} = - for i in 0 .. <t.data.len: + for i in 0 ..< t.data.len: when compiles(t.data[i].hcode): # CountTable records don't contain a hcode t.data[i].hcode = 0 t.data[i].key = default(type(t.data[i].key)) diff --git a/lib/pure/collections/tables.nim b/lib/pure/collections/tables.nim index 01a42efab..777beabc3 100644 --- a/lib/pure/collections/tables.nim +++ b/lib/pure/collections/tables.nim @@ -308,6 +308,7 @@ proc `[]=`*[A, B](t: var Table[A, B], key: A, val: B) = proc add*[A, B](t: var Table[A, B], key: A, val: B) = ## puts a new (key, value)-pair into `t` even if ``t[key]`` already exists. + ## This can introduce duplicate keys into the table! addImpl(enlarge) proc len*[A, B](t: TableRef[A, B]): int = @@ -337,9 +338,9 @@ template dollarImpl(): untyped {.dirty.} = result = "{" for key, val in pairs(t): if result.len > 1: result.add(", ") - result.add($key) + result.addQuoted(key) result.add(": ") - result.add($val) + result.addQuoted(val) result.add("}") proc `$`*[A, B](t: Table[A, B]): string = @@ -430,6 +431,7 @@ proc `[]=`*[A, B](t: TableRef[A, B], key: A, val: B) = proc add*[A, B](t: TableRef[A, B], key: A, val: B) = ## puts a new (key, value)-pair into `t` even if ``t[key]`` already exists. + ## This can introduce duplicate keys into the table! t[].add(key, val) proc del*[A, B](t: TableRef[A, B], key: A) = @@ -604,6 +606,7 @@ proc `[]=`*[A, B](t: var OrderedTable[A, B], key: A, val: B) = proc add*[A, B](t: var OrderedTable[A, B], key: A, val: B) = ## puts a new (key, value)-pair into `t` even if ``t[key]`` already exists. + ## This can introduce duplicate keys into the table! addImpl(enlarge) proc mgetOrPut*[A, B](t: var OrderedTable[A, B], key: A, val: B): var B = @@ -770,6 +773,7 @@ proc `[]=`*[A, B](t: OrderedTableRef[A, B], key: A, val: B) = proc add*[A, B](t: OrderedTableRef[A, B], key: A, val: B) = ## puts a new (key, value)-pair into `t` even if ``t[key]`` already exists. + ## This can introduce duplicate keys into the table! t[].add(key, val) proc newOrderedTable*[A, B](initialSize=64): OrderedTableRef[A, B] = @@ -962,9 +966,10 @@ proc initCountTable*[A](initialSize=64): CountTable[A] = newSeq(result.data, initialSize) proc toCountTable*[A](keys: openArray[A]): CountTable[A] = - ## creates a new count table with every key in `keys` having a count of 1. + ## creates a new count table with every key in `keys` having a count + ## of how many times it occurs in `keys`. result = initCountTable[A](rightSize(keys.len)) - for key in items(keys): result[key] = 1 + for key in items(keys): result.inc key proc `$`*[A](t: CountTable[A]): string = ## The `$` operator for count tables. @@ -989,9 +994,10 @@ proc inc*[A](t: var CountTable[A], key: A, val = 1) = proc smallest*[A](t: CountTable[A]): tuple[key: A, val: int] = ## returns the (key,val)-pair with the smallest `val`. Efficiency: O(n) assert t.len > 0 - var minIdx = 0 - for h in 1..high(t.data): - if t.data[h].val > 0 and t.data[minIdx].val > t.data[h].val: minIdx = h + var minIdx = -1 + for h in 0..high(t.data): + if t.data[h].val > 0 and (minIdx == -1 or t.data[minIdx].val > t.data[h].val): + minIdx = h result.key = t.data[minIdx].key result.val = t.data[minIdx].val @@ -1325,3 +1331,7 @@ when isMainModule: assert((a == b) == true) assert((b == a) == true) + block: # CountTable.smallest + var t = initCountTable[int]() + for v in items([0, 0, 5, 5, 5]): t.inc(v) + doAssert t.smallest == (0, 2) diff --git a/lib/pure/concurrency/cpuinfo.nim b/lib/pure/concurrency/cpuinfo.nim index 603fee080..f01488811 100644 --- a/lib/pure/concurrency/cpuinfo.nim +++ b/lib/pure/concurrency/cpuinfo.nim @@ -45,8 +45,25 @@ proc countProcessors*(): int {.rtl, extern: "ncpi$1".} = ## returns the numer of the processors/cores the machine has. ## Returns 0 if it cannot be detected. when defined(windows): - var x = getEnv("NUMBER_OF_PROCESSORS") - if x.len > 0: result = parseInt(x.string) + type + SYSTEM_INFO {.final, pure.} = object + u1: int32 + dwPageSize: int32 + lpMinimumApplicationAddress: pointer + lpMaximumApplicationAddress: pointer + dwActiveProcessorMask: ptr int32 + dwNumberOfProcessors: int32 + dwProcessorType: int32 + dwAllocationGranularity: int32 + wProcessorLevel: int16 + wProcessorRevision: int16 + + proc GetSystemInfo(lpSystemInfo: var SYSTEM_INFO) {.stdcall, dynlib: "kernel32", importc: "GetSystemInfo".} + + var + si: SYSTEM_INFO + GetSystemInfo(si) + result = si.dwNumberOfProcessors elif defined(macosx) or defined(bsd): var mib: array[0..3, cint] diff --git a/lib/pure/concurrency/threadpool.nim b/lib/pure/concurrency/threadpool.nim index 0f23b7e85..a5eaec86e 100644 --- a/lib/pure/concurrency/threadpool.nim +++ b/lib/pure/concurrency/threadpool.nim @@ -149,7 +149,7 @@ proc selectWorker(w: ptr Worker; fn: WorkerProc; data: pointer): bool = proc cleanFlowVars(w: ptr Worker) = let q = addr(w.q) acquire(q.lock) - for i in 0 .. <q.len: + for i in 0 ..< q.len: GC_unref(cast[RootRef](q.data[i])) #echo "GC_unref" q.len = 0 @@ -401,7 +401,7 @@ proc setup() = gCpus = p currentPoolSize = min(p, MaxThreadPoolSize) readyWorker = addr(workersData[0]) - for i in 0.. <currentPoolSize: activateWorkerThread(i) + for i in 0..<currentPoolSize: activateWorkerThread(i) proc preferSpawn*(): bool = ## Use this proc to determine quickly if a 'spawn' or a direct call is @@ -446,7 +446,7 @@ proc nimSpawn3(fn: WorkerProc; data: pointer) {.compilerProc.} = # implementation of 'spawn' that is used by the code generator. while true: if selectWorker(readyWorker, fn, data): return - for i in 0.. <currentPoolSize: + for i in 0..<currentPoolSize: if selectWorker(addr(workersData[i]), fn, data): return # determine what to do, but keep in mind this is expensive too: @@ -543,7 +543,7 @@ proc sync*() = var toRelease = 0 while true: var allReady = true - for i in 0 .. <currentPoolSize: + for i in 0 ..< currentPoolSize: if not allReady: break allReady = allReady and workersData[i].ready if allReady: break diff --git a/lib/pure/cookies.nim b/lib/pure/cookies.nim index 7d850798c..8f16717ac 100644 --- a/lib/pure/cookies.nim +++ b/lib/pure/cookies.nim @@ -13,6 +13,15 @@ import strtabs, times proc parseCookies*(s: string): StringTableRef = ## parses cookies into a string table. + ## + ## The proc is meant to parse the Cookie header set by a client, not the + ## "Set-Cookie" header set by servers. + ## + ## Example: + ## + ## .. code-block::Nim + ## doAssert parseCookies("a=1; foo=bar") == {"a": 1, "foo": "bar"}.newStringTable + result = newStringTable(modeCaseInsensitive) var i = 0 while true: @@ -42,7 +51,7 @@ proc setCookie*(key, value: string, domain = "", path = "", if secure: result.add("; Secure") if httpOnly: result.add("; HttpOnly") -proc setCookie*(key, value: string, expires: TimeInfo, +proc setCookie*(key, value: string, expires: DateTime, domain = "", path = "", noName = false, secure = false, httpOnly = false): string = ## Creates a command in the format of @@ -54,9 +63,9 @@ proc setCookie*(key, value: string, expires: TimeInfo, noname, secure, httpOnly) when isMainModule: - var tim = Time(int(getTime()) + 76 * (60 * 60 * 24)) + var tim = fromUnix(getTime().toUnix + 76 * (60 * 60 * 24)) - let cookie = setCookie("test", "value", tim.getGMTime()) + let cookie = setCookie("test", "value", tim.utc) when not defined(testing): echo cookie let start = "Set-Cookie: test=value; Expires=" diff --git a/lib/pure/cstrutils.nim b/lib/pure/cstrutils.nim new file mode 100644 index 000000000..437140892 --- /dev/null +++ b/lib/pure/cstrutils.nim @@ -0,0 +1,79 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2017 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module supports helper routines for working with ``cstring`` +## without having to convert ``cstring`` to ``string`` in order to +## save allocations. + +include "system/inclrtl" + +proc toLowerAscii(c: char): char {.inline.} = + if c in {'A'..'Z'}: + result = chr(ord(c) + (ord('a') - ord('A'))) + else: + result = c + +proc startsWith*(s, prefix: cstring): bool {.noSideEffect, + rtl, extern: "csuStartsWith".} = + ## Returns true iff ``s`` starts with ``prefix``. + ## + ## If ``prefix == ""`` true is returned. + var i = 0 + while true: + if prefix[i] == '\0': return true + if s[i] != prefix[i]: return false + inc(i) + +proc endsWith*(s, suffix: cstring): bool {.noSideEffect, + rtl, extern: "csuEndsWith".} = + ## Returns true iff ``s`` ends with ``suffix``. + ## + ## If ``suffix == ""`` true is returned. + let slen = s.len + var i = 0 + var j = slen - len(suffix) + while i+j <% slen: + if s[i+j] != suffix[i]: return false + inc(i) + if suffix[i] == '\0': return true + +proc cmpIgnoreStyle*(a, b: cstring): int {.noSideEffect, + rtl, extern: "csuCmpIgnoreStyle".} = + ## Compares two strings normalized (i.e. case and + ## underscores do not matter). Returns: + ## + ## | 0 iff a == b + ## | < 0 iff a < b + ## | > 0 iff a > b + var i = 0 + var j = 0 + while true: + while a[i] == '_': inc(i) + while b[j] == '_': inc(j) # BUGFIX: typo + var aa = toLowerAscii(a[i]) + var bb = toLowerAscii(b[j]) + result = ord(aa) - ord(bb) + if result != 0 or aa == '\0': break + inc(i) + inc(j) + +proc cmpIgnoreCase*(a, b: cstring): int {.noSideEffect, + rtl, extern: "csuCmpIgnoreCase".} = + ## Compares two strings in a case insensitive manner. Returns: + ## + ## | 0 iff a == b + ## | < 0 iff a < b + ## | > 0 iff a > b + var i = 0 + while true: + var aa = toLowerAscii(a[i]) + var bb = toLowerAscii(b[i]) + result = ord(aa) - ord(bb) + if result != 0 or aa == '\0': break + inc(i) diff --git a/lib/pure/future.nim b/lib/pure/future.nim index 2a6d29933..1a3ab688d 100644 --- a/lib/pure/future.nim +++ b/lib/pure/future.nim @@ -22,7 +22,7 @@ proc createProcType(p, b: NimNode): NimNode {.compileTime.} = case p.kind of nnkPar: - for i in 0 .. <p.len: + for i in 0 ..< p.len: let ident = p[i] var identDefs = newNimNode(nnkIdentDefs) case ident.kind @@ -77,7 +77,7 @@ macro `=>`*(p, b: untyped): untyped = if c[0].kind == nnkIdent and c[0].ident == !"->": var procTy = createProcType(c[1], c[2]) params[0] = procTy[0][0] - for i in 1 .. <procTy[0].len: + for i in 1 ..< procTy[0].len: params.add(procTy[0][i]) else: error("Expected proc type (->) got (" & $c[0].ident & ").") @@ -96,7 +96,7 @@ macro `=>`*(p, b: untyped): untyped = if p[0].kind == nnkIdent and p[0].ident == !"->": var procTy = createProcType(p[1], p[2]) params[0] = procTy[0][0] - for i in 1 .. <procTy[0].len: + for i in 1 ..< procTy[0].len: params.add(procTy[0][i]) else: error("Expected proc type (->) got (" & $p[0].ident & ").") @@ -197,4 +197,4 @@ macro dump*(x: typed): untyped = let s = x.toStrLit let r = quote do: debugEcho `s`, " = ", `x` - return r \ No newline at end of file + return r diff --git a/lib/pure/gentabs.nim b/lib/pure/gentabs.nim deleted file mode 100644 index 928ff8fe0..000000000 --- a/lib/pure/gentabs.nim +++ /dev/null @@ -1,211 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2012 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## The ``gentabs`` module implements an efficient hash table that is a -## key-value mapping. The keys are required to be strings, but the values -## may be any Nim or user defined type. This module supports matching -## of keys in case-sensitive, case-insensitive and style-insensitive modes. -## -## **Warning:** This module is deprecated, new code shouldn't use it! - -{.deprecated.} - -import - os, hashes, strutils - -type - GenTableMode* = enum ## describes the table's key matching mode - modeCaseSensitive, ## case sensitive matching of keys - modeCaseInsensitive, ## case insensitive matching of keys - modeStyleInsensitive ## style sensitive matching of keys - - GenKeyValuePair[T] = tuple[key: string, val: T] - GenKeyValuePairSeq[T] = seq[GenKeyValuePair[T]] - GenTable*[T] = object of RootObj - counter: int - data: GenKeyValuePairSeq[T] - mode: GenTableMode - - PGenTable*[T] = ref GenTable[T] ## use this type to declare hash tables - -{.deprecated: [TGenTableMode: GenTableMode, TGenKeyValuePair: GenKeyValuePair, - TGenKeyValuePairSeq: GenKeyValuePairSeq, TGenTable: GenTable].} - -const - growthFactor = 2 - startSize = 64 - - -proc len*[T](tbl: PGenTable[T]): int {.inline.} = - ## returns the number of keys in `tbl`. - result = tbl.counter - -iterator pairs*[T](tbl: PGenTable[T]): tuple[key: string, value: T] = - ## iterates over any (key, value) pair in the table `tbl`. - for h in 0..high(tbl.data): - if not isNil(tbl.data[h].key): - yield (tbl.data[h].key, tbl.data[h].val) - -proc myhash[T](tbl: PGenTable[T], key: string): Hash = - case tbl.mode - of modeCaseSensitive: result = hashes.hash(key) - of modeCaseInsensitive: result = hashes.hashIgnoreCase(key) - of modeStyleInsensitive: result = hashes.hashIgnoreStyle(key) - -proc myCmp[T](tbl: PGenTable[T], a, b: string): bool = - case tbl.mode - of modeCaseSensitive: result = cmp(a, b) == 0 - of modeCaseInsensitive: result = cmpIgnoreCase(a, b) == 0 - of modeStyleInsensitive: result = cmpIgnoreStyle(a, b) == 0 - -proc mustRehash(length, counter: int): bool = - assert(length > counter) - result = (length * 2 < counter * 3) or (length - counter < 4) - -proc newGenTable*[T](mode: GenTableMode): PGenTable[T] = - ## creates a new generic hash table that is empty. - new(result) - result.mode = mode - result.counter = 0 - newSeq(result.data, startSize) - -proc nextTry(h, maxHash: Hash): Hash {.inline.} = - result = ((5 * h) + 1) and maxHash - -proc rawGet[T](tbl: PGenTable[T], key: string): int = - var h: Hash - h = myhash(tbl, key) and high(tbl.data) # start with real hash value - while not isNil(tbl.data[h].key): - if myCmp(tbl, tbl.data[h].key, key): - return h - h = nextTry(h, high(tbl.data)) - result = - 1 - -proc rawInsert[T](tbl: PGenTable[T], data: var GenKeyValuePairSeq[T], - key: string, val: T) = - var h: Hash - h = myhash(tbl, key) and high(data) - while not isNil(data[h].key): - h = nextTry(h, high(data)) - data[h].key = key - data[h].val = val - -proc enlarge[T](tbl: PGenTable[T]) = - var n: GenKeyValuePairSeq[T] - newSeq(n, len(tbl.data) * growthFactor) - for i in countup(0, high(tbl.data)): - if not isNil(tbl.data[i].key): - rawInsert[T](tbl, n, tbl.data[i].key, tbl.data[i].val) - swap(tbl.data, n) - -proc hasKey*[T](tbl: PGenTable[T], key: string): bool = - ## returns true iff `key` is in the table `tbl`. - result = rawGet(tbl, key) >= 0 - -proc `[]`*[T](tbl: PGenTable[T], key: string): T = - ## retrieves the value at ``tbl[key]``. If `key` is not in `tbl`, - ## default(T) is returned and no exception is raised. One can check - ## with ``hasKey`` whether the key exists. - var index = rawGet(tbl, key) - if index >= 0: result = tbl.data[index].val - -proc `[]=`*[T](tbl: PGenTable[T], key: string, val: T) = - ## puts a (key, value)-pair into `tbl`. - var index = rawGet(tbl, key) - if index >= 0: - tbl.data[index].val = val - else: - if mustRehash(len(tbl.data), tbl.counter): enlarge(tbl) - rawInsert(tbl, tbl.data, key, val) - inc(tbl.counter) - - -when isMainModule: - # - # Verify tables of integer values (string keys) - # - var x = newGenTable[int](modeCaseInsensitive) - x["one"] = 1 - x["two"] = 2 - x["three"] = 3 - x["four"] = 4 - x["five"] = 5 - assert(len(x) == 5) # length procedure works - assert(x["one"] == 1) # case-sensitive lookup works - assert(x["ONE"] == 1) # case-insensitive should work for this table - assert(x["one"]+x["two"] == 3) # make sure we're getting back ints - assert(x.hasKey("one")) # hasKey should return 'true' for a key - # of "one"... - assert(not x.hasKey("NOPE")) # ...but key "NOPE" is not in the table. - for k,v in pairs(x): # make sure the 'pairs' iterator works - assert(x[k]==v) - - # - # Verify a table of user-defined types - # - type - MyType = tuple[first, second: string] # a pair of strings - {.deprecated: [TMyType: MyType].} - - var y = newGenTable[MyType](modeCaseInsensitive) # hash table where each - # value is MyType tuple - - #var junk: MyType = ("OK", "Here") - - #echo junk.first, " ", junk.second - - y["Hello"] = ("Hello", "World") - y["Goodbye"] = ("Goodbye", "Everyone") - #y["Hello"] = MyType( ("Hello", "World") ) - #y["Goodbye"] = MyType( ("Goodbye", "Everyone") ) - - assert( not isNil(y["Hello"].first) ) - assert( y["Hello"].first == "Hello" ) - assert( y["Hello"].second == "World" ) - - # - # Verify table of tables - # - var z: PGenTable[ PGenTable[int] ] # hash table where each value is - # a hash table of ints - - z = newGenTable[PGenTable[int]](modeCaseInsensitive) - z["first"] = newGenTable[int](modeCaseInsensitive) - z["first"]["one"] = 1 - z["first"]["two"] = 2 - z["first"]["three"] = 3 - - z["second"] = newGenTable[int](modeCaseInsensitive) - z["second"]["red"] = 10 - z["second"]["blue"] = 20 - - assert(len(z) == 2) # length of outer table - assert(len(z["first"]) == 3) # length of "first" table - assert(len(z["second"]) == 2) # length of "second" table - assert( z["first"]["one"] == 1) # retrieve from first inner table - assert( z["second"]["red"] == 10) # retrieve from second inner table - - when false: - # disabled: depends on hash order: - var output = "" - for k, v in pairs(z): - output.add( "$# ($#) ->\L" % [k,$len(v)] ) - for k2,v2 in pairs(v): - output.add( " $# <-> $#\L" % [k2,$v2] ) - - let expected = unindent """ - first (3) -> - two <-> 2 - three <-> 3 - one <-> 1 - second (2) -> - red <-> 10 - blue <-> 20 - """ - assert output == expected diff --git a/lib/pure/htmlgen.nim b/lib/pure/htmlgen.nim index ad199a215..c0934a45b 100644 --- a/lib/pure/htmlgen.nim +++ b/lib/pure/htmlgen.nim @@ -59,8 +59,8 @@ proc xmlCheckedTag*(e: NimNode, tag: string, optAttr = "", reqAttr = "", # copy the attributes; when iterating over them these lists # will be modified, so that each attribute is only given one value - var req = split(reqAttr) - var opt = split(optAttr) + var req = splitWhitespace(reqAttr) + var opt = splitWhitespace(optAttr) result = newNimNode(nnkBracket, e) result.add(newStrLitNode("<")) result.add(newStrLitNode(tag)) diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index de1d332a3..54a8498fa 100644 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -469,7 +469,7 @@ proc request*(url: string, httpMethod: string, extraHeaders = "", ## **Deprecated since version 0.15.0**: use ``HttpClient.request`` instead. var r = if proxy == nil: parseUri(url) else: proxy.url var hostUrl = if proxy == nil: r else: parseUri(url) - var headers = httpMethod.toUpper() + var headers = httpMethod.toUpperAscii() # TODO: Use generateHeaders further down once it supports proxies. var s = newSocket() @@ -713,10 +713,10 @@ proc downloadFile*(url: string, outputFilename: string, proc generateHeaders(requestUrl: Uri, httpMethod: string, headers: HttpHeaders, body: string, proxy: Proxy): string = # GET - result = httpMethod.toUpper() + result = httpMethod.toUpperAscii() result.add ' ' - if proxy.isNil: + if proxy.isNil or (not proxy.isNil and requestUrl.scheme == "https"): # /path?query if requestUrl.path[0] != '/': result.add '/' result.add(requestUrl.path) @@ -1048,7 +1048,11 @@ proc newConnection(client: HttpClient | AsyncHttpClient, client.currentURL.scheme != url.scheme or client.currentURL.port != url.port or (not client.connected): - let isSsl = url.scheme.toLowerAscii() == "https" + # Connect to proxy if specified + let connectionUrl = + if client.proxy.isNil: url else: client.proxy.url + + let isSsl = connectionUrl.scheme.toLowerAscii() == "https" if isSsl and not defined(ssl): raise newException(HttpRequestError, @@ -1056,31 +1060,55 @@ proc newConnection(client: HttpClient | AsyncHttpClient, if client.connected: client.close() + client.connected = false # TODO: I should be able to write 'net.Port' here... let port = - if url.port == "": + if connectionUrl.port == "": if isSsl: nativesockets.Port(443) else: nativesockets.Port(80) - else: nativesockets.Port(url.port.parseInt) + else: nativesockets.Port(connectionUrl.port.parseInt) when client is HttpClient: - client.socket = await net.dial(url.hostname, port) + client.socket = await net.dial(connectionUrl.hostname, port) elif client is AsyncHttpClient: - client.socket = await asyncnet.dial(url.hostname, port) + client.socket = await asyncnet.dial(connectionUrl.hostname, port) else: {.fatal: "Unsupported client type".} when defined(ssl): if isSsl: try: client.sslContext.wrapConnectedSocket( - client.socket, handshakeAsClient, url.hostname) + client.socket, handshakeAsClient, connectionUrl.hostname) except: client.socket.close() raise getCurrentException() + # If need to CONNECT through proxy + if url.scheme == "https" and not client.proxy.isNil: + when defined(ssl): + # Pass only host:port for CONNECT + var connectUrl = initUri() + connectUrl.hostname = url.hostname + connectUrl.port = if url.port != "": url.port else: "443" + + let proxyHeaderString = generateHeaders(connectUrl, $HttpConnect, newHttpHeaders(), "", client.proxy) + await client.socket.send(proxyHeaderString) + let proxyResp = await parseResponse(client, false) + + if not proxyResp.status.startsWith("200"): + raise newException(HttpRequestError, + "The proxy server rejected a CONNECT request, " & + "so a secure connection could not be established.") + client.sslContext.wrapConnectedSocket( + client.socket, handshakeAsClient, url.hostname) + else: + raise newException(HttpRequestError, + "SSL support is not available. Cannot connect over SSL.") + + # May be connected through proxy but remember actual URL being accessed client.currentURL = url client.connected = true @@ -1100,32 +1128,9 @@ proc requestAux(client: HttpClient | AsyncHttpClient, url: string, headers: HttpHeaders = nil): Future[Response | AsyncResponse] {.multisync.} = # Helper that actually makes the request. Does not handle redirects. - let connectionUrl = - if client.proxy.isNil: parseUri(url) else: client.proxy.url let requestUrl = parseUri(url) - let savedProxy = client.proxy # client's proxy may be overwritten. - - if requestUrl.scheme == "https" and not client.proxy.isNil: - when defined(ssl): - client.proxy.url = connectionUrl - var connectUrl = requestUrl - connectUrl.scheme = "http" - connectUrl.port = "443" - let proxyResp = await requestAux(client, $connectUrl, $HttpConnect) - - if not proxyResp.status.startsWith("200"): - raise newException(HttpRequestError, - "The proxy server rejected a CONNECT request, " & - "so a secure connection could not be established.") - client.sslContext.wrapConnectedSocket( - client.socket, handshakeAsClient, requestUrl.hostname) - client.proxy = nil - else: - raise newException(HttpRequestError, - "SSL support not available. Cannot connect to https site over proxy.") - else: - await newConnection(client, connectionUrl) + await newConnection(client, requestUrl) let effectiveHeaders = client.headers.override(headers) @@ -1143,9 +1148,6 @@ proc requestAux(client: HttpClient | AsyncHttpClient, url: string, client.getBody result = await parseResponse(client, getBody) - # Restore the clients proxy in case it was overwritten. - client.proxy = savedProxy - proc request*(client: HttpClient | AsyncHttpClient, url: string, httpMethod: string, body = "", headers: HttpHeaders = nil): Future[Response | AsyncResponse] diff --git a/lib/pure/httpcore.nim b/lib/pure/httpcore.nim index a5ab40ca4..f150fa1c1 100644 --- a/lib/pure/httpcore.nim +++ b/lib/pure/httpcore.nim @@ -113,6 +113,9 @@ proc newHttpHeaders*(keyValuePairs: new result result.table = newTable[string, seq[string]](pairs) +proc `$`*(headers: HttpHeaders): string = + return $headers.table + proc clear*(headers: HttpHeaders) = headers.table.clear() diff --git a/lib/pure/includes/osenv.nim b/lib/pure/includes/osenv.nim index 8d2fc235a..ae62a5c4e 100644 --- a/lib/pure/includes/osenv.nim +++ b/lib/pure/includes/osenv.nim @@ -94,7 +94,7 @@ proc findEnvVar(key: string): int = if startsWith(environment[i], temp): return i return -1 -proc getEnv*(key: string): TaintedString {.tags: [ReadEnvEffect].} = +proc getEnv*(key: string, default = ""): TaintedString {.tags: [ReadEnvEffect].} = ## Returns the value of the `environment variable`:idx: named `key`. ## ## If the variable does not exist, "" is returned. To distinguish @@ -108,7 +108,7 @@ proc getEnv*(key: string): TaintedString {.tags: [ReadEnvEffect].} = return TaintedString(substr(environment[i], find(environment[i], '=')+1)) else: var env = c_getenv(key) - if env == nil: return TaintedString("") + if env == nil: return TaintedString(default) result = TaintedString($env) proc existsEnv*(key: string): bool {.tags: [ReadEnvEffect].} = diff --git a/lib/pure/includes/oserr.nim b/lib/pure/includes/oserr.nim index dbb709f1b..0889d7383 100644 --- a/lib/pure/includes/oserr.nim +++ b/lib/pure/includes/oserr.nim @@ -56,9 +56,6 @@ proc raiseOSError*(msg: string = "") {.noinline, rtl, extern: "nos$1", raise newException(OSError, msg) {.pop.} -when not defined(nimfix): - {.deprecated: [osError: raiseOSError].} - proc `==`*(err1, err2: OSErrorCode): bool {.borrow.} proc `$`*(err: OSErrorCode): string {.borrow.} diff --git a/lib/pure/ioselectors.nim b/lib/pure/ioselectors.nim deleted file mode 100644 index ef8072221..000000000 --- a/lib/pure/ioselectors.nim +++ /dev/null @@ -1,293 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2016 Eugene Kabanov -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## This module allows high-level and efficient I/O multiplexing. -## -## Supported OS primitives: ``epoll``, ``kqueue``, ``poll`` and -## Windows ``select``. -## -## To use threadsafe version of this module, it needs to be compiled -## with both ``-d:threadsafe`` and ``--threads:on`` options. -## -## Supported features: files, sockets, pipes, timers, processes, signals -## and user events. -## -## Fully supported OS: MacOSX, FreeBSD, OpenBSD, NetBSD, Linux (except -## for Android). -## -## Partially supported OS: Windows (only sockets and user events), -## Solaris (files, sockets, handles and user events). -## Android (files, sockets, handles and user events). -## -## TODO: ``/dev/poll``, ``event ports`` and filesystem events. - -import os - -const hasThreadSupport = compileOption("threads") and defined(threadsafe) - -const ioselSupportedPlatform* = defined(macosx) or defined(freebsd) or - defined(netbsd) or defined(openbsd) or - defined(dragonfly) or - (defined(linux) and not defined(android)) - ## This constant is used to determine whether the destination platform is - ## fully supported by ``ioselectors`` module. - -const bsdPlatform = defined(macosx) or defined(freebsd) or - defined(netbsd) or defined(openbsd) or - defined(dragonfly) - -when defined(nimdoc): - type - Selector*[T] = ref object - ## An object which holds descriptors to be checked for read/write status - - Event* {.pure.} = enum - ## An enum which hold event types - Read, ## Descriptor is available for read - Write, ## Descriptor is available for write - Timer, ## Timer descriptor is completed - Signal, ## Signal is raised - Process, ## Process is finished - Vnode, ## BSD specific file change happens - User, ## User event is raised - Error, ## Error happens while waiting, for descriptor - VnodeWrite, ## NOTE_WRITE (BSD specific, write to file occurred) - VnodeDelete, ## NOTE_DELETE (BSD specific, unlink of file occurred) - VnodeExtend, ## NOTE_EXTEND (BSD specific, file extended) - VnodeAttrib, ## NOTE_ATTRIB (BSD specific, file attributes changed) - VnodeLink, ## NOTE_LINK (BSD specific, file link count changed) - VnodeRename, ## NOTE_RENAME (BSD specific, file renamed) - VnodeRevoke ## NOTE_REVOKE (BSD specific, file revoke occurred) - - ReadyKey* = object - ## An object which holds result for descriptor - fd* : int ## file/socket descriptor - events*: set[Event] ## set of events - - SelectEvent* = object - ## An object which holds user defined event - - proc newSelector*[T](): Selector[T] = - ## Creates a new selector - - proc close*[T](s: Selector[T]) = - ## Closes selector - - proc registerHandle*[T](s: Selector[T], fd: SocketHandle, events: set[Event], - data: T) = - ## Registers file/socket descriptor ``fd`` to selector ``s`` - ## with events set in ``events``. The ``data`` is application-defined - ## data, which to be passed when event happens. - - proc updateHandle*[T](s: Selector[T], fd: SocketHandle, events: set[Event]) = - ## Update file/socket descriptor ``fd``, registered in selector - ## ``s`` with new events set ``event``. - - proc registerTimer*[T](s: Selector[T], timeout: int, oneshot: bool, - data: T): int {.discardable.} = - ## Registers timer notification with ``timeout`` in milliseconds - ## to selector ``s``. - ## If ``oneshot`` is ``true`` timer will be notified only once. - ## Set ``oneshot`` to ``false`` if your want periodic notifications. - ## The ``data`` is application-defined data, which to be passed, when - ## time limit expired. - - proc registerSignal*[T](s: Selector[T], signal: int, - data: T): int {.discardable.} = - ## Registers Unix signal notification with ``signal`` to selector - ## ``s``. The ``data`` is application-defined data, which to be - ## passed, when signal raises. - ## - ## This function is not supported for ``Windows``. - - proc registerProcess*[T](s: Selector[T], pid: int, - data: T): int {.discardable.} = - ## Registers process id (pid) notification when process has - ## exited to selector ``s``. - ## The ``data`` is application-defined data, which to be passed, when - ## process with ``pid`` has exited. - - proc registerEvent*[T](s: Selector[T], ev: SelectEvent, data: T) = - ## Registers selector event ``ev`` to selector ``s``. - ## ``data`` application-defined data, which to be passed, when - ## ``ev`` happens. - - proc registerVnode*[T](s: Selector[T], fd: cint, events: set[Event], - data: T) = - ## Registers selector BSD/MacOSX specific vnode events for file - ## descriptor ``fd`` and events ``events``. - ## ``data`` application-defined data, which to be passed, when - ## vnode event happens. - ## - ## This function is supported only by BSD and MacOSX. - - proc newSelectEvent*(): SelectEvent = - ## Creates new event ``SelectEvent``. - - proc setEvent*(ev: SelectEvent) = - ## Trigger event ``ev``. - - proc close*(ev: SelectEvent) = - ## Closes selector event ``ev``. - - proc unregister*[T](s: Selector[T], ev: SelectEvent) = - ## Unregisters event ``ev`` from selector ``s``. - - proc unregister*[T](s: Selector[T], fd: int|SocketHandle|cint) = - ## Unregisters file/socket descriptor ``fd`` from selector ``s``. - - proc selectInto*[T](s: Selector[T], timeout: int, - results: var openarray[ReadyKey]): int = - ## Process call waiting for events registered in selector ``s``. - ## The ``timeout`` argument specifies the minimum number of milliseconds - ## the function will be blocked, if no events are not ready. Specifying a - ## timeout of ``-1`` causes function to block indefinitely. - ## All available events will be stored in ``results`` array. - ## - ## Function returns number of triggered events. - - proc select*[T](s: Selector[T], timeout: int): seq[ReadyKey] = - ## Process call waiting for events registered in selector ``s``. - ## The ``timeout`` argument specifies the minimum number of milliseconds - ## the function will be blocked, if no events are not ready. Specifying a - ## timeout of -1 causes function to block indefinitely. - ## - ## Function returns sequence of triggered events. - - proc getData*[T](s: Selector[T], fd: SocketHandle|int): T = - ## Retrieves application-defined ``data`` associated with descriptor ``fd``. - ## If specified descriptor ``fd`` is not registered, empty/default value - ## will be returned. - - proc setData*[T](s: Selector[T], fd: SocketHandle|int, data: var T): bool = - ## Associate application-defined ``data`` with descriptor ``fd``. - ## - ## Returns ``true``, if data was succesfully updated, ``false`` otherwise. - - template isEmpty*[T](s: Selector[T]): bool = - ## Returns ``true``, if there no registered events or descriptors - ## in selector. - - template withData*[T](s: Selector[T], fd: SocketHandle|int, value, - body: untyped) = - ## Retrieves the application-data assigned with descriptor ``fd`` - ## to ``value``. This ``value`` can be modified in the scope of - ## the ``withData`` call. - ## - ## .. code-block:: nim - ## - ## s.withData(fd, value) do: - ## # block is executed only if ``fd`` registered in selector ``s`` - ## value.uid = 1000 - ## - - template withData*[T](s: Selector[T], fd: SocketHandle|int, value, - body1, body2: untyped) = - ## Retrieves the application-data assigned with descriptor ``fd`` - ## to ``value``. This ``value`` can be modified in the scope of - ## the ``withData`` call. - ## - ## .. code-block:: nim - ## - ## s.withData(fd, value) do: - ## # block is executed only if ``fd`` registered in selector ``s``. - ## value.uid = 1000 - ## do: - ## # block is executed if ``fd`` not registered in selector ``s``. - ## raise - ## - -else: - when hasThreadSupport: - import locks - - type - SharedArray[T] = UncheckedArray[T] - - proc allocSharedArray[T](nsize: int): ptr SharedArray[T] = - result = cast[ptr SharedArray[T]](allocShared0(sizeof(T) * nsize)) - - proc deallocSharedArray[T](sa: ptr SharedArray[T]) = - deallocShared(cast[pointer](sa)) - type - Event* {.pure.} = enum - Read, Write, Timer, Signal, Process, Vnode, User, Error, Oneshot, - Finished, VnodeWrite, VnodeDelete, VnodeExtend, VnodeAttrib, VnodeLink, - VnodeRename, VnodeRevoke - - type - IOSelectorsException* = object of Exception - - ReadyKey* = object - fd* : int - events*: set[Event] - - SelectorKey[T] = object - ident: int - events: set[Event] - param: int - data: T - - proc raiseIOSelectorsError[T](message: T) = - var msg = "" - when T is string: - msg.add(message) - elif T is OSErrorCode: - msg.add(osErrorMsg(message) & " (code: " & $int(message) & ")") - else: - msg.add("Internal Error\n") - var err = newException(IOSelectorsException, msg) - raise err - - when not defined(windows): - import posix - - proc setNonBlocking(fd: cint) {.inline.} = - var x = fcntl(fd, F_GETFL, 0) - if x == -1: - raiseIOSelectorsError(osLastError()) - else: - var mode = x or O_NONBLOCK - if fcntl(fd, F_SETFL, mode) == -1: - raiseIOSelectorsError(osLastError()) - - template setKey(s, pident, pevents, pparam, pdata: untyped) = - var skey = addr(s.fds[pident]) - skey.ident = pident - skey.events = pevents - skey.param = pparam - skey.data = data - - when ioselSupportedPlatform: - template blockSignals(newmask: var Sigset, oldmask: var Sigset) = - when hasThreadSupport: - if posix.pthread_sigmask(SIG_BLOCK, newmask, oldmask) == -1: - raiseIOSelectorsError(osLastError()) - else: - if posix.sigprocmask(SIG_BLOCK, newmask, oldmask) == -1: - raiseIOSelectorsError(osLastError()) - - template unblockSignals(newmask: var Sigset, oldmask: var Sigset) = - when hasThreadSupport: - if posix.pthread_sigmask(SIG_UNBLOCK, newmask, oldmask) == -1: - raiseIOSelectorsError(osLastError()) - else: - if posix.sigprocmask(SIG_UNBLOCK, newmask, oldmask) == -1: - raiseIOSelectorsError(osLastError()) - - when defined(linux): - include ioselects/ioselectors_epoll - elif bsdPlatform: - include ioselects/ioselectors_kqueue - elif defined(windows): - include ioselects/ioselectors_select - elif defined(solaris): - include ioselects/ioselectors_poll # need to replace it with event ports - else: - include ioselects/ioselectors_poll diff --git a/lib/pure/ioselects/ioselectors_epoll.nim b/lib/pure/ioselects/ioselectors_epoll.nim index 3a5cbc87a..8827f239f 100644 --- a/lib/pure/ioselects/ioselectors_epoll.nim +++ b/lib/pure/ioselects/ioselectors_epoll.nim @@ -9,7 +9,7 @@ # This module implements Linux epoll(). -import posix, times +import posix, times, epoll # Maximum number of events that can be returned const MAX_EPOLL_EVENTS = 64 @@ -36,35 +36,6 @@ when not defined(android): ssi_addr*: uint64 pad* {.importc: "__pad".}: array[0..47, uint8] -type - eventFdData {.importc: "eventfd_t", - header: "<sys/eventfd.h>", pure, final.} = uint64 - epoll_data {.importc: "union epoll_data", header: "<sys/epoll.h>", - pure, final.} = object - u64 {.importc: "u64".}: uint64 - epoll_event {.importc: "struct epoll_event", - header: "<sys/epoll.h>", pure, final.} = object - events: uint32 # Epoll events - data: epoll_data # User data variable - -const - EPOLL_CTL_ADD = 1 # Add a file descriptor to the interface. - EPOLL_CTL_DEL = 2 # Remove a file descriptor from the interface. - EPOLL_CTL_MOD = 3 # Change file descriptor epoll_event structure. - EPOLLIN = 0x00000001 - EPOLLOUT = 0x00000004 - EPOLLERR = 0x00000008 - EPOLLHUP = 0x00000010 - EPOLLRDHUP = 0x00002000 - EPOLLONESHOT = 1 shl 30 - -proc epoll_create(size: cint): cint - {.importc: "epoll_create", header: "<sys/epoll.h>".} -proc epoll_ctl(epfd: cint; op: cint; fd: cint; event: ptr epoll_event): cint - {.importc: "epoll_ctl", header: "<sys/epoll.h>".} -proc epoll_wait(epfd: cint; events: ptr epoll_event; maxevents: cint; - timeout: cint): cint - {.importc: "epoll_wait", header: "<sys/epoll.h>".} proc timerfd_create(clock_id: ClockId, flags: cint): cint {.cdecl, importc: "timerfd_create", header: "<sys/timerfd.h>".} proc timerfd_settime(ufd: cint, flags: cint, @@ -80,26 +51,26 @@ when not defined(android): var RLIMIT_NOFILE {.importc: "RLIMIT_NOFILE", header: "<sys/resource.h>".}: cint type - rlimit {.importc: "struct rlimit", + RLimit {.importc: "struct rlimit", header: "<sys/resource.h>", pure, final.} = object rlim_cur: int rlim_max: int -proc getrlimit(resource: cint, rlp: var rlimit): cint +proc getrlimit(resource: cint, rlp: var RLimit): cint {.importc: "getrlimit",header: "<sys/resource.h>".} when hasThreadSupport: type SelectorImpl[T] = object - epollFD : cint - maxFD : int + epollFD: cint + maxFD: int fds: ptr SharedArray[SelectorKey[T]] count: int Selector*[T] = ptr SelectorImpl[T] else: type SelectorImpl[T] = object - epollFD : cint - maxFD : int + epollFD: cint + maxFD: int fds: seq[SelectorKey[T]] count: int Selector*[T] = ref SelectorImpl[T] @@ -109,7 +80,8 @@ type SelectEvent* = ptr SelectEventImpl proc newSelector*[T](): Selector[T] = - var a = rlimit() + # Retrieve the maximum fd count (for current OS) via getrlimit() + var a = RLimit() if getrlimit(RLIMIT_NOFILE, a) != 0: raiseOsError(osLastError()) var maxFD = int(a.rlim_max) @@ -152,8 +124,8 @@ proc newSelectEvent*(): SelectEvent = result = cast[SelectEvent](allocShared0(sizeof(SelectEventImpl))) result.efd = fdci -proc setEvent*(ev: SelectEvent) = - var data : uint64 = 1 +proc trigger*(ev: SelectEvent) = + var data: uint64 = 1 if posix.write(ev.efd, addr data, sizeof(uint64)) == -1: raiseIOSelectorsError(osLastError()) @@ -164,6 +136,8 @@ proc close*(ev: SelectEvent) = raiseIOSelectorsError(osLastError()) template checkFd(s, f) = + # TODO: I don't see how this can ever happen. You won't be able to create an + # FD if there is too many. -- DP if f >= s.maxFD: raiseIOSelectorsError("Maximum number of descriptors is exhausted!") @@ -171,10 +145,10 @@ proc registerHandle*[T](s: Selector[T], fd: SocketHandle, events: set[Event], data: T) = let fdi = int(fd) s.checkFd(fdi) - doAssert(s.fds[fdi].ident == 0) + doAssert(s.fds[fdi].ident == 0, "Descriptor $# already registered" % $fdi) s.setKey(fdi, events, 0, data) if events != {}: - var epv = epoll_event(events: EPOLLRDHUP) + var epv = EpollEvent(events: EPOLLRDHUP) epv.data.u64 = fdi.uint if Event.Read in events: epv.events = epv.events or EPOLLIN if Event.Write in events: epv.events = epv.events or EPOLLOUT @@ -189,10 +163,10 @@ proc updateHandle*[T](s: Selector[T], fd: SocketHandle, events: set[Event]) = s.checkFd(fdi) var pkey = addr(s.fds[fdi]) doAssert(pkey.ident != 0, - "Descriptor [" & $fdi & "] is not registered in the queue!") + "Descriptor $# is not registered in the selector!" % $fdi) doAssert(pkey.events * maskEvents == {}) if pkey.events != events: - var epv = epoll_event(events: EPOLLRDHUP) + var epv = EpollEvent(events: EPOLLRDHUP) epv.data.u64 = fdi.uint if Event.Read in events: epv.events = epv.events or EPOLLIN @@ -217,24 +191,25 @@ proc unregister*[T](s: Selector[T], fd: int|SocketHandle) = s.checkFd(fdi) var pkey = addr(s.fds[fdi]) doAssert(pkey.ident != 0, - "Descriptor [" & $fdi & "] is not registered in the queue!") + "Descriptor $# is not registered in the selector!" % $fdi) if pkey.events != {}: when not defined(android): if pkey.events * {Event.Read, Event.Write} != {}: - var epv = epoll_event() + var epv = EpollEvent() + # TODO: Refactor all these EPOLL_CTL_DEL + dec(s.count) into a proc. if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) != 0: raiseIOSelectorsError(osLastError()) dec(s.count) elif Event.Timer in pkey.events: if Event.Finished notin pkey.events: - var epv = epoll_event() + var epv = EpollEvent() if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) != 0: raiseIOSelectorsError(osLastError()) dec(s.count) if posix.close(cint(fdi)) != 0: raiseIOSelectorsError(osLastError()) elif Event.Signal in pkey.events: - var epv = epoll_event() + var epv = EpollEvent() if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) != 0: raiseIOSelectorsError(osLastError()) var nmask, omask: Sigset @@ -247,7 +222,7 @@ proc unregister*[T](s: Selector[T], fd: int|SocketHandle) = raiseIOSelectorsError(osLastError()) elif Event.Process in pkey.events: if Event.Finished notin pkey.events: - var epv = epoll_event() + var epv = EpollEvent() if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) != 0: raiseIOSelectorsError(osLastError()) var nmask, omask: Sigset @@ -260,13 +235,13 @@ proc unregister*[T](s: Selector[T], fd: int|SocketHandle) = raiseIOSelectorsError(osLastError()) else: if pkey.events * {Event.Read, Event.Write} != {}: - var epv = epoll_event() + var epv = EpollEvent() if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) != 0: raiseIOSelectorsError(osLastError()) dec(s.count) elif Event.Timer in pkey.events: if Event.Finished notin pkey.events: - var epv = epoll_event() + var epv = EpollEvent() if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) != 0: raiseIOSelectorsError(osLastError()) dec(s.count) @@ -280,7 +255,7 @@ proc unregister*[T](s: Selector[T], ev: SelectEvent) = var pkey = addr(s.fds[fdi]) doAssert(pkey.ident != 0, "Event is not registered in the queue!") doAssert(Event.User in pkey.events) - var epv = epoll_event() + var epv = EpollEvent() if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) != 0: raiseIOSelectorsError(osLastError()) dec(s.count) @@ -300,17 +275,18 @@ proc registerTimer*[T](s: Selector[T], timeout: int, oneshot: bool, doAssert(s.fds[fdi].ident == 0) var events = {Event.Timer} - var epv = epoll_event(events: EPOLLIN or EPOLLRDHUP) + var epv = EpollEvent(events: EPOLLIN or EPOLLRDHUP) epv.data.u64 = fdi.uint + if oneshot: - new_ts.it_interval.tv_sec = 0.Time + new_ts.it_interval.tv_sec = posix.Time(0) new_ts.it_interval.tv_nsec = 0 - new_ts.it_value.tv_sec = (timeout div 1_000).Time + new_ts.it_value.tv_sec = posix.Time(timeout div 1_000) new_ts.it_value.tv_nsec = (timeout %% 1_000) * 1_000_000 incl(events, Event.Oneshot) epv.events = epv.events or EPOLLONESHOT else: - new_ts.it_interval.tv_sec = (timeout div 1000).Time + new_ts.it_interval.tv_sec = posix.Time(timeout div 1000) new_ts.it_interval.tv_nsec = (timeout %% 1_000) * 1_000_000 new_ts.it_value.tv_sec = new_ts.it_interval.tv_sec new_ts.it_value.tv_nsec = new_ts.it_interval.tv_nsec @@ -343,7 +319,7 @@ when not defined(android): s.checkFd(fdi) doAssert(s.fds[fdi].ident == 0) - var epv = epoll_event(events: EPOLLIN or EPOLLRDHUP) + var epv = EpollEvent(events: EPOLLIN or EPOLLRDHUP) epv.data.u64 = fdi.uint if epoll_ctl(s.epollFD, EPOLL_CTL_ADD, fdi.cint, addr epv) != 0: raiseIOSelectorsError(osLastError()) @@ -370,7 +346,7 @@ when not defined(android): s.checkFd(fdi) doAssert(s.fds[fdi].ident == 0) - var epv = epoll_event(events: EPOLLIN or EPOLLRDHUP) + var epv = EpollEvent(events: EPOLLIN or EPOLLRDHUP) epv.data.u64 = fdi.uint epv.events = EPOLLIN or EPOLLRDHUP if epoll_ctl(s.epollFD, EPOLL_CTL_ADD, fdi.cint, addr epv) != 0: @@ -383,7 +359,7 @@ proc registerEvent*[T](s: Selector[T], ev: SelectEvent, data: T) = let fdi = int(ev.efd) doAssert(s.fds[fdi].ident == 0, "Event is already registered in the queue!") s.setKey(fdi, {Event.User}, 0, data) - var epv = epoll_event(events: EPOLLIN or EPOLLRDHUP) + var epv = EpollEvent(events: EPOLLIN or EPOLLRDHUP) epv.data.u64 = ev.efd.uint if epoll_ctl(s.epollFD, EPOLL_CTL_ADD, ev.efd, addr epv) != 0: raiseIOSelectorsError(osLastError()) @@ -392,7 +368,7 @@ proc registerEvent*[T](s: Selector[T], ev: SelectEvent, data: T) = proc selectInto*[T](s: Selector[T], timeout: int, results: var openarray[ReadyKey]): int = var - resTable: array[MAX_EPOLL_EVENTS, epoll_event] + resTable: array[MAX_EPOLL_EVENTS, EpollEvent] maxres = MAX_EPOLL_EVENTS i, k: int @@ -482,7 +458,7 @@ proc selectInto*[T](s: Selector[T], timeout: int, rkey.events.incl(Event.User) if Event.Oneshot in pkey.events: - var epv = epoll_event() + var epv = EpollEvent() if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, cint(fdi), addr epv) != 0: raiseIOSelectorsError(osLastError()) # we will not clear key until it will be unregistered, so @@ -505,16 +481,19 @@ proc select*[T](s: Selector[T], timeout: int): seq[ReadyKey] = template isEmpty*[T](s: Selector[T]): bool = (s.count == 0) -proc getData*[T](s: Selector[T], fd: SocketHandle|int): T = +proc contains*[T](s: Selector[T], fd: SocketHandle|int): bool {.inline.} = + return s.fds[fd].ident != 0 + +proc getData*[T](s: Selector[T], fd: SocketHandle|int): var T = let fdi = int(fd) s.checkFd(fdi) - if s.fds[fdi].ident != 0: + if fdi in s: result = s.fds[fdi].data proc setData*[T](s: Selector[T], fd: SocketHandle|int, data: T): bool = let fdi = int(fd) s.checkFd(fdi) - if s.fds[fdi].ident != 0: + if fdi in s: s.fds[fdi].data = data result = true @@ -523,8 +502,8 @@ template withData*[T](s: Selector[T], fd: SocketHandle|int, value, mixin checkFd let fdi = int(fd) s.checkFd(fdi) - if s.fds[fdi].ident != 0: - var value = addr(s.fds[fdi].data) + if fdi in s: + var value = addr(s.getData(fdi)) body template withData*[T](s: Selector[T], fd: SocketHandle|int, value, body1, @@ -532,8 +511,8 @@ template withData*[T](s: Selector[T], fd: SocketHandle|int, value, body1, mixin checkFd let fdi = int(fd) s.checkFd(fdi) - if s.fds[fdi].ident != 0: - var value = addr(s.fds[fdi].data) + if fdi in s: + var value = addr(s.getData(fdi)) body1 else: body2 diff --git a/lib/pure/ioselects/ioselectors_kqueue.nim b/lib/pure/ioselects/ioselectors_kqueue.nim index 7786de46a..af5aa15df 100644 --- a/lib/pure/ioselects/ioselectors_kqueue.nim +++ b/lib/pure/ioselects/ioselectors_kqueue.nim @@ -144,7 +144,7 @@ proc newSelectEvent*(): SelectEvent = result.rfd = fds[0] result.wfd = fds[1] -proc setEvent*(ev: SelectEvent) = +proc trigger*(ev: SelectEvent) = var data: uint64 = 1 if posix.write(ev.wfd, addr data, sizeof(uint64)) != sizeof(uint64): raiseIOSelectorsError(osLastError()) @@ -243,7 +243,7 @@ proc updateHandle*[T](s: Selector[T], fd: SocketHandle, s.checkFd(fdi) var pkey = addr(s.fds[fdi]) doAssert(pkey.ident != 0, - "Descriptor [" & $fdi & "] is not registered in the queue!") + "Descriptor $# is not registered in the queue!" % $fdi) doAssert(pkey.events * maskEvents == {}) if pkey.events != events: @@ -452,10 +452,10 @@ proc selectInto*[T](s: Selector[T], timeout: int, if timeout != -1: if timeout >= 1000: - tv.tv_sec = (timeout div 1_000).Time + tv.tv_sec = posix.Time(timeout div 1_000) tv.tv_nsec = (timeout %% 1_000) * 1_000_000 else: - tv.tv_sec = 0.Time + tv.tv_sec = posix.Time(0) tv.tv_nsec = timeout * 1_000_000 else: ptv = nil @@ -584,16 +584,19 @@ proc select*[T](s: Selector[T], timeout: int): seq[ReadyKey] = template isEmpty*[T](s: Selector[T]): bool = (s.count == 0) -proc getData*[T](s: Selector[T], fd: SocketHandle|int): T = +proc contains*[T](s: Selector[T], fd: SocketHandle|int): bool {.inline.} = + return s.fds[fd].ident != 0 + +proc getData*[T](s: Selector[T], fd: SocketHandle|int): var T = let fdi = int(fd) s.checkFd(fdi) - if s.fds[fdi].ident != 0: + if fdi in s: result = s.fds[fdi].data proc setData*[T](s: Selector[T], fd: SocketHandle|int, data: T): bool = let fdi = int(fd) s.checkFd(fdi) - if s.fds[fdi].ident != 0: + if fdi in s: s.fds[fdi].data = data result = true @@ -602,8 +605,8 @@ template withData*[T](s: Selector[T], fd: SocketHandle|int, value, mixin checkFd let fdi = int(fd) s.checkFd(fdi) - if s.fds[fdi].ident != 0: - var value = addr(s.fds[fdi].data) + if fdi in s: + var value = addr(s.getData(fdi)) body template withData*[T](s: Selector[T], fd: SocketHandle|int, value, body1, @@ -611,8 +614,8 @@ template withData*[T](s: Selector[T], fd: SocketHandle|int, value, body1, mixin checkFd let fdi = int(fd) s.checkFd(fdi) - if s.fds[fdi].ident != 0: - var value = addr(s.fds[fdi].data) + if fdi in s: + var value = addr(s.getData(fdi)) body1 else: body2 diff --git a/lib/pure/ioselects/ioselectors_poll.nim b/lib/pure/ioselects/ioselectors_poll.nim index 1b90e0806..cc06aa592 100644 --- a/lib/pure/ioselects/ioselectors_poll.nim +++ b/lib/pure/ioselects/ioselectors_poll.nim @@ -208,7 +208,7 @@ proc newSelectEvent*(): SelectEvent = result.rfd = fds[0] result.wfd = fds[1] -proc setEvent*(ev: SelectEvent) = +proc trigger*(ev: SelectEvent) = var data: uint64 = 1 if posix.write(ev.wfd, addr data, sizeof(uint64)) != sizeof(uint64): raiseIOSelectorsError(osLastError()) @@ -279,16 +279,19 @@ proc select*[T](s: Selector[T], timeout: int): seq[ReadyKey] = template isEmpty*[T](s: Selector[T]): bool = (s.count == 0) -proc getData*[T](s: Selector[T], fd: SocketHandle|int): T = +proc contains*[T](s: Selector[T], fd: SocketHandle|int): bool {.inline.} = + return s.fds[fd].ident != 0 + +proc getData*[T](s: Selector[T], fd: SocketHandle|int): var T = let fdi = int(fd) s.checkFd(fdi) - if s.fds[fdi].ident != 0: + if fdi in s: result = s.fds[fdi].data proc setData*[T](s: Selector[T], fd: SocketHandle|int, data: T): bool = let fdi = int(fd) s.checkFd(fdi) - if s.fds[fdi].ident != 0: + if fdi in s: s.fds[fdi].data = data result = true @@ -297,8 +300,8 @@ template withData*[T](s: Selector[T], fd: SocketHandle|int, value, mixin checkFd let fdi = int(fd) s.checkFd(fdi) - if s.fds[fdi].ident != 0: - var value = addr(s.fds[fdi].data) + if fdi in s: + var value = addr(s.getData(fdi)) body template withData*[T](s: Selector[T], fd: SocketHandle|int, value, body1, @@ -306,8 +309,8 @@ template withData*[T](s: Selector[T], fd: SocketHandle|int, value, body1, mixin checkFd let fdi = int(fd) s.checkFd(fdi) - if s.fds[fdi].ident != 0: - var value = addr(s.fds[fdi].data) + if fdi in s: + var value = addr(s.getData(fdi)) body1 else: body2 diff --git a/lib/pure/ioselects/ioselectors_select.nim b/lib/pure/ioselects/ioselectors_select.nim index dc3451d52..c787f0070 100644 --- a/lib/pure/ioselects/ioselectors_select.nim +++ b/lib/pure/ioselects/ioselectors_select.nim @@ -154,7 +154,7 @@ when defined(windows): result.rsock = rsock result.wsock = wsock - proc setEvent*(ev: SelectEvent) = + proc trigger*(ev: SelectEvent) = var data: uint64 = 1 if winlean.send(ev.wsock, cast[pointer](addr data), cint(sizeof(uint64)), 0) != sizeof(uint64): @@ -178,7 +178,7 @@ else: result.rsock = SocketHandle(fds[0]) result.wsock = SocketHandle(fds[1]) - proc setEvent*(ev: SelectEvent) = + proc trigger*(ev: SelectEvent) = var data: uint64 = 1 if posix.write(cint(ev.wsock), addr data, sizeof(uint64)) != sizeof(uint64): raiseIOSelectorsError(osLastError()) @@ -279,8 +279,9 @@ proc updateHandle*[T](s: Selector[T], fd: SocketHandle, inc(s.count) pkey.events = events -proc unregister*[T](s: Selector[T], fd: SocketHandle) = +proc unregister*[T](s: Selector[T], fd: SocketHandle|int) = s.withSelectLock(): + let fd = fd.SocketHandle var pkey = s.getKey(fd) if Event.Read in pkey.events: IOFD_CLR(fd, addr s.rSet) @@ -379,6 +380,16 @@ proc flush*[T](s: Selector[T]) = discard template isEmpty*[T](s: Selector[T]): bool = (s.count == 0) +proc contains*[T](s: Selector[T], fd: SocketHandle|int): bool {.inline.} = + s.withSelectLock(): + result = false + + let fdi = int(fd) + for i in 0..<FD_SETSIZE: + if s.fds[i].ident == fdi: + return true + inc(i) + when hasThreadSupport: template withSelectLock[T](s: Selector[T], body: untyped) = acquire(s.lock) @@ -391,15 +402,12 @@ else: template withSelectLock[T](s: Selector[T], body: untyped) = body -proc getData*[T](s: Selector[T], fd: SocketHandle|int): T = +proc getData*[T](s: Selector[T], fd: SocketHandle|int): var T = s.withSelectLock(): let fdi = int(fd) - var i = 0 - while i < FD_SETSIZE: + for i in 0..<FD_SETSIZE: if s.fds[i].ident == fdi: - result = s.fds[i].data - break - inc(i) + return s.fds[i].data proc setData*[T](s: Selector[T], fd: SocketHandle|int, data: T): bool = s.withSelectLock(): @@ -431,16 +439,17 @@ template withData*[T](s: Selector[T], fd: SocketHandle|int, value, body1, body2: untyped) = mixin withSelectLock s.withSelectLock(): - var value: ptr T - let fdi = int(fd) - var i = 0 - while i < FD_SETSIZE: - if s.fds[i].ident == fdi: - value = addr(s.fds[i].data) - break - inc(i) - if i != FD_SETSIZE: - body1 - else: - body2 + block: + var value: ptr T + let fdi = int(fd) + var i = 0 + while i < FD_SETSIZE: + if s.fds[i].ident == fdi: + value = addr(s.fds[i].data) + break + inc(i) + if i != FD_SETSIZE: + body1 + else: + body2 diff --git a/lib/pure/json.nim b/lib/pure/json.nim index fd7a3af03..b5b84863a 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -823,13 +823,13 @@ proc toJson(x: NimNode): NimNode {.compiletime.} = of nnkBracket: # array if x.len == 0: return newCall(bindSym"newJArray") result = newNimNode(nnkBracket) - for i in 0 .. <x.len: + for i in 0 ..< x.len: result.add(toJson(x[i])) result = newCall(bindSym"%", result) of nnkTableConstr: # object if x.len == 0: return newCall(bindSym"newJObject") result = newNimNode(nnkTableConstr) - for i in 0 .. <x.len: + for i in 0 ..< x.len: x[i].expectKind nnkExprColonExpr result.add newTree(nnkExprColonExpr, x[i][0], toJson(x[i][1])) result = newCall(bindSym"%", result) @@ -1303,7 +1303,7 @@ else: case getVarType(x) of JArray: result = newJArray() - for i in 0 .. <x.len: + for i in 0 ..< x.len: result.add(x[i].convertObject()) of JObject: result = newJObject() @@ -1346,6 +1346,16 @@ proc createJsonIndexer(jsonNode: NimNode, indexNode ) +proc transformJsonIndexer(jsonNode: NimNode): NimNode = + case jsonNode.kind + of nnkBracketExpr: + result = newNimNode(nnkCurlyExpr) + else: + result = jsonNode.copy() + + for child in jsonNode: + result.add(transformJsonIndexer(child)) + template verifyJsonKind(node: JsonNode, kinds: set[JsonNodeKind], ast: string) = if node.kind notin kinds: @@ -1449,7 +1459,7 @@ proc processElseBranch(recCaseNode, elseBranch, jsonNode, kindType, # We need to build up a list of conditions from each ``of`` branch so that # we can then negate it to get ``else``. var cond = newIdentNode("false") - for i in 1 .. <len(recCaseNode): + for i in 1 ..< len(recCaseNode): if recCaseNode[i].kind == nnkElse: break @@ -1511,7 +1521,7 @@ proc processObjField(field, jsonNode: NimNode): seq[NimNode] = exprColonExpr.add(getEnumCall) # Iterate through each `of` branch. - for i in 1 .. <field.len: + for i in 1 ..< field.len: case field[i].kind of nnkOfBranch: result.add processOfBranch(field[i], jsonNode, kindType, kindJsonNode) @@ -1524,6 +1534,35 @@ proc processObjField(field, jsonNode: NimNode): seq[NimNode] = doAssert result.len > 0 +proc processFields(obj: NimNode, + jsonNode: NimNode): seq[NimNode] {.compileTime.} = + ## Process all the fields of an ``ObjectTy`` and any of its + ## parent type's fields (via inheritance). + result = @[] + case obj.kind + of nnkObjectTy: + expectKind(obj[2], nnkRecList) + for field in obj[2]: + let nodes = processObjField(field, jsonNode) + result.add(nodes) + + # process parent type fields + case obj[1].kind + of nnkBracketExpr: + assert $obj[1][0] == "ref" + result.add(processFields(getType(obj[1][1]), jsonNode)) + of nnkSym: + result.add(processFields(getType(obj[1]), jsonNode)) + else: + discard + of nnkTupleTy: + for identDefs in obj: + expectKind(identDefs, nnkIdentDefs) + let nodes = processObjField(identDefs[0], jsonNode) + result.add(nodes) + else: + doAssert false, "Unable to process field type: " & $obj.kind + proc processType(typeName: NimNode, obj: NimNode, jsonNode: NimNode, isRef: bool): NimNode {.compileTime.} = ## Process a type such as ``Sym "float"`` or ``ObjectTy ...``. @@ -1533,20 +1572,21 @@ proc processType(typeName: NimNode, obj: NimNode, ## .. code-block::plain ## ObjectTy ## Empty - ## Empty + ## InheritanceInformation ## RecList ## Sym "events" case obj.kind - of nnkObjectTy: + of nnkObjectTy, nnkTupleTy: # Create object constructor. - result = newNimNode(nnkObjConstr) - result.add(typeName) # Name of the type to construct. + result = + if obj.kind == nnkObjectTy: newNimNode(nnkObjConstr) + else: newNimNode(nnkPar) - # Process each object field and add it as an exprColonExpr - expectKind(obj[2], nnkRecList) - for field in obj[2]: - let nodes = processObjField(field, jsonNode) - result.add(nodes) + if obj.kind == nnkObjectTy: + result.add(typeName) # Name of the type to construct. + + # Process each object/tuple field and add it as an exprColonExpr + result.add(processFields(obj, jsonNode)) # Object might be null. So we need to check for that. if isRef: @@ -1569,25 +1609,14 @@ proc processType(typeName: NimNode, obj: NimNode, `getEnumCall` ) of nnkSym: - case ($typeName).normalize - of "float": - result = quote do: - ( - verifyJsonKind(`jsonNode`, {JFloat, JInt}, astToStr(`jsonNode`)); - if `jsonNode`.kind == JFloat: `jsonNode`.fnum else: `jsonNode`.num.float - ) + let name = ($typeName).normalize + case name of "string": result = quote do: ( verifyJsonKind(`jsonNode`, {JString, JNull}, astToStr(`jsonNode`)); if `jsonNode`.kind == JNull: nil else: `jsonNode`.str ) - of "int": - result = quote do: - ( - verifyJsonKind(`jsonNode`, {JInt}, astToStr(`jsonNode`)); - `jsonNode`.num.int - ) of "biggestint": result = quote do: ( @@ -1601,12 +1630,36 @@ proc processType(typeName: NimNode, obj: NimNode, `jsonNode`.bval ) else: - doAssert false, "Unable to process nnkSym " & $typeName + if name.startsWith("int") or name.startsWith("uint"): + result = quote do: + ( + verifyJsonKind(`jsonNode`, {JInt}, astToStr(`jsonNode`)); + `jsonNode`.num.`obj` + ) + elif name.startsWith("float"): + result = quote do: + ( + verifyJsonKind(`jsonNode`, {JInt, JFloat}, astToStr(`jsonNode`)); + if `jsonNode`.kind == JFloat: `jsonNode`.fnum.`obj` else: `jsonNode`.num.`obj` + ) + else: + doAssert false, "Unable to process nnkSym " & $typeName else: doAssert false, "Unable to process type: " & $obj.kind doAssert(not result.isNil(), "processType not initialised.") +import options +proc workaroundMacroNone[T](): Option[T] = + none(T) + +proc depth(n: NimNode, current = 0): int = + result = 1 + for child in n: + let d = 1 + child.depth(current + 1) + if d > result: + result = d + proc createConstructor(typeSym, jsonNode: NimNode): NimNode = ## Accepts a type description, i.e. "ref Type", "seq[Type]", "Type" etc. ## @@ -1616,10 +1669,50 @@ proc createConstructor(typeSym, jsonNode: NimNode): NimNode = # echo("--createConsuctor-- \n", treeRepr(typeSym)) # echo() + if depth(jsonNode) > 150: + error("The `to` macro does not support ref objects with cycles.", jsonNode) + case typeSym.kind of nnkBracketExpr: var bracketName = ($typeSym[0]).normalize case bracketName + of "option": + # TODO: Would be good to verify that this is Option[T] from + # options module I suppose. + let lenientJsonNode = transformJsonIndexer(jsonNode) + + let optionGeneric = typeSym[1] + let value = createConstructor(typeSym[1], jsonNode) + let workaround = bindSym("workaroundMacroNone") # TODO: Nim Bug: This shouldn't be necessary. + + result = quote do: + ( + if `lenientJsonNode`.isNil: `workaround`[`optionGeneric`]() else: some[`optionGeneric`](`value`) + ) + of "table", "orderedtable": + let tableKeyType = typeSym[1] + if ($tableKeyType).cmpIgnoreStyle("string") != 0: + error("JSON doesn't support keys of type " & $tableKeyType) + let tableValueType = typeSym[2] + + let forLoopKey = genSym(nskForVar, "key") + let indexerNode = createJsonIndexer(jsonNode, forLoopKey) + let constructorNode = createConstructor(tableValueType, indexerNode) + + let tableInit = + if bracketName == "table": + bindSym("initTable") + else: + bindSym("initOrderedTable") + + # Create a statement expression containing a for loop. + result = quote do: + ( + var map = `tableInit`[`tableKeyType`, `tableValueType`](); + verifyJsonKind(`jsonNode`, {JObject}, astToStr(`jsonNode`)); + for `forLoopKey` in keys(`jsonNode`.fields): map[`forLoopKey`] = `constructorNode`; + map + ) of "ref": # Ref type. var typeName = $typeSym[1] @@ -1640,7 +1733,7 @@ proc createConstructor(typeSym, jsonNode: NimNode): NimNode = ( var list: `typeSym` = @[]; verifyJsonKind(`jsonNode`, {JArray}, astToStr(`jsonNode`)); - for `forLoopI` in 0 .. <`jsonNode`.len: list.add(`constructorNode`); + for `forLoopI` in 0 ..< `jsonNode`.len: list.add(`constructorNode`); list ) of "array": @@ -1654,7 +1747,7 @@ proc createConstructor(typeSym, jsonNode: NimNode): NimNode = ( var list: `typeSym`; verifyJsonKind(`jsonNode`, {JArray}, astToStr(`jsonNode`)); - for `forLoopI` in 0 .. <`jsonNode`.len: list[`forLoopI`] =`constructorNode`; + for `forLoopI` in 0 ..< `jsonNode`.len: list[`forLoopI`] =`constructorNode`; list ) @@ -1663,12 +1756,23 @@ proc createConstructor(typeSym, jsonNode: NimNode): NimNode = let obj = getType(typeSym) result = processType(typeSym, obj, jsonNode, false) of nnkSym: + # Handle JsonNode. + if ($typeSym).cmpIgnoreStyle("jsonnode") == 0: + return jsonNode + + # Handle all other types. let obj = getType(typeSym) if obj.kind == nnkBracketExpr: # When `Sym "Foo"` turns out to be a `ref object`. result = createConstructor(obj, jsonNode) else: result = processType(typeSym, obj, jsonNode, false) + of nnkTupleTy: + result = processType(typeSym, typeSym, jsonNode, false) + of nnkPar: + # TODO: The fact that `jsonNode` here works to give a good line number + # is weird. Specifying typeSym should work but doesn't. + error("Use a named tuple instead of: " & $toStrLit(typeSym), jsonNode) else: doAssert false, "Unable to create constructor for: " & $typeSym.kind @@ -1683,7 +1787,7 @@ proc postProcessValue(value: NimNode): NimNode = result = postProcess(value) else: result = value - for i in 0 .. <len(result): + for i in 0 ..< len(result): result[i] = postProcessValue(result[i]) proc postProcessExprColonExpr(exprColonExpr, resIdent: NimNode): NimNode = @@ -1796,10 +1900,18 @@ macro to*(node: JsonNode, T: typedesc): untyped = expectKind(typeNode, nnkBracketExpr) doAssert(($typeNode[0]).normalize == "typedesc") - result = createConstructor(typeNode[1], node) + # Create `temp` variable to store the result in case the user calls this + # on `parseJson` (see bug #6604). + result = newNimNode(nnkStmtListExpr) + let temp = genSym(nskLet, "temp") + result.add quote do: + let `temp` = `node` + + let constructor = createConstructor(typeNode[1], temp) # TODO: Rename postProcessValue and move it (?) - result = postProcessValue(result) + result.add(postProcessValue(constructor)) + # echo(treeRepr(result)) # echo(toStrLit(result)) when false: @@ -1849,8 +1961,8 @@ when isMainModule: discard parseJson"""{ invalid""" except: discard - # memory diff should less than 2M - doAssert(abs(getOccupiedMem() - startMemory) < 2 * 1024 * 1024) + # memory diff should less than 4M + doAssert(abs(getOccupiedMem() - startMemory) < 4 * 1024 * 1024) # test `$` diff --git a/lib/pure/lenientops.nim b/lib/pure/lenientops.nim new file mode 100644 index 000000000..b124a9512 --- /dev/null +++ b/lib/pure/lenientops.nim @@ -0,0 +1,60 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2017 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module offers implementations of common binary operations +## like ``+``, ``-``, ``*``, ``/`` and comparison operations, +## which work for mixed float/int operands. +## All operations convert the integer operand into the +## type of the float operand. For numerical expressions, the return +## type is always the type of the float involved in the expresssion, +## i.e., there is no auto conversion from float32 to float64. +## +## Note: In general, auto-converting from int to float loses +## information, which is why these operators live in a separate +## module. Use with care. +## +## Regarding binary comparison, this module only provides unequal operators. +## The equality operator ``==`` is omitted, because depending on the use case +## either casting to float or rounding to int might be preferred, and users +## should make an explicit choice. + +import typetraits + +proc `+`*[I: SomeInteger, F: SomeReal](i: I, f: F): F {.noSideEffect, inline.} = + F(i) + f +proc `+`*[I: SomeInteger, F: SomeReal](f: F, i: I): F {.noSideEffect, inline.} = + f + F(i) + +proc `-`*[I: SomeInteger, F: SomeReal](i: I, f: F): F {.noSideEffect, inline.} = + F(i) - f +proc `-`*[I: SomeInteger, F: SomeReal](f: F, i: I): F {.noSideEffect, inline.} = + f - F(i) + +proc `*`*[I: SomeInteger, F: SomeReal](i: I, f: F): F {.noSideEffect, inline.} = + F(i) * f +proc `*`*[I: SomeInteger, F: SomeReal](f: F, i: I): F {.noSideEffect, inline.} = + f * F(i) + +proc `/`*[I: SomeInteger, F: SomeReal](i: I, f: F): F {.noSideEffect, inline.} = + F(i) / f +proc `/`*[I: SomeInteger, F: SomeReal](f: F, i: I): F {.noSideEffect, inline.} = + f / F(i) + +proc `<`*[I: SomeInteger, F: SomeReal](i: I, f: F): bool {.noSideEffect, inline.} = + F(i) < f +proc `<`*[I: SomeInteger, F: SomeReal](f: F, i: I): bool {.noSideEffect, inline.} = + f < F(i) +proc `<=`*[I: SomeInteger, F: SomeReal](i: I, f: F): bool {.noSideEffect, inline.} = + F(i) <= f +proc `<=`*[I: SomeInteger, F: SomeReal](f: F, i: I): bool {.noSideEffect, inline.} = + f <= F(i) + +# Note that we must not defined `>=` and `>`, because system.nim already has a +# template with signature (x, y: untyped): untyped, which would lead to +# ambigous calls. diff --git a/lib/pure/logging.nim b/lib/pure/logging.nim index e2a5bed96..830820fd1 100644 --- a/lib/pure/logging.nim +++ b/lib/pure/logging.nim @@ -202,13 +202,17 @@ when not defined(js): proc countLogLines(logger: RollingFileLogger): int = result = 0 - for line in logger.file.lines(): + let fp = open(logger.baseName, fmRead) + for line in fp.lines(): result.inc() + fp.close() proc countFiles(filename: string): int = # Example: file.log.1 result = 0 - let (dir, name, ext) = splitFile(filename) + var (dir, name, ext) = splitFile(filename) + if dir == "": + dir = "." for kind, path in walkDir(dir): if kind == pcFile: let llfn = name & ext & ExtSep diff --git a/lib/pure/marshal.nim b/lib/pure/marshal.nim index c4c731acf..6ee830786 100644 --- a/lib/pure/marshal.nim +++ b/lib/pure/marshal.nim @@ -283,7 +283,7 @@ proc to*[T](data: string): T = loadAny(newStringStream(data), toAny(result), tab) when not defined(testing) and isMainModule: - template testit(x: expr) = echo($$to[type(x)]($$x)) + template testit(x: untyped) = echo($$to[type(x)]($$x)) var x: array[0..4, array[0..4, string]] = [ ["test", "1", "2", "3", "4"], ["test", "1", "2", "3", "4"], diff --git a/lib/pure/matchers.nim b/lib/pure/matchers.nim index 7022c21d9..6366fee1a 100644 --- a/lib/pure/matchers.nim +++ b/lib/pure/matchers.nim @@ -48,7 +48,7 @@ proc validEmailAddress*(s: string): bool {.noSideEffect, "aero", "jobs", "museum": return true else: return false -proc parseInt*(s: string, value: var int, validRange: Slice[int]) {. +proc parseInt*(s: string, value: var int, validRange: HSlice[int, int]) {. noSideEffect, rtl, extern: "nmatchParseInt".} = ## parses `s` into an integer in the range `validRange`. If successful, ## `value` is modified to contain the result. Otherwise no exception is diff --git a/lib/pure/math.nim b/lib/pure/math.nim index 8037b31b0..cbd04a145 100644 --- a/lib/pure/math.nim +++ b/lib/pure/math.nim @@ -184,6 +184,8 @@ when not defined(JS): proc pow*(x, y: float32): float32 {.importc: "powf", header: "<math.h>".} proc pow*(x, y: float64): float64 {.importc: "pow", header: "<math.h>".} ## computes x to power raised of y. + ## + ## To compute power between integers, use `^` e.g. 2 ^ 6 proc erf*(x: float32): float32 {.importc: "erff", header: "<math.h>".} proc erf*(x: float64): float64 {.importc: "erf", header: "<math.h>".} @@ -289,6 +291,8 @@ when not defined(JS): ## echo fmod(-2.5, 0.3) ## -0.1 else: + proc trunc*(x: float32): float32 {.importc: "Math.trunc", nodecl.} + proc trunc*(x: float64): float64 {.importc: "Math.trunc", nodecl.} proc floor*(x: float32): float32 {.importc: "Math.floor", nodecl.} proc floor*(x: float64): float64 {.importc: "Math.floor", nodecl.} proc ceil*(x: float32): float32 {.importc: "Math.ceil", nodecl.} @@ -347,15 +351,19 @@ proc round*[T: float32|float64](x: T, places: int = 0): T = result = round0(x*mult)/mult when not defined(JS): - proc frexp*(x: float32, exponent: var int): float32 {. + proc c_frexp*(x: float32, exponent: var int32): float32 {. importc: "frexp", header: "<math.h>".} - proc frexp*(x: float64, exponent: var int): float64 {. + proc c_frexp*(x: float64, exponent: var int32): float64 {. importc: "frexp", header: "<math.h>".} + proc frexp*[T, U](x: T, exponent: var U): T = ## Split a number into mantissa and exponent. ## `frexp` calculates the mantissa m (a float greater than or equal to 0.5 ## and less than 1) and the integer value n such that `x` (the original ## float value) equals m * 2**n. frexp stores n in `exponent` and returns ## m. + var exp: int32 + result = c_frexp(x, exp) + exponent = exp else: proc frexp*[T: float32|float64](x: T, exponent: var int): T = if x == 0.0: @@ -364,9 +372,14 @@ else: elif x < 0.0: result = -frexp(-x, exponent) else: - var ex = floor(log2(x)) - exponent = round(ex) + var ex = trunc(log2(x)) + exponent = int(ex) result = x / pow(2.0, ex) + if abs(result) >= 1: + inc(exponent) + result = result / 2 + if exponent == 1024 and result == 0.0: + result = 0.99999999999999988898 proc splitDecimal*[T: float32|float64](x: T): tuple[intpart: T, floatpart: T] = ## Breaks `x` into an integral and a fractional part. diff --git a/lib/pure/memfiles.nim b/lib/pure/memfiles.nim index 9b2d25267..5c73381ff 100644 --- a/lib/pure/memfiles.nim +++ b/lib/pure/memfiles.nim @@ -257,10 +257,13 @@ proc close*(f: var MemFile) = when defined(windows): if f.wasOpened: error = unmapViewOfFile(f.mem) == 0 - lastErr = osLastError() - error = (closeHandle(f.mapHandle) == 0) or error - if f.fHandle != INVALID_HANDLE_VALUE: - error = (closeHandle(f.fHandle) == 0) or error + if not error: + error = closeHandle(f.mapHandle) == 0 + if not error and f.fHandle != INVALID_HANDLE_VALUE: + discard closeHandle(f.fHandle) + f.fHandle = INVALID_HANDLE_VALUE + if error: + lastErr = osLastError() else: error = munmap(f.mem, f.size) != 0 lastErr = osLastError() diff --git a/lib/pure/mersenne.nim b/lib/pure/mersenne.nim index f18cf5b90..6ac0c4e56 100644 --- a/lib/pure/mersenne.nim +++ b/lib/pure/mersenne.nim @@ -17,7 +17,7 @@ type proc newMersenneTwister*(seed: uint32): MersenneTwister = result.index = 0 result.mt[0] = seed - for i in 1..623'u32: + for i in 1'u32 .. 623'u32: result.mt[i] = (0x6c078965'u32 * (result.mt[i-1] xor (result.mt[i-1] shr 30'u32)) + i) proc generateNumbers(m: var MersenneTwister) = diff --git a/lib/pure/mimetypes.nim b/lib/pure/mimetypes.nim index 1e315afb4..b397ef47b 100644 --- a/lib/pure/mimetypes.nim +++ b/lib/pure/mimetypes.nim @@ -491,6 +491,8 @@ const mimes* = { "vrml": "x-world/x-vrml", "wrl": "x-world/x-vrml"} +from strutils import startsWith + proc newMimetypes*(): MimeDB = ## Creates a new Mimetypes database. The database will contain the most ## common mimetypes. @@ -498,8 +500,11 @@ proc newMimetypes*(): MimeDB = proc getMimetype*(mimedb: MimeDB, ext: string, default = "text/plain"): string = ## Gets mimetype which corresponds to ``ext``. Returns ``default`` if ``ext`` - ## could not be found. - result = mimedb.mimes.getOrDefault(ext) + ## could not be found. ``ext`` can start with an optional dot which is ignored. + if ext.startsWith("."): + result = mimedb.mimes.getOrDefault(ext.substr(1)) + else: + result = mimedb.mimes.getOrDefault(ext) if result == "": return default diff --git a/lib/pure/net.nim b/lib/pure/net.nim index 215a301b6..aad6ab3e8 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -58,7 +58,7 @@ ## You can then begin accepting connections using the ``accept`` procedure. ## ## .. code-block:: Nim -## var client = newSocket() +## var client = new Socket ## var address = "" ## while true: ## socket.acceptAddr(client, address) @@ -145,7 +145,7 @@ type SOBool* = enum ## Boolean socket options. OptAcceptConn, OptBroadcast, OptDebug, OptDontRoute, OptKeepAlive, - OptOOBInline, OptReuseAddr, OptReusePort + OptOOBInline, OptReuseAddr, OptReusePort, OptNoDelay ReadLineResult* = enum ## result for readLineAsync ReadFullLine, ReadPartialLine, ReadDisconnected, ReadNone @@ -857,15 +857,23 @@ proc close*(socket: Socket) = # shutdown i.e not wait for the peers "close notify" alert with a second # call to SSLShutdown let res = SSLShutdown(socket.sslHandle) - SSLFree(socket.sslHandle) - socket.sslHandle = nil if res == 0: discard elif res != 1: socketError(socket, res) finally: + when defineSsl: + if socket.isSSL and socket.sslHandle != nil: + SSLFree(socket.sslHandle) + socket.sslHandle = nil + socket.fd.close() +when defined(posix): + from posix import TCP_NODELAY +else: + from winlean import TCP_NODELAY + proc toCInt*(opt: SOBool): cint = ## Converts a ``SOBool`` into its Socket Option cint representation. case opt @@ -877,6 +885,7 @@ proc toCInt*(opt: SOBool): cint = of OptOOBInline: SO_OOBINLINE of OptReuseAddr: SO_REUSEADDR of OptReusePort: SO_REUSEPORT + of OptNoDelay: TCP_NODELAY proc getSockOpt*(socket: Socket, opt: SOBool, level = SOL_SOCKET): bool {. tags: [ReadIOEffect].} = @@ -899,6 +908,12 @@ proc getPeerAddr*(socket: Socket): (string, Port) = proc setSockOpt*(socket: Socket, opt: SOBool, value: bool, level = SOL_SOCKET) {. tags: [WriteIOEffect].} = ## Sets option ``opt`` to a boolean value specified by ``value``. + ## + ## .. code-block:: Nim + ## var socket = newSocket() + ## socket.setSockOpt(OptReusePort, true) + ## socket.setSockOpt(OptNoDelay, true, level=IPPROTO_TCP.toInt) + ## var valuei = cint(if value: 1 else: 0) setSockOptInt(socket.fd, cint(level), toCInt(opt), valuei) @@ -1120,11 +1135,11 @@ proc recv*(socket: Socket, data: var string, size: int, timeout = -1, ## ## When 0 is returned the socket's connection has been closed. ## - ## This function will throw an EOS exception when an error occurs. A value + ## This function will throw an OSError exception when an error occurs. A value ## lower than 0 is never returned. ## ## A timeout may be specified in milliseconds, if enough data is not received - ## within the time specified an ETimeout exception will be raised. + ## within the time specified an TimeoutError exception will be raised. ## ## **Note**: ``data`` must be initialised. ## diff --git a/lib/pure/numeric.nim b/lib/pure/numeric.nim deleted file mode 100644 index ccda3a146..000000000 --- a/lib/pure/numeric.nim +++ /dev/null @@ -1,87 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2013 Robert Persson -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## **Warning:** This module will be moved out of the stdlib and into a -## Nimble package, don't use it. - -type OneVarFunction* = proc (x: float): float - -{.deprecated: [TOneVarFunction: OneVarFunction].} - -proc brent*(xmin,xmax:float, function:OneVarFunction, tol:float,maxiter=1000): - tuple[rootx, rooty: float, success: bool]= - ## Searches `function` for a root between `xmin` and `xmax` - ## using brents method. If the function value at `xmin`and `xmax` has the - ## same sign, `rootx`/`rooty` is set too the extrema value closest to x-axis - ## and succes is set to false. - ## Otherwise there exists at least one root and success is set to true. - ## This root is searched for at most `maxiter` iterations. - ## If `tol` tolerance is reached within `maxiter` iterations - ## the root refinement stops and success=true. - - # see http://en.wikipedia.org/wiki/Brent%27s_method - var - a=xmin - b=xmax - c=a - d=1.0e308 - fa=function(a) - fb=function(b) - fc=fa - s=0.0 - fs=0.0 - mflag:bool - i=0 - tmp2:float - - if fa*fb>=0: - if abs(fa)<abs(fb): - return (a,fa,false) - else: - return (b,fb,false) - - if abs(fa)<abs(fb): - swap(fa,fb) - swap(a,b) - - while fb!=0.0 and abs(a-b)>tol: - if fa!=fc and fb!=fc: # inverse quadratic interpolation - s = a * fb * fc / (fa - fb) / (fa - fc) + b * fa * fc / (fb - fa) / (fb - fc) + c * fa * fb / (fc - fa) / (fc - fb) - else: #secant rule - s = b - fb * (b - a) / (fb - fa) - tmp2 = (3.0 * a + b) / 4.0 - if not((s > tmp2 and s < b) or (s < tmp2 and s > b)) or - (mflag and abs(s - b) >= (abs(b - c) / 2.0)) or - (not mflag and abs(s - b) >= abs(c - d) / 2.0): - s=(a+b)/2.0 - mflag=true - else: - if (mflag and (abs(b - c) < tol)) or (not mflag and (abs(c - d) < tol)): - s=(a+b)/2.0 - mflag=true - else: - mflag=false - fs = function(s) - d = c - c = b - fc = fb - if fa * fs<0.0: - b=s - fb=fs - else: - a=s - fa=fs - if abs(fa)<abs(fb): - swap(a,b) - swap(fa,fb) - inc i - if i>maxiter: - break - - return (b,fb,true) diff --git a/lib/pure/oids.nim b/lib/pure/oids.nim index 60b53dbe0..427a68964 100644 --- a/lib/pure/oids.nim +++ b/lib/pure/oids.nim @@ -88,7 +88,7 @@ proc generatedTime*(oid: Oid): Time = var tmp: int32 var dummy = oid.time bigEndian32(addr(tmp), addr(dummy)) - result = Time(tmp) + result = fromUnix(tmp) when not defined(testing) and isMainModule: let xo = genOid() diff --git a/lib/pure/os.nim b/lib/pure/os.nim index a1ae4e250..c18d03289 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -173,33 +173,33 @@ proc findExe*(exe: string, followSymlinks: bool = true; return x result = "" -proc getLastModificationTime*(file: string): Time {.rtl, extern: "nos$1".} = +proc getLastModificationTime*(file: string): times.Time {.rtl, extern: "nos$1".} = ## Returns the `file`'s last modification time. when defined(posix): var res: Stat if stat(file, res) < 0'i32: raiseOSError(osLastError()) - return res.st_mtime + return fromUnix(res.st_mtime.int64) else: var f: WIN32_FIND_DATA var h = findFirstFile(file, f) if h == -1'i32: raiseOSError(osLastError()) - result = winTimeToUnixTime(rdFileTime(f.ftLastWriteTime)) + result = fromUnix(winTimeToUnixTime(rdFileTime(f.ftLastWriteTime)).int64) findClose(h) -proc getLastAccessTime*(file: string): Time {.rtl, extern: "nos$1".} = +proc getLastAccessTime*(file: string): times.Time {.rtl, extern: "nos$1".} = ## Returns the `file`'s last read or write access time. when defined(posix): var res: Stat if stat(file, res) < 0'i32: raiseOSError(osLastError()) - return res.st_atime + return fromUnix(res.st_atime.int64) else: var f: WIN32_FIND_DATA var h = findFirstFile(file, f) if h == -1'i32: raiseOSError(osLastError()) - result = winTimeToUnixTime(rdFileTime(f.ftLastAccessTime)) + result = fromUnix(winTimeToUnixTime(rdFileTime(f.ftLastAccessTime)).int64) findClose(h) -proc getCreationTime*(file: string): Time {.rtl, extern: "nos$1".} = +proc getCreationTime*(file: string): times.Time {.rtl, extern: "nos$1".} = ## Returns the `file`'s creation time. ## ## **Note:** Under POSIX OS's, the returned time may actually be the time at @@ -208,12 +208,12 @@ proc getCreationTime*(file: string): Time {.rtl, extern: "nos$1".} = when defined(posix): var res: Stat if stat(file, res) < 0'i32: raiseOSError(osLastError()) - return res.st_ctime + return fromUnix(res.st_ctime.int64) else: var f: WIN32_FIND_DATA var h = findFirstFile(file, f) if h == -1'i32: raiseOSError(osLastError()) - result = winTimeToUnixTime(rdFileTime(f.ftCreationTime)) + result = fromUnix(winTimeToUnixTime(rdFileTime(f.ftCreationTime)).int64) findClose(h) proc fileNewer*(a, b: string): bool {.rtl, extern: "nos$1".} = @@ -630,7 +630,7 @@ proc execShellCmd*(command: string): int {.rtl, extern: "nos$1", ## the process has finished. To execute a program without having a ## shell involved, use the `execProcess` proc of the `osproc` ## module. - when defined(linux): + when defined(posix): result = c_system(command) shr 8 else: result = c_system(command) @@ -816,32 +816,40 @@ iterator walkDir*(dir: string; relative=false): tuple[kind: PathComponent, path: k = getSymlinkFileKind(y) yield (k, y) -iterator walkDirRec*(dir: string, filter={pcFile, pcDir}): string {. - tags: [ReadDirEffect].} = - ## Recursively walks over the directory `dir` and yields for each file in `dir`. - ## The full path for each file is returned. Directories are not returned. +iterator walkDirRec*(dir: string, yieldFilter = {pcFile}, + followFilter = {pcDir}): string {.tags: [ReadDirEffect].} = + ## Recursively walks over the directory `dir` and yields for each file + ## or directory in `dir`. + ## The full path for each file or directory is returned. ## **Warning**: ## Modifying the directory structure while the iterator ## is traversing may result in undefined behavior! ## - ## Walking is recursive. `filter` controls the behaviour of the iterator: + ## Walking is recursive. `filters` controls the behaviour of the iterator: ## ## --------------------- --------------------------------------------- - ## filter meaning + ## yieldFilter meaning ## --------------------- --------------------------------------------- ## ``pcFile`` yield real files ## ``pcLinkToFile`` yield symbolic links to files + ## ``pcDir`` yield real directories + ## ``pcLinkToDir`` yield symbolic links to directories + ## --------------------- --------------------------------------------- + ## + ## --------------------- --------------------------------------------- + ## followFilter meaning + ## --------------------- --------------------------------------------- ## ``pcDir`` follow real directories ## ``pcLinkToDir`` follow symbolic links to directories ## --------------------- --------------------------------------------- ## var stack = @[dir] while stack.len > 0: - for k,p in walkDir(stack.pop()): - if k in filter: - case k - of pcFile, pcLinkToFile: yield p - of pcDir, pcLinkToDir: stack.add(p) + for k, p in walkDir(stack.pop()): + if k in {pcDir, pcLinkToDir} and k in followFilter: + stack.add(p) + if k in yieldFilter: + yield p proc rawRemoveDir(dir: string) = when defined(windows): @@ -1195,14 +1203,15 @@ when defined(nimdoc): ## Returns the number of `command line arguments`:idx: given to the ## application. ## - ## If your binary was called without parameters this will return zero. You - ## can later query each individual paramater with `paramStr() <#paramStr>`_ + ## Unlike `argc`:idx: in C, if your binary was called without parameters this + ## will return zero. + ## You can query each individual paramater with `paramStr() <#paramStr>`_ ## or retrieve all of them in one go with `commandLineParams() ## <#commandLineParams>`_. ## - ## **Availability**: On Posix there is no portable way to get the command - ## line from a DLL and thus the proc isn't defined in this environment. You - ## can test for its availability with `declared() <system.html#declared>`_. + ## **Availability**: When generating a dynamic library (see --app:lib) on + ## Posix this proc is not defined. + ## Test for availability using `declared() <system.html#declared>`_. ## Example: ## ## .. code-block:: nim @@ -1219,13 +1228,14 @@ when defined(nimdoc): ## `paramCount() <#paramCount>`_ with this proc you can call the ## convenience `commandLineParams() <#commandLineParams>`_. ## - ## It is possible to call ``paramStr(0)`` but this will return OS specific + ## Similarly to `argv`:idx: in C, + ## it is possible to call ``paramStr(0)`` but this will return OS specific ## contents (usually the name of the invoked executable). You should avoid ## this and call `getAppFilename() <#getAppFilename>`_ instead. ## - ## **Availability**: On Posix there is no portable way to get the command - ## line from a DLL and thus the proc isn't defined in this environment. You - ## can test for its availability with `declared() <system.html#declared>`_. + ## **Availability**: When generating a dynamic library (see --app:lib) on + ## Posix this proc is not defined. + ## Test for availability using `declared() <system.html#declared>`_. ## Example: ## ## .. code-block:: nim @@ -1441,7 +1451,7 @@ proc sleep*(milsecs: int) {.rtl, extern: "nos$1", tags: [TimeEffect].} = winlean.sleep(int32(milsecs)) else: var a, b: Timespec - a.tv_sec = Time(milsecs div 1000) + a.tv_sec = posix.Time(milsecs div 1000) a.tv_nsec = (milsecs mod 1000) * 1000 * 1000 discard posix.nanosleep(a, b) @@ -1479,16 +1489,17 @@ type size*: BiggestInt # Size of file. permissions*: set[FilePermission] # File permissions linkCount*: BiggestInt # Number of hard links the file object has. - lastAccessTime*: Time # Time file was last accessed. - lastWriteTime*: Time # Time file was last modified/written to. - creationTime*: Time # Time file was created. Not supported on all systems! + lastAccessTime*: times.Time # Time file was last accessed. + lastWriteTime*: times.Time # Time file was last modified/written to. + creationTime*: times.Time # Time file was created. Not supported on all systems! template rawToFormalFileInfo(rawInfo, path, formalInfo): untyped = ## Transforms the native file info structure into the one nim uses. ## 'rawInfo' is either a 'TBY_HANDLE_FILE_INFORMATION' structure on Windows, ## or a 'Stat' structure on posix when defined(Windows): - template toTime(e: FILETIME): untyped {.gensym.} = winTimeToUnixTime(rdFileTime(e)) # local templates default to bind semantics + template toTime(e: FILETIME): untyped {.gensym.} = + fromUnix(winTimeToUnixTime(rdFileTime(e)).int64) # local templates default to bind semantics template merge(a, b): untyped = a or (b shl 32) formalInfo.id.device = rawInfo.dwVolumeSerialNumber formalInfo.id.file = merge(rawInfo.nFileIndexLow, rawInfo.nFileIndexHigh) @@ -1520,9 +1531,9 @@ template rawToFormalFileInfo(rawInfo, path, formalInfo): untyped = formalInfo.id = (rawInfo.st_dev, rawInfo.st_ino) formalInfo.size = rawInfo.st_size formalInfo.linkCount = rawInfo.st_Nlink.BiggestInt - formalInfo.lastAccessTime = rawInfo.st_atime - formalInfo.lastWriteTime = rawInfo.st_mtime - formalInfo.creationTime = rawInfo.st_ctime + formalInfo.lastAccessTime = fromUnix(rawInfo.st_atime.int64) + formalInfo.lastWriteTime = fromUnix(rawInfo.st_mtime.int64) + formalInfo.creationTime = fromUnix(rawInfo.st_ctime.int64) result.permissions = {} checkAndIncludeMode(S_IRUSR, fpUserRead) diff --git a/lib/pure/ospaths.nim b/lib/pure/ospaths.nim index dcb785c83..0d638abb9 100644 --- a/lib/pure/ospaths.nim +++ b/lib/pure/ospaths.nim @@ -558,3 +558,67 @@ proc expandTilde*(path: string): string {. result = getHomeDir() / path.substr(2) else: result = path + +proc quoteShellWindows*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".} = + ## Quote s, so it can be safely passed to Windows API. + ## Based on Python's subprocess.list2cmdline + ## See http://msdn.microsoft.com/en-us/library/17w5ykft.aspx + let needQuote = {' ', '\t'} in s or s.len == 0 + + result = "" + var backslashBuff = "" + if needQuote: + result.add("\"") + + for c in s: + if c == '\\': + backslashBuff.add(c) + elif c == '\"': + result.add(backslashBuff) + result.add(backslashBuff) + backslashBuff.setLen(0) + result.add("\\\"") + else: + if backslashBuff.len != 0: + result.add(backslashBuff) + backslashBuff.setLen(0) + result.add(c) + + if needQuote: + result.add("\"") + +proc quoteShellPosix*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".} = + ## Quote ``s``, so it can be safely passed to POSIX shell. + ## Based on Python's pipes.quote + const safeUnixChars = {'%', '+', '-', '.', '/', '_', ':', '=', '@', + '0'..'9', 'A'..'Z', 'a'..'z'} + if s.len == 0: + return "''" + + let safe = s.allCharsInSet(safeUnixChars) + + if safe: + return s + else: + return "'" & s.replace("'", "'\"'\"'") & "'" + +when defined(windows) or defined(posix): + proc quoteShell*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".} = + ## Quote ``s``, so it can be safely passed to shell. + when defined(windows): + return quoteShellWindows(s) + else: + return quoteShellPosix(s) + +when isMainModule: + assert quoteShellWindows("aaa") == "aaa" + assert quoteShellWindows("aaa\"") == "aaa\\\"" + assert quoteShellWindows("") == "\"\"" + + assert quoteShellPosix("aaa") == "aaa" + assert quoteShellPosix("aaa a") == "'aaa a'" + assert quoteShellPosix("") == "''" + assert quoteShellPosix("a'a") == "'a'\"'\"'a'" + + when defined(posix): + assert quoteShell("") == "''" diff --git a/lib/pure/osproc.nim b/lib/pure/osproc.nim index 07429b9a9..1625845d1 100644 --- a/lib/pure/osproc.nim +++ b/lib/pure/osproc.nim @@ -15,6 +15,9 @@ include "system/inclrtl" import strutils, os, strtabs, streams, cpuinfo +from ospaths import quoteShell, quoteShellWindows, quoteShellPosix +export quoteShell, quoteShellWindows, quoteShellPosix + when defined(windows): import winlean else: @@ -38,10 +41,13 @@ type ## Windows: Named pipes are used so that you can peek ## at the process' output streams. poDemon ## Windows: The program creates no Window. + ## Unix: Start the program as a demon. This is still + ## work in progress! ProcessObj = object of RootObj when defined(windows): fProcessHandle: Handle + fThreadHandle: Handle inHandle, outHandle, errHandle: FileHandle id: Handle else: @@ -49,6 +55,7 @@ type inStream, outStream, errStream: Stream id: Pid exitStatus: cint + exitFlag: bool options: set[ProcessOption] Process* = ref ProcessObj ## represents an operating system process @@ -60,58 +67,6 @@ type const poUseShell* {.deprecated.} = poUsePath ## Deprecated alias for poUsePath. -proc quoteShellWindows*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".} = - ## Quote s, so it can be safely passed to Windows API. - ## Based on Python's subprocess.list2cmdline - ## See http://msdn.microsoft.com/en-us/library/17w5ykft.aspx - let needQuote = {' ', '\t'} in s or s.len == 0 - - result = "" - var backslashBuff = "" - if needQuote: - result.add("\"") - - for c in s: - if c == '\\': - backslashBuff.add(c) - elif c == '\"': - result.add(backslashBuff) - result.add(backslashBuff) - backslashBuff.setLen(0) - result.add("\\\"") - else: - if backslashBuff.len != 0: - result.add(backslashBuff) - backslashBuff.setLen(0) - result.add(c) - - if needQuote: - result.add("\"") - -proc quoteShellPosix*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".} = - ## Quote ``s``, so it can be safely passed to POSIX shell. - ## Based on Python's pipes.quote - const safeUnixChars = {'%', '+', '-', '.', '/', '_', ':', '=', '@', - '0'..'9', 'A'..'Z', 'a'..'z'} - if s.len == 0: - return "''" - - let safe = s.allCharsInSet(safeUnixChars) - - if safe: - return s - else: - return "'" & s.replace("'", "'\"'\"'") & "'" - -proc quoteShell*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".} = - ## Quote ``s``, so it can be safely passed to shell. - when defined(Windows): - return quoteShellWindows(s) - elif defined(posix): - return quoteShellPosix(s) - else: - {.error:"quoteShell is not supported on your system".} - proc execProcess*(command: string, args: openArray[string] = [], env: StringTableRef = nil, @@ -216,8 +171,7 @@ proc waitForExit*(p: Process, timeout: int = -1): int {.rtl, ## On posix, if the process has exited because of a signal, 128 + signal ## number will be returned. - -proc peekExitCode*(p: Process): int {.tags: [].} +proc peekExitCode*(p: Process): int {.rtl, extern: "nosp$1", tags: [].} ## return -1 if the process is still running. Otherwise the process' exit code ## ## On posix, if the process has exited because of a signal, 128 + signal @@ -280,55 +234,98 @@ proc execProcesses*(cmds: openArray[string], ## executes the commands `cmds` in parallel. Creates `n` processes ## that execute in parallel. The highest return value of all processes ## is returned. Runs `beforeRunEvent` before running each command. - when defined(posix): - # poParentStreams causes problems on Posix, so we simply disable it: - var options = options - {poParentStreams} assert n > 0 if n > 1: - var q: seq[Process] - newSeq(q, n) - var m = min(n, cmds.len) - for i in 0..m-1: + var i = 0 + var q = newSeq[Process](n) + + when defined(windows): + var w: WOHandleArray + var m = min(min(n, MAXIMUM_WAIT_OBJECTS), cmds.len) + var wcount = m + else: + var m = min(n, cmds.len) + + while i < m: if beforeRunEvent != nil: beforeRunEvent(i) - q[i] = startProcess(cmds[i], options=options + {poEvalCommand}) - when defined(noBusyWaiting): - var r = 0 - for i in m..high(cmds): - when defined(debugExecProcesses): - var err = "" - var outp = outputStream(q[r]) - while running(q[r]) or not atEnd(outp): - err.add(outp.readLine()) - err.add("\n") - echo(err) - result = max(waitForExit(q[r]), result) - if afterRunEvent != nil: afterRunEvent(r, q[r]) - if q[r] != nil: close(q[r]) - if beforeRunEvent != nil: - beforeRunEvent(i) - q[r] = startProcess(cmds[i], options=options + {poEvalCommand}) - r = (r + 1) mod n - else: - var i = m - while i <= high(cmds): - sleep(50) - for r in 0..n-1: - if not running(q[r]): - #echo(outputStream(q[r]).readLine()) - result = max(waitForExit(q[r]), result) - if afterRunEvent != nil: afterRunEvent(r, q[r]) - if q[r] != nil: close(q[r]) - if beforeRunEvent != nil: - beforeRunEvent(i) - q[r] = startProcess(cmds[i], options=options + {poEvalCommand}) - inc(i) - if i > high(cmds): break - for j in 0..m-1: - result = max(waitForExit(q[j]), result) - if afterRunEvent != nil: afterRunEvent(j, q[j]) - if q[j] != nil: close(q[j]) + q[i] = startProcess(cmds[i], options = options + {poEvalCommand}) + when defined(windows): + w[i] = q[i].fProcessHandle + inc(i) + + var ecount = len(cmds) + while ecount > 0: + var rexit = -1 + when defined(windows): + # waiting for all children, get result if any child exits + var ret = waitForMultipleObjects(int32(wcount), addr(w), 0'i32, + INFINITE) + if ret == WAIT_TIMEOUT: + # must not be happen + discard + elif ret == WAIT_FAILED: + raiseOSError(osLastError()) + else: + var status: int32 + for r in 0..m-1: + if not isNil(q[r]) and q[r].fProcessHandle == w[ret]: + discard getExitCodeProcess(q[r].fProcessHandle, status) + q[r].exitFlag = true + q[r].exitStatus = status + rexit = r + break + else: + var status: cint = 1 + # waiting for all children, get result if any child exits + let res = waitpid(-1, status, 0) + if res > 0: + for r in 0..m-1: + if not isNil(q[r]) and q[r].id == res: + if WIFEXITED(status) or WIFSIGNALED(status): + q[r].exitFlag = true + q[r].exitStatus = status + rexit = r + break + else: + let err = osLastError() + if err == OSErrorCode(ECHILD): + # some child exits, we need to check our childs exit codes + for r in 0..m-1: + if (not isNil(q[r])) and (not running(q[r])): + q[r].exitFlag = true + q[r].exitStatus = status + rexit = r + break + elif err == OSErrorCode(EINTR): + # signal interrupted our syscall, lets repeat it + continue + else: + # all other errors are exceptions + raiseOSError(err) + + if rexit >= 0: + result = max(result, q[rexit].peekExitCode()) + if afterRunEvent != nil: afterRunEvent(rexit, q[rexit]) + close(q[rexit]) + if i < len(cmds): + if beforeRunEvent != nil: beforeRunEvent(i) + q[rexit] = startProcess(cmds[i], + options = options + {poEvalCommand}) + when defined(windows): + w[rexit] = q[rexit].fProcessHandle + inc(i) + else: + when defined(windows): + for k in 0..wcount - 1: + if w[k] == q[rexit].fProcessHandle: + w[k] = w[wcount - 1] + w[wcount - 1] = 0 + dec(wcount) + break + q[rexit] = nil + dec(ecount) else: for i in 0..high(cmds): if beforeRunEvent != nil: @@ -370,6 +367,8 @@ when not defined(useNimRtl): elif not running(p): break close(p) +template streamAccess(p) = + assert poParentStreams notin p.options, "API usage error: stream access not allowed when you use poParentStreams" when defined(Windows) and not defined(useNimRtl): # We need to implement a handle stream for Windows: @@ -513,6 +512,7 @@ when defined(Windows) and not defined(useNimRtl): hi, ho, he: Handle new(result) result.options = options + result.exitFlag = true si.cb = sizeof(si).cint if poParentStreams notin options: si.dwFlags = STARTF_USESTDHANDLES # STARTF_USESHOWWINDOW or @@ -581,28 +581,31 @@ when defined(Windows) and not defined(useNimRtl): "Requested command not found: '$1'. OS error:" % command) else: raiseOSError(lastError, command) - # Close the handle now so anyone waiting is woken: - discard closeHandle(procInfo.hThread) result.fProcessHandle = procInfo.hProcess + result.fThreadHandle = procInfo.hThread result.id = procInfo.dwProcessId + result.exitFlag = false proc close(p: Process) = - if poInteractive in p.options: - # somehow this is not always required on Windows: + if poParentStreams notin p.options: discard closeHandle(p.inHandle) discard closeHandle(p.outHandle) discard closeHandle(p.errHandle) - #discard closeHandle(p.FProcessHandle) + discard closeHandle(p.fThreadHandle) + discard closeHandle(p.fProcessHandle) proc suspend(p: Process) = - discard suspendThread(p.fProcessHandle) + discard suspendThread(p.fThreadHandle) proc resume(p: Process) = - discard resumeThread(p.fProcessHandle) + discard resumeThread(p.fThreadHandle) proc running(p: Process): bool = - var x = waitForSingleObject(p.fProcessHandle, 50) - return x == WAIT_TIMEOUT + if p.exitFlag: + return false + else: + var x = waitForSingleObject(p.fProcessHandle, 0) + return x == WAIT_TIMEOUT proc terminate(p: Process) = if running(p): @@ -612,28 +615,48 @@ when defined(Windows) and not defined(useNimRtl): terminate(p) proc waitForExit(p: Process, timeout: int = -1): int = - discard waitForSingleObject(p.fProcessHandle, timeout.int32) - - var res: int32 - discard getExitCodeProcess(p.fProcessHandle, res) - result = res - discard closeHandle(p.fProcessHandle) + if p.exitFlag: + return p.exitStatus + + let res = waitForSingleObject(p.fProcessHandle, timeout.int32) + if res == WAIT_TIMEOUT: + terminate(p) + var status: int32 + discard getExitCodeProcess(p.fProcessHandle, status) + if status != STILL_ACTIVE: + p.exitFlag = true + p.exitStatus = status + discard closeHandle(p.fThreadHandle) + discard closeHandle(p.fProcessHandle) + result = status + else: + result = -1 proc peekExitCode(p: Process): int = - var b = waitForSingleObject(p.fProcessHandle, 50) == WAIT_TIMEOUT - if b: result = -1 - else: - var res: int32 - discard getExitCodeProcess(p.fProcessHandle, res) - return res + if p.exitFlag: + return p.exitStatus + + result = -1 + var b = waitForSingleObject(p.fProcessHandle, 0) == WAIT_TIMEOUT + if not b: + var status: int32 + discard getExitCodeProcess(p.fProcessHandle, status) + p.exitFlag = true + p.exitStatus = status + discard closeHandle(p.fThreadHandle) + discard closeHandle(p.fProcessHandle) + result = status proc inputStream(p: Process): Stream = + streamAccess(p) result = newFileHandleStream(p.inHandle) proc outputStream(p: Process): Stream = + streamAccess(p) result = newFileHandleStream(p.outHandle) proc errorStream(p: Process): Stream = + streamAccess(p) result = newFileHandleStream(p.errHandle) proc execCmd(command: string): int = @@ -729,9 +752,7 @@ elif not defined(useNimRtl): sysEnv: cstringArray workingDir: cstring pStdin, pStdout, pStderr, pErrorPipe: array[0..1, cint] - optionPoUsePath: bool - optionPoParentStreams: bool - optionPoStdErrToStdOut: bool + options: set[ProcessOption] {.deprecated: [TStartProcessData: StartProcessData].} const useProcessAuxSpawn = declared(posix_spawn) and not defined(useFork) and @@ -756,7 +777,8 @@ elif not defined(useNimRtl): pStdin, pStdout, pStderr: array[0..1, cint] new(result) result.options = options - result.exitStatus = -3 # for ``waitForExit`` + result.exitFlag = true + if poParentStreams notin options: if pipe(pStdin) != 0'i32 or pipe(pStdout) != 0'i32 or pipe(pStderr) != 0'i32: @@ -765,7 +787,9 @@ elif not defined(useNimRtl): var sysCommand: string var sysArgsRaw: seq[string] if poEvalCommand in options: - const useShPath {.strdefine.} = "/bin/sh" + const useShPath {.strdefine.} = + when not defined(android): "/bin/sh" + else: "/system/bin/sh" sysCommand = useShPath sysArgsRaw = @[sysCommand, "-c", command] assert args.len == 0, "`args` has to be empty when using poEvalCommand." @@ -794,10 +818,8 @@ elif not defined(useNimRtl): data.pStdin = pStdin data.pStdout = pStdout data.pStderr = pStderr - data.optionPoParentStreams = poParentStreams in options - data.optionPoUsePath = poUsePath in options - data.optionPoStdErrToStdOut = poStdErrToStdOut in options data.workingDir = workingDir + data.options = options when useProcessAuxSpawn: var currentDir = getCurrentDir() @@ -811,6 +833,7 @@ elif not defined(useNimRtl): if poEchoCmd in options: echo(command, " ", join(args, " ")) result.id = pid + result.exitFlag = false if poParentStreams in options: # does not make much sense, but better than nothing: @@ -837,7 +860,7 @@ elif not defined(useNimRtl): var attr: Tposix_spawnattr var fops: Tposix_spawn_file_actions - template chck(e: expr) = + template chck(e: untyped) = if e != 0'i32: raiseOSError(osLastError()) chck posix_spawn_file_actions_init(fops) @@ -846,19 +869,22 @@ elif not defined(useNimRtl): var mask: Sigset chck sigemptyset(mask) chck posix_spawnattr_setsigmask(attr, mask) - chck posix_spawnattr_setpgroup(attr, 0'i32) + if poDemon in data.options: + chck posix_spawnattr_setpgroup(attr, 0'i32) - chck posix_spawnattr_setflags(attr, POSIX_SPAWN_USEVFORK or - POSIX_SPAWN_SETSIGMASK or - POSIX_SPAWN_SETPGROUP) + var flags = POSIX_SPAWN_USEVFORK or + POSIX_SPAWN_SETSIGMASK + if poDemon in data.options: + flags = flags or POSIX_SPAWN_SETPGROUP + chck posix_spawnattr_setflags(attr, flags) - if not data.optionPoParentStreams: + if not (poParentStreams in data.options): chck posix_spawn_file_actions_addclose(fops, data.pStdin[writeIdx]) chck posix_spawn_file_actions_adddup2(fops, data.pStdin[readIdx], readIdx) chck posix_spawn_file_actions_addclose(fops, data.pStdout[readIdx]) chck posix_spawn_file_actions_adddup2(fops, data.pStdout[writeIdx], writeIdx) chck posix_spawn_file_actions_addclose(fops, data.pStderr[readIdx]) - if data.optionPoStdErrToStdOut: + if (poStdErrToStdOut in data.options): chck posix_spawn_file_actions_adddup2(fops, data.pStdout[writeIdx], 2) else: chck posix_spawn_file_actions_adddup2(fops, data.pStderr[writeIdx], 2) @@ -868,7 +894,7 @@ elif not defined(useNimRtl): setCurrentDir($data.workingDir) var pid: Pid - if data.optionPoUsePath: + if (poUsePath in data.options): res = posix_spawnp(pid, data.sysCommand, fops, attr, data.sysArgs, data.sysEnv) else: res = posix_spawn(pid, data.sysCommand, fops, attr, data.sysArgs, data.sysEnv) @@ -930,7 +956,7 @@ elif not defined(useNimRtl): # Warning: no GC here! # Or anything that touches global structures - all called nim procs # must be marked with stackTrace:off. Inspect C code after making changes. - if not data.optionPoParentStreams: + if not (poParentStreams in data.options): discard close(data.pStdin[writeIdx]) if dup2(data.pStdin[readIdx], readIdx) < 0: startProcessFail(data) @@ -938,7 +964,7 @@ elif not defined(useNimRtl): if dup2(data.pStdout[writeIdx], writeIdx) < 0: startProcessFail(data) discard close(data.pStderr[readIdx]) - if data.optionPoStdErrToStdOut: + if (poStdErrToStdOut in data.options): if dup2(data.pStdout[writeIdx], 2) < 0: startProcessFail(data) else: @@ -952,7 +978,7 @@ elif not defined(useNimRtl): discard close(data.pErrorPipe[readIdx]) discard fcntl(data.pErrorPipe[writeIdx], F_SETFD, FD_CLOEXEC) - if data.optionPoUsePath: + if (poUsePath in data.options): when defined(uClibc) or defined(linux): # uClibc environment (OpenWrt included) doesn't have the full execvpe let exe = findExe(data.sysCommand) @@ -984,19 +1010,22 @@ elif not defined(useNimRtl): if kill(p.id, SIGCONT) != 0'i32: raiseOsError(osLastError()) proc running(p: Process): bool = - var ret : int - var status : cint = 1 - ret = waitpid(p.id, status, WNOHANG) - if ret == int(p.id): - if isExitStatus(status): - p.exitStatus = status - return false - else: - return true - elif ret == 0: - return true # Can't establish status. Assume running. - else: + if p.exitFlag: return false + else: + var status: cint = 1 + let ret = waitpid(p.id, status, WNOHANG) + if ret == int(p.id): + if isExitStatus(status): + p.exitFlag = true + p.exitStatus = status + return false + else: + return true + elif ret == 0: + return true # Can't establish status. Assume running. + else: + raiseOSError(osLastError()) proc terminate(p: Process) = if kill(p.id, SIGTERM) != 0'i32: @@ -1011,13 +1040,14 @@ elif not defined(useNimRtl): import kqueue, times proc waitForExit(p: Process, timeout: int = -1): int = - if p.exitStatus != -3: + if p.exitFlag: return exitStatus(p.exitStatus) if timeout == -1: - var status : cint = 1 + var status: cint = 1 if waitpid(p.id, status, 0) < 0: raiseOSError(osLastError()) + p.exitFlag = true p.exitStatus = status else: var kqFD = kqueue() @@ -1030,15 +1060,15 @@ elif not defined(useNimRtl): var tmspec: Timespec if timeout >= 1000: - tmspec.tv_sec = (timeout div 1_000).Time + tmspec.tv_sec = posix.Time(timeout div 1_000) tmspec.tv_nsec = (timeout %% 1_000) * 1_000_000 else: - tmspec.tv_sec = 0.Time + tmspec.tv_sec = posix.Time(0) tmspec.tv_nsec = (timeout * 1_000_000) try: while true: - var status : cint = 1 + var status: cint = 1 var count = kevent(kqFD, addr(kevIn), 1, addr(kevOut), 1, addr(tmspec)) if count < 0: @@ -1051,12 +1081,14 @@ elif not defined(useNimRtl): raiseOSError(osLastError()) if waitpid(p.id, status, 0) < 0: raiseOSError(osLastError()) + p.exitFlag = true p.exitStatus = status break else: if kevOut.ident == p.id.uint and kevOut.filter == EVFILT_PROC: if waitpid(p.id, status, 0) < 0: raiseOSError(osLastError()) + p.exitFlag = true p.exitStatus = status break else: @@ -1077,36 +1109,33 @@ elif not defined(useNimRtl): var b: Timespec b.tv_sec = e.tv_sec b.tv_nsec = e.tv_nsec - e.tv_sec = (e.tv_sec - s.tv_sec).Time + e.tv_sec = e.tv_sec - s.tv_sec if e.tv_nsec >= s.tv_nsec: e.tv_nsec -= s.tv_nsec else: - if e.tv_sec == 0.Time: + if e.tv_sec == posix.Time(0): raise newException(ValueError, "System time was modified") else: diff = s.tv_nsec - e.tv_nsec e.tv_nsec = 1_000_000_000 - diff - t.tv_sec = (t.tv_sec - e.tv_sec).Time + t.tv_sec = t.tv_sec - e.tv_sec if t.tv_nsec >= e.tv_nsec: t.tv_nsec -= e.tv_nsec else: - t.tv_sec = (int(t.tv_sec) - 1).Time + t.tv_sec = t.tv_sec - posix.Time(1) diff = e.tv_nsec - t.tv_nsec t.tv_nsec = 1_000_000_000 - diff s.tv_sec = b.tv_sec s.tv_nsec = b.tv_nsec - #if waitPid(p.id, p.exitStatus, 0) == int(p.id): - # ``waitPid`` fails if the process is not running anymore. But then - # ``running`` probably set ``p.exitStatus`` for us. Since ``p.exitStatus`` is - # initialized with -3, wrong success exit codes are prevented. - if p.exitStatus != -3: + if p.exitFlag: return exitStatus(p.exitStatus) if timeout == -1: - var status : cint = 1 + var status: cint = 1 if waitpid(p.id, status, 0) < 0: raiseOSError(osLastError()) + p.exitFlag = true p.exitStatus = status else: var nmask, omask: Sigset @@ -1125,10 +1154,10 @@ elif not defined(useNimRtl): raiseOSError(osLastError()) if timeout >= 1000: - tmspec.tv_sec = (timeout div 1_000).Time + tmspec.tv_sec = posix.Time(timeout div 1_000) tmspec.tv_nsec = (timeout %% 1_000) * 1_000_000 else: - tmspec.tv_sec = 0.Time + tmspec.tv_sec = posix.Time(0) tmspec.tv_nsec = (timeout * 1_000_000) try: @@ -1138,9 +1167,10 @@ elif not defined(useNimRtl): let res = sigtimedwait(nmask, sinfo, tmspec) if res == SIGCHLD: if sinfo.si_pid == p.id: - var status : cint = 1 + var status: cint = 1 if waitpid(p.id, status, 0) < 0: raiseOSError(osLastError()) + p.exitFlag = true p.exitStatus = status break else: @@ -1161,9 +1191,10 @@ elif not defined(useNimRtl): # timeout expired, so we trying to kill process if posix.kill(p.id, SIGKILL) == -1: raiseOSError(osLastError()) - var status : cint = 1 + var status: cint = 1 if waitpid(p.id, status, 0) < 0: raiseOSError(osLastError()) + p.exitFlag = true p.exitStatus = status break else: @@ -1181,12 +1212,13 @@ elif not defined(useNimRtl): proc peekExitCode(p: Process): int = var status = cint(0) result = -1 - if p.exitStatus != -3: + if p.exitFlag: return exitStatus(p.exitStatus) var ret = waitpid(p.id, status, WNOHANG) if ret > 0: if isExitStatus(status): + p.exitFlag = true p.exitStatus = status result = exitStatus(status) @@ -1197,16 +1229,19 @@ elif not defined(useNimRtl): stream = newFileStream(f) proc inputStream(p: Process): Stream = + streamAccess(p) if p.inStream == nil: createStream(p.inStream, p.inHandle, fmWrite) return p.inStream proc outputStream(p: Process): Stream = + streamAccess(p) if p.outStream == nil: createStream(p.outStream, p.outHandle, fmRead) return p.outStream proc errorStream(p: Process): Stream = + streamAccess(p) if p.errStream == nil: createStream(p.errStream, p.errHandle, fmRead) return p.errStream @@ -1287,16 +1322,3 @@ proc execCmdEx*(command: string, options: set[ProcessOption] = { result[1] = peekExitCode(p) if result[1] != -1: break close(p) - -when isMainModule: - assert quoteShellWindows("aaa") == "aaa" - assert quoteShellWindows("aaa\"") == "aaa\\\"" - assert quoteShellWindows("") == "\"\"" - - assert quoteShellPosix("aaa") == "aaa" - assert quoteShellPosix("aaa a") == "'aaa a'" - assert quoteShellPosix("") == "''" - assert quoteShellPosix("a'a") == "'a'\"'\"'a'" - - when defined(posix): - assert quoteShell("") == "''" diff --git a/lib/pure/parsecsv.nim b/lib/pure/parsecsv.nim index 77b145a73..071858b7c 100644 --- a/lib/pure/parsecsv.nim +++ b/lib/pure/parsecsv.nim @@ -32,7 +32,7 @@ ## import parsecsv ## import os ## # Prepare a file -## var csv_content = """One,Two,Three,Four +## var content = """One,Two,Three,Four ## 1,2,3,4 ## 10,20,30,40 ## 100,200,300,400 @@ -72,7 +72,10 @@ proc raiseEInvalidCsv(filename: string, line, col: int, msg: string) {.noreturn.} = var e: ref CsvError new(e) - e.msg = filename & "(" & $line & ", " & $col & ") Error: " & msg + if filename.len == 0: + e.msg = "Error: " & msg + else: + e.msg = filename & "(" & $line & ", " & $col & ") Error: " & msg raise e proc error(my: CsvParser, pos: int, msg: string) = diff --git a/lib/pure/parseopt2.nim b/lib/pure/parseopt2.nim index 2e8dbe140..a2ff9bf0c 100644 --- a/lib/pure/parseopt2.nim +++ b/lib/pure/parseopt2.nim @@ -35,7 +35,7 @@ type cmd: seq[string] pos: int remainingShortOptions: string - kind*: CmdLineKind ## the dected command line token + kind*: CmdLineKind ## the detected command line token key*, val*: TaintedString ## key and value pair; ``key`` is the option ## or the argument, ``value`` is not "" if ## the option was given a value diff --git a/lib/pure/parsesql.nim b/lib/pure/parsesql.nim index 00d007d01..ae192ab9a 100644 --- a/lib/pure/parsesql.nim +++ b/lib/pure/parsesql.nim @@ -55,6 +55,13 @@ const ";", ":", ",", "(", ")", "[", "]", "." ] + reservedKeywords = @[ + # statements + "select", "from", "where", "group", "limit", "having", + # functions + "count", + ] + proc open(L: var SqlLexer, input: Stream, filename: string) = lexbase.open(L, input) L.filename = filename @@ -274,16 +281,16 @@ proc getSymbol(c: var SqlLexer, tok: var Token) = c.bufpos = pos tok.kind = tkIdentifier -proc getQuotedIdentifier(c: var SqlLexer, tok: var Token) = +proc getQuotedIdentifier(c: var SqlLexer, tok: var Token, quote='\"') = var pos = c.bufpos + 1 var buf = c.buf tok.kind = tkQuotedIdentifier while true: var ch = buf[pos] - if ch == '\"': - if buf[pos+1] == '\"': + if ch == quote: + if buf[pos+1] == quote: inc(pos, 2) - add(tok.literal, '\"') + add(tok.literal, quote) else: inc(pos) break @@ -442,7 +449,8 @@ proc getTok(c: var SqlLexer, tok: var Token) = add(tok.literal, '.') of '0'..'9': getNumeric(c, tok) of '\'': getString(c, tok, tkStringConstant) - of '"': getQuotedIdentifier(c, tok) + of '"': getQuotedIdentifier(c, tok, '"') + of '`': getQuotedIdentifier(c, tok, '`') of lexbase.EndOfFile: tok.kind = tkEof tok.literal = "[EOF]" @@ -450,7 +458,7 @@ proc getTok(c: var SqlLexer, tok: var Token) = '\128'..'\255': getSymbol(c, tok) of '+', '-', '*', '/', '<', '>', '=', '~', '!', '@', '#', '%', - '^', '&', '|', '`', '?': + '^', '&', '|', '?': getOperator(c, tok) else: add(tok.literal, c.buf[c.bufpos]) @@ -462,27 +470,27 @@ proc errorStr(L: SqlLexer, msg: string): string = # ----------------------------- parser ---------------------------------------- -# Operator/Element Associativity Description -# . left table/column name separator -# :: left PostgreSQL-style typecast -# [ ] left array element selection -# - right unary minus -# ^ left exponentiation -# * / % left multiplication, division, modulo -# + - left addition, subtraction -# IS IS TRUE, IS FALSE, IS UNKNOWN, IS NULL -# ISNULL test for null -# NOTNULL test for not null -# (any other) left all other native and user-defined oprs -# IN set membership -# BETWEEN range containment -# OVERLAPS time interval overlap -# LIKE ILIKE SIMILAR string pattern matching -# < > less than, greater than -# = right equality, assignment -# NOT right logical negation -# AND left logical conjunction -# OR left logical disjunction +# Operator/Element Associativity Description +# . left table/column name separator +# :: left PostgreSQL-style typecast +# [ ] left array element selection +# - right unary minus +# ^ left exponentiation +# * / % left multiplication, division, modulo +# + - left addition, subtraction +# IS IS TRUE, IS FALSE, IS UNKNOWN, IS NULL +# ISNULL test for null +# NOTNULL test for not null +# (any other) left all other native and user-defined oprs +# IN set membership +# BETWEEN range containment +# OVERLAPS time interval overlap +# LIKE ILIKE SIMILAR string pattern matching +# < > less than, greater than +# = right equality, assignment +# NOT right logical negation +# AND left logical conjunction +# OR left logical disjunction type SqlNodeKind* = enum ## kind of SQL abstract syntax tree @@ -504,6 +512,7 @@ type nkPrefix, nkInfix, nkCall, + nkPrGroup, nkColumnReference, nkReferences, nkDefault, @@ -518,11 +527,15 @@ type nkSelect, nkSelectDistinct, nkSelectColumns, + nkSelectPair, nkAsgn, nkFrom, + nkFromItemPair, nkGroup, + nkLimit, nkHaving, nkOrder, + nkJoin, nkDesc, nkUnion, nkIntersect, @@ -658,10 +671,12 @@ proc getPrecedence(p: SqlParser): int = elif isOpr(p, "=") or isOpr(p, "<") or isOpr(p, ">") or isOpr(p, ">=") or isOpr(p, "<=") or isOpr(p, "<>") or isOpr(p, "!=") or isKeyw(p, "is") or isKeyw(p, "like"): - result = 3 + result = 4 elif isKeyw(p, "and"): - result = 2 + result = 3 elif isKeyw(p, "or"): + result = 2 + elif isKeyw(p, "between"): result = 1 elif p.tok.kind == tkOperator: # user-defined operator: @@ -670,6 +685,7 @@ proc getPrecedence(p: SqlParser): int = result = - 1 proc parseExpr(p: var SqlParser): SqlNode +proc parseSelect(p: var SqlParser): SqlNode proc identOrLiteral(p: var SqlParser): SqlNode = case p.tok.kind @@ -693,7 +709,8 @@ proc identOrLiteral(p: var SqlParser): SqlNode = getTok(p) of tkParLe: getTok(p) - result = parseExpr(p) + result = newNode(nkPrGroup) + result.add(parseExpr(p)) eat(p, tkParRi) else: sqlError(p, "expression expected") @@ -745,7 +762,7 @@ proc lowestExprAux(p: var SqlParser, v: var SqlNode, limit: int): int = result = opPred while opPred > limit: node = newNode(nkInfix) - opNode = newNode(nkIdent, p.tok.literal) + opNode = newNode(nkIdent, p.tok.literal.toLower()) getTok(p) result = lowestExprAux(p, v2, opPred) node.add(opNode) @@ -921,6 +938,19 @@ proc parseWhere(p: var SqlParser): SqlNode = result = newNode(nkWhere) result.add(parseExpr(p)) +proc parseFromItem(p: var SqlParser): SqlNode = + result = newNode(nkFromItemPair) + if p.tok.kind == tkParLe: + getTok(p) + var select = parseSelect(p) + result.add(select) + eat(p, tkParRi) + else: + result.add(parseExpr(p)) + if isKeyw(p, "as"): + getTok(p) + result.add(parseExpr(p)) + proc parseIndexDef(p: var SqlParser): SqlNode = result = parseIfNotExists(p, nkCreateIndex) if isKeyw(p, "primary"): @@ -956,6 +986,7 @@ proc parseInsert(p: var SqlParser): SqlNode = if p.tok.kind == tkParLe: var n = newNode(nkColumnList) parseParIdentList(p, n) + result.add n else: result.add(nil) if isKeyw(p, "default"): @@ -996,6 +1027,8 @@ proc parseUpdate(p: var SqlParser): SqlNode = proc parseDelete(p: var SqlParser): SqlNode = getTok(p) + if isOpr(p, "*"): + getTok(p) result = newNode(nkDelete) eat(p, "from") result.add(primary(p)) @@ -1018,7 +1051,12 @@ proc parseSelect(p: var SqlParser): SqlNode = a.add(newNode(nkIdent, "*")) getTok(p) else: - a.add(parseExpr(p)) + var pair = newNode(nkSelectPair) + pair.add(parseExpr(p)) + a.add(pair) + if isKeyw(p, "as"): + getTok(p) + pair.add(parseExpr(p)) if p.tok.kind != tkComma: break getTok(p) result.add(a) @@ -1026,7 +1064,7 @@ proc parseSelect(p: var SqlParser): SqlNode = var f = newNode(nkFrom) while true: getTok(p) - f.add(parseExpr(p)) + f.add(parseFromItem(p)) if p.tok.kind != tkComma: break result.add(f) if isKeyw(p, "where"): @@ -1040,6 +1078,11 @@ proc parseSelect(p: var SqlParser): SqlNode = if p.tok.kind != tkComma: break getTok(p) result.add(g) + if isKeyw(p, "limit"): + getTok(p) + var l = newNode(nkLimit) + l.add(parseExpr(p)) + result.add(l) if isKeyw(p, "having"): var h = newNode(nkHaving) while true: @@ -1072,6 +1115,19 @@ proc parseSelect(p: var SqlParser): SqlNode = if p.tok.kind != tkComma: break getTok(p) result.add(n) + if isKeyw(p, "join") or isKeyw(p, "inner") or isKeyw(p, "outer") or isKeyw(p, "cross"): + var join = newNode(nkJoin) + result.add(join) + if isKeyw(p, "join"): + join.add(newNode(nkIdent, "")) + getTok(p) + else: + join.add(newNode(nkIdent, p.tok.literal.toLower())) + getTok(p) + eat(p, "join") + join.add(parseFromItem(p)) + eat(p, "on") + join.add(parseExpr(p)) proc parseStmt(p: var SqlParser; parent: SqlNode) = if isKeyw(p, "create"): @@ -1103,7 +1159,7 @@ proc parseStmt(p: var SqlParser; parent: SqlNode) = elif isKeyw(p, "begin"): getTok(p) else: - sqlError(p, "CREATE expected") + sqlError(p, "SELECT, CREATE, UPDATE or DELETE expected") proc open(p: var SqlParser, input: Stream, filename: string) = ## opens the parser `p` and assigns the input stream `input` to it. @@ -1115,13 +1171,13 @@ proc open(p: var SqlParser, input: Stream, filename: string) = proc parse(p: var SqlParser): SqlNode = ## parses the content of `p`'s input stream and returns the SQL AST. - ## Syntax errors raise an `EInvalidSql` exception. + ## Syntax errors raise an `SqlParseError` exception. result = newNode(nkStmtList) while p.tok.kind != tkEof: parseStmt(p, result) + if p.tok.kind == tkEof: + break eat(p, tkSemicolon) - if result.len == 1: - result = result.sons[0] proc close(p: var SqlParser) = ## closes the parser `p`. The associated input stream is closed too. @@ -1130,7 +1186,7 @@ proc close(p: var SqlParser) = proc parseSQL*(input: Stream, filename: string): SqlNode = ## parses the SQL from `input` into an AST and returns the AST. ## `filename` is only used for error messages. - ## Syntax errors raise an `EInvalidSql` exception. + ## Syntax errors raise an `SqlParseError` exception. var p: SqlParser open(p, input, filename) try: @@ -1138,29 +1194,74 @@ proc parseSQL*(input: Stream, filename: string): SqlNode = finally: close(p) -proc ra(n: SqlNode, s: var string, indent: int) +proc parseSQL*(input: string, filename=""): SqlNode = + ## parses the SQL from `input` into an AST and returns the AST. + ## `filename` is only used for error messages. + ## Syntax errors raise an `SqlParseError` exception. + parseSQL(newStringStream(input), "") + + +type + SqlWriter = object + indent: int + upperCase: bool + buffer: string + +proc add(s: var SqlWriter, thing: char) = + s.buffer.add(thing) + +proc add(s: var SqlWriter, thing: string) = + if s.buffer.len > 0 and s.buffer[^1] notin {' ', '\L', '(', '.'}: + s.buffer.add(" ") + s.buffer.add(thing) + +proc addKeyw(s: var SqlWriter, thing: string) = + var keyw = thing + if s.upperCase: + keyw = keyw.toUpper() + s.add(keyw) + +proc addIden(s: var SqlWriter, thing: string) = + var iden = thing + if iden.toLower() in reservedKeywords: + iden = '"' & iden & '"' + s.add(iden) + +proc ra(n: SqlNode, s: var SqlWriter) + +proc rs(n: SqlNode, s: var SqlWriter, prefix = "(", suffix = ")", sep = ", ") = + if n.len > 0: + s.add(prefix) + for i in 0 .. n.len-1: + if i > 0: s.add(sep) + ra(n.sons[i], s) + s.add(suffix) + +proc addMulti(s: var SqlWriter, n: SqlNode, sep = ',') = + if n.len > 0: + for i in 0 .. n.len-1: + if i > 0: s.add(sep) + ra(n.sons[i], s) -proc rs(n: SqlNode, s: var string, indent: int, - prefix = "(", suffix = ")", - sep = ", ") = +proc addMulti(s: var SqlWriter, n: SqlNode, sep = ',', prefix, suffix: char) = if n.len > 0: s.add(prefix) for i in 0 .. n.len-1: if i > 0: s.add(sep) - ra(n.sons[i], s, indent) + ra(n.sons[i], s) s.add(suffix) -proc ra(n: SqlNode, s: var string, indent: int) = +proc ra(n: SqlNode, s: var SqlWriter) = if n == nil: return case n.kind of nkNone: discard of nkIdent: - if allCharsInSet(n.strVal, {'\33'..'\127'}): + if allCharsInSet(n.strVal, {'\33'..'\127'}) and n.strVal.toLower() notin reservedKeywords: s.add(n.strVal) else: s.add("\"" & replace(n.strVal, "\"", "\"\"") & "\"") of nkStringLit: - s.add(escape(n.strVal, "e'", "'")) + s.add(escape(n.strVal, "'", "'")) of nkBitStringLit: s.add("b'" & n.strVal & "'") of nkHexStringLit: @@ -1168,217 +1269,206 @@ proc ra(n: SqlNode, s: var string, indent: int) = of nkIntegerLit, nkNumericLit: s.add(n.strVal) of nkPrimaryKey: - s.add(" primary key") - rs(n, s, indent) + s.addKeyw("primary key") + rs(n, s) of nkForeignKey: - s.add(" foreign key") - rs(n, s, indent) + s.addKeyw("foreign key") + rs(n, s) of nkNotNull: - s.add(" not null") + s.addKeyw("not null") of nkNull: - s.add(" null") + s.addKeyw("null") of nkDot: - ra(n.sons[0], s, indent) - s.add(".") - ra(n.sons[1], s, indent) + ra(n.sons[0], s) + s.add('.') + ra(n.sons[1], s) of nkDotDot: - ra(n.sons[0], s, indent) + ra(n.sons[0], s) s.add(". .") - ra(n.sons[1], s, indent) + ra(n.sons[1], s) of nkPrefix: - s.add('(') - ra(n.sons[0], s, indent) + ra(n.sons[0], s) s.add(' ') - ra(n.sons[1], s, indent) - s.add(')') + ra(n.sons[1], s) of nkInfix: - s.add('(') - ra(n.sons[1], s, indent) + ra(n.sons[1], s) s.add(' ') - ra(n.sons[0], s, indent) + ra(n.sons[0], s) s.add(' ') - ra(n.sons[2], s, indent) - s.add(')') + ra(n.sons[2], s) of nkCall, nkColumnReference: - ra(n.sons[0], s, indent) + ra(n.sons[0], s) s.add('(') for i in 1..n.len-1: - if i > 1: s.add(", ") - ra(n.sons[i], s, indent) + if i > 1: s.add(',') + ra(n.sons[i], s) + s.add(')') + of nkPrGroup: + s.add('(') + s.addMulti(n) s.add(')') of nkReferences: - s.add(" references ") - ra(n.sons[0], s, indent) + s.addKeyw("references") + ra(n.sons[0], s) of nkDefault: - s.add(" default ") - ra(n.sons[0], s, indent) + s.addKeyw("default") + ra(n.sons[0], s) of nkCheck: - s.add(" check ") - ra(n.sons[0], s, indent) + s.addKeyw("check") + ra(n.sons[0], s) of nkConstraint: - s.add(" constraint ") - ra(n.sons[0], s, indent) - s.add(" check ") - ra(n.sons[1], s, indent) + s.addKeyw("constraint") + ra(n.sons[0], s) + s.addKeyw("check") + ra(n.sons[1], s) of nkUnique: - s.add(" unique") - rs(n, s, indent) + s.addKeyw("unique") + rs(n, s) of nkIdentity: - s.add(" identity") + s.addKeyw("identity") of nkColumnDef: - s.add("\n ") - rs(n, s, indent, "", "", " ") + rs(n, s, "", "", " ") of nkStmtList: for i in 0..n.len-1: - ra(n.sons[i], s, indent) - s.add("\n") + ra(n.sons[i], s) + s.add(';') of nkInsert: assert n.len == 3 - s.add("insert into ") - ra(n.sons[0], s, indent) - ra(n.sons[1], s, indent) + s.addKeyw("insert into") + ra(n.sons[0], s) + s.add(' ') + ra(n.sons[1], s) if n.sons[2].kind == nkDefault: - s.add("default values") + s.addKeyw("default values") else: - s.add("\nvalues ") - ra(n.sons[2], s, indent) - s.add(';') + ra(n.sons[2], s) of nkUpdate: - s.add("update ") - ra(n.sons[0], s, indent) - s.add(" set ") + s.addKeyw("update") + ra(n.sons[0], s) + s.addKeyw("set") var L = n.len for i in 1 .. L-2: if i > 1: s.add(", ") var it = n.sons[i] assert it.kind == nkAsgn - ra(it, s, indent) - ra(n.sons[L-1], s, indent) - s.add(';') + ra(it, s) + ra(n.sons[L-1], s) of nkDelete: - s.add("delete from ") - ra(n.sons[0], s, indent) - ra(n.sons[1], s, indent) - s.add(';') + s.addKeyw("delete from") + ra(n.sons[0], s) + ra(n.sons[1], s) of nkSelect, nkSelectDistinct: - s.add("select ") + s.addKeyw("select") if n.kind == nkSelectDistinct: - s.add("distinct ") - rs(n.sons[0], s, indent, "", "", ", ") - for i in 1 .. n.len-1: ra(n.sons[i], s, indent) - s.add(';') + s.addKeyw("distinct") + s.addMulti(n.sons[0]) + for i in 1 .. n.len-1: + ra(n.sons[i], s) of nkSelectColumns: assert(false) + of nkSelectPair: + ra(n.sons[0], s) + if n.sons.len == 2: + s.addKeyw("as") + ra(n.sons[1], s) + of nkFromItemPair: + if n.sons[0].kind == nkIdent: + ra(n.sons[0], s) + else: + assert n.sons[0].kind == nkSelect + s.add('(') + ra(n.sons[0], s) + s.add(')') + if n.sons.len == 2: + s.addKeyw("as") + ra(n.sons[1], s) of nkAsgn: - ra(n.sons[0], s, indent) + ra(n.sons[0], s) s.add(" = ") - ra(n.sons[1], s, indent) + ra(n.sons[1], s) of nkFrom: - s.add("\nfrom ") - rs(n, s, indent, "", "", ", ") + s.addKeyw("from") + s.addMulti(n) of nkGroup: - s.add("\ngroup by") - rs(n, s, indent, "", "", ", ") + s.addKeyw("group by") + s.addMulti(n) + of nkLimit: + s.addKeyw("limit") + s.addMulti(n) of nkHaving: - s.add("\nhaving") - rs(n, s, indent, "", "", ", ") + s.addKeyw("having") + s.addMulti(n) of nkOrder: - s.add("\norder by ") - rs(n, s, indent, "", "", ", ") + s.addKeyw("order by") + s.addMulti(n) + of nkJoin: + var joinType = n.sons[0].strVal + if joinType == "": + joinType = "join" + else: + joinType &= " " & "join" + s.addKeyw(joinType) + ra(n.sons[1], s) + s.addKeyw("on") + ra(n.sons[2], s) of nkDesc: - ra(n.sons[0], s, indent) - s.add(" desc") + ra(n.sons[0], s) + s.addKeyw("desc") of nkUnion: - s.add(" union") + s.addKeyw("union") of nkIntersect: - s.add(" intersect") + s.addKeyw("intersect") of nkExcept: - s.add(" except") + s.addKeyw("except") of nkColumnList: - rs(n, s, indent) + rs(n, s) of nkValueList: - s.add("values ") - rs(n, s, indent) + s.addKeyw("values") + rs(n, s) of nkWhere: - s.add("\nwhere ") - ra(n.sons[0], s, indent) + s.addKeyw("where") + ra(n.sons[0], s) of nkCreateTable, nkCreateTableIfNotExists: - s.add("create table ") + s.addKeyw("create table") if n.kind == nkCreateTableIfNotExists: - s.add("if not exists ") - ra(n.sons[0], s, indent) + s.addKeyw("if not exists") + ra(n.sons[0], s) s.add('(') for i in 1..n.len-1: - if i > 1: s.add(", ") - ra(n.sons[i], s, indent) + if i > 1: s.add(',') + ra(n.sons[i], s) s.add(");") of nkCreateType, nkCreateTypeIfNotExists: - s.add("create type ") + s.addKeyw("create type") if n.kind == nkCreateTypeIfNotExists: - s.add("if not exists ") - ra(n.sons[0], s, indent) - s.add(" as ") - ra(n.sons[1], s, indent) - s.add(';') + s.addKeyw("if not exists") + ra(n.sons[0], s) + s.addKeyw("as") + ra(n.sons[1], s) of nkCreateIndex, nkCreateIndexIfNotExists: - s.add("create index ") + s.addKeyw("create index") if n.kind == nkCreateIndexIfNotExists: - s.add("if not exists ") - ra(n.sons[0], s, indent) - s.add(" on ") - ra(n.sons[1], s, indent) + s.addKeyw("if not exists") + ra(n.sons[0], s) + s.addKeyw("on") + ra(n.sons[1], s) s.add('(') for i in 2..n.len-1: if i > 2: s.add(", ") - ra(n.sons[i], s, indent) + ra(n.sons[i], s) s.add(");") of nkEnumDef: - s.add("enum ") - rs(n, s, indent) + s.addKeyw("enum") + rs(n, s) -# What I want: -# -#select(columns = [T1.all, T2.name], -# fromm = [T1, T2], -# where = T1.name ==. T2.name, -# orderby = [name]): -# -#for row in dbQuery(db, """select x, y, z -# from a, b -# where a.name = b.name"""): -# - -#select x, y, z: -# fromm: Table1, Table2 -# where: x.name == y.name -#db.select(fromm = [t1, t2], where = t1.name == t2.name): -#for x, y, z in db.select(fromm = a, b where = a.name == b.name): -# writeLine x, y, z - -proc renderSQL*(n: SqlNode): string = +proc renderSQL*(n: SqlNode, upperCase=false): string = ## Converts an SQL abstract syntax tree to its string representation. - result = "" - ra(n, result, 0) + var s: SqlWriter + s.buffer = "" + s.upperCase = upperCase + ra(n, s) + return s.buffer proc `$`*(n: SqlNode): string = ## an alias for `renderSQL`. renderSQL(n) - -when not defined(testing) and isMainModule: - echo(renderSQL(parseSQL(newStringStream(""" - CREATE TYPE happiness AS ENUM ('happy', 'very happy', 'ecstatic'); - CREATE TABLE holidays ( - num_weeks int, - happiness happiness - ); - CREATE INDEX table1_attr1 ON table1(attr1); - - SELECT * FROM myTab WHERE col1 = 'happy'; - """), "stdin"))) - -# CREATE TYPE happiness AS ENUM ('happy', 'very happy', 'ecstatic'); -# CREATE TABLE holidays ( -# num_weeks int, -# happiness happiness -# ); -# CREATE INDEX table1_attr1 ON table1(attr1) diff --git a/lib/pure/parseutils.nim b/lib/pure/parseutils.nim index b78e8d000..57387e62e 100644 --- a/lib/pure/parseutils.nim +++ b/lib/pure/parseutils.nim @@ -8,6 +8,8 @@ # ## This module contains helpers for parsing tokens, numbers, identifiers, etc. +## +## To unpack raw bytes look at the `streams <streams.html>`_ module. {.deadCodeElim: on.} @@ -85,6 +87,23 @@ proc parseOct*(s: string, number: var int, start = 0): int {. inc(i) if foundDigit: result = i-start +proc parseBin*(s: string, number: var int, start = 0): int {. + rtl, extern: "npuParseBin", noSideEffect.} = + ## parses an binary number and stores its value in ``number``. Returns + ## the number of the parsed characters or 0 in case of an error. + var i = start + var foundDigit = false + if s[i] == '0' and (s[i+1] == 'b' or s[i+1] == 'B'): inc(i, 2) + while true: + case s[i] + of '_': discard + of '0'..'1': + number = number shl 1 or (ord(s[i]) - ord('0')) + foundDigit = true + else: break + inc(i) + if foundDigit: result = i-start + proc parseIdent*(s: string, ident: var string, start = 0): int = ## parses an identifier and stores it in ``ident``. Returns ## the number of the parsed characters or 0 in case of an error. @@ -250,6 +269,31 @@ proc parseInt*(s: string, number: var int, start = 0): int {. elif result != 0: number = int(res) +proc parseSaturatedNatural*(s: string, b: var int, start = 0): int = + ## parses a natural number into ``b``. This cannot raise an overflow + ## error. Instead of an ``Overflow`` exception ``high(int)`` is returned. + ## The number of processed character is returned. + ## This is usually what you really want to use instead of `parseInt`:idx:. + ## Example: + ## + ## .. code-block:: nim + ## var res = 0 + ## discard parseSaturatedNatural("848", res) + ## doAssert res == 848 + var i = start + if s[i] == '+': inc(i) + if s[i] in {'0'..'9'}: + b = 0 + while s[i] in {'0'..'9'}: + let c = ord(s[i]) - ord('0') + if b <= (high(int) - c) div 10: + b = b * 10 + c + else: + b = high(int) + inc(i) + while s[i] == '_': inc(i) # underscores are allowed and ignored + result = i - start + # overflowChecks doesn't work with BiggestUInt proc rawParseUInt(s: string, b: var BiggestUInt, start = 0): int = var @@ -391,16 +435,43 @@ when isMainModule: let input = "$test{} $this is ${an{ example}} " let expected = @[(ikVar, "test"), (ikStr, "{} "), (ikVar, "this"), (ikStr, " is "), (ikExpr, "an{ example}"), (ikStr, " ")] - assert toSeq(interpolatedFragments(input)) == expected + doAssert toSeq(interpolatedFragments(input)) == expected var value = 0 discard parseHex("0x38", value) - assert value == 56 + doAssert value == 56 discard parseHex("0x34", value) - assert value == 56 * 256 + 52 + doAssert value == 56 * 256 + 52 value = -1 discard parseHex("0x38", value) - assert value == -200 + doAssert value == -200 + + value = -1 + doAssert(parseSaturatedNatural("848", value) == 3) + doAssert value == 848 + + value = -1 + discard parseSaturatedNatural("84899999999999999999324234243143142342135435342532453", value) + doAssert value == high(int) + value = -1 + discard parseSaturatedNatural("9223372036854775808", value) + doAssert value == high(int) + + value = -1 + discard parseSaturatedNatural("9223372036854775807", value) + doAssert value == high(int) + + value = -1 + discard parseSaturatedNatural("18446744073709551616", value) + doAssert value == high(int) + + value = -1 + discard parseSaturatedNatural("18446744073709551615", value) + doAssert value == high(int) + + value = -1 + doAssert(parseSaturatedNatural("1_000_000", value) == 9) + doAssert value == 1_000_000 {.pop.} diff --git a/lib/pure/poly.nim b/lib/pure/poly.nim deleted file mode 100644 index e286c5d17..000000000 --- a/lib/pure/poly.nim +++ /dev/null @@ -1,371 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2013 Robert Persson -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## **Warning:** This module will be moved out of the stdlib and into a -## Nimble package, don't use it. - -import math -import strutils -import numeric - -type - Poly* = object - cofs:seq[float] - -{.deprecated: [TPoly: Poly].} - -proc degree*(p:Poly):int= - ## Returns the degree of the polynomial, - ## that is the number of coefficients-1 - return p.cofs.len-1 - - -proc eval*(p:Poly,x:float):float= - ## Evaluates a polynomial function value for `x` - ## quickly using Horners method - var n=p.degree - result=p.cofs[n] - dec n - while n>=0: - result = result*x+p.cofs[n] - dec n - -proc `[]` *(p:Poly;idx:int):float= - ## Gets a coefficient of the polynomial. - ## p[2] will returns the quadric term, p[3] the cubic etc. - ## Out of bounds index will return 0.0. - if idx<0 or idx>p.degree: - return 0.0 - return p.cofs[idx] - -proc `[]=` *(p:var Poly;idx:int,v:float)= - ## Sets an coefficient of the polynomial by index. - ## p[2] set the quadric term, p[3] the cubic etc. - ## If index is out of range for the coefficients, - ## the polynomial grows to the smallest needed degree. - assert(idx>=0) - - if idx>p.degree: #polynomial must grow - var oldlen=p.cofs.len - p.cofs.setLen(idx+1) - for q in oldlen.. <high(p.cofs): - p.cofs[q]=0.0 #new-grown coefficients set to zero - - p.cofs[idx]=v - - -iterator items*(p:Poly):float= - ## Iterates through the coefficients of the polynomial. - var i=p.degree - while i>=0: - yield p[i] - dec i - -proc clean*(p:var Poly;zerotol=0.0)= - ## Removes leading zero coefficients of the polynomial. - ## An optional tolerance can be given for what's considered zero. - var n=p.degree - var relen=false - - while n>0 and abs(p[n])<=zerotol: # >0 => keep at least one coefficient - dec n - relen=true - - if relen: p.cofs.setLen(n+1) - - -proc `$` *(p:Poly):string = - ## Gets a somewhat reasonable string representation of the polynomial - ## The format should be compatible with most online function plotters, - ## for example directly in google search - result="" - var first=true #might skip + sign if first coefficient - - for idx in countdown(p.degree,0): - let a=p[idx] - - if a==0.0: - continue - - if a>= 0.0 and not first: - result.add('+') - first=false - - if a!=1.0 or idx==0: - result.add(formatFloat(a,ffDefault,0)) - if idx>=2: - result.add("x^" & $idx) - elif idx==1: - result.add("x") - - if result=="": - result="0" - - -proc derivative*(p: Poly): Poly= - ## Returns a new polynomial, which is the derivative of `p` - newSeq[float](result.cofs,p.degree) - for idx in 0..high(result.cofs): - result.cofs[idx]=p.cofs[idx+1]*float(idx+1) - -proc diff*(p:Poly,x:float):float= - ## Evaluates the differentiation of a polynomial with - ## respect to `x` quickly using a modifed Horners method - var n=p.degree - result=p[n]*float(n) - dec n - while n>=1: - result = result*x+p[n]*float(n) - dec n - -proc integral*(p:Poly):Poly= - ## Returns a new polynomial which is the indefinite - ## integral of `p`. The constant term is set to 0.0 - newSeq(result.cofs,p.cofs.len+1) - result.cofs[0]=0.0 #constant arbitrary term, use 0.0 - for i in 1..high(result.cofs): - result.cofs[i]=p.cofs[i-1]/float(i) - - -proc integrate*(p:Poly;xmin,xmax:float):float= - ## Computes the definite integral of `p` between `xmin` and `xmax` - ## quickly using a modified version of Horners method - var - n=p.degree - s1=p[n]/float(n+1) - s2=s1 - fac:float - - dec n - while n>=0: - fac=p[n]/float(n+1) - s1 = s1*xmin+fac - s2 = s2*xmax+fac - dec n - - result=s2*xmax-s1*xmin - -proc initPoly*(cofs:varargs[float]):Poly= - ## Initializes a polynomial with given coefficients. - ## The most significant coefficient is first, so to create x^2-2x+3: - ## intiPoly(1.0,-2.0,3.0) - if len(cofs)<=0: - result.cofs= @[0.0] #need at least one coefficient - else: - # reverse order of coefficients so indexing matches degree of - # coefficient... - result.cofs= @[] - for idx in countdown(cofs.len-1,0): - result.cofs.add(cofs[idx]) - - result.clean #remove leading zero terms - - -proc divMod*(p,d:Poly;q,r:var Poly)= - ## Divides `p` with `d`, and stores the quotinent in `q` and - ## the remainder in `d` - var - pdeg=p.degree - ddeg=d.degree - power=p.degree-d.degree - ratio:float - - r.cofs = p.cofs #initial remainder=numerator - if power<0: #denominator is larger than numerator - q.cofs= @ [0.0] #quotinent is 0.0 - return # keep remainder as numerator - - q.cofs=newSeq[float](power+1) - - for i in countdown(pdeg,ddeg): - ratio=r.cofs[i]/d.cofs[ddeg] - - q.cofs[i-ddeg]=ratio - r.cofs[i]=0.0 - - for j in countup(0,<ddeg): - var idx=i-ddeg+j - r.cofs[idx] = r.cofs[idx] - d.cofs[j]*ratio - - r.clean # drop zero coefficients in remainder - -proc `+` *(p1:Poly,p2:Poly):Poly= - ## Adds two polynomials - var n=max(p1.cofs.len,p2.cofs.len) - newSeq(result.cofs,n) - - for idx in countup(0,n-1): - result[idx]=p1[idx]+p2[idx] - - result.clean # drop zero coefficients in remainder - -proc `*` *(p1:Poly,p2:Poly):Poly= - ## Multiplies the polynomial `p1` with `p2` - var - d1=p1.degree - d2=p2.degree - n=d1+d2 - idx:int - - newSeq(result.cofs,n) - - for i1 in countup(0,d1): - for i2 in countup(0,d2): - idx=i1+i2 - result[idx]=result[idx]+p1[i1]*p2[i2] - - result.clean - -proc `*` *(p:Poly,f:float):Poly= - ## Multiplies the polynomial `p` with a real number - newSeq(result.cofs,p.cofs.len) - for i in 0..high(p.cofs): - result[i]=p.cofs[i]*f - result.clean - -proc `*` *(f:float,p:Poly):Poly= - ## Multiplies a real number with a polynomial - return p*f - -proc `-`*(p:Poly):Poly= - ## Negates a polynomial - result=p - for i in countup(0,<result.cofs.len): - result.cofs[i]= -result.cofs[i] - -proc `-` *(p1:Poly,p2:Poly):Poly= - ## Subtract `p1` with `p2` - var n=max(p1.cofs.len,p2.cofs.len) - newSeq(result.cofs,n) - - for idx in countup(0,n-1): - result[idx]=p1[idx]-p2[idx] - - result.clean # drop zero coefficients in remainder - -proc `/`*(p:Poly,f:float):Poly= - ## Divides polynomial `p` with a real number `f` - newSeq(result.cofs,p.cofs.len) - for i in 0..high(p.cofs): - result[i]=p.cofs[i]/f - result.clean - -proc `/` *(p,q:Poly):Poly= - ## Divides polynomial `p` with polynomial `q` - var dummy:Poly - p.divMod(q,result,dummy) - -proc `mod` *(p,q:Poly):Poly= - ## Computes the polynomial modulo operation, - ## that is the remainder of `p`/`q` - var dummy:Poly - p.divMod(q,dummy,result) - - -proc normalize*(p:var Poly)= - ## Multiplies the polynomial inplace by a term so that - ## the leading term is 1.0. - ## This might lead to an unstable polynomial - ## if the leading term is zero. - p=p/p[p.degree] - - -proc solveQuadric*(a,b,c:float;zerotol=0.0):seq[float]= - ## Solves the quadric equation `ax^2+bx+c`, with a possible - ## tolerance `zerotol` to find roots of curves just 'touching' - ## the x axis. Returns sequence with 0,1 or 2 solutions. - - var p,q,d:float - - p=b/(2.0*a) - - if p==Inf or p==NegInf: #linear equation.. - var linrt= -c/b - if linrt==Inf or linrt==NegInf: #constant only - return @[] - return @[linrt] - - q=c/a - d=p*p-q - - if d<0.0: - #check for inside zerotol range for neg. roots - var err=a*p*p-b*p+c #evaluate error at parabola center axis - if(err<=zerotol): return @[-p] - return @[] - else: - var sr=sqrt(d) - result= @[-sr-p,sr-p] - -proc getRangeForRoots(p:Poly):tuple[xmin,xmax:float]= - ## helper function for `roots` function - ## quickly computes a range, guaranteed to contain - ## all the real roots of the polynomial - # see http://www.mathsisfun.com/algebra/polynomials-bounds-zeros.html - - var deg=p.degree - var d=p[deg] - var bound1,bound2:float - - for i in countup(0,deg): - var c=abs(p.cofs[i]/d) - bound1=max(bound1,c+1.0) - bound2=bound2+c - - bound2=max(1.0,bound2) - result.xmax=min(bound1,bound2) - result.xmin= -result.xmax - - -proc addRoot(p:Poly,res:var seq[float],xp0,xp1,tol,zerotol,mergetol:float,maxiter:int)= - ## helper function for `roots` function - ## try to do a numeric search for a single root in range xp0-xp1, - ## adding it to `res` (allocating `res` if nil) - var br=brent(xp0,xp1, proc(x:float):float=p.eval(x),tol) - if br.success: - if res.len==0 or br.rootx>=res[high(res)]+mergetol: #dont add equal roots. - res.add(br.rootx) - else: - #this might be a 'touching' case, check function value against - #zero tolerance - if abs(br.rooty)<=zerotol: - if res.len==0 or br.rootx>=res[high(res)]+mergetol: #dont add equal roots. - res.add(br.rootx) - - -proc roots*(p:Poly,tol=1.0e-9,zerotol=1.0e-6,mergetol=1.0e-12,maxiter=1000):seq[float]= - ## Computes the real roots of the polynomial `p` - ## `tol` is the tolerance used to break searching for each root when reached. - ## `zerotol` is the tolerance, which is 'close enough' to zero to be considered a root - ## and is used to find roots for curves that only 'touch' the x-axis. - ## `mergetol` is the tolerance, of which two x-values are considered being the same root. - ## `maxiter` can be used to limit the number of iterations for each root. - ## Returns a (possibly empty) sorted sequence with the solutions. - var deg=p.degree - if deg<=0: #constant only => no roots - return @[] - elif p.degree==1: #linear - var linrt= -p.cofs[0]/p.cofs[1] - if linrt==Inf or linrt==NegInf: - return @[] #constant only => no roots - return @[linrt] - elif p.degree==2: - return solveQuadric(p.cofs[2],p.cofs[1],p.cofs[0],zerotol) - else: - # degree >=3 , find min/max points of polynomial with recursive - # derivative and do a numerical search for root between each min/max - var range=p.getRangeForRoots() - var minmax=p.derivative.roots(tol,zerotol,mergetol) - result= @[] - if minmax!=nil: #ie. we have minimas/maximas in this function - for x in minmax.items: - addRoot(p,result,range.xmin,x,tol,zerotol,mergetol,maxiter) - range.xmin=x - addRoot(p,result,range.xmin,range.xmax,tol,zerotol,mergetol,maxiter) - diff --git a/lib/pure/random.nim b/lib/pure/random.nim index 27fbfad45..de419b9fb 100644 --- a/lib/pure/random.nim +++ b/lib/pure/random.nim @@ -7,16 +7,16 @@ # distribution, for details about the copyright. # -## Nim's standard random number generator. Based on the ``xoroshiro128+`` (xor/rotate/shift/rotate) library. +## Nim's standard random number generator. Based on +## the ``xoroshiro128+`` (xor/rotate/shift/rotate) library. ## * More information: http://xoroshiro.di.unimi.it/ ## * C implementation: http://xoroshiro.di.unimi.it/xoroshiro128plus.c ## -## Do not use this module for cryptographic use! +## **Do not use this module for cryptographic purposes!** include "system/inclrtl" {.push debugger:off.} -# XXX Expose RandomGenState when defined(JS): type ui = uint32 @@ -27,31 +27,34 @@ else: const randMax = 18_446_744_073_709_551_615u64 type - RandomGenState = object + Rand* = object ## State of the random number generator. + ## The procs that use the default state + ## are **not** thread-safe! a0, a1: ui when defined(JS): - var state = RandomGenState( + var state = Rand( a0: 0x69B4C98Cu32, a1: 0xFED1DD30u32) # global for backwards compatibility else: # racy for multi-threading but good enough for now: - var state = RandomGenState( + var state = Rand( a0: 0x69B4C98CB8530805u64, a1: 0xFED1DD3004688D67CAu64) # global for backwards compatibility proc rotl(x, k: ui): ui = result = (x shl k) or (x shr (ui(64) - k)) -proc next(s: var RandomGenState): uint64 = - let s0 = s.a0 - var s1 = s.a1 +proc next*(r: var Rand): uint64 = + ## Uses the state to compute a new ``uint64`` random number. + let s0 = r.a0 + var s1 = r.a1 result = s0 + s1 s1 = s1 xor s0 - s.a0 = rotl(s0, 55) xor s1 xor (s1 shl 14) # a, b - s.a1 = rotl(s1, 36) # c + r.a0 = rotl(s0, 55) xor s1 xor (s1 shl 14) # a, b + r.a1 = rotl(s1, 36) # c -proc skipRandomNumbers(s: var RandomGenState) = +proc skipRandomNumbers*(s: var Rand) = ## This is the jump function for the generator. It is equivalent ## to 2^64 calls to next(); it can be used to generate 2^64 ## non-overlapping subsequences for parallel computations. @@ -71,21 +74,23 @@ proc skipRandomNumbers(s: var RandomGenState) = s.a0 = s0 s.a1 = s1 -proc random*(max: int): int {.benign.} = +proc random*(max: int): int {.benign, deprecated.} = ## Returns a random number in the range 0..max-1. The sequence of ## random number is always the same, unless `randomize` is called ## which initializes the random number generator with a "random" - ## number, i.e. a tickcount. + ## number, i.e. a tickcount. **Deprecated since version 0.18.0**. + ## Use ``rand`` instead. while true: let x = next(state) if x < randMax - (randMax mod ui(max)): return int(x mod uint64(max)) -proc random*(max: float): float {.benign.} = +proc random*(max: float): float {.benign, deprecated.} = ## Returns a random number in the range 0..<max. The sequence of ## random number is always the same, unless `randomize` is called ## which initializes the random number generator with a "random" - ## number, i.e. a tickcount. + ## number, i.e. a tickcount. **Deprecated since version 0.18.0**. + ## Use ``rand`` instead. let x = next(state) when defined(JS): result = (float(x) / float(high(uint32))) * max @@ -93,38 +98,100 @@ proc random*(max: float): float {.benign.} = let u = (0x3FFu64 shl 52u64) or (x shr 12u64) result = (cast[float](u) - 1.0) * max -proc random*[T](x: Slice[T]): T = +proc random*[T](x: HSlice[T, T]): T {.deprecated.} = ## For a slice `a .. b` returns a value in the range `a .. b-1`. + ## **Deprecated since version 0.18.0**. + ## Use ``rand`` instead. result = T(random(x.b - x.a)) + x.a -proc random*[T](a: openArray[T]): T = +proc random*[T](a: openArray[T]): T {.deprecated.} = ## returns a random element from the openarray `a`. + ## **Deprecated since version 0.18.0**. + ## Use ``rand`` instead. result = a[random(a.low..a.len)] +proc rand*(r: var Rand; max: int): int {.benign.} = + ## Returns a random number in the range 0..max. The sequence of + ## random number is always the same, unless `randomize` is called + ## which initializes the random number generator with a "random" + ## number, i.e. a tickcount. + while true: + let x = next(r) + if x <= randMax - (randMax mod ui(max)): + return int(x mod (uint64(max)+1u64)) + +proc rand*(max: int): int {.benign.} = + ## Returns a random number in the range 0..max. The sequence of + ## random number is always the same, unless `randomize` is called + ## which initializes the random number generator with a "random" + ## number, i.e. a tickcount. + rand(state, max) + +proc rand*(r: var Rand; max: float): float {.benign.} = + ## Returns a random number in the range 0..max. The sequence of + ## random number is always the same, unless `randomize` is called + ## which initializes the random number generator with a "random" + ## number, i.e. a tickcount. + let x = next(r) + when defined(JS): + result = (float(x) / float(high(uint32))) * max + else: + let u = (0x3FFu64 shl 52u64) or (x shr 12u64) + result = (cast[float](u) - 1.0) * max + +proc rand*(max: float): float {.benign.} = + ## Returns a random number in the range 0..max. The sequence of + ## random number is always the same, unless `randomize` is called + ## which initializes the random number generator with a "random" + ## number, i.e. a tickcount. + rand(state, max) + +proc rand*[T](r: var Rand; x: HSlice[T, T]): T = + ## For a slice `a .. b` returns a value in the range `a .. b`. + result = T(rand(r, x.b - x.a)) + x.a + +proc rand*[T](x: HSlice[T, T]): T = + ## For a slice `a .. b` returns a value in the range `a .. b`. + result = rand(state, x) + +proc rand*[T](r: var Rand; a: openArray[T]): T = + ## returns a random element from the openarray `a`. + result = a[rand(r, a.low..a.high)] + +proc rand*[T](a: openArray[T]): T = + ## returns a random element from the openarray `a`. + result = a[rand(a.low..a.high)] + + +proc initRand*(seed: int64): Rand = + ## Creates a new ``Rand`` state from ``seed``. + result.a0 = ui(seed shr 16) + result.a1 = ui(seed and 0xffff) + discard next(result) + proc randomize*(seed: int64) {.benign.} = - ## Initializes the random number generator with a specific seed. - state.a0 = ui(seed shr 16) - state.a1 = ui(seed and 0xffff) - discard next(state) + ## Initializes the default random number generator + ## with a specific seed. + state = initRand(seed) -proc shuffle*[T](x: var openArray[T]) = - ## Will randomly swap the positions of elements in a sequence. +proc shuffle*[T](r: var Rand; x: var openArray[T]) = + ## Swaps the positions of elements in a sequence randomly. for i in countdown(x.high, 1): - let j = random(i + 1) + let j = r.rand(i) swap(x[i], x[j]) +proc shuffle*[T](x: var openArray[T]) = + ## Swaps the positions of elements in a sequence randomly. + shuffle(state, x) + when not defined(nimscript): import times proc randomize*() {.benign.} = ## Initializes the random number generator with a "random" ## number, i.e. a tickcount. Note: Does not work for NimScript. - when defined(JS): - proc getMil(t: Time): int {.importcpp: "getTime", nodecl.} - randomize(getMil times.getTime()) - else: - let time = int64(times.epochTime() * 1_000_000_000) - randomize(time) + let time = int64(times.epochTime() * 1_000_000_000) + randomize(time) {.pop.} @@ -134,12 +201,12 @@ when isMainModule: var x = 8234 for i in 0..100_000: - x = random(len(occur)) # myrand(x) + x = rand(high(occur)) inc occur[x] for i, oc in occur: if oc < 69: doAssert false, "too few occurrences of " & $i - elif oc > 130: + elif oc > 150: doAssert false, "too many occurrences of " & $i var a = [0, 1] diff --git a/lib/pure/rationals.nim b/lib/pure/rationals.nim index c2ba2b1f3..7907b4e6c 100644 --- a/lib/pure/rationals.nim +++ b/lib/pure/rationals.nim @@ -39,47 +39,13 @@ proc toRational*[T:SomeInteger](x: T): Rational[T] = result.num = x result.den = 1 -proc toRationalSub(x: float, n: int): Rational[int] = - var - a = 0'i64 - b, c, d = 1'i64 - result = 0 // 1 # rational 0 - while b <= n and d <= n: - let ac = (a+c) - let bd = (b+d) - # scale by 1000 so not overflow for high precision - let mediant = (ac.float/1000) / (bd.float/1000) - if x == mediant: - if bd <= n: - result.num = ac.int - result.den = bd.int - return result - elif d > b: - result.num = c.int - result.den = d.int - return result - else: - result.num = a.int - result.den = b.int - return result - elif x > mediant: - a = ac - b = bd - else: - c = ac - d = bd - if (b > n): - return initRational(c.int, d.int) - return initRational(a.int, b.int) - -proc toRational*(x: float, n: int = high(int)): Rational[int] = - ## Calculate the best rational numerator and denominator +proc toRational*(x: float, n: int = high(int) shr (sizeof(int) div 2 * 8)): Rational[int] = + ## Calculates the best rational numerator and denominator ## that approximates to `x`, where the denominator is ## smaller than `n` (default is the largest possible - ## int to give maximum resolution) + ## int to give maximum resolution). ## - ## The algorithm is based on the Farey sequence named - ## after John Farey + ## The algorithm is based on the theory of continued fractions. ## ## .. code-block:: Nim ## import math, rationals @@ -88,13 +54,24 @@ proc toRational*(x: float, n: int = high(int)): Rational[int] = ## let x = toRational(PI, t) ## let newPI = x.num / x.den ## echo x, " ", newPI, " error: ", PI - newPI, " ", t - if x > 1: - result = toRationalSub(1.0/x, n) - swap(result.num, result.den) - elif x == 1.0: - result = 1 // 1 - else: - result = toRationalSub(x, n) + + # David Eppstein / UC Irvine / 8 Aug 1993 + # With corrections from Arno Formella, May 2008 + var + m11, m22 = 1 + m12, m21 = 0 + ai = int(x) + x = x + while m21 * ai + m22 <= n: + swap m12, m11 + swap m22, m21 + m11 = m12 * ai + m11 + m21 = m22 * ai + m21 + if x == float(ai): break # division by zero + x = 1/(x - float(ai)) + if x > float(high(int32)): break # representation failure + ai = int(x) + result = m11 // m21 proc toFloat*[T](x: Rational[T]): float = ## Convert a rational number `x` to a float. @@ -346,7 +323,19 @@ when isMainModule: assert abs(toFloat(y) - 0.4814814814814815) < 1.0e-7 assert toInt(z) == 0 - assert toRational(0.98765432) == 12345679 // 12500000 - assert toRational(0.1, 1000000) == 1 // 10 - assert toRational(0.9, 1000000) == 9 // 10 - #assert toRational(PI) == 80143857 // 25510582 + when sizeof(int) == 8: + assert toRational(0.98765432) == 2111111029 // 2137499919 + assert toRational(PI) == 817696623 // 260280919 + when sizeof(int) == 4: + assert toRational(0.98765432) == 80 // 81 + assert toRational(PI) == 355 // 113 + + assert toRational(0.1) == 1 // 10 + assert toRational(0.9) == 9 // 10 + + assert toRational(0.0) == 0 // 1 + assert toRational(-0.25) == 1 // -4 + assert toRational(3.2) == 16 // 5 + assert toRational(0.33) == 33 // 100 + assert toRational(0.22) == 11 // 50 + assert toRational(10.0) == 10 // 1 diff --git a/lib/pure/romans.nim b/lib/pure/romans.nim deleted file mode 100644 index aa047d1cc..000000000 --- a/lib/pure/romans.nim +++ /dev/null @@ -1,59 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2011 Philippe Lhoste -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## Module for converting an integer to a Roman numeral. -## See http://en.wikipedia.org/wiki/Roman_numerals for reference. -## -## **Warning:** This module will be moved out of the stdlib and into a -## Nimble package, don't use it. - -const - RomanNumeralDigits* = {'I', 'i', 'V', 'v', 'X', 'x', 'L', 'l', 'C', 'c', - 'D', 'd', 'M', 'm'} ## set of all characters a Roman numeral may consist of - -proc romanToDecimal*(romanVal: string): int = - ## Converts a Roman numeral to its int representation. - result = 0 - var prevVal = 0 - for i in countdown(romanVal.len - 1, 0): - var val = 0 - case romanVal[i] - of 'I', 'i': val = 1 - of 'V', 'v': val = 5 - of 'X', 'x': val = 10 - of 'L', 'l': val = 50 - of 'C', 'c': val = 100 - of 'D', 'd': val = 500 - of 'M', 'm': val = 1000 - else: - raise newException(EInvalidValue, "invalid roman numeral: " & $romanVal) - if val >= prevVal: - inc(result, val) - else: - dec(result, val) - prevVal = val - -proc decimalToRoman*(number: range[1..3_999]): string = - ## Converts a number to a Roman numeral. - const romanComposites = [ - ("M", 1000), ("CM", 900), - ("D", 500), ("CD", 400), ("C", 100), - ("XC", 90), ("L", 50), ("XL", 40), ("X", 10), ("IX", 9), - ("V", 5), ("IV", 4), ("I", 1)] - result = "" - var decVal: int = number - for key, val in items(romanComposites): - while decVal >= val: - dec(decVal, val) - result.add(key) - -when isMainModule: - for i in 1 .. 3_999: - assert i == i.decimalToRoman.romanToDecimal - diff --git a/lib/pure/ropes.nim b/lib/pure/ropes.nim index 6e97237e0..6ddd61afa 100644 --- a/lib/pure/ropes.nim +++ b/lib/pure/ropes.nim @@ -17,6 +17,7 @@ ## runtime efficiency. include "system/inclrtl" +import streams {.deadCodeElim: on.} @@ -130,7 +131,7 @@ proc insertInCache(s: string, tree: Rope): Rope = result.left = t t.right = nil -proc rope*(s: string): Rope {.rtl, extern: "nro$1Str".} = +proc rope*(s: string = nil): Rope {.rtl, extern: "nro$1Str".} = ## Converts a string to a rope. if s.len == 0: result = nil @@ -242,10 +243,13 @@ proc write*(f: File, r: Rope) {.rtl, extern: "nro$1".} = ## writes a rope to a file. for s in leaves(r): write(f, s) +proc write*(s: Stream, r: Rope) {.rtl, extern: "nroWriteStream".} = + ## writes a rope to a stream. + for rs in leaves(r): write(s, rs) + proc `$`*(r: Rope): string {.rtl, extern: "nroToString".}= ## converts a rope back to a string. - result = newString(r.len) - setLen(result, 0) + result = newStringOfCap(r.len) for s in leaves(r): add(result, s) when false: diff --git a/lib/pure/scgi.nim b/lib/pure/scgi.nim index 711e4a897..1ff26954e 100644 --- a/lib/pure/scgi.nim +++ b/lib/pure/scgi.nim @@ -95,7 +95,7 @@ type AsyncScgiState* = ref AsyncScgiStateObj {.deprecated: [EScgi: ScgiError, TScgiState: ScgiState, - PAsyncScgiState: AsyncScgiState, scgiError: raiseScgiError].} + PAsyncScgiState: AsyncScgiState].} proc recvBuffer(s: var ScgiState, L: int) = if L > s.bufLen: diff --git a/lib/pure/securehash.nim b/lib/pure/securehash.nim index c19146669..57c1f3631 100644 --- a/lib/pure/securehash.nim +++ b/lib/pure/securehash.nim @@ -181,7 +181,7 @@ proc `$`*(self: SecureHash): string = result.add(toHex(int(v), 2)) proc parseSecureHash*(hash: string): SecureHash = - for i in 0.. <Sha1DigestSize: + for i in 0 ..< Sha1DigestSize: Sha1Digest(result)[i] = uint8(parseHexInt(hash[i*2] & hash[i*2 + 1])) proc `==`*(a, b: SecureHash): bool = diff --git a/lib/pure/selectors.nim b/lib/pure/selectors.nim index 506b2cec0..518cc4bd5 100644 --- a/lib/pure/selectors.nim +++ b/lib/pure/selectors.nim @@ -1,420 +1,313 @@ # # # Nim's Runtime Library -# (c) Copyright 2015 Dominik Picheta +# (c) Copyright 2016 Eugene Kabanov # # See the file "copying.txt", included in this # distribution, for details about the copyright. # -# TODO: Docs. - -import os, hashes - -when defined(linux): - import posix, epoll -elif defined(macosx) or defined(freebsd) or defined(openbsd) or defined(netbsd): - import posix, kqueue, times -elif defined(windows): - import winlean -else: - import posix - -const MultiThreaded = defined(useStdlibThreading) - -when MultiThreaded: - import sharedtables - - type SelectorData = pointer -else: - import tables - - type SelectorData = RootRef - -proc hash*(x: SocketHandle): Hash {.borrow.} -proc `$`*(x: SocketHandle): string {.borrow.} - -type - Event* = enum - EvRead, EvWrite, EvError - - SelectorKey* = object - fd*: SocketHandle - events*: set[Event] ## The events which ``fd`` listens for. - data*: SelectorData ## User object. - - ReadyInfo* = tuple[key: SelectorKey, events: set[Event]] +## This module allows high-level and efficient I/O multiplexing. +## +## Supported OS primitives: ``epoll``, ``kqueue``, ``poll`` and +## Windows ``select``. +## +## To use threadsafe version of this module, it needs to be compiled +## with both ``-d:threadsafe`` and ``--threads:on`` options. +## +## Supported features: files, sockets, pipes, timers, processes, signals +## and user events. +## +## Fully supported OS: MacOSX, FreeBSD, OpenBSD, NetBSD, Linux (except +## for Android). +## +## Partially supported OS: Windows (only sockets and user events), +## Solaris (files, sockets, handles and user events). +## Android (files, sockets, handles and user events). +## +## TODO: ``/dev/poll``, ``event ports`` and filesystem events. + +import os, strutils, nativesockets + +const hasThreadSupport = compileOption("threads") and defined(threadsafe) + +const ioselSupportedPlatform* = defined(macosx) or defined(freebsd) or + defined(netbsd) or defined(openbsd) or + defined(dragonfly) or + (defined(linux) and not defined(android)) + ## This constant is used to determine whether the destination platform is + ## fully supported by ``ioselectors`` module. + +const bsdPlatform = defined(macosx) or defined(freebsd) or + defined(netbsd) or defined(openbsd) or + defined(dragonfly) when defined(nimdoc): type - Selector* = ref object - ## An object which holds file descriptors to be checked for read/write - ## status. - - proc register*(s: Selector, fd: SocketHandle, events: set[Event], - data: SelectorData): SelectorKey {.discardable.} = - ## Registers file descriptor ``fd`` to selector ``s`` with a set of Event - ## ``events``. - - proc update*(s: Selector, fd: SocketHandle, - events: set[Event]): SelectorKey {.discardable.} = - ## Updates the events which ``fd`` wants notifications for. - - proc unregister*(s: Selector, fd: SocketHandle): SelectorKey {.discardable.} = - ## Unregisters file descriptor ``fd`` from selector ``s``. - - proc close*(s: Selector) = - ## Closes the selector - - proc select*(s: Selector, timeout: int): seq[ReadyInfo] = - ## The ``events`` field of the returned ``key`` contains the original events - ## for which the ``fd`` was bound. This is contrary to the ``events`` field - ## of the ``ReadyInfo`` tuple which determines which events are ready - ## on the ``fd``. - - proc newSelector*(): Selector = + Selector*[T] = ref object + ## An object which holds descriptors to be checked for read/write status + + Event* {.pure.} = enum + ## An enum which hold event types + Read, ## Descriptor is available for read + Write, ## Descriptor is available for write + Timer, ## Timer descriptor is completed + Signal, ## Signal is raised + Process, ## Process is finished + Vnode, ## BSD specific file change happens + User, ## User event is raised + Error, ## Error happens while waiting, for descriptor + VnodeWrite, ## NOTE_WRITE (BSD specific, write to file occurred) + VnodeDelete, ## NOTE_DELETE (BSD specific, unlink of file occurred) + VnodeExtend, ## NOTE_EXTEND (BSD specific, file extended) + VnodeAttrib, ## NOTE_ATTRIB (BSD specific, file attributes changed) + VnodeLink, ## NOTE_LINK (BSD specific, file link count changed) + VnodeRename, ## NOTE_RENAME (BSD specific, file renamed) + VnodeRevoke ## NOTE_REVOKE (BSD specific, file revoke occurred) + + ReadyKey* = object + ## An object which holds result for descriptor + fd* : int ## file/socket descriptor + events*: set[Event] ## set of events + + SelectEvent* = object + ## An object which holds user defined event + + proc newSelector*[T](): Selector[T] = ## Creates a new selector - proc contains*(s: Selector, fd: SocketHandle): bool = + proc close*[T](s: Selector[T]) = + ## Closes the selector. + + proc registerHandle*[T](s: Selector[T], fd: SocketHandle, events: set[Event], + data: T) = + ## Registers file/socket descriptor ``fd`` to selector ``s`` + ## with events set in ``events``. The ``data`` is application-defined + ## data, which will be passed when an event is triggered. + + proc updateHandle*[T](s: Selector[T], fd: SocketHandle, events: set[Event]) = + ## Update file/socket descriptor ``fd``, registered in selector + ## ``s`` with new events set ``event``. + + proc registerTimer*[T](s: Selector[T], timeout: int, oneshot: bool, + data: T): int {.discardable.} = + ## Registers timer notification with ``timeout`` (in milliseconds) + ## to selector ``s``. + ## + ## If ``oneshot`` is ``true``, timer will be notified only once. + ## + ## Set ``oneshot`` to ``false`` if you want periodic notifications. + ## + ## The ``data`` is application-defined data, which will be passed, when + ## the timer is triggered. + ## + ## Returns the file descriptor for the registered timer. + + proc registerSignal*[T](s: Selector[T], signal: int, + data: T): int {.discardable.} = + ## Registers Unix signal notification with ``signal`` to selector + ## ``s``. + ## + ## The ``data`` is application-defined data, which will be + ## passed when signal raises. + ## + ## Returns the file descriptor for the registered signal. + ## + ## **Note:** This function is not supported on ``Windows``. + + proc registerProcess*[T](s: Selector[T], pid: int, + data: T): int {.discardable.} = + ## Registers a process id (pid) notification (when process has + ## exited) in selector ``s``. + ## + ## The ``data`` is application-defined data, which will be passed when + ## process with ``pid`` has exited. + ## + ## Returns the file descriptor for the registered signal. + + proc registerEvent*[T](s: Selector[T], ev: SelectEvent, data: T) = + ## Registers selector event ``ev`` in selector ``s``. + ## + ## The ``data`` is application-defined data, which will be passed when + ## ``ev`` happens. + + proc registerVnode*[T](s: Selector[T], fd: cint, events: set[Event], + data: T) = + ## Registers selector BSD/MacOSX specific vnode events for file + ## descriptor ``fd`` and events ``events``. + ## ``data`` application-defined data, which to be passed, when + ## vnode event happens. + ## + ## **Note:** This function is supported only by BSD and MacOSX. + + proc newSelectEvent*(): SelectEvent = + ## Creates a new user-defined event. + + proc trigger*(ev: SelectEvent) = + ## Trigger event ``ev``. + + proc close*(ev: SelectEvent) = + ## Closes user-defined event ``ev``. + + proc unregister*[T](s: Selector[T], ev: SelectEvent) = + ## Unregisters user-defined event ``ev`` from selector ``s``. + + proc unregister*[T](s: Selector[T], fd: int|SocketHandle|cint) = + ## Unregisters file/socket descriptor ``fd`` from selector ``s``. + + proc selectInto*[T](s: Selector[T], timeout: int, + results: var openarray[ReadyKey]): int = + ## Waits for events registered in selector ``s``. + ## + ## The ``timeout`` argument specifies the maximum number of milliseconds + ## the function will be blocked for if no events are ready. Specifying a + ## timeout of ``-1`` causes the function to block indefinitely. + ## All available events will be stored in ``results`` array. + ## + ## Returns number of triggered events. + + proc select*[T](s: Selector[T], timeout: int): seq[ReadyKey] = + ## Waits for events registered in selector ``s``. + ## + ## The ``timeout`` argument specifies the maximum number of milliseconds + ## the function will be blocked for if no events are ready. Specifying a + ## timeout of ``-1`` causes the function to block indefinitely. + ## + ## Returns a list of triggered events. + + proc getData*[T](s: Selector[T], fd: SocketHandle|int): var T = + ## Retrieves application-defined ``data`` associated with descriptor ``fd``. + ## If specified descriptor ``fd`` is not registered, empty/default value + ## will be returned. + + proc setData*[T](s: Selector[T], fd: SocketHandle|int, data: var T): bool = + ## Associate application-defined ``data`` with descriptor ``fd``. + ## + ## Returns ``true``, if data was succesfully updated, ``false`` otherwise. + + template isEmpty*[T](s: Selector[T]): bool = # TODO: Why is this a template? + ## Returns ``true``, if there are no registered events or descriptors + ## in selector. + + template withData*[T](s: Selector[T], fd: SocketHandle|int, value, + body: untyped) = + ## Retrieves the application-data assigned with descriptor ``fd`` + ## to ``value``. This ``value`` can be modified in the scope of + ## the ``withData`` call. + ## + ## .. code-block:: nim + ## + ## s.withData(fd, value) do: + ## # block is executed only if ``fd`` registered in selector ``s`` + ## value.uid = 1000 + ## + + template withData*[T](s: Selector[T], fd: SocketHandle|int, value, + body1, body2: untyped) = + ## Retrieves the application-data assigned with descriptor ``fd`` + ## to ``value``. This ``value`` can be modified in the scope of + ## the ``withData`` call. + ## + ## .. code-block:: nim + ## + ## s.withData(fd, value) do: + ## # block is executed only if ``fd`` registered in selector ``s``. + ## value.uid = 1000 + ## do: + ## # block is executed if ``fd`` not registered in selector ``s``. + ## raise + ## + + proc contains*[T](s: Selector[T], fd: SocketHandle|int): bool {.inline.} = ## Determines whether selector contains a file descriptor. - proc `[]`*(s: Selector, fd: SocketHandle): SelectorKey = - ## Retrieves the selector key for ``fd``. - -elif defined(linux): - type - Selector* = object - epollFD: cint - events: array[64, epoll_event] - when MultiThreaded: - fds: SharedTable[SocketHandle, SelectorKey] - else: - fds: Table[SocketHandle, SelectorKey] - - proc createEventStruct(events: set[Event], fd: SocketHandle): epoll_event = - if EvRead in events: - result.events = EPOLLIN - if EvWrite in events: - result.events = result.events or EPOLLOUT - result.events = result.events or EPOLLRDHUP - result.data.fd = fd.cint - - proc register*(s: var Selector, fd: SocketHandle, events: set[Event], - data: SelectorData) = - var event = createEventStruct(events, fd) - if events != {}: - if epoll_ctl(s.epollFD, EPOLL_CTL_ADD, fd, addr(event)) != 0: - raiseOSError(osLastError()) - - s.fds[fd] = SelectorKey(fd: fd, events: events, data: data) - - proc update*(s: var Selector, fd: SocketHandle, events: set[Event]) = - if s.fds[fd].events != events: - if events == {}: - # This fd is idle -- it should not be registered to epoll. - # But it should remain a part of this selector instance. - # This is to prevent epoll_wait from returning immediately - # because its got fds which are waiting for no events and - # are therefore constantly ready. (leading to 100% CPU usage). - if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fd, nil) != 0: - raiseOSError(osLastError()) - s.fds[fd].events = events - else: - var event = createEventStruct(events, fd) - if s.fds[fd].events == {}: - # This fd is idle. It's not a member of this epoll instance and must - # be re-registered. - if epoll_ctl(s.epollFD, EPOLL_CTL_ADD, fd, addr(event)) != 0: - raiseOSError(osLastError()) - else: - if epoll_ctl(s.epollFD, EPOLL_CTL_MOD, fd, addr(event)) != 0: - raiseOSError(osLastError()) - s.fds[fd].events = events - - proc unregister*(s: var Selector, fd: SocketHandle) = - if s.fds[fd].events != {}: - if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fd, nil) != 0: - let err = osLastError() - if err.cint notin {ENOENT, EBADF}: - # TODO: Why do we sometimes get an EBADF? Is this normal? - raiseOSError(err) - s.fds.del(fd) - - proc close*(s: var Selector) = - when MultiThreaded: deinitSharedTable(s.fds) - if s.epollFD.close() != 0: raiseOSError(osLastError()) - - proc epollHasFd(s: Selector, fd: SocketHandle): bool = - result = true - var event = createEventStruct(s.fds[fd].events, fd) - if epoll_ctl(s.epollFD, EPOLL_CTL_MOD, fd, addr(event)) != 0: - let err = osLastError() - if err.cint in {ENOENT, EBADF}: - return false - raiseOSError(err) - - proc select*(s: var Selector, timeout: int): seq[ReadyInfo] = - result = @[] - let evNum = epoll_wait(s.epollFD, addr s.events[0], 64.cint, timeout.cint) - if evNum < 0: - let err = osLastError() - if err.cint == EINTR: - return @[] - raiseOSError(err) - if evNum == 0: return @[] - for i in 0 .. <evNum: - let fd = s.events[i].data.fd.SocketHandle - - var evSet: set[Event] = {} - if (s.events[i].events and EPOLLERR) != 0 or (s.events[i].events and EPOLLHUP) != 0: evSet = evSet + {EvError} - if (s.events[i].events and EPOLLIN) != 0: evSet = evSet + {EvRead} - if (s.events[i].events and EPOLLOUT) != 0: evSet = evSet + {EvWrite} - let selectorKey = s.fds[fd] - assert selectorKey.fd != 0.SocketHandle - result.add((selectorKey, evSet)) - - #echo("Epoll: ", result[i].key.fd, " ", result[i].events, " ", result[i].key.events) - - proc newSelector*(): Selector = - result.epollFD = epoll_create(64) - if result.epollFD < 0: - raiseOSError(osLastError()) - when MultiThreaded: - result.fds = initSharedTable[SocketHandle, SelectorKey]() - else: - result.fds = initTable[SocketHandle, SelectorKey]() - - proc contains*(s: Selector, fd: SocketHandle): bool = - ## Determines whether selector contains a file descriptor. - if s.fds.hasKey(fd): - # Ensure the underlying epoll instance still contains this fd. - if s.fds[fd].events != {}: - result = epollHasFd(s, fd) - else: - result = true - else: - return false +else: + when hasThreadSupport: + import locks - proc `[]`*(s: Selector, fd: SocketHandle): SelectorKey = - ## Retrieves the selector key for ``fd``. - return s.fds[fd] -elif defined(macosx) or defined(freebsd) or defined(openbsd) or defined(netbsd): - type - Selector* = object - kqFD: cint - events: array[64, KEvent] - when MultiThreaded: - fds: SharedTable[SocketHandle, SelectorKey] - else: - fds: Table[SocketHandle, SelectorKey] - - template modifyKQueue(kqFD: cint, fd: SocketHandle, event: Event, - op: cushort) = - var kev = KEvent(ident: fd.cuint, - filter: if event == EvRead: EVFILT_READ else: EVFILT_WRITE, - flags: op) - if kevent(kqFD, addr kev, 1, nil, 0, nil) == -1: - raiseOSError(osLastError()) - - proc register*(s: var Selector, fd: SocketHandle, events: set[Event], - data: SelectorData) = - for event in events: - modifyKQueue(s.kqFD, fd, event, EV_ADD) - s.fds[fd] = SelectorKey(fd: fd, events: events, data: data) - - proc update*(s: var Selector, fd: SocketHandle, events: set[Event]) = - let previousEvents = s.fds[fd].events - if previousEvents != events: - for event in events-previousEvents: - modifyKQueue(s.kqFD, fd, event, EV_ADD) - for event in previousEvents-events: - modifyKQueue(s.kqFD, fd, event, EV_DELETE) - s.fds[fd].events = events - - proc unregister*(s: var Selector, fd: SocketHandle) = - for event in s.fds[fd].events: - modifyKQueue(s.kqFD, fd, event, EV_DELETE) - s.fds.del(fd) - - proc close*(s: var Selector) = - when MultiThreaded: deinitSharedTable(s.fds) - if s.kqFD.close() != 0: raiseOSError(osLastError()) - - proc select*(s: var Selector, timeout: int): seq[ReadyInfo] = - result = @[] - var tv = - if timeout >= 1000: Timespec(tv_sec: (timeout div 1000).Time, tv_nsec: 0) - else: Timespec(tv_sec: 0.Time, tv_nsec: timeout * 1000000) - let evNum = kevent(s.kqFD, nil, 0, addr s.events[0], 64.cint, addr tv) - if evNum < 0: - let err = osLastError() - if err.cint == EINTR: - return @[] - raiseOSError(err) - if evNum == 0: return @[] - for i in 0 .. <evNum: - let fd = s.events[i].ident.SocketHandle - - var evSet: set[Event] = {} - if (s.events[i].flags and EV_EOF) != 0: evSet = evSet + {EvError} - if s.events[i].filter == EVFILT_READ: evSet = evSet + {EvRead} - elif s.events[i].filter == EVFILT_WRITE: evSet = evSet + {EvWrite} - let selectorKey = s.fds[fd] - assert selectorKey.fd != 0.SocketHandle - result.add((selectorKey, evSet)) - - proc newSelector*(): Selector = - result.kqFD = kqueue() - if result.kqFD < 0: - raiseOSError(osLastError()) - when MultiThreaded: - result.fds = initSharedTable[SocketHandle, SelectorKey]() - else: - result.fds = initTable[SocketHandle, SelectorKey]() + type + SharedArray[T] = UncheckedArray[T] - proc contains*(s: Selector, fd: SocketHandle): bool = - ## Determines whether selector contains a file descriptor. - s.fds.hasKey(fd) # and s.fds[fd].events != {} + proc allocSharedArray[T](nsize: int): ptr SharedArray[T] = + result = cast[ptr SharedArray[T]](allocShared0(sizeof(T) * nsize)) - proc `[]`*(s: Selector, fd: SocketHandle): SelectorKey = - ## Retrieves the selector key for ``fd``. - return s.fds[fd] + proc deallocSharedArray[T](sa: ptr SharedArray[T]) = + deallocShared(cast[pointer](sa)) -elif not defined(nimdoc): - # TODO: kqueue for bsd/mac os x. type - Selector* = object - when MultiThreaded: - fds: SharedTable[SocketHandle, SelectorKey] - else: - fds: Table[SocketHandle, SelectorKey] - - proc register*(s: var Selector, fd: SocketHandle, events: set[Event], - data: SelectorData) = - let result = SelectorKey(fd: fd, events: events, data: data) - if s.fds.hasKeyOrPut(fd, result): - raise newException(ValueError, "File descriptor already exists.") - - proc update*(s: var Selector, fd: SocketHandle, events: set[Event]) = - #if not s.fds.hasKey(fd): - # raise newException(ValueError, "File descriptor not found.") - s.fds[fd].events = events - - proc unregister*(s: var Selector, fd: SocketHandle) = - s.fds.del(fd) - - proc close*(s: var Selector) = - when MultiThreaded: deinitSharedTable(s.fds) - - proc timeValFromMilliseconds(timeout: int): TimeVal = - if timeout != -1: - var seconds = timeout div 1000 - result.tv_sec = seconds.int32 - result.tv_usec = ((timeout - seconds * 1000) * 1000).int32 - - proc createFdSet(rd, wr: var TFdSet, s: Selector, m: var int) = - FD_ZERO(rd); FD_ZERO(wr) - for k, v in pairs(s.fds): - if EvRead in v.events: - m = max(m, int(k)) - FD_SET(k, rd) - if EvWrite in v.events: - m = max(m, int(k)) - FD_SET(k, wr) - - proc getReadyFDs(rd, wr: var TFdSet, - s: var Selector): seq[ReadyInfo] = - result = @[] - for k, v in pairs(s.fds): - var events: set[Event] = {} - if FD_ISSET(k, rd) != 0'i32: - events = events + {EvRead} - if FD_ISSET(k, wr) != 0'i32: - events = events + {EvWrite} - result.add((v, events)) - - proc select*(s: var Selector, timeout: int): seq[ReadyInfo] = - var tv {.noInit.}: TimeVal = timeValFromMilliseconds(timeout) - - var rd, wr: TFdSet - var m = 0 - createFdSet(rd, wr, s, m) - - var retCode = 0 - if timeout != -1: - retCode = int(select(cint(m+1), addr(rd), addr(wr), nil, addr(tv))) - else: - retCode = int(select(cint(m+1), addr(rd), addr(wr), nil, nil)) - - if retCode < 0: - raiseOSError(osLastError()) - elif retCode == 0: - return @[] - else: - return getReadyFDs(rd, wr, s) + Event* {.pure.} = enum + Read, Write, Timer, Signal, Process, Vnode, User, Error, Oneshot, + Finished, VnodeWrite, VnodeDelete, VnodeExtend, VnodeAttrib, VnodeLink, + VnodeRename, VnodeRevoke - proc newSelector*(): Selector = - when MultiThreaded: - result.fds = initSharedTable[SocketHandle, SelectorKey]() + type + IOSelectorsException* = object of Exception + + ReadyKey* = object + fd* : int + events*: set[Event] + + SelectorKey[T] = object + ident: int + events: set[Event] + param: int + data: T + + proc raiseIOSelectorsError[T](message: T) = + var msg = "" + when T is string: + msg.add(message) + elif T is OSErrorCode: + msg.add(osErrorMsg(message) & " (code: " & $int(message) & ")") else: - result.fds = initTable[SocketHandle, SelectorKey]() - - proc contains*(s: Selector, fd: SocketHandle): bool = - return s.fds.hasKey(fd) - - proc `[]`*(s: Selector, fd: SocketHandle): SelectorKey = - return s.fds[fd] - -proc contains*(s: Selector, key: SelectorKey): bool = - ## Determines whether selector contains this selector key. More accurate - ## than checking if the file descriptor is in the selector because it - ## ensures that the keys are equal. File descriptors may not always be - ## unique especially when an fd is closed and then a new one is opened, - ## the new one may have the same value. - when not defined(nimdoc): - return key.fd in s and s.fds[key.fd] == key - -proc len*(s: Selector): int = - ## Retrieves the number of registered file descriptors in this Selector. - when not defined(nimdoc): - return s.fds.len - -{.deprecated: [TEvent: Event, PSelectorKey: SelectorKey, - TReadyInfo: ReadyInfo, PSelector: Selector].} - - -when not defined(testing) and isMainModule and not defined(nimdoc): - # Select() - import sockets + msg.add("Internal Error\n") + var err = newException(IOSelectorsException, msg) + raise err + + proc setNonBlocking(fd: cint) {.inline.} = + setBlocking(fd.SocketHandle, false) + + when not defined(windows): + import posix + + template setKey(s, pident, pevents, pparam, pdata: untyped) = + var skey = addr(s.fds[pident]) + skey.ident = pident + skey.events = pevents + skey.param = pparam + skey.data = data + + when ioselSupportedPlatform: + template blockSignals(newmask: var Sigset, oldmask: var Sigset) = + when hasThreadSupport: + if posix.pthread_sigmask(SIG_BLOCK, newmask, oldmask) == -1: + raiseIOSelectorsError(osLastError()) + else: + if posix.sigprocmask(SIG_BLOCK, newmask, oldmask) == -1: + raiseIOSelectorsError(osLastError()) - when MultiThreaded: - type - SockWrapper = object - sock: Socket - else: - type - SockWrapper = ref object of RootObj - sock: Socket - - var sock = socket() - if sock == sockets.invalidSocket: raiseOSError(osLastError()) - #sock.setBlocking(false) - sock.connect("irc.freenode.net", Port(6667)) - - var selector = newSelector() - var data = SockWrapper(sock: sock) - when MultiThreaded: - selector.register(sock.getFD, {EvWrite}, addr data) + template unblockSignals(newmask: var Sigset, oldmask: var Sigset) = + when hasThreadSupport: + if posix.pthread_sigmask(SIG_UNBLOCK, newmask, oldmask) == -1: + raiseIOSelectorsError(osLastError()) + else: + if posix.sigprocmask(SIG_UNBLOCK, newmask, oldmask) == -1: + raiseIOSelectorsError(osLastError()) + + when defined(linux): + include ioselects/ioselectors_epoll + elif bsdPlatform: + include ioselects/ioselectors_kqueue + elif defined(windows): + include ioselects/ioselectors_select + elif defined(solaris): + include ioselects/ioselectors_poll # need to replace it with event ports + elif defined(genode): + include ioselects/ioselectors_select # TODO: use the native VFS layer else: - selector.register(sock.getFD, {EvWrite}, data) - var i = 0 - while true: - let ready = selector.select(1000) - echo ready.len - if ready.len > 0: echo ready[0].events - i.inc - if i == 6: - selector.unregister(sock.getFD) - selector.close() - break + include ioselects/ioselectors_poll + +{.deprecated: [setEvent: trigger].} +{.deprecated: [register: registerHandle].} +{.deprecated: [update: updateHandle].} diff --git a/lib/pure/streams.nim b/lib/pure/streams.nim index 69f673990..354e07da3 100644 --- a/lib/pure/streams.nim +++ b/lib/pure/streams.nim @@ -224,6 +224,38 @@ proc peekInt64*(s: Stream): int64 = ## peeks an int64 from the stream `s`. Raises `EIO` if an error occurred. peek(s, result) +proc readUint8*(s: Stream): uint8 = + ## reads an uint8 from the stream `s`. Raises `EIO` if an error occurred. + read(s, result) + +proc peekUint8*(s: Stream): uint8 = + ## peeks an uint8 from the stream `s`. Raises `EIO` if an error occurred. + peek(s, result) + +proc readUint16*(s: Stream): uint16 = + ## reads an uint16 from the stream `s`. Raises `EIO` if an error occurred. + read(s, result) + +proc peekUint16*(s: Stream): uint16 = + ## peeks an uint16 from the stream `s`. Raises `EIO` if an error occurred. + peek(s, result) + +proc readUint32*(s: Stream): uint32 = + ## reads an uint32 from the stream `s`. Raises `EIO` if an error occurred. + read(s, result) + +proc peekUint32*(s: Stream): uint32 = + ## peeks an uint32 from the stream `s`. Raises `EIO` if an error occurred. + peek(s, result) + +proc readUint64*(s: Stream): uint64 = + ## reads an uint64 from the stream `s`. Raises `EIO` if an error occurred. + read(s, result) + +proc peekUint64*(s: Stream): uint64 = + ## peeks an uint64 from the stream `s`. Raises `EIO` if an error occurred. + peek(s, result) + proc readFloat32*(s: Stream): float32 = ## reads a float32 from the stream `s`. Raises `EIO` if an error occurred. read(s, result) diff --git a/lib/pure/strformat.nim b/lib/pure/strformat.nim new file mode 100644 index 000000000..180cbcbec --- /dev/null +++ b/lib/pure/strformat.nim @@ -0,0 +1,619 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2017 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +##[ +String `interpolation`:idx: / `format`:idx: inspired by +Python's ``f``-strings. + +Examples: + +.. code-block:: nim + + doAssert fmt"""{"abc":>4}""" == " abc" + doAssert fmt"""{"abc":<4}""" == "abc " + + doAssert fmt"{-12345:08}" == "-0012345" + doAssert fmt"{-1:3}" == " -1" + doAssert fmt"{-1:03}" == "-01" + doAssert fmt"{16:#X}" == "0x10" + + doAssert fmt"{123.456}" == "123.456" + doAssert fmt"{123.456:>9.3f}" == " 123.456" + doAssert fmt"{123.456:9.3f}" == " 123.456" + doAssert fmt"{123.456:9.4f}" == " 123.4560" + doAssert fmt"{123.456:>9.0f}" == " 123." + doAssert fmt"{123.456:<9.4f}" == "123.4560 " + + doAssert fmt"{123.456:e}" == "1.234560e+02" + doAssert fmt"{123.456:>13e}" == " 1.234560e+02" + doAssert fmt"{123.456:13e}" == " 1.234560e+02" + + +An expression like ``fmt"{key} is {value:arg} {{z}}"`` is transformed into: + +.. code-block:: nim + var temp = newStringOfCap(educatedCapGuess) + format(key, temp) + format(" is ", temp) + format(value, arg, temp) + format(" {z}", temp) + temp + +Parts of the string that are enclosed in the curly braces are interpreted +as Nim code, to escape an ``{`` or ``}`` double it. + +``fmt`` delegates most of the work to an open overloaded set +of ``format`` procs. The required signature for a type ``T`` that supports +formatting is usually ``proc format(x: T; result: var string)`` for efficiency +but can also be ``proc format(x: T): string``. ``add`` and ``$`` procs are +used as the fallback implementation. + +This is the concrete lookup algorithm that ``fmt`` uses: + +.. code-block:: nim + + when compiles(format(arg, res)): + format(arg, res) + elif compiles(format(arg)): + res.add format(arg) + elif compiles(add(res, arg)): + res.add(arg) + else: + res.add($arg) + + +The subexpression after the colon +(``arg`` in ``fmt"{key} is {value:arg} {{z}}"``) is an optional argument +passed to ``format``. + +If an optional argument is present the following lookup algorithm is used: + +.. code-block:: nim + + when compiles(format(arg, option, res)): + format(arg, option, res) + else: + res.add format(arg, option) + + +For strings and numeric types the optional argument is a so-called +"standard format specifier". + + +Standard format specifier +========================= + + +The general form of a standard format specifier is:: + + [[fill]align][sign][#][0][minimumwidth][.precision][type] + +The square brackets ``[]`` indicate an optional element. + +The optional align flag can be one of the following: + +'<' + Forces the field to be left-aligned within the available + space. (This is the default for strings.) + +'>' + Forces the field to be right-aligned within the available space. + (This is the default for numbers.) + +'^' + Forces the field to be centered within the available space. + +Note that unless a minimum field width is defined, the field width +will always be the same size as the data to fill it, so that the alignment +option has no meaning in this case. + +The optional 'fill' character defines the character to be used to pad +the field to the minimum width. The fill character, if present, must be +followed by an alignment flag. + +The 'sign' option is only valid for numeric types, and can be one of the following: + +================= ==================================================== + Sign Meaning +================= ==================================================== +``+`` Indicates that a sign should be used for both + positive as well as negative numbers. +``-`` Indicates that a sign should be used only for + negative numbers (this is the default behavior). +(space) Indicates that a leading space should be used on + positive numbers. +================= ==================================================== + +If the '#' character is present, integers use the 'alternate form' for formatting. +This means that binary, octal, and hexadecimal output will be prefixed +with '0b', '0o', and '0x', respectively. + +'width' is a decimal integer defining the minimum field width. If not specified, +then the field width will be determined by the content. + +If the width field is preceded by a zero ('0') character, this enables +zero-padding. + +The 'precision' is a decimal number indicating how many digits should be displayed +after the decimal point in a floating point conversion. For non-numeric types the +field indicates the maximum field size - in other words, how many characters will +be used from the field content. The precision is ignored for integer conversions. + +Finally, the 'type' determines how the data should be presented. + +The available integer presentation types are: + + +================= ==================================================== + Type Result +================= ==================================================== +``b`` Binary. Outputs the number in base 2. +``d`` Decimal Integer. Outputs the number in base 10. +``o`` Octal format. Outputs the number in base 8. +``x`` Hex format. Outputs the number in base 16, using + lower-case letters for the digits above 9. +``X`` Hex format. Outputs the number in base 16, using + uppercase letters for the digits above 9. +(None) the same as 'd' +================= ==================================================== + + +The available floating point presentation types are: + +================= ==================================================== + Type Result +================= ==================================================== +``e`` Exponent notation. Prints the number in scientific + notation using the letter 'e' to indicate the + exponent. +``E`` Exponent notation. Same as 'e' except it converts + the number to uppercase. +``f`` Fixed point. Displays the number as a fixed-point + number. +``F`` Fixed point. Same as 'f' except it converts the + number to uppercase. +``g`` General format. This prints the number as a + fixed-point number, unless the number is too + large, in which case it switches to 'e' + exponent notation. +``G`` General format. Same as 'g' except switches to 'E' + if the number gets to large. +(None) similar to 'g', except that it prints at least one + digit after the decimal point. +================= ==================================================== + + +Future directions +================= + +A curly expression with commas in it like ``{x, argA, argB}`` could be +transformed to ``format(x, argA, argB, res)`` in order to support +formatters that do not need to parse a custom language within a custom +language but instead prefer to use Nim's existing syntax. This also +helps in readability since there is only so much you can cram into +single letter DSLs. + +]## + +import macros, parseutils, unicode +import strutils + +template callFormat(res, arg) {.dirty.} = + when arg is string: + # workaround in order to circumvent 'strutils.format' which matches + # too but doesn't adhere to our protocol. + res.add arg + elif compiles(format(arg, res)): + format(arg, res) + elif compiles(format(arg)): + res.add format(arg) + elif compiles(add(res, arg)): + res.add(arg) + else: + res.add($arg) + +template callFormatOption(res, arg, option) {.dirty.} = + when compiles(format(arg, option, res)): + format(arg, option, res) + else: + res.add format(arg, option) + +macro fmt*(pattern: string): untyped = + ## For a specification of the ``fmt`` macro, see the module level documentation. + runnableExamples: + template check(actual, expected: string) = + doAssert actual == expected + + from strutils import toUpperAscii, repeat + + # Basic tests + let s = "string" + check fmt"{0} {s}", "0 string" + check fmt"{s[0..2].toUpperAscii}", "STR" + check fmt"{-10:04}", "-010" + check fmt"{-10:<04}", "-010" + check fmt"{-10:>04}", "-010" + check fmt"0x{10:02X}", "0x0A" + + check fmt"{10:#04X}", "0x0A" + + check fmt"""{"test":#>5}""", "#test" + check fmt"""{"test":>5}""", " test" + + check fmt"""{"test":#^7}""", "#test##" + + check fmt"""{"test": <5}""", "test " + check fmt"""{"test":<5}""", "test " + check fmt"{1f:.3f}", "1.000" + check fmt"Hello, {s}!", "Hello, string!" + + # Tests for identifers without parenthesis + check fmt"{s} works{s}", "string worksstring" + check fmt"{s:>7}", " string" + doAssert(not compiles(fmt"{s_works}")) # parsed as identifier `s_works` + + # Misc general tests + check fmt"{{}}", "{}" + check fmt"{0}%", "0%" + check fmt"{0}%asdf", "0%asdf" + check fmt("\n{\"\\n\"}\n"), "\n\n\n" + check fmt"""{"abc"}s""", "abcs" + + # String tests + check fmt"""{"abc"}""", "abc" + check fmt"""{"abc":>4}""", " abc" + check fmt"""{"abc":<4}""", "abc " + check fmt"""{"":>4}""", " " + check fmt"""{"":<4}""", " " + + # Int tests + check fmt"{12345}", "12345" + check fmt"{ - 12345}", "-12345" + check fmt"{12345:6}", " 12345" + check fmt"{12345:>6}", " 12345" + check fmt"{12345:4}", "12345" + check fmt"{12345:08}", "00012345" + check fmt"{-12345:08}", "-0012345" + check fmt"{0:0}", "0" + check fmt"{0:02}", "00" + check fmt"{-1:3}", " -1" + check fmt"{-1:03}", "-01" + check fmt"{10}", "10" + check fmt"{16:#X}", "0x10" + check fmt"{16:^#7X}", " 0x10 " + check fmt"{16:^+#7X}", " +0x10 " + + # Hex tests + check fmt"{0:x}", "0" + check fmt"{-0:x}", "0" + check fmt"{255:x}", "ff" + check fmt"{255:X}", "FF" + check fmt"{-255:x}", "-ff" + check fmt"{-255:X}", "-FF" + check fmt"{255:x} uNaffeCteD CaSe", "ff uNaffeCteD CaSe" + check fmt"{255:X} uNaffeCteD CaSe", "FF uNaffeCteD CaSe" + check fmt"{255:4x}", " ff" + check fmt"{255:04x}", "00ff" + check fmt"{-255:4x}", " -ff" + check fmt"{-255:04x}", "-0ff" + + # Float tests + check fmt"{123.456}", "123.456" + check fmt"{-123.456}", "-123.456" + check fmt"{123.456:.3f}", "123.456" + check fmt"{123.456:+.3f}", "+123.456" + check fmt"{-123.456:+.3f}", "-123.456" + check fmt"{-123.456:.3f}", "-123.456" + check fmt"{123.456:1g}", "123.456" + check fmt"{123.456:.1f}", "123.5" + check fmt"{123.456:.0f}", "123." + #check fmt"{123.456:.0f}", "123." + check fmt"{123.456:>9.3f}", " 123.456" + check fmt"{123.456:9.3f}", " 123.456" + check fmt"{123.456:>9.4f}", " 123.4560" + check fmt"{123.456:>9.0f}", " 123." + check fmt"{123.456:<9.4f}", "123.4560 " + + # Float (scientific) tests + check fmt"{123.456:e}", "1.234560e+02" + check fmt"{123.456:>13e}", " 1.234560e+02" + check fmt"{123.456:<13e}", "1.234560e+02 " + check fmt"{123.456:.1e}", "1.2e+02" + check fmt"{123.456:.2e}", "1.23e+02" + check fmt"{123.456:.3e}", "1.235e+02" + + # Note: times.format adheres to the format protocol. Test that this + # works: + import times + + var nullTime: TimeInfo + check fmt"{nullTime:yyyy-mm-dd}", "0000-00-00" + + # Unicode string tests + check fmt"""{"αβγ"}""", "αβγ" + check fmt"""{"αβγ":>5}""", " αβγ" + check fmt"""{"αβγ":<5}""", "αβγ " + check fmt"""a{"a"}α{"α"}€{"€"}𐍈{"𐍈"}""", "aaαα€€𐍈𐍈" + check fmt"""a{"a":2}α{"α":2}€{"€":2}𐍈{"𐍈":2}""", "aa αα €€ 𐍈𐍈 " + # Invalid unicode sequences should be handled as plain strings. + # Invalid examples taken from: https://stackoverflow.com/a/3886015/1804173 + let invalidUtf8 = [ + "\xc3\x28", "\xa0\xa1", + "\xe2\x28\xa1", "\xe2\x82\x28", + "\xf0\x28\x8c\xbc", "\xf0\x90\x28\xbc", "\xf0\x28\x8c\x28" + ] + for s in invalidUtf8: + check fmt"{s:>5}", repeat(" ", 5-s.len) & s + + if pattern.kind notin {nnkStrLit..nnkTripleStrLit}: + error "fmt only works with string literals", pattern + let f = pattern.strVal + var i = 0 + let res = genSym(nskVar, "fmtRes") + result = newNimNode(nnkStmtListExpr, lineInfoFrom=pattern) + result.add newVarStmt(res, newCall(bindSym"newStringOfCap", newLit(f.len + count(f, '{')*10))) + var strlit = "" + while i < f.len: + if f[i] == '{': + inc i + if f[i] == '{': + inc i + strlit.add '{' + else: + if strlit.len > 0: + result.add newCall(bindSym"add", res, newLit(strlit)) + strlit = "" + + var subexpr = "" + while i < f.len and f[i] != '}' and f[i] != ':': + subexpr.add f[i] + inc i + let x = parseExpr(subexpr) + + if f[i] == ':': + inc i + var options = "" + while i < f.len and f[i] != '}': + options.add f[i] + inc i + result.add getAst(callFormatOption(res, x, newLit(options))) + else: + result.add getAst(callFormat(res, x)) + if f[i] == '}': + inc i + else: + doAssert false, "invalid format string: missing '}'" + elif f[i] == '}': + if f[i+1] == '}': + strlit.add '}' + inc i, 2 + else: + doAssert false, "invalid format string: '}' instead of '}}'" + inc i + else: + strlit.add f[i] + inc i + if strlit.len > 0: + result.add newCall(bindSym"add", res, newLit(strlit)) + result.add res + when defined(debugFmtDsl): + echo repr result + +proc mkDigit(v: int, typ: char): string {.inline.} = + assert(v < 26) + if v < 10: + result = $chr(ord('0') + v) + else: + result = $chr(ord(if typ == 'x': 'a' else: 'A') + v - 10) + +proc alignString*(s: string, minimumWidth: int; align = '\0'; fill = ' '): string = + ## Aligns ``s`` using ``fill`` char. + ## This is only of interest if you want to write a custom ``format`` proc that + ## should support the standard format specifiers. + if minimumWidth == 0: + result = s + else: + let sRuneLen = if s.validateUtf8 == -1: s.runeLen else: s.len + let toFill = minimumWidth - sRuneLen + if toFill <= 0: + result = s + elif align == '<' or align == '\0': + result = s & repeat(fill, toFill) + elif align == '^': + let half = toFill div 2 + result = repeat(fill, half) & s & repeat(fill, toFill - half) + else: + result = repeat(fill, toFill) & s + +type + StandardFormatSpecifier* = object ## Type that describes "standard format specifiers". + fill*, align*: char ## Desired fill and alignment. + sign*: char ## Desired sign. + alternateForm*: bool ## Whether to prefix binary, octal and hex numbers + ## with ``0b``, ``0o``, ``0x``. + padWithZero*: bool ## Whether to pad with zeros rather than spaces. + minimumWidth*, precision*: int ## Desired minium width and precision. + typ*: char ## Type like 'f', 'g' or 'd'. + endPosition*: int ## End position in the format specifier after + ## ``parseStandardFormatSpecifier`` returned. + +proc formatInt(n: SomeNumber; radix: int; spec: StandardFormatSpecifier): string = + ## Converts ``n`` to string. If ``n`` is `SomeReal`, it casts to `int64`. + ## Conversion is done using ``radix``. If result's length is lesser than + ## ``minimumWidth``, it aligns result to the right or left (depending on ``a``) + ## with ``fill`` char. + when n is SomeUnsignedInt: + var v = n.uint64 + let negative = false + else: + var v = n.int64 + let negative = v.int64 < 0 + if negative: + # FIXME: overflow error for low(int64) + v = v * -1 + + var xx = "" + if spec.alternateForm: + case spec.typ + of 'X': xx = "0x" + of 'x': xx = "0x" + of 'b': xx = "0b" + of 'o': xx = "0o" + else: discard + + if v == 0: + result = "0" + else: + result = "" + while v > type(v)(0): + let d = v mod type(v)(radix) + v = v div type(v)(radix) + result.add(mkDigit(d.int, spec.typ)) + for idx in 0..<(result.len div 2): + swap result[idx], result[result.len - idx - 1] + if spec.padWithZero: + let sign = negative or spec.sign != '-' + let toFill = spec.minimumWidth - result.len - xx.len - ord(sign) + if toFill > 0: + result = repeat('0', toFill) & result + + if negative: + result = "-" & xx & result + elif spec.sign != '-': + result = spec.sign & xx & result + else: + result = xx & result + + if spec.align == '<': + for i in result.len..<spec.minimumWidth: + result.add(spec.fill) + else: + let toFill = spec.minimumWidth - result.len + if spec.align == '^': + let half = toFill div 2 + result = repeat(spec.fill, half) & result & repeat(spec.fill, toFill - half) + else: + if toFill > 0: + result = repeat(spec.fill, toFill) & result + +proc parseStandardFormatSpecifier*(s: string; start = 0; + ignoreUnknownSuffix = false): StandardFormatSpecifier = + ## An exported helper proc that parses the "standard format specifiers", + ## as specified by the grammar:: + ## + ## [[fill]align][sign][#][0][minimumwidth][.precision][type] + ## + ## This is only of interest if you want to write a custom ``format`` proc that + ## should support the standard format specifiers. If ``ignoreUnknownSuffix`` is true, + ## an unknown suffix after the ``type`` field is not an error. + const alignChars = {'<', '>', '^'} + result.fill = ' ' + result.align = '\0' + result.sign = '-' + var i = start + if i + 1 < s.len and s[i+1] in alignChars: + result.fill = s[i] + result.align = s[i+1] + inc i, 2 + elif i < s.len and s[i] in alignChars: + result.align = s[i] + inc i + + if i < s.len and s[i] in {'-', '+', ' '}: + result.sign = s[i] + inc i + + if i < s.len and s[i] == '#': + result.alternateForm = true + inc i + + if i+1 < s.len and s[i] == '0' and s[i+1] in {'0'..'9'}: + result.padWithZero = true + inc i + + let parsedLength = parseSaturatedNatural(s, result.minimumWidth, i) + inc i, parsedLength + if i < s.len and s[i] == '.': + inc i + let parsedLengthB = parseSaturatedNatural(s, result.precision, i) + inc i, parsedLengthB + else: + result.precision = -1 + + if i < s.len and s[i] in {'A'..'Z', 'a'..'z'}: + result.typ = s[i] + inc i + result.endPosition = i + if i != s.len and not ignoreUnknownSuffix: + raise newException(ValueError, + "invalid format string, cannot parse: " & s[i..^1]) + + +proc format*(value: SomeInteger; specifier: string; res: var string) = + ## Standard format implementation for ``SomeInteger``. It makes little + ## sense to call this directly, but it is required to exist + ## by the ``fmt`` macro. + let spec = parseStandardFormatSpecifier(specifier) + var radix = 10 + case spec.typ + of 'x', 'X': radix = 16 + of 'd', '\0': discard + of 'b': radix = 2 + of 'o': radix = 8 + else: + raise newException(ValueError, + "invalid type in format string for number, expected one " & + " of 'x', 'X', 'b', 'd', 'o' but got: " & spec.typ) + res.add formatInt(value, radix, spec) + +proc format*(value: SomeReal; specifier: string; res: var string) = + ## Standard format implementation for ``SomeReal``. It makes little + ## sense to call this directly, but it is required to exist + ## by the ``fmt`` macro. + let spec = parseStandardFormatSpecifier(specifier) + + var fmode = ffDefault + case spec.typ + of 'e', 'E': + fmode = ffScientific + of 'f', 'F': + fmode = ffDecimal + of 'g', 'G': + fmode = ffDefault + of '\0': discard + else: + raise newException(ValueError, + "invalid type in format string for number, expected one " & + " of 'e', 'E', 'f', 'F', 'g', 'G' but got: " & spec.typ) + + var f = formatBiggestFloat(value, fmode, spec.precision) + if value >= 0.0 and spec.sign != '-': + f = spec.sign & f + # the default for numbers is right-alignment: + let align = if spec.align == '\0': '>' else: spec.align + let result = alignString(f, spec.minimumWidth, + align, spec.fill) + if spec.typ in {'A'..'Z'}: + res.add toUpperAscii(result) + else: + res.add result + +proc format*(value: string; specifier: string; res: var string) = + ## Standard format implementation for ``string``. It makes little + ## sense to call this directly, but it is required to exist + ## by the ``fmt`` macro. + let spec = parseStandardFormatSpecifier(specifier) + var fmode = ffDefault + case spec.typ + of 's', '\0': discard + else: + raise newException(ValueError, + "invalid type in format string for string, expected 's', but got " & + spec.typ) + res.add alignString(value, spec.minimumWidth, spec.align, spec.fill) diff --git a/lib/pure/strscans.nim b/lib/pure/strscans.nim index a54556915..2bd87837f 100644 --- a/lib/pure/strscans.nim +++ b/lib/pure/strscans.nim @@ -31,7 +31,10 @@ As can be seen from the examples, strings are matched verbatim except for substrings starting with ``$``. These constructions are available: ================= ======================================================== -``$i`` Matches an integer. This uses ``parseutils.parseInt``. +``$b`` Matches a binary integer. This uses ``parseutils.parseBin``. +``$o`` Matches an octal integer. This uses ``parseutils.parseOct``. +``$i`` Matches a decimal integer. This uses ``parseutils.parseInt``. +``$h`` Matches a hex integer. This uses ``parseutils.parseHex``. ``$f`` Matches a floating pointer number. Uses ``parseFloat``. ``$w`` Matches an ASCII identifier: ``[A-Z-a-z_][A-Za-z_0-9]*``. ``$s`` Skips optional whitespace. @@ -330,19 +333,37 @@ macro scanf*(input: string; pattern: static[string]; results: varargs[typed]): b conds.add resLen.notZero conds.add resLen of 'w': - if i < results.len or getType(results[i]).typeKind != ntyString: + if i < results.len and getType(results[i]).typeKind == ntyString: matchBind "parseIdent" else: error("no string var given for $w") inc i + of 'b': + if i < results.len and getType(results[i]).typeKind == ntyInt: + matchBind "parseBin" + else: + error("no int var given for $b") + inc i + of 'o': + if i < results.len and getType(results[i]).typeKind == ntyInt: + matchBind "parseOct" + else: + error("no int var given for $o") + inc i of 'i': - if i < results.len or getType(results[i]).typeKind != ntyInt: + if i < results.len and getType(results[i]).typeKind == ntyInt: matchBind "parseInt" else: - error("no int var given for $d") + error("no int var given for $i") + inc i + of 'h': + if i < results.len and getType(results[i]).typeKind == ntyInt: + matchBind "parseHex" + else: + error("no int var given for $h") inc i of 'f': - if i < results.len or getType(results[i]).typeKind != ntyFloat: + if i < results.len and getType(results[i]).typeKind == ntyFloat: matchBind "parseFloat" else: error("no float var given for $f") @@ -357,7 +378,7 @@ macro scanf*(input: string; pattern: static[string]; results: varargs[typed]): b else: error("invalid format string") of '*', '+': - if i < results.len or getType(results[i]).typeKind != ntyString: + if i < results.len and getType(results[i]).typeKind == ntyString: var min = ord(pattern[p] == '+') var q=p+1 var token = "" @@ -441,7 +462,7 @@ template success*(x: int): bool = x != 0 template nxt*(input: string; idx, step: int = 1) = inc(idx, step) macro scanp*(input, idx: typed; pattern: varargs[untyped]): bool = - ## See top level documentation of his module of how ``scanp`` works. + ## ``scanp`` is currently undocumented. type StmtTriple = tuple[init, cond, action: NimNode] template interf(x): untyped = bindSym(x, brForceOpen) @@ -563,12 +584,12 @@ macro scanp*(input, idx: typed; pattern: varargs[untyped]): bool = of nnkCurlyExpr: if it.len == 3 and it[1].kind == nnkIntLit and it[2].kind == nnkIntLit: var h = newTree(nnkPar, it[0]) - for count in 2..it[1].intVal: h.add(it[0]) + for count in 2i64 .. it[1].intVal: h.add(it[0]) for count in it[1].intVal .. it[2].intVal-1: h.add(newTree(nnkPrefix, ident"?", it[0])) result = atm(h, input, idx, attached) elif it.len == 2 and it[1].kind == nnkIntLit: var h = newTree(nnkPar, it[0]) - for count in 2..it[1].intVal: h.add(it[0]) + for count in 2i64 .. it[1].intVal: h.add(it[0]) result = atm(h, input, idx, attached) else: error("invalid pattern") @@ -645,6 +666,14 @@ when isMainModule: doAssert intval == 89 doAssert floatVal == 33.25 + var binval: int + var octval: int + var hexval: int + doAssert scanf("0b0101 0o1234 0xabcd", "$b$s$o$s$h", binval, octval, hexval) + doAssert binval == 0b0101 + doAssert octval == 0o1234 + doAssert hexval == 0xabcd + let xx = scanf("$abc", "$$$i", intval) doAssert xx == false diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index cc0f474f4..dbb4db781 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -32,10 +32,6 @@ when defined(nimOldSplit): else: {.pragma: deprecatedSplit.} -type - CharSet* {.deprecated.} = set[char] # for compatibility with Nim -{.deprecated: [TCharSet: CharSet].} - const Whitespace* = {' ', '\t', '\v', '\r', '\l', '\f'} ## All the characters that count as whitespace. @@ -78,40 +74,40 @@ proc isAlphaAscii*(c: char): bool {.noSideEffect, procvar, return c in Letters proc isAlphaNumeric*(c: char): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsAlphaNumericChar".}= + rtl, extern: "nsuIsAlphaNumericChar".} = ## Checks whether or not `c` is alphanumeric. ## ## This checks a-z, A-Z, 0-9 ASCII characters only. - return c in Letters or c in Digits + return c in Letters+Digits proc isDigit*(c: char): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsDigitChar".}= + rtl, extern: "nsuIsDigitChar".} = ## Checks whether or not `c` is a number. ## ## This checks 0-9 ASCII characters only. return c in Digits proc isSpaceAscii*(c: char): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsSpaceAsciiChar".}= + rtl, extern: "nsuIsSpaceAsciiChar".} = ## Checks whether or not `c` is a whitespace character. return c in Whitespace proc isLowerAscii*(c: char): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsLowerAsciiChar".}= + rtl, extern: "nsuIsLowerAsciiChar".} = ## Checks whether or not `c` is a lower case character. ## ## This checks ASCII characters only. return c in {'a'..'z'} proc isUpperAscii*(c: char): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsUpperAsciiChar".}= + rtl, extern: "nsuIsUpperAsciiChar".} = ## Checks whether or not `c` is an upper case character. ## ## This checks ASCII characters only. return c in {'A'..'Z'} proc isAlphaAscii*(s: string): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsAlphaAsciiStr".}= + rtl, extern: "nsuIsAlphaAsciiStr".} = ## Checks whether or not `s` is alphabetical. ## ## This checks a-z, A-Z ASCII characters only. @@ -123,10 +119,10 @@ proc isAlphaAscii*(s: string): bool {.noSideEffect, procvar, result = true for c in s: - result = c.isAlphaAscii() and result + if not c.isAlphaAscii(): return false proc isAlphaNumeric*(s: string): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsAlphaNumericStr".}= + rtl, extern: "nsuIsAlphaNumericStr".} = ## Checks whether or not `s` is alphanumeric. ## ## This checks a-z, A-Z, 0-9 ASCII characters only. @@ -138,10 +134,11 @@ proc isAlphaNumeric*(s: string): bool {.noSideEffect, procvar, result = true for c in s: - result = c.isAlphaNumeric() and result + if not c.isAlphaNumeric(): + return false proc isDigit*(s: string): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsDigitStr".}= + rtl, extern: "nsuIsDigitStr".} = ## Checks whether or not `s` is a numeric value. ## ## This checks 0-9 ASCII characters only. @@ -153,10 +150,11 @@ proc isDigit*(s: string): bool {.noSideEffect, procvar, result = true for c in s: - result = c.isDigit() and result + if not c.isDigit(): + return false proc isSpaceAscii*(s: string): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsSpaceAsciiStr".}= + rtl, extern: "nsuIsSpaceAsciiStr".} = ## Checks whether or not `s` is completely whitespace. ## ## Returns true if all characters in `s` are whitespace @@ -170,7 +168,7 @@ proc isSpaceAscii*(s: string): bool {.noSideEffect, procvar, return false proc isLowerAscii*(s: string): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsLowerAsciiStr".}= + rtl, extern: "nsuIsLowerAsciiStr".} = ## Checks whether or not `s` contains all lower case characters. ## ## This checks ASCII characters only. @@ -185,7 +183,7 @@ proc isLowerAscii*(s: string): bool {.noSideEffect, procvar, true proc isUpperAscii*(s: string): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsUpperAsciiStr".}= + rtl, extern: "nsuIsUpperAsciiStr".} = ## Checks whether or not `s` contains all upper case characters. ## ## This checks ASCII characters only. @@ -504,16 +502,15 @@ template splitCommon(s, sep, maxsplit, sepLen) = var last = 0 var splits = maxsplit - if len(s) > 0: - while last <= len(s): - var first = last - while last < len(s) and not stringHasSep(s, last, sep): - inc(last) - if splits == 0: last = len(s) - yield substr(s, first, last-1) - if splits == 0: break - dec(splits) - inc(last, sepLen) + while last <= len(s): + var first = last + while last < len(s) and not stringHasSep(s, last, sep): + inc(last) + if splits == 0: last = len(s) + yield substr(s, first, last-1) + if splits == 0: break + dec(splits) + inc(last, sepLen) template oldSplit(s, seps, maxsplit) = var last = 0 @@ -576,15 +573,46 @@ iterator split*(s: string, seps: set[char] = Whitespace, else: splitCommon(s, seps, maxsplit, 1) -iterator splitWhitespace*(s: string): string = - ## Splits at whitespace. - oldSplit(s, Whitespace, -1) +iterator splitWhitespace*(s: string, maxsplit: int = -1): string = + ## Splits the string ``s`` at whitespace stripping leading and trailing + ## whitespace if necessary. If ``maxsplit`` is specified and is positive, + ## no more than ``maxsplit`` splits is made. + ## + ## The following code: + ## + ## .. code-block:: nim + ## let s = " foo \t bar baz " + ## for ms in [-1, 1, 2, 3]: + ## echo "------ maxsplit = ", ms, ":" + ## for item in s.splitWhitespace(maxsplit=ms): + ## echo '"', item, '"' + ## + ## ...results in: + ## + ## .. code-block:: + ## ------ maxsplit = -1: + ## "foo" + ## "bar" + ## "baz" + ## ------ maxsplit = 1: + ## "foo" + ## "bar baz " + ## ------ maxsplit = 2: + ## "foo" + ## "bar" + ## "baz " + ## ------ maxsplit = 3: + ## "foo" + ## "bar" + ## "baz" + ## + oldSplit(s, Whitespace, maxsplit) -proc splitWhitespace*(s: string): seq[string] {.noSideEffect, +proc splitWhitespace*(s: string, maxsplit: int = -1): seq[string] {.noSideEffect, rtl, extern: "nsuSplitWhitespace".} = - ## The same as the `splitWhitespace <#splitWhitespace.i,string>`_ + ## The same as the `splitWhitespace <#splitWhitespace.i,string,int>`_ ## iterator, but is a proc that returns a sequence of substrings. - accumulateResult(splitWhitespace(s)) + accumulateResult(splitWhitespace(s, maxsplit)) iterator split*(s: string, sep: char, maxsplit: int = -1): string = ## Splits the string `s` into substrings using a single separator. @@ -640,36 +668,35 @@ template rsplitCommon(s, sep, maxsplit, sepLen) = splits = maxsplit startPos = 0 - if len(s) > 0: - # go to -1 in order to get separators at the beginning - while first >= -1: - while first >= 0 and not stringHasSep(s, first, sep): - dec(first) + # go to -1 in order to get separators at the beginning + while first >= -1: + while first >= 0 and not stringHasSep(s, first, sep): + dec(first) - if splits == 0: - # No more splits means set first to the beginning - first = -1 + if splits == 0: + # No more splits means set first to the beginning + first = -1 - if first == -1: - startPos = 0 - else: - startPos = first + sepLen + if first == -1: + startPos = 0 + else: + startPos = first + sepLen - yield substr(s, startPos, last) + yield substr(s, startPos, last) - if splits == 0: - break + if splits == 0: + break - dec(splits) - dec(first) + dec(splits) + dec(first) - last = first + last = first iterator rsplit*(s: string, seps: set[char] = Whitespace, maxsplit: int = -1): string = ## Splits the string `s` into substrings from the right using a ## string separator. Works exactly the same as `split iterator - ## <#split.i,string,char>`_ except in reverse order. + ## <#split.i,string,char,int>`_ except in reverse order. ## ## .. code-block:: nim ## for piece in "foo bar".rsplit(WhiteSpace): @@ -689,7 +716,7 @@ iterator rsplit*(s: string, sep: char, maxsplit: int = -1): string = ## Splits the string `s` into substrings from the right using a ## string separator. Works exactly the same as `split iterator - ## <#split.i,string,char>`_ except in reverse order. + ## <#split.i,string,char,int>`_ except in reverse order. ## ## .. code-block:: nim ## for piece in "foo:bar".rsplit(':'): @@ -708,7 +735,7 @@ iterator rsplit*(s: string, sep: string, maxsplit: int = -1, keepSeparators: bool = false): string = ## Splits the string `s` into substrings from the right using a ## string separator. Works exactly the same as `split iterator - ## <#split.i,string,string>`_ except in reverse order. + ## <#split.i,string,string,int>`_ except in reverse order. ## ## .. code-block:: nim ## for piece in "foothebar".rsplit("the"): @@ -789,14 +816,20 @@ proc countLines*(s: string): int {.noSideEffect, proc split*(s: string, seps: set[char] = Whitespace, maxsplit: int = -1): seq[string] {. noSideEffect, rtl, extern: "nsuSplitCharSet".} = - ## The same as the `split iterator <#split.i,string,set[char]>`_, but is a + ## The same as the `split iterator <#split.i,string,set[char],int>`_, but is a ## proc that returns a sequence of substrings. + runnableExamples: + doAssert "a,b;c".split({',', ';'}) == @["a", "b", "c"] + doAssert "".split({' '}) == @[""] accumulateResult(split(s, seps, maxsplit)) proc split*(s: string, sep: char, maxsplit: int = -1): seq[string] {.noSideEffect, rtl, extern: "nsuSplitChar".} = - ## The same as the `split iterator <#split.i,string,char>`_, but is a proc + ## The same as the `split iterator <#split.i,string,char,int>`_, but is a proc ## that returns a sequence of substrings. + runnableExamples: + doAssert "a,b,c".split(',') == @["a", "b", "c"] + doAssert "".split(' ') == @[""] accumulateResult(split(s, sep, maxsplit)) proc split*(s: string, sep: string, maxsplit: int = -1): seq[string] {.noSideEffect, @@ -804,7 +837,14 @@ proc split*(s: string, sep: string, maxsplit: int = -1): seq[string] {.noSideEff ## Splits the string `s` into substrings using a string separator. ## ## Substrings are separated by the string `sep`. This is a wrapper around the - ## `split iterator <#split.i,string,string>`_. + ## `split iterator <#split.i,string,string,int>`_. + runnableExamples: + doAssert "a,b,c".split(",") == @["a", "b", "c"] + doAssert "a man a plan a canal panama".split("a ") == @["", "man ", "plan ", "canal panama"] + doAssert "".split("Elon Musk") == @[""] + doAssert "a largely spaced sentence".split(" ") == @["a", "", "largely", "", "", "", "spaced", "sentence"] + + doAssert "a largely spaced sentence".split(" ", maxsplit=1) == @["a", " largely spaced sentence"] doAssert(sep.len > 0) accumulateResult(split(s, sep, maxsplit)) @@ -812,7 +852,7 @@ proc split*(s: string, sep: string, maxsplit: int = -1): seq[string] {.noSideEff proc rsplit*(s: string, seps: set[char] = Whitespace, maxsplit: int = -1): seq[string] {.noSideEffect, rtl, extern: "nsuRSplitCharSet".} = - ## The same as the `rsplit iterator <#rsplit.i,string,set[char]>`_, but is a + ## The same as the `rsplit iterator <#rsplit.i,string,set[char],int>`_, but is a ## proc that returns a sequence of substrings. ## ## A possible common use case for `rsplit` is path manipulation, @@ -834,7 +874,7 @@ proc rsplit*(s: string, seps: set[char] = Whitespace, proc rsplit*(s: string, sep: char, maxsplit: int = -1): seq[string] {.noSideEffect, rtl, extern: "nsuRSplitChar".} = - ## The same as the `split iterator <#rsplit.i,string,char>`_, but is a proc + ## The same as the `rsplit iterator <#rsplit.i,string,char,int>`_, but is a proc ## that returns a sequence of substrings. ## ## A possible common use case for `rsplit` is path manipulation, @@ -856,7 +896,7 @@ proc rsplit*(s: string, sep: char, maxsplit: int = -1): seq[string] proc rsplit*(s: string, sep: string, maxsplit: int = -1): seq[string] {.noSideEffect, rtl, extern: "nsuRSplitString".} = - ## The same as the `split iterator <#rsplit.i,string,string>`_, but is a proc + ## The same as the `rsplit iterator <#rsplit.i,string,string,int>`_, but is a proc ## that returns a sequence of substrings. ## ## A possible common use case for `rsplit` is path manipulation, @@ -873,6 +913,13 @@ proc rsplit*(s: string, sep: string, maxsplit: int = -1): seq[string] ## .. code-block:: nim ## @["Root#Object#Method", "Index"] ## + runnableExamples: + doAssert "a largely spaced sentence".rsplit(" ", maxsplit=1) == @["a largely spaced", "sentence"] + + doAssert "a,b,c".rsplit(",") == @["a", "b", "c"] + doAssert "a man a plan a canal panama".rsplit("a ") == @["", "man ", "plan ", "canal panama"] + doAssert "".rsplit("Elon Musk") == @[""] + doAssert "a largely spaced sentence".rsplit(" ") == @["a", "", "largely", "", "", "", "spaced", "sentence"] accumulateResult(rsplit(s, sep, maxsplit)) result.reverse() @@ -1062,8 +1109,8 @@ proc align*(s: string, count: Natural, padding = ' '): string {. ## ## `padding` characters (by default spaces) are added before `s` resulting in ## right alignment. If ``s.len >= count``, no spaces are added and `s` is - ## returned unchanged. If you need to left align a string use the `repeatChar - ## proc <#repeatChar>`_. Example: + ## returned unchanged. If you need to left align a string use the `alignLeft + ## proc <#alignLeft>`_. Example: ## ## .. code-block:: nim ## assert align("abc", 4) == " abc" @@ -1078,6 +1125,28 @@ proc align*(s: string, count: Natural, padding = ' '): string {. else: result = s +proc alignLeft*(s: string, count: Natural, padding = ' '): string {.noSideEffect.} = + ## Left-Aligns a string `s` with `padding`, so that it is of length `count`. + ## + ## `padding` characters (by default spaces) are added after `s` resulting in + ## left alignment. If ``s.len >= count``, no spaces are added and `s` is + ## returned unchanged. If you need to right align a string use the `align + ## proc <#align>`_. Example: + ## + ## .. code-block:: nim + ## assert alignLeft("abc", 4) == "abc " + ## assert alignLeft("a", 0) == "a" + ## assert alignLeft("1232", 6) == "1232 " + ## assert alignLeft("1232", 6, '#') == "1232##" + if s.len < count: + result = newString(count) + if s.len > 0: + result[0 .. (s.len - 1)] = s + for i in s.len ..< count: + result[i] = padding + else: + result = s + iterator tokenize*(s: string, seps: set[char] = Whitespace): tuple[ token: string, isSep: bool] = ## Tokenizes the string `s` into substrings. @@ -1175,7 +1244,7 @@ proc unindent*(s: string, count: Natural, padding: string = " "): string var indentCount = 0 for j in 0..<count.int: indentCount.inc - if line[j .. j + <padding.len] != padding: + if line[j .. j + padding.len-1] != padding: indentCount = j break result.add(line[indentCount*padding.len .. ^1]) @@ -1250,14 +1319,13 @@ proc addSep*(dest: var string, sep = ", ", startLen: Natural = 0) ## This is often useful for generating some code where the items need to ## be *separated* by `sep`. `sep` is only added if `dest` is longer than ## `startLen`. The following example creates a string describing - ## an array of integers: - ## - ## .. code-block:: nim - ## var arr = "[" - ## for x in items([2, 3, 5, 7, 11]): - ## addSep(arr, startLen=len("[")) - ## add(arr, $x) - ## add(arr, "]") + ## an array of integers. + runnableExamples: + var arr = "[" + for x in items([2, 3, 5, 7, 11]): + addSep(arr, startLen=len("[")) + add(arr, $x) + add(arr, "]") if dest.len > startLen: add(dest, sep) proc allCharsInSet*(s: string, theSet: set[char]): bool = @@ -1306,18 +1374,36 @@ proc join*[T: not string](a: openArray[T], sep: string = ""): string {. add(result, $x) type - SkipTable = array[char, int] - -{.push profiler: off.} -proc preprocessSub(sub: string, a: var SkipTable) = - var m = len(sub) - for i in 0..0xff: a[chr(i)] = m+1 - for i in 0..m-1: a[sub[i]] = m-i -{.pop.} + SkipTable* = array[char, int] -proc findAux(s, sub: string, start, last: int, a: SkipTable): int = - # Fast "quick search" algorithm: - var +proc initSkipTable*(a: var SkipTable, sub: string) + {.noSideEffect, rtl, extern: "nsuInitSkipTable".} = + ## Preprocess table `a` for `sub`. + let m = len(sub) + let m1 = m + 1 + var i = 0 + while i <= 0xff-7: + a[chr(i + 0)] = m1 + a[chr(i + 1)] = m1 + a[chr(i + 2)] = m1 + a[chr(i + 3)] = m1 + a[chr(i + 4)] = m1 + a[chr(i + 5)] = m1 + a[chr(i + 6)] = m1 + a[chr(i + 7)] = m1 + i += 8 + + for i in 0..m-1: + a[sub[i]] = m-i + +proc find*(a: SkipTable, s, sub: string, start: Natural = 0, last: Natural = 0): int + {.noSideEffect, rtl, extern: "nsuFindStrA".} = + ## Searches for `sub` in `s` inside range `start`..`last` using preprocessed table `a`. + ## If `last` is unspecified, it defaults to `s.high`. + ## + ## Searching is case-sensitive. If `sub` is not in `s`, -1 is returned. + let + last = if last==0: s.high else: last m = len(sub) n = last + 1 # search: @@ -1337,17 +1423,6 @@ when not (defined(js) or defined(nimdoc) or defined(nimscript)): else: const hasCStringBuiltin = false -proc find*(s, sub: string, start: Natural = 0, last: Natural = 0): int {.noSideEffect, - rtl, extern: "nsuFindStr".} = - ## Searches for `sub` in `s` inside range `start`..`last`. - ## If `last` is unspecified, it defaults to `s.high`. - ## - ## Searching is case-sensitive. If `sub` is not in `s`, -1 is returned. - var a {.noinit.}: SkipTable - let last = if last==0: s.high else: last - preprocessSub(sub, a) - result = findAux(s, sub, start, last, a) - proc find*(s: string, sub: char, start: Natural = 0, last: Natural = 0): int {.noSideEffect, rtl, extern: "nsuFindChar".} = ## Searches for `sub` in `s` inside range `start`..`last`. @@ -1366,9 +1441,24 @@ proc find*(s: string, sub: char, start: Natural = 0, last: Natural = 0): int {.n else: for i in start..last: if sub == s[i]: return i - return -1 +proc find*(s, sub: string, start: Natural = 0, last: Natural = 0): int {.noSideEffect, + rtl, extern: "nsuFindStr".} = + ## Searches for `sub` in `s` inside range `start`..`last`. + ## If `last` is unspecified, it defaults to `s.high`. + ## + ## Searching is case-sensitive. If `sub` is not in `s`, -1 is returned. + if sub.len > s.len: + return -1 + + if sub.len == 1: + return find(s, sub[0], start, last) + + var a {.noinit.}: SkipTable + initSkipTable(a, sub) + result = find(a, s, sub, start, last) + proc find*(s: string, chars: set[char], start: Natural = 0, last: Natural = 0): int {.noSideEffect, rtl, extern: "nsuFindCharSet".} = ## Searches for `chars` in `s` inside range `start`..`last`. @@ -1500,11 +1590,11 @@ proc replace*(s, sub: string, by = ""): string {.noSideEffect, ## Replaces `sub` in `s` by the string `by`. var a {.noinit.}: SkipTable result = "" - preprocessSub(sub, a) + initSkipTable(a, sub) let last = s.high var i = 0 while true: - var j = findAux(s, sub, i, last, a) + var j = find(a, s, sub, i, last) if j < 0: break add result, substr(s, i, j - 1) add result, by @@ -1534,11 +1624,11 @@ proc replaceWord*(s, sub: string, by = ""): string {.noSideEffect, const wordChars = {'a'..'z', 'A'..'Z', '0'..'9', '_', '\128'..'\255'} var a {.noinit.}: SkipTable result = "" - preprocessSub(sub, a) + initSkipTable(a, sub) var i = 0 let last = s.high while true: - var j = findAux(s, sub, i, last, a) + var j = find(a, s, sub, i, last) if j < 0: break # word boundary? if (j == 0 or s[j-1] notin wordChars) and @@ -1653,7 +1743,9 @@ proc insertSep*(s: string, sep = '_', digits = 3): string {.noSideEffect, ## ## Even though the algorithm works with any string `s`, it is only useful ## if `s` contains a number. - ## Example: ``insertSep("1000000") == "1_000_000"`` + runnableExamples: + doAssert insertSep("1000000") == "1_000_000" + var L = (s.len-1) div digits + s.len result = newString(L) var j = 0 @@ -1669,29 +1761,15 @@ proc insertSep*(s: string, sep = '_', digits = 3): string {.noSideEffect, proc escape*(s: string, prefix = "\"", suffix = "\""): string {.noSideEffect, rtl, extern: "nsuEscape".} = - ## Escapes a string `s`. - ## - ## This does these operations (at the same time): - ## * replaces any ``\`` by ``\\`` - ## * replaces any ``'`` by ``\'`` - ## * replaces any ``"`` by ``\"`` - ## * replaces any other character in the set ``{'\0'..'\31', '\127'..'\255'}`` - ## by ``\xHH`` where ``HH`` is its hexadecimal value. - ## The procedure has been designed so that its output is usable for many - ## different common syntaxes. The resulting string is prefixed with - ## `prefix` and suffixed with `suffix`. Both may be empty strings. - ## **Note**: This is not correct for producing Ansi C code! + ## Escapes a string `s`. See `system.addEscapedChar <system.html#addEscapedChar>`_ + ## for the escaping scheme. + ## + ## The resulting string is prefixed with `prefix` and suffixed with `suffix`. + ## Both may be empty strings. result = newStringOfCap(s.len + s.len shr 2) result.add(prefix) for c in items(s): - case c - of '\0'..'\31', '\127'..'\255': - add(result, "\\x") - add(result, toHex(ord(c), 2)) - of '\\': add(result, "\\\\") - of '\'': add(result, "\\'") - of '\"': add(result, "\\\"") - else: add(result, c) + result.addEscapedChar(c) add(result, suffix) proc unescape*(s: string, prefix = "\"", suffix = "\""): string {.noSideEffect, @@ -1741,6 +1819,8 @@ proc validIdentifier*(s: string): bool {.noSideEffect, ## ## A valid identifier starts with a character of the set `IdentStartChars` ## and is followed by any number of characters of the set `IdentChars`. + runnableExamples: + doAssert "abc_def08".validIdentifier if s[0] in IdentStartChars: for i in 1..s.len-1: if s[i] notin IdentChars: return false @@ -1751,7 +1831,7 @@ proc editDistance*(a, b: string): int {.noSideEffect, ## Returns the edit distance between `a` and `b`. ## ## This uses the `Levenshtein`:idx: distance algorithm with only a linear - ## memory overhead. This implementation is highly optimized! + ## memory overhead. var len1 = a.len var len2 = b.len if len1 > len2: @@ -1850,7 +1930,7 @@ type {.deprecated: [TFloatFormat: FloatFormatMode].} proc formatBiggestFloat*(f: BiggestFloat, format: FloatFormatMode = ffDefault, - precision: range[0..32] = 16; + precision: range[-1..32] = 16; decimalSep = '.'): string {. noSideEffect, rtl, extern: "nsu$1".} = ## Converts a floating point value `f` to a string. @@ -1862,7 +1942,7 @@ proc formatBiggestFloat*(f: BiggestFloat, format: FloatFormatMode = ffDefault, ## `precision`'s default value is the maximum number of meaningful digits ## after the decimal point for Nim's ``biggestFloat`` type. ## - ## If ``precision == 0``, it tries to format it nicely. + ## If ``precision == -1``, it tries to format it nicely. when defined(js): var res: cstring case format @@ -1884,7 +1964,7 @@ proc formatBiggestFloat*(f: BiggestFloat, format: FloatFormatMode = ffDefault, buf {.noinit.}: array[0..2500, char] L: cint frmtstr[0] = '%' - if precision > 0: + if precision >= 0: frmtstr[1] = '#' frmtstr[2] = '.' frmtstr[3] = '*' @@ -1907,9 +1987,18 @@ proc formatBiggestFloat*(f: BiggestFloat, format: FloatFormatMode = ffDefault, # but nothing else is possible: if buf[i] in {'.', ','}: result[i] = decimalsep else: result[i] = buf[i] + when defined(windows): + # VS pre 2015 violates the C standard: "The exponent always contains at + # least two digits, and only as many more digits as necessary to + # represent the exponent." [C11 §7.21.6.1] + # The following post-processing fixes this behavior. + if result.len > 4 and result[^4] == '+' and result[^3] == '0': + result[^3] = result[^2] + result[^2] = result[^1] + result.setLen(result.len - 1) proc formatFloat*(f: float, format: FloatFormatMode = ffDefault, - precision: range[0..32] = 16; decimalSep = '.'): string {. + precision: range[-1..32] = 16; decimalSep = '.'): string {. noSideEffect, rtl, extern: "nsu$1".} = ## Converts a floating point value `f` to a string. ## @@ -1920,7 +2009,12 @@ proc formatFloat*(f: float, format: FloatFormatMode = ffDefault, ## `precision`'s default value is the maximum number of meaningful digits ## after the decimal point for Nim's ``float`` type. ## - ## If ``precision == 0``, it tries to format it nicely. + ## If ``precision == -1``, it tries to format it nicely. + runnableExamples: + let x = 123.456 + doAssert x.formatFloat() == "123.4560000000000" + doAssert x.formatFloat(ffDecimal, 4) == "123.4560" + doAssert x.formatFloat(ffScientific, 2) == "1.23e+02" result = formatBiggestFloat(f, format, precision, decimalSep) proc trimZeros*(x: var string) {.noSideEffect.} = @@ -1955,18 +2049,13 @@ proc formatSize*(bytes: int64, ## ## `includeSpace` can be set to true to include the (SI preferred) space ## between the number and the unit (e.g. 1 KiB). - ## - ## Examples: - ## - ## .. code-block:: nim - ## - ## formatSize((1'i64 shl 31) + (300'i64 shl 20)) == "2.293GiB" - ## formatSize((2.234*1024*1024).int) == "2.234MiB" - ## formatSize(4096, includeSpace=true) == "4 KiB" - ## formatSize(4096, prefix=bpColloquial, includeSpace=true) == "4 kB" - ## formatSize(4096) == "4KiB" - ## formatSize(5_378_934, prefix=bpColloquial, decimalSep=',') == "5,13MB" - ## + runnableExamples: + doAssert formatSize((1'i64 shl 31) + (300'i64 shl 20)) == "2.293GiB" + doAssert formatSize((2.234*1024*1024).int) == "2.234MiB" + doAssert formatSize(4096, includeSpace=true) == "4 KiB" + doAssert formatSize(4096, prefix=bpColloquial, includeSpace=true) == "4 kB" + doAssert formatSize(4096) == "4KiB" + doAssert formatSize(5_378_934, prefix=bpColloquial, decimalSep=',') == "5,13MB" const iecPrefixes = ["", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi", "Yi"] const collPrefixes = ["", "k", "M", "G", "T", "P", "E", "Z", "Y"] var @@ -2060,7 +2149,7 @@ proc formatEng*(f: BiggestFloat, ## formatEng(4100, unit="V") == "4.1e3 V" ## formatEng(4100, unit="") == "4.1e3 " # Space with unit="" ## - ## `decimalSep` is used as the decimal separator + ## `decimalSep` is used as the decimal separator. var absolute: BiggestFloat significand: BiggestFloat @@ -2271,69 +2360,116 @@ proc format*(formatstr: string, a: varargs[string, `$`]): string {.noSideEffect, proc removeSuffix*(s: var string, chars: set[char] = Newlines) {. rtl, extern: "nsuRemoveSuffixCharSet".} = - ## Removes the first matching character from the string (in-place) given a - ## set of characters. If the set of characters is only equal to `Newlines` - ## then it will remove both the newline and return feed. - ## .. code-block:: nim - ## var - ## userInput = "Hello World!\r\n" - ## otherInput = "Hello!?!" - ## userInput.removeSuffix - ## userInput == "Hello World!" - ## userInput.removeSuffix({'!', '?'}) - ## userInput == "Hello World" - ## otherInput.removeSuffix({'!', '?'}) - ## otherInput == "Hello!?" + ## Removes all characters from `chars` from the end of the string `s` + ## (in-place). + runnableExamples: + var userInput = "Hello World!*~\r\n" + userInput.removeSuffix + doAssert userInput == "Hello World!*~" + userInput.removeSuffix({'~', '*'}) + doAssert userInput == "Hello World!" + + var otherInput = "Hello!?!" + otherInput.removeSuffix({'!', '?'}) + doAssert otherInput == "Hello" if s.len == 0: return - var last = len(s) - 1 - if chars == Newlines: - if s[last] == '\10': - last -= 1 - if s[last] == '\13': - last -= 1 - else: - if s[last] in chars: - last -= 1 + var last = s.high + while last > -1 and s[last] in chars: last -= 1 s.setLen(last + 1) proc removeSuffix*(s: var string, c: char) {. rtl, extern: "nsuRemoveSuffixChar".} = - ## Removes a single character (in-place) from a string. - ## .. code-block:: nim - ## var - ## table = "users" - ## table.removeSuffix('s') - ## table == "user" + ## Removes all occurrences of a single character (in-place) from the end + ## of a string. + ## + runnableExamples: + var table = "users" + table.removeSuffix('s') + doAssert table == "user" + + var dots = "Trailing dots......." + dots.removeSuffix('.') + doAssert dots == "Trailing dots" removeSuffix(s, chars = {c}) proc removeSuffix*(s: var string, suffix: string) {. rtl, extern: "nsuRemoveSuffixString".} = ## Remove the first matching suffix (in-place) from a string. - ## .. code-block:: nim - ## var - ## answers = "yeses" - ## answers.removeSuffix("es") - ## answers == "yes" + runnableExamples: + var answers = "yeses" + answers.removeSuffix("es") + doAssert answers == "yes" var newLen = s.len if s.endsWith(suffix): newLen -= len(suffix) s.setLen(newLen) +proc removePrefix*(s: var string, chars: set[char] = Newlines) {. + rtl, extern: "nsuRemovePrefixCharSet".} = + ## Removes all characters from `chars` from the start of the string `s` + ## (in-place). + ## + runnableExamples: + var userInput = "\r\n*~Hello World!" + userInput.removePrefix + doAssert userInput == "*~Hello World!" + userInput.removePrefix({'~', '*'}) + doAssert userInput == "Hello World!" + + var otherInput = "?!?Hello!?!" + otherInput.removePrefix({'!', '?'}) + doAssert otherInput == "Hello!?!" + var start = 0 + while start < s.len and s[start] in chars: start += 1 + if start > 0: s.delete(0, start - 1) + +proc removePrefix*(s: var string, c: char) {. + rtl, extern: "nsuRemovePrefixChar".} = + ## Removes all occurrences of a single character (in-place) from the start + ## of a string. + ## + runnableExamples: + var ident = "pControl" + ident.removePrefix('p') + doAssert ident == "Control" + removePrefix(s, chars = {c}) + +proc removePrefix*(s: var string, prefix: string) {. + rtl, extern: "nsuRemovePrefixString".} = + ## Remove the first matching prefix (in-place) from a string. + ## + runnableExamples: + var answers = "yesyes" + answers.removePrefix("yes") + doAssert answers == "yes" + if s.startsWith(prefix): + s.delete(0, prefix.len - 1) + when isMainModule: doAssert align("abc", 4) == " abc" doAssert align("a", 0) == "a" doAssert align("1232", 6) == " 1232" doAssert align("1232", 6, '#') == "##1232" + doAssert alignLeft("abc", 4) == "abc " + doAssert alignLeft("a", 0) == "a" + doAssert alignLeft("1232", 6) == "1232 " + doAssert alignLeft("1232", 6, '#') == "1232##" + let inp = """ this is a long text -- muchlongerthan10chars and here it goes""" outp = " this is a\nlong text\n--\nmuchlongerthan10chars\nand here\nit goes" doAssert wordWrap(inp, 10, false) == outp + doAssert formatBiggestFloat(1234.567, ffDecimal, -1) == "1234.567000" + doAssert formatBiggestFloat(1234.567, ffDecimal, 0) == "1235." + doAssert formatBiggestFloat(1234.567, ffDecimal, 1) == "1234.6" doAssert formatBiggestFloat(0.00000000001, ffDecimal, 11) == "0.00000000001" doAssert formatBiggestFloat(0.00000000001, ffScientific, 1, ',') in ["1,0e-11", "1,0e-011"] + # bug #6589 + doAssert formatFloat(123.456, ffScientific, precision = -1) == "1.234560e+02" doAssert "$# $3 $# $#" % ["a", "b", "c"] == "a c b c" doAssert "${1}12 ${-1}$2" % ["a", "b"] == "a12 bb" @@ -2497,6 +2633,12 @@ bar doAssert s.split(' ', maxsplit=1) == @["", "this is an example "] doAssert s.split(" ", maxsplit=4) == @["", "this", "is", "an", "example "] + doAssert s.splitWhitespace() == @["this", "is", "an", "example"] + doAssert s.splitWhitespace(maxsplit=1) == @["this", "is an example "] + doAssert s.splitWhitespace(maxsplit=2) == @["this", "is", "an example "] + doAssert s.splitWhitespace(maxsplit=3) == @["this", "is", "an", "example "] + doAssert s.splitWhitespace(maxsplit=4) == @["this", "is", "an", "example"] + block: # formatEng tests doAssert formatEng(0, 2, trim=false) == "0.00" doAssert formatEng(0, 2) == "0" diff --git a/lib/pure/terminal.nim b/lib/pure/terminal.nim index 871ac5d39..f15cee66a 100644 --- a/lib/pure/terminal.nim +++ b/lib/pure/terminal.nim @@ -391,8 +391,8 @@ proc eraseLine*(f: File) = origin.X = 0'i16 if setConsoleCursorPosition(h, origin) == 0: raiseOSError(osLastError()) - var ht = scrbuf.dwSize.Y - origin.Y - var wt = scrbuf.dwSize.X - origin.X + var ht: DWORD = scrbuf.dwSize.Y - origin.Y + var wt: DWORD = scrbuf.dwSize.X - origin.X if fillConsoleOutputCharacter(h, ' ', ht*wt, origin, addr(numwrote)) == 0: raiseOSError(osLastError()) @@ -634,7 +634,10 @@ proc getch*(): char = doAssert(readConsoleInput(fd, addr(keyEvent), 1, addr(numRead)) != 0) if numRead == 0 or keyEvent.eventType != 1 or keyEvent.bKeyDown == 0: continue - return char(keyEvent.uChar) + if keyEvent.uChar == 0: + return char(keyEvent.wVirtualKeyCode) + else: + return char(keyEvent.uChar) else: let fd = getFileHandle(stdin) var oldMode: Termios @@ -650,10 +653,10 @@ template setCursorPos*(x, y: int) = setCursorPos(stdout, x, y) template setCursorXPos*(x: int) = setCursorXPos(stdout, x) when defined(windows): template setCursorYPos(x: int) = setCursorYPos(stdout, x) -template cursorUp*(count=1) = cursorUp(stdout, f) -template cursorDown*(count=1) = cursorDown(stdout, f) -template cursorForward*(count=1) = cursorForward(stdout, f) -template cursorBackward*(count=1) = cursorBackward(stdout, f) +template cursorUp*(count=1) = cursorUp(stdout, count) +template cursorDown*(count=1) = cursorDown(stdout, count) +template cursorForward*(count=1) = cursorForward(stdout, count) +template cursorBackward*(count=1) = cursorBackward(stdout, count) template eraseLine*() = eraseLine(stdout) template eraseScreen*() = eraseScreen(stdout) template setStyle*(style: set[Style]) = diff --git a/lib/pure/times.nim b/lib/pure/times.nim index 96668c4f8..42e89e7ce 100644 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -10,27 +10,25 @@ ## This module contains routines and types for dealing with time. ## This module is available for the `JavaScript target -## <backends.html#the-javascript-target>`_. +## <backends.html#the-javascript-target>`_. The proleptic Gregorian calendar is the only calendar supported. ## ## Examples: ## ## .. code-block:: nim ## ## import times, os -## var -## t = cpuTime() +## let time = cpuTime() ## ## sleep(100) # replace this with something to be timed -## echo "Time taken: ",cpuTime() - t +## echo "Time taken: ",cpuTime() - time ## -## echo "My formatted time: ", format(getLocalTime(getTime()), "d MMMM yyyy HH:mm") +## echo "My formatted time: ", format(now(), "d MMMM yyyy HH:mm") ## echo "Using predefined formats: ", getClockStr(), " ", getDateStr() ## ## echo "epochTime() float value: ", epochTime() -## echo "getTime() float value: ", toSeconds(getTime()) ## echo "cpuTime() float value: ", cpuTime() -## echo "An hour from now : ", getLocalTime(getTime()) + 1.hours -## echo "An hour from (UTC) now: ", getGmTime(getTime()) + initInterval(0,0,0,1) +## echo "An hour from now : ", now() + 1.hours +## echo "An hour from (UTC) now: ", getTime().utc + initInterval(0,0,0,1) {.push debugger:off.} # the user does not want to trace a part # of the standard library! @@ -40,132 +38,85 @@ import include "system/inclrtl" -type - Month* = enum ## represents a month - mJan, mFeb, mMar, mApr, mMay, mJun, mJul, mAug, mSep, mOct, mNov, mDec - WeekDay* = enum ## represents a weekday - dMon, dTue, dWed, dThu, dFri, dSat, dSun - -when defined(posix) and not defined(JS): - when defined(linux) and defined(amd64): - type - TimeImpl {.importc: "time_t", header: "<time.h>".} = clong - Time* = distinct TimeImpl ## distinct type that represents a time - ## measured as number of seconds since the epoch - - Timeval {.importc: "struct timeval", - header: "<sys/select.h>".} = object ## struct timeval - tv_sec: clong ## Seconds. - tv_usec: clong ## Microseconds. - else: - type - TimeImpl {.importc: "time_t", header: "<time.h>".} = int - Time* = distinct TimeImpl ## distinct type that represents a time - ## measured as number of seconds since the epoch - - Timeval {.importc: "struct timeval", - header: "<sys/select.h>".} = object ## struct timeval - tv_sec: int ## Seconds. - tv_usec: int ## Microseconds. +when defined(posix): + import posix - # we cannot import posix.nim here, because posix.nim depends on times.nim. - # Ok, we could, but I don't want circular dependencies. - # And gettimeofday() is not defined in the posix module anyway. Sigh. + type CTime = posix.Time proc posix_gettimeofday(tp: var Timeval, unused: pointer = nil) {. importc: "gettimeofday", header: "<sys/time.h>".} when not defined(freebsd) and not defined(netbsd) and not defined(openbsd): var timezone {.importc, header: "<time.h>".}: int - proc tzset(): void {.importc, header: "<time.h>".} tzset() elif defined(windows): import winlean # newest version of Visual C++ defines time_t to be of 64 bits - type TimeImpl {.importc: "time_t", header: "<time.h>".} = int64 + type CTime {.importc: "time_t", header: "<time.h>".} = distinct int64 # visual c's c runtime exposes these under a different name - var - timezone {.importc: "_timezone", header: "<time.h>".}: int - - type - Time* = distinct TimeImpl + var timezone {.importc: "_timezone", header: "<time.h>".}: int +type + Month* = enum ## Represents a month. Note that the enum starts at ``1``, so ``ord(month)`` will give + ## the month number in the range ``[1..12]``. + mJan = 1, mFeb, mMar, mApr, mMay, mJun, mJul, mAug, mSep, mOct, mNov, mDec -elif defined(JS): - type - TimeBase = float - Time* = distinct TimeBase - - proc getDay(t: Time): int {.tags: [], raises: [], benign, importcpp.} - proc getFullYear(t: Time): int {.tags: [], raises: [], benign, importcpp.} - proc getHours(t: Time): int {.tags: [], raises: [], benign, importcpp.} - proc getMilliseconds(t: Time): int {.tags: [], raises: [], benign, importcpp.} - proc getMinutes(t: Time): int {.tags: [], raises: [], benign, importcpp.} - proc getMonth(t: Time): int {.tags: [], raises: [], benign, importcpp.} - proc getSeconds(t: Time): int {.tags: [], raises: [], benign, importcpp.} - proc getTime(t: Time): int {.tags: [], raises: [], noSideEffect, benign, importcpp.} - proc getTimezoneOffset(t: Time): int {.tags: [], raises: [], benign, importcpp.} - proc getDate(t: Time): int {.tags: [], raises: [], benign, importcpp.} - proc getUTCDate(t: Time): int {.tags: [], raises: [], benign, importcpp.} - proc getUTCFullYear(t: Time): int {.tags: [], raises: [], benign, importcpp.} - proc getUTCHours(t: Time): int {.tags: [], raises: [], benign, importcpp.} - proc getUTCMilliseconds(t: Time): int {.tags: [], raises: [], benign, importcpp.} - proc getUTCMinutes(t: Time): int {.tags: [], raises: [], benign, importcpp.} - proc getUTCMonth(t: Time): int {.tags: [], raises: [], benign, importcpp.} - proc getUTCSeconds(t: Time): int {.tags: [], raises: [], benign, importcpp.} - proc getUTCDay(t: Time): int {.tags: [], raises: [], benign, importcpp.} - proc getYear(t: Time): int {.tags: [], raises: [], benign, importcpp.} - proc parse(t: Time; s: cstring): Time {.tags: [], raises: [], benign, importcpp.} - proc setDate(t: Time; x: int) {.tags: [], raises: [], benign, importcpp.} - proc setFullYear(t: Time; x: int) {.tags: [], raises: [], benign, importcpp.} - proc setHours(t: Time; x: int) {.tags: [], raises: [], benign, importcpp.} - proc setMilliseconds(t: Time; x: int) {.tags: [], raises: [], benign, importcpp.} - proc setMinutes(t: Time; x: int) {.tags: [], raises: [], benign, importcpp.} - proc setMonth(t: Time; x: int) {.tags: [], raises: [], benign, importcpp.} - proc setSeconds(t: Time; x: int) {.tags: [], raises: [], benign, importcpp.} - proc setTime(t: Time; x: int) {.tags: [], raises: [], benign, importcpp.} - proc setUTCDate(t: Time; x: int) {.tags: [], raises: [], benign, importcpp.} - proc setUTCFullYear(t: Time; x: int) {.tags: [], raises: [], benign, importcpp.} - proc setUTCHours(t: Time; x: int) {.tags: [], raises: [], benign, importcpp.} - proc setUTCMilliseconds(t: Time; x: int) {.tags: [], raises: [], benign, importcpp.} - proc setUTCMinutes(t: Time; x: int) {.tags: [], raises: [], benign, importcpp.} - proc setUTCMonth(t: Time; x: int) {.tags: [], raises: [], benign, importcpp.} - proc setUTCSeconds(t: Time; x: int) {.tags: [], raises: [], benign, importcpp.} - proc setYear(t: Time; x: int) {.tags: [], raises: [], benign, importcpp.} - proc toGMTString(t: Time): cstring {.tags: [], raises: [], benign, importcpp.} - proc toLocaleString(t: Time): cstring {.tags: [], raises: [], benign, importcpp.} + WeekDay* = enum ## Represents a weekday. + dMon, dTue, dWed, dThu, dFri, dSat, dSun -type - TimeInfo* = object of RootObj ## represents a time in different parts - second*: range[0..61] ## The number of seconds after the minute, + MonthdayRange* = range[1..31] + HourRange* = range[0..23] + MinuteRange* = range[0..59] + SecondRange* = range[0..60] + YeardayRange* = range[0..365] + + TimeImpl = int64 + + Time* = distinct TimeImpl ## Represents a point in time. + ## This is currently implemented as a ``int64`` representing + ## seconds since ``1970-01-01T00:00:00Z``, but don't + ## rely on this knowledge because it might change + ## in the future to allow for higher precision. + ## Use the procs ``toUnix`` and ``fromUnix`` to + ## work with unix timestamps instead. + + DateTime* = object of RootObj ## Represents a time in different parts. + ## Although this type can represent leap + ## seconds, they are generally not supported + ## in this module. They are not ignored, + ## but the ``DateTime``'s returned by + ## procedures in this module will never have + ## a leap second. + second*: SecondRange ## The number of seconds after the minute, ## normally in the range 0 to 59, but can - ## be up to 61 to allow for leap seconds. - minute*: range[0..59] ## The number of minutes after the hour, + ## be up to 60 to allow for a leap second. + minute*: MinuteRange ## The number of minutes after the hour, ## in the range 0 to 59. - hour*: range[0..23] ## The number of hours past midnight, + hour*: HourRange ## The number of hours past midnight, ## in the range 0 to 23. - monthday*: range[1..31] ## The day of the month, in the range 1 to 31. + monthday*: MonthdayRange ## The day of the month, in the range 1 to 31. month*: Month ## The current month. - year*: int ## The current year. + year*: int ## The current year, using astronomical year numbering + ## (meaning that before year 1 is year 0, then year -1 and so on). weekday*: WeekDay ## The current day of the week. - yearday*: range[0..365] ## The number of days since January 1, + yearday*: YeardayRange ## The number of days since January 1, ## in the range 0 to 365. - ## Always 0 if the target is JS. - isDST*: bool ## Determines whether DST is in effect. - ## Semantically, this adds another negative hour - ## offset to the time in addition to the timezone. - timezone*: int ## The offset of the (non-DST) timezone in seconds - ## west of UTC. Note that the sign of this number - ## is the opposite of the one in a formatted - ## timezone string like ``+01:00`` (which would be - ## parsed into the timezone ``-3600``). - - ## I make some assumptions about the data in here. Either - ## everything should be positive or everything negative. Zero is - ## fine too. Mixed signs will lead to unexpected results. - TimeInterval* = object ## a time interval + isDst*: bool ## Determines whether DST is in effect. + ## Always false for the JavaScript backend. + timezone*: Timezone ## The timezone represented as an implementation of ``Timezone``. + utcOffset*: int ## The offset in seconds west of UTC, including any offset due to DST. + ## Note that the sign of this number is the opposite + ## of the one in a formatted offset string like ``+01:00`` + ## (which would be parsed into the UTC offset ``-3600``). + + TimeInterval* = object ## Represents a duration of time. Can be used to add and subtract + ## from a ``DateTime`` or ``Time``. + ## Note that a ``TimeInterval`` doesn't represent a fixed duration of time, + ## since the duration of some units depend on the context (e.g a year + ## can be either 365 or 366 days long). The non-fixed time units are years, + ## months and days. milliseconds*: int ## The number of milliseconds seconds*: int ## The number of seconds minutes*: int ## The number of minutes @@ -174,92 +125,394 @@ type months*: int ## The number of months years*: int ## The number of years + Timezone* = object ## Timezone interface for supporting ``DateTime``'s of arbritary timezones. + ## The ``times`` module only supplies implementations for the systems local time and UTC. + ## The members ``zoneInfoFromUtc`` and ``zoneInfoFromTz`` should not be accessed directly + ## and are only exported so that ``Timezone`` can be implemented by other modules. + zoneInfoFromUtc*: proc (time: Time): ZonedTime {.nimcall, tags: [], raises: [], benign .} + zoneInfoFromTz*: proc (adjTime: Time): ZonedTime {.nimcall, tags: [], raises: [], benign .} + name*: string ## The name of the timezone, f.ex 'Europe/Stockholm' or 'Etc/UTC'. Used for checking equality. + ## Se also: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones + ZonedTime* = object ## Represents a zooned instant in time that is not associated with any calendar. + ## This type is only used for implementing timezones. + adjTime*: Time ## Time adjusted to a timezone. + utcOffset*: int + isDst*: bool + {.deprecated: [TMonth: Month, TWeekDay: WeekDay, TTime: Time, - TTimeInterval: TimeInterval, TTimeInfo: TimeInfo].} + TTimeInterval: TimeInterval, TTimeInfo: DateTime, TimeInfo: DateTime].} -proc getTime*(): Time {.tags: [TimeEffect], benign.} - ## gets the current calendar time as a UNIX epoch value (number of seconds - ## elapsed since 1970) with integer precission. Use epochTime for higher - ## resolution. -proc getLocalTime*(t: Time): TimeInfo {.tags: [TimeEffect], raises: [], benign.} - ## converts the calendar time `t` to broken-time representation, - ## expressed relative to the user's specified time zone. -proc getGMTime*(t: Time): TimeInfo {.tags: [TimeEffect], raises: [], benign.} - ## converts the calendar time `t` to broken-down time representation, - ## expressed in Coordinated Universal Time (UTC). - -proc timeInfoToTime*(timeInfo: TimeInfo): Time - {.tags: [TimeEffect], benign, deprecated.} - ## converts a broken-down time structure to - ## calendar time representation. The function ignores the specified - ## contents of the structure members `weekday` and `yearday` and recomputes - ## them from the other information in the broken-down time structure. - ## - ## **Warning:** This procedure is deprecated since version 0.14.0. - ## Use ``toTime`` instead. +const + secondsInMin = 60 + secondsInHour = 60*60 + secondsInDay = 60*60*24 + minutesInHour = 60 -proc toTime*(timeInfo: TimeInfo): Time {.tags: [TimeEffect], benign.} - ## converts a broken-down time structure to - ## calendar time representation. The function ignores the specified - ## contents of the structure members `weekday` and `yearday` and recomputes - ## them from the other information in the broken-down time structure. +proc fromUnix*(unix: int64): Time {.benign, tags: [], raises: [], noSideEffect.} = + ## Convert a unix timestamp (seconds since ``1970-01-01T00:00:00Z``) to a ``Time``. + Time(unix) -proc fromSeconds*(since1970: float): Time {.tags: [], raises: [], benign.} - ## Takes a float which contains the number of seconds since the unix epoch and - ## returns a time object. +proc toUnix*(t: Time): int64 {.benign, tags: [], raises: [], noSideEffect.} = + ## Convert ``t`` to a unix timestamp (seconds since ``1970-01-01T00:00:00Z``). + t.int64 -proc fromSeconds*(since1970: int64): Time {.tags: [], raises: [], benign.} = - ## Takes an int which contains the number of seconds since the unix epoch and - ## returns a time object. - fromSeconds(float(since1970)) +proc isLeapYear*(year: int): bool = + ## Returns true if ``year`` is a leap year. + year mod 4 == 0 and (year mod 100 != 0 or year mod 400 == 0) -proc toSeconds*(time: Time): float {.tags: [], raises: [], benign.} - ## Returns the time in seconds since the unix epoch. +proc getDaysInMonth*(month: Month, year: int): int = + ## Get the number of days in a ``month`` of a ``year``. + # http://www.dispersiondesign.com/articles/time/number_of_days_in_a_month + case month + of mFeb: result = if isLeapYear(year): 29 else: 28 + of mApr, mJun, mSep, mNov: result = 30 + else: result = 31 + +proc getDaysInYear*(year: int): int = + ## Get the number of days in a ``year`` + result = 365 + (if isLeapYear(year): 1 else: 0) + +proc assertValidDate(monthday: MonthdayRange, month: Month, year: int) {.inline.} = + assert monthday <= getDaysInMonth(month, year), + $year & "-" & $ord(month) & "-" & $monthday & " is not a valid date" + +proc toEpochDay(monthday: MonthdayRange, month: Month, year: int): int64 = + ## Get the epoch day from a year/month/day date. + ## The epoch day is the number of days since 1970/01/01 (it might be negative). + assertValidDate monthday, month, year + # Based on http://howardhinnant.github.io/date_algorithms.html + var (y, m, d) = (year, ord(month), monthday.int) + if m <= 2: + y.dec + + let era = (if y >= 0: y else: y-399) div 400 + let yoe = y - era * 400 + let doy = (153 * (m + (if m > 2: -3 else: 9)) + 2) div 5 + d-1 + let doe = yoe * 365 + yoe div 4 - yoe div 100 + doy + return era * 146097 + doe - 719468 + +proc fromEpochDay(epochday: int64): tuple[monthday: MonthdayRange, month: Month, year: int] = + ## Get the year/month/day date from a epoch day. + ## The epoch day is the number of days since 1970/01/01 (it might be negative). + # Based on http://howardhinnant.github.io/date_algorithms.html + var z = epochday + z.inc 719468 + let era = (if z >= 0: z else: z - 146096) div 146097 + let doe = z - era * 146097 + let yoe = (doe - doe div 1460 + doe div 36524 - doe div 146096) div 365 + let y = yoe + era * 400; + let doy = doe - (365 * yoe + yoe div 4 - yoe div 100) + let mp = (5 * doy + 2) div 153 + let d = doy - (153 * mp + 2) div 5 + 1 + let m = mp + (if mp < 10: 3 else: -9) + return (d.MonthdayRange, m.Month, (y + ord(m <= 2)).int) + +proc getDayOfYear*(monthday: MonthdayRange, month: Month, year: int): YeardayRange {.tags: [], raises: [], benign .} = + ## Returns the day of the year. + ## Equivalent with ``initDateTime(day, month, year).yearday``. + assertValidDate monthday, month, year + const daysUntilMonth: array[Month, int] = [0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334] + const daysUntilMonthLeap: array[Month, int] = [0, 31, 60, 91, 121, 152, 182, 213, 244, 274, 305, 335] + + if isLeapYear(year): + result = daysUntilMonthLeap[month] + monthday - 1 + else: + result = daysUntilMonth[month] + monthday - 1 + +proc getDayOfWeek*(monthday: MonthdayRange, month: Month, year: int): WeekDay {.tags: [], raises: [], benign .} = + ## Returns the day of the week enum from day, month and year. + ## Equivalent with ``initDateTime(day, month, year).weekday``. + assertValidDate monthday, month, year + # 1970-01-01 is a Thursday, we adjust to the previous Monday + let days = toEpochday(monthday, month, year) - 3 + let weeks = (if days >= 0: days else: days - 6) div 7 + let wd = days - weeks * 7 + # The value of d is 0 for a Sunday, 1 for a Monday, 2 for a Tuesday, etc. + # so we must correct for the WeekDay type. + result = if wd == 0: dSun else: WeekDay(wd - 1) + +# Forward declarations +proc utcZoneInfoFromUtc(time: Time): ZonedTime {.tags: [], raises: [], benign .} +proc utcZoneInfoFromTz(adjTime: Time): ZonedTime {.tags: [], raises: [], benign .} +proc localZoneInfoFromUtc(time: Time): ZonedTime {.tags: [], raises: [], benign .} +proc localZoneInfoFromTz(adjTime: Time): ZonedTime {.tags: [], raises: [], benign .} proc `-`*(a, b: Time): int64 {. - rtl, extern: "ntDiffTime", tags: [], raises: [], noSideEffect, benign.} - ## computes the difference of two calendar times. Result is in seconds. + rtl, extern: "ntDiffTime", tags: [], raises: [], noSideEffect, benign, deprecated.} = + ## Computes the difference of two calendar times. Result is in seconds. + ## This is deprecated because it will need to change when sub second time resolution is implemented. + ## Use ``a.toUnix - b.toUnix`` instead. ## ## .. code-block:: nim ## let a = fromSeconds(1_000_000_000) ## let b = fromSeconds(1_500_000_000) ## echo initInterval(seconds=int(b - a)) - ## # (milliseconds: 0, seconds: 20, minutes: 53, hours: 0, days: 5787, months: 0, years: 0) + ## # (milliseconds: 0, seconds: 20, minutes: 53, hours: 0, days: 5787, months: 0, years: 0) + a.toUnix - b.toUnix proc `<`*(a, b: Time): bool {. - rtl, extern: "ntLtTime", tags: [], raises: [], noSideEffect.} = - ## returns true iff ``a < b``, that is iff a happened before b. - when defined(js): - result = TimeBase(a) < TimeBase(b) - else: - result = a - b < 0 + rtl, extern: "ntLtTime", tags: [], raises: [], noSideEffect, borrow.} + ## Returns true iff ``a < b``, that is iff a happened before b. proc `<=` * (a, b: Time): bool {. - rtl, extern: "ntLeTime", tags: [], raises: [], noSideEffect.}= - ## returns true iff ``a <= b``. - when defined(js): - result = TimeBase(a) <= TimeBase(b) - else: - result = a - b <= 0 + rtl, extern: "ntLeTime", tags: [], raises: [], noSideEffect, borrow.} + ## Returns true iff ``a <= b``. proc `==`*(a, b: Time): bool {. - rtl, extern: "ntEqTime", tags: [], raises: [], noSideEffect.} = - ## returns true if ``a == b``, that is if both times represent the same value - when defined(js): - result = TimeBase(a) == TimeBase(b) + rtl, extern: "ntEqTime", tags: [], raises: [], noSideEffect, borrow.} + ## Returns true if ``a == b``, that is if both times represent the same point in time. + +proc toTime*(dt: DateTime): Time {.tags: [], raises: [], benign.} = + ## Converts a broken-down time structure to + ## calendar time representation. + let epochDay = toEpochday(dt.monthday, dt.month, dt.year) + result = Time(epochDay * secondsInDay) + result.inc dt.hour * secondsInHour + result.inc dt.minute * 60 + result.inc dt.second + # The code above ignores the UTC offset of `timeInfo`, + # so we need to compensate for that here. + result.inc dt.utcOffset + +proc initDateTime(zt: ZonedTime, zone: Timezone): DateTime = + let adjTime = zt.adjTime.int64 + let epochday = (if adjTime >= 0: adjTime else: adjTime - (secondsInDay - 1)) div secondsInDay + var rem = zt.adjTime.int64 - epochday * secondsInDay + let hour = rem div secondsInHour + rem = rem - hour * secondsInHour + let minute = rem div secondsInMin + rem = rem - minute * secondsInMin + let second = rem + + let (d, m, y) = fromEpochday(epochday) + + DateTime( + year: y, + month: m, + monthday: d, + hour: hour, + minute: minute, + second: second, + weekday: getDayOfWeek(d, m, y), + yearday: getDayOfYear(d, m, y), + isDst: zt.isDst, + timezone: zone, + utcOffset: zt.utcOffset + ) + +proc inZone*(time: Time, zone: Timezone): DateTime {.tags: [], raises: [], benign.} = + ## Break down ``time`` into a ``DateTime`` using ``zone`` as the timezone. + let zoneInfo = zone.zoneInfoFromUtc(time) + result = initDateTime(zoneInfo, zone) + +proc inZone*(dt: DateTime, zone: Timezone): DateTime {.tags: [], raises: [], benign.} = + ## Convert ``dt`` into a ``DateTime`` using ``zone`` as the timezone. + dt.toTime.inZone(zone) + +proc `$`*(zone: Timezone): string = + ## Returns the name of the timezone. + zone.name + +proc `==`*(zone1, zone2: Timezone): bool = + ## Two ``Timezone``'s are considered equal if their name is equal. + zone1.name == zone2.name + +proc toAdjTime(dt: DateTime): Time = + let epochDay = toEpochday(dt.monthday, dt.month, dt.year) + result = Time(epochDay * secondsInDay) + result.inc dt.hour * secondsInHour + result.inc dt.minute * secondsInMin + result.inc dt.second + +when defined(JS): + type JsDate = object + proc newDate(year, month, date, hours, minutes, seconds, milliseconds: int): JsDate {.tags: [], raises: [], importc: "new Date".} + proc newDate(): JsDate {.importc: "new Date".} + proc newDate(value: float): JsDate {.importc: "new Date".} + proc getTimezoneOffset(js: JsDate): int {.tags: [], raises: [], benign, importcpp.} + proc getDay(js: JsDate): int {.tags: [], raises: [], benign, importcpp.} + proc getFullYear(js: JsDate): int {.tags: [], raises: [], benign, importcpp.} + proc getHours(js: JsDate): int {.tags: [], raises: [], benign, importcpp.} + proc getMilliseconds(js: JsDate): int {.tags: [], raises: [], benign, importcpp.} + proc getMinutes(js: JsDate): int {.tags: [], raises: [], benign, importcpp.} + proc getMonth(js: JsDate): int {.tags: [], raises: [], benign, importcpp.} + proc getSeconds(js: JsDate): int {.tags: [], raises: [], benign, importcpp.} + proc getTime(js: JsDate): int {.tags: [], raises: [], noSideEffect, benign, importcpp.} + proc getDate(js: JsDate): int {.tags: [], raises: [], benign, importcpp.} + proc getUTCDate(js: JsDate): int {.tags: [], raises: [], benign, importcpp.} + proc getUTCFullYear(js: JsDate): int {.tags: [], raises: [], benign, importcpp.} + proc getUTCHours(js: JsDate): int {.tags: [], raises: [], benign, importcpp.} + proc getUTCMilliseconds(js: JsDate): int {.tags: [], raises: [], benign, importcpp.} + proc getUTCMinutes(js: JsDate): int {.tags: [], raises: [], benign, importcpp.} + proc getUTCMonth(js: JsDate): int {.tags: [], raises: [], benign, importcpp.} + proc getUTCSeconds(js: JsDate): int {.tags: [], raises: [], benign, importcpp.} + proc getUTCDay(js: JsDate): int {.tags: [], raises: [], benign, importcpp.} + proc getYear(js: JsDate): int {.tags: [], raises: [], benign, importcpp.} + proc setFullYear(js: JsDate, year: int): void {.tags: [], raises: [], benign, importcpp.} + + proc localZoneInfoFromUtc(time: Time): ZonedTime = + let jsDate = newDate(time.float * 1000) + let offset = jsDate.getTimezoneOffset() * secondsInMin + result.adjTime = Time(time.int64 - offset) + result.utcOffset = offset + result.isDst = false + + proc localZoneInfoFromTz(adjTime: Time): ZonedTime = + let utcDate = newDate(adjTime.float * 1000) + let localDate = newDate(utcDate.getUTCFullYear(), utcDate.getUTCMonth(), utcDate.getUTCDate(), + utcDate.getUTCHours(), utcDate.getUTCMinutes(), utcDate.getUTCSeconds(), 0) + + # This is as dumb as it looks - JS doesn't support years in the range 0-99 in the constructor + # because they are assumed to be 19xx... + # Because JS doesn't support timezone history, it doesn't really matter in practice. + if utcDate.getUTCFullYear() in 0 .. 99: + localDate.setFullYear(utcDate.getUTCFullYear()) + + result.adjTime = adjTime + result.utcOffset = localDate.getTimezoneOffset() * secondsInMin + result.isDst = false + +else: + when defined(freebsd) or defined(netbsd) or defined(openbsd) or + defined(macosx): + type + StructTm {.importc: "struct tm".} = object + second {.importc: "tm_sec".}, + minute {.importc: "tm_min".}, + hour {.importc: "tm_hour".}, + monthday {.importc: "tm_mday".}, + month {.importc: "tm_mon".}, + year {.importc: "tm_year".}, + weekday {.importc: "tm_wday".}, + yearday {.importc: "tm_yday".}, + isdst {.importc: "tm_isdst".}: cint + gmtoff {.importc: "tm_gmtoff".}: clong else: - result = a - b == 0 + type + StructTm {.importc: "struct tm".} = object + second {.importc: "tm_sec".}, + minute {.importc: "tm_min".}, + hour {.importc: "tm_hour".}, + monthday {.importc: "tm_mday".}, + month {.importc: "tm_mon".}, + year {.importc: "tm_year".}, + weekday {.importc: "tm_wday".}, + yearday {.importc: "tm_yday".}, + isdst {.importc: "tm_isdst".}: cint + when defined(linux) and defined(amd64): + gmtoff {.importc: "tm_gmtoff".}: clong + zone {.importc: "tm_zone".}: cstring + type + StructTmPtr = ptr StructTm + + proc localtime(timer: ptr CTime): StructTmPtr {. importc: "localtime", header: "<time.h>", tags: [].} + + proc toAdjTime(tm: StructTm): Time = + let epochDay = toEpochday(tm.monthday, (tm.month + 1).Month, tm.year.int + 1900) + result = Time(epochDay * secondsInDay) + result.inc tm.hour * secondsInHour + result.inc tm.minute * 60 + result.inc tm.second + + proc getStructTm(time: Time | int64): StructTm = + let timei64 = time.int64 + var a = + if timei64 < low(CTime): + CTime(low(CTime)) + elif timei64 > high(CTime): + CTime(high(CTime)) + else: + CTime(timei64) + result = localtime(addr(a))[] + + proc localZoneInfoFromUtc(time: Time): ZonedTime = + let tm = getStructTm(time) + let adjTime = tm.toAdjTime + result.adjTime = adjTime + result.utcOffset = (time.toUnix - adjTime.toUnix).int + result.isDst = tm.isdst > 0 + + proc localZoneInfoFromTz(adjTime: Time): ZonedTime = + var adjTimei64 = adjTime.int64 + let past = adjTimei64 - secondsInDay + var tm = getStructTm(past) + let pastOffset = past - tm.toAdjTime.int64 + + let future = adjTimei64 + secondsInDay + tm = getStructTm(future) + let futureOffset = future - tm.toAdjTime.int64 + + var utcOffset: int + if pastOffset == futureOffset: + utcOffset = pastOffset.int + else: + if pastOffset > futureOffset: + adjTimei64 -= secondsInHour + + adjTimei64 += pastOffset + utcOffset = (adjTimei64 - getStructTm(adjTimei64).toAdjTime.int64).int + + # This extra roundtrip is needed to normalize any impossible datetimes + # as a result of offset changes (normally due to dst) + let utcTime = adjTime.int64 + utcOffset + tm = getStructTm(utcTime) + result.adjTime = tm.toAdjTime + result.utcOffset = (utcTime - result.adjTime.int64).int + result.isDst = tm.isdst > 0 + +proc utcZoneInfoFromUtc(time: Time): ZonedTime = + result.adjTime = time + result.utcOffset = 0 + result.isDst = false + +proc utcZoneInfoFromTz(adjTime: Time): ZonedTime = + utcZoneInfoFromUtc(adjTime) # adjTime == time since we are in UTC + +proc utc*(): TimeZone = + ## Get the ``Timezone`` implementation for the UTC timezone. + ## + ## .. code-block:: nim + ## doAssert now().utc.timezone == utc() + ## doAssert utc().name == "Etc/UTC" + Timezone(zoneInfoFromUtc: utcZoneInfoFromUtc, zoneInfoFromTz: utcZoneInfoFromTz, name: "Etc/UTC") + +proc local*(): TimeZone = + ## Get the ``Timezone`` implementation for the local timezone. + ## + ## .. code-block:: nim + ## doAssert now().timezone == local() + ## doAssert local().name == "LOCAL" + Timezone(zoneInfoFromUtc: localZoneInfoFromUtc, zoneInfoFromTz: localZoneInfoFromTz, name: "LOCAL") -proc getTimezone*(): int {.tags: [TimeEffect], raises: [], benign.} - ## returns the offset of the local (non-DST) timezone in seconds west of UTC. +proc utc*(dt: DateTime): DateTime = + ## Shorthand for ``dt.inZone(utc())``. + dt.inZone(utc()) -proc getStartMilsecs*(): int {.deprecated, tags: [TimeEffect], benign.} - ## get the milliseconds from the start of the program. **Deprecated since - ## version 0.8.10.** Use ``epochTime`` or ``cpuTime`` instead. +proc local*(dt: DateTime): DateTime = + ## Shorthand for ``dt.inZone(local())``. + dt.inZone(local()) + +proc utc*(t: Time): DateTime = + ## Shorthand for ``t.inZone(utc())``. + t.inZone(utc()) + +proc local*(t: Time): DateTime = + ## Shorthand for ``t.inZone(local())``. + t.inZone(local()) + +proc getTime*(): Time {.tags: [TimeEffect], benign.} + ## Gets the current time as a ``Time`` with second resolution. Use epochTime for higher + ## resolution. + +proc now*(): DateTime {.tags: [TimeEffect], benign.} = + ## Get the current time as a ``DateTime`` in the local timezone. + ## + ## Shorthand for ``getTime().local``. + getTime().local proc initInterval*(milliseconds, seconds, minutes, hours, days, months, years: int = 0): TimeInterval = - ## creates a new ``TimeInterval``. + ## Creates a new ``TimeInterval``. ## ## You can also use the convenience procedures called ``milliseconds``, ## ``seconds``, ``minutes``, ``hours``, ``days``, ``months``, and ``years``. @@ -269,46 +522,33 @@ proc initInterval*(milliseconds, seconds, minutes, hours, days, months, ## .. code-block:: nim ## ## let day = initInterval(hours=24) - ## let tomorrow = getTime() + day - ## echo(tomorrow) - var carryO = 0 - result.milliseconds = `mod`(milliseconds, 1000) - carryO = `div`(milliseconds, 1000) - result.seconds = `mod`(carryO + seconds, 60) - carryO = `div`(carryO + seconds, 60) - result.minutes = `mod`(carryO + minutes, 60) - carryO = `div`(carryO + minutes, 60) - result.hours = `mod`(carryO + hours, 24) - carryO = `div`(carryO + hours, 24) - result.days = carryO + days - - result.months = `mod`(months, 12) - carryO = `div`(months, 12) - result.years = carryO + years + ## let dt = initDateTime(01, mJan, 2000, 12, 00, 00, utc()) + ## doAssert $(dt + day) == "2000-01-02T12-00-00+00:00" + result.milliseconds = milliseconds + result.seconds = seconds + result.minutes = minutes + result.hours = hours + result.days = days + result.months = months + result.years = years proc `+`*(ti1, ti2: TimeInterval): TimeInterval = ## Adds two ``TimeInterval`` objects together. - var carryO = 0 - result.milliseconds = `mod`(ti1.milliseconds + ti2.milliseconds, 1000) - carryO = `div`(ti1.milliseconds + ti2.milliseconds, 1000) - result.seconds = `mod`(carryO + ti1.seconds + ti2.seconds, 60) - carryO = `div`(carryO + ti1.seconds + ti2.seconds, 60) - result.minutes = `mod`(carryO + ti1.minutes + ti2.minutes, 60) - carryO = `div`(carryO + ti1.minutes + ti2.minutes, 60) - result.hours = `mod`(carryO + ti1.hours + ti2.hours, 24) - carryO = `div`(carryO + ti1.hours + ti2.hours, 24) - result.days = carryO + ti1.days + ti2.days - - result.months = `mod`(ti1.months + ti2.months, 12) - carryO = `div`(ti1.months + ti2.months, 12) - result.years = carryO + ti1.years + ti2.years + result.milliseconds = ti1.milliseconds + ti2.milliseconds + result.seconds = ti1.seconds + ti2.seconds + result.minutes = ti1.minutes + ti2.minutes + result.hours = ti1.hours + ti2.hours + result.days = ti1.days + ti2.days + result.months = ti1.months + ti2.months + result.years = ti1.years + ti2.years proc `-`*(ti: TimeInterval): TimeInterval = ## Reverses a time interval + ## ## .. code-block:: nim ## ## let day = -initInterval(hours=24) - ## echo day # -> (milliseconds: 0, seconds: 0, minutes: 0, hours: 0, days: -1, months: 0, years: 0) + ## echo day # -> (milliseconds: 0, seconds: 0, minutes: 0, hours: -24, days: 0, months: 0, years: 0) result = TimeInterval( milliseconds: -ti.milliseconds, seconds: -ti.seconds, @@ -325,123 +565,100 @@ proc `-`*(ti1, ti2: TimeInterval): TimeInterval = ## Time components are compared one-by-one, see output: ## ## .. code-block:: nim - ## let a = fromSeconds(1_000_000_000) - ## let b = fromSeconds(1_500_000_000) + ## let a = fromUnix(1_000_000_000) + ## let b = fromUnix(1_500_000_000) ## echo b.toTimeInterval - a.toTimeInterval - ## # (milliseconds: 0, seconds: -40, minutes: -6, hours: 1, days: -2, months: -2, years: 16) + ## # (milliseconds: 0, seconds: -40, minutes: -6, hours: 1, days: 5, months: -2, years: 16) result = ti1 + (-ti2) -proc isLeapYear*(year: int): bool = - ## returns true if ``year`` is a leap year - - if year mod 400 == 0: - return true - elif year mod 100 == 0: - return false - elif year mod 4 == 0: - return true - else: - return false - -proc getDaysInMonth*(month: Month, year: int): int = - ## Get the number of days in a ``month`` of a ``year`` +proc evaluateInterval(dt: DateTime, interval: TimeInterval): tuple[adjDiff, absDiff: int64] = + ## Evaluates how many seconds the interval is worth + ## in the context of ``dt``. + ## The result in split into an adjusted diff and an absolute diff. - # http://www.dispersiondesign.com/articles/time/number_of_days_in_a_month - case month - of mFeb: result = if isLeapYear(year): 29 else: 28 - of mApr, mJun, mSep, mNov: result = 30 - else: result = 31 - -proc getDaysInYear*(year: int): int = - ## Get the number of days in a ``year`` - result = 365 + (if isLeapYear(year): 1 else: 0) - -proc toSeconds(a: TimeInfo, interval: TimeInterval): float = - ## Calculates how many seconds the interval is worth by adding up - ## all the fields - - var anew = a + var anew = dt var newinterv = interval - result = 0 newinterv.months += interval.years * 12 var curMonth = anew.month - if newinterv.months < 0: # subtracting + # Subtracting + if newinterv.months < 0: for mth in countDown(-1 * newinterv.months, 1): - result -= float(getDaysInMonth(curMonth, anew.year) * 24 * 60 * 60) if curMonth == mJan: curMonth = mDec anew.year.dec() else: curMonth.dec() - else: # adding + result.adjDiff -= getDaysInMonth(curMonth, anew.year) * secondsInDay + # Adding + else: for mth in 1 .. newinterv.months: - result += float(getDaysInMonth(curMonth, anew.year) * 24 * 60 * 60) + result.adjDiff += getDaysInMonth(curMonth, anew.year) * secondsInDay if curMonth == mDec: curMonth = mJan anew.year.inc() else: curMonth.inc() - result += float(newinterv.days * 24 * 60 * 60) - result += float(newinterv.hours * 60 * 60) - result += float(newinterv.minutes * 60) - result += float(newinterv.seconds) - result += newinterv.milliseconds / 1000 - -proc `+`*(a: TimeInfo, interval: TimeInterval): TimeInfo = - ## adds ``interval`` time from TimeInfo ``a``. + result.adjDiff += newinterv.days * secondsInDay + result.absDiff += newinterv.hours * secondsInHour + result.absDiff += newinterv.minutes * secondsInMin + result.absDiff += newinterv.seconds + result.absDiff += newinterv.milliseconds div 1000 + +proc `+`*(dt: DateTime, interval: TimeInterval): DateTime = + ## Adds ``interval`` to ``dt``. Components from ``interval`` are added + ## in the order of their size, i.e first the ``years`` component, then the ``months`` + ## component and so on. The returned ``DateTime`` will have the same timezone as the input. ## - ## **Note:** This has been only briefly tested and it may not be - ## very accurate. - let t = toSeconds(toTime(a)) - let secs = toSeconds(a, interval) - if a.timezone == 0: - result = getGMTime(fromSeconds(t + secs)) - else: - result = getLocalTime(fromSeconds(t + secs)) - -proc `-`*(a: TimeInfo, interval: TimeInterval): TimeInfo = - ## subtracts ``interval`` time from TimeInfo ``a``. + ## Note that when adding months, monthday overflow is allowed. This means that if the resulting + ## month doesn't have enough days it, the month will be incremented and the monthday will be + ## set to the number of days overflowed. So adding one month to `31 October` will result in `31 November`, + ## which will overflow and result in `1 December`. ## - ## **Note:** This has been only briefly tested, it is inaccurate especially - ## when you subtract so much that you reach the Julian calendar. - let - t = toSeconds(toTime(a)) - secs = toSeconds(a, -interval) - if a.timezone == 0: - result = getGMTime(fromSeconds(t + secs)) + ## .. code-block:: nim + ## let dt = initDateTime(30, mMar, 2017, 00, 00, 00, utc()) + ## doAssert $(dt + 1.months) == "2017-04-30T00:00:00+00:00" + ## # This is correct and happens due to monthday overflow. + ## doAssert $(dt - 1.months) == "2017-03-02T00:00:00+00:00" + let (adjDiff, absDiff) = evaluateInterval(dt, interval) + + if adjDiff.int64 != 0: + let zInfo = dt.timezone.zoneInfoFromTz(Time(dt.toAdjTime.int64 + adjDiff)) + + if absDiff != 0: + let time = Time(zInfo.adjTime.int64 + zInfo.utcOffset + absDiff) + result = initDateTime(dt.timezone.zoneInfoFromUtc(time), dt.timezone) + else: + result = initDateTime(zInfo, dt.timezone) else: - result = getLocalTime(fromSeconds(t + secs)) + result = initDateTime(dt.timezone.zoneInfoFromUtc(Time(dt.toTime.int64 + absDiff)), dt.timezone) -proc miliseconds*(t: TimeInterval): int {.deprecated.} = t.milliseconds - -proc `miliseconds=`*(t: var TimeInterval, milliseconds: int) {.deprecated.} = - ## An alias for a misspelled field in ``TimeInterval``. - ## - ## **Warning:** This should not be used! It will be removed in the next - ## version. - t.milliseconds = milliseconds +proc `-`*(dt: DateTime, interval: TimeInterval): DateTime = + ## Subtract ``interval`` from ``dt``. Components from ``interval`` are subtracted + ## in the order of their size, i.e first the ``years`` component, then the ``months`` + ## component and so on. The returned ``DateTime`` will have the same timezone as the input. + dt + (-interval) proc getDateStr*(): string {.rtl, extern: "nt$1", tags: [TimeEffect].} = - ## gets the current date as a string of the format ``YYYY-MM-DD``. - var ti = getLocalTime(getTime()) - result = $ti.year & '-' & intToStr(ord(ti.month)+1, 2) & + ## Gets the current date as a string of the format ``YYYY-MM-DD``. + var ti = now() + result = $ti.year & '-' & intToStr(ord(ti.month), 2) & '-' & intToStr(ti.monthday, 2) proc getClockStr*(): string {.rtl, extern: "nt$1", tags: [TimeEffect].} = - ## gets the current clock time as a string of the format ``HH:MM:SS``. - var ti = getLocalTime(getTime()) + ## Gets the current clock time as a string of the format ``HH:MM:SS``. + var ti = now() result = intToStr(ti.hour, 2) & ':' & intToStr(ti.minute, 2) & ':' & intToStr(ti.second, 2) proc `$`*(day: WeekDay): string = - ## stingify operator for ``WeekDay``. + ## Stringify operator for ``WeekDay``. const lookup: array[WeekDay, string] = ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"] return lookup[day] proc `$`*(m: Month): string = - ## stingify operator for ``Month``. + ## Stringify operator for ``Month``. const lookup: array[Month, string] = ["January", "February", "March", "April", "May", "June", "July", "August", "September", "October", "November", "December"] @@ -450,74 +667,68 @@ proc `$`*(m: Month): string = proc milliseconds*(ms: int): TimeInterval {.inline.} = ## TimeInterval of `ms` milliseconds ## - ## Note: not all time functions have millisecond resolution - initInterval(`mod`(ms,1000), `div`(ms,1000)) + ## Note: not all time procedures have millisecond resolution + initInterval(milliseconds = ms) proc seconds*(s: int): TimeInterval {.inline.} = ## TimeInterval of `s` seconds ## ## ``echo getTime() + 5.second`` - initInterval(0,`mod`(s,60), `div`(s,60)) + initInterval(seconds = s) proc minutes*(m: int): TimeInterval {.inline.} = ## TimeInterval of `m` minutes ## ## ``echo getTime() + 5.minutes`` - initInterval(0,0,`mod`(m,60), `div`(m,60)) + initInterval(minutes = m) proc hours*(h: int): TimeInterval {.inline.} = ## TimeInterval of `h` hours ## ## ``echo getTime() + 2.hours`` - initInterval(0,0,0,`mod`(h,24),`div`(h,24)) + initInterval(hours = h) proc days*(d: int): TimeInterval {.inline.} = ## TimeInterval of `d` days ## ## ``echo getTime() + 2.days`` - initInterval(0,0,0,0,d) + initInterval(days = d) proc months*(m: int): TimeInterval {.inline.} = ## TimeInterval of `m` months ## ## ``echo getTime() + 2.months`` - initInterval(0,0,0,0,0,`mod`(m,12),`div`(m,12)) + initInterval(months = m) proc years*(y: int): TimeInterval {.inline.} = ## TimeInterval of `y` years ## ## ``echo getTime() + 2.years`` - initInterval(0,0,0,0,0,0,y) + initInterval(years = y) -proc `+=`*(t: var Time, ti: TimeInterval) = - ## modifies `t` by adding the interval `ti` - t = toTime(getLocalTime(t) + ti) +proc `+=`*(time: var Time, interval: TimeInterval) = + ## Modifies `time` by adding `interval`. + time = toTime(time.local + interval) -proc `+`*(t: Time, ti: TimeInterval): Time = - ## adds the interval `ti` to Time `t` - ## by converting to localTime, adding the interval, and converting back +proc `+`*(time: Time, interval: TimeInterval): Time = + ## Adds `interval` to `time` + ## by converting to a ``DateTime`` in the local timezone, + ## adding the interval, and converting back to ``Time``. ## ## ``echo getTime() + 1.day`` - result = toTime(getLocalTime(t) + ti) + result = toTime(time.local + interval) -proc `-=`*(t: var Time, ti: TimeInterval) = - ## modifies `t` by subtracting the interval `ti` - t = toTime(getLocalTime(t) - ti) +proc `-=`*(time: var Time, interval: TimeInterval) = + ## Modifies `time` by subtracting `interval`. + time = toTime(time.local - interval) -proc `-`*(t: Time, ti: TimeInterval): Time = - ## subtracts the interval `ti` from Time `t` +proc `-`*(time: Time, interval: TimeInterval): Time = + ## Subtracts `interval` from Time `time`. ## ## ``echo getTime() - 1.day`` - result = toTime(getLocalTime(t) - ti) + result = toTime(time.local - interval) -const - secondsInMin = 60 - secondsInHour = 60*60 - secondsInDay = 60*60*24 - minutesInHour = 60 - epochStartYear = 1970 - -proc formatToken(info: TimeInfo, token: string, buf: var string) = +proc formatToken(dt: DateTime, token: string, buf: var string) = ## Helper of the format proc to parse individual tokens. ## ## Pass the found token in the user input string, and the buffer where the @@ -525,96 +736,96 @@ proc formatToken(info: TimeInfo, token: string, buf: var string) = ## formatting tokens require modifying the previous characters. case token of "d": - buf.add($info.monthday) + buf.add($dt.monthday) of "dd": - if info.monthday < 10: + if dt.monthday < 10: buf.add("0") - buf.add($info.monthday) + buf.add($dt.monthday) of "ddd": - buf.add(($info.weekday)[0 .. 2]) + buf.add(($dt.weekday)[0 .. 2]) of "dddd": - buf.add($info.weekday) + buf.add($dt.weekday) of "h": - buf.add($(if info.hour > 12: info.hour - 12 else: info.hour)) + buf.add($(if dt.hour > 12: dt.hour - 12 else: dt.hour)) of "hh": - let amerHour = if info.hour > 12: info.hour - 12 else: info.hour + let amerHour = if dt.hour > 12: dt.hour - 12 else: dt.hour if amerHour < 10: buf.add('0') buf.add($amerHour) of "H": - buf.add($info.hour) + buf.add($dt.hour) of "HH": - if info.hour < 10: + if dt.hour < 10: buf.add('0') - buf.add($info.hour) + buf.add($dt.hour) of "m": - buf.add($info.minute) + buf.add($dt.minute) of "mm": - if info.minute < 10: + if dt.minute < 10: buf.add('0') - buf.add($info.minute) + buf.add($dt.minute) of "M": - buf.add($(int(info.month)+1)) + buf.add($ord(dt.month)) of "MM": - if info.month < mOct: + if dt.month < mOct: buf.add('0') - buf.add($(int(info.month)+1)) + buf.add($ord(dt.month)) of "MMM": - buf.add(($info.month)[0..2]) + buf.add(($dt.month)[0..2]) of "MMMM": - buf.add($info.month) + buf.add($dt.month) of "s": - buf.add($info.second) + buf.add($dt.second) of "ss": - if info.second < 10: + if dt.second < 10: buf.add('0') - buf.add($info.second) + buf.add($dt.second) of "t": - if info.hour >= 12: + if dt.hour >= 12: buf.add('P') else: buf.add('A') of "tt": - if info.hour >= 12: + if dt.hour >= 12: buf.add("PM") else: buf.add("AM") of "y": - var fr = ($info.year).len()-1 + var fr = ($dt.year).len()-1 if fr < 0: fr = 0 - buf.add(($info.year)[fr .. ($info.year).len()-1]) + buf.add(($dt.year)[fr .. ($dt.year).len()-1]) of "yy": - var fr = ($info.year).len()-2 + var fr = ($dt.year).len()-2 if fr < 0: fr = 0 - var fyear = ($info.year)[fr .. ($info.year).len()-1] + var fyear = ($dt.year)[fr .. ($dt.year).len()-1] if fyear.len != 2: fyear = repeat('0', 2-fyear.len()) & fyear buf.add(fyear) of "yyy": - var fr = ($info.year).len()-3 + var fr = ($dt.year).len()-3 if fr < 0: fr = 0 - var fyear = ($info.year)[fr .. ($info.year).len()-1] + var fyear = ($dt.year)[fr .. ($dt.year).len()-1] if fyear.len != 3: fyear = repeat('0', 3-fyear.len()) & fyear buf.add(fyear) of "yyyy": - var fr = ($info.year).len()-4 + var fr = ($dt.year).len()-4 if fr < 0: fr = 0 - var fyear = ($info.year)[fr .. ($info.year).len()-1] + var fyear = ($dt.year)[fr .. ($dt.year).len()-1] if fyear.len != 4: fyear = repeat('0', 4-fyear.len()) & fyear buf.add(fyear) of "yyyyy": - var fr = ($info.year).len()-5 + var fr = ($dt.year).len()-5 if fr < 0: fr = 0 - var fyear = ($info.year)[fr .. ($info.year).len()-1] + var fyear = ($dt.year)[fr .. ($dt.year).len()-1] if fyear.len != 5: fyear = repeat('0', 5-fyear.len()) & fyear buf.add(fyear) of "z": let - nonDstTz = info.timezone - int(info.isDst) * secondsInHour + nonDstTz = dt.utcOffset hours = abs(nonDstTz) div secondsInHour if nonDstTz <= 0: buf.add('+') else: buf.add('-') buf.add($hours) of "zz": let - nonDstTz = info.timezone - int(info.isDst) * secondsInHour + nonDstTz = dt.utcOffset hours = abs(nonDstTz) div secondsInHour if nonDstTz <= 0: buf.add('+') else: buf.add('-') @@ -622,7 +833,7 @@ proc formatToken(info: TimeInfo, token: string, buf: var string) = buf.add($hours) of "zzz": let - nonDstTz = info.timezone - int(info.isDst) * secondsInHour + nonDstTz = dt.utcOffset hours = abs(nonDstTz) div secondsInHour minutes = (abs(nonDstTz) div secondsInMin) mod minutesInHour if nonDstTz <= 0: buf.add('+') @@ -638,8 +849,8 @@ proc formatToken(info: TimeInfo, token: string, buf: var string) = raise newException(ValueError, "Invalid format string: " & token) -proc format*(info: TimeInfo, f: string): string = - ## This function formats `info` as specified by `f`. The following format +proc format*(dt: DateTime, f: string): string {.tags: [].}= + ## This procedure formats `dt` as specified by `f`. The following format ## specifiers are available: ## ## ========== ================================================================================= ================================================ @@ -683,7 +894,7 @@ proc format*(info: TimeInfo, f: string): string = while true: case f[i] of ' ', '-', '/', ':', '\'', '\0', '(', ')', '[', ']', ',': - formatToken(info, currentF, result) + formatToken(dt, currentF, result) currentF = "" if f[i] == '\0': break @@ -700,187 +911,187 @@ proc format*(info: TimeInfo, f: string): string = if currentF.len < 1 or currentF[high(currentF)] == f[i]: currentF.add(f[i]) else: - formatToken(info, currentF, result) + formatToken(dt, currentF, result) dec(i) # Move position back to re-process the character separately. currentF = "" inc(i) -proc `$`*(timeInfo: TimeInfo): string {.tags: [], raises: [], benign.} = - ## converts a `TimeInfo` object to a string representation. +proc `$`*(dt: DateTime): string {.tags: [], raises: [], benign.} = + ## Converts a `DateTime` object to a string representation. ## It uses the format ``yyyy-MM-dd'T'HH-mm-sszzz``. try: - result = format(timeInfo, "yyyy-MM-dd'T'HH:mm:sszzz") # todo: optimize this + result = format(dt, "yyyy-MM-dd'T'HH:mm:sszzz") # todo: optimize this except ValueError: assert false # cannot happen because format string is valid -proc `$`*(time: Time): string {.tags: [TimeEffect], raises: [], benign.} = +proc `$`*(time: Time): string {.tags: [], raises: [], benign.} = ## converts a `Time` value to a string representation. It will use the local ## time zone and use the format ``yyyy-MM-dd'T'HH-mm-sszzz``. - $getLocalTime(time) + $time.local {.pop.} -proc parseToken(info: var TimeInfo; token, value: string; j: var int) = +proc parseToken(dt: var DateTime; token, value: string; j: var int) = ## Helper of the parse proc to parse individual tokens. var sv: int case token of "d": var pd = parseInt(value[j..j+1], sv) - info.monthday = sv + dt.monthday = sv j += pd of "dd": - info.monthday = value[j..j+1].parseInt() + dt.monthday = value[j..j+1].parseInt() j += 2 of "ddd": case value[j..j+2].toLowerAscii() - of "sun": info.weekday = dSun - of "mon": info.weekday = dMon - of "tue": info.weekday = dTue - of "wed": info.weekday = dWed - of "thu": info.weekday = dThu - of "fri": info.weekday = dFri - of "sat": info.weekday = dSat + of "sun": dt.weekday = dSun + of "mon": dt.weekday = dMon + of "tue": dt.weekday = dTue + of "wed": dt.weekday = dWed + of "thu": dt.weekday = dThu + of "fri": dt.weekday = dFri + of "sat": dt.weekday = dSat else: raise newException(ValueError, "Couldn't parse day of week (ddd), got: " & value[j..j+2]) j += 3 of "dddd": if value.len >= j+6 and value[j..j+5].cmpIgnoreCase("sunday") == 0: - info.weekday = dSun + dt.weekday = dSun j += 6 elif value.len >= j+6 and value[j..j+5].cmpIgnoreCase("monday") == 0: - info.weekday = dMon + dt.weekday = dMon j += 6 elif value.len >= j+7 and value[j..j+6].cmpIgnoreCase("tuesday") == 0: - info.weekday = dTue + dt.weekday = dTue j += 7 elif value.len >= j+9 and value[j..j+8].cmpIgnoreCase("wednesday") == 0: - info.weekday = dWed + dt.weekday = dWed j += 9 elif value.len >= j+8 and value[j..j+7].cmpIgnoreCase("thursday") == 0: - info.weekday = dThu + dt.weekday = dThu j += 8 elif value.len >= j+6 and value[j..j+5].cmpIgnoreCase("friday") == 0: - info.weekday = dFri + dt.weekday = dFri j += 6 elif value.len >= j+8 and value[j..j+7].cmpIgnoreCase("saturday") == 0: - info.weekday = dSat + dt.weekday = dSat j += 8 else: raise newException(ValueError, "Couldn't parse day of week (dddd), got: " & value) of "h", "H": var pd = parseInt(value[j..j+1], sv) - info.hour = sv + dt.hour = sv j += pd of "hh", "HH": - info.hour = value[j..j+1].parseInt() + dt.hour = value[j..j+1].parseInt() j += 2 of "m": var pd = parseInt(value[j..j+1], sv) - info.minute = sv + dt.minute = sv j += pd of "mm": - info.minute = value[j..j+1].parseInt() + dt.minute = value[j..j+1].parseInt() j += 2 of "M": var pd = parseInt(value[j..j+1], sv) - info.month = Month(sv-1) + dt.month = sv.Month j += pd of "MM": var month = value[j..j+1].parseInt() j += 2 - info.month = Month(month-1) + dt.month = month.Month of "MMM": case value[j..j+2].toLowerAscii(): - of "jan": info.month = mJan - of "feb": info.month = mFeb - of "mar": info.month = mMar - of "apr": info.month = mApr - of "may": info.month = mMay - of "jun": info.month = mJun - of "jul": info.month = mJul - of "aug": info.month = mAug - of "sep": info.month = mSep - of "oct": info.month = mOct - of "nov": info.month = mNov - of "dec": info.month = mDec + of "jan": dt.month = mJan + of "feb": dt.month = mFeb + of "mar": dt.month = mMar + of "apr": dt.month = mApr + of "may": dt.month = mMay + of "jun": dt.month = mJun + of "jul": dt.month = mJul + of "aug": dt.month = mAug + of "sep": dt.month = mSep + of "oct": dt.month = mOct + of "nov": dt.month = mNov + of "dec": dt.month = mDec else: raise newException(ValueError, "Couldn't parse month (MMM), got: " & value) j += 3 of "MMMM": if value.len >= j+7 and value[j..j+6].cmpIgnoreCase("january") == 0: - info.month = mJan + dt.month = mJan j += 7 elif value.len >= j+8 and value[j..j+7].cmpIgnoreCase("february") == 0: - info.month = mFeb + dt.month = mFeb j += 8 elif value.len >= j+5 and value[j..j+4].cmpIgnoreCase("march") == 0: - info.month = mMar + dt.month = mMar j += 5 elif value.len >= j+5 and value[j..j+4].cmpIgnoreCase("april") == 0: - info.month = mApr + dt.month = mApr j += 5 elif value.len >= j+3 and value[j..j+2].cmpIgnoreCase("may") == 0: - info.month = mMay + dt.month = mMay j += 3 elif value.len >= j+4 and value[j..j+3].cmpIgnoreCase("june") == 0: - info.month = mJun + dt.month = mJun j += 4 elif value.len >= j+4 and value[j..j+3].cmpIgnoreCase("july") == 0: - info.month = mJul + dt.month = mJul j += 4 elif value.len >= j+6 and value[j..j+5].cmpIgnoreCase("august") == 0: - info.month = mAug + dt.month = mAug j += 6 elif value.len >= j+9 and value[j..j+8].cmpIgnoreCase("september") == 0: - info.month = mSep + dt.month = mSep j += 9 elif value.len >= j+7 and value[j..j+6].cmpIgnoreCase("october") == 0: - info.month = mOct + dt.month = mOct j += 7 elif value.len >= j+8 and value[j..j+7].cmpIgnoreCase("november") == 0: - info.month = mNov + dt.month = mNov j += 8 elif value.len >= j+8 and value[j..j+7].cmpIgnoreCase("december") == 0: - info.month = mDec + dt.month = mDec j += 8 else: raise newException(ValueError, "Couldn't parse month (MMMM), got: " & value) of "s": var pd = parseInt(value[j..j+1], sv) - info.second = sv + dt.second = sv j += pd of "ss": - info.second = value[j..j+1].parseInt() + dt.second = value[j..j+1].parseInt() j += 2 of "t": - if value[j] == 'P' and info.hour > 0 and info.hour < 12: - info.hour += 12 + if value[j] == 'P' and dt.hour > 0 and dt.hour < 12: + dt.hour += 12 j += 1 of "tt": - if value[j..j+1] == "PM" and info.hour > 0 and info.hour < 12: - info.hour += 12 + if value[j..j+1] == "PM" and dt.hour > 0 and dt.hour < 12: + dt.hour += 12 j += 2 of "yy": # Assumes current century var year = value[j..j+1].parseInt() - var thisCen = getLocalTime(getTime()).year div 100 - info.year = thisCen*100 + year + var thisCen = now().year div 100 + dt.year = thisCen*100 + year j += 2 of "yyyy": - info.year = value[j..j+3].parseInt() + dt.year = value[j..j+3].parseInt() j += 4 of "z": - info.isDST = false + dt.isDst = false if value[j] == '+': - info.timezone = 0 - parseInt($value[j+1]) * secondsInHour + dt.utcOffset = 0 - parseInt($value[j+1]) * secondsInHour elif value[j] == '-': - info.timezone = parseInt($value[j+1]) * secondsInHour + dt.utcOffset = parseInt($value[j+1]) * secondsInHour elif value[j] == 'Z': - info.timezone = 0 + dt.utcOffset = 0 j += 1 return else: @@ -888,13 +1099,13 @@ proc parseToken(info: var TimeInfo; token, value: string; j: var int) = "Couldn't parse timezone offset (z), got: " & value[j]) j += 2 of "zz": - info.isDST = false + dt.isDst = false if value[j] == '+': - info.timezone = 0 - value[j+1..j+2].parseInt() * secondsInHour + dt.utcOffset = 0 - value[j+1..j+2].parseInt() * secondsInHour elif value[j] == '-': - info.timezone = value[j+1..j+2].parseInt() * secondsInHour + dt.utcOffset = value[j+1..j+2].parseInt() * secondsInHour elif value[j] == 'Z': - info.timezone = 0 + dt.utcOffset = 0 j += 1 return else: @@ -902,31 +1113,33 @@ proc parseToken(info: var TimeInfo; token, value: string; j: var int) = "Couldn't parse timezone offset (zz), got: " & value[j]) j += 3 of "zzz": - info.isDST = false + dt.isDst = false var factor = 0 if value[j] == '+': factor = -1 elif value[j] == '-': factor = 1 elif value[j] == 'Z': - info.timezone = 0 + dt.utcOffset = 0 j += 1 return else: raise newException(ValueError, "Couldn't parse timezone offset (zzz), got: " & value[j]) - info.timezone = factor * value[j+1..j+2].parseInt() * secondsInHour + dt.utcOffset = factor * value[j+1..j+2].parseInt() * secondsInHour j += 4 - info.timezone += factor * value[j..j+1].parseInt() * 60 + dt.utcOffset += factor * value[j..j+1].parseInt() * 60 j += 2 else: # Ignore the token and move forward in the value string by the same length j += token.len -proc parse*(value, layout: string): TimeInfo = - ## This function parses a date/time string using the standard format - ## identifiers as listed below. The function defaults information not provided - ## in the format string from the running program (timezone, month, year, etc). - ## Daylight saving time is only set if no timezone is given and the given date - ## lies within the DST period of the current locale. +proc parse*(value, layout: string, zone: Timezone = local()): DateTime = + ## This procedure parses a date/time string using the standard format + ## identifiers as listed below. The procedure defaults information not provided + ## in the format string from the running program (month, year, etc). + ## + ## The return value will always be in the `zone` timezone. If no UTC offset was + ## parsed, then the input will be assumed to be specified in the `zone` timezone + ## already, so no timezone conversion will be done in that case. ## ## ========== ================================================================================= ================================================ ## Specifier Description Example @@ -965,17 +1178,17 @@ proc parse*(value, layout: string): TimeInfo = var j = 0 # pointer for value string var token = "" # Assumes current day of month, month and year, but time is reset to 00:00:00. Weekday will be reset after parsing. - var info = getLocalTime(getTime()) - info.hour = 0 - info.minute = 0 - info.second = 0 - info.isDST = true # using this is flag for checking whether a timezone has \ + var dt = now() + dt.hour = 0 + dt.minute = 0 + dt.second = 0 + dt.isDst = true # using this is flag for checking whether a timezone has \ # been read (because DST is always false when a tz is parsed) while true: case layout[i] of ' ', '-', '/', ':', '\'', '\0', '(', ')', '[', ']', ',': if token.len > 0: - parseToken(info, token, value, j) + parseToken(dt, token, value, j) # Reset token token = "" # Break if at end of line @@ -997,26 +1210,15 @@ proc parse*(value, layout: string): TimeInfo = token.add(layout[i]) inc(i) else: - parseToken(info, token, value, j) + parseToken(dt, token, value, j) token = "" - if info.isDST: - # means that no timezone has been parsed. In this case, we need to check - # whether the date is within DST of the local time. - let tmp = getLocalTime(toTime(info)) - # correctly set isDST so that the following step works on the correct time - info.isDST = tmp.isDST - - # Correct weekday and yearday; transform timestamp to local time. - # There currently is no way of returning this with the original (parsed) - # timezone while also setting weekday and yearday (we are depending on stdlib - # to provide this calculation). - return getLocalTime(toTime(info)) - -# Leap year calculations are adapted from: -# http://www.codeproject.com/Articles/7358/Ultra-fast-Algorithms-for-Working-with-Leap-Years -# The dayOfTheWeek procs are adapated from: -# http://stason.org/TULARC/society/calendars/2-5-What-day-of-the-week-was-2-August-1953.html + if dt.isDst: + # No timezone parsed - assume timezone is `zone` + result = initDateTime(zone.zoneInfoFromTz(dt.toAdjTime), zone) + else: + # Otherwise convert to `zone` + result = dt.toTime.inZone(zone) proc countLeapYears*(yearSpan: int): int = ## Returns the number of leap years spanned by a given number of years. @@ -1025,7 +1227,7 @@ proc countLeapYears*(yearSpan: int): int = ## counts the number of leap years up to January 1st of a given year. ## Keep in mind that if specified year is a leap year, the leap day ## has not happened before January 1st of that year. - (((yearSpan - 1) / 4) - ((yearSpan - 1) / 100) + ((yearSpan - 1) / 400)).int + (yearSpan - 1) div 4 - (yearSpan - 1) div 100 + (yearSpan - 1) div 400 proc countDays*(yearSpan: int): int = ## Returns the number of days spanned by a given number of years. @@ -1042,75 +1244,7 @@ proc countYearsAndDays*(daySpan: int): tuple[years: int, days: int] = result.years = days div 365 result.days = days mod 365 -proc getDayOfWeek*(day, month, year: int): WeekDay = - ## Returns the day of the week enum from day, month and year. - # Day & month start from one. - let - a = (14 - month) div 12 - y = year - a - m = month + (12*a) - 2 - d = (day + y + (y div 4) - (y div 100) + (y div 400) + (31*m) div 12) mod 7 - # The value of d is 0 for a Sunday, 1 for a Monday, 2 for a Tuesday, etc. - # so we must correct for the WeekDay type. - if d == 0: return dSun - result = (d-1).WeekDay - -proc getDayOfWeekJulian*(day, month, year: int): WeekDay = - ## Returns the day of the week enum from day, month and year, - ## according to the Julian calendar. - # Day & month start from one. - let - a = (14 - month) div 12 - y = year - a - m = month + (12*a) - 2 - d = (5 + day + y + (y div 4) + (31*m) div 12) mod 7 - result = d.WeekDay - -proc timeToTimeInfo*(t: Time): TimeInfo {.deprecated.} = - ## Converts a Time to TimeInfo. - ## - ## **Warning:** This procedure is deprecated since version 0.14.0. - ## Use ``getLocalTime`` or ``getGMTime`` instead. - let - secs = t.toSeconds().int - daysSinceEpoch = secs div secondsInDay - (yearsSinceEpoch, daysRemaining) = countYearsAndDays(daysSinceEpoch) - daySeconds = secs mod secondsInDay - - y = yearsSinceEpoch + epochStartYear - - var - mon = mJan - days = daysRemaining - daysInMonth = getDaysInMonth(mon, y) - - # calculate month and day remainder - while days > daysInMonth and mon <= mDec: - days -= daysInMonth - mon.inc - daysInMonth = getDaysInMonth(mon, y) - - let - yd = daysRemaining - m = mon # month is zero indexed enum - md = days - # NB: month is zero indexed but dayOfWeek expects 1 indexed. - wd = getDayOfWeek(days, mon.int + 1, y).Weekday - h = daySeconds div secondsInHour + 1 - mi = (daySeconds mod secondsInHour) div secondsInMin - s = daySeconds mod secondsInMin - result = TimeInfo(year: y, yearday: yd, month: m, monthday: md, weekday: wd, hour: h, minute: mi, second: s) - -proc timeToTimeInterval*(t: Time): TimeInterval {.deprecated.} = - ## Converts a Time to a TimeInterval. - ## - ## **Warning:** This procedure is deprecated since version 0.14.0. - ## Use ``toTimeInterval`` instead. - # Milliseconds not available from Time - var tInfo = t.getLocalTime() - initInterval(0, tInfo.second, tInfo.minute, tInfo.hour, tInfo.weekday.ord, tInfo.month.ord, tInfo.year) - -proc toTimeInterval*(t: Time): TimeInterval = +proc toTimeInterval*(time: Time): TimeInterval = ## Converts a Time to a TimeInterval. ## ## To be used when diffing times. @@ -1121,10 +1255,25 @@ proc toTimeInterval*(t: Time): TimeInterval = ## echo a, " ", b # real dates ## echo a.toTimeInterval # meaningless value, don't use it by itself ## echo b.toTimeInterval - a.toTimeInterval - ## # (milliseconds: 0, seconds: -40, minutes: -6, hours: 1, days: -2, months: -2, years: 16) + ## # (milliseconds: 0, seconds: -40, minutes: -6, hours: 1, days: 5, months: -2, years: 16) # Milliseconds not available from Time - var tInfo = t.getLocalTime() - initInterval(0, tInfo.second, tInfo.minute, tInfo.hour, tInfo.weekday.ord, tInfo.month.ord, tInfo.year) + var dt = time.local + initInterval(0, dt.second, dt.minute, dt.hour, dt.monthday, dt.month.ord - 1, dt.year) + +proc initDateTime*(monthday: MonthdayRange, month: Month, year: int, + hour: HourRange, minute: MinuteRange, second: SecondRange, zone: Timezone = local()): DateTime = + ## Create a new ``DateTime`` in the specified timezone. + assertValidDate monthday, month, year + doAssert monthday <= getDaysInMonth(month, year), "Invalid date: " & $month & " " & $monthday & ", " & $year + let dt = DateTime( + monthday: monthday, + year: year, + month: month, + hour: hour, + minute: minute, + second: second + ) + result = initDateTime(zone.zoneInfoFromTz(dt.toAdjTime), zone) when not defined(JS): proc epochTime*(): float {.rtl, extern: "nt$1", tags: [TimeEffect].} @@ -1145,160 +1294,39 @@ when not defined(JS): ## doWork() ## echo "CPU time [s] ", cpuTime() - t0 -when not defined(JS): - # C wrapper: - when defined(freebsd) or defined(netbsd) or defined(openbsd) or - defined(macosx): - type - StructTM {.importc: "struct tm".} = object - second {.importc: "tm_sec".}, - minute {.importc: "tm_min".}, - hour {.importc: "tm_hour".}, - monthday {.importc: "tm_mday".}, - month {.importc: "tm_mon".}, - year {.importc: "tm_year".}, - weekday {.importc: "tm_wday".}, - yearday {.importc: "tm_yday".}, - isdst {.importc: "tm_isdst".}: cint - gmtoff {.importc: "tm_gmtoff".}: clong - else: - type - StructTM {.importc: "struct tm".} = object - second {.importc: "tm_sec".}, - minute {.importc: "tm_min".}, - hour {.importc: "tm_hour".}, - monthday {.importc: "tm_mday".}, - month {.importc: "tm_mon".}, - year {.importc: "tm_year".}, - weekday {.importc: "tm_wday".}, - yearday {.importc: "tm_yday".}, - isdst {.importc: "tm_isdst".}: cint - when defined(linux) and defined(amd64): - gmtoff {.importc: "tm_gmtoff".}: clong - zone {.importc: "tm_zone".}: cstring +when defined(JS): + proc getTime(): Time = + (newDate().getTime() div 1000).Time + + proc epochTime*(): float {.tags: [TimeEffect].} = + newDate().getTime() / 1000 + +else: type - TimeInfoPtr = ptr StructTM Clock {.importc: "clock_t".} = distinct int - when not defined(windows): - # This is not ANSI C, but common enough - proc timegm(t: StructTM): Time {. - importc: "timegm", header: "<time.h>", tags: [].} - - proc localtime(timer: ptr Time): TimeInfoPtr {. - importc: "localtime", header: "<time.h>", tags: [].} - proc gmtime(timer: ptr Time): TimeInfoPtr {. - importc: "gmtime", header: "<time.h>", tags: [].} - proc timec(timer: ptr Time): Time {. + proc timec(timer: ptr CTime): CTime {. importc: "time", header: "<time.h>", tags: [].} - proc mktime(t: StructTM): Time {. - importc: "mktime", header: "<time.h>", tags: [].} + proc getClock(): Clock {.importc: "clock", header: "<time.h>", tags: [TimeEffect].} - proc difftime(a, b: Time): float {.importc: "difftime", header: "<time.h>", - tags: [].} var clocksPerSec {.importc: "CLOCKS_PER_SEC", nodecl.}: int - # our own procs on top of that: - proc tmToTimeInfo(tm: StructTM, local: bool): TimeInfo = - const - weekDays: array[0..6, WeekDay] = [ - dSun, dMon, dTue, dWed, dThu, dFri, dSat] - TimeInfo(second: int(tm.second), - minute: int(tm.minute), - hour: int(tm.hour), - monthday: int(tm.monthday), - month: Month(tm.month), - year: tm.year + 1900'i32, - weekday: weekDays[int(tm.weekday)], - yearday: int(tm.yearday), - isDST: tm.isdst > 0, - timezone: if local: getTimezone() else: 0 - ) - - - proc timeInfoToTM(t: TimeInfo): StructTM = - const - weekDays: array[WeekDay, int8] = [1'i8,2'i8,3'i8,4'i8,5'i8,6'i8,0'i8] - result.second = t.second - result.minute = t.minute - result.hour = t.hour - result.monthday = t.monthday - result.month = ord(t.month) - result.year = cint(t.year - 1900) - result.weekday = weekDays[t.weekday] - result.yearday = t.yearday - result.isdst = if t.isDST: 1 else: 0 - - when not defined(useNimRtl): - proc `-` (a, b: Time): int64 = - return toBiggestInt(difftime(a, b)) - - proc getStartMilsecs(): int = - #echo "clocks per sec: ", clocksPerSec, "clock: ", int(getClock()) - #return getClock() div (clocksPerSec div 1000) - when defined(macosx): - result = toInt(toFloat(int(getClock())) / (toFloat(clocksPerSec) / 1000.0)) - else: - result = int(getClock()) div (clocksPerSec div 1000) - when false: - var a: Timeval - posix_gettimeofday(a) - result = a.tv_sec * 1000'i64 + a.tv_usec div 1000'i64 - #echo "result: ", result - - proc getTime(): Time = return timec(nil) - proc getLocalTime(t: Time): TimeInfo = - var a = t - let lt = localtime(addr(a)) - assert(not lt.isNil) - result = tmToTimeInfo(lt[], true) - # copying is needed anyway to provide reentrancity; thus - # the conversion is not expensive - - proc getGMTime(t: Time): TimeInfo = - var a = t - result = tmToTimeInfo(gmtime(addr(a))[], false) - # copying is needed anyway to provide reentrancity; thus - # the conversion is not expensive - - proc toTime(timeInfo: TimeInfo): Time = - var cTimeInfo = timeInfo # for C++ we have to make a copy - # because the header of mktime is broken in my version of libc - - result = mktime(timeInfoToTM(cTimeInfo)) - # mktime is defined to interpret the input as local time. As timeInfoToTM - # does ignore the timezone, we need to adjust this here. - result = Time(TimeImpl(result) - getTimezone() + timeInfo.timezone) - - proc timeInfoToTime(timeInfo: TimeInfo): Time = toTime(timeInfo) + proc getTime(): Time = + timec(nil).Time const epochDiff = 116444736000000000'i64 rateDiff = 10000000'i64 # 100 nsecs - proc unixTimeToWinTime*(t: Time): int64 = + proc unixTimeToWinTime*(time: CTime): int64 = ## converts a UNIX `Time` (``time_t``) to a Windows file time - result = int64(t) * rateDiff + epochDiff + result = int64(time) * rateDiff + epochDiff - proc winTimeToUnixTime*(t: int64): Time = + proc winTimeToUnixTime*(time: int64): CTime = ## converts a Windows time to a UNIX `Time` (``time_t``) - result = Time((t - epochDiff) div rateDiff) - - proc getTimezone(): int = - when defined(freebsd) or defined(netbsd) or defined(openbsd): - var a = timec(nil) - let lt = localtime(addr(a)) - # BSD stores in `gmtoff` offset east of UTC in seconds, - # but posix systems using west of UTC in seconds - return -(lt.gmtoff) - else: - return timezone - - proc fromSeconds(since1970: float): Time = Time(since1970) - - proc toSeconds(time: Time): float = float(time) + result = CTime((time - epochDiff) div rateDiff) when not defined(useNimRtl): proc epochTime(): float = @@ -1319,83 +1347,138 @@ when not defined(JS): proc cpuTime(): float = result = toFloat(int(getClock())) / toFloat(clocksPerSec) -elif defined(JS): - proc newDate(): Time {.importc: "new Date".} - proc internGetTime(): Time {.importc: "new Date", tags: [].} +# Deprecated procs - proc newDate(value: float): Time {.importc: "new Date".} - proc newDate(value: cstring): Time {.importc: "new Date".} - proc getTime(): Time = - # Warning: This is something different in JS. - return newDate() +proc fromSeconds*(since1970: float): Time {.tags: [], raises: [], benign, deprecated.} = + ## Takes a float which contains the number of seconds since the unix epoch and + ## returns a time object. + ## + ## **Deprecated since v0.18.0:** use ``fromUnix`` instead + Time(since1970) - const - weekDays: array[0..6, WeekDay] = [ - dSun, dMon, dTue, dWed, dThu, dFri, dSat] - - proc getLocalTime(t: Time): TimeInfo = - result.second = t.getSeconds() - result.minute = t.getMinutes() - result.hour = t.getHours() - result.monthday = t.getDate() - result.month = Month(t.getMonth()) - result.year = t.getFullYear() - result.weekday = weekDays[t.getDay()] - result.timezone = getTimezone() - - result.yearday = result.monthday - 1 - for month in mJan..<result.month: - result.yearday += getDaysInMonth(month, result.year) - - proc getGMTime(t: Time): TimeInfo = - result.second = t.getUTCSeconds() - result.minute = t.getUTCMinutes() - result.hour = t.getUTCHours() - result.monthday = t.getUTCDate() - result.month = Month(t.getUTCMonth()) - result.year = t.getUTCFullYear() - result.weekday = weekDays[t.getUTCDay()] - - result.yearday = result.monthday - 1 - for month in mJan..<result.month: - result.yearday += getDaysInMonth(month, result.year) - - proc timeInfoToTime(timeInfo: TimeInfo): Time = toTime(timeInfo) - - proc toTime*(timeInfo: TimeInfo): Time = newDate($timeInfo) - - proc `-` (a, b: Time): int64 = - return a.getTime() - b.getTime() +proc fromSeconds*(since1970: int64): Time {.tags: [], raises: [], benign, deprecated.} = + ## Takes an int which contains the number of seconds since the unix epoch and + ## returns a time object. + ## + ## **Deprecated since v0.18.0:** use ``fromUnix`` instead + Time(since1970) - var - startMilsecs = getTime() +proc toSeconds*(time: Time): float {.tags: [], raises: [], benign, deprecated.} = + ## Returns the time in seconds since the unix epoch. + ## + ## **Deprecated since v0.18.0:** use ``toUnix`` instead + float(time) + +proc getLocalTime*(time: Time): DateTime {.tags: [], raises: [], benign, deprecated.} = + ## Converts the calendar time `time` to broken-time representation, + ## expressed relative to the user's specified time zone. + ## + ## **Deprecated since v0.18.0:** use ``local`` instead + time.local - proc getStartMilsecs(): int = - ## get the milliseconds from the start of the program - return int(getTime() - startMilsecs) +proc getGMTime*(time: Time): DateTime {.tags: [], raises: [], benign, deprecated.} = + ## Converts the calendar time `time` to broken-down time representation, + ## expressed in Coordinated Universal Time (UTC). + ## + ## **Deprecated since v0.18.0:** use ``utc`` instead + time.utc - proc fromSeconds(since1970: float): Time = result = newDate(since1970 * 1000) +proc getTimezone*(): int {.tags: [TimeEffect], raises: [], benign, deprecated.} = + ## Returns the offset of the local (non-DST) timezone in seconds west of UTC. + ## + ## **Deprecated since v0.18.0:** use ``now().utcOffset`` to get the current + ## utc offset (including DST). + when defined(JS): + return newDate().getTimezoneOffset() * 60 + elif defined(freebsd) or defined(netbsd) or defined(openbsd): + var a = timec(nil) + let lt = localtime(addr(a)) + # BSD stores in `gmtoff` offset east of UTC in seconds, + # but posix systems using west of UTC in seconds + return -(lt.gmtoff) + else: + return timezone - proc toSeconds(time: Time): float = result = time.getTime() / 1000 +proc timeInfoToTime*(dt: DateTime): Time {.tags: [], benign, deprecated.} = + ## Converts a broken-down time structure to calendar time representation. + ## + ## **Warning:** This procedure is deprecated since version 0.14.0. + ## Use ``toTime`` instead. + dt.toTime + +when defined(JS): + var startMilsecs = getTime() + proc getStartMilsecs*(): int {.deprecated, tags: [TimeEffect], benign.} = + ## get the milliseconds from the start of the program. **Deprecated since + ## version 0.8.10.** Use ``epochTime`` or ``cpuTime`` instead. + when defined(JS): + ## get the milliseconds from the start of the program + return int(getTime() - startMilsecs) +else: + proc getStartMilsecs*(): int {.deprecated, tags: [TimeEffect], benign.} = + when defined(macosx): + result = toInt(toFloat(int(getClock())) / (toFloat(clocksPerSec) / 1000.0)) + else: + result = int(getClock()) div (clocksPerSec div 1000) + +proc miliseconds*(t: TimeInterval): int {.deprecated.} = + t.milliseconds + +proc timeToTimeInterval*(t: Time): TimeInterval {.deprecated.} = + ## Converts a Time to a TimeInterval. + ## + ## **Warning:** This procedure is deprecated since version 0.14.0. + ## Use ``toTimeInterval`` instead. + # Milliseconds not available from Time + t.toTimeInterval() - proc getTimezone(): int = result = newDate().getTimezoneOffset() * 60 +proc timeToTimeInfo*(t: Time): DateTime {.deprecated.} = + ## Converts a Time to DateTime. + ## + ## **Warning:** This procedure is deprecated since version 0.14.0. + ## Use ``inZone`` instead. + const epochStartYear = 1970 - proc epochTime*(): float {.tags: [TimeEffect].} = newDate().toSeconds() + let + secs = t.toSeconds().int + daysSinceEpoch = secs div secondsInDay + (yearsSinceEpoch, daysRemaining) = countYearsAndDays(daysSinceEpoch) + daySeconds = secs mod secondsInDay + y = yearsSinceEpoch + epochStartYear -when isMainModule: - # this is testing non-exported function var - t4 = getGMTime(fromSeconds(876124714)) # Mon 6 Oct 08:58:34 BST 1997 - t4L = getLocalTime(fromSeconds(876124714)) - assert toSeconds(t4, initInterval(seconds=0)) == 0.0 - assert toSeconds(t4L, initInterval(milliseconds=1)) == toSeconds(t4, initInterval(milliseconds=1)) - assert toSeconds(t4L, initInterval(seconds=1)) == toSeconds(t4, initInterval(seconds=1)) - assert toSeconds(t4L, initInterval(minutes=1)) == toSeconds(t4, initInterval(minutes=1)) - assert toSeconds(t4L, initInterval(hours=1)) == toSeconds(t4, initInterval(hours=1)) - assert toSeconds(t4L, initInterval(days=1)) == toSeconds(t4, initInterval(days=1)) - assert toSeconds(t4L, initInterval(months=1)) == toSeconds(t4, initInterval(months=1)) - assert toSeconds(t4L, initInterval(years=1)) == toSeconds(t4, initInterval(years=1)) - - # Further tests are in tests/stdlib/ttime.nim - # koch test c stdlib + mon = mJan + days = daysRemaining + daysInMonth = getDaysInMonth(mon, y) + + # calculate month and day remainder + while days > daysInMonth and mon <= mDec: + days -= daysInMonth + mon.inc + daysInMonth = getDaysInMonth(mon, y) + + let + yd = daysRemaining + m = mon # month is zero indexed enum + md = days + # NB: month is zero indexed but dayOfWeek expects 1 indexed. + wd = getDayOfWeek(days, mon, y).Weekday + h = daySeconds div secondsInHour + 1 + mi = (daySeconds mod secondsInHour) div secondsInMin + s = daySeconds mod secondsInMin + result = DateTime(year: y, yearday: yd, month: m, monthday: md, weekday: wd, hour: h, minute: mi, second: s) + +proc getDayOfWeek*(day, month, year: int): WeekDay {.tags: [], raises: [], benign, deprecated.} = + getDayOfWeek(day, month.Month, year) + +proc getDayOfWeekJulian*(day, month, year: int): WeekDay {.deprecated.} = + ## Returns the day of the week enum from day, month and year, + ## according to the Julian calendar. + # Day & month start from one. + let + a = (14 - month) div 12 + y = year - a + m = month + (12*a) - 2 + d = (5 + day + y + (y div 4) + (31*m) div 12) mod 7 + result = d.WeekDay diff --git a/lib/pure/typetraits.nim b/lib/pure/typetraits.nim index 55c4bf038..2047abda4 100644 --- a/lib/pure/typetraits.nim +++ b/lib/pure/typetraits.nim @@ -21,7 +21,7 @@ proc name*(t: typedesc): string {.magic: "TypeTrait".} ## ## proc `$`*(T: typedesc): string = name(T) ## - ## template test(x): stmt = + ## template test(x): typed = ## echo "type: ", type(x), ", value: ", x ## ## test 42 @@ -31,6 +31,10 @@ proc name*(t: typedesc): string {.magic: "TypeTrait".} ## test(@['A','B']) ## # --> type: seq[char], value: @[A, B] +proc `$`*(t: typedesc): string = + ## An alias for `name`. + name(t) + proc arity*(t: typedesc): int {.magic: "TypeTrait".} ## Returns the arity of the given type @@ -49,3 +53,15 @@ proc stripGenericParams*(t: typedesc): typedesc {.magic: "TypeTrait".} ## This trait is similar to `genericHead`, but instead of producing ## error for non-generic types, it will just return them unmodified +proc supportsCopyMem*(t: typedesc): bool {.magic: "TypeTrait".} + ## This trait returns true iff the type ``t`` is safe to use for + ## `copyMem`:idx:. Other languages name a type like these `blob`:idx:. + + +when isMainModule: + # echo type(42) + import streams + var ss = newStringStream() + ss.write($type(42)) # needs `$` + ss.setPosition(0) + doAssert ss.readAll() == "int" diff --git a/lib/pure/unicode.nim b/lib/pure/unicode.nim index 7d9c3108b..257c620f7 100644 --- a/lib/pure/unicode.nim +++ b/lib/pure/unicode.nim @@ -293,33 +293,33 @@ proc runeSubStr*(s: string, pos:int, len:int = int.high): string = if pos < 0: let (o, rl) = runeReverseOffset(s, -pos) if len >= rl: - result = s[o.. s.len-1] + result = s.substr(o, s.len-1) elif len < 0: let e = rl + len if e < 0: result = "" else: - result = s[o.. runeOffset(s, e-(rl+pos) , o)-1] + result = s.substr(o, runeOffset(s, e-(rl+pos) , o)-1) else: - result = s[o.. runeOffset(s, len, o)-1] + result = s.substr(o, runeOffset(s, len, o)-1) else: let o = runeOffset(s, pos) if o < 0: result = "" elif len == int.high: - result = s[o.. s.len-1] + result = s.substr(o, s.len-1) elif len < 0: let (e, rl) = runeReverseOffset(s, -len) discard rl if e <= 0: result = "" else: - result = s[o.. e-1] + result = s.substr(o, e-1) else: var e = runeOffset(s, len, o) if e < 0: e = s.len - result = s[o.. e-1] + result = s.substr(o, e-1) const alphaRanges = [ diff --git a/lib/pure/unittest.nim b/lib/pure/unittest.nim index 3772a213a..fbce087ff 100644 --- a/lib/pure/unittest.nim +++ b/lib/pure/unittest.nim @@ -21,13 +21,41 @@ ## ``nim c -r <testfile.nim>`` exits with 0 or 1 ## ## Running a single test -## --------------------- +## ===================== ## -## Simply specify the test name as a command line argument. +## Specify the test name as a command line argument. ## ## .. code:: ## -## nim c -r test "my super awesome test name" +## nim c -r test "my test name" "another test" +## +## Multiple arguments can be used. +## +## Running a single test suite +## =========================== +## +## Specify the suite name delimited by ``"::"``. +## +## .. code:: +## +## nim c -r test "my test name::" +## +## Selecting tests by pattern +## ========================== +## +## A single ``"*"`` can be used for globbing. +## +## Delimit the end of a suite name with ``"::"``. +## +## Tests matching **any** of the arguments are executed. +## +## .. code:: +## +## nim c -r test fast_suite::mytest1 fast_suite::mytest2 +## nim c -r test "fast_suite::mytest*" +## nim c -r test "auth*::" "crypto::hashing*" +## # Run suites starting with 'bug #' and standalone tests starting with '#' +## nim c -r test 'bug #*::' '::#*' ## ## Example ## ------- @@ -121,7 +149,7 @@ var checkpoints {.threadvar.}: seq[string] formatters {.threadvar.}: seq[OutputFormatter] - testsToRun {.threadvar.}: HashSet[string] + testsFilters {.threadvar.}: HashSet[string] when declared(stdout): abortOnError = existsEnv("NIMTEST_ABORT_ON_ERROR") @@ -300,22 +328,63 @@ method testEnded*(formatter: JUnitOutputFormatter, testResult: TestResult) = method suiteEnded*(formatter: JUnitOutputFormatter) = formatter.stream.writeLine("\t</testsuite>") -proc shouldRun(testName: string): bool = - if testsToRun.len == 0: +proc glob(matcher, filter: string): bool = + ## Globbing using a single `*`. Empty `filter` matches everything. + if filter.len == 0: return true - result = testName in testsToRun + if not filter.contains('*'): + return matcher == filter + + let beforeAndAfter = filter.split('*', maxsplit=1) + if beforeAndAfter.len == 1: + # "foo*" + return matcher.startswith(beforeAndAfter[0]) + + if matcher.len < filter.len - 1: + return false # "12345" should not match "123*345" + + return matcher.startsWith(beforeAndAfter[0]) and matcher.endsWith(beforeAndAfter[1]) + +proc matchFilter(suiteName, testName, filter: string): bool = + if filter == "": + return true + if testName == filter: + # corner case for tests containing "::" in their name + return true + let suiteAndTestFilters = filter.split("::", maxsplit=1) + + if suiteAndTestFilters.len == 1: + # no suite specified + let test_f = suiteAndTestFilters[0] + return glob(testName, test_f) + + return glob(suiteName, suiteAndTestFilters[0]) and glob(testName, suiteAndTestFilters[1]) + +when defined(testing): export matchFilter + +proc shouldRun(currentSuiteName, testName: string): bool = + ## Check if a test should be run by matching suiteName and testName against + ## test filters. + if testsFilters.len == 0: + return true + + for f in testsFilters: + if matchFilter(currentSuiteName, testName, f): + return true + + return false proc ensureInitialized() = if formatters == nil: formatters = @[OutputFormatter(defaultConsoleFormatter())] - if not testsToRun.isValid: - testsToRun.init() - when declared(os): + if not testsFilters.isValid: + testsFilters.init() + when declared(paramCount): # Read tests to run from the command line. for i in 1 .. paramCount(): - testsToRun.incl(paramStr(i)) + testsFilters.incl(paramStr(i)) # These two procs are added as workarounds for # https://github.com/nim-lang/Nim/issues/5549 @@ -395,7 +464,7 @@ template test*(name, body) {.dirty.} = ensureInitialized() - if shouldRun(name): + if shouldRun(when declared(testSuiteName): testSuiteName else: "", name): checkpoints = @[] var testStatusIMPL {.inject.} = OK diff --git a/lib/pure/uri.nim b/lib/pure/uri.nim index 4b2e4e052..a651530c3 100644 --- a/lib/pure/uri.nim +++ b/lib/pure/uri.nim @@ -47,6 +47,49 @@ proc add*(url: var Url, a: Url) {.deprecated.} = url = url / a {.pop.} +proc encodeUrl*(s: string): string = + ## Encodes a value to be HTTP safe: This means that characters in the set + ## ``{'A'..'Z', 'a'..'z', '0'..'9', '_'}`` are carried over to the result, + ## a space is converted to ``'+'`` and every other character is encoded as + ## ``'%xx'`` where ``xx`` denotes its hexadecimal value. + result = newStringOfCap(s.len + s.len shr 2) # assume 12% non-alnum-chars + for i in 0..s.len-1: + case s[i] + of 'a'..'z', 'A'..'Z', '0'..'9', '_': add(result, s[i]) + of ' ': add(result, '+') + else: + add(result, '%') + add(result, toHex(ord(s[i]), 2)) + +proc decodeUrl*(s: string): string = + ## Decodes a value from its HTTP representation: This means that a ``'+'`` + ## is converted to a space, ``'%xx'`` (where ``xx`` denotes a hexadecimal + ## value) is converted to the character with ordinal number ``xx``, and + ## and every other character is carried over. + proc handleHexChar(c: char, x: var int) {.inline.} = + case c + of '0'..'9': x = (x shl 4) or (ord(c) - ord('0')) + of 'a'..'f': x = (x shl 4) or (ord(c) - ord('a') + 10) + of 'A'..'F': x = (x shl 4) or (ord(c) - ord('A') + 10) + else: assert(false) + + result = newString(s.len) + var i = 0 + var j = 0 + while i < s.len: + case s[i] + of '%': + var x = 0 + handleHexChar(s[i+1], x) + handleHexChar(s[i+2], x) + inc(i, 2) + result[j] = chr(x) + of '+': result[j] = ' ' + else: result[j] = s[i] + inc(i) + inc(j) + setLen(result, j) + proc parseAuthority(authority: string, result: var Uri) = var i = 0 var inPort = false @@ -250,7 +293,7 @@ proc combine*(base: Uri, reference: Uri): Uri = proc combine*(uris: varargs[Uri]): Uri = ## Combines multiple URIs together. result = uris[0] - for i in 1 .. <uris.len: + for i in 1 ..< uris.len: result = combine(result, uris[i]) proc isAbsolute*(uri: Uri): bool = @@ -308,11 +351,16 @@ proc `$`*(u: Uri): string = result.add(":") result.add(u.password) result.add("@") - result.add(u.hostname) + if u.hostname.endswith('/'): + result.add(u.hostname[0..^2]) + else: + result.add(u.hostname) if u.port.len > 0: result.add(":") result.add(u.port) if u.path.len > 0: + if u.hostname.len > 0 and u.path[0] != '/': + result.add('/') result.add(u.path) if u.query.len > 0: result.add("?") @@ -323,6 +371,11 @@ proc `$`*(u: Uri): string = when isMainModule: block: + const test1 = "abc\L+def xyz" + doAssert encodeUrl(test1) == "abc%0A%2Bdef+xyz" + doAssert decodeUrl(encodeUrl(test1)) == test1 + + block: let str = "http://localhost" let test = parseUri(str) doAssert test.path == "" @@ -483,6 +536,34 @@ when isMainModule: let foo = parseUri("http://localhost:9515") / "status" doAssert $foo == "http://localhost:9515/status" + # bug #6649 #6652 + block: + var foo = parseUri("http://example.com") + foo.hostname = "example.com" + foo.path = "baz" + doAssert $foo == "http://example.com/baz" + + foo.hostname = "example.com/" + foo.path = "baz" + doAssert $foo == "http://example.com/baz" + + foo.hostname = "example.com" + foo.path = "/baz" + doAssert $foo == "http://example.com/baz" + + foo.hostname = "example.com/" + foo.path = "/baz" + doAssert $foo == "http://example.com/baz" + + foo.hostname = "example.com/" + foo.port = "8000" + foo.path = "baz" + doAssert $foo == "http://example.com:8000/baz" + + foo = parseUri("file:/dir/file") + foo.path = "relative" + doAssert $foo == "file:relative" + # isAbsolute tests block: doAssert "www.google.com".parseUri().isAbsolute() == false @@ -524,4 +605,4 @@ when isMainModule: doAssert "https://example.com/about/staff.html?".parseUri().isAbsolute == true doAssert "https://example.com/about/staff.html?parameters".parseUri().isAbsolute == true - echo("All good!") \ No newline at end of file + echo("All good!") |