diff options
Diffstat (limited to 'lib/pure')
98 files changed, 9583 insertions, 5176 deletions
diff --git a/lib/pure/algorithm.nim b/lib/pure/algorithm.nim index fdf2d7cbb..81badfae6 100644 --- a/lib/pure/algorithm.nim +++ b/lib/pure/algorithm.nim @@ -13,9 +13,6 @@ type SortOrder* = enum ## sort order Descending, Ascending -{.deprecated: [TSortOrder: SortOrder].} - - proc `*`*(x: int, order: SortOrder): int {.inline.} = ## flips `x` if ``order == Descending``; ## if ``order == Ascending`` then `x` is returned. @@ -24,16 +21,20 @@ proc `*`*(x: int, order: SortOrder): int {.inline.} = var y = order.ord - 1 result = (x xor y) - y -proc fill*[T](a: var openArray[T], first, last: Natural, value: T) = - ## fills the array ``a[first..last]`` with `value`. +template fillImpl[T](a: var openArray[T], first, last: int, value: T) = var x = first while x <= last: a[x] = value inc(x) +proc fill*[T](a: var openArray[T], first, last: Natural, value: T) = + ## fills the array ``a[first..last]`` with `value`. + fillImpl(a, first, last, value) + proc fill*[T](a: var openArray[T], value: T) = ## fills the array `a` with `value`. - fill(a, 0, a.high, value) + fillImpl(a, 0, a.high, value) + proc reverse*[T](a: var openArray[T], first, last: Natural) = ## reverses the array ``a[first..last]``. @@ -63,36 +64,72 @@ proc reversed*[T](a: openArray[T]): seq[T] = ## returns the reverse of the array `a`. reversed(a, 0, a.high) +proc binarySearch*[T, K](a: openArray[T], key: K, + cmp: proc (x: T, y: K): int {.closure.}): int = + ## binary search for `key` in `a`. Returns -1 if not found. + ## + ## `cmp` is the comparator function to use, the expected return values are + ## the same as that of system.cmp. + if a.len == 0: + return -1 + + let len = a.len + + if len == 1: + if cmp(a[0], key) == 0: + return 0 + else: + return -1 + + if (len and (len - 1)) == 0: + # when `len` is a power of 2, a faster shr can be used. + var step = len shr 1 + var cmpRes: int + while step > 0: + let i = result or step + cmpRes = cmp(a[i], key) + if cmpRes == 0: + return i + + if cmpRes < 1: + result = i + step = step shr 1 + if cmp(a[result], key) != 0: result = -1 + else: + var b = len + var cmpRes: int + while result < b: + var mid = (result + b) shr 1 + cmpRes = cmp(a[mid], key) + if cmpRes == 0: + return mid + + if cmpRes < 0: + result = mid + 1 + else: + b = mid + if result >= len or cmp(a[result], key) != 0: result = -1 + proc binarySearch*[T](a: openArray[T], key: T): int = ## binary search for `key` in `a`. Returns -1 if not found. - var b = len(a) - while result < b: - var mid = (result + b) div 2 - if a[mid] < key: result = mid + 1 - else: b = mid - if result >= len(a) or a[result] != key: result = -1 - -proc smartBinarySearch*[T](a: openArray[T], key: T): int = - ## ``a.len`` must be a power of 2 for this to work. - var step = a.len div 2 - while step > 0: - if a[result or step] <= key: - result = result or step - step = step shr 1 - if a[result] != key: result = -1 + binarySearch(a, key, cmp[T]) + +proc smartBinarySearch*[T](a: openArray[T], key: T): int {.deprecated.} = + ## **Deprecated since version 0.18.1**; Use ``binarySearch`` instead. + binarySearch(a, key, cmp[T]) const onlySafeCode = true -proc lowerBound*[T](a: openArray[T], key: T, cmp: proc(x,y: T): int {.closure.}): int = - ## same as binarySearch except that if key is not in `a` then this - ## returns the location where `key` would be if it were. In other - ## words if you have a sorted sequence and you call +proc lowerBound*[T, K](a: openArray[T], key: K, cmp: proc(x: T, k: K): int {.closure.}): int = + ## Returns a position to the first element in the `a` that is greater than `key`, or last + ## if no such element is found. In other words if you have a sorted sequence and you call ## insert(thing, elm, lowerBound(thing, elm)) ## the sequence will still be sorted. ## - ## `cmp` is the comparator function to use, the expected return values are + ## The first version uses `cmp` to compare the elements. The expected return values are ## the same as that of system.cmp. + ## The second version uses the default comparison function `cmp`. ## ## example:: ## @@ -103,7 +140,7 @@ proc lowerBound*[T](a: openArray[T], key: T, cmp: proc(x,y: T): int {.closure.}) var count = a.high - a.low + 1 var step, pos: int while count != 0: - step = count div 2 + step = count shr 1 pos = result + step if cmp(a[pos], key) < 0: result = pos + 1 @@ -113,6 +150,36 @@ proc lowerBound*[T](a: openArray[T], key: T, cmp: proc(x,y: T): int {.closure.}) proc lowerBound*[T](a: openArray[T], key: T): int = lowerBound(a, key, cmp[T]) +proc upperBound*[T, K](a: openArray[T], key: K, cmp: proc(x: T, k: K): int {.closure.}): int = + ## Returns a position to the first element in the `a` that is not less + ## (i.e. greater or equal to) than `key`, or last if no such element is found. + ## In other words if you have a sorted sequence and you call + ## insert(thing, elm, upperBound(thing, elm)) + ## the sequence will still be sorted. + ## + ## The first version uses `cmp` to compare the elements. The expected return values are + ## the same as that of system.cmp. + ## The second version uses the default comparison function `cmp`. + ## + ## example:: + ## + ## var arr = @[1,2,3,4,6,7,8,9] + ## arr.insert(5, arr.upperBound(4)) + ## # after running the above arr is `[1,2,3,4,5,6,7,8,9]` + result = a.low + var count = a.high - a.low + 1 + var step, pos: int + while count != 0: + step = count shr 1 + pos = result + step + if cmp(a[pos], key) <= 0: + result = pos + 1 + count -= step + 1 + else: + count = step + +proc upperBound*[T](a: openArray[T], key: T): int = upperBound(a, key, cmp[T]) + template `<-` (a, b) = when false: a = b @@ -359,10 +426,11 @@ when isMainModule: var srt1 = [1,2,3,4,4,4,4,5] var srt2 = ["iello","hello"] var srt3 = [1.0,1.0,1.0] - var srt4: seq[int] = @[] + var srt4: seq[int] assert srt1.isSorted(cmp) == true assert srt2.isSorted(cmp) == false assert srt3.isSorted(cmp) == true + assert srt4.isSorted(cmp) == true var srtseq = newSeq[int]() assert srtseq.isSorted(cmp) == true # Tests for reversed @@ -391,7 +459,7 @@ proc rotateInternal[T](arg: var openarray[T]; first, middle, last: int): int = swap(arg[mFirst], arg[next]) mFirst += 1 - next += 1 + next += 1 if mFirst == mMiddle: mMiddle = next @@ -443,7 +511,7 @@ proc rotateLeft*[T](arg: var openarray[T]; slice: HSlice[int, int]; dist: int): ## the distance in amount of elements that the data should be rotated. Can be negative, can be any number. ## ## .. code-block:: nim - ## var list = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + ## var list = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] ## list.rotateLeft(1 .. 8, 3) ## doAssert list == [0, 4, 5, 6, 7, 8, 1, 2, 3, 9, 10] let sliceLen = slice.b + 1 - slice.a @@ -472,12 +540,12 @@ proc rotatedLeft*[T](arg: openarray[T]; dist: int): seq[T] = arg.rotatedInternal(0, distLeft, arg.len) when isMainModule: - var list = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] - let list2 = list.rotatedLeft(1 ..< 9, 3) + var list = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + let list2 = list.rotatedLeft(1 ..< 9, 3) let expected = [0, 4, 5, 6, 7, 8, 1, 2, 3, 9, 10] doAssert list.rotateLeft(1 ..< 9, 3) == 6 - doAssert list == expected + doAssert list == expected doAssert list2 == @expected var s0,s1,s2,s3,s4,s5 = "xxxabcdefgxxx" @@ -494,3 +562,45 @@ when isMainModule: doAssert s4 == "xxxefgabcdxxx" doAssert s5.rotateLeft(3 ..< 10, 11) == 6 doAssert s5 == "xxxefgabcdxxx" + + block product: + doAssert product(newSeq[seq[int]]()) == newSeq[seq[int]](), "empty input" + doAssert product(@[newSeq[int](), @[], @[]]) == newSeq[seq[int]](), "bit more empty input" + doAssert product(@[@[1,2]]) == @[@[1,2]], "a simple case of one element" + doAssert product(@[@[1,2], @[3,4]]) == @[@[2,4],@[1,4],@[2,3],@[1,3]], "two elements" + doAssert product(@[@[1,2], @[3,4], @[5,6]]) == @[@[2,4,6],@[1,4,6],@[2,3,6],@[1,3,6], @[2,4,5],@[1,4,5],@[2,3,5],@[1,3,5]], "three elements" + doAssert product(@[@[1,2], @[]]) == newSeq[seq[int]](), "two elements, but one empty" + + block lowerBound: + doAssert lowerBound([1,2,4], 3, system.cmp[int]) == 2 + doAssert lowerBound([1,2,2,3], 4, system.cmp[int]) == 4 + doAssert lowerBound([1,2,3,10], 11) == 4 + + block upperBound: + doAssert upperBound([1,2,4], 3, system.cmp[int]) == 2 + doAssert upperBound([1,2,2,3], 3, system.cmp[int]) == 4 + doAssert upperBound([1,2,3,5], 3) == 3 + + block fillEmptySeq: + var s = newSeq[int]() + s.fill(0) + + block testBinarySearch: + var noData: seq[int] + doAssert binarySearch(noData, 7) == -1 + let oneData = @[1] + doAssert binarySearch(oneData, 1) == 0 + doAssert binarySearch(onedata, 7) == -1 + let someData = @[1,3,4,7] + doAssert binarySearch(someData, 1) == 0 + doAssert binarySearch(somedata, 7) == 3 + doAssert binarySearch(someData, -1) == -1 + doAssert binarySearch(someData, 5) == -1 + doAssert binarySearch(someData, 13) == -1 + let moreData = @[1,3,5,7,4711] + doAssert binarySearch(moreData, -1) == -1 + doAssert binarySearch(moreData, 1) == 0 + doAssert binarySearch(moreData, 5) == 2 + doAssert binarySearch(moreData, 6) == -1 + doAssert binarySearch(moreData, 4711) == 4 + doAssert binarySearch(moreData, 4712) == -1 diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index a52c667fc..820f34703 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -10,6 +10,7 @@ include "system/inclrtl" import os, tables, strutils, times, heapqueue, lists, options, asyncstreams +import options, math import asyncfutures except callSoon import nativesockets, net, deques @@ -157,9 +158,6 @@ export asyncfutures, asyncstreams ## ---------------- ## ## * The effect system (``raises: []``) does not work with async procedures. -## * Can't await in a ``except`` body -## * Forward declarations for async procs are broken, -## link includes workaround: https://github.com/nim-lang/Nim/issues/3182. # TODO: Check if yielded future is nil and throw a more meaningful exception @@ -168,8 +166,10 @@ type timers*: HeapQueue[tuple[finishAt: float, fut: Future[void]]] callbacks*: Deque[proc ()] -proc processTimers(p: PDispatcherBase; didSomeWork: var bool) {.inline.} = - #Process just part if timers at a step +proc processTimers( + p: PDispatcherBase, didSomeWork: var bool +): Option[int] {.inline.} = + # Pop the timers in the order in which they will expire (smaller `finishAt`). var count = p.timers.len let t = epochTime() while count > 0 and t >= p.timers[0].finishAt: @@ -177,22 +177,25 @@ proc processTimers(p: PDispatcherBase; didSomeWork: var bool) {.inline.} = dec count didSomeWork = true + # Return the number of miliseconds in which the next timer will expire. + if p.timers.len == 0: return + + let milisecs = (p.timers[0].finishAt - epochTime()) * 1000 + return some(ceil(milisecs).int) + proc processPendingCallbacks(p: PDispatcherBase; didSomeWork: var bool) = while p.callbacks.len > 0: var cb = p.callbacks.popFirst() cb() didSomeWork = true -proc adjustedTimeout(p: PDispatcherBase, timeout: int): int {.inline.} = - # If dispatcher has active timers this proc returns the timeout - # of the nearest timer. Returns `timeout` otherwise. - result = timeout - if p.timers.len > 0: - let timerTimeout = p.timers[0].finishAt - let curTime = epochTime() - if timeout == -1 or (curTime + (timeout / 1000)) > timerTimeout: - result = int((timerTimeout - curTime) * 1000) - if result < 0: result = 0 +proc adjustTimeout(pollTimeout: int, nextTimer: Option[int]): int {.inline.} = + if nextTimer.isNone(): + return pollTimeout + + result = nextTimer.get() + if pollTimeout == -1: return + result = min(pollTimeout, result) proc callSoon(cbproc: proc ()) {.gcsafe.} @@ -299,53 +302,53 @@ when defined(windows) or defined(nimdoc): "No handles or timers registered in dispatcher.") result = false - if p.handles.len != 0: - let at = p.adjustedTimeout(timeout) - var llTimeout = - if at == -1: winlean.INFINITE - else: at.int32 - - var lpNumberOfBytesTransferred: Dword - var lpCompletionKey: ULONG_PTR - var customOverlapped: PCustomOverlapped - let res = getQueuedCompletionStatus(p.ioPort, - addr lpNumberOfBytesTransferred, addr lpCompletionKey, - cast[ptr POVERLAPPED](addr customOverlapped), llTimeout).bool - result = true - - # http://stackoverflow.com/a/12277264/492186 - # TODO: http://www.serverframework.com/handling-multiple-pending-socket-read-and-write-operations.html - if res: - # This is useful for ensuring the reliability of the overlapped struct. + let nextTimer = processTimers(p, result) + let at = adjustTimeout(timeout, nextTimer) + var llTimeout = + if at == -1: winlean.INFINITE + else: at.int32 + + var lpNumberOfBytesTransferred: Dword + var lpCompletionKey: ULONG_PTR + var customOverlapped: PCustomOverlapped + let res = getQueuedCompletionStatus(p.ioPort, + addr lpNumberOfBytesTransferred, addr lpCompletionKey, + cast[ptr POVERLAPPED](addr customOverlapped), llTimeout).bool + result = true + + # http://stackoverflow.com/a/12277264/492186 + # TODO: http://www.serverframework.com/handling-multiple-pending-socket-read-and-write-operations.html + if res: + # This is useful for ensuring the reliability of the overlapped struct. + assert customOverlapped.data.fd == lpCompletionKey.AsyncFD + + customOverlapped.data.cb(customOverlapped.data.fd, + lpNumberOfBytesTransferred, OSErrorCode(-1)) + + # If cell.data != nil, then system.protect(rawEnv(cb)) was called, + # so we need to dispose our `cb` environment, because it is not needed + # anymore. + if customOverlapped.data.cell.data != nil: + system.dispose(customOverlapped.data.cell) + + GC_unref(customOverlapped) + else: + let errCode = osLastError() + if customOverlapped != nil: assert customOverlapped.data.fd == lpCompletionKey.AsyncFD - customOverlapped.data.cb(customOverlapped.data.fd, - lpNumberOfBytesTransferred, OSErrorCode(-1)) - - # If cell.data != nil, then system.protect(rawEnv(cb)) was called, - # so we need to dispose our `cb` environment, because it is not needed - # anymore. + lpNumberOfBytesTransferred, errCode) if customOverlapped.data.cell.data != nil: system.dispose(customOverlapped.data.cell) - GC_unref(customOverlapped) else: - let errCode = osLastError() - if customOverlapped != nil: - assert customOverlapped.data.fd == lpCompletionKey.AsyncFD - customOverlapped.data.cb(customOverlapped.data.fd, - lpNumberOfBytesTransferred, errCode) - if customOverlapped.data.cell.data != nil: - system.dispose(customOverlapped.data.cell) - GC_unref(customOverlapped) - else: - if errCode.int32 == WAIT_TIMEOUT: - # Timed out - result = false - else: raiseOSError(errCode) + if errCode.int32 == WAIT_TIMEOUT: + # Timed out + result = false + else: raiseOSError(errCode) # Timer processing. - processTimers(p, result) + discard processTimers(p, result) # Callback queue processing processPendingCallbacks(p, result) @@ -1231,48 +1234,48 @@ else: "No handles or timers registered in dispatcher.") result = false - if not p.selector.isEmpty(): - var keys: array[64, ReadyKey] - var count = p.selector.selectInto(p.adjustedTimeout(timeout), keys) - for i in 0..<count: - var custom = false - let fd = keys[i].fd - let events = keys[i].events - var rLength = 0 # len(data.readList) after callback - var wLength = 0 # len(data.writeList) after callback - - if Event.Read in events or events == {Event.Error}: - processBasicCallbacks(fd, readList) - result = true - - if Event.Write in events or events == {Event.Error}: - processBasicCallbacks(fd, writeList) - result = true - - if Event.User in events: - processBasicCallbacks(fd, readList) + var keys: array[64, ReadyKey] + let nextTimer = processTimers(p, result) + var count = p.selector.selectInto(adjustTimeout(timeout, nextTimer), keys) + for i in 0..<count: + var custom = false + let fd = keys[i].fd + let events = keys[i].events + var rLength = 0 # len(data.readList) after callback + var wLength = 0 # len(data.writeList) after callback + + if Event.Read in events or events == {Event.Error}: + processBasicCallbacks(fd, readList) + result = true + + if Event.Write in events or events == {Event.Error}: + processBasicCallbacks(fd, writeList) + result = true + + if Event.User in events: + processBasicCallbacks(fd, readList) + custom = true + if rLength == 0: + p.selector.unregister(fd) + result = true + + when ioselSupportedPlatform: + if (customSet * events) != {}: custom = true - if rLength == 0: - p.selector.unregister(fd) + processCustomCallbacks(fd) result = true - when ioselSupportedPlatform: - if (customSet * events) != {}: - custom = true - processCustomCallbacks(fd) - result = true - - # because state `data` can be modified in callback we need to update - # descriptor events with currently registered callbacks. - if not custom: - var newEvents: set[Event] = {} - if rLength != -1 and wLength != -1: - if rLength > 0: incl(newEvents, Event.Read) - if wLength > 0: incl(newEvents, Event.Write) - p.selector.updateHandle(SocketHandle(fd), newEvents) + # because state `data` can be modified in callback we need to update + # descriptor events with currently registered callbacks. + if not custom: + var newEvents: set[Event] = {} + if rLength != -1 and wLength != -1: + if rLength > 0: incl(newEvents, Event.Read) + if wLength > 0: incl(newEvents, Event.Write) + p.selector.updateHandle(SocketHandle(fd), newEvents) # Timer processing. - processTimers(p, result) + discard processTimers(p, result) # Callback queue processing processPendingCallbacks(p, result) @@ -1513,7 +1516,7 @@ proc poll*(timeout = 500) = # Common procedures between current and upcoming asyncdispatch include includes.asynccommon -proc sleepAsync*(ms: int): Future[void] = +proc sleepAsync*(ms: int | float): Future[void] = ## Suspends the execution of the current async procedure for the next ## ``ms`` milliseconds. var retFuture = newFuture[void]("sleepAsync") @@ -1533,7 +1536,11 @@ proc withTimeout*[T](fut: Future[T], timeout: int): Future[bool] = var timeoutFuture = sleepAsync(timeout) fut.callback = proc () = - if not retFuture.finished: retFuture.complete(true) + if not retFuture.finished: + if fut.failed: + retFuture.fail(fut.error) + else: + retFuture.complete(true) timeoutFuture.callback = proc () = if not retFuture.finished: retFuture.complete(false) diff --git a/lib/pure/asyncfile.nim b/lib/pure/asyncfile.nim index 97bec2815..37339d3d1 100644 --- a/lib/pure/asyncfile.nim +++ b/lib/pure/asyncfile.nim @@ -78,7 +78,10 @@ proc getFileSize*(f: AsyncFile): int64 = raiseOSError(osLastError()) result = (high shl 32) or low else: + let curPos = lseek(f.fd.cint, 0, SEEK_CUR) result = lseek(f.fd.cint, 0, SEEK_END) + f.offset = lseek(f.fd.cint, curPos, SEEK_SET) + assert(f.offset == curPos) proc newAsyncFile*(fd: AsyncFd): AsyncFile = ## Creates `AsyncFile` with a previously opened file descriptor `fd`. @@ -88,7 +91,7 @@ proc newAsyncFile*(fd: AsyncFd): AsyncFile = proc openAsync*(filename: string, mode = fmRead): AsyncFile = ## Opens a file specified by the path in ``filename`` using - ## the specified ``mode`` asynchronously. + ## the specified FileMode ``mode`` asynchronously. when defined(windows) or defined(nimdoc): let flags = FILE_FLAG_OVERLAPPED or FILE_ATTRIBUTE_NORMAL let desiredAccess = getDesiredAccess(mode) @@ -281,6 +284,7 @@ proc read*(f: AsyncFile, size: int): Future[string] = result = false # We still want this callback to be called. elif res == 0: # EOF + f.offset = lseek(fd.cint, 0, SEEK_CUR) retFuture.complete("") else: readBuffer.setLen(res) diff --git a/lib/pure/asyncftpclient.nim b/lib/pure/asyncftpclient.nim index 019a18f55..3d6a9a015 100644 --- a/lib/pure/asyncftpclient.nim +++ b/lib/pure/asyncftpclient.nim @@ -304,6 +304,7 @@ proc retrFile*(ftp: AsyncFtpClient, file, dest: string, raise newException(ReplyError, "Reply has no file size.") await getFile(ftp, destFile, fileSize, onProgressChanged) + destFile.close() proc doUpload(ftp: AsyncFtpClient, file: File, onProgressChanged: ProgressChangedProc) {.async.} = diff --git a/lib/pure/asyncfutures.nim b/lib/pure/asyncfutures.nim index 6df6527d5..5bf9183ed 100644 --- a/lib/pure/asyncfutures.nim +++ b/lib/pure/asyncfutures.nim @@ -27,8 +27,6 @@ type FutureError* = object of Exception cause*: FutureBase -{.deprecated: [PFutureBase: FutureBase, PFuture: Future].} - when not defined(release): var currentID = 0 @@ -177,7 +175,7 @@ proc fail*[T](future: Future[T], error: ref Exception) = if getStackTrace(error) == "": getStackTrace() else: getStackTrace(error) future.callbacks.call() -proc clearCallbacks(future: FutureBase) = +proc clearCallbacks*(future: FutureBase) = future.callbacks.function = nil future.callbacks.next = nil @@ -221,10 +219,10 @@ proc getHint(entry: StackTraceEntry): string = ## We try to provide some hints about stack trace entries that the user ## may not be familiar with, in particular calls inside the stdlib. result = "" - if entry.procname == "processPendingCallbacks": + if entry.procname == cstring"processPendingCallbacks": if cmpIgnoreStyle(entry.filename, "asyncdispatch.nim") == 0: return "Executes pending callbacks" - elif entry.procname == "poll": + elif entry.procname == cstring"poll": if cmpIgnoreStyle(entry.filename, "asyncdispatch.nim") == 0: return "Processes asynchronous completion events" diff --git a/lib/pure/asynchttpserver.nim b/lib/pure/asynchttpserver.nim index ba1615651..d27c2fb9c 100644 --- a/lib/pure/asynchttpserver.nim +++ b/lib/pure/asynchttpserver.nim @@ -60,9 +60,6 @@ type reusePort: bool maxBody: int ## The maximum content-length that will be read for the body. -{.deprecated: [TRequest: Request, PAsyncHttpServer: AsyncHttpServer, - THttpCode: HttpCode, THttpVersion: HttpVersion].} - proc newAsyncHttpServer*(reuseAddr = true, reusePort = false, maxBody = 8388608): AsyncHttpServer = ## Creates a new ``AsyncHttpServer`` instance. @@ -132,6 +129,20 @@ proc parseProtocol(protocol: string): tuple[orig: string, major, minor: int] = proc sendStatus(client: AsyncSocket, status: string): Future[void] = client.send("HTTP/1.1 " & status & "\c\L\c\L") +proc parseUppercaseMethod(name: string): HttpMethod = + result = + case name + of "GET": HttpGet + of "POST": HttpPost + of "HEAD": HttpHead + of "PUT": HttpPut + of "DELETE": HttpDelete + of "PATCH": HttpPatch + of "OPTIONS": HttpOptions + of "CONNECT": HttpConnect + of "TRACE": HttpTrace + else: raise newException(ValueError, "Invalid HTTP method " & name) + proc processRequest(server: AsyncHttpServer, req: FutureVar[Request], client: AsyncSocket, address: string, lineFut: FutureVar[string], @@ -175,8 +186,7 @@ proc processRequest(server: AsyncHttpServer, req: FutureVar[Request], case i of 0: try: - # TODO: this is likely slow. - request.reqMethod = parseEnum[HttpMethod]("http" & linePart) + request.reqMethod = parseUppercaseMethod(linePart) except ValueError: asyncCheck request.respondError(Http400) return diff --git a/lib/pure/asyncmacro.nim b/lib/pure/asyncmacro.nim index f70714309..9e0893a4d 100644 --- a/lib/pure/asyncmacro.nim +++ b/lib/pure/asyncmacro.nim @@ -11,7 +11,7 @@ ## ************* ## `asyncdispatch` module depends on the `asyncmacro` module to work properly. -import macros, strutils +import macros, strutils, asyncfutures proc skipUntilStmtList(node: NimNode): NimNode {.compileTime.} = # Skips a nest of StmtList's. @@ -26,6 +26,8 @@ proc skipStmtList(node: NimNode): NimNode {.compileTime.} = template createCb(retFutureSym, iteratorNameSym, strName, identName, futureVarCompletions: untyped) = + bind finished + var nameIterVar = iteratorNameSym #{.push stackTrace: off.} proc identName {.closure.} = @@ -60,52 +62,6 @@ template createCb(retFutureSym, iteratorNameSym, identName() #{.pop.} -proc generateExceptionCheck(futSym, - tryStmt, rootReceiver, fromNode: NimNode): NimNode {.compileTime.} = - if tryStmt.kind == nnkNilLit: - result = rootReceiver - else: - var exceptionChecks: seq[tuple[cond, body: NimNode]] = @[] - let errorNode = newDotExpr(futSym, newIdentNode("error")) - for i in 1 ..< tryStmt.len: - let exceptBranch = tryStmt[i] - if exceptBranch[0].kind == nnkStmtList: - exceptionChecks.add((newIdentNode("true"), exceptBranch[0])) - else: - var exceptIdentCount = 0 - var ifCond: NimNode - for i in 0 ..< exceptBranch.len: - let child = exceptBranch[i] - if child.kind == nnkIdent: - let cond = infix(errorNode, "of", child) - if exceptIdentCount == 0: - ifCond = cond - else: - ifCond = infix(ifCond, "or", cond) - else: - break - exceptIdentCount.inc - - expectKind(exceptBranch[exceptIdentCount], nnkStmtList) - exceptionChecks.add((ifCond, exceptBranch[exceptIdentCount])) - # -> -> else: raise futSym.error - exceptionChecks.add((newIdentNode("true"), - newNimNode(nnkRaiseStmt).add(errorNode))) - # Read the future if there is no error. - # -> else: futSym.read - let elseNode = newNimNode(nnkElse, fromNode) - elseNode.add newNimNode(nnkStmtList, fromNode) - elseNode[0].add rootReceiver - - let ifBody = newStmtList() - ifBody.add newCall(newIdentNode("setCurrentException"), errorNode) - ifBody.add newIfStmt(exceptionChecks) - ifBody.add newCall(newIdentNode("setCurrentException"), newNilLit()) - - result = newIfStmt( - (newDotExpr(futSym, newIdentNode("failed")), ifBody) - ) - result.add elseNode template useVar(result: var NimNode, futureVarNode: NimNode, valueReceiver, rootReceiver: untyped, fromNode: NimNode) = @@ -121,8 +77,7 @@ template useVar(result: var NimNode, futureVarNode: NimNode, valueReceiver, result.add newNimNode(nnkYieldStmt, fromNode).add(futureVarNode) # -> future<x>.read valueReceiver = newDotExpr(futureVarNode, newIdentNode("read")) - result.add generateExceptionCheck(futureVarNode, tryStmt, rootReceiver, - fromNode) + result.add rootReceiver template createVar(result: var NimNode, futSymName: string, asyncProc: NimNode, @@ -152,8 +107,8 @@ proc createFutureVarCompletions(futureVarIdents: seq[NimNode], ) proc processBody(node, retFutureSym: NimNode, - subTypeIsVoid: bool, futureVarIdents: seq[NimNode], - tryStmt: NimNode): NimNode {.compileTime.} = + subTypeIsVoid: bool, + futureVarIdents: seq[NimNode]): NimNode {.compileTime.} = #echo(node.treeRepr) result = node case node.kind @@ -171,7 +126,7 @@ proc processBody(node, retFutureSym: NimNode, result.add newCall(newIdentNode("complete"), retFutureSym) else: let x = node[0].processBody(retFutureSym, subTypeIsVoid, - futureVarIdents, tryStmt) + futureVarIdents) if x.kind == nnkYieldStmt: result.add x else: result.add newCall(newIdentNode("complete"), retFutureSym, x) @@ -179,7 +134,7 @@ proc processBody(node, retFutureSym: NimNode, result.add newNimNode(nnkReturnStmt, node).add(newNilLit()) return # Don't process the children of this return stmt of nnkCommand, nnkCall: - if node[0].kind == nnkIdent and node[0].ident == !"await": + if node[0].kind == nnkIdent and node[0].eqIdent("await"): case node[1].kind of nnkIdent, nnkInfix, nnkDotExpr, nnkCall, nnkCommand: # await x @@ -192,7 +147,7 @@ proc processBody(node, retFutureSym: NimNode, else: error("Invalid node kind in 'await', got: " & $node[1].kind) elif node.len > 1 and node[1].kind == nnkCommand and - node[1][0].kind == nnkIdent and node[1][0].ident == !"await": + node[1][0].kind == nnkIdent and node[1][0].eqIdent("await"): # foo await x var newCommand = node result.createVar("future" & $node[0].toStrLit, node[1][1], newCommand[1], @@ -201,16 +156,16 @@ proc processBody(node, retFutureSym: NimNode, of nnkVarSection, nnkLetSection: case node[0][2].kind of nnkCommand: - if node[0][2][0].kind == nnkIdent and node[0][2][0].ident == !"await": + if node[0][2][0].kind == nnkIdent and node[0][2][0].eqIdent("await"): # var x = await y var newVarSection = node # TODO: Should this use copyNimNode? - result.createVar("future" & $node[0][0].ident, node[0][2][1], + result.createVar("future" & node[0][0].strVal, node[0][2][1], newVarSection[0][2], newVarSection, node) else: discard of nnkAsgn: case node[1].kind of nnkCommand: - if node[1][0].ident == !"await": + if node[1][0].eqIdent("await"): # x = await y var newAsgn = node result.createVar("future" & $node[0].toStrLit, node[1][1], newAsgn[1], newAsgn, node) @@ -218,74 +173,22 @@ proc processBody(node, retFutureSym: NimNode, of nnkDiscardStmt: # discard await x if node[0].kind == nnkCommand and node[0][0].kind == nnkIdent and - node[0][0].ident == !"await": + node[0][0].eqIdent("await"): var newDiscard = node result.createVar("futureDiscard_" & $toStrLit(node[0][1]), node[0][1], newDiscard[0], newDiscard, node) - of nnkTryStmt: - # try: await x; except: ... - result = newNimNode(nnkStmtList, node) - template wrapInTry(n, tryBody: untyped) = - var temp = n - n[0] = tryBody - tryBody = temp - - # Transform ``except`` body. - # TODO: Could we perform some ``await`` transformation here to get it - # working in ``except``? - tryBody[1] = processBody(n[1], retFutureSym, subTypeIsVoid, - futureVarIdents, nil) - - proc processForTry(n: NimNode, i: var int, - res: NimNode): bool {.compileTime.} = - ## Transforms the body of the tryStmt. Does not transform the - ## body in ``except``. - ## Returns true if the tryStmt node was transformed into an ifStmt. - result = false - var skipped = n.skipStmtList() - while i < skipped.len: - var processed = processBody(skipped[i], retFutureSym, - subTypeIsVoid, futureVarIdents, n) - - # Check if we transformed the node into an exception check. - # This suggests skipped[i] contains ``await``. - if processed.kind != skipped[i].kind or processed.len != skipped[i].len: - processed = processed.skipUntilStmtList() - expectKind(processed, nnkStmtList) - expectKind(processed[2][1], nnkElse) - i.inc - - if not processForTry(n, i, processed[2][1][0]): - # We need to wrap the nnkElse nodes back into a tryStmt. - # As they are executed if an exception does not happen - # inside the awaited future. - # The following code will wrap the nodes inside the - # original tryStmt. - wrapInTry(n, processed[2][1][0]) - - res.add processed - result = true - else: - res.add skipped[i] - i.inc - var i = 0 - if not processForTry(node, i, result): - # If the tryStmt hasn't been transformed we can just put the body - # back into it. - wrapInTry(node, result) - return else: discard for i in 0 ..< result.len: result[i] = processBody(result[i], retFutureSym, subTypeIsVoid, - futureVarIdents, nil) + futureVarIdents) proc getName(node: NimNode): string {.compileTime.} = case node.kind of nnkPostfix: - return $node[1].ident + return node[1].strVal of nnkIdent: - return $node.ident + return node.strVal of nnkEmpty: return "anonymous" else: @@ -296,7 +199,7 @@ proc getFutureVarIdents(params: NimNode): seq[NimNode] {.compileTime.} = for i in 1 ..< len(params): expectKind(params[i], nnkIdentDefs) if params[i][1].kind == nnkBracketExpr and - ($params[i][1][0].ident).normalize == "futurevar": + params[i][1][0].eqIdent("futurevar"): result.add(params[i][0]) proc isInvalidReturnType(typeName: string): bool = @@ -323,7 +226,7 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = let fut = repr(returnType[0]) verifyReturnType(fut) baseType = returnType[1] - elif returnType.kind in nnkCallKinds and $returnType[0] == "[]": + elif returnType.kind in nnkCallKinds and returnType[0].eqIdent("[]"): let fut = repr(returnType[1]) verifyReturnType(fut) baseType = returnType[2] @@ -333,7 +236,7 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = verifyReturnType(repr(returnType)) let subtypeIsVoid = returnType.kind == nnkEmpty or - (baseType.kind == nnkIdent and returnType[1].ident == !"void") + (baseType.kind == nnkIdent and returnType[1].eqIdent("void")) let futureVarIdents = getFutureVarIdents(prc.params) @@ -348,7 +251,7 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = newVarStmt(retFutureSym, newCall( newNimNode(nnkBracketExpr, prc.body).add( - newIdentNode(!"newFuture"), # TODO: Strange bug here? Remove the `!`. + newIdentNode("newFuture"), subRetType), newLit(prcName)))) # Get type from return type of this proc @@ -360,7 +263,7 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = # -> complete(retFuture, result) var iteratorNameSym = genSym(nskIterator, $prcName & "Iter") var procBody = prc.body.processBody(retFutureSym, subtypeIsVoid, - futureVarIdents, nil) + futureVarIdents) # don't do anything with forward bodies (empty) if procBody.kind != nnkEmpty: procBody.add(createFutureVarCompletions(futureVarIdents, nil)) @@ -424,8 +327,8 @@ macro async*(prc: untyped): untyped = ## Macro which processes async procedures into the appropriate ## iterators and yield statements. if prc.kind == nnkStmtList: + result = newStmtList() for oneProc in prc: - result = newStmtList() result.add asyncSingleProc(oneProc) else: result = asyncSingleProc(prc) @@ -448,30 +351,30 @@ proc stripAwait(node: NimNode): NimNode = case node.kind of nnkCommand, nnkCall: - if node[0].kind == nnkIdent and node[0].ident == !"await": + if node[0].kind == nnkIdent and node[0].eqIdent("await"): node[0] = emptyNoopSym elif node.len > 1 and node[1].kind == nnkCommand and - node[1][0].kind == nnkIdent and node[1][0].ident == !"await": + node[1][0].kind == nnkIdent and node[1][0].eqIdent("await"): # foo await x node[1][0] = emptyNoopSym of nnkVarSection, nnkLetSection: case node[0][2].kind of nnkCommand: - if node[0][2][0].kind == nnkIdent and node[0][2][0].ident == !"await": + if node[0][2][0].kind == nnkIdent and node[0][2][0].eqIdent("await"): # var x = await y node[0][2][0] = emptyNoopSym else: discard of nnkAsgn: case node[1].kind of nnkCommand: - if node[1][0].ident == !"await": + if node[1][0].eqIdent("await"): # x = await y node[1][0] = emptyNoopSym else: discard of nnkDiscardStmt: # discard await x if node[0].kind == nnkCommand and node[0][0].kind == nnkIdent and - node[0][0].ident == !"await": + node[0][0].eqIdent("await"): node[0][0] = emptyNoopSym else: discard @@ -480,9 +383,9 @@ proc stripAwait(node: NimNode): NimNode = proc splitParamType(paramType: NimNode, async: bool): NimNode = result = paramType - if paramType.kind == nnkInfix and $paramType[0].ident in ["|", "or"]: - let firstAsync = "async" in ($paramType[1].ident).normalize - let secondAsync = "async" in ($paramType[2].ident).normalize + if paramType.kind == nnkInfix and paramType[0].strVal in ["|", "or"]: + let firstAsync = "async" in paramType[1].strVal.normalize + let secondAsync = "async" in paramType[2].strVal.normalize if firstAsync: result = paramType[if async: 1 else: 2] diff --git a/lib/pure/asyncnet.nim b/lib/pure/asyncnet.nim index a75f9daac..71a1600dc 100644 --- a/lib/pure/asyncnet.nim +++ b/lib/pure/asyncnet.nim @@ -134,8 +134,6 @@ type protocol: Protocol AsyncSocket* = ref AsyncSocketDesc -{.deprecated: [PAsyncSocket: AsyncSocket].} - proc newAsyncSocket*(fd: AsyncFD, domain: Domain = AF_INET, sockType: SockType = SOCK_STREAM, protocol: Protocol = IPPROTO_TCP, buffered = true): AsyncSocket = @@ -201,7 +199,7 @@ when defineSsl: flags: set[SocketFlag]) {.async.} = let len = bioCtrlPending(socket.bioOut) if len > 0: - var data = newStringOfCap(len) + var data = newString(len) let read = bioRead(socket.bioOut, addr data[0], len) assert read != 0 if read < 0: @@ -495,8 +493,6 @@ proc recvLineInto*(socket: AsyncSocket, resString: FutureVar[string], ## **Warning**: ``recvLineInto`` on unbuffered sockets assumes that the ## protocol uses ``\r\L`` to delimit a new line. assert SocketFlag.Peek notin flags ## TODO: - assert(not resString.mget.isNil(), - "String inside resString future needs to be initialised") result = newFuture[void]("asyncnet.recvLineInto") # TODO: Make the async transformation check for FutureVar params and complete @@ -659,7 +655,7 @@ when defineSsl: proc wrapConnectedSocket*(ctx: SslContext, socket: AsyncSocket, handshake: SslHandshakeType, - hostname: string = nil) = + hostname: string = "") = ## Wraps a connected socket in an SSL context. This function effectively ## turns ``socket`` into an SSL socket. ## ``hostname`` should be specified so that the client knows which hostname @@ -674,7 +670,7 @@ when defineSsl: case handshake of handshakeAsClient: - if not hostname.isNil and not isIpAddress(hostname): + if hostname.len > 0 and not isIpAddress(hostname): # Set the SNI address for this connection. This call can fail if # we're not using TLSv1+. discard SSL_set_tlsext_host_name(socket.sslHandle, hostname) diff --git a/lib/pure/base64.nim b/lib/pure/base64.nim index 4b0d08292..bfb8a1666 100644 --- a/lib/pure/base64.nim +++ b/lib/pure/base64.nim @@ -52,7 +52,7 @@ template encodeInternal(s: typed, lineLen: int, newLine: string): untyped = if numLines > 0: inc(total, (numLines - 1) * newLine.len) result = newString(total) - var + var i = 0 r = 0 currLine = 0 @@ -76,7 +76,7 @@ template encodeInternal(s: typed, lineLen: int, newLine: string): untyped = currLine = 0 if i < s.len-1: - let + let a = ord(s[i]) b = ord(s[i+1]) result[r] = cb64[a shr 2] @@ -130,11 +130,11 @@ proc decode*(s: string): string = # total is an upper bound, as we will skip arbitrary whitespace: result = newString(total) - var + var i = 0 r = 0 while true: - while s[i] in Whitespace: inc(i) + while i < s.len and s[i] in Whitespace: inc(i) if i < s.len-3: let a = s[i].decodeByte diff --git a/lib/pure/cgi.nim b/lib/pure/cgi.nim index 5de6aa487..101146ace 100644 --- a/lib/pure/cgi.nim +++ b/lib/pure/cgi.nim @@ -64,8 +64,6 @@ type methodPost, ## query uses the POST method methodGet ## query uses the GET method -{.deprecated: [TRequestMethod: RequestMethod, ECgi: CgiError].} - proc cgiError*(msg: string) {.noreturn.} = ## raises an ECgi exception with message `msg`. var e: ref CgiError @@ -97,11 +95,10 @@ iterator decodeData*(data: string): tuple[key, value: TaintedString] = var name = "" var value = "" # decode everything in one pass: - while data[i] != '\0': + while i < data.len: setLen(name, 0) # reuse memory - while true: + while i < data.len: case data[i] - of '\0': break of '%': var x = 0 handleHexChar(data[i+1], x) @@ -112,15 +109,16 @@ iterator decodeData*(data: string): tuple[key, value: TaintedString] = of '=', '&': break else: add(name, data[i]) inc(i) - if data[i] != '=': cgiError("'=' expected") + if i >= data.len or data[i] != '=': cgiError("'=' expected") inc(i) # skip '=' setLen(value, 0) # reuse memory - while true: + while i < data.len: case data[i] of '%': var x = 0 - handleHexChar(data[i+1], x) - handleHexChar(data[i+2], x) + if i+2 < data.len: + handleHexChar(data[i+1], x) + handleHexChar(data[i+2], x) inc(i, 2) add(value, chr(x)) of '+': add(value, ' ') @@ -128,19 +126,18 @@ iterator decodeData*(data: string): tuple[key, value: TaintedString] = else: add(value, data[i]) inc(i) yield (name.TaintedString, value.TaintedString) - if data[i] == '&': inc(i) - elif data[i] == '\0': break - else: cgiError("'&' expected") + if i < data.len: + if data[i] == '&': inc(i) + else: cgiError("'&' expected") iterator decodeData*(allowedMethods: set[RequestMethod] = {methodNone, methodPost, methodGet}): tuple[key, value: TaintedString] = ## Reads and decodes CGI data and yields the (name, value) pairs the ## data consists of. If the client does not use a method listed in the ## `allowedMethods` set, an `ECgi` exception is raised. - var data = getEncodedData(allowedMethods) - if not isNil(data): - for key, value in decodeData(data): - yield (key, value) + let data = getEncodedData(allowedMethods) + for key, value in decodeData(data): + yield (key, value) proc readData*(allowedMethods: set[RequestMethod] = {methodNone, methodPost, methodGet}): StringTableRef = diff --git a/lib/pure/collections/critbits.nim b/lib/pure/collections/critbits.nim index 34f5c5470..c94e08098 100644 --- a/lib/pure/collections/critbits.nim +++ b/lib/pure/collections/critbits.nim @@ -74,18 +74,19 @@ proc rawInsert[T](c: var CritBitTree[T], key: string): Node[T] = var newByte = 0 block blockX: while newbyte < key.len: - if it.key[newbyte] != key[newbyte]: - newotherbits = it.key[newbyte].ord xor key[newbyte].ord + let ch = if newbyte < it.key.len: it.key[newbyte] else: '\0' + if ch != key[newbyte]: + newotherbits = ch.ord xor key[newbyte].ord break blockX inc newbyte - if it.key[newbyte] != '\0': + if newbyte < it.key.len: newotherbits = it.key[newbyte].ord else: return it while (newOtherBits and (newOtherBits-1)) != 0: newOtherBits = newOtherBits and (newOtherBits-1) newOtherBits = newOtherBits xor 255 - let ch = it.key[newByte] + let ch = if newByte < it.key.len: it.key[newByte] else: '\0' let dir = (1 + (ord(ch) or newOtherBits)) shr 8 var inner: Node[T] @@ -162,18 +163,20 @@ proc containsOrIncl*(c: var CritBitTree[void], key: string): bool = var n = rawInsert(c, key) result = c.count == oldCount -proc inc*(c: var CritBitTree[int]; key: string) = - ## counts the 'key'. - let oldCount = c.count +proc inc*(c: var CritBitTree[int]; key: string, val: int = 1) = + ## increments `c[key]` by `val`. var n = rawInsert(c, key) - if c.count == oldCount: - # not a new key: - inc n.val + inc n.val, val proc incl*(c: var CritBitTree[void], key: string) = ## includes `key` in `c`. discard rawInsert(c, key) +proc incl*[T](c: var CritBitTree[T], key: string, val: T) = + ## inserts `key` with value `val` into `c`. + var n = rawInsert(c, key) + n.val = val + proc `[]=`*[T](c: var CritBitTree[T], key: string, val: T) = ## puts a (key, value)-pair into `t`. var n = rawInsert(c, key) @@ -321,10 +324,14 @@ proc `$`*[T](c: CritBitTree[T]): string = const avgItemLen = 16 result = newStringOfCap(c.count * avgItemLen) result.add("{") - for key, val in pairs(c): - if result.len > 1: result.add(", ") - result.add($key) - when T isnot void: + when T is void: + for key in keys(c): + if result.len > 1: result.add(", ") + result.addQuoted(key) + else: + for key, val in pairs(c): + if result.len > 1: result.add(", ") + result.addQuoted(key) result.add(": ") result.addQuoted(val) result.add("}") @@ -351,3 +358,37 @@ when isMainModule: assert toSeq(r.items) == @["abc", "definition", "prefix", "xyz"] assert toSeq(r.itemsWithPrefix("de")) == @["definition"] + var c = CritBitTree[int]() + + c.inc("a") + assert c["a"] == 1 + + c.inc("a", 4) + assert c["a"] == 5 + + c.inc("a", -5) + assert c["a"] == 0 + + c.inc("b", 2) + assert c["b"] == 2 + + c.inc("c", 3) + assert c["c"] == 3 + + c.inc("a", 1) + assert c["a"] == 1 + + var cf = CritBitTree[float]() + + cf.incl("a", 1.0) + assert cf["a"] == 1.0 + + cf.incl("b", 2.0) + assert cf["b"] == 2.0 + + cf.incl("c", 3.0) + assert cf["c"] == 3.0 + + assert cf.len == 3 + cf.excl("c") + assert cf.len == 2 diff --git a/lib/pure/collections/deques.nim b/lib/pure/collections/deques.nim index 328308a9b..e8342e208 100644 --- a/lib/pure/collections/deques.nim +++ b/lib/pure/collections/deques.nim @@ -38,7 +38,7 @@ ## Note: For inter thread communication use ## a `Channel <channels.html>`_ instead. -import math +import math, typetraits type Deque*[T] = object @@ -160,16 +160,15 @@ proc peekLast*[T](deq: Deque[T]): T {.inline.} = emptyCheck(deq) result = deq.data[(deq.tail - 1) and deq.mask] -template default[T](t: typedesc[T]): T = - var v: T - v +template destroy(x: untyped) = + reset(x) proc popFirst*[T](deq: var Deque[T]): T {.inline, discardable.} = ## Remove and returns the first element of the `deq`. emptyCheck(deq) dec deq.count result = deq.data[deq.head] - deq.data[deq.head] = default(type(result)) + destroy(deq.data[deq.head]) deq.head = (deq.head + 1) and deq.mask proc popLast*[T](deq: var Deque[T]): T {.inline, discardable.} = @@ -178,7 +177,34 @@ proc popLast*[T](deq: var Deque[T]): T {.inline, discardable.} = dec deq.count deq.tail = (deq.tail - 1) and deq.mask result = deq.data[deq.tail] - deq.data[deq.tail] = default(type(result)) + destroy(deq.data[deq.tail]) + +proc clear*[T](deq: var Deque[T]) {.inline.} = + ## Resets the deque so that it is empty. + for el in mitems(deq): destroy(el) + deq.count = 0 + deq.tail = deq.head + +proc shrink*[T](deq: var Deque[T], fromFirst = 0, fromLast = 0) = + ## Remove `fromFirst` elements from the front of the deque and + ## `fromLast` elements from the back. If the supplied number of + ## elements exceeds the total number of elements in the deque, + ## the deque will remain empty. + ## + ## Any user defined destructors + if fromFirst + fromLast > deq.count: + clear(deq) + return + + for i in 0 ..< fromFirst: + destroy(deq.data[deq.head]) + deq.head = (deq.head + 1) and deq.mask + + for i in 0 ..< fromLast: + destroy(deq.data[deq.tail]) + deq.tail = (deq.tail - 1) and deq.mask + + dec deq.count, fromFirst + fromLast proc `$`*[T](deq: Deque[T]): string = ## Turn a deque into its string representation. @@ -215,6 +241,22 @@ when isMainModule: assert deq.find(6) >= 0 assert deq.find(789) < 0 + block: + var d = initDeque[int](1) + d.addLast 7 + d.addLast 8 + d.addLast 10 + d.addFirst 5 + d.addFirst 2 + d.addFirst 1 + d.addLast 20 + d.shrink(fromLast = 2) + doAssert($d == "[1, 2, 5, 7, 8]") + d.shrink(2, 1) + doAssert($d == "[5, 7]") + d.shrink(2, 2) + doAssert d.len == 0 + for i in -2 .. 10: if i in deq: assert deq.contains(i) and deq.find(i) >= 0 diff --git a/lib/pure/collections/intsets.nim b/lib/pure/collections/intsets.nim index bfecfe447..545958977 100644 --- a/lib/pure/collections/intsets.nim +++ b/lib/pure/collections/intsets.nim @@ -184,7 +184,7 @@ proc missingOrExcl*(s: var IntSet, key: int) : bool = ## `key` is removed from `s` and false is returned. var count = s.elems exclImpl(s, key) - result = count == s.elems + result = count == s.elems proc containsOrIncl*(s: var IntSet, key: int): bool = ## returns true if `s` contains `key`, otherwise `key` is included in `s` @@ -212,7 +212,10 @@ proc initIntSet*: IntSet = #newSeq(result.data, InitIntSetSize) #result.max = InitIntSetSize-1 - result.data = nil + when defined(nimNoNilSeqs): + result.data = @[] + else: + result.data = nil result.max = 0 result.counter = 0 result.head = nil @@ -222,7 +225,10 @@ proc clear*(result: var IntSet) = #setLen(result.data, InitIntSetSize) #for i in 0..InitIntSetSize-1: result.data[i] = nil #result.max = InitIntSetSize-1 - result.data = nil + when defined(nimNoNilSeqs): + result.data = @[] + else: + result.data = nil result.max = 0 result.counter = 0 result.head = nil @@ -234,7 +240,10 @@ proc assign*(dest: var IntSet, src: IntSet) = ## copies `src` to `dest`. `dest` does not need to be initialized by ## `initIntSet`. if src.elems <= src.a.len: - dest.data = nil + when defined(nimNoNilSeqs): + dest.data = @[] + else: + dest.data = nil dest.max = 0 dest.counter = src.counter dest.head = nil @@ -247,11 +256,9 @@ proc assign*(dest: var IntSet, src: IntSet) = var it = src.head while it != nil: - var h = it.key and dest.max while dest.data[h] != nil: h = nextTry(h, dest.max) assert(dest.data[h] == nil) - var n: PTrunk new(n) n.next = dest.head @@ -259,7 +266,6 @@ proc assign*(dest: var IntSet, src: IntSet) = n.bits = it.bits dest.head = n dest.data[h] = n - it = it.next proc union*(s1, s2: IntSet): IntSet = @@ -315,7 +321,7 @@ proc len*(s: IntSet): int {.inline.} = for _ in s: inc(result) -proc card*(s: IntSet): int {.inline.} = +proc card*(s: IntSet): int {.inline.} = ## alias for `len() <#len>` _. result = s.len() @@ -361,7 +367,7 @@ when isMainModule: x.incl(1056) x.incl(1044) - x.excl(1044) + x.excl(1044) assert x.containsOrIncl(888) == false assert 888 in x diff --git a/lib/pure/collections/sequtils.nim b/lib/pure/collections/sequtils.nim index 511020228..63d910a8e 100644 --- a/lib/pure/collections/sequtils.nim +++ b/lib/pure/collections/sequtils.nim @@ -25,6 +25,28 @@ import macros when not defined(nimhygiene): {.pragma: dirty.} + +macro evalOnceAs(expAlias, exp: untyped, letAssigneable: static[bool]): untyped = + ## Injects ``expAlias`` in caller scope, to avoid bugs involving multiple + ## substitution in macro arguments such as + ## https://github.com/nim-lang/Nim/issues/7187 + ## ``evalOnceAs(myAlias, myExp)`` will behave as ``let myAlias = myExp`` + ## except when ``letAssigneable`` is false (eg to handle openArray) where + ## it just forwards ``exp`` unchanged + expectKind(expAlias, nnkIdent) + var val = exp + + result = newStmtList() + # If `exp` is not a symbol we evaluate it once here and then use the temporary + # symbol as alias + if exp.kind != nnkSym and letAssigneable: + val = genSym() + result.add(newLetStmt(val, exp)) + + result.add( + newProc(name = genSym(nskTemplate, $expAlias), params = [getType(untyped)], + body = val, procType = nnkTemplateDef)) + proc concat*[T](seqs: varargs[seq[T]]): seq[T] = ## Takes several sequences' items and returns them inside a new sequence. ## @@ -161,7 +183,6 @@ proc distribute*[T](s: seq[T], num: Positive, spread = true): seq[seq[T]] = ## assert numbers.distribute(3, false) == @[@[1, 2, 3], @[4, 5, 6], @[7]] ## assert numbers.distribute(6)[0] == @[1, 2] ## assert numbers.distribute(6)[5] == @[7] - assert(not s.isNil, "`s` can't be nil") if num < 2: result = @[s] return @@ -496,13 +517,17 @@ template toSeq*(iter: untyped): untyped = ## result = true) ## assert odd_numbers == @[1, 3, 5, 7, 9] + # Note: see also `mapIt` for explanation of some of the implementation + # subtleties. when compiles(iter.len): - var i = 0 - var result = newSeq[type(iter)](iter.len) - for x in iter: - result[i] = x - inc i - result + block: + evalOnceAs(iter2, iter, true) + var result = newSeq[type(iter)](iter2.len) + var i = 0 + for x in iter2: + result[i] = x + inc i + result else: var result: seq[type(iter)] = @[] for x in iter: @@ -635,8 +660,7 @@ template mapIt*(s, typ, op: untyped): untyped = result.add(op) result - -template mapIt*(s, op: untyped): untyped = +template mapIt*(s: typed, op: untyped): untyped = ## Convenience template around the ``map`` proc to reduce typing. ## ## The template injects the ``it`` variable which you can use directly in an @@ -653,19 +677,24 @@ template mapIt*(s, op: untyped): untyped = block: var it{.inject.}: type(items(s)); op)) - var result: seq[outType] when compiles(s.len): - let t = s - var i = 0 - result = newSeq[outType](s.len) - for it {.inject.} in t: - result[i] = op - i += 1 + block: # using a block avoids https://github.com/nim-lang/Nim/issues/8580 + + # BUG: `evalOnceAs(s2, s, false)` would lead to C compile errors + # (`error: use of undeclared identifier`) instead of Nim compile errors + evalOnceAs(s2, s, compiles((let _ = s))) + + var i = 0 + var result = newSeq[outType](s2.len) + for it {.inject.} in s2: + result[i] = op + i += 1 + result else: - result = @[] + var result: seq[outType] = @[] for it {.inject.} in s: result.add(op) - result + result template applyIt*(varSeq, op: untyped) = ## Convenience template around the mutable ``apply`` proc to reduce typing. @@ -752,6 +781,14 @@ macro mapLiterals*(constructor, op: untyped; when isMainModule: import strutils + + # helper for testing double substitution side effects which are handled + # by `evalOnceAs` + var counter = 0 + proc identity[T](a:T):auto= + counter.inc + a + block: # concat test let s1 = @[1, 2, 3] @@ -854,20 +891,20 @@ when isMainModule: doAssert numbers.distribute(6)[0] == @[1, 2] doAssert numbers.distribute(6)[5] == @[7] let a = @[1, 2, 3, 4, 5, 6, 7] - doAssert a.distribute(1, true) == @[@[1, 2, 3, 4, 5, 6, 7]] - doAssert a.distribute(1, false) == @[@[1, 2, 3, 4, 5, 6, 7]] - doAssert a.distribute(2, true) == @[@[1, 2, 3, 4], @[5, 6, 7]] - doAssert a.distribute(2, false) == @[@[1, 2, 3, 4], @[5, 6, 7]] - doAssert a.distribute(3, true) == @[@[1, 2, 3], @[4, 5], @[6, 7]] - doAssert a.distribute(3, false) == @[@[1, 2, 3], @[4, 5, 6], @[7]] - doAssert a.distribute(4, true) == @[@[1, 2], @[3, 4], @[5, 6], @[7]] - doAssert a.distribute(4, false) == @[@[1, 2], @[3, 4], @[5, 6], @[7]] - doAssert a.distribute(5, true) == @[@[1, 2], @[3, 4], @[5], @[6], @[7]] - doAssert a.distribute(5, false) == @[@[1, 2], @[3, 4], @[5, 6], @[7], @[]] - doAssert a.distribute(6, true) == @[@[1, 2], @[3], @[4], @[5], @[6], @[7]] - doAssert a.distribute(6, false) == @[ + doAssert a.distribute(1, true) == @[@[1, 2, 3, 4, 5, 6, 7]] + doAssert a.distribute(1, false) == @[@[1, 2, 3, 4, 5, 6, 7]] + doAssert a.distribute(2, true) == @[@[1, 2, 3, 4], @[5, 6, 7]] + doAssert a.distribute(2, false) == @[@[1, 2, 3, 4], @[5, 6, 7]] + doAssert a.distribute(3, true) == @[@[1, 2, 3], @[4, 5], @[6, 7]] + doAssert a.distribute(3, false) == @[@[1, 2, 3], @[4, 5, 6], @[7]] + doAssert a.distribute(4, true) == @[@[1, 2], @[3, 4], @[5, 6], @[7]] + doAssert a.distribute(4, false) == @[@[1, 2], @[3, 4], @[5, 6], @[7]] + doAssert a.distribute(5, true) == @[@[1, 2], @[3, 4], @[5], @[6], @[7]] + doAssert a.distribute(5, false) == @[@[1, 2], @[3, 4], @[5, 6], @[7], @[]] + doAssert a.distribute(6, true) == @[@[1, 2], @[3], @[4], @[5], @[6], @[7]] + doAssert a.distribute(6, false) == @[ @[1, 2], @[3, 4], @[5, 6], @[7], @[], @[]] - doAssert a.distribute(8, false) == a.distribute(8, true) + doAssert a.distribute(8, false) == a.distribute(8, true) doAssert a.distribute(90, false) == a.distribute(90, true) var b = @[0] for f in 1 .. 25: b.add(f) @@ -997,6 +1034,12 @@ when isMainModule: result = true) assert odd_numbers == @[1, 3, 5, 7, 9] + block: + # tests https://github.com/nim-lang/Nim/issues/7187 + counter = 0 + let ret = toSeq(@[1, 2, 3].identity().filter(proc (x: int): bool = x < 3)) + doAssert ret == @[1, 2] + doAssert counter == 1 block: # foldl tests let numbers = @[5, 9, 11] @@ -1023,10 +1066,12 @@ when isMainModule: assert multiplication == 495, "Multiplication is (5*(9*(11)))" assert concatenation == "nimiscool" - block: # mapIt tests + block: # mapIt + applyIt test + counter = 0 var nums = @[1, 2, 3, 4] - strings = nums.mapIt($(4 * it)) + strings = nums.identity.mapIt($(4 * it)) + doAssert counter == 1 nums.applyIt(it * 3) assert nums[0] + nums[3] == 15 assert strings[2] == "12" @@ -1044,5 +1089,51 @@ when isMainModule: doAssert mapLiterals((1, ("abc"), 2), float, nested=false) == (float(1), "abc", float(2)) doAssert mapLiterals(([1], ("abc"), 2), `$`, nested=true) == (["1"], "abc", "2") + block: # mapIt with openArray + counter = 0 + proc foo(x: openArray[int]): seq[int] = x.mapIt(it * 10) + doAssert foo([identity(1),identity(2)]) == @[10, 20] + doAssert counter == 2 + + block: # mapIt with direct openArray + proc foo1(x: openArray[int]): seq[int] = x.mapIt(it * 10) + counter = 0 + doAssert foo1(openArray[int]([identity(1),identity(2)])) == @[10,20] + doAssert counter == 2 + + # Corner cases (openArray litterals should not be common) + template foo2(x: openArray[int]): seq[int] = x.mapIt(it * 10) + counter = 0 + doAssert foo2(openArray[int]([identity(1),identity(2)])) == @[10,20] + # TODO: this fails; not sure how to fix this case + # doAssert counter == 2 + + counter = 0 + doAssert openArray[int]([identity(1), identity(2)]).mapIt(it) == @[1,2] + # ditto + # doAssert counter == 2 + + block: # mapIt empty test, see https://github.com/nim-lang/Nim/pull/8584#pullrequestreview-144723468 + # NOTE: `[].mapIt(it)` is illegal, just as `let a = @[]` is (lacks type + # of elements) + doAssert: not compiles(mapIt(@[], it)) + doAssert: not compiles(mapIt([], it)) + doAssert newSeq[int](0).mapIt(it) == @[] + + block: # mapIt redifinition check, see https://github.com/nim-lang/Nim/issues/8580 + let s2 = [1,2].mapIt(it) + doAssert s2 == @[1,2] + + block: + counter = 0 + doAssert [1,2].identity().mapIt(it*2).mapIt(it*10) == @[20, 40] + # https://github.com/nim-lang/Nim/issues/7187 test case + doAssert counter == 1 + + block: # mapIt with invalid RHS for `let` (#8566) + type X = enum + A, B + doAssert mapIt(X, $it) == @["A", "B"] + when not defined(testing): echo "Finished doc tests" diff --git a/lib/pure/collections/sets.nim b/lib/pure/collections/sets.nim index 9e9152fc8..31ca56963 100644 --- a/lib/pure/collections/sets.nim +++ b/lib/pure/collections/sets.nim @@ -11,7 +11,7 @@ ## ordered hash set. ## ## Hash sets are different from the `built in set type -## <manual.html#set-type>`_. Sets allow you to store any value that can be +## <manual.html#types-set-type>`_. Sets allow you to store any value that can be ## `hashed <hashes.html>`_ and they don't contain duplicate entries. ## ## **Note**: The data types declared here have *value semantics*: This means @@ -74,7 +74,7 @@ proc isValid*[A](s: HashSet[A]): bool = ## proc savePreferences(options: HashSet[string]) = ## assert options.isValid, "Pass an initialized set!" ## # Do stuff here, may crash in release builds! - result = not s.data.isNil + result = s.data.len > 0 proc len*[A](s: HashSet[A]): int = ## Returns the number of keys in `s`. @@ -120,6 +120,13 @@ iterator items*[A](s: HashSet[A]): A = for h in 0..high(s.data): if isFilled(s.data[h].hcode): yield s.data[h].key +proc hash*[A](s: HashSet[A]): Hash = + ## hashing of HashSet + assert s.isValid, "The set needs to be initialized." + for h in 0..high(s.data): + result = result xor s.data[h].hcode + result = !$result + const growthFactor = 2 @@ -340,6 +347,18 @@ proc excl*[A](s: var HashSet[A], other: HashSet[A]) = assert other.isValid, "The set `other` needs to be initialized." for item in other: discard exclImpl(s, item) +proc pop*[A](s: var HashSet[A]): A = + ## Remove and return an arbitrary element from the set `s`. + ## + ## Raises KeyError if the set `s` is empty. + ## + for h in 0..high(s.data): + if isFilled(s.data[h].hcode): + result = s.data[h].key + excl(s, result) + return result + raise newException(KeyError, "set is empty") + proc containsOrIncl*[A](s: var HashSet[A], key: A): bool = ## Includes `key` in the set `s` and tells if `key` was added to `s`. ## @@ -635,7 +654,7 @@ proc isValid*[A](s: OrderedSet[A]): bool = ## proc saveTarotCards(cards: OrderedSet[int]) = ## assert cards.isValid, "Pass an initialized set!" ## # Do stuff here, may crash in release builds! - result = not s.data.isNil + result = s.data.len > 0 proc len*[A](s: OrderedSet[A]): int {.inline.} = ## Returns the number of keys in `s`. @@ -690,6 +709,13 @@ iterator items*[A](s: OrderedSet[A]): A = forAllOrderedPairs: yield s.data[h].key +proc hash*[A](s: OrderedSet[A]): Hash = + ## hashing of OrderedSet + assert s.isValid, "The set needs to be initialized." + forAllOrderedPairs: + result = result !& s.data[h].hcode + result = !$result + iterator pairs*[A](s: OrderedSet[A]): tuple[a: int, b: A] = assert s.isValid, "The set needs to be initialized" forAllOrderedPairs: diff --git a/lib/pure/collections/sharedtables.nim b/lib/pure/collections/sharedtables.nim index 4f311af87..0292a27a2 100644 --- a/lib/pure/collections/sharedtables.nim +++ b/lib/pure/collections/sharedtables.nim @@ -136,11 +136,13 @@ proc withKey*[A, B](t: var SharedTable[A, B], key: A, ## procedure. ## ## The ``mapper`` takes 3 arguments: - ## #. ``key`` - the current key, if it exists, or the key passed to - ## ``withKey`` otherwise; - ## #. ``val`` - the current value, if the key exists, or default value - ## of the type otherwise; - ## #. ``pairExists`` - ``true`` if the key exists, ``false`` otherwise. + ## + ## 1. ``key`` - the current key, if it exists, or the key passed to + ## ``withKey`` otherwise; + ## 2. ``val`` - the current value, if the key exists, or default value + ## of the type otherwise; + ## 3. ``pairExists`` - ``true`` if the key exists, ``false`` otherwise. + ## ## The ``mapper`` can can modify ``val`` and ``pairExists`` values to change ## the mapping of the key or delete it from the table. ## When adding a value, make sure to set ``pairExists`` to ``true`` along diff --git a/lib/pure/collections/tables.nim b/lib/pure/collections/tables.nim index c97846f92..f85de7546 100644 --- a/lib/pure/collections/tables.nim +++ b/lib/pure/collections/tables.nim @@ -158,6 +158,12 @@ template getOrDefaultImpl(t, key): untyped = var index = rawGet(t, key, hc) if index >= 0: result = t.data[index].val +template getOrDefaultImpl(t, key, default: untyped): untyped = + mixin rawGet + var hc: Hash + var index = rawGet(t, key, hc) + result = if index >= 0: t.data[index].val else: default + proc `[]`*[A, B](t: Table[A, B], key: A): B {.deprecatedGet.} = ## retrieves the value at ``t[key]``. If `key` is not in `t`, the ## ``KeyError`` exception is raised. One can check with ``hasKey`` whether @@ -175,10 +181,18 @@ proc mget*[A, B](t: var Table[A, B], key: A): var B {.deprecated.} = ## instead. get(t, key) -proc getOrDefault*[A, B](t: Table[A, B], key: A): B = getOrDefaultImpl(t, key) +proc getOrDefault*[A, B](t: Table[A, B], key: A): B = + ## retrieves the value at ``t[key]`` iff `key` is in `t`. Otherwise, the + ## default initialization value for type `B` is returned (e.g. 0 for any + ## integer type). + getOrDefaultImpl(t, key) -template withValue*[A, B](t: var Table[A, B], key: A, - value, body: untyped) = +proc getOrDefault*[A, B](t: Table[A, B], key: A, default: B): B = + ## retrieves the value at ``t[key]`` iff `key` is in `t`. Otherwise, `default` + ## is returned. + getOrDefaultImpl(t, key, default) + +template withValue*[A, B](t: var Table[A, B], key: A, value, body: untyped) = ## retrieves the value at ``t[key]``. ## `value` can be modified in the scope of the ``withValue`` call. ## @@ -266,7 +280,7 @@ iterator mvalues*[A, B](t: var Table[A, B]): var B = if isFilled(t.data[h].hcode): yield t.data[h].val proc del*[A, B](t: var Table[A, B], key: A) = - ## deletes `key` from hash table `t`. + ## deletes `key` from hash table `t`. Does nothing if the key does not exist. delImpl() proc take*[A, B](t: var Table[A, B], key: A, val: var B): bool = @@ -325,8 +339,7 @@ proc initTable*[A, B](initialSize=64): Table[A, B] = result.counter = 0 newSeq(result.data, initialSize) -proc toTable*[A, B](pairs: openArray[(A, - B)]): Table[A, B] = +proc toTable*[A, B](pairs: openArray[(A, B)]): Table[A, B] = ## creates a new hash table that contains the given `pairs`. result = initTable[A, B](rightSize(pairs.len)) for key, val in items(pairs): result[key] = val @@ -410,7 +423,16 @@ proc mget*[A, B](t: TableRef[A, B], key: A): var B {.deprecated.} = ## Use ```[]``` instead. t[][key] -proc getOrDefault*[A, B](t: TableRef[A, B], key: A): B = getOrDefault(t[], key) +proc getOrDefault*[A, B](t: TableRef[A, B], key: A): B = + ## retrieves the value at ``t[key]`` iff `key` is in `t`. Otherwise, the + ## default initialization value for type `B` is returned (e.g. 0 for any + ## integer type). + getOrDefault(t[], key) + +proc getOrDefault*[A, B](t: TableRef[A, B], key: A, default: B): B = + ## retrieves the value at ``t[key]`` iff `key` is in `t`. Otherwise, `default` + ## is returned. + getOrDefault(t[], key, default) proc mgetOrPut*[A, B](t: TableRef[A, B], key: A, val: B): var B = ## retrieves value at ``t[key]`` or puts ``val`` if not present, either way @@ -435,7 +457,7 @@ proc add*[A, B](t: TableRef[A, B], key: A, val: B) = t[].add(key, val) proc del*[A, B](t: TableRef[A, B], key: A) = - ## deletes `key` from hash table `t`. + ## deletes `key` from hash table `t`. Does nothing if the key does not exist. t[].del(key) proc take*[A, B](t: TableRef[A, B], key: A, val: var B): bool = @@ -562,8 +584,15 @@ proc mget*[A, B](t: var OrderedTable[A, B], key: A): var B {.deprecated.} = get(t, key) proc getOrDefault*[A, B](t: OrderedTable[A, B], key: A): B = + ## retrieves the value at ``t[key]`` iff `key` is in `t`. Otherwise, the + ## default initialization value for type `B` is returned (e.g. 0 for any + ## integer type). getOrDefaultImpl(t, key) +proc getOrDefault*[A, B](t: OrderedTable[A, B], key: A, default: B): B = + ## retrieves the value at ``t[key]`` iff `key` is in `t`. Otherwise, `default` + ## is returned. + getOrDefaultImpl(t, key, default) proc hasKey*[A, B](t: OrderedTable[A, B], key: A): bool = ## returns true iff `key` is in the table `t`. @@ -630,8 +659,7 @@ proc initOrderedTable*[A, B](initialSize=64): OrderedTable[A, B] = result.last = -1 newSeq(result.data, initialSize) -proc toOrderedTable*[A, B](pairs: openArray[(A, - B)]): OrderedTable[A, B] = +proc toOrderedTable*[A, B](pairs: openArray[(A, B)]): OrderedTable[A, B] = ## creates a new ordered hash table that contains the given `pairs`. result = initOrderedTable[A, B](rightSize(pairs.len)) for key, val in items(pairs): result[key] = val @@ -657,8 +685,7 @@ proc `==`*[A, B](s, t: OrderedTable[A, B]): bool = hs = nxts return true -proc sort*[A, B](t: var OrderedTable[A, B], - cmp: proc (x,y: (A, B)): int) = +proc sort*[A, B](t: var OrderedTable[A, B], cmp: proc (x,y: (A, B)): int) = ## sorts `t` according to `cmp`. This modifies the internal list ## that kept the insertion order, so insertion order is lost after this ## call but key lookup and insertions remain possible after `sort` (in @@ -748,8 +775,16 @@ proc mget*[A, B](t: OrderedTableRef[A, B], key: A): var B {.deprecated.} = result = t[][key] proc getOrDefault*[A, B](t: OrderedTableRef[A, B], key: A): B = + ## retrieves the value at ``t[key]`` iff `key` is in `t`. Otherwise, the + ## default initialization value for type `B` is returned (e.g. 0 for any + ## integer type). getOrDefault(t[], key) +proc getOrDefault*[A, B](t: OrderedTableRef[A, B], key: A, default: B): B = + ## retrieves the value at ``t[key]`` iff `key` is in `t`. Otherwise, `default` + ## is returned. + getOrDefault(t[], key, default) + proc mgetOrPut*[A, B](t: OrderedTableRef[A, B], key: A, val: B): var B = ## retrieves value at ``t[key]`` or puts ``val`` if not present, either way ## returning a value which can be modified. @@ -802,8 +837,7 @@ proc `==`*[A, B](s, t: OrderedTableRef[A, B]): bool = elif isNil(t): result = false else: result = s[] == t[] -proc sort*[A, B](t: OrderedTableRef[A, B], - cmp: proc (x,y: (A, B)): int) = +proc sort*[A, B](t: OrderedTableRef[A, B], cmp: proc (x,y: (A, B)): int) = ## sorts `t` according to `cmp`. This modifies the internal list ## that kept the insertion order, so insertion order is lost after this ## call but key lookup and insertions remain possible after `sort` (in @@ -811,7 +845,8 @@ proc sort*[A, B](t: OrderedTableRef[A, B], t[].sort(cmp) proc del*[A, B](t: var OrderedTable[A, B], key: A) = - ## deletes `key` from ordered hash table `t`. O(n) complexity. + ## deletes `key` from ordered hash table `t`. O(n) complexity. Does nothing + ## if the key does not exist. var n: OrderedKeyValuePairSeq[A, B] newSeq(n, len(t.data)) var h = t.first @@ -830,7 +865,8 @@ proc del*[A, B](t: var OrderedTable[A, B], key: A) = h = nxt proc del*[A, B](t: var OrderedTableRef[A, B], key: A) = - ## deletes `key` from ordered hash table `t`. O(n) complexity. + ## deletes `key` from ordered hash table `t`. O(n) complexity. Does nothing + ## if the key does not exist. t[].del(key) # ------------------------------ count tables ------------------------------- @@ -916,9 +952,17 @@ proc mget*[A](t: var CountTable[A], key: A): var int {.deprecated.} = ctget(t, key) proc getOrDefault*[A](t: CountTable[A], key: A): int = + ## retrieves the value at ``t[key]`` iff `key` is in `t`. Otherwise, 0 (the + ## default initialization value of `int`), is returned. var index = rawGet(t, key) if index >= 0: result = t.data[index].val +proc getOrDefault*[A](t: CountTable[A], key: A, default: int): int = + ## retrieves the value at ``t[key]`` iff `key` is in `t`. Otherwise, the + ## integer value of `default` is returned. + var index = rawGet(t, key) + result = if index >= 0: t.data[index].val else: default + proc hasKey*[A](t: CountTable[A], key: A): bool = ## returns true iff `key` is in the table `t`. result = rawGet(t, key) >= 0 @@ -1073,8 +1117,15 @@ proc mget*[A](t: CountTableRef[A], key: A): var int {.deprecated.} = result = t[][key] proc getOrDefault*[A](t: CountTableRef[A], key: A): int = + ## retrieves the value at ``t[key]`` iff `key` is in `t`. Otherwise, 0 (the + ## default initialization value of `int`), is returned. result = t[].getOrDefault(key) +proc getOrDefault*[A](t: CountTableRef[A], key: A, default: int): int = + ## retrieves the value at ``t[key]`` iff `key` is in `t`. Otherwise, the + ## integer value of `default` is returned. + result = t[].getOrDefault(key, default) + proc hasKey*[A](t: CountTableRef[A], key: A): bool = ## returns true iff `key` is in the table `t`. result = t[].hasKey(key) @@ -1267,7 +1318,7 @@ when isMainModule: #lib/pure/collections/tables.nim(117, 21) template/generic instantiation from here #lib/pure/collections/tableimpl.nim(32, 27) Error: undeclared field: 'hcode doAssert 0 == t.getOrDefault(testKey) - t.inc(testKey,3) + t.inc(testKey, 3) doAssert 3 == t.getOrDefault(testKey) block: @@ -1278,7 +1329,7 @@ when isMainModule: doAssert clearTable[42] == "asd" clearTable.clear() doAssert(not clearTable.hasKey(123123)) - doAssert clearTable.getOrDefault(42) == nil + doAssert clearTable.getOrDefault(42) == "" block: #5482 var a = [("wrong?","foo"), ("wrong?", "foo2")].newOrderedTable() @@ -1334,3 +1385,21 @@ when isMainModule: block: # CountTable.smallest let t = toCountTable([0, 0, 5, 5, 5]) doAssert t.smallest == (0, 2) + + block: + var tp: Table[string, string] = initTable[string, string]() + doAssert "test1" == tp.getOrDefault("test1", "test1") + tp["test2"] = "test2" + doAssert "test2" == tp.getOrDefault("test2", "test1") + var tr: TableRef[string, string] = newTable[string, string]() + doAssert "test1" == tr.getOrDefault("test1", "test1") + tr["test2"] = "test2" + doAssert "test2" == tr.getOrDefault("test2", "test1") + var op: OrderedTable[string, string] = initOrderedTable[string, string]() + doAssert "test1" == op.getOrDefault("test1", "test1") + op["test2"] = "test2" + doAssert "test2" == op.getOrDefault("test2", "test1") + var orf: OrderedTableRef[string, string] = newOrderedTable[string, string]() + doAssert "test1" == orf.getOrDefault("test1", "test1") + orf["test2"] = "test2" + doAssert "test2" == orf.getOrDefault("test2", "test1") diff --git a/lib/pure/colors.nim b/lib/pure/colors.nim index 4ec76dee0..843f29a63 100644 --- a/lib/pure/colors.nim +++ b/lib/pure/colors.nim @@ -10,12 +10,11 @@ ## the ``graphics`` module. import strutils +from algorithm import binarySearch type Color* = distinct int ## a color stored as RGB -{.deprecated: [TColor: Color].} - proc `==` *(a, b: Color): bool {.borrow.} ## compares two colors. @@ -373,37 +372,28 @@ proc `$`*(c: Color): string = ## converts a color into its textual representation. Example: ``#00FF00``. result = '#' & toHex(int(c), 6) -proc binaryStrSearch(x: openArray[tuple[name: string, col: Color]], - y: string): int = - var a = 0 - var b = len(x) - 1 - while a <= b: - var mid = (a + b) div 2 - var c = cmp(x[mid].name, y) - if c < 0: a = mid + 1 - elif c > 0: b = mid - 1 - else: return mid - result = - 1 +proc colorNameCmp(x: tuple[name: string, col: Color], y: string): int = + result = cmpIgnoreCase(x.name, y) proc parseColor*(name: string): Color = ## parses `name` to a color value. If no valid color could be - ## parsed ``EInvalidValue`` is raised. + ## parsed ``EInvalidValue`` is raised. Case insensitive. if name[0] == '#': result = Color(parseHexInt(name)) else: - var idx = binaryStrSearch(colorNames, name) + var idx = binarySearch(colorNames, name, colorNameCmp) if idx < 0: raise newException(ValueError, "unknown color: " & name) result = colorNames[idx][1] proc isColor*(name: string): bool = ## returns true if `name` is a known color name or a hexadecimal color - ## prefixed with ``#``. + ## prefixed with ``#``. Case insensitive. if name[0] == '#': for i in 1 .. name.len-1: if name[i] notin {'0'..'9', 'a'..'f', 'A'..'F'}: return false result = true else: - result = binaryStrSearch(colorNames, name) >= 0 + result = binarySearch(colorNames, name, colorNameCmp) >= 0 proc rgb*(r, g, b: range[0..255]): Color = ## constructs a color from RGB values. diff --git a/lib/pure/complex.nim b/lib/pure/complex.nim index ccde5ee0a..ba5c571ce 100644 --- a/lib/pure/complex.nim +++ b/lib/pure/complex.nim @@ -25,7 +25,9 @@ type Complex* = tuple[re, im: float] ## a complex number, consisting of a real and an imaginary part -{.deprecated: [TComplex: Complex].} +const + im*: Complex = (re: 0.0, im: 1.0) + ## The imaginary unit. √-1. proc toComplex*(x: SomeInteger): Complex = ## Convert some integer ``x`` to a complex number. diff --git a/lib/pure/concurrency/cpuinfo.nim b/lib/pure/concurrency/cpuinfo.nim index f01488811..541265da9 100644 --- a/lib/pure/concurrency/cpuinfo.nim +++ b/lib/pure/concurrency/cpuinfo.nim @@ -38,8 +38,18 @@ when defined(macosx) or defined(bsd): importc: "sysctl", nodecl.} when defined(genode): - proc affinitySpaceTotal(): cuint {. - importcpp: "genodeEnv->cpu().affinity_space().total()".} + include genode/env + + proc affinitySpaceTotal(env: GenodeEnvPtr): cuint {. + importcpp: "@->cpu().affinity_space().total()".} + +when defined(haiku): + {.emit: "#include <OS.h>".} + type + SystemInfo {.importc: "system_info", bycopy.} = object + cpuCount {.importc: "cpu_count".}: uint32 + + proc getSystemInfo(info: ptr SystemInfo): int32 {.importc: "get_system_info".} proc countProcessors*(): int {.rtl, extern: "ncpi$1".} = ## returns the numer of the processors/cores the machine has. @@ -83,7 +93,11 @@ proc countProcessors*(): int {.rtl, extern: "ncpi$1".} = var SC_NPROC_ONLN {.importc: "_SC_NPROC_ONLN", header: "<unistd.h>".}: cint result = sysconf(SC_NPROC_ONLN) elif defined(genode): - result = affinitySpaceTotal().int + result = runtimeEnv.affinitySpaceTotal().int + elif defined(haiku): + var sysinfo: SystemInfo + if getSystemInfo(addr sysinfo) == 0: + result = sysinfo.cpuCount.int else: result = sysconf(SC_NPROCESSORS_ONLN) if result <= 0: result = 0 diff --git a/lib/pure/concurrency/threadpool.nim b/lib/pure/concurrency/threadpool.nim index a5eaec86e..f3b13fac5 100644 --- a/lib/pure/concurrency/threadpool.nim +++ b/lib/pure/concurrency/threadpool.nim @@ -30,7 +30,7 @@ proc destroySemaphore(cv: var Semaphore) {.inline.} = deinitCond(cv.c) deinitLock(cv.L) -proc await(cv: var Semaphore) = +proc blockUntil(cv: var Semaphore) = acquire(cv.L) while cv.counter <= 0: wait(cv.c, cv.L) @@ -81,7 +81,7 @@ proc closeBarrier(b: ptr Barrier) {.compilerProc.} = fence() b.interest = true fence() - while b.left != b.entered: await(b.cv) + while b.left != b.entered: blockUntil(b.cv) destroySemaphore(b.cv) {.pop.} @@ -89,8 +89,6 @@ proc closeBarrier(b: ptr Barrier) {.compilerProc.} = # ---------------------------------------------------------------------------- type - foreign* = object ## a region that indicates the pointer comes from a - ## foreign thread heap. AwaitInfo = object cv: Semaphore idx: int @@ -99,7 +97,7 @@ type FlowVarBaseObj = object of RootObj ready, usesSemaphore, awaited: bool cv: Semaphore #\ - # for 'awaitAny' support + # for 'blockUntilAny' support ai: ptr AwaitInfo idx: int data: pointer # we incRef and unref it to keep it alive; note this MUST NOT @@ -130,12 +128,12 @@ type q: ToFreeQueue readyForTask: Semaphore -proc await*(fv: FlowVarBase) = +proc blockUntil*(fv: FlowVarBase) = ## waits until the value for the flowVar arrives. Usually it is not necessary ## to call this explicitly. if fv.usesSemaphore and not fv.awaited: fv.awaited = true - await(fv.cv) + blockUntil(fv.cv) destroySemaphore(fv.cv) proc selectWorker(w: ptr Worker; fn: WorkerProc; data: pointer): bool = @@ -143,7 +141,7 @@ proc selectWorker(w: ptr Worker; fn: WorkerProc; data: pointer): bool = w.data = data w.f = fn signal(w.taskArrived) - await(w.taskStarted) + blockUntil(w.taskStarted) result = true proc cleanFlowVars(w: ptr Worker) = @@ -168,12 +166,21 @@ proc wakeupWorkerToProcessQueue(w: ptr Worker) = signal(w.q.empty) signal(w.taskArrived) +proc attach(fv: FlowVarBase; i: int): bool = + acquire(fv.cv.L) + if fv.cv.counter <= 0: + fv.idx = i + result = true + else: + result = false + release(fv.cv.L) + proc finished(fv: FlowVarBase) = - doAssert fv.ai.isNil, "flowVar is still attached to an 'awaitAny'" + doAssert fv.ai.isNil, "flowVar is still attached to an 'blockUntilAny'" # we have to protect against the rare cases where the owner of the flowVar # simply disregards the flowVar and yet the "flowVar" has not yet written # anything to it: - await(fv) + blockUntil(fv) if fv.data.isNil: return let owner = cast[ptr Worker](fv.owner) let q = addr(owner.q) @@ -182,7 +189,7 @@ proc finished(fv: FlowVarBase) = #echo "EXHAUSTED!" release(q.lock) wakeupWorkerToProcessQueue(owner) - await(q.empty) + blockUntil(q.empty) acquire(q.lock) q.data[q.len] = cast[pointer](fv.data) inc q.len @@ -213,7 +220,7 @@ proc awaitAndThen*[T](fv: FlowVar[T]; action: proc (x: T) {.closure.}) = ## to ``action``. Note that due to Nim's parameter passing semantics this ## means that ``T`` doesn't need to be copied and so ``awaitAndThen`` can ## sometimes be more efficient than ``^``. - await(fv) + blockUntil(fv) when T is string or T is seq: action(cast[T](fv.data)) elif T is ref: @@ -222,49 +229,50 @@ proc awaitAndThen*[T](fv: FlowVar[T]; action: proc (x: T) {.closure.}) = action(fv.blob) finished(fv) -proc unsafeRead*[T](fv: FlowVar[ref T]): foreign ptr T = +proc unsafeRead*[T](fv: FlowVar[ref T]): ptr T = ## blocks until the value is available and then returns this value. - await(fv) - result = cast[foreign ptr T](fv.data) + blockUntil(fv) + result = cast[ptr T](fv.data) proc `^`*[T](fv: FlowVar[ref T]): ref T = ## blocks until the value is available and then returns this value. - await(fv) + blockUntil(fv) let src = cast[ref T](fv.data) deepCopy result, src proc `^`*[T](fv: FlowVar[T]): T = ## blocks until the value is available and then returns this value. - await(fv) + blockUntil(fv) when T is string or T is seq: # XXX closures? deepCopy? result = cast[T](fv.data) else: result = fv.blob -proc awaitAny*(flowVars: openArray[FlowVarBase]): int = +proc blockUntilAny*(flowVars: openArray[FlowVarBase]): int = ## awaits any of the given flowVars. Returns the index of one flowVar for - ## which a value arrived. A flowVar only supports one call to 'awaitAny' at - ## the same time. That means if you await([a,b]) and await([b,c]) the second - ## call will only await 'c'. If there is no flowVar left to be able to wait + ## which a value arrived. A flowVar only supports one call to 'blockUntilAny' at + ## the same time. That means if you blockUntilAny([a,b]) and blockUntilAny([b,c]) the second + ## call will only blockUntil 'c'. If there is no flowVar left to be able to wait ## on, -1 is returned. - ## **Note**: This results in non-deterministic behaviour and so should be - ## avoided. + ## **Note**: This results in non-deterministic behaviour and should be avoided. var ai: AwaitInfo ai.cv.initSemaphore() var conflicts = 0 + result = -1 for i in 0 .. flowVars.high: if cas(addr flowVars[i].ai, nil, addr ai): - flowVars[i].idx = i + if not attach(flowVars[i], i): + result = i + break else: inc conflicts if conflicts < flowVars.len: - await(ai.cv) - result = ai.idx + if result < 0: + blockUntil(ai.cv) + result = ai.idx for i in 0 .. flowVars.high: discard cas(addr flowVars[i].ai, addr ai, nil) - else: - result = -1 destroySemaphore(ai.cv) proc isReady*(fv: FlowVarBase): bool = @@ -318,10 +326,10 @@ proc slave(w: ptr Worker) {.thread.} = w.ready = true readyWorker = w signal(gSomeReady) - await(w.taskArrived) + blockUntil(w.taskArrived) # XXX Somebody needs to look into this (why does this assertion fail # in Visual Studio?) - when not defined(vcc): assert(not w.ready) + when not defined(vcc) and not defined(tcc): assert(not w.ready) withLock numSlavesLock: inc numSlavesRunning @@ -343,7 +351,7 @@ proc distinguishedSlave(w: ptr Worker) {.thread.} = else: w.ready = true signal(w.readyForTask) - await(w.taskArrived) + blockUntil(w.taskArrived) assert(not w.ready) w.f(w, w.data) if w.q.len != 0: w.cleanFlowVars @@ -491,7 +499,7 @@ proc nimSpawn3(fn: WorkerProc; data: pointer) {.compilerProc.} = # on the current thread instead. var self = addr(workersData[localThreadId-1]) fn(self, data) - await(self.taskStarted) + blockUntil(self.taskStarted) return if isSlave: @@ -516,7 +524,7 @@ proc nimSpawn3(fn: WorkerProc; data: pointer) {.compilerProc.} = inc numSlavesWaiting - await(gSomeReady) + blockUntil(gSomeReady) if isSlave: withLock numSlavesLock: @@ -534,7 +542,7 @@ proc nimSpawn4(fn: WorkerProc; data: pointer; id: ThreadId) {.compilerProc.} = release(distinguishedLock) while true: if selectWorker(addr(distinguishedData[id]), fn, data): break - await(distinguishedData[id].readyForTask) + blockUntil(distinguishedData[id].readyForTask) proc sync*() = @@ -547,7 +555,7 @@ proc sync*() = if not allReady: break allReady = allReady and workersData[i].ready if allReady: break - await(gSomeReady) + blockUntil(gSomeReady) inc toRelease for i in 0 ..< toRelease: diff --git a/lib/pure/cookies.nim b/lib/pure/cookies.nim index 8f16717ac..eaff86ae6 100644 --- a/lib/pure/cookies.nim +++ b/lib/pure/cookies.nim @@ -25,16 +25,16 @@ proc parseCookies*(s: string): StringTableRef = result = newStringTable(modeCaseInsensitive) var i = 0 while true: - while s[i] == ' ' or s[i] == '\t': inc(i) + while i < s.len and (s[i] == ' ' or s[i] == '\t'): inc(i) var keystart = i - while s[i] != '=' and s[i] != '\0': inc(i) + while i < s.len and s[i] != '=': inc(i) var keyend = i-1 - if s[i] == '\0': break + if i >= s.len: break inc(i) # skip '=' var valstart = i - while s[i] != ';' and s[i] != '\0': inc(i) + while i < s.len and s[i] != ';': inc(i) result[substr(s, keystart, keyend)] = substr(s, valstart, i-1) - if s[i] == '\0': break + if i >= s.len: break inc(i) # skip ';' proc setCookie*(key, value: string, domain = "", path = "", @@ -51,26 +51,26 @@ proc setCookie*(key, value: string, domain = "", path = "", if secure: result.add("; Secure") if httpOnly: result.add("; HttpOnly") -proc setCookie*(key, value: string, expires: DateTime, +proc setCookie*(key, value: string, expires: DateTime|Time, domain = "", path = "", noName = false, secure = false, httpOnly = false): string = ## Creates a command in the format of ## ``Set-Cookie: key=value; Domain=...; ...`` - ## - ## **Note:** UTC is assumed as the timezone for ``expires``. return setCookie(key, value, domain, path, - format(expires, "ddd',' dd MMM yyyy HH:mm:ss 'GMT'"), + format(expires.utc, "ddd',' dd MMM yyyy HH:mm:ss 'GMT'"), noname, secure, httpOnly) when isMainModule: - var tim = fromUnix(getTime().toUnix + 76 * (60 * 60 * 24)) + let expire = fromUnix(0) + 1.seconds - let cookie = setCookie("test", "value", tim.utc) - when not defined(testing): - echo cookie - let start = "Set-Cookie: test=value; Expires=" - assert cookie[0..start.high] == start + let cookies = [ + setCookie("test", "value", expire), + setCookie("test", "value", expire.local), + setCookie("test", "value", expire.utc) + ] + let expected = "Set-Cookie: test=value; Expires=Thu, 01 Jan 1970 00:00:01 GMT" + doAssert cookies == [expected, expected, expected] let table = parseCookies("uid=1; kp=2") - assert table["uid"] == "1" - assert table["kp"] == "2" + doAssert table["uid"] == "1" + doAssert table["kp"] == "2" diff --git a/lib/pure/coro.nim b/lib/pure/coro.nim index b6ef30e7c..2fe34ed40 100644 --- a/lib/pure/coro.nim +++ b/lib/pure/coro.nim @@ -43,6 +43,10 @@ when defined(windows): {.warning: "ucontext coroutine backend is not available on windows, defaulting to fibers.".} when defined(nimCoroutinesSetjmp): {.warning: "setjmp coroutine backend is not available on windows, defaulting to fibers.".} +elif defined(haiku): + const coroBackend = CORO_BACKEND_SETJMP + when defined(nimCoroutinesUcontext): + {.warning: "ucontext coroutine backend is not available on haiku, defaulting to setjmp".} elif defined(nimCoroutinesSetjmp) or defined(nimCoroutinesSetjmpBundled): const coroBackend = CORO_BACKEND_SETJMP else: @@ -55,21 +59,21 @@ when coroBackend == CORO_BACKEND_FIBERS: elif coroBackend == CORO_BACKEND_UCONTEXT: type - stack_t {.importc, header: "<sys/ucontext.h>".} = object + stack_t {.importc, header: "<ucontext.h>".} = object ss_sp: pointer ss_flags: int ss_size: int - ucontext_t {.importc, header: "<sys/ucontext.h>".} = object + ucontext_t {.importc, header: "<ucontext.h>".} = object uc_link: ptr ucontext_t uc_stack: stack_t Context = ucontext_t - proc getcontext(context: var ucontext_t): int32 {.importc, header: "<sys/ucontext.h>".} - proc setcontext(context: var ucontext_t): int32 {.importc, header: "<sys/ucontext.h>".} - proc swapcontext(fromCtx, toCtx: var ucontext_t): int32 {.importc, header: "<sys/ucontext.h>".} - proc makecontext(context: var ucontext_t, fn: pointer, argc: int32) {.importc, header: "<sys/ucontext.h>", varargs.} + proc getcontext(context: var ucontext_t): int32 {.importc, header: "<ucontext.h>".} + proc setcontext(context: var ucontext_t): int32 {.importc, header: "<ucontext.h>".} + proc swapcontext(fromCtx, toCtx: var ucontext_t): int32 {.importc, header: "<ucontext.h>".} + proc makecontext(context: var ucontext_t, fn: pointer, argc: int32) {.importc, header: "<ucontext.h>", varargs.} elif coroBackend == CORO_BACKEND_SETJMP: proc coroExecWithStack*(fn: pointer, stack: pointer) {.noreturn, importc: "narch_$1", fastcall.} @@ -111,7 +115,12 @@ elif coroBackend == CORO_BACKEND_SETJMP: when defined(unix): # GLibc fails with "*** longjmp causes uninitialized stack frame ***" because # our custom stacks are not initialized to a magic value. - {.passC: "-U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=0"} + when defined(osx): + # workaround: error: The deprecated ucontext routines require _XOPEN_SOURCE to be defined + const extra = " -D_XOPEN_SOURCE" + else: + const extra = "" + {.passC: "-U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=0" & extra.} const CORO_CREATED = 0 @@ -190,7 +199,7 @@ proc switchTo(current, to: CoroutinePtr) = elif to.state == CORO_CREATED: # Coroutine is started. coroExecWithStack(runCurrentTask, to.stack.bottom) - doAssert false + #doAssert false else: {.error: "Invalid coroutine backend set.".} # Execution was just resumed. Restore frame information and set active stack. diff --git a/lib/pure/cstrutils.nim b/lib/pure/cstrutils.nim index 437140892..fe9ceb68b 100644 --- a/lib/pure/cstrutils.nim +++ b/lib/pure/cstrutils.nim @@ -45,8 +45,10 @@ proc endsWith*(s, suffix: cstring): bool {.noSideEffect, proc cmpIgnoreStyle*(a, b: cstring): int {.noSideEffect, rtl, extern: "csuCmpIgnoreStyle".} = - ## Compares two strings normalized (i.e. case and - ## underscores do not matter). Returns: + ## Semantically the same as ``cmp(normalize($a), normalize($b))``. It + ## is just optimized to not allocate temporary strings. This should + ## NOT be used to compare Nim identifier names. use `macros.eqIdent` + ## for that. Returns: ## ## | 0 iff a == b ## | < 0 iff a < b diff --git a/lib/pure/db_common.nim b/lib/pure/db_common.nim index 957389605..f8689024b 100644 --- a/lib/pure/db_common.nim +++ b/lib/pure/db_common.nim @@ -83,9 +83,6 @@ type foreignKey*: bool ## is this a foreign key? DbColumns* = seq[DbColumn] -{.deprecated: [EDb: DbError, TSqlQuery: SqlQuery, FDb: DbEffect, - FReadDb: ReadDbEffect, FWriteDb: WriteDbEffect].} - template sql*(query: string): SqlQuery = ## constructs a SqlQuery from the string `query`. This is supposed to be ## used as a raw-string-literal modifier: diff --git a/lib/pure/distros.nim b/lib/pure/distros.nim index 0adba5b1e..5847cfadb 100644 --- a/lib/pure/distros.nim +++ b/lib/pure/distros.nim @@ -126,6 +126,8 @@ type OpenBSD DragonFlyBSD + Haiku + const LacksDevPackages* = {Distribution.Gentoo, Distribution.Slackware, @@ -166,6 +168,8 @@ proc detectOsImpl(d: Distribution): bool = of Distribution.Solaris: let uname = toLowerAscii(uname()) result = ("sun" in uname) or ("solaris" in uname) + of Distribution.Haiku: + result = defined(haiku) else: let dd = toLowerAscii($d) result = dd in toLowerAscii(uname()) or dd in toLowerAscii(release()) @@ -224,6 +228,8 @@ proc foreignDepInstallCmd*(foreignPackageName: string): (string, bool) = result = ("pacman -S " & p, true) else: result = ("<your package manager here> install " & p, true) + elif defined(haiku): + result = ("pkgman install " & p, true) else: result = ("brew install " & p, false) diff --git a/lib/pure/dynlib.nim b/lib/pure/dynlib.nim index fda41dadb..d030ad532 100644 --- a/lib/pure/dynlib.nim +++ b/lib/pure/dynlib.nim @@ -10,14 +10,55 @@ ## This module implements the ability to access symbols from shared ## libraries. On POSIX this uses the ``dlsym`` mechanism, on ## Windows ``LoadLibrary``. +## +## Examples +## -------- +## +## Loading a simple C function +## ^^^^^^^^^^^^^^^^^^^^^^^^^^^ +## +## The following example demonstrates loading a function called 'greet' +## from a library that is determined at runtime based upon a language choice. +## If the library fails to load or the function 'greet' is not found, +## it quits with a failure error code. +## +## .. code-block::nim +## +## import dynlib +## +## type +## greetFunction = proc(): cstring {.gcsafe, stdcall.} +## +## let lang = stdin.readLine() +## +## let lib = case lang +## of "french": +## loadLib("french.dll") +## else: +## loadLib("english.dll") +## +## if lib == nil: +## echo "Error loading library" +## quit(QuitFailure) +## +## let greet = cast[greetFunction](lib.symAddr("greet")) +## +## if greet == nil: +## echo "Error loading 'greet' function from library" +## quit(QuitFailure) +## +## let greeting = greet() +## +## echo greeting +## +## unloadLib(lib) +## import strutils type LibHandle* = pointer ## a handle to a dynamically loaded library -{.deprecated: [TLibHandle: LibHandle].} - proc loadLib*(path: string, global_symbols=false): LibHandle {.gcsafe.} ## loads a library from `path`. Returns nil if the library could not ## be loaded. @@ -96,6 +137,28 @@ when defined(posix): proc symAddr(lib: LibHandle, name: cstring): pointer = return dlsym(lib, name) +elif defined(nintendoswitch): + # + # ========================================================================= + # Nintendo switch DevkitPro sdk does not have these. Raise an error if called. + # ========================================================================= + # + + proc dlclose(lib: LibHandle) = + raise newException(OSError, "dlclose not implemented on Nintendo Switch!") + proc dlopen(path: cstring, mode: int): LibHandle = + raise newException(OSError, "dlopen not implemented on Nintendo Switch!") + proc dlsym(lib: LibHandle, name: cstring): pointer = + raise newException(OSError, "dlsym not implemented on Nintendo Switch!") + proc loadLib(path: string, global_symbols=false): LibHandle = + raise newException(OSError, "loadLib not implemented on Nintendo Switch!") + proc loadLib(): LibHandle = + raise newException(OSError, "loadLib not implemented on Nintendo Switch!") + proc unloadLib(lib: LibHandle) = + raise newException(OSError, "unloadLib not implemented on Nintendo Switch!") + proc symAddr(lib: LibHandle, name: cstring): pointer = + raise newException(OSError, "symAddr not implemented on Nintendo Switch!") + elif defined(windows) or defined(dos): # # ======================================================================= diff --git a/lib/pure/encodings.nim b/lib/pure/encodings.nim index 5840d443d..2039a31be 100644 --- a/lib/pure/encodings.nim +++ b/lib/pure/encodings.nim @@ -27,8 +27,6 @@ type EncodingError* = object of ValueError ## exception that is raised ## for encoding errors -{.deprecated: [EInvalidEncoding: EncodingError, PConverter: EncodingConverter].} - when defined(windows): proc eqEncodingNames(a, b: string): bool = var i = 0 @@ -36,7 +34,7 @@ when defined(windows): while i < a.len and j < b.len: if a[i] in {'-', '_'}: inc i if b[j] in {'-', '_'}: inc j - if a[i].toLower != b[j].toLower: return false + if i < a.len and j < b.len and a[i].toLowerAscii != b[j].toLowerAscii: return false inc i inc j result = i == a.len and j == b.len @@ -257,7 +255,7 @@ when defined(windows): else: when defined(haiku): - const iconvDll = "(libc.so.6|libiconv.so|libtextencoding.so)" + const iconvDll = "libiconv.so" elif defined(macosx): const iconvDll = "libiconv.dylib" else: @@ -274,6 +272,8 @@ else: const EILSEQ = 86.cint elif defined(solaris): const EILSEQ = 88.cint + elif defined(haiku): + const EILSEQ = -2147454938.cint var errno {.importc, header: "<errno.h>".}: cint @@ -334,7 +334,7 @@ when defined(windows): if s.len == 0: return "" # educated guess of capacity: var cap = s.len + s.len shr 2 - result = newStringOfCap(cap*2) + result = newString(cap*2) # convert to utf-16 LE var m = multiByteToWideChar(codePage = c.src, dwFlags = 0'i32, lpMultiByteStr = cstring(s), @@ -349,7 +349,7 @@ when defined(windows): lpWideCharStr = nil, cchWideChar = cint(0)) # and do the conversion properly: - result = newStringOfCap(cap*2) + result = newString(cap*2) m = multiByteToWideChar(codePage = c.src, dwFlags = 0'i32, lpMultiByteStr = cstring(s), cbMultiByte = cint(s.len), @@ -366,7 +366,7 @@ when defined(windows): if int(c.dest) == 1200: return # otherwise the fun starts again: cap = s.len + s.len shr 2 - var res = newStringOfCap(cap) + var res = newString(cap) m = wideCharToMultiByte( codePage = c.dest, dwFlags = 0'i32, @@ -384,7 +384,7 @@ when defined(windows): lpMultiByteStr = nil, cbMultiByte = cint(0)) # and do the conversion properly: - res = newStringOfCap(cap) + res = newString(cap) m = wideCharToMultiByte( codePage = c.dest, dwFlags = 0'i32, diff --git a/lib/pure/events.nim b/lib/pure/events.nim index 23a8a2c58..a39dbe3bf 100644 --- a/lib/pure/events.nim +++ b/lib/pure/events.nim @@ -43,9 +43,6 @@ type s: seq[EventHandler] EventError* = object of ValueError -{.deprecated: [TEventArgs: EventArgs, TEventHandler: EventHandler, - TEventEmitter: EventEmitter, EInvalidEvent: EventError].} - proc initEventHandler*(name: string): EventHandler = ## Initializes an EventHandler with the specified name and returns it. result.handlers = @[] diff --git a/lib/pure/fenv.nim b/lib/pure/fenv.nim index f8f115ecc..0725973ca 100644 --- a/lib/pure/fenv.nim +++ b/lib/pure/fenv.nim @@ -10,9 +10,9 @@ ## Floating-point environment. Handling of floating-point rounding and ## exceptions (overflow, division by zero, etc.). -{.deadCodeElim:on.} +{.deadCodeElim: on.} # dce option deprecated -when defined(Posix) and not defined(haiku): +when defined(Posix): {.passl: "-lm".} var @@ -102,32 +102,33 @@ proc feupdateenv*(envp: ptr Tfenv): cint {.importc, header: "<fenv.h>".} ## represented by object pointed to by `envp` and raise exceptions ## according to saved exceptions. -var FP_RADIX_INTERNAL {. importc: "FLT_RADIX" header: "<float.h>" .} : int - -template fpRadix* : int = FP_RADIX_INTERNAL +const + FLT_RADIX = 2 ## the radix of the exponent representation + + FLT_MANT_DIG = 24 ## the number of base FLT_RADIX digits in the mantissa part of a float + FLT_DIG = 6 ## the number of digits of precision of a float + FLT_MIN_EXP = -125 # the minimum value of base FLT_RADIX in the exponent part of a float + FLT_MAX_EXP = 128 # the maximum value of base FLT_RADIX in the exponent part of a float + FLT_MIN_10_EXP = -37 ## the minimum value in base 10 of the exponent part of a float + FLT_MAX_10_EXP = 38 ## the maximum value in base 10 of the exponent part of a float + FLT_MIN = 1.17549435e-38'f32 ## the minimum value of a float + FLT_MAX = 3.40282347e+38'f32 ## the maximum value of a float + FLT_EPSILON = 1.19209290e-07'f32 ## the difference between 1 and the least value greater than 1 of a float + + DBL_MANT_DIG = 53 ## the number of base FLT_RADIX digits in the mantissa part of a double + DBL_DIG = 15 ## the number of digits of precision of a double + DBL_MIN_EXP = -1021 ## the minimum value of base FLT_RADIX in the exponent part of a double + DBL_MAX_EXP = 1024 ## the maximum value of base FLT_RADIX in the exponent part of a double + DBL_MIN_10_EXP = -307 ## the minimum value in base 10 of the exponent part of a double + DBL_MAX_10_EXP = 308 ## the maximum value in base 10 of the exponent part of a double + DBL_MIN = 2.2250738585072014E-308 ## the minimal value of a double + DBL_MAX = 1.7976931348623157E+308 ## the minimal value of a double + DBL_EPSILON = 2.2204460492503131E-16 ## the difference between 1 and the least value greater than 1 of a double + +template fpRadix* : int = FLT_RADIX ## The (integer) value of the radix used to represent any floating ## point type on the architecture used to build the program. -var FLT_MANT_DIG {. importc: "FLT_MANT_DIG" header: "<float.h>" .} : int -var FLT_DIG {. importc: "FLT_DIG" header: "<float.h>" .} : int -var FLT_MIN_EXP {. importc: "FLT_MIN_EXP" header: "<float.h>" .} : int -var FLT_MAX_EXP {. importc: "FLT_MAX_EXP" header: "<float.h>" .} : int -var FLT_MIN_10_EXP {. importc: "FLT_MIN_10_EXP" header: "<float.h>" .} : int -var FLT_MAX_10_EXP {. importc: "FLT_MAX_10_EXP" header: "<float.h>" .} : int -var FLT_MIN {. importc: "FLT_MIN" header: "<float.h>" .} : cfloat -var FLT_MAX {. importc: "FLT_MAX" header: "<float.h>" .} : cfloat -var FLT_EPSILON {. importc: "FLT_EPSILON" header: "<float.h>" .} : cfloat - -var DBL_MANT_DIG {. importc: "DBL_MANT_DIG" header: "<float.h>" .} : int -var DBL_DIG {. importc: "DBL_DIG" header: "<float.h>" .} : int -var DBL_MIN_EXP {. importc: "DBL_MIN_EXP" header: "<float.h>" .} : int -var DBL_MAX_EXP {. importc: "DBL_MAX_EXP" header: "<float.h>" .} : int -var DBL_MIN_10_EXP {. importc: "DBL_MIN_10_EXP" header: "<float.h>" .} : int -var DBL_MAX_10_EXP {. importc: "DBL_MAX_10_EXP" header: "<float.h>" .} : int -var DBL_MIN {. importc: "DBL_MIN" header: "<float.h>" .} : cdouble -var DBL_MAX {. importc: "DBL_MAX" header: "<float.h>" .} : cdouble -var DBL_EPSILON {. importc: "DBL_EPSILON" header: "<float.h>" .} : cdouble - template mantissaDigits*(T : typedesc[float32]) : int = FLT_MANT_DIG ## Number of digits (in base ``floatingPointRadix``) in the mantissa ## of 32-bit floating-point numbers. @@ -179,3 +180,11 @@ template maximumPositiveValue*(T : typedesc[float64]) : float64 = DBL_MAX template epsilon*(T : typedesc[float64]): float64 = DBL_EPSILON ## The difference between 1.0 and the smallest number greater than ## 1.0 that can be represented in a 64-bit floating-point type. + + +when isMainModule: + func is_significant(x: float): bool = + if x > minimumPositiveValue(float) and x < maximumPositiveValue(float): true + else: false + + doAssert is_significant(10.0) diff --git a/lib/pure/future.nim b/lib/pure/future.nim index 1a3ab688d..40dba7846 100644 --- a/lib/pure/future.nim +++ b/lib/pure/future.nim @@ -1,200 +1,4 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Dominik Picheta -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# -## This module implements experimental features which may soon be moved to -## the system module (or other more appropriate modules). +{.deprecated: "Use the new 'sugar' module instead".} -import macros - -proc createProcType(p, b: NimNode): NimNode {.compileTime.} = - #echo treeRepr(p) - #echo treeRepr(b) - result = newNimNode(nnkProcTy) - var formalParams = newNimNode(nnkFormalParams) - - formalParams.add b - - case p.kind - of nnkPar: - for i in 0 ..< p.len: - let ident = p[i] - var identDefs = newNimNode(nnkIdentDefs) - case ident.kind - of nnkExprColonExpr: - identDefs.add ident[0] - identDefs.add ident[1] - else: - identDefs.add newIdentNode("i" & $i) - identDefs.add(ident) - identDefs.add newEmptyNode() - formalParams.add identDefs - else: - var identDefs = newNimNode(nnkIdentDefs) - identDefs.add newIdentNode("i0") - identDefs.add(p) - identDefs.add newEmptyNode() - formalParams.add identDefs - - result.add formalParams - result.add newEmptyNode() - #echo(treeRepr(result)) - #echo(result.toStrLit()) - -macro `=>`*(p, b: untyped): untyped = - ## Syntax sugar for anonymous procedures. - ## - ## .. code-block:: nim - ## - ## proc passTwoAndTwo(f: (int, int) -> int): int = - ## f(2, 2) - ## - ## passTwoAndTwo((x, y) => x + y) # 4 - - #echo treeRepr(p) - #echo(treeRepr(b)) - var params: seq[NimNode] = @[newIdentNode("auto")] - - case p.kind - of nnkPar: - for c in children(p): - var identDefs = newNimNode(nnkIdentDefs) - case c.kind - of nnkExprColonExpr: - identDefs.add(c[0]) - identDefs.add(c[1]) - identDefs.add(newEmptyNode()) - of nnkIdent: - identDefs.add(c) - identDefs.add(newIdentNode("auto")) - identDefs.add(newEmptyNode()) - of nnkInfix: - if c[0].kind == nnkIdent and c[0].ident == !"->": - var procTy = createProcType(c[1], c[2]) - params[0] = procTy[0][0] - for i in 1 ..< procTy[0].len: - params.add(procTy[0][i]) - else: - error("Expected proc type (->) got (" & $c[0].ident & ").") - break - else: - echo treeRepr c - error("Incorrect procedure parameter list.") - params.add(identDefs) - of nnkIdent: - var identDefs = newNimNode(nnkIdentDefs) - identDefs.add(p) - identDefs.add(newIdentNode("auto")) - identDefs.add(newEmptyNode()) - params.add(identDefs) - of nnkInfix: - if p[0].kind == nnkIdent and p[0].ident == !"->": - var procTy = createProcType(p[1], p[2]) - params[0] = procTy[0][0] - for i in 1 ..< procTy[0].len: - params.add(procTy[0][i]) - else: - error("Expected proc type (->) got (" & $p[0].ident & ").") - else: - error("Incorrect procedure parameter list.") - result = newProc(params = params, body = b, procType = nnkLambda) - #echo(result.treeRepr) - #echo(result.toStrLit()) - #return result # TODO: Bug? - -macro `->`*(p, b: untyped): untyped = - ## Syntax sugar for procedure types. - ## - ## .. code-block:: nim - ## - ## proc pass2(f: (float, float) -> float): float = - ## f(2, 2) - ## - ## # is the same as: - ## - ## proc pass2(f: proc (x, y: float): float): float = - ## f(2, 2) - - result = createProcType(p, b) - -type ListComprehension = object -var lc*: ListComprehension - -macro `[]`*(lc: ListComprehension, comp, typ: untyped): untyped = - ## List comprehension, returns a sequence. `comp` is the actual list - ## comprehension, for example ``x | (x <- 1..10, x mod 2 == 0)``. `typ` is - ## the type that will be stored inside the result seq. - ## - ## .. code-block:: nim - ## - ## echo lc[x | (x <- 1..10, x mod 2 == 0), int] - ## - ## const n = 20 - ## echo lc[(x,y,z) | (x <- 1..n, y <- x..n, z <- y..n, x*x + y*y == z*z), - ## tuple[a,b,c: int]] - - expectLen(comp, 3) - expectKind(comp, nnkInfix) - expectKind(comp[0], nnkIdent) - assert($comp[0].ident == "|") - - result = newCall( - newDotExpr( - newIdentNode("result"), - newIdentNode("add")), - comp[1]) - - for i in countdown(comp[2].len-1, 0): - let x = comp[2][i] - expectMinLen(x, 1) - if x[0].kind == nnkIdent and $x[0].ident == "<-": - expectLen(x, 3) - result = newNimNode(nnkForStmt).add(x[1], x[2], result) - else: - result = newIfStmt((x, result)) - - result = newNimNode(nnkCall).add( - newNimNode(nnkPar).add( - newNimNode(nnkLambda).add( - newEmptyNode(), - newEmptyNode(), - newEmptyNode(), - newNimNode(nnkFormalParams).add( - newNimNode(nnkBracketExpr).add( - newIdentNode("seq"), - typ)), - newEmptyNode(), - newEmptyNode(), - newStmtList( - newAssignment( - newIdentNode("result"), - newNimNode(nnkPrefix).add( - newIdentNode("@"), - newNimNode(nnkBracket))), - result)))) - - -macro dump*(x: typed): untyped = - ## Dumps the content of an expression, useful for debugging. - ## It accepts any expression and prints a textual representation - ## of the tree representing the expression - as it would appear in - ## source code - together with the value of the expression. - ## - ## As an example, - ## - ## .. code-block:: nim - ## let - ## x = 10 - ## y = 20 - ## dump(x + y) - ## - ## will print ``x + y = 30``. - let s = x.toStrLit - let r = quote do: - debugEcho `s`, " = ", `x` - return r +include sugar diff --git a/lib/pure/hashes.nim b/lib/pure/hashes.nim index b015ed311..6535bb0d3 100644 --- a/lib/pure/hashes.nim +++ b/lib/pure/hashes.nim @@ -45,7 +45,6 @@ type Hash* = int ## a hash value; hash tables using these values should ## always have a size of a power of two and can use the ``and`` ## operator instead of ``mod`` for truncation of the hash value. -{.deprecated: [THash: Hash].} proc `!&`*(h: Hash, val: int): Hash {.inline.} = ## mixes a hash value `h` with `val` to produce a new hash value. This is diff --git a/lib/pure/htmlgen.nim b/lib/pure/htmlgen.nim index c0934a45b..a2e224f44 100644 --- a/lib/pure/htmlgen.nim +++ b/lib/pure/htmlgen.nim @@ -31,14 +31,23 @@ import macros, strutils const - coreAttr* = " id class title style " - eventAttr* = " onclick ondblclick onmousedown onmouseup " & - "onmouseover onmousemove onmouseout onkeypress onkeydown onkeyup onload " - commonAttr* = coreAttr & eventAttr + coreAttr* = " accesskey class contenteditable dir hidden id lang " & + "spellcheck style tabindex title translate " + eventAttr* = "onabort onblur oncancel oncanplay oncanplaythrough onchange " & + "onclick oncuechange ondblclick ondurationchange onemptied onended " & + "onerror onfocus oninput oninvalid onkeydown onkeypress onkeyup onload " & + "onloadeddata onloadedmetadata onloadstart onmousedown onmouseenter " & + "onmouseleave onmousemove onmouseout onmouseover onmouseup onmousewheel " & + "onpause onplay onplaying onprogress onratechange onreset onresize " & + "onscroll onseeked onseeking onselect onshow onstalled onsubmit " & + "onsuspend ontimeupdate ontoggle onvolumechange onwaiting " + ariaAttr* = " role " + commonAttr* = coreAttr & eventAttr & ariaAttr proc getIdent(e: NimNode): string {.compileTime.} = case e.kind - of nnkIdent: result = normalize($e.ident) + of nnkIdent: + result = e.strVal.normalize of nnkAccQuoted: result = getIdent(e[0]) for i in 1 .. e.len-1: @@ -92,19 +101,21 @@ proc xmlCheckedTag*(e: NimNode, tag: string, optAttr = "", reqAttr = "", result.add(newStrLitNode("</")) result.add(newStrLitNode(tag)) result.add(newStrLitNode(">")) - result = nestList(!"&", result) - + when compiles(nestList(ident"&", result)): + result = nestList(ident"&", result) + else: + result = nestList(!"&", result) macro a*(e: varargs[untyped]): untyped = ## generates the HTML ``a`` element. let e = callsite() - result = xmlCheckedTag(e, "a", "href charset type hreflang rel rev " & - "accesskey tabindex" & commonAttr) + result = xmlCheckedTag(e, "a", "href target download rel hreflang type " & + commonAttr) -macro acronym*(e: varargs[untyped]): untyped = - ## generates the HTML ``acronym`` element. +macro abbr*(e: varargs[untyped]): untyped = + ## generates the HTML ``abbr`` element. let e = callsite() - result = xmlCheckedTag(e, "acronym", commonAttr) + result = xmlCheckedTag(e, "abbr", commonAttr) macro address*(e: varargs[untyped]): untyped = ## generates the HTML ``address`` element. @@ -114,8 +125,24 @@ macro address*(e: varargs[untyped]): untyped = macro area*(e: varargs[untyped]): untyped = ## generates the HTML ``area`` element. let e = callsite() - result = xmlCheckedTag(e, "area", "shape coords href nohref" & - " accesskey tabindex" & commonAttr, "alt", true) + result = xmlCheckedTag(e, "area", "coords download href hreflang rel " & + "shape target type" & commonAttr, "alt", true) + +macro article*(e: varargs[untyped]): untyped = + ## generates the HTML ``article`` element. + let e = callsite() + result = xmlCheckedTag(e, "article", commonAttr) + +macro aside*(e: varargs[untyped]): untyped = + ## generates the HTML ``aside`` element. + let e = callsite() + result = xmlCheckedTag(e, "aside", commonAttr) + +macro audio*(e: varargs[untyped]): untyped = + ## generates the HTML ``audio`` element. + let e = callsite() + result = xmlCheckedTag(e, "audio", "src crossorigin preload " & + "autoplay mediagroup loop muted controls" & commonAttr) macro b*(e: varargs[untyped]): untyped = ## generates the HTML ``b`` element. @@ -125,7 +152,17 @@ macro b*(e: varargs[untyped]): untyped = macro base*(e: varargs[untyped]): untyped = ## generates the HTML ``base`` element. let e = callsite() - result = xmlCheckedTag(e, "base", "", "href", true) + result = xmlCheckedTag(e, "base", "href target" & commonAttr, "", true) + +macro bdi*(e: varargs[untyped]): untyped = + ## generates the HTML ``bdi`` element. + let e = callsite() + result = xmlCheckedTag(e, "bdi", commonAttr) + +macro bdo*(e: varargs[untyped]): untyped = + ## generates the HTML ``bdo`` element. + let e = callsite() + result = xmlCheckedTag(e, "bdo", commonAttr) macro big*(e: varargs[untyped]): untyped = ## generates the HTML ``big`` element. @@ -140,18 +177,26 @@ macro blockquote*(e: varargs[untyped]): untyped = macro body*(e: varargs[untyped]): untyped = ## generates the HTML ``body`` element. let e = callsite() - result = xmlCheckedTag(e, "body", commonAttr) + result = xmlCheckedTag(e, "body", "onafterprint onbeforeprint " & + "onbeforeunload onhashchange onmessage onoffline ononline onpagehide " & + "onpageshow onpopstate onstorage onunload" & commonAttr) macro br*(e: varargs[untyped]): untyped = ## generates the HTML ``br`` element. let e = callsite() - result = xmlCheckedTag(e, "br", "", "", true) + result = xmlCheckedTag(e, "br", commonAttr, "", true) macro button*(e: varargs[untyped]): untyped = ## generates the HTML ``button`` element. let e = callsite() - result = xmlCheckedTag(e, "button", "accesskey tabindex " & - "disabled name type value" & commonAttr) + result = xmlCheckedTag(e, "button", "autofocus disabled form formaction " & + "formenctype formmethod formnovalidate formtarget menu name type value" & + commonAttr) + +macro canvas*(e: varargs[untyped]): untyped = + ## generates the HTML ``canvas`` element. + let e = callsite() + result = xmlCheckedTag(e, "canvas", "width height" & commonAttr) macro caption*(e: varargs[untyped]): untyped = ## generates the HTML ``caption`` element. @@ -171,12 +216,22 @@ macro code*(e: varargs[untyped]): untyped = macro col*(e: varargs[untyped]): untyped = ## generates the HTML ``col`` element. let e = callsite() - result = xmlCheckedTag(e, "col", "span align valign" & commonAttr, "", true) + result = xmlCheckedTag(e, "col", "span" & commonAttr, "", true) macro colgroup*(e: varargs[untyped]): untyped = ## generates the HTML ``colgroup`` element. let e = callsite() - result = xmlCheckedTag(e, "colgroup", "span align valign" & commonAttr) + result = xmlCheckedTag(e, "colgroup", "span" & commonAttr) + +macro data*(e: varargs[untyped]): untyped = + ## generates the HTML ``data`` element. + let e = callsite() + result = xmlCheckedTag(e, "data", "value" & commonAttr) + +macro datalist*(e: varargs[untyped]): untyped = + ## generates the HTML ``datalist`` element. + let e = callsite() + result = xmlCheckedTag(e, "datalist", commonAttr) macro dd*(e: varargs[untyped]): untyped = ## generates the HTML ``dd`` element. @@ -213,16 +268,37 @@ macro em*(e: varargs[untyped]): untyped = let e = callsite() result = xmlCheckedTag(e, "em", commonAttr) +macro embed*(e: varargs[untyped]): untyped = + ## generates the HTML ``embed`` element. + let e = callsite() + result = xmlCheckedTag(e, "embed", "src type height width" & + commonAttr, "", true) + macro fieldset*(e: varargs[untyped]): untyped = ## generates the HTML ``fieldset`` element. let e = callsite() - result = xmlCheckedTag(e, "fieldset", commonAttr) + result = xmlCheckedTag(e, "fieldset", "disabled form name" & commonAttr) + +macro figure*(e: varargs[untyped]): untyped = + ## generates the HTML ``figure`` element. + let e = callsite() + result = xmlCheckedTag(e, "figure", commonAttr) + +macro figcaption*(e: varargs[untyped]): untyped = + ## generates the HTML ``figcaption`` element. + let e = callsite() + result = xmlCheckedTag(e, "figcaption", commonAttr) + +macro footer*(e: varargs[untyped]): untyped = + ## generates the HTML ``footer`` element. + let e = callsite() + result = xmlCheckedTag(e, "footer", commonAttr) macro form*(e: varargs[untyped]): untyped = ## generates the HTML ``form`` element. let e = callsite() - result = xmlCheckedTag(e, "form", "method encype accept accept-charset" & - commonAttr, "action") + result = xmlCheckedTag(e, "form", "accept-charset action autocomplete " & + "enctype method name novalidate target" & commonAttr) macro h1*(e: varargs[untyped]): untyped = ## generates the HTML ``h1`` element. @@ -257,7 +333,12 @@ macro h6*(e: varargs[untyped]): untyped = macro head*(e: varargs[untyped]): untyped = ## generates the HTML ``head`` element. let e = callsite() - result = xmlCheckedTag(e, "head", "profile") + result = xmlCheckedTag(e, "head", commonAttr) + +macro header*(e: varargs[untyped]): untyped = + ## generates the HTML ``header`` element. + let e = callsite() + result = xmlCheckedTag(e, "header", commonAttr) macro html*(e: varargs[untyped]): untyped = ## generates the HTML ``html`` element. @@ -274,16 +355,26 @@ macro i*(e: varargs[untyped]): untyped = let e = callsite() result = xmlCheckedTag(e, "i", commonAttr) +macro iframe*(e: varargs[untyped]): untyped = + ## generates the HTML ``iframe`` element. + let e = callsite() + result = xmlCheckedTag(e, "iframe", "src srcdoc name sandbox width height" & + commonAttr) + macro img*(e: varargs[untyped]): untyped = ## generates the HTML ``img`` element. let e = callsite() - result = xmlCheckedTag(e, "img", "longdesc height width", "src alt", true) + result = xmlCheckedTag(e, "img", "crossorigin usemap ismap height width" & + commonAttr, "src alt", true) macro input*(e: varargs[untyped]): untyped = ## generates the HTML ``input`` element. let e = callsite() - result = xmlCheckedTag(e, "input", "name type value checked maxlength src" & - " alt accept disabled readonly accesskey tabindex" & commonAttr, "", true) + result = xmlCheckedTag(e, "input", "accept alt autocomplete autofocus " & + "checked dirname disabled form formaction formenctype formmethod " & + "formnovalidate formtarget height inputmode list max maxlength min " & + "minlength multiple name pattern placeholder readonly required size " & + "src step type value width" & commonAttr, "", true) macro ins*(e: varargs[untyped]): untyped = ## generates the HTML ``ins`` element. @@ -295,36 +386,64 @@ macro kbd*(e: varargs[untyped]): untyped = let e = callsite() result = xmlCheckedTag(e, "kbd", commonAttr) +macro keygen*(e: varargs[untyped]): untyped = + ## generates the HTML ``keygen`` element. + let e = callsite() + result = xmlCheckedTag(e, "keygen", "autofocus challenge disabled " & + "form keytype name" & commonAttr) + macro label*(e: varargs[untyped]): untyped = ## generates the HTML ``label`` element. let e = callsite() - result = xmlCheckedTag(e, "label", "for accesskey" & commonAttr) + result = xmlCheckedTag(e, "label", "form for" & commonAttr) macro legend*(e: varargs[untyped]): untyped = ## generates the HTML ``legend`` element. let e = callsite() - result = xmlCheckedTag(e, "legend", "accesskey" & commonAttr) + result = xmlCheckedTag(e, "legend", commonAttr) macro li*(e: varargs[untyped]): untyped = ## generates the HTML ``li`` element. let e = callsite() - result = xmlCheckedTag(e, "li", commonAttr) + result = xmlCheckedTag(e, "li", "value" & commonAttr) macro link*(e: varargs[untyped]): untyped = ## generates the HTML ``link`` element. let e = callsite() - result = xmlCheckedTag(e, "link", "href charset hreflang type rel rev media" & - commonAttr, "", true) + result = xmlCheckedTag(e, "link", "href crossorigin rel media hreflang " & + "type sizes" & commonAttr, "", true) + +macro main*(e: varargs[untyped]): untyped = + ## generates the HTML ``main`` element. + let e = callsite() + result = xmlCheckedTag(e, "main", commonAttr) macro map*(e: varargs[untyped]): untyped = ## generates the HTML ``map`` element. let e = callsite() - result = xmlCheckedTag(e, "map", "class title" & eventAttr, "id", false) + result = xmlCheckedTag(e, "map", "name" & commonAttr) + +macro mark*(e: varargs[untyped]): untyped = + ## generates the HTML ``mark`` element. + let e = callsite() + result = xmlCheckedTag(e, "mark", commonAttr) macro meta*(e: varargs[untyped]): untyped = ## generates the HTML ``meta`` element. let e = callsite() - result = xmlCheckedTag(e, "meta", "name http-equiv scheme", "content", true) + result = xmlCheckedTag(e, "meta", "name http-equiv content charset" & + commonAttr, "", true) + +macro meter*(e: varargs[untyped]): untyped = + ## generates the HTML ``meter`` element. + let e = callsite() + result = xmlCheckedTag(e, "meter", "value min max low high optimum" & + commonAttr) + +macro nav*(e: varargs[untyped]): untyped = + ## generates the HTML ``nav`` element. + let e = callsite() + result = xmlCheckedTag(e, "nav", commonAttr) macro noscript*(e: varargs[untyped]): untyped = ## generates the HTML ``noscript`` element. @@ -334,13 +453,13 @@ macro noscript*(e: varargs[untyped]): untyped = macro `object`*(e: varargs[untyped]): untyped = ## generates the HTML ``object`` element. let e = callsite() - result = xmlCheckedTag(e, "object", "classid data codebase declare type " & - "codetype archive standby width height name tabindex" & commonAttr) + result = xmlCheckedTag(e, "object", "data type typemustmatch name usemap " & + "form width height" & commonAttr) macro ol*(e: varargs[untyped]): untyped = ## generates the HTML ``ol`` element. let e = callsite() - result = xmlCheckedTag(e, "ol", commonAttr) + result = xmlCheckedTag(e, "ol", "reversed start type" & commonAttr) macro optgroup*(e: varargs[untyped]): untyped = ## generates the HTML ``optgroup`` element. @@ -350,7 +469,13 @@ macro optgroup*(e: varargs[untyped]): untyped = macro option*(e: varargs[untyped]): untyped = ## generates the HTML ``option`` element. let e = callsite() - result = xmlCheckedTag(e, "option", "selected value" & commonAttr) + result = xmlCheckedTag(e, "option", "disabled label selected value" & + commonAttr) + +macro output*(e: varargs[untyped]): untyped = + ## generates the HTML ``output`` element. + let e = callsite() + result = xmlCheckedTag(e, "output", "for form name" & commonAttr) macro p*(e: varargs[untyped]): untyped = ## generates the HTML ``p`` element. @@ -360,18 +485,53 @@ macro p*(e: varargs[untyped]): untyped = macro param*(e: varargs[untyped]): untyped = ## generates the HTML ``param`` element. let e = callsite() - result = xmlCheckedTag(e, "param", "value id type valuetype", "name", true) + result = xmlCheckedTag(e, "param", commonAttr, "name value", true) macro pre*(e: varargs[untyped]): untyped = ## generates the HTML ``pre`` element. let e = callsite() result = xmlCheckedTag(e, "pre", commonAttr) +macro progress*(e: varargs[untyped]): untyped = + ## generates the HTML ``progress`` element. + let e = callsite() + result = xmlCheckedTag(e, "progress", "value max" & commonAttr) + macro q*(e: varargs[untyped]): untyped = ## generates the HTML ``q`` element. let e = callsite() result = xmlCheckedTag(e, "q", "cite" & commonAttr) +macro rb*(e: varargs[untyped]): untyped = + ## generates the HTML ``rb`` element. + let e = callsite() + result = xmlCheckedTag(e, "rb", commonAttr) + +macro rp*(e: varargs[untyped]): untyped = + ## generates the HTML ``rp`` element. + let e = callsite() + result = xmlCheckedTag(e, "rp", commonAttr) + +macro rt*(e: varargs[untyped]): untyped = + ## generates the HTML ``rt`` element. + let e = callsite() + result = xmlCheckedTag(e, "rt", commonAttr) + +macro rtc*(e: varargs[untyped]): untyped = + ## generates the HTML ``rtc`` element. + let e = callsite() + result = xmlCheckedTag(e, "rtc", commonAttr) + +macro ruby*(e: varargs[untyped]): untyped = + ## generates the HTML ``ruby`` element. + let e = callsite() + result = xmlCheckedTag(e, "ruby", commonAttr) + +macro s*(e: varargs[untyped]): untyped = + ## generates the HTML ``s`` element. + let e = callsite() + result = xmlCheckedTag(e, "s", commonAttr) + macro samp*(e: varargs[untyped]): untyped = ## generates the HTML ``samp`` element. let e = callsite() @@ -380,19 +540,30 @@ macro samp*(e: varargs[untyped]): untyped = macro script*(e: varargs[untyped]): untyped = ## generates the HTML ``script`` element. let e = callsite() - result = xmlCheckedTag(e, "script", "src charset defer", "type", false) + result = xmlCheckedTag(e, "script", "src type charset async defer " & + "crossorigin" & commonAttr) + +macro section*(e: varargs[untyped]): untyped = + ## generates the HTML ``section`` element. + let e = callsite() + result = xmlCheckedTag(e, "section", commonAttr) macro select*(e: varargs[untyped]): untyped = ## generates the HTML ``select`` element. let e = callsite() - result = xmlCheckedTag(e, "select", "name size multiple disabled tabindex" & - commonAttr) + result = xmlCheckedTag(e, "select", "autofocus disabled form multiple " & + "name required size" & commonAttr) macro small*(e: varargs[untyped]): untyped = ## generates the HTML ``small`` element. let e = callsite() result = xmlCheckedTag(e, "small", commonAttr) +macro source*(e: varargs[untyped]): untyped = + ## generates the HTML ``source`` element. + let e = callsite() + result = xmlCheckedTag(e, "source", "type" & commonAttr, "src", true) + macro span*(e: varargs[untyped]): untyped = ## generates the HTML ``span`` element. let e = callsite() @@ -406,7 +577,7 @@ macro strong*(e: varargs[untyped]): untyped = macro style*(e: varargs[untyped]): untyped = ## generates the HTML ``style`` element. let e = callsite() - result = xmlCheckedTag(e, "style", "media title", "type") + result = xmlCheckedTag(e, "style", "media type" & commonAttr) macro sub*(e: varargs[untyped]): untyped = ## generates the HTML ``sub`` element. @@ -421,57 +592,77 @@ macro sup*(e: varargs[untyped]): untyped = macro table*(e: varargs[untyped]): untyped = ## generates the HTML ``table`` element. let e = callsite() - result = xmlCheckedTag(e, "table", "summary border cellpadding cellspacing" & - " frame rules width" & commonAttr) + result = xmlCheckedTag(e, "table", "border sortable" & commonAttr) macro tbody*(e: varargs[untyped]): untyped = ## generates the HTML ``tbody`` element. let e = callsite() - result = xmlCheckedTag(e, "tbody", "align valign" & commonAttr) + result = xmlCheckedTag(e, "tbody", commonAttr) macro td*(e: varargs[untyped]): untyped = ## generates the HTML ``td`` element. let e = callsite() - result = xmlCheckedTag(e, "td", "colspan rowspan abbr axis headers scope" & - " align valign" & commonAttr) + result = xmlCheckedTag(e, "td", "colspan rowspan headers" & commonAttr) + +macro `template`*(e: varargs[untyped]): untyped = + ## generates the HTML ``template`` element. + let e = callsite() + result = xmlCheckedTag(e, "template", commonAttr) macro textarea*(e: varargs[untyped]): untyped = ## generates the HTML ``textarea`` element. let e = callsite() - result = xmlCheckedTag(e, "textarea", " name disabled readonly accesskey" & - " tabindex" & commonAttr, "rows cols", false) + result = xmlCheckedTag(e, "textarea", "autocomplete autofocus cols " & + "dirname disabled form inputmode maxlength minlength name placeholder " & + "readonly required rows wrap" & commonAttr) macro tfoot*(e: varargs[untyped]): untyped = ## generates the HTML ``tfoot`` element. let e = callsite() - result = xmlCheckedTag(e, "tfoot", "align valign" & commonAttr) + result = xmlCheckedTag(e, "tfoot", commonAttr) macro th*(e: varargs[untyped]): untyped = ## generates the HTML ``th`` element. let e = callsite() - result = xmlCheckedTag(e, "th", "colspan rowspan abbr axis headers scope" & - " align valign" & commonAttr) + result = xmlCheckedTag(e, "th", "colspan rowspan headers abbr scope axis" & + " sorted" & commonAttr) macro thead*(e: varargs[untyped]): untyped = ## generates the HTML ``thead`` element. let e = callsite() - result = xmlCheckedTag(e, "thead", "align valign" & commonAttr) + result = xmlCheckedTag(e, "thead", commonAttr) + +macro time*(e: varargs[untyped]): untyped = + ## generates the HTML ``time`` element. + let e = callsite() + result = xmlCheckedTag(e, "time", "datetime" & commonAttr) macro title*(e: varargs[untyped]): untyped = ## generates the HTML ``title`` element. let e = callsite() - result = xmlCheckedTag(e, "title") + result = xmlCheckedTag(e, "title", commonAttr) macro tr*(e: varargs[untyped]): untyped = ## generates the HTML ``tr`` element. let e = callsite() - result = xmlCheckedTag(e, "tr", "align valign" & commonAttr) + result = xmlCheckedTag(e, "tr", commonAttr) + +macro track*(e: varargs[untyped]): untyped = + ## generates the HTML ``track`` element. + let e = callsite() + result = xmlCheckedTag(e, "track", "kind srclang label default" & + commonAttr, "src", true) macro tt*(e: varargs[untyped]): untyped = ## generates the HTML ``tt`` element. let e = callsite() result = xmlCheckedTag(e, "tt", commonAttr) +macro u*(e: varargs[untyped]): untyped = + ## generates the HTML ``u`` element. + let e = callsite() + result = xmlCheckedTag(e, "u", commonAttr) + macro ul*(e: varargs[untyped]): untyped = ## generates the HTML ``ul`` element. let e = callsite() @@ -482,6 +673,17 @@ macro `var`*(e: varargs[untyped]): untyped = let e = callsite() result = xmlCheckedTag(e, "var", commonAttr) +macro video*(e: varargs[untyped]): untyped = + ## generates the HTML ``video`` element. + let e = callsite() + result = xmlCheckedTag(e, "video", "src crossorigin poster preload " & + "autoplay mediagroup loop muted controls width height" & commonAttr) + +macro wbr*(e: varargs[untyped]): untyped = + ## generates the HTML ``wbr`` element. + let e = callsite() + result = xmlCheckedTag(e, "wbr", commonAttr, "", true) + when isMainModule: let nim = "Nim" assert h1(a(href="http://nim-lang.org", nim)) == diff --git a/lib/pure/htmlparser.nim b/lib/pure/htmlparser.nim index a718a10fd..fbf2b8e73 100644 --- a/lib/pure/htmlparser.nim +++ b/lib/pure/htmlparser.nim @@ -178,7 +178,6 @@ type tagVar, ## the HTML ``var`` element tagVideo, ## the HTML ``video`` element tagWbr ## the HTML ``wbr`` element -{.deprecated: [THtmlTag: HtmlTag].} const tagToStr* = [ @@ -218,79 +217,6 @@ const tagBr, tagCol, tagFrame, tagHr, tagImg, tagIsindex, tagLink, tagMeta, tagParam, tagWbr} - Entities = [ - ("nbsp", 0x00A0), ("iexcl", 0x00A1), ("cent", 0x00A2), ("pound", 0x00A3), - ("curren", 0x00A4), ("yen", 0x00A5), ("brvbar", 0x00A6), ("sect", 0x00A7), - ("uml", 0x00A8), ("copy", 0x00A9), ("ordf", 0x00AA), ("laquo", 0x00AB), - ("not", 0x00AC), ("shy", 0x00AD), ("reg", 0x00AE), ("macr", 0x00AF), - ("deg", 0x00B0), ("plusmn", 0x00B1), ("sup2", 0x00B2), ("sup3", 0x00B3), - ("acute", 0x00B4), ("micro", 0x00B5), ("para", 0x00B6), ("middot", 0x00B7), - ("cedil", 0x00B8), ("sup1", 0x00B9), ("ordm", 0x00BA), ("raquo", 0x00BB), - ("frac14", 0x00BC), ("frac12", 0x00BD), ("frac34", 0x00BE), - ("iquest", 0x00BF), ("Agrave", 0x00C0), ("Aacute", 0x00C1), - ("Acirc", 0x00C2), ("Atilde", 0x00C3), ("Auml", 0x00C4), ("Aring", 0x00C5), - ("AElig", 0x00C6), ("Ccedil", 0x00C7), ("Egrave", 0x00C8), - ("Eacute", 0x00C9), ("Ecirc", 0x00CA), ("Euml", 0x00CB), ("Igrave", 0x00CC), - ("Iacute", 0x00CD), ("Icirc", 0x00CE), ("Iuml", 0x00CF), ("ETH", 0x00D0), - ("Ntilde", 0x00D1), ("Ograve", 0x00D2), ("Oacute", 0x00D3), - ("Ocirc", 0x00D4), ("Otilde", 0x00D5), ("Ouml", 0x00D6), ("times", 0x00D7), - ("Oslash", 0x00D8), ("Ugrave", 0x00D9), ("Uacute", 0x00DA), - ("Ucirc", 0x00DB), ("Uuml", 0x00DC), ("Yacute", 0x00DD), ("THORN", 0x00DE), - ("szlig", 0x00DF), ("agrave", 0x00E0), ("aacute", 0x00E1), - ("acirc", 0x00E2), ("atilde", 0x00E3), ("auml", 0x00E4), ("aring", 0x00E5), - ("aelig", 0x00E6), ("ccedil", 0x00E7), ("egrave", 0x00E8), - ("eacute", 0x00E9), ("ecirc", 0x00EA), ("euml", 0x00EB), ("igrave", 0x00EC), - ("iacute", 0x00ED), ("icirc", 0x00EE), ("iuml", 0x00EF), ("eth", 0x00F0), - ("ntilde", 0x00F1), ("ograve", 0x00F2), ("oacute", 0x00F3), - ("ocirc", 0x00F4), ("otilde", 0x00F5), ("ouml", 0x00F6), ("divide", 0x00F7), - ("oslash", 0x00F8), ("ugrave", 0x00F9), ("uacute", 0x00FA), - ("ucirc", 0x00FB), ("uuml", 0x00FC), ("yacute", 0x00FD), ("thorn", 0x00FE), - ("yuml", 0x00FF), ("OElig", 0x0152), ("oelig", 0x0153), ("Scaron", 0x0160), - ("scaron", 0x0161), ("Yuml", 0x0178), ("fnof", 0x0192), ("circ", 0x02C6), - ("tilde", 0x02DC), ("Alpha", 0x0391), ("Beta", 0x0392), ("Gamma", 0x0393), - ("Delta", 0x0394), ("Epsilon", 0x0395), ("Zeta", 0x0396), ("Eta", 0x0397), - ("Theta", 0x0398), ("Iota", 0x0399), ("Kappa", 0x039A), ("Lambda", 0x039B), - ("Mu", 0x039C), ("Nu", 0x039D), ("Xi", 0x039E), ("Omicron", 0x039F), - ("Pi", 0x03A0), ("Rho", 0x03A1), ("Sigma", 0x03A3), ("Tau", 0x03A4), - ("Upsilon", 0x03A5), ("Phi", 0x03A6), ("Chi", 0x03A7), ("Psi", 0x03A8), - ("Omega", 0x03A9), ("alpha", 0x03B1), ("beta", 0x03B2), ("gamma", 0x03B3), - ("delta", 0x03B4), ("epsilon", 0x03B5), ("zeta", 0x03B6), ("eta", 0x03B7), - ("theta", 0x03B8), ("iota", 0x03B9), ("kappa", 0x03BA), ("lambda", 0x03BB), - ("mu", 0x03BC), ("nu", 0x03BD), ("xi", 0x03BE), ("omicron", 0x03BF), - ("pi", 0x03C0), ("rho", 0x03C1), ("sigmaf", 0x03C2), ("sigma", 0x03C3), - ("tau", 0x03C4), ("upsilon", 0x03C5), ("phi", 0x03C6), ("chi", 0x03C7), - ("psi", 0x03C8), ("omega", 0x03C9), ("thetasym", 0x03D1), ("upsih", 0x03D2), - ("piv", 0x03D6), ("ensp", 0x2002), ("emsp", 0x2003), ("thinsp", 0x2009), - ("zwnj", 0x200C), ("zwj", 0x200D), ("lrm", 0x200E), ("rlm", 0x200F), - ("ndash", 0x2013), ("mdash", 0x2014), ("lsquo", 0x2018), ("rsquo", 0x2019), - ("sbquo", 0x201A), ("ldquo", 0x201C), ("rdquo", 0x201D), ("bdquo", 0x201E), - ("dagger", 0x2020), ("Dagger", 0x2021), ("bull", 0x2022), - ("hellip", 0x2026), ("permil", 0x2030), ("prime", 0x2032), - ("Prime", 0x2033), ("lsaquo", 0x2039), ("rsaquo", 0x203A), - ("oline", 0x203E), ("frasl", 0x2044), ("euro", 0x20AC), - ("image", 0x2111), ("weierp", 0x2118), ("real", 0x211C), - ("trade", 0x2122), ("alefsym", 0x2135), ("larr", 0x2190), - ("uarr", 0x2191), ("rarr", 0x2192), ("darr", 0x2193), - ("harr", 0x2194), ("crarr", 0x21B5), ("lArr", 0x21D0), - ("uArr", 0x21D1), ("rArr", 0x21D2), ("dArr", 0x21D3), - ("hArr", 0x21D4), ("forall", 0x2200), ("part", 0x2202), - ("exist", 0x2203), ("empty", 0x2205), ("nabla", 0x2207), - ("isin", 0x2208), ("notin", 0x2209), ("ni", 0x220B), - ("prod", 0x220F), ("sum", 0x2211), ("minus", 0x2212), - ("lowast", 0x2217), ("radic", 0x221A), ("prop", 0x221D), - ("infin", 0x221E), ("ang", 0x2220), ("and", 0x2227), - ("or", 0x2228), ("cap", 0x2229), ("cup", 0x222A), - ("int", 0x222B), ("there4", 0x2234), ("sim", 0x223C), - ("cong", 0x2245), ("asymp", 0x2248), ("ne", 0x2260), - ("equiv", 0x2261), ("le", 0x2264), ("ge", 0x2265), - ("sub", 0x2282), ("sup", 0x2283), ("nsub", 0x2284), - ("sube", 0x2286), ("supe", 0x2287), ("oplus", 0x2295), - ("otimes", 0x2297), ("perp", 0x22A5), ("sdot", 0x22C5), - ("lceil", 0x2308), ("rceil", 0x2309), ("lfloor", 0x230A), - ("rfloor", 0x230B), ("lang", 0x2329), ("rang", 0x232A), - ("loz", 0x25CA), ("spades", 0x2660), ("clubs", 0x2663), - ("hearts", 0x2665), ("diams", 0x2666)] - proc allLower(s: string): bool = for c in s: if c < 'a' or c > 'z': return false @@ -436,18 +362,1538 @@ proc htmlTag*(s: string): HtmlTag = let s = if allLower(s): s else: toLowerAscii(s) result = toHtmlTag(s) +proc runeToEntity*(rune: Rune): string = + ## converts a Rune to its numeric HTML entity equivalent. + runnableExamples: + import unicode + doAssert runeToEntity(Rune(0)) == "" + doAssert runeToEntity(Rune(-1)) == "" + doAssert runeToEntity("Ü".runeAt(0)) == "#220" + doAssert runeToEntity("∈".runeAt(0)) == "#8712" + if rune.ord <= 0: result = "" + else: result = '#' & $rune.ord + +proc entityToRune*(entity: string): Rune = + ## Converts an HTML entity name like ``Ü`` or values like ``Ü`` + ## or ``Ü`` to its UTF-8 equivalent. + ## Rune(0) is returned if the entity name is unknown. + runnableExamples: + import unicode + doAssert entityToRune("") == Rune(0) + doAssert entityToRune("a") == Rune(0) + doAssert entityToRune("gt") == ">".runeAt(0) + doAssert entityToRune("Uuml") == "Ü".runeAt(0) + doAssert entityToRune("quest") == "?".runeAt(0) + doAssert entityToRune("#x0003F") == "?".runeAt(0) + if entity.len < 2: return # smallest entity has length 2 + if entity[0] == '#': + var runeValue = 0 + case entity[1] + of '0'..'9': + try: runeValue = parseInt(entity[1..^1]) + except: discard + of 'x', 'X': # not case sensitive here + try: runeValue = parseHexInt(entity[2..^1]) + except: discard + else: discard # other entities are not defined with prefix ``#`` + if runeValue notin 0..0x10FFFF: runeValue = 0 # only return legal values + return Rune(runeValue) + case entity # entity names are case sensitive + of "Tab": Rune(0x00009) + of "NewLine": Rune(0x0000A) + of "excl": Rune(0x00021) + of "quot", "QUOT": Rune(0x00022) + of "num": Rune(0x00023) + of "dollar": Rune(0x00024) + of "percnt": Rune(0x00025) + of "amp", "AMP": Rune(0x00026) + of "apos": Rune(0x00027) + of "lpar": Rune(0x00028) + of "rpar": Rune(0x00029) + of "ast", "midast": Rune(0x0002A) + of "plus": Rune(0x0002B) + of "comma": Rune(0x0002C) + of "period": Rune(0x0002E) + of "sol": Rune(0x0002F) + of "colon": Rune(0x0003A) + of "semi": Rune(0x0003B) + of "lt", "LT": Rune(0x0003C) + of "equals": Rune(0x0003D) + of "gt", "GT": Rune(0x0003E) + of "quest": Rune(0x0003F) + of "commat": Rune(0x00040) + of "lsqb", "lbrack": Rune(0x0005B) + of "bsol": Rune(0x0005C) + of "rsqb", "rbrack": Rune(0x0005D) + of "Hat": Rune(0x0005E) + of "lowbar": Rune(0x0005F) + of "grave", "DiacriticalGrave": Rune(0x00060) + of "lcub", "lbrace": Rune(0x0007B) + of "verbar", "vert", "VerticalLine": Rune(0x0007C) + of "rcub", "rbrace": Rune(0x0007D) + of "nbsp", "NonBreakingSpace": Rune(0x000A0) + of "iexcl": Rune(0x000A1) + of "cent": Rune(0x000A2) + of "pound": Rune(0x000A3) + of "curren": Rune(0x000A4) + of "yen": Rune(0x000A5) + of "brvbar": Rune(0x000A6) + of "sect": Rune(0x000A7) + of "Dot", "die", "DoubleDot", "uml": Rune(0x000A8) + of "copy", "COPY": Rune(0x000A9) + of "ordf": Rune(0x000AA) + of "laquo": Rune(0x000AB) + of "not": Rune(0x000AC) + of "shy": Rune(0x000AD) + of "reg", "circledR", "REG": Rune(0x000AE) + of "macr", "OverBar", "strns": Rune(0x000AF) + of "deg": Rune(0x000B0) + of "plusmn", "pm", "PlusMinus": Rune(0x000B1) + of "sup2": Rune(0x000B2) + of "sup3": Rune(0x000B3) + of "acute", "DiacriticalAcute": Rune(0x000B4) + of "micro": Rune(0x000B5) + of "para": Rune(0x000B6) + of "middot", "centerdot", "CenterDot": Rune(0x000B7) + of "cedil", "Cedilla": Rune(0x000B8) + of "sup1": Rune(0x000B9) + of "ordm": Rune(0x000BA) + of "raquo": Rune(0x000BB) + of "frac14": Rune(0x000BC) + of "frac12", "half": Rune(0x000BD) + of "frac34": Rune(0x000BE) + of "iquest": Rune(0x000BF) + of "Agrave": Rune(0x000C0) + of "Aacute": Rune(0x000C1) + of "Acirc": Rune(0x000C2) + of "Atilde": Rune(0x000C3) + of "Auml": Rune(0x000C4) + of "Aring": Rune(0x000C5) + of "AElig": Rune(0x000C6) + of "Ccedil": Rune(0x000C7) + of "Egrave": Rune(0x000C8) + of "Eacute": Rune(0x000C9) + of "Ecirc": Rune(0x000CA) + of "Euml": Rune(0x000CB) + of "Igrave": Rune(0x000CC) + of "Iacute": Rune(0x000CD) + of "Icirc": Rune(0x000CE) + of "Iuml": Rune(0x000CF) + of "ETH": Rune(0x000D0) + of "Ntilde": Rune(0x000D1) + of "Ograve": Rune(0x000D2) + of "Oacute": Rune(0x000D3) + of "Ocirc": Rune(0x000D4) + of "Otilde": Rune(0x000D5) + of "Ouml": Rune(0x000D6) + of "times": Rune(0x000D7) + of "Oslash": Rune(0x000D8) + of "Ugrave": Rune(0x000D9) + of "Uacute": Rune(0x000DA) + of "Ucirc": Rune(0x000DB) + of "Uuml": Rune(0x000DC) + of "Yacute": Rune(0x000DD) + of "THORN": Rune(0x000DE) + of "szlig": Rune(0x000DF) + of "agrave": Rune(0x000E0) + of "aacute": Rune(0x000E1) + of "acirc": Rune(0x000E2) + of "atilde": Rune(0x000E3) + of "auml": Rune(0x000E4) + of "aring": Rune(0x000E5) + of "aelig": Rune(0x000E6) + of "ccedil": Rune(0x000E7) + of "egrave": Rune(0x000E8) + of "eacute": Rune(0x000E9) + of "ecirc": Rune(0x000EA) + of "euml": Rune(0x000EB) + of "igrave": Rune(0x000EC) + of "iacute": Rune(0x000ED) + of "icirc": Rune(0x000EE) + of "iuml": Rune(0x000EF) + of "eth": Rune(0x000F0) + of "ntilde": Rune(0x000F1) + of "ograve": Rune(0x000F2) + of "oacute": Rune(0x000F3) + of "ocirc": Rune(0x000F4) + of "otilde": Rune(0x000F5) + of "ouml": Rune(0x000F6) + of "divide", "div": Rune(0x000F7) + of "oslash": Rune(0x000F8) + of "ugrave": Rune(0x000F9) + of "uacute": Rune(0x000FA) + of "ucirc": Rune(0x000FB) + of "uuml": Rune(0x000FC) + of "yacute": Rune(0x000FD) + of "thorn": Rune(0x000FE) + of "yuml": Rune(0x000FF) + of "Amacr": Rune(0x00100) + of "amacr": Rune(0x00101) + of "Abreve": Rune(0x00102) + of "abreve": Rune(0x00103) + of "Aogon": Rune(0x00104) + of "aogon": Rune(0x00105) + of "Cacute": Rune(0x00106) + of "cacute": Rune(0x00107) + of "Ccirc": Rune(0x00108) + of "ccirc": Rune(0x00109) + of "Cdot": Rune(0x0010A) + of "cdot": Rune(0x0010B) + of "Ccaron": Rune(0x0010C) + of "ccaron": Rune(0x0010D) + of "Dcaron": Rune(0x0010E) + of "dcaron": Rune(0x0010F) + of "Dstrok": Rune(0x00110) + of "dstrok": Rune(0x00111) + of "Emacr": Rune(0x00112) + of "emacr": Rune(0x00113) + of "Edot": Rune(0x00116) + of "edot": Rune(0x00117) + of "Eogon": Rune(0x00118) + of "eogon": Rune(0x00119) + of "Ecaron": Rune(0x0011A) + of "ecaron": Rune(0x0011B) + of "Gcirc": Rune(0x0011C) + of "gcirc": Rune(0x0011D) + of "Gbreve": Rune(0x0011E) + of "gbreve": Rune(0x0011F) + of "Gdot": Rune(0x00120) + of "gdot": Rune(0x00121) + of "Gcedil": Rune(0x00122) + of "Hcirc": Rune(0x00124) + of "hcirc": Rune(0x00125) + of "Hstrok": Rune(0x00126) + of "hstrok": Rune(0x00127) + of "Itilde": Rune(0x00128) + of "itilde": Rune(0x00129) + of "Imacr": Rune(0x0012A) + of "imacr": Rune(0x0012B) + of "Iogon": Rune(0x0012E) + of "iogon": Rune(0x0012F) + of "Idot": Rune(0x00130) + of "imath", "inodot": Rune(0x00131) + of "IJlig": Rune(0x00132) + of "ijlig": Rune(0x00133) + of "Jcirc": Rune(0x00134) + of "jcirc": Rune(0x00135) + of "Kcedil": Rune(0x00136) + of "kcedil": Rune(0x00137) + of "kgreen": Rune(0x00138) + of "Lacute": Rune(0x00139) + of "lacute": Rune(0x0013A) + of "Lcedil": Rune(0x0013B) + of "lcedil": Rune(0x0013C) + of "Lcaron": Rune(0x0013D) + of "lcaron": Rune(0x0013E) + of "Lmidot": Rune(0x0013F) + of "lmidot": Rune(0x00140) + of "Lstrok": Rune(0x00141) + of "lstrok": Rune(0x00142) + of "Nacute": Rune(0x00143) + of "nacute": Rune(0x00144) + of "Ncedil": Rune(0x00145) + of "ncedil": Rune(0x00146) + of "Ncaron": Rune(0x00147) + of "ncaron": Rune(0x00148) + of "napos": Rune(0x00149) + of "ENG": Rune(0x0014A) + of "eng": Rune(0x0014B) + of "Omacr": Rune(0x0014C) + of "omacr": Rune(0x0014D) + of "Odblac": Rune(0x00150) + of "odblac": Rune(0x00151) + of "OElig": Rune(0x00152) + of "oelig": Rune(0x00153) + of "Racute": Rune(0x00154) + of "racute": Rune(0x00155) + of "Rcedil": Rune(0x00156) + of "rcedil": Rune(0x00157) + of "Rcaron": Rune(0x00158) + of "rcaron": Rune(0x00159) + of "Sacute": Rune(0x0015A) + of "sacute": Rune(0x0015B) + of "Scirc": Rune(0x0015C) + of "scirc": Rune(0x0015D) + of "Scedil": Rune(0x0015E) + of "scedil": Rune(0x0015F) + of "Scaron": Rune(0x00160) + of "scaron": Rune(0x00161) + of "Tcedil": Rune(0x00162) + of "tcedil": Rune(0x00163) + of "Tcaron": Rune(0x00164) + of "tcaron": Rune(0x00165) + of "Tstrok": Rune(0x00166) + of "tstrok": Rune(0x00167) + of "Utilde": Rune(0x00168) + of "utilde": Rune(0x00169) + of "Umacr": Rune(0x0016A) + of "umacr": Rune(0x0016B) + of "Ubreve": Rune(0x0016C) + of "ubreve": Rune(0x0016D) + of "Uring": Rune(0x0016E) + of "uring": Rune(0x0016F) + of "Udblac": Rune(0x00170) + of "udblac": Rune(0x00171) + of "Uogon": Rune(0x00172) + of "uogon": Rune(0x00173) + of "Wcirc": Rune(0x00174) + of "wcirc": Rune(0x00175) + of "Ycirc": Rune(0x00176) + of "ycirc": Rune(0x00177) + of "Yuml": Rune(0x00178) + of "Zacute": Rune(0x00179) + of "zacute": Rune(0x0017A) + of "Zdot": Rune(0x0017B) + of "zdot": Rune(0x0017C) + of "Zcaron": Rune(0x0017D) + of "zcaron": Rune(0x0017E) + of "fnof": Rune(0x00192) + of "imped": Rune(0x001B5) + of "gacute": Rune(0x001F5) + of "jmath": Rune(0x00237) + of "circ": Rune(0x002C6) + of "caron", "Hacek": Rune(0x002C7) + of "breve", "Breve": Rune(0x002D8) + of "dot", "DiacriticalDot": Rune(0x002D9) + of "ring": Rune(0x002DA) + of "ogon": Rune(0x002DB) + of "tilde", "DiacriticalTilde": Rune(0x002DC) + of "dblac", "DiacriticalDoubleAcute": Rune(0x002DD) + of "DownBreve": Rune(0x00311) + of "UnderBar": Rune(0x00332) + of "Alpha": Rune(0x00391) + of "Beta": Rune(0x00392) + of "Gamma": Rune(0x00393) + of "Delta": Rune(0x00394) + of "Epsilon": Rune(0x00395) + of "Zeta": Rune(0x00396) + of "Eta": Rune(0x00397) + of "Theta": Rune(0x00398) + of "Iota": Rune(0x00399) + of "Kappa": Rune(0x0039A) + of "Lambda": Rune(0x0039B) + of "Mu": Rune(0x0039C) + of "Nu": Rune(0x0039D) + of "Xi": Rune(0x0039E) + of "Omicron": Rune(0x0039F) + of "Pi": Rune(0x003A0) + of "Rho": Rune(0x003A1) + of "Sigma": Rune(0x003A3) + of "Tau": Rune(0x003A4) + of "Upsilon": Rune(0x003A5) + of "Phi": Rune(0x003A6) + of "Chi": Rune(0x003A7) + of "Psi": Rune(0x003A8) + of "Omega": Rune(0x003A9) + of "alpha": Rune(0x003B1) + of "beta": Rune(0x003B2) + of "gamma": Rune(0x003B3) + of "delta": Rune(0x003B4) + of "epsiv", "varepsilon", "epsilon": Rune(0x003B5) + of "zeta": Rune(0x003B6) + of "eta": Rune(0x003B7) + of "theta": Rune(0x003B8) + of "iota": Rune(0x003B9) + of "kappa": Rune(0x003BA) + of "lambda": Rune(0x003BB) + of "mu": Rune(0x003BC) + of "nu": Rune(0x003BD) + of "xi": Rune(0x003BE) + of "omicron": Rune(0x003BF) + of "pi": Rune(0x003C0) + of "rho": Rune(0x003C1) + of "sigmav", "varsigma", "sigmaf": Rune(0x003C2) + of "sigma": Rune(0x003C3) + of "tau": Rune(0x003C4) + of "upsi", "upsilon": Rune(0x003C5) + of "phi", "phiv", "varphi": Rune(0x003C6) + of "chi": Rune(0x003C7) + of "psi": Rune(0x003C8) + of "omega": Rune(0x003C9) + of "thetav", "vartheta", "thetasym": Rune(0x003D1) + of "Upsi", "upsih": Rune(0x003D2) + of "straightphi": Rune(0x003D5) + of "piv", "varpi": Rune(0x003D6) + of "Gammad": Rune(0x003DC) + of "gammad", "digamma": Rune(0x003DD) + of "kappav", "varkappa": Rune(0x003F0) + of "rhov", "varrho": Rune(0x003F1) + of "epsi", "straightepsilon": Rune(0x003F5) + of "bepsi", "backepsilon": Rune(0x003F6) + of "IOcy": Rune(0x00401) + of "DJcy": Rune(0x00402) + of "GJcy": Rune(0x00403) + of "Jukcy": Rune(0x00404) + of "DScy": Rune(0x00405) + of "Iukcy": Rune(0x00406) + of "YIcy": Rune(0x00407) + of "Jsercy": Rune(0x00408) + of "LJcy": Rune(0x00409) + of "NJcy": Rune(0x0040A) + of "TSHcy": Rune(0x0040B) + of "KJcy": Rune(0x0040C) + of "Ubrcy": Rune(0x0040E) + of "DZcy": Rune(0x0040F) + of "Acy": Rune(0x00410) + of "Bcy": Rune(0x00411) + of "Vcy": Rune(0x00412) + of "Gcy": Rune(0x00413) + of "Dcy": Rune(0x00414) + of "IEcy": Rune(0x00415) + of "ZHcy": Rune(0x00416) + of "Zcy": Rune(0x00417) + of "Icy": Rune(0x00418) + of "Jcy": Rune(0x00419) + of "Kcy": Rune(0x0041A) + of "Lcy": Rune(0x0041B) + of "Mcy": Rune(0x0041C) + of "Ncy": Rune(0x0041D) + of "Ocy": Rune(0x0041E) + of "Pcy": Rune(0x0041F) + of "Rcy": Rune(0x00420) + of "Scy": Rune(0x00421) + of "Tcy": Rune(0x00422) + of "Ucy": Rune(0x00423) + of "Fcy": Rune(0x00424) + of "KHcy": Rune(0x00425) + of "TScy": Rune(0x00426) + of "CHcy": Rune(0x00427) + of "SHcy": Rune(0x00428) + of "SHCHcy": Rune(0x00429) + of "HARDcy": Rune(0x0042A) + of "Ycy": Rune(0x0042B) + of "SOFTcy": Rune(0x0042C) + of "Ecy": Rune(0x0042D) + of "YUcy": Rune(0x0042E) + of "YAcy": Rune(0x0042F) + of "acy": Rune(0x00430) + of "bcy": Rune(0x00431) + of "vcy": Rune(0x00432) + of "gcy": Rune(0x00433) + of "dcy": Rune(0x00434) + of "iecy": Rune(0x00435) + of "zhcy": Rune(0x00436) + of "zcy": Rune(0x00437) + of "icy": Rune(0x00438) + of "jcy": Rune(0x00439) + of "kcy": Rune(0x0043A) + of "lcy": Rune(0x0043B) + of "mcy": Rune(0x0043C) + of "ncy": Rune(0x0043D) + of "ocy": Rune(0x0043E) + of "pcy": Rune(0x0043F) + of "rcy": Rune(0x00440) + of "scy": Rune(0x00441) + of "tcy": Rune(0x00442) + of "ucy": Rune(0x00443) + of "fcy": Rune(0x00444) + of "khcy": Rune(0x00445) + of "tscy": Rune(0x00446) + of "chcy": Rune(0x00447) + of "shcy": Rune(0x00448) + of "shchcy": Rune(0x00449) + of "hardcy": Rune(0x0044A) + of "ycy": Rune(0x0044B) + of "softcy": Rune(0x0044C) + of "ecy": Rune(0x0044D) + of "yucy": Rune(0x0044E) + of "yacy": Rune(0x0044F) + of "iocy": Rune(0x00451) + of "djcy": Rune(0x00452) + of "gjcy": Rune(0x00453) + of "jukcy": Rune(0x00454) + of "dscy": Rune(0x00455) + of "iukcy": Rune(0x00456) + of "yicy": Rune(0x00457) + of "jsercy": Rune(0x00458) + of "ljcy": Rune(0x00459) + of "njcy": Rune(0x0045A) + of "tshcy": Rune(0x0045B) + of "kjcy": Rune(0x0045C) + of "ubrcy": Rune(0x0045E) + of "dzcy": Rune(0x0045F) + of "ensp": Rune(0x02002) + of "emsp": Rune(0x02003) + of "emsp13": Rune(0x02004) + of "emsp14": Rune(0x02005) + of "numsp": Rune(0x02007) + of "puncsp": Rune(0x02008) + of "thinsp", "ThinSpace": Rune(0x02009) + of "hairsp", "VeryThinSpace": Rune(0x0200A) + of "ZeroWidthSpace", "NegativeVeryThinSpace", "NegativeThinSpace", + "NegativeMediumSpace", "NegativeThickSpace": Rune(0x0200B) + of "zwnj": Rune(0x0200C) + of "zwj": Rune(0x0200D) + of "lrm": Rune(0x0200E) + of "rlm": Rune(0x0200F) + of "hyphen", "dash": Rune(0x02010) + of "ndash": Rune(0x02013) + of "mdash": Rune(0x02014) + of "horbar": Rune(0x02015) + of "Verbar", "Vert": Rune(0x02016) + of "lsquo", "OpenCurlyQuote": Rune(0x02018) + of "rsquo", "rsquor", "CloseCurlyQuote": Rune(0x02019) + of "lsquor", "sbquo": Rune(0x0201A) + of "ldquo", "OpenCurlyDoubleQuote": Rune(0x0201C) + of "rdquo", "rdquor", "CloseCurlyDoubleQuote": Rune(0x0201D) + of "ldquor", "bdquo": Rune(0x0201E) + of "dagger": Rune(0x02020) + of "Dagger", "ddagger": Rune(0x02021) + of "bull", "bullet": Rune(0x02022) + of "nldr": Rune(0x02025) + of "hellip", "mldr": Rune(0x02026) + of "permil": Rune(0x02030) + of "pertenk": Rune(0x02031) + of "prime": Rune(0x02032) + of "Prime": Rune(0x02033) + of "tprime": Rune(0x02034) + of "bprime", "backprime": Rune(0x02035) + of "lsaquo": Rune(0x02039) + of "rsaquo": Rune(0x0203A) + of "oline": Rune(0x0203E) + of "caret": Rune(0x02041) + of "hybull": Rune(0x02043) + of "frasl": Rune(0x02044) + of "bsemi": Rune(0x0204F) + of "qprime": Rune(0x02057) + of "MediumSpace": Rune(0x0205F) + of "NoBreak": Rune(0x02060) + of "ApplyFunction", "af": Rune(0x02061) + of "InvisibleTimes", "it": Rune(0x02062) + of "InvisibleComma", "ic": Rune(0x02063) + of "euro": Rune(0x020AC) + of "tdot", "TripleDot": Rune(0x020DB) + of "DotDot": Rune(0x020DC) + of "Copf", "complexes": Rune(0x02102) + of "incare": Rune(0x02105) + of "gscr": Rune(0x0210A) + of "hamilt", "HilbertSpace", "Hscr": Rune(0x0210B) + of "Hfr", "Poincareplane": Rune(0x0210C) + of "quaternions", "Hopf": Rune(0x0210D) + of "planckh": Rune(0x0210E) + of "planck", "hbar", "plankv", "hslash": Rune(0x0210F) + of "Iscr", "imagline": Rune(0x02110) + of "image", "Im", "imagpart", "Ifr": Rune(0x02111) + of "Lscr", "lagran", "Laplacetrf": Rune(0x02112) + of "ell": Rune(0x02113) + of "Nopf", "naturals": Rune(0x02115) + of "numero": Rune(0x02116) + of "copysr": Rune(0x02117) + of "weierp", "wp": Rune(0x02118) + of "Popf", "primes": Rune(0x02119) + of "rationals", "Qopf": Rune(0x0211A) + of "Rscr", "realine": Rune(0x0211B) + of "real", "Re", "realpart", "Rfr": Rune(0x0211C) + of "reals", "Ropf": Rune(0x0211D) + of "rx": Rune(0x0211E) + of "trade", "TRADE": Rune(0x02122) + of "integers", "Zopf": Rune(0x02124) + of "ohm": Rune(0x02126) + of "mho": Rune(0x02127) + of "Zfr", "zeetrf": Rune(0x02128) + of "iiota": Rune(0x02129) + of "angst": Rune(0x0212B) + of "bernou", "Bernoullis", "Bscr": Rune(0x0212C) + of "Cfr", "Cayleys": Rune(0x0212D) + of "escr": Rune(0x0212F) + of "Escr", "expectation": Rune(0x02130) + of "Fscr", "Fouriertrf": Rune(0x02131) + of "phmmat", "Mellintrf", "Mscr": Rune(0x02133) + of "order", "orderof", "oscr": Rune(0x02134) + of "alefsym", "aleph": Rune(0x02135) + of "beth": Rune(0x02136) + of "gimel": Rune(0x02137) + of "daleth": Rune(0x02138) + of "CapitalDifferentialD", "DD": Rune(0x02145) + of "DifferentialD", "dd": Rune(0x02146) + of "ExponentialE", "exponentiale", "ee": Rune(0x02147) + of "ImaginaryI", "ii": Rune(0x02148) + of "frac13": Rune(0x02153) + of "frac23": Rune(0x02154) + of "frac15": Rune(0x02155) + of "frac25": Rune(0x02156) + of "frac35": Rune(0x02157) + of "frac45": Rune(0x02158) + of "frac16": Rune(0x02159) + of "frac56": Rune(0x0215A) + of "frac18": Rune(0x0215B) + of "frac38": Rune(0x0215C) + of "frac58": Rune(0x0215D) + of "frac78": Rune(0x0215E) + of "larr", "leftarrow", "LeftArrow", "slarr", + "ShortLeftArrow": Rune(0x02190) + of "uarr", "uparrow", "UpArrow", "ShortUpArrow": Rune(0x02191) + of "rarr", "rightarrow", "RightArrow", "srarr", + "ShortRightArrow": Rune(0x02192) + of "darr", "downarrow", "DownArrow", + "ShortDownArrow": Rune(0x02193) + of "harr", "leftrightarrow", "LeftRightArrow": Rune(0x02194) + of "varr", "updownarrow", "UpDownArrow": Rune(0x02195) + of "nwarr", "UpperLeftArrow", "nwarrow": Rune(0x02196) + of "nearr", "UpperRightArrow", "nearrow": Rune(0x02197) + of "searr", "searrow", "LowerRightArrow": Rune(0x02198) + of "swarr", "swarrow", "LowerLeftArrow": Rune(0x02199) + of "nlarr", "nleftarrow": Rune(0x0219A) + of "nrarr", "nrightarrow": Rune(0x0219B) + of "rarrw", "rightsquigarrow": Rune(0x0219D) + of "Larr", "twoheadleftarrow": Rune(0x0219E) + of "Uarr": Rune(0x0219F) + of "Rarr", "twoheadrightarrow": Rune(0x021A0) + of "Darr": Rune(0x021A1) + of "larrtl", "leftarrowtail": Rune(0x021A2) + of "rarrtl", "rightarrowtail": Rune(0x021A3) + of "LeftTeeArrow", "mapstoleft": Rune(0x021A4) + of "UpTeeArrow", "mapstoup": Rune(0x021A5) + of "map", "RightTeeArrow", "mapsto": Rune(0x021A6) + of "DownTeeArrow", "mapstodown": Rune(0x021A7) + of "larrhk", "hookleftarrow": Rune(0x021A9) + of "rarrhk", "hookrightarrow": Rune(0x021AA) + of "larrlp", "looparrowleft": Rune(0x021AB) + of "rarrlp", "looparrowright": Rune(0x021AC) + of "harrw", "leftrightsquigarrow": Rune(0x021AD) + of "nharr", "nleftrightarrow": Rune(0x021AE) + of "lsh", "Lsh": Rune(0x021B0) + of "rsh", "Rsh": Rune(0x021B1) + of "ldsh": Rune(0x021B2) + of "rdsh": Rune(0x021B3) + of "crarr": Rune(0x021B5) + of "cularr", "curvearrowleft": Rune(0x021B6) + of "curarr", "curvearrowright": Rune(0x021B7) + of "olarr", "circlearrowleft": Rune(0x021BA) + of "orarr", "circlearrowright": Rune(0x021BB) + of "lharu", "LeftVector", "leftharpoonup": Rune(0x021BC) + of "lhard", "leftharpoondown", "DownLeftVector": Rune(0x021BD) + of "uharr", "upharpoonright", "RightUpVector": Rune(0x021BE) + of "uharl", "upharpoonleft", "LeftUpVector": Rune(0x021BF) + of "rharu", "RightVector", "rightharpoonup": Rune(0x021C0) + of "rhard", "rightharpoondown", "DownRightVector": Rune(0x021C1) + of "dharr", "RightDownVector", "downharpoonright": Rune(0x021C2) + of "dharl", "LeftDownVector", "downharpoonleft": Rune(0x021C3) + of "rlarr", "rightleftarrows", "RightArrowLeftArrow": Rune(0x021C4) + of "udarr", "UpArrowDownArrow": Rune(0x021C5) + of "lrarr", "leftrightarrows", "LeftArrowRightArrow": Rune(0x021C6) + of "llarr", "leftleftarrows": Rune(0x021C7) + of "uuarr", "upuparrows": Rune(0x021C8) + of "rrarr", "rightrightarrows": Rune(0x021C9) + of "ddarr", "downdownarrows": Rune(0x021CA) + of "lrhar", "ReverseEquilibrium", + "leftrightharpoons": Rune(0x021CB) + of "rlhar", "rightleftharpoons", "Equilibrium": Rune(0x021CC) + of "nlArr", "nLeftarrow": Rune(0x021CD) + of "nhArr", "nLeftrightarrow": Rune(0x021CE) + of "nrArr", "nRightarrow": Rune(0x021CF) + of "lArr", "Leftarrow", "DoubleLeftArrow": Rune(0x021D0) + of "uArr", "Uparrow", "DoubleUpArrow": Rune(0x021D1) + of "rArr", "Rightarrow", "Implies", + "DoubleRightArrow": Rune(0x021D2) + of "dArr", "Downarrow", "DoubleDownArrow": Rune(0x021D3) + of "hArr", "Leftrightarrow", "DoubleLeftRightArrow", + "iff": Rune(0x021D4) + of "vArr", "Updownarrow", "DoubleUpDownArrow": Rune(0x021D5) + of "nwArr": Rune(0x021D6) + of "neArr": Rune(0x021D7) + of "seArr": Rune(0x021D8) + of "swArr": Rune(0x021D9) + of "lAarr", "Lleftarrow": Rune(0x021DA) + of "rAarr", "Rrightarrow": Rune(0x021DB) + of "zigrarr": Rune(0x021DD) + of "larrb", "LeftArrowBar": Rune(0x021E4) + of "rarrb", "RightArrowBar": Rune(0x021E5) + of "duarr", "DownArrowUpArrow": Rune(0x021F5) + of "loarr": Rune(0x021FD) + of "roarr": Rune(0x021FE) + of "hoarr": Rune(0x021FF) + of "forall", "ForAll": Rune(0x02200) + of "comp", "complement": Rune(0x02201) + of "part", "PartialD": Rune(0x02202) + of "exist", "Exists": Rune(0x02203) + of "nexist", "NotExists", "nexists": Rune(0x02204) + of "empty", "emptyset", "emptyv", "varnothing": Rune(0x02205) + of "nabla", "Del": Rune(0x02207) + of "isin", "isinv", "Element", "in": Rune(0x02208) + of "notin", "NotElement", "notinva": Rune(0x02209) + of "niv", "ReverseElement", "ni", "SuchThat": Rune(0x0220B) + of "notni", "notniva", "NotReverseElement": Rune(0x0220C) + of "prod", "Product": Rune(0x0220F) + of "coprod", "Coproduct": Rune(0x02210) + of "sum", "Sum": Rune(0x02211) + of "minus": Rune(0x02212) + of "mnplus", "mp", "MinusPlus": Rune(0x02213) + of "plusdo", "dotplus": Rune(0x02214) + of "setmn", "setminus", "Backslash", "ssetmn", + "smallsetminus": Rune(0x02216) + of "lowast": Rune(0x02217) + of "compfn", "SmallCircle": Rune(0x02218) + of "radic", "Sqrt": Rune(0x0221A) + of "prop", "propto", "Proportional", "vprop", + "varpropto": Rune(0x0221D) + of "infin": Rune(0x0221E) + of "angrt": Rune(0x0221F) + of "ang", "angle": Rune(0x02220) + of "angmsd", "measuredangle": Rune(0x02221) + of "angsph": Rune(0x02222) + of "mid", "VerticalBar", "smid", "shortmid": Rune(0x02223) + of "nmid", "NotVerticalBar", "nsmid", "nshortmid": Rune(0x02224) + of "par", "parallel", "DoubleVerticalBar", "spar", + "shortparallel": Rune(0x02225) + of "npar", "nparallel", "NotDoubleVerticalBar", "nspar", + "nshortparallel": Rune(0x02226) + of "and", "wedge": Rune(0x02227) + of "or", "vee": Rune(0x02228) + of "cap": Rune(0x02229) + of "cup": Rune(0x0222A) + of "int", "Integral": Rune(0x0222B) + of "Int": Rune(0x0222C) + of "tint", "iiint": Rune(0x0222D) + of "conint", "oint", "ContourIntegral": Rune(0x0222E) + of "Conint", "DoubleContourIntegral": Rune(0x0222F) + of "Cconint": Rune(0x02230) + of "cwint": Rune(0x02231) + of "cwconint", "ClockwiseContourIntegral": Rune(0x02232) + of "awconint", "CounterClockwiseContourIntegral": Rune(0x02233) + of "there4", "therefore", "Therefore": Rune(0x02234) + of "becaus", "because", "Because": Rune(0x02235) + of "ratio": Rune(0x02236) + of "Colon", "Proportion": Rune(0x02237) + of "minusd", "dotminus": Rune(0x02238) + of "mDDot": Rune(0x0223A) + of "homtht": Rune(0x0223B) + of "sim", "Tilde", "thksim", "thicksim": Rune(0x0223C) + of "bsim", "backsim": Rune(0x0223D) + of "ac", "mstpos": Rune(0x0223E) + of "acd": Rune(0x0223F) + of "wreath", "VerticalTilde", "wr": Rune(0x02240) + of "nsim", "NotTilde": Rune(0x02241) + of "esim", "EqualTilde", "eqsim": Rune(0x02242) + of "sime", "TildeEqual", "simeq": Rune(0x02243) + of "nsime", "nsimeq", "NotTildeEqual": Rune(0x02244) + of "cong", "TildeFullEqual": Rune(0x02245) + of "simne": Rune(0x02246) + of "ncong", "NotTildeFullEqual": Rune(0x02247) + of "asymp", "ap", "TildeTilde", "approx", "thkap", + "thickapprox": Rune(0x02248) + of "nap", "NotTildeTilde", "napprox": Rune(0x02249) + of "ape", "approxeq": Rune(0x0224A) + of "apid": Rune(0x0224B) + of "bcong", "backcong": Rune(0x0224C) + of "asympeq", "CupCap": Rune(0x0224D) + of "bump", "HumpDownHump", "Bumpeq": Rune(0x0224E) + of "bumpe", "HumpEqual", "bumpeq": Rune(0x0224F) + of "esdot", "DotEqual", "doteq": Rune(0x02250) + of "eDot", "doteqdot": Rune(0x02251) + of "efDot", "fallingdotseq": Rune(0x02252) + of "erDot", "risingdotseq": Rune(0x02253) + of "colone", "coloneq", "Assign": Rune(0x02254) + of "ecolon", "eqcolon": Rune(0x02255) + of "ecir", "eqcirc": Rune(0x02256) + of "cire", "circeq": Rune(0x02257) + of "wedgeq": Rune(0x02259) + of "veeeq": Rune(0x0225A) + of "trie", "triangleq": Rune(0x0225C) + of "equest", "questeq": Rune(0x0225F) + of "ne", "NotEqual": Rune(0x02260) + of "equiv", "Congruent": Rune(0x02261) + of "nequiv", "NotCongruent": Rune(0x02262) + of "le", "leq": Rune(0x02264) + of "ge", "GreaterEqual", "geq": Rune(0x02265) + of "lE", "LessFullEqual", "leqq": Rune(0x02266) + of "gE", "GreaterFullEqual", "geqq": Rune(0x02267) + of "lnE", "lneqq": Rune(0x02268) + of "gnE", "gneqq": Rune(0x02269) + of "Lt", "NestedLessLess", "ll": Rune(0x0226A) + of "Gt", "NestedGreaterGreater", "gg": Rune(0x0226B) + of "twixt", "between": Rune(0x0226C) + of "NotCupCap": Rune(0x0226D) + of "nlt", "NotLess", "nless": Rune(0x0226E) + of "ngt", "NotGreater", "ngtr": Rune(0x0226F) + of "nle", "NotLessEqual", "nleq": Rune(0x02270) + of "nge", "NotGreaterEqual", "ngeq": Rune(0x02271) + of "lsim", "LessTilde", "lesssim": Rune(0x02272) + of "gsim", "gtrsim", "GreaterTilde": Rune(0x02273) + of "nlsim", "NotLessTilde": Rune(0x02274) + of "ngsim", "NotGreaterTilde": Rune(0x02275) + of "lg", "lessgtr", "LessGreater": Rune(0x02276) + of "gl", "gtrless", "GreaterLess": Rune(0x02277) + of "ntlg", "NotLessGreater": Rune(0x02278) + of "ntgl", "NotGreaterLess": Rune(0x02279) + of "pr", "Precedes", "prec": Rune(0x0227A) + of "sc", "Succeeds", "succ": Rune(0x0227B) + of "prcue", "PrecedesSlantEqual", "preccurlyeq": Rune(0x0227C) + of "sccue", "SucceedsSlantEqual", "succcurlyeq": Rune(0x0227D) + of "prsim", "precsim", "PrecedesTilde": Rune(0x0227E) + of "scsim", "succsim", "SucceedsTilde": Rune(0x0227F) + of "npr", "nprec", "NotPrecedes": Rune(0x02280) + of "nsc", "nsucc", "NotSucceeds": Rune(0x02281) + of "sub", "subset": Rune(0x02282) + of "sup", "supset", "Superset": Rune(0x02283) + of "nsub": Rune(0x02284) + of "nsup": Rune(0x02285) + of "sube", "SubsetEqual", "subseteq": Rune(0x02286) + of "supe", "supseteq", "SupersetEqual": Rune(0x02287) + of "nsube", "nsubseteq", "NotSubsetEqual": Rune(0x02288) + of "nsupe", "nsupseteq", "NotSupersetEqual": Rune(0x02289) + of "subne", "subsetneq": Rune(0x0228A) + of "supne", "supsetneq": Rune(0x0228B) + of "cupdot": Rune(0x0228D) + of "uplus", "UnionPlus": Rune(0x0228E) + of "sqsub", "SquareSubset", "sqsubset": Rune(0x0228F) + of "sqsup", "SquareSuperset", "sqsupset": Rune(0x02290) + of "sqsube", "SquareSubsetEqual", "sqsubseteq": Rune(0x02291) + of "sqsupe", "SquareSupersetEqual", "sqsupseteq": Rune(0x02292) + of "sqcap", "SquareIntersection": Rune(0x02293) + of "sqcup", "SquareUnion": Rune(0x02294) + of "oplus", "CirclePlus": Rune(0x02295) + of "ominus", "CircleMinus": Rune(0x02296) + of "otimes", "CircleTimes": Rune(0x02297) + of "osol": Rune(0x02298) + of "odot", "CircleDot": Rune(0x02299) + of "ocir", "circledcirc": Rune(0x0229A) + of "oast", "circledast": Rune(0x0229B) + of "odash", "circleddash": Rune(0x0229D) + of "plusb", "boxplus": Rune(0x0229E) + of "minusb", "boxminus": Rune(0x0229F) + of "timesb", "boxtimes": Rune(0x022A0) + of "sdotb", "dotsquare": Rune(0x022A1) + of "vdash", "RightTee": Rune(0x022A2) + of "dashv", "LeftTee": Rune(0x022A3) + of "top", "DownTee": Rune(0x022A4) + of "bottom", "bot", "perp", "UpTee": Rune(0x022A5) + of "models": Rune(0x022A7) + of "vDash", "DoubleRightTee": Rune(0x022A8) + of "Vdash": Rune(0x022A9) + of "Vvdash": Rune(0x022AA) + of "VDash": Rune(0x022AB) + of "nvdash": Rune(0x022AC) + of "nvDash": Rune(0x022AD) + of "nVdash": Rune(0x022AE) + of "nVDash": Rune(0x022AF) + of "prurel": Rune(0x022B0) + of "vltri", "vartriangleleft", "LeftTriangle": Rune(0x022B2) + of "vrtri", "vartriangleright", "RightTriangle": Rune(0x022B3) + of "ltrie", "trianglelefteq", "LeftTriangleEqual": Rune(0x022B4) + of "rtrie", "trianglerighteq", "RightTriangleEqual": Rune(0x022B5) + of "origof": Rune(0x022B6) + of "imof": Rune(0x022B7) + of "mumap", "multimap": Rune(0x022B8) + of "hercon": Rune(0x022B9) + of "intcal", "intercal": Rune(0x022BA) + of "veebar": Rune(0x022BB) + of "barvee": Rune(0x022BD) + of "angrtvb": Rune(0x022BE) + of "lrtri": Rune(0x022BF) + of "xwedge", "Wedge", "bigwedge": Rune(0x022C0) + of "xvee", "Vee", "bigvee": Rune(0x022C1) + of "xcap", "Intersection", "bigcap": Rune(0x022C2) + of "xcup", "Union", "bigcup": Rune(0x022C3) + of "diam", "diamond", "Diamond": Rune(0x022C4) + of "sdot": Rune(0x022C5) + of "sstarf", "Star": Rune(0x022C6) + of "divonx", "divideontimes": Rune(0x022C7) + of "bowtie": Rune(0x022C8) + of "ltimes": Rune(0x022C9) + of "rtimes": Rune(0x022CA) + of "lthree", "leftthreetimes": Rune(0x022CB) + of "rthree", "rightthreetimes": Rune(0x022CC) + of "bsime", "backsimeq": Rune(0x022CD) + of "cuvee", "curlyvee": Rune(0x022CE) + of "cuwed", "curlywedge": Rune(0x022CF) + of "Sub", "Subset": Rune(0x022D0) + of "Sup", "Supset": Rune(0x022D1) + of "Cap": Rune(0x022D2) + of "Cup": Rune(0x022D3) + of "fork", "pitchfork": Rune(0x022D4) + of "epar": Rune(0x022D5) + of "ltdot", "lessdot": Rune(0x022D6) + of "gtdot", "gtrdot": Rune(0x022D7) + of "Ll": Rune(0x022D8) + of "Gg", "ggg": Rune(0x022D9) + of "leg", "LessEqualGreater", "lesseqgtr": Rune(0x022DA) + of "gel", "gtreqless", "GreaterEqualLess": Rune(0x022DB) + of "cuepr", "curlyeqprec": Rune(0x022DE) + of "cuesc", "curlyeqsucc": Rune(0x022DF) + of "nprcue", "NotPrecedesSlantEqual": Rune(0x022E0) + of "nsccue", "NotSucceedsSlantEqual": Rune(0x022E1) + of "nsqsube", "NotSquareSubsetEqual": Rune(0x022E2) + of "nsqsupe", "NotSquareSupersetEqual": Rune(0x022E3) + of "lnsim": Rune(0x022E6) + of "gnsim": Rune(0x022E7) + of "prnsim", "precnsim": Rune(0x022E8) + of "scnsim", "succnsim": Rune(0x022E9) + of "nltri", "ntriangleleft", "NotLeftTriangle": Rune(0x022EA) + of "nrtri", "ntriangleright", "NotRightTriangle": Rune(0x022EB) + of "nltrie", "ntrianglelefteq", + "NotLeftTriangleEqual": Rune(0x022EC) + of "nrtrie", "ntrianglerighteq", + "NotRightTriangleEqual": Rune(0x022ED) + of "vellip": Rune(0x022EE) + of "ctdot": Rune(0x022EF) + of "utdot": Rune(0x022F0) + of "dtdot": Rune(0x022F1) + of "disin": Rune(0x022F2) + of "isinsv": Rune(0x022F3) + of "isins": Rune(0x022F4) + of "isindot": Rune(0x022F5) + of "notinvc": Rune(0x022F6) + of "notinvb": Rune(0x022F7) + of "isinE": Rune(0x022F9) + of "nisd": Rune(0x022FA) + of "xnis": Rune(0x022FB) + of "nis": Rune(0x022FC) + of "notnivc": Rune(0x022FD) + of "notnivb": Rune(0x022FE) + of "barwed", "barwedge": Rune(0x02305) + of "Barwed", "doublebarwedge": Rune(0x02306) + of "lceil", "LeftCeiling": Rune(0x02308) + of "rceil", "RightCeiling": Rune(0x02309) + of "lfloor", "LeftFloor": Rune(0x0230A) + of "rfloor", "RightFloor": Rune(0x0230B) + of "drcrop": Rune(0x0230C) + of "dlcrop": Rune(0x0230D) + of "urcrop": Rune(0x0230E) + of "ulcrop": Rune(0x0230F) + of "bnot": Rune(0x02310) + of "profline": Rune(0x02312) + of "profsurf": Rune(0x02313) + of "telrec": Rune(0x02315) + of "target": Rune(0x02316) + of "ulcorn", "ulcorner": Rune(0x0231C) + of "urcorn", "urcorner": Rune(0x0231D) + of "dlcorn", "llcorner": Rune(0x0231E) + of "drcorn", "lrcorner": Rune(0x0231F) + of "frown", "sfrown": Rune(0x02322) + of "smile", "ssmile": Rune(0x02323) + of "cylcty": Rune(0x0232D) + of "profalar": Rune(0x0232E) + of "topbot": Rune(0x02336) + of "ovbar": Rune(0x0233D) + of "solbar": Rune(0x0233F) + of "angzarr": Rune(0x0237C) + of "lmoust", "lmoustache": Rune(0x023B0) + of "rmoust", "rmoustache": Rune(0x023B1) + of "tbrk", "OverBracket": Rune(0x023B4) + of "bbrk", "UnderBracket": Rune(0x023B5) + of "bbrktbrk": Rune(0x023B6) + of "OverParenthesis": Rune(0x023DC) + of "UnderParenthesis": Rune(0x023DD) + of "OverBrace": Rune(0x023DE) + of "UnderBrace": Rune(0x023DF) + of "trpezium": Rune(0x023E2) + of "elinters": Rune(0x023E7) + of "blank": Rune(0x02423) + of "oS", "circledS": Rune(0x024C8) + of "boxh", "HorizontalLine": Rune(0x02500) + of "boxv": Rune(0x02502) + of "boxdr": Rune(0x0250C) + of "boxdl": Rune(0x02510) + of "boxur": Rune(0x02514) + of "boxul": Rune(0x02518) + of "boxvr": Rune(0x0251C) + of "boxvl": Rune(0x02524) + of "boxhd": Rune(0x0252C) + of "boxhu": Rune(0x02534) + of "boxvh": Rune(0x0253C) + of "boxH": Rune(0x02550) + of "boxV": Rune(0x02551) + of "boxdR": Rune(0x02552) + of "boxDr": Rune(0x02553) + of "boxDR": Rune(0x02554) + of "boxdL": Rune(0x02555) + of "boxDl": Rune(0x02556) + of "boxDL": Rune(0x02557) + of "boxuR": Rune(0x02558) + of "boxUr": Rune(0x02559) + of "boxUR": Rune(0x0255A) + of "boxuL": Rune(0x0255B) + of "boxUl": Rune(0x0255C) + of "boxUL": Rune(0x0255D) + of "boxvR": Rune(0x0255E) + of "boxVr": Rune(0x0255F) + of "boxVR": Rune(0x02560) + of "boxvL": Rune(0x02561) + of "boxVl": Rune(0x02562) + of "boxVL": Rune(0x02563) + of "boxHd": Rune(0x02564) + of "boxhD": Rune(0x02565) + of "boxHD": Rune(0x02566) + of "boxHu": Rune(0x02567) + of "boxhU": Rune(0x02568) + of "boxHU": Rune(0x02569) + of "boxvH": Rune(0x0256A) + of "boxVh": Rune(0x0256B) + of "boxVH": Rune(0x0256C) + of "uhblk": Rune(0x02580) + of "lhblk": Rune(0x02584) + of "block": Rune(0x02588) + of "blk14": Rune(0x02591) + of "blk12": Rune(0x02592) + of "blk34": Rune(0x02593) + of "squ", "square", "Square": Rune(0x025A1) + of "squf", "squarf", "blacksquare", + "FilledVerySmallSquare": Rune(0x025AA) + of "EmptyVerySmallSquare": Rune(0x025AB) + of "rect": Rune(0x025AD) + of "marker": Rune(0x025AE) + of "fltns": Rune(0x025B1) + of "xutri", "bigtriangleup": Rune(0x025B3) + of "utrif", "blacktriangle": Rune(0x025B4) + of "utri", "triangle": Rune(0x025B5) + of "rtrif", "blacktriangleright": Rune(0x025B8) + of "rtri", "triangleright": Rune(0x025B9) + of "xdtri", "bigtriangledown": Rune(0x025BD) + of "dtrif", "blacktriangledown": Rune(0x025BE) + of "dtri", "triangledown": Rune(0x025BF) + of "ltrif", "blacktriangleleft": Rune(0x025C2) + of "ltri", "triangleleft": Rune(0x025C3) + of "loz", "lozenge": Rune(0x025CA) + of "cir": Rune(0x025CB) + of "tridot": Rune(0x025EC) + of "xcirc", "bigcirc": Rune(0x025EF) + of "ultri": Rune(0x025F8) + of "urtri": Rune(0x025F9) + of "lltri": Rune(0x025FA) + of "EmptySmallSquare": Rune(0x025FB) + of "FilledSmallSquare": Rune(0x025FC) + of "starf", "bigstar": Rune(0x02605) + of "star": Rune(0x02606) + of "phone": Rune(0x0260E) + of "female": Rune(0x02640) + of "male": Rune(0x02642) + of "spades", "spadesuit": Rune(0x02660) + of "clubs", "clubsuit": Rune(0x02663) + of "hearts", "heartsuit": Rune(0x02665) + of "diams", "diamondsuit": Rune(0x02666) + of "sung": Rune(0x0266A) + of "flat": Rune(0x0266D) + of "natur", "natural": Rune(0x0266E) + of "sharp": Rune(0x0266F) + of "check", "checkmark": Rune(0x02713) + of "cross": Rune(0x02717) + of "malt", "maltese": Rune(0x02720) + of "sext": Rune(0x02736) + of "VerticalSeparator": Rune(0x02758) + of "lbbrk": Rune(0x02772) + of "rbbrk": Rune(0x02773) + of "lobrk", "LeftDoubleBracket": Rune(0x027E6) + of "robrk", "RightDoubleBracket": Rune(0x027E7) + of "lang", "LeftAngleBracket", "langle": Rune(0x027E8) + of "rang", "RightAngleBracket", "rangle": Rune(0x027E9) + of "Lang": Rune(0x027EA) + of "Rang": Rune(0x027EB) + of "loang": Rune(0x027EC) + of "roang": Rune(0x027ED) + of "xlarr", "longleftarrow", "LongLeftArrow": Rune(0x027F5) + of "xrarr", "longrightarrow", "LongRightArrow": Rune(0x027F6) + of "xharr", "longleftrightarrow", + "LongLeftRightArrow": Rune(0x027F7) + of "xlArr", "Longleftarrow", "DoubleLongLeftArrow": Rune(0x027F8) + of "xrArr", "Longrightarrow", "DoubleLongRightArrow": Rune(0x027F9) + of "xhArr", "Longleftrightarrow", + "DoubleLongLeftRightArrow": Rune(0x027FA) + of "xmap", "longmapsto": Rune(0x027FC) + of "dzigrarr": Rune(0x027FF) + of "nvlArr": Rune(0x02902) + of "nvrArr": Rune(0x02903) + of "nvHarr": Rune(0x02904) + of "Map": Rune(0x02905) + of "lbarr": Rune(0x0290C) + of "rbarr", "bkarow": Rune(0x0290D) + of "lBarr": Rune(0x0290E) + of "rBarr", "dbkarow": Rune(0x0290F) + of "RBarr", "drbkarow": Rune(0x02910) + of "DDotrahd": Rune(0x02911) + of "UpArrowBar": Rune(0x02912) + of "DownArrowBar": Rune(0x02913) + of "Rarrtl": Rune(0x02916) + of "latail": Rune(0x02919) + of "ratail": Rune(0x0291A) + of "lAtail": Rune(0x0291B) + of "rAtail": Rune(0x0291C) + of "larrfs": Rune(0x0291D) + of "rarrfs": Rune(0x0291E) + of "larrbfs": Rune(0x0291F) + of "rarrbfs": Rune(0x02920) + of "nwarhk": Rune(0x02923) + of "nearhk": Rune(0x02924) + of "searhk", "hksearow": Rune(0x02925) + of "swarhk", "hkswarow": Rune(0x02926) + of "nwnear": Rune(0x02927) + of "nesear", "toea": Rune(0x02928) + of "seswar", "tosa": Rune(0x02929) + of "swnwar": Rune(0x0292A) + of "rarrc": Rune(0x02933) + of "cudarrr": Rune(0x02935) + of "ldca": Rune(0x02936) + of "rdca": Rune(0x02937) + of "cudarrl": Rune(0x02938) + of "larrpl": Rune(0x02939) + of "curarrm": Rune(0x0293C) + of "cularrp": Rune(0x0293D) + of "rarrpl": Rune(0x02945) + of "harrcir": Rune(0x02948) + of "Uarrocir": Rune(0x02949) + of "lurdshar": Rune(0x0294A) + of "ldrushar": Rune(0x0294B) + of "LeftRightVector": Rune(0x0294E) + of "RightUpDownVector": Rune(0x0294F) + of "DownLeftRightVector": Rune(0x02950) + of "LeftUpDownVector": Rune(0x02951) + of "LeftVectorBar": Rune(0x02952) + of "RightVectorBar": Rune(0x02953) + of "RightUpVectorBar": Rune(0x02954) + of "RightDownVectorBar": Rune(0x02955) + of "DownLeftVectorBar": Rune(0x02956) + of "DownRightVectorBar": Rune(0x02957) + of "LeftUpVectorBar": Rune(0x02958) + of "LeftDownVectorBar": Rune(0x02959) + of "LeftTeeVector": Rune(0x0295A) + of "RightTeeVector": Rune(0x0295B) + of "RightUpTeeVector": Rune(0x0295C) + of "RightDownTeeVector": Rune(0x0295D) + of "DownLeftTeeVector": Rune(0x0295E) + of "DownRightTeeVector": Rune(0x0295F) + of "LeftUpTeeVector": Rune(0x02960) + of "LeftDownTeeVector": Rune(0x02961) + of "lHar": Rune(0x02962) + of "uHar": Rune(0x02963) + of "rHar": Rune(0x02964) + of "dHar": Rune(0x02965) + of "luruhar": Rune(0x02966) + of "ldrdhar": Rune(0x02967) + of "ruluhar": Rune(0x02968) + of "rdldhar": Rune(0x02969) + of "lharul": Rune(0x0296A) + of "llhard": Rune(0x0296B) + of "rharul": Rune(0x0296C) + of "lrhard": Rune(0x0296D) + of "udhar", "UpEquilibrium": Rune(0x0296E) + of "duhar", "ReverseUpEquilibrium": Rune(0x0296F) + of "RoundImplies": Rune(0x02970) + of "erarr": Rune(0x02971) + of "simrarr": Rune(0x02972) + of "larrsim": Rune(0x02973) + of "rarrsim": Rune(0x02974) + of "rarrap": Rune(0x02975) + of "ltlarr": Rune(0x02976) + of "gtrarr": Rune(0x02978) + of "subrarr": Rune(0x02979) + of "suplarr": Rune(0x0297B) + of "lfisht": Rune(0x0297C) + of "rfisht": Rune(0x0297D) + of "ufisht": Rune(0x0297E) + of "dfisht": Rune(0x0297F) + of "lopar": Rune(0x02985) + of "ropar": Rune(0x02986) + of "lbrke": Rune(0x0298B) + of "rbrke": Rune(0x0298C) + of "lbrkslu": Rune(0x0298D) + of "rbrksld": Rune(0x0298E) + of "lbrksld": Rune(0x0298F) + of "rbrkslu": Rune(0x02990) + of "langd": Rune(0x02991) + of "rangd": Rune(0x02992) + of "lparlt": Rune(0x02993) + of "rpargt": Rune(0x02994) + of "gtlPar": Rune(0x02995) + of "ltrPar": Rune(0x02996) + of "vzigzag": Rune(0x0299A) + of "vangrt": Rune(0x0299C) + of "angrtvbd": Rune(0x0299D) + of "ange": Rune(0x029A4) + of "range": Rune(0x029A5) + of "dwangle": Rune(0x029A6) + of "uwangle": Rune(0x029A7) + of "angmsdaa": Rune(0x029A8) + of "angmsdab": Rune(0x029A9) + of "angmsdac": Rune(0x029AA) + of "angmsdad": Rune(0x029AB) + of "angmsdae": Rune(0x029AC) + of "angmsdaf": Rune(0x029AD) + of "angmsdag": Rune(0x029AE) + of "angmsdah": Rune(0x029AF) + of "bemptyv": Rune(0x029B0) + of "demptyv": Rune(0x029B1) + of "cemptyv": Rune(0x029B2) + of "raemptyv": Rune(0x029B3) + of "laemptyv": Rune(0x029B4) + of "ohbar": Rune(0x029B5) + of "omid": Rune(0x029B6) + of "opar": Rune(0x029B7) + of "operp": Rune(0x029B9) + of "olcross": Rune(0x029BB) + of "odsold": Rune(0x029BC) + of "olcir": Rune(0x029BE) + of "ofcir": Rune(0x029BF) + of "olt": Rune(0x029C0) + of "ogt": Rune(0x029C1) + of "cirscir": Rune(0x029C2) + of "cirE": Rune(0x029C3) + of "solb": Rune(0x029C4) + of "bsolb": Rune(0x029C5) + of "boxbox": Rune(0x029C9) + of "trisb": Rune(0x029CD) + of "rtriltri": Rune(0x029CE) + of "LeftTriangleBar": Rune(0x029CF) + of "RightTriangleBar": Rune(0x029D0) + of "race": Rune(0x029DA) + of "iinfin": Rune(0x029DC) + of "infintie": Rune(0x029DD) + of "nvinfin": Rune(0x029DE) + of "eparsl": Rune(0x029E3) + of "smeparsl": Rune(0x029E4) + of "eqvparsl": Rune(0x029E5) + of "lozf", "blacklozenge": Rune(0x029EB) + of "RuleDelayed": Rune(0x029F4) + of "dsol": Rune(0x029F6) + of "xodot", "bigodot": Rune(0x02A00) + of "xoplus", "bigoplus": Rune(0x02A01) + of "xotime", "bigotimes": Rune(0x02A02) + of "xuplus", "biguplus": Rune(0x02A04) + of "xsqcup", "bigsqcup": Rune(0x02A06) + of "qint", "iiiint": Rune(0x02A0C) + of "fpartint": Rune(0x02A0D) + of "cirfnint": Rune(0x02A10) + of "awint": Rune(0x02A11) + of "rppolint": Rune(0x02A12) + of "scpolint": Rune(0x02A13) + of "npolint": Rune(0x02A14) + of "pointint": Rune(0x02A15) + of "quatint": Rune(0x02A16) + of "intlarhk": Rune(0x02A17) + of "pluscir": Rune(0x02A22) + of "plusacir": Rune(0x02A23) + of "simplus": Rune(0x02A24) + of "plusdu": Rune(0x02A25) + of "plussim": Rune(0x02A26) + of "plustwo": Rune(0x02A27) + of "mcomma": Rune(0x02A29) + of "minusdu": Rune(0x02A2A) + of "loplus": Rune(0x02A2D) + of "roplus": Rune(0x02A2E) + of "Cross": Rune(0x02A2F) + of "timesd": Rune(0x02A30) + of "timesbar": Rune(0x02A31) + of "smashp": Rune(0x02A33) + of "lotimes": Rune(0x02A34) + of "rotimes": Rune(0x02A35) + of "otimesas": Rune(0x02A36) + of "Otimes": Rune(0x02A37) + of "odiv": Rune(0x02A38) + of "triplus": Rune(0x02A39) + of "triminus": Rune(0x02A3A) + of "tritime": Rune(0x02A3B) + of "iprod", "intprod": Rune(0x02A3C) + of "amalg": Rune(0x02A3F) + of "capdot": Rune(0x02A40) + of "ncup": Rune(0x02A42) + of "ncap": Rune(0x02A43) + of "capand": Rune(0x02A44) + of "cupor": Rune(0x02A45) + of "cupcap": Rune(0x02A46) + of "capcup": Rune(0x02A47) + of "cupbrcap": Rune(0x02A48) + of "capbrcup": Rune(0x02A49) + of "cupcup": Rune(0x02A4A) + of "capcap": Rune(0x02A4B) + of "ccups": Rune(0x02A4C) + of "ccaps": Rune(0x02A4D) + of "ccupssm": Rune(0x02A50) + of "And": Rune(0x02A53) + of "Or": Rune(0x02A54) + of "andand": Rune(0x02A55) + of "oror": Rune(0x02A56) + of "orslope": Rune(0x02A57) + of "andslope": Rune(0x02A58) + of "andv": Rune(0x02A5A) + of "orv": Rune(0x02A5B) + of "andd": Rune(0x02A5C) + of "ord": Rune(0x02A5D) + of "wedbar": Rune(0x02A5F) + of "sdote": Rune(0x02A66) + of "simdot": Rune(0x02A6A) + of "congdot": Rune(0x02A6D) + of "easter": Rune(0x02A6E) + of "apacir": Rune(0x02A6F) + of "apE": Rune(0x02A70) + of "eplus": Rune(0x02A71) + of "pluse": Rune(0x02A72) + of "Esim": Rune(0x02A73) + of "Colone": Rune(0x02A74) + of "Equal": Rune(0x02A75) + of "eDDot", "ddotseq": Rune(0x02A77) + of "equivDD": Rune(0x02A78) + of "ltcir": Rune(0x02A79) + of "gtcir": Rune(0x02A7A) + of "ltquest": Rune(0x02A7B) + of "gtquest": Rune(0x02A7C) + of "les", "LessSlantEqual", "leqslant": Rune(0x02A7D) + of "ges", "GreaterSlantEqual", "geqslant": Rune(0x02A7E) + of "lesdot": Rune(0x02A7F) + of "gesdot": Rune(0x02A80) + of "lesdoto": Rune(0x02A81) + of "gesdoto": Rune(0x02A82) + of "lesdotor": Rune(0x02A83) + of "gesdotol": Rune(0x02A84) + of "lap", "lessapprox": Rune(0x02A85) + of "gap", "gtrapprox": Rune(0x02A86) + of "lne", "lneq": Rune(0x02A87) + of "gne", "gneq": Rune(0x02A88) + of "lnap", "lnapprox": Rune(0x02A89) + of "gnap", "gnapprox": Rune(0x02A8A) + of "lEg", "lesseqqgtr": Rune(0x02A8B) + of "gEl", "gtreqqless": Rune(0x02A8C) + of "lsime": Rune(0x02A8D) + of "gsime": Rune(0x02A8E) + of "lsimg": Rune(0x02A8F) + of "gsiml": Rune(0x02A90) + of "lgE": Rune(0x02A91) + of "glE": Rune(0x02A92) + of "lesges": Rune(0x02A93) + of "gesles": Rune(0x02A94) + of "els", "eqslantless": Rune(0x02A95) + of "egs", "eqslantgtr": Rune(0x02A96) + of "elsdot": Rune(0x02A97) + of "egsdot": Rune(0x02A98) + of "el": Rune(0x02A99) + of "eg": Rune(0x02A9A) + of "siml": Rune(0x02A9D) + of "simg": Rune(0x02A9E) + of "simlE": Rune(0x02A9F) + of "simgE": Rune(0x02AA0) + of "LessLess": Rune(0x02AA1) + of "GreaterGreater": Rune(0x02AA2) + of "glj": Rune(0x02AA4) + of "gla": Rune(0x02AA5) + of "ltcc": Rune(0x02AA6) + of "gtcc": Rune(0x02AA7) + of "lescc": Rune(0x02AA8) + of "gescc": Rune(0x02AA9) + of "smt": Rune(0x02AAA) + of "lat": Rune(0x02AAB) + of "smte": Rune(0x02AAC) + of "late": Rune(0x02AAD) + of "bumpE": Rune(0x02AAE) + of "pre", "preceq", "PrecedesEqual": Rune(0x02AAF) + of "sce", "succeq", "SucceedsEqual": Rune(0x02AB0) + of "prE": Rune(0x02AB3) + of "scE": Rune(0x02AB4) + of "prnE", "precneqq": Rune(0x02AB5) + of "scnE", "succneqq": Rune(0x02AB6) + of "prap", "precapprox": Rune(0x02AB7) + of "scap", "succapprox": Rune(0x02AB8) + of "prnap", "precnapprox": Rune(0x02AB9) + of "scnap", "succnapprox": Rune(0x02ABA) + of "Pr": Rune(0x02ABB) + of "Sc": Rune(0x02ABC) + of "subdot": Rune(0x02ABD) + of "supdot": Rune(0x02ABE) + of "subplus": Rune(0x02ABF) + of "supplus": Rune(0x02AC0) + of "submult": Rune(0x02AC1) + of "supmult": Rune(0x02AC2) + of "subedot": Rune(0x02AC3) + of "supedot": Rune(0x02AC4) + of "subE", "subseteqq": Rune(0x02AC5) + of "supE", "supseteqq": Rune(0x02AC6) + of "subsim": Rune(0x02AC7) + of "supsim": Rune(0x02AC8) + of "subnE", "subsetneqq": Rune(0x02ACB) + of "supnE", "supsetneqq": Rune(0x02ACC) + of "csub": Rune(0x02ACF) + of "csup": Rune(0x02AD0) + of "csube": Rune(0x02AD1) + of "csupe": Rune(0x02AD2) + of "subsup": Rune(0x02AD3) + of "supsub": Rune(0x02AD4) + of "subsub": Rune(0x02AD5) + of "supsup": Rune(0x02AD6) + of "suphsub": Rune(0x02AD7) + of "supdsub": Rune(0x02AD8) + of "forkv": Rune(0x02AD9) + of "topfork": Rune(0x02ADA) + of "mlcp": Rune(0x02ADB) + of "Dashv", "DoubleLeftTee": Rune(0x02AE4) + of "Vdashl": Rune(0x02AE6) + of "Barv": Rune(0x02AE7) + of "vBar": Rune(0x02AE8) + of "vBarv": Rune(0x02AE9) + of "Vbar": Rune(0x02AEB) + of "Not": Rune(0x02AEC) + of "bNot": Rune(0x02AED) + of "rnmid": Rune(0x02AEE) + of "cirmid": Rune(0x02AEF) + of "midcir": Rune(0x02AF0) + of "topcir": Rune(0x02AF1) + of "nhpar": Rune(0x02AF2) + of "parsim": Rune(0x02AF3) + of "parsl": Rune(0x02AFD) + of "fflig": Rune(0x0FB00) + of "filig": Rune(0x0FB01) + of "fllig": Rune(0x0FB02) + of "ffilig": Rune(0x0FB03) + of "ffllig": Rune(0x0FB04) + of "Ascr": Rune(0x1D49C) + of "Cscr": Rune(0x1D49E) + of "Dscr": Rune(0x1D49F) + of "Gscr": Rune(0x1D4A2) + of "Jscr": Rune(0x1D4A5) + of "Kscr": Rune(0x1D4A6) + of "Nscr": Rune(0x1D4A9) + of "Oscr": Rune(0x1D4AA) + of "Pscr": Rune(0x1D4AB) + of "Qscr": Rune(0x1D4AC) + of "Sscr": Rune(0x1D4AE) + of "Tscr": Rune(0x1D4AF) + of "Uscr": Rune(0x1D4B0) + of "Vscr": Rune(0x1D4B1) + of "Wscr": Rune(0x1D4B2) + of "Xscr": Rune(0x1D4B3) + of "Yscr": Rune(0x1D4B4) + of "Zscr": Rune(0x1D4B5) + of "ascr": Rune(0x1D4B6) + of "bscr": Rune(0x1D4B7) + of "cscr": Rune(0x1D4B8) + of "dscr": Rune(0x1D4B9) + of "fscr": Rune(0x1D4BB) + of "hscr": Rune(0x1D4BD) + of "iscr": Rune(0x1D4BE) + of "jscr": Rune(0x1D4BF) + of "kscr": Rune(0x1D4C0) + of "lscr": Rune(0x1D4C1) + of "mscr": Rune(0x1D4C2) + of "nscr": Rune(0x1D4C3) + of "pscr": Rune(0x1D4C5) + of "qscr": Rune(0x1D4C6) + of "rscr": Rune(0x1D4C7) + of "sscr": Rune(0x1D4C8) + of "tscr": Rune(0x1D4C9) + of "uscr": Rune(0x1D4CA) + of "vscr": Rune(0x1D4CB) + of "wscr": Rune(0x1D4CC) + of "xscr": Rune(0x1D4CD) + of "yscr": Rune(0x1D4CE) + of "zscr": Rune(0x1D4CF) + of "Afr": Rune(0x1D504) + of "Bfr": Rune(0x1D505) + of "Dfr": Rune(0x1D507) + of "Efr": Rune(0x1D508) + of "Ffr": Rune(0x1D509) + of "Gfr": Rune(0x1D50A) + of "Jfr": Rune(0x1D50D) + of "Kfr": Rune(0x1D50E) + of "Lfr": Rune(0x1D50F) + of "Mfr": Rune(0x1D510) + of "Nfr": Rune(0x1D511) + of "Ofr": Rune(0x1D512) + of "Pfr": Rune(0x1D513) + of "Qfr": Rune(0x1D514) + of "Sfr": Rune(0x1D516) + of "Tfr": Rune(0x1D517) + of "Ufr": Rune(0x1D518) + of "Vfr": Rune(0x1D519) + of "Wfr": Rune(0x1D51A) + of "Xfr": Rune(0x1D51B) + of "Yfr": Rune(0x1D51C) + of "afr": Rune(0x1D51E) + of "bfr": Rune(0x1D51F) + of "cfr": Rune(0x1D520) + of "dfr": Rune(0x1D521) + of "efr": Rune(0x1D522) + of "ffr": Rune(0x1D523) + of "gfr": Rune(0x1D524) + of "hfr": Rune(0x1D525) + of "ifr": Rune(0x1D526) + of "jfr": Rune(0x1D527) + of "kfr": Rune(0x1D528) + of "lfr": Rune(0x1D529) + of "mfr": Rune(0x1D52A) + of "nfr": Rune(0x1D52B) + of "ofr": Rune(0x1D52C) + of "pfr": Rune(0x1D52D) + of "qfr": Rune(0x1D52E) + of "rfr": Rune(0x1D52F) + of "sfr": Rune(0x1D530) + of "tfr": Rune(0x1D531) + of "ufr": Rune(0x1D532) + of "vfr": Rune(0x1D533) + of "wfr": Rune(0x1D534) + of "xfr": Rune(0x1D535) + of "yfr": Rune(0x1D536) + of "zfr": Rune(0x1D537) + of "Aopf": Rune(0x1D538) + of "Bopf": Rune(0x1D539) + of "Dopf": Rune(0x1D53B) + of "Eopf": Rune(0x1D53C) + of "Fopf": Rune(0x1D53D) + of "Gopf": Rune(0x1D53E) + of "Iopf": Rune(0x1D540) + of "Jopf": Rune(0x1D541) + of "Kopf": Rune(0x1D542) + of "Lopf": Rune(0x1D543) + of "Mopf": Rune(0x1D544) + of "Oopf": Rune(0x1D546) + of "Sopf": Rune(0x1D54A) + of "Topf": Rune(0x1D54B) + of "Uopf": Rune(0x1D54C) + of "Vopf": Rune(0x1D54D) + of "Wopf": Rune(0x1D54E) + of "Xopf": Rune(0x1D54F) + of "Yopf": Rune(0x1D550) + of "aopf": Rune(0x1D552) + of "bopf": Rune(0x1D553) + of "copf": Rune(0x1D554) + of "dopf": Rune(0x1D555) + of "eopf": Rune(0x1D556) + of "fopf": Rune(0x1D557) + of "gopf": Rune(0x1D558) + of "hopf": Rune(0x1D559) + of "iopf": Rune(0x1D55A) + of "jopf": Rune(0x1D55B) + of "kopf": Rune(0x1D55C) + of "lopf": Rune(0x1D55D) + of "mopf": Rune(0x1D55E) + of "nopf": Rune(0x1D55F) + of "oopf": Rune(0x1D560) + of "popf": Rune(0x1D561) + of "qopf": Rune(0x1D562) + of "ropf": Rune(0x1D563) + of "sopf": Rune(0x1D564) + of "topf": Rune(0x1D565) + of "uopf": Rune(0x1D566) + of "vopf": Rune(0x1D567) + of "wopf": Rune(0x1D568) + of "xopf": Rune(0x1D569) + of "yopf": Rune(0x1D56A) + of "zopf": Rune(0x1D56B) + else: Rune(0) + proc entityToUtf8*(entity: string): string = - ## Converts an HTML entity name like ``Ü`` to its UTF-8 equivalent. + ## Converts an HTML entity name like ``Ü`` or values like ``Ü`` + ## or ``Ü`` to its UTF-8 equivalent. ## "" is returned if the entity name is unknown. The HTML parser ## already converts entities to UTF-8. - for name, val in items(Entities): - if name == entity: return toUTF8(Rune(val)) - result = "" + runnableExamples: + const sigma = "Σ" + doAssert entityToUtf8("") == "" + doAssert entityToUtf8("a") == "" + doAssert entityToUtf8("gt") == ">" + doAssert entityToUtf8("Uuml") == "Ü" + doAssert entityToUtf8("quest") == "?" + doAssert entityToUtf8("#63") == "?" + doAssert entityToUtf8("Sigma") == sigma + doAssert entityToUtf8("#931") == sigma + doAssert entityToUtf8("#0931") == sigma + doAssert entityToUtf8("#x3A3") == sigma + doAssert entityToUtf8("#x03A3") == sigma + doAssert entityToUtf8("#x3a3") == sigma + doAssert entityToUtf8("#X3a3") == sigma + let rune = entityToRune(entity) + if rune.ord <= 0: result = "" + else: result = toUTF8(rune) proc addNode(father, son: XmlNode) = if son != nil: add(father, son) -proc parse(x: var XmlParser, errors: var seq[string]): XmlNode +proc parse(x: var XmlParser, errors: var seq[string]): XmlNode {.gcsafe.} proc expected(x: var XmlParser, n: XmlNode): string = result = errorMsg(x, "</" & n.tag & "> expected") diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index 648481ca5..139d4bb50 100644 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -73,12 +73,18 @@ ## progress of the HTTP request. ## ## .. code-block:: Nim -## var client = newAsyncHttpClient() +## import asyncdispatch, httpclient +## ## proc onProgressChanged(total, progress, speed: BiggestInt) {.async.} = ## echo("Downloaded ", progress, " of ", total) ## echo("Current rate: ", speed div 1000, "kb/s") -## client.onProgressChanged = onProgressChanged -## discard await client.getContent("http://speedtest-ams2.digitalocean.com/100mb.test") +## +## proc asyncProc() {.async.} = +## var client = newAsyncHttpClient() +## client.onProgressChanged = onProgressChanged +## discard await client.getContent("http://speedtest-ams2.digitalocean.com/100mb.test") +## +## waitFor asyncProc() ## ## If you would like to remove the callback simply set it to ``nil``. ## @@ -179,7 +185,7 @@ proc body*(response: Response): string = ## Retrieves the specified response's body. ## ## The response's body stream is read synchronously. - if response.body.isNil(): + if response.body.len == 0: response.body = response.bodyStream.readAll() return response.body @@ -192,7 +198,7 @@ proc `body=`*(response: Response, value: string) {.deprecated.} = proc body*(response: AsyncResponse): Future[string] {.async.} = ## Reads the response's body and caches it. The read is performed only ## once. - if response.body.isNil: + if response.body.len == 0: response.body = await readAll(response.bodyStream) return response.body @@ -213,10 +219,6 @@ type ## and ``postContent`` proc, ## when the server returns an error -{.deprecated: [TResponse: Response, PProxy: Proxy, - EInvalidProtocol: ProtocolError, EHttpRequestErr: HttpRequestError -].} - const defUserAgent* = "Nim httpclient/" & NimVersion proc httpError(msg: string) = @@ -241,7 +243,7 @@ proc parseChunks(s: Socket, timeout: int): string = var i = 0 if chunkSizeStr == "": httpError("Server terminated connection prematurely") - while true: + while i < chunkSizeStr.len: case chunkSizeStr[i] of '0'..'9': chunkSize = chunkSize shl 4 or (ord(chunkSizeStr[i]) - ord('0')) @@ -249,8 +251,6 @@ proc parseChunks(s: Socket, timeout: int): string = chunkSize = chunkSize shl 4 or (ord(chunkSizeStr[i]) - ord('a') + 10) of 'A'..'F': chunkSize = chunkSize shl 4 or (ord(chunkSizeStr[i]) - ord('A') + 10) - of '\0': - break of ';': # http://tools.ietf.org/html/rfc2616#section-3.6.1 # We don't care about chunk-extensions. @@ -358,21 +358,17 @@ proc parseResponse(s: Socket, getBody: bool, timeout: int): Response = else: result.body = "" -{.deprecated: [THttpMethod: HttpMethod].} - when not defined(ssl): type SSLContext = ref object var defaultSSLContext {.threadvar.}: SSLContext -when defined(ssl): - defaultSSLContext = newContext(verifyMode = CVerifyNone) - template contextOrDefault(ctx: SSLContext): SSLContext = - var result = ctx - if ctx == nil: - if defaultSSLContext == nil: - defaultSSLContext = newContext(verifyMode = CVerifyNone) +proc getDefaultSSL(): SSLContext = + result = defaultSslContext + when defined(ssl): + if result == nil: + defaultSSLContext = newContext(verifyMode = CVerifyNone) result = defaultSSLContext - result + doAssert result != nil, "failure to initialize the SSL context" proc newProxy*(url: string, auth = ""): Proxy = ## Constructs a new ``TProxy`` object. @@ -382,23 +378,23 @@ proc newMultipartData*: MultipartData = ## Constructs a new ``MultipartData`` object. MultipartData(content: @[]) -proc add*(p: var MultipartData, name, content: string, filename: string = nil, - contentType: string = nil) = +proc add*(p: var MultipartData, name, content: string, filename: string = "", + contentType: string = "") = ## Add a value to the multipart data. Raises a `ValueError` exception if ## `name`, `filename` or `contentType` contain newline characters. if {'\c','\L'} in name: raise newException(ValueError, "name contains a newline character") - if filename != nil and {'\c','\L'} in filename: + if {'\c','\L'} in filename: raise newException(ValueError, "filename contains a newline character") - if contentType != nil and {'\c','\L'} in contentType: + if {'\c','\L'} in contentType: raise newException(ValueError, "contentType contains a newline character") var str = "Content-Disposition: form-data; name=\"" & name & "\"" - if filename != nil: + if filename.len > 0: str.add("; filename=\"" & filename & "\"") str.add("\c\L") - if contentType != nil: + if contentType.len > 0: str.add("Content-Type: " & contentType & "\c\L") str.add("\c\L" & content & "\c\L") @@ -438,7 +434,7 @@ proc addFiles*(p: var MultipartData, xs: openarray[tuple[name, file: string]]): var contentType: string let (_, fName, ext) = splitFile(file) if ext.len > 0: - contentType = m.getMimetype(ext[1..ext.high], nil) + contentType = m.getMimetype(ext[1..ext.high], "") p.add(name, readFile(file), fName & ext, contentType) result = p @@ -461,7 +457,7 @@ proc `[]=`*(p: var MultipartData, name: string, p.add(name, file.content, file.name, file.contentType) proc format(p: MultipartData): tuple[contentType, body: string] = - if p == nil or p.content == nil or p.content.len == 0: + if p == nil or p.content.len == 0: return ("", "") # Create boundary that is not in the data to be formatted @@ -482,9 +478,9 @@ proc format(p: MultipartData): tuple[contentType, body: string] = result.body.add("--" & bound & "--\c\L") proc request*(url: string, httpMethod: string, extraHeaders = "", - body = "", sslContext = defaultSSLContext, timeout = -1, + body = "", sslContext = getDefaultSSL(), timeout = -1, userAgent = defUserAgent, proxy: Proxy = nil): Response - {.deprecated.} = + {.deprecated: "use HttpClient.request instead".} = ## | Requests ``url`` with the custom method string specified by the ## | ``httpMethod`` parameter. ## | Extra headers can be specified and must be separated by ``\c\L`` @@ -581,8 +577,8 @@ proc request*(url: string, httpMethod: string, extraHeaders = "", result = parseResponse(s, httpMethod != "HEAD", timeout) -proc request*(url: string, httpMethod = httpGET, extraHeaders = "", - body = "", sslContext = defaultSSLContext, timeout = -1, +proc request*(url: string, httpMethod = HttpGET, extraHeaders = "", + body = "", sslContext = getDefaultSSL(), timeout = -1, userAgent = defUserAgent, proxy: Proxy = nil): Response {.deprecated.} = ## | Requests ``url`` with the specified ``httpMethod``. @@ -613,7 +609,7 @@ proc getNewLocation(lastURL: string, headers: HttpHeaders): string = result = $parsed proc get*(url: string, extraHeaders = "", maxRedirects = 5, - sslContext: SSLContext = defaultSSLContext, + sslContext: SSLContext = getDefaultSSL(), timeout = -1, userAgent = defUserAgent, proxy: Proxy = nil): Response {.deprecated.} = ## | GETs the ``url`` and returns a ``Response`` object @@ -622,19 +618,19 @@ proc get*(url: string, extraHeaders = "", maxRedirects = 5, ## | An optional timeout can be specified in milliseconds, if reading from the ## server takes longer than specified an ETimeout exception will be raised. ## - ## ## **Deprecated since version 0.15.0**: use ``HttpClient.get`` instead. - result = request(url, httpGET, extraHeaders, "", sslContext, timeout, + ## **Deprecated since version 0.15.0**: use ``HttpClient.get`` instead. + result = request(url, HttpGET, extraHeaders, "", sslContext, timeout, userAgent, proxy) var lastURL = url for i in 1..maxRedirects: if result.status.redirection(): let redirectTo = getNewLocation(lastURL, result.headers) - result = request(redirectTo, httpGET, extraHeaders, "", sslContext, + result = request(redirectTo, HttpGET, extraHeaders, "", sslContext, timeout, userAgent, proxy) lastURL = redirectTo proc getContent*(url: string, extraHeaders = "", maxRedirects = 5, - sslContext: SSLContext = defaultSSLContext, + sslContext: SSLContext = getDefaultSSL(), timeout = -1, userAgent = defUserAgent, proxy: Proxy = nil): string {.deprecated.} = ## | GETs the body and returns it as a string. @@ -653,7 +649,7 @@ proc getContent*(url: string, extraHeaders = "", maxRedirects = 5, proc post*(url: string, extraHeaders = "", body = "", maxRedirects = 5, - sslContext: SSLContext = defaultSSLContext, + sslContext: SSLContext = getDefaultSSL(), timeout = -1, userAgent = defUserAgent, proxy: Proxy = nil, multipart: MultipartData = nil): Response {.deprecated.} = @@ -683,20 +679,20 @@ proc post*(url: string, extraHeaders = "", body = "", if not multipart.isNil: xh.add(withNewLine("Content-Type: " & mpContentType)) - result = request(url, httpPOST, xh, xb, sslContext, timeout, userAgent, + result = request(url, HttpPOST, xh, xb, sslContext, timeout, userAgent, proxy) var lastURL = url for i in 1..maxRedirects: if result.status.redirection(): let redirectTo = getNewLocation(lastURL, result.headers) - var meth = if result.status != "307": httpGet else: httpPost + var meth = if result.status != "307": HttpGet else: HttpPost result = request(redirectTo, meth, xh, xb, sslContext, timeout, userAgent, proxy) lastURL = redirectTo proc postContent*(url: string, extraHeaders = "", body = "", maxRedirects = 5, - sslContext: SSLContext = defaultSSLContext, + sslContext: SSLContext = getDefaultSSL(), timeout = -1, userAgent = defUserAgent, proxy: Proxy = nil, multipart: MultipartData = nil): string @@ -719,7 +715,7 @@ proc postContent*(url: string, extraHeaders = "", body = "", return r.body proc downloadFile*(url: string, outputFilename: string, - sslContext: SSLContext = defaultSSLContext, + sslContext: SSLContext = getDefaultSSL(), timeout = -1, userAgent = defUserAgent, proxy: Proxy = nil) {.deprecated.} = ## | Downloads ``url`` and saves it to ``outputFilename`` @@ -739,12 +735,13 @@ proc downloadFile*(url: string, outputFilename: string, proc generateHeaders(requestUrl: Uri, httpMethod: string, headers: HttpHeaders, body: string, proxy: Proxy): string = # GET - result = httpMethod.toUpperAscii() + let upperMethod = httpMethod.toUpperAscii() + result = upperMethod result.add ' ' - if proxy.isNil or (not proxy.isNil and requestUrl.scheme == "https"): + if proxy.isNil or requestUrl.scheme == "https": # /path?query - if requestUrl.path[0] != '/': result.add '/' + if not requestUrl.path.startsWith("/"): result.add '/' result.add(requestUrl.path) if requestUrl.query.len > 0: result.add("?" & requestUrl.query) @@ -768,7 +765,9 @@ proc generateHeaders(requestUrl: Uri, httpMethod: string, add(result, "Connection: Keep-Alive\c\L") # Content length header. - if body.len > 0 and not headers.hasKey("Content-Length"): + const requiresBody = ["POST", "PUT", "PATCH"] + let needsContentLength = body.len > 0 or upperMethod in requiresBody + if needsContentLength and not headers.hasKey("Content-Length"): add(result, "Content-Length: " & $body.len & "\c\L") # Proxy auth header. @@ -808,6 +807,7 @@ type lastProgressReport: float when SocketType is AsyncSocket: bodyStream: FutureStream[string] + parseBodyFut: Future[void] else: bodyStream: Stream getBody: bool ## When `false`, the body is never read in requestAux. @@ -816,7 +816,7 @@ type HttpClient* = HttpClientBase[Socket] proc newHttpClient*(userAgent = defUserAgent, - maxRedirects = 5, sslContext = defaultSslContext, proxy: Proxy = nil, + maxRedirects = 5, sslContext = getDefaultSSL(), proxy: Proxy = nil, timeout = -1): HttpClient = ## Creates a new HttpClient instance. ## @@ -843,7 +843,7 @@ proc newHttpClient*(userAgent = defUserAgent, result.bodyStream = newStringStream() result.getBody = true when defined(ssl): - result.sslContext = contextOrDefault(sslContext) + result.sslContext = sslContext type AsyncHttpClient* = HttpClientBase[AsyncSocket] @@ -851,7 +851,7 @@ type {.deprecated: [PAsyncHttpClient: AsyncHttpClient].} proc newAsyncHttpClient*(userAgent = defUserAgent, - maxRedirects = 5, sslContext = defaultSslContext, + maxRedirects = 5, sslContext = getDefaultSSL(), proxy: Proxy = nil): AsyncHttpClient = ## Creates a new AsyncHttpClient instance. ## @@ -875,7 +875,7 @@ proc newAsyncHttpClient*(userAgent = defUserAgent, result.bodyStream = newFutureStream[string]("newAsyncHttpClient") result.getBody = true when defined(ssl): - result.sslContext = contextOrDefault(sslContext) + result.sslContext = sslContext proc close*(client: HttpClient | AsyncHttpClient) = ## Closes any connections held by the HTTP client. @@ -929,7 +929,7 @@ proc parseChunks(client: HttpClient | AsyncHttpClient): Future[void] var i = 0 if chunkSizeStr == "": httpError("Server terminated connection prematurely") - while true: + while i < chunkSizeStr.len: case chunkSizeStr[i] of '0'..'9': chunkSize = chunkSize shl 4 or (ord(chunkSizeStr[i]) - ord('0')) @@ -937,8 +937,6 @@ proc parseChunks(client: HttpClient | AsyncHttpClient): Future[void] chunkSize = chunkSize shl 4 or (ord(chunkSizeStr[i]) - ord('a') + 10) of 'A'..'F': chunkSize = chunkSize shl 4 or (ord(chunkSizeStr[i]) - ord('A') + 10) - of '\0': - break of ';': # http://tools.ietf.org/html/rfc2616#section-3.6.1 # We don't care about chunk-extensions. @@ -1069,10 +1067,14 @@ proc parseResponse(client: HttpClient | AsyncHttpClient, if getBody: when client is HttpClient: client.bodyStream = newStringStream() + result.bodyStream = client.bodyStream + parseBody(client, result.headers, result.version) else: client.bodyStream = newFutureStream[string]("parseResponse") - await parseBody(client, result.headers, result.version) - result.bodyStream = client.bodyStream + result.bodyStream = client.bodyStream + assert(client.parseBodyFut.isNil or client.parseBodyFut.finished) + client.parseBodyFut = parseBody(client, result.headers, result.version) + # do not wait here for the body request to complete proc newConnection(client: HttpClient | AsyncHttpClient, url: Uri) {.multisync.} = @@ -1162,6 +1164,12 @@ proc requestAux(client: HttpClient | AsyncHttpClient, url: string, # Helper that actually makes the request. Does not handle redirects. let requestUrl = parseUri(url) + when client is AsyncHttpClient: + if not client.parseBodyFut.isNil: + # let the current operation finish before making another request + await client.parseBodyFut + client.parseBodyFut = nil + await newConnection(client, requestUrl) let effectiveHeaders = client.headers.override(headers) @@ -1187,7 +1195,7 @@ proc request*(client: HttpClient | AsyncHttpClient, url: string, ## Connects to the hostname specified by the URL and performs a request ## using the custom method string specified by ``httpMethod``. ## - ## Connection will kept alive. Further requests on the same ``client`` to + ## Connection will be kept alive. Further requests on the same ``client`` to ## the same hostname will not require a new connection to be made. The ## connection can be closed by using the ``close`` procedure. ## @@ -1199,7 +1207,7 @@ proc request*(client: HttpClient | AsyncHttpClient, url: string, for i in 1..client.maxRedirects: if result.status.redirection(): let redirectTo = getNewLocation(lastURL, result.headers) - result = await client.request(redirectTo, httpMethod, body, headers) + result = await client.requestAux(redirectTo, httpMethod, body, headers) lastURL = redirectTo diff --git a/lib/pure/httpcore.nim b/lib/pure/httpcore.nim index f150fa1c1..f85375111 100644 --- a/lib/pure/httpcore.nim +++ b/lib/pure/httpcore.nim @@ -45,10 +45,6 @@ type ## TCP/IP tunnel, usually used for proxies. HttpPatch ## Applies partial modifications to a resource. -{.deprecated: [httpGet: HttpGet, httpHead: HttpHead, httpPost: HttpPost, - httpPut: HttpPut, httpDelete: HttpDelete, httpTrace: HttpTrace, - httpOptions: HttpOptions, httpConnect: HttpConnect].} - const Http100* = HttpCode(100) @@ -190,11 +186,11 @@ proc len*(headers: HttpHeaders): int = return headers.table.len proc parseList(line: string, list: var seq[string], start: int): int = var i = 0 var current = "" - while line[start + i] notin {'\c', '\l', '\0'}: + while start+i < line.len and line[start + i] notin {'\c', '\l'}: i += line.skipWhitespace(start + i) i += line.parseUntil(current, {'\c', '\l', ','}, start + i) list.add(current) - if line[start + i] == ',': + if start+i < line.len and line[start + i] == ',': i.inc # Skip , current.setLen(0) diff --git a/lib/pure/httpserver.nim b/lib/pure/httpserver.nim index 632eb198a..a81e8c0a8 100644 --- a/lib/pure/httpserver.nim +++ b/lib/pure/httpserver.nim @@ -110,7 +110,6 @@ when false: # TODO: Fix this, or get rid of it. type RequestMethod = enum reqGet, reqPost - {.deprecated: [TRequestMethod: RequestMethod].} proc executeCgi(client: Socket, path, query: string, meth: RequestMethod) = var env = newStringTable(modeCaseInsensitive) @@ -126,7 +125,7 @@ when false: var dataAvail = false while dataAvail: dataAvail = recvLine(client, buf) # TODO: This is incorrect. - var L = toLower(buf.string) + var L = toLowerAscii(buf.string) if L.startsWith("content-length:"): var i = len("content-length:") while L[i] in Whitespace: inc(i) @@ -199,7 +198,7 @@ when false: notFound(client) else: when defined(Windows): - var ext = splitFile(path).ext.toLower + var ext = splitFile(path).ext.toLowerAscii if ext == ".exe" or ext == ".cgi": # XXX: extract interpreter information here? cgi = true @@ -225,7 +224,6 @@ type PAsyncHTTPServer* = ref AsyncHTTPServer AsyncHTTPServer = object of Server asyncSocket: AsyncSocket -{.deprecated: [TAsyncHTTPServer: AsyncHTTPServer, TServer: Server].} proc open*(s: var Server, port = Port(80), reuseAddr = false) = ## creates a new server at port `port`. If ``port == 0`` a free port is @@ -303,7 +301,7 @@ proc next*(s: var Server) = if s.reqMethod == "POST": # Check for Expect header if s.headers.hasKey("Expect"): - if s.headers["Expect"].toLower == "100-continue": + if s.headers["Expect"].toLowerAscii == "100-continue": s.client.sendStatus("100 Continue") else: s.client.sendStatus("417 Expectation Failed") @@ -427,7 +425,7 @@ proc nextAsync(s: PAsyncHTTPServer) = if s.reqMethod == "POST": # Check for Expect header if s.headers.hasKey("Expect"): - if s.headers["Expect"].toLower == "100-continue": + if s.headers["Expect"].toLowerAscii == "100-continue": s.client.sendStatus("100 Continue") else: s.client.sendStatus("417 Expectation Failed") diff --git a/lib/pure/includes/asynccommon.nim b/lib/pure/includes/asynccommon.nim index 45b10c584..06f4958c6 100644 --- a/lib/pure/includes/asynccommon.nim +++ b/lib/pure/includes/asynccommon.nim @@ -18,13 +18,13 @@ proc createAsyncNativeSocket*(domain: Domain = Domain.AF_INET, createAsyncNativeSocketImpl(domain, sockType, protocol) proc newAsyncNativeSocket*(domain: cint, sockType: cint, - protocol: cint): AsyncFD {.deprecated.} = + protocol: cint): AsyncFD {.deprecated: "use createAsyncNativeSocket instead".} = createAsyncNativeSocketImpl(domain, sockType, protocol) proc newAsyncNativeSocket*(domain: Domain = Domain.AF_INET, sockType: SockType = SOCK_STREAM, protocol: Protocol = IPPROTO_TCP): AsyncFD - {.deprecated.} = + {.deprecated: "use createAsyncNativeSocket instead".} = createAsyncNativeSocketImpl(domain, sockType, protocol) when defined(windows) or defined(nimdoc): diff --git a/lib/pure/includes/oserr.nim b/lib/pure/includes/oserr.nim index 0889d7383..493e8e174 100644 --- a/lib/pure/includes/oserr.nim +++ b/lib/pure/includes/oserr.nim @@ -12,50 +12,6 @@ when not defined(nimscript): when defined(windows): import winlean -proc osErrorMsg*(): string {.rtl, extern: "nos$1", deprecated.} = - ## Retrieves the operating system's error flag, ``errno``. - ## On Windows ``GetLastError`` is checked before ``errno``. - ## Returns "" if no error occurred. - ## - ## **Deprecated since version 0.9.4**: use the other ``osErrorMsg`` proc. - - result = "" - when defined(Windows) and not defined(nimscript): - var err = getLastError() - if err != 0'i32: - when useWinUnicode: - var msgbuf: WideCString - if formatMessageW(0x00000100 or 0x00001000 or 0x00000200 or 0x000000FF, - nil, err, 0, addr(msgbuf), 0, nil) != 0'i32: - result = $msgbuf - if msgbuf != nil: localFree(cast[pointer](msgbuf)) - else: - var msgbuf: cstring - if formatMessageA(0x00000100 or 0x00001000 or 0x00000200 or 0x000000FF, - nil, err, 0, addr(msgbuf), 0, nil) != 0'i32: - result = $msgbuf - if msgbuf != nil: localFree(msgbuf) - when not defined(nimscript): - if errno != 0'i32: - result = $c_strerror(errno) - -{.push warning[deprecated]: off.} -proc raiseOSError*(msg: string = "") {.noinline, rtl, extern: "nos$1", - deprecated.} = - ## raises an OSError exception with the given message ``msg``. - ## If ``msg == ""``, the operating system's error flag - ## (``errno``) is converted to a readable error message. On Windows - ## ``GetLastError`` is checked before ``errno``. - ## If no error flag is set, the message ``unknown OS error`` is used. - ## - ## **Deprecated since version 0.9.4**: use the other ``raiseOSError`` proc. - if len(msg) == 0: - var m = osErrorMsg() - raise newException(OSError, if m.len > 0: m else: "unknown OS error") - else: - raise newException(OSError, msg) -{.pop.} - proc `==`*(err1, err2: OSErrorCode): bool {.borrow.} proc `$`*(err: OSErrorCode): string {.borrow.} @@ -104,13 +60,13 @@ proc raiseOSError*(errorCode: OSErrorCode; additionalInfo = "") {.noinline.} = if additionalInfo.len == 0: e.msg = osErrorMsg(errorCode) else: - e.msg = osErrorMsg(errorCode) & "\nAdditional info: " & additionalInfo + e.msg = osErrorMsg(errorCode) & "\nAdditional info: '" & additionalInfo & "'" if e.msg == "": e.msg = "unknown OS error" raise e {.push stackTrace:off.} -proc osLastError*(): OSErrorCode = +proc osLastError*(): OSErrorCode {.sideEffect.} = ## Retrieves the last operating system error code. ## ## This procedure is useful in the event when an OS call fails. In that case diff --git a/lib/pure/ioselects/ioselectors_epoll.nim b/lib/pure/ioselects/ioselectors_epoll.nim index 98b8a2b2b..8b3f14f34 100644 --- a/lib/pure/ioselects/ioselectors_epoll.nim +++ b/lib/pure/ioselects/ioselectors_epoll.nim @@ -48,16 +48,6 @@ when not defined(android): proc signalfd(fd: cint, mask: var Sigset, flags: cint): cint {.cdecl, importc: "signalfd", header: "<sys/signalfd.h>".} -var RLIMIT_NOFILE {.importc: "RLIMIT_NOFILE", - header: "<sys/resource.h>".}: cint -type - RLimit {.importc: "struct rlimit", - header: "<sys/resource.h>", pure, final.} = object - rlim_cur: int - rlim_max: int -proc getrlimit(resource: cint, rlp: var RLimit): cint - {.importc: "getrlimit",header: "<sys/resource.h>".} - when hasThreadSupport: type SelectorImpl[T] = object @@ -82,7 +72,7 @@ type proc newSelector*[T](): Selector[T] = # Retrieve the maximum fd count (for current OS) via getrlimit() var a = RLimit() - if getrlimit(RLIMIT_NOFILE, a) != 0: + if getrlimit(posix.RLIMIT_NOFILE, a) != 0: raiseOsError(osLastError()) var maxFD = int(a.rlim_max) doAssert(maxFD > 0) @@ -102,6 +92,9 @@ proc newSelector*[T](): Selector[T] = result.maxFD = maxFD result.fds = newSeq[SelectorKey[T]](maxFD) + for i in 0 ..< maxFD: + result.fds[i].ident = InvalidIdent + proc close*[T](s: Selector[T]) = let res = posix.close(s.epollFD) when hasThreadSupport: @@ -110,12 +103,6 @@ proc close*[T](s: Selector[T]) = if res != 0: raiseIOSelectorsError(osLastError()) -template clearKey[T](key: ptr SelectorKey[T]) = - var empty: T - key.ident = 0 - key.events = {} - key.data = empty - proc newSelectEvent*(): SelectEvent = let fdci = eventfd(0, 0) if fdci == -1: @@ -145,7 +132,7 @@ proc registerHandle*[T](s: Selector[T], fd: int | SocketHandle, events: set[Event], data: T) = let fdi = int(fd) s.checkFd(fdi) - doAssert(s.fds[fdi].ident == 0, "Descriptor $# already registered" % $fdi) + doAssert(s.fds[fdi].ident == InvalidIdent, "Descriptor $# already registered" % $fdi) s.setKey(fdi, events, 0, data) if events != {}: var epv = EpollEvent(events: EPOLLRDHUP) @@ -162,7 +149,7 @@ proc updateHandle*[T](s: Selector[T], fd: int | SocketHandle, events: set[Event] let fdi = int(fd) s.checkFd(fdi) var pkey = addr(s.fds[fdi]) - doAssert(pkey.ident != 0, + doAssert(pkey.ident != InvalidIdent, "Descriptor $# is not registered in the selector!" % $fdi) doAssert(pkey.events * maskEvents == {}) if pkey.events != events: @@ -190,7 +177,7 @@ proc unregister*[T](s: Selector[T], fd: int|SocketHandle) = let fdi = int(fd) s.checkFd(fdi) var pkey = addr(s.fds[fdi]) - doAssert(pkey.ident != 0, + doAssert(pkey.ident != InvalidIdent, "Descriptor $# is not registered in the selector!" % $fdi) if pkey.events != {}: when not defined(android): @@ -253,7 +240,7 @@ proc unregister*[T](s: Selector[T], ev: SelectEvent) = let fdi = int(ev.efd) s.checkFd(fdi) var pkey = addr(s.fds[fdi]) - doAssert(pkey.ident != 0, "Event is not registered in the queue!") + doAssert(pkey.ident != InvalidIdent, "Event is not registered in the queue!") doAssert(Event.User in pkey.events) var epv = EpollEvent() if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) != 0: @@ -272,7 +259,7 @@ proc registerTimer*[T](s: Selector[T], timeout: int, oneshot: bool, setNonBlocking(fdi.cint) s.checkFd(fdi) - doAssert(s.fds[fdi].ident == 0) + doAssert(s.fds[fdi].ident == InvalidIdent) var events = {Event.Timer} var epv = EpollEvent(events: EPOLLIN or EPOLLRDHUP) @@ -317,7 +304,7 @@ when not defined(android): setNonBlocking(fdi.cint) s.checkFd(fdi) - doAssert(s.fds[fdi].ident == 0) + doAssert(s.fds[fdi].ident == InvalidIdent) var epv = EpollEvent(events: EPOLLIN or EPOLLRDHUP) epv.data.u64 = fdi.uint @@ -344,7 +331,7 @@ when not defined(android): setNonBlocking(fdi.cint) s.checkFd(fdi) - doAssert(s.fds[fdi].ident == 0) + doAssert(s.fds[fdi].ident == InvalidIdent) var epv = EpollEvent(events: EPOLLIN or EPOLLRDHUP) epv.data.u64 = fdi.uint @@ -357,7 +344,7 @@ when not defined(android): proc registerEvent*[T](s: Selector[T], ev: SelectEvent, data: T) = let fdi = int(ev.efd) - doAssert(s.fds[fdi].ident == 0, "Event is already registered in the queue!") + doAssert(s.fds[fdi].ident == InvalidIdent, "Event is already registered in the queue!") s.setKey(fdi, {Event.User}, 0, data) var epv = EpollEvent(events: EPOLLIN or EPOLLRDHUP) epv.data.u64 = ev.efd.uint @@ -391,7 +378,7 @@ proc selectInto*[T](s: Selector[T], timeout: int, let fdi = int(resTable[i].data.u64) let pevents = resTable[i].events var pkey = addr(s.fds[fdi]) - doAssert(pkey.ident != 0) + doAssert(pkey.ident != InvalidIdent) var rkey = ReadyKey(fd: fdi, events: {}) if (pevents and EPOLLERR) != 0 or (pevents and EPOLLHUP) != 0: @@ -492,7 +479,7 @@ template isEmpty*[T](s: Selector[T]): bool = (s.count == 0) proc contains*[T](s: Selector[T], fd: SocketHandle|int): bool {.inline.} = - return s.fds[fd.int].ident != 0 + return s.fds[fd.int].ident != InvalidIdent proc getData*[T](s: Selector[T], fd: SocketHandle|int): var T = let fdi = int(fd) @@ -528,4 +515,4 @@ template withData*[T](s: Selector[T], fd: SocketHandle|int, value, body1, body2 proc getFd*[T](s: Selector[T]): int = - return s.epollFd.int \ No newline at end of file + return s.epollFd.int diff --git a/lib/pure/ioselects/ioselectors_kqueue.nim b/lib/pure/ioselects/ioselectors_kqueue.nim index 10e23c072..0e133f650 100644 --- a/lib/pure/ioselects/ioselectors_kqueue.nim +++ b/lib/pure/ioselects/ioselectors_kqueue.nim @@ -114,6 +114,9 @@ proc newSelector*[T](): Selector[T] = result.fds = newSeq[SelectorKey[T]](maxFD) result.changes = newSeqOfCap[KEvent](MAX_KQUEUE_EVENTS) + for i in 0 ..< maxFD: + result.fds[i].ident = InvalidIdent + result.sock = usock result.kqFD = kqFD result.maxFD = maxFD.int @@ -128,12 +131,6 @@ proc close*[T](s: Selector[T]) = if res1 != 0 or res2 != 0: raiseIOSelectorsError(osLastError()) -template clearKey[T](key: ptr SelectorKey[T]) = - var empty: T - key.ident = 0 - key.events = {} - key.data = empty - proc newSelectEvent*(): SelectEvent = var fds: array[2, cint] if posix.pipe(fds) != 0: @@ -221,7 +218,7 @@ proc registerHandle*[T](s: Selector[T], fd: int | SocketHandle, events: set[Event], data: T) = let fdi = int(fd) s.checkFd(fdi) - doAssert(s.fds[fdi].ident == 0) + doAssert(s.fds[fdi].ident == InvalidIdent) s.setKey(fdi, events, 0, data) if events != {}: @@ -242,7 +239,7 @@ proc updateHandle*[T](s: Selector[T], fd: int | SocketHandle, let fdi = int(fd) s.checkFd(fdi) var pkey = addr(s.fds[fdi]) - doAssert(pkey.ident != 0, + doAssert(pkey.ident != InvalidIdent, "Descriptor $# is not registered in the queue!" % $fdi) doAssert(pkey.events * maskEvents == {}) @@ -269,7 +266,7 @@ proc registerTimer*[T](s: Selector[T], timeout: int, oneshot: bool, data: T): int {.discardable.} = let fdi = getUnique(s) s.checkFd(fdi) - doAssert(s.fds[fdi].ident == 0) + doAssert(s.fds[fdi].ident == InvalidIdent) let events = if oneshot: {Event.Timer, Event.Oneshot} else: {Event.Timer} let flags: cushort = if oneshot: EV_ONESHOT or EV_ADD else: EV_ADD @@ -291,7 +288,7 @@ proc registerSignal*[T](s: Selector[T], signal: int, data: T): int {.discardable.} = let fdi = getUnique(s) s.checkFd(fdi) - doAssert(s.fds[fdi].ident == 0) + doAssert(s.fds[fdi].ident == InvalidIdent) s.setKey(fdi, {Event.Signal}, signal, data) var nmask, omask: Sigset @@ -315,7 +312,7 @@ proc registerProcess*[T](s: Selector[T], pid: int, data: T): int {.discardable.} = let fdi = getUnique(s) s.checkFd(fdi) - doAssert(s.fds[fdi].ident == 0) + doAssert(s.fds[fdi].ident == InvalidIdent) var kflags: cushort = EV_ONESHOT or EV_ADD setKey(s, fdi, {Event.Process, Event.Oneshot}, pid, data) @@ -331,7 +328,7 @@ proc registerProcess*[T](s: Selector[T], pid: int, proc registerEvent*[T](s: Selector[T], ev: SelectEvent, data: T) = let fdi = ev.rfd.int - doAssert(s.fds[fdi].ident == 0, "Event is already registered in the queue!") + doAssert(s.fds[fdi].ident == InvalidIdent, "Event is already registered in the queue!") setKey(s, fdi, {Event.User}, 0, data) modifyKQueue(s, fdi.uint, EVFILT_READ, EV_ADD, 0, 0, nil) @@ -374,7 +371,7 @@ proc unregister*[T](s: Selector[T], fd: int|SocketHandle) = let fdi = int(fd) s.checkFd(fdi) var pkey = addr(s.fds[fdi]) - doAssert(pkey.ident != 0, + doAssert(pkey.ident != InvalidIdent, "Descriptor [" & $fdi & "] is not registered in the queue!") if pkey.events != {}: @@ -434,7 +431,7 @@ proc unregister*[T](s: Selector[T], ev: SelectEvent) = let fdi = int(ev.rfd) s.checkFd(fdi) var pkey = addr(s.fds[fdi]) - doAssert(pkey.ident != 0, "Event is not registered in the queue!") + doAssert(pkey.ident != InvalidIdent, "Event is not registered in the queue!") doAssert(Event.User in pkey.events) modifyKQueue(s, uint(fdi), EVFILT_READ, EV_DELETE, 0, 0, nil) when not declared(CACHE_EVENTS): @@ -570,8 +567,11 @@ proc selectInto*[T](s: Selector[T], timeout: int, doAssert(true, "Unsupported kqueue filter in the queue!") if (kevent.flags and EV_EOF) != 0: + # TODO this error handling needs to be rethought. + # `fflags` can sometimes be `0x80000000` and thus we use 'cast' + # here: if kevent.fflags != 0: - rkey.errorCode = kevent.fflags.OSErrorCode + rkey.errorCode = cast[OSErrorCode](kevent.fflags) else: # This assumes we are dealing with sockets. # TODO: For future-proofing it might be a good idea to give the @@ -593,7 +593,7 @@ template isEmpty*[T](s: Selector[T]): bool = (s.count == 0) proc contains*[T](s: Selector[T], fd: SocketHandle|int): bool {.inline.} = - return s.fds[fd.int].ident != 0 + return s.fds[fd.int].ident != InvalidIdent proc getData*[T](s: Selector[T], fd: SocketHandle|int): var T = let fdi = int(fd) diff --git a/lib/pure/ioselects/ioselectors_poll.nim b/lib/pure/ioselects/ioselectors_poll.nim index 66d52b352..9d708b0c1 100644 --- a/lib/pure/ioselects/ioselectors_poll.nim +++ b/lib/pure/ioselects/ioselectors_poll.nim @@ -40,16 +40,6 @@ type wfd: cint SelectEvent* = ptr SelectEventImpl -var RLIMIT_NOFILE {.importc: "RLIMIT_NOFILE", - header: "<sys/resource.h>".}: cint -type - rlimit {.importc: "struct rlimit", - header: "<sys/resource.h>", pure, final.} = object - rlim_cur: int - rlim_max: int -proc getrlimit(resource: cint, rlp: var rlimit): cint - {.importc: "getrlimit",header: "<sys/resource.h>".} - when hasThreadSupport: template withPollLock[T](s: Selector[T], body: untyped) = acquire(s.lock) @@ -63,8 +53,8 @@ else: body proc newSelector*[T](): Selector[T] = - var a = rlimit() - if getrlimit(RLIMIT_NOFILE, a) != 0: + var a = RLimit() + if getrlimit(posix.RLIMIT_NOFILE, a) != 0: raiseIOSelectorsError(osLastError()) var maxFD = int(a.rlim_max) @@ -80,6 +70,9 @@ proc newSelector*[T](): Selector[T] = result.fds = newSeq[SelectorKey[T]](maxFD) result.pollfds = newSeq[TPollFd](maxFD) + for i in 0 ..< maxFD: + result.fds[i].ident = InvalidIdent + proc close*[T](s: Selector[T]) = when hasThreadSupport: deinitLock(s.lock) @@ -87,12 +80,6 @@ proc close*[T](s: Selector[T]) = deallocSharedArray(s.pollfds) deallocShared(cast[pointer](s)) -template clearKey[T](key: ptr SelectorKey[T]) = - var empty: T - key.ident = 0 - key.events = {} - key.data = empty - template pollAdd[T](s: Selector[T], sock: cint, events: set[Event]) = withPollLock(s): var pollev: cshort = 0 @@ -145,7 +132,7 @@ proc registerHandle*[T](s: Selector[T], fd: int | SocketHandle, events: set[Event], data: T) = var fdi = int(fd) s.checkFd(fdi) - doAssert(s.fds[fdi].ident == 0) + doAssert(s.fds[fdi].ident == InvalidIdent) setKey(s, fdi, events, 0, data) if events != {}: s.pollAdd(fdi.cint, events) @@ -156,7 +143,7 @@ proc updateHandle*[T](s: Selector[T], fd: int | SocketHandle, let fdi = int(fd) s.checkFd(fdi) var pkey = addr(s.fds[fdi]) - doAssert(pkey.ident != 0, + doAssert(pkey.ident != InvalidIdent, "Descriptor [" & $fdi & "] is not registered in the queue!") doAssert(pkey.events * maskEvents == {}) @@ -172,7 +159,7 @@ proc updateHandle*[T](s: Selector[T], fd: int | SocketHandle, proc registerEvent*[T](s: Selector[T], ev: SelectEvent, data: T) = var fdi = int(ev.rfd) - doAssert(s.fds[fdi].ident == 0, "Event is already registered in the queue!") + doAssert(s.fds[fdi].ident == InvalidIdent, "Event is already registered in the queue!") var events = {Event.User} setKey(s, fdi, events, 0, data) events.incl(Event.Read) @@ -182,9 +169,9 @@ proc unregister*[T](s: Selector[T], fd: int|SocketHandle) = let fdi = int(fd) s.checkFd(fdi) var pkey = addr(s.fds[fdi]) - doAssert(pkey.ident != 0, + doAssert(pkey.ident != InvalidIdent, "Descriptor [" & $fdi & "] is not registered in the queue!") - pkey.ident = 0 + pkey.ident = InvalidIdent pkey.events = {} s.pollRemove(fdi.cint) @@ -192,9 +179,9 @@ proc unregister*[T](s: Selector[T], ev: SelectEvent) = let fdi = int(ev.rfd) s.checkFd(fdi) var pkey = addr(s.fds[fdi]) - doAssert(pkey.ident != 0, "Event is not registered in the queue!") + doAssert(pkey.ident != InvalidIdent, "Event is not registered in the queue!") doAssert(Event.User in pkey.events) - pkey.ident = 0 + pkey.ident = InvalidIdent pkey.events = {} s.pollRemove(fdi.cint) @@ -280,7 +267,7 @@ template isEmpty*[T](s: Selector[T]): bool = (s.count == 0) proc contains*[T](s: Selector[T], fd: SocketHandle|int): bool {.inline.} = - return s.fds[fd.int].ident != 0 + return s.fds[fd.int].ident != InvalidIdent proc getData*[T](s: Selector[T], fd: SocketHandle|int): var T = let fdi = int(fd) @@ -317,4 +304,4 @@ template withData*[T](s: Selector[T], fd: SocketHandle|int, value, body1, proc getFd*[T](s: Selector[T]): int = - return -1 \ No newline at end of file + return -1 diff --git a/lib/pure/ioselects/ioselectors_select.nim b/lib/pure/ioselects/ioselectors_select.nim index 7ed250307..521b31a64 100644 --- a/lib/pure/ioselects/ioselectors_select.nim +++ b/lib/pure/ioselects/ioselectors_select.nim @@ -99,6 +99,9 @@ proc newSelector*[T](): Selector[T] = result = Selector[T]() result.fds = newSeq[SelectorKey[T]](FD_SETSIZE) + for i in 0 ..< FD_SETSIZE: + result.fds[i].ident = InvalidIdent + IOFD_ZERO(addr result.rSet) IOFD_ZERO(addr result.wSet) IOFD_ZERO(addr result.eSet) @@ -195,7 +198,7 @@ proc setSelectKey[T](s: Selector[T], fd: SocketHandle, events: set[Event], var i = 0 let fdi = int(fd) while i < FD_SETSIZE: - if s.fds[i].ident == 0: + if s.fds[i].ident == InvalidIdent: var pkey = addr(s.fds[i]) pkey.ident = fdi pkey.events = events @@ -221,7 +224,7 @@ proc delKey[T](s: Selector[T], fd: SocketHandle) = var i = 0 while i < FD_SETSIZE: if s.fds[i].ident == fd.int: - s.fds[i].ident = 0 + s.fds[i].ident = InvalidIdent s.fds[i].events = {} s.fds[i].data = empty break @@ -307,7 +310,10 @@ proc selectInto*[T](s: Selector[T], timeout: int, var rset, wset, eset: FdSet if timeout != -1: - tv.tv_sec = timeout.int32 div 1_000 + when defined(genode): + tv.tv_sec = Time(timeout div 1_000) + else: + tv.tv_sec = timeout.int32 div 1_000 tv.tv_usec = (timeout.int32 %% 1_000) * 1_000 else: ptv = nil @@ -335,7 +341,7 @@ proc selectInto*[T](s: Selector[T], timeout: int, var k = 0 while (i < FD_SETSIZE) and (k < count): - if s.fds[i].ident != 0: + if s.fds[i].ident != InvalidIdent: var flag = false var pkey = addr(s.fds[i]) var rkey = ReadyKey(fd: int(pkey.ident), events: {}) @@ -388,7 +394,6 @@ proc contains*[T](s: Selector[T], fd: SocketHandle|int): bool {.inline.} = for i in 0..<FD_SETSIZE: if s.fds[i].ident == fdi: return true - inc(i) when hasThreadSupport: template withSelectLock[T](s: Selector[T], body: untyped) = diff --git a/lib/pure/json.nim b/lib/pure/json.nim index bbde4db5f..9279fea77 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -89,510 +89,22 @@ ## echo j2 import - hashes, tables, strutils, lexbase, streams, unicode, macros + hashes, tables, strutils, lexbase, streams, unicode, macros, parsejson export tables.`$` +export + parsejson.JsonEventKind, parsejson.JsonError, JsonParser, JsonKindError, + open, close, str, getInt, getFloat, kind, getColumn, getLine, getFilename, + errorMsg, errorMsgExpected, next, JsonParsingError, raiseParseErr + when defined(nimJsonGet): {.pragma: deprecatedGet, deprecated.} else: {.pragma: deprecatedGet.} type - JsonEventKind* = enum ## enumeration of all events that may occur when parsing - jsonError, ## an error occurred during parsing - jsonEof, ## end of file reached - jsonString, ## a string literal - jsonInt, ## an integer literal - jsonFloat, ## a float literal - jsonTrue, ## the value ``true`` - jsonFalse, ## the value ``false`` - jsonNull, ## the value ``null`` - jsonObjectStart, ## start of an object: the ``{`` token - jsonObjectEnd, ## end of an object: the ``}`` token - jsonArrayStart, ## start of an array: the ``[`` token - jsonArrayEnd ## start of an array: the ``]`` token - - TokKind = enum # must be synchronized with TJsonEventKind! - tkError, - tkEof, - tkString, - tkInt, - tkFloat, - tkTrue, - tkFalse, - tkNull, - tkCurlyLe, - tkCurlyRi, - tkBracketLe, - tkBracketRi, - tkColon, - tkComma - - JsonError* = enum ## enumeration that lists all errors that can occur - errNone, ## no error - errInvalidToken, ## invalid token - errStringExpected, ## string expected - errColonExpected, ## ``:`` expected - errCommaExpected, ## ``,`` expected - errBracketRiExpected, ## ``]`` expected - errCurlyRiExpected, ## ``}`` expected - errQuoteExpected, ## ``"`` or ``'`` expected - errEOC_Expected, ## ``*/`` expected - errEofExpected, ## EOF expected - errExprExpected ## expr expected - - ParserState = enum - stateEof, stateStart, stateObject, stateArray, stateExpectArrayComma, - stateExpectObjectComma, stateExpectColon, stateExpectValue - - JsonParser* = object of BaseLexer ## the parser object. - a: string - tok: TokKind - kind: JsonEventKind - err: JsonError - state: seq[ParserState] - filename: string - - JsonKindError* = object of ValueError ## raised by the ``to`` macro if the - ## JSON kind is incorrect. - -{.deprecated: [TJsonEventKind: JsonEventKind, TJsonError: JsonError, - TJsonParser: JsonParser, TTokKind: TokKind].} - -const - errorMessages: array[JsonError, string] = [ - "no error", - "invalid token", - "string expected", - "':' expected", - "',' expected", - "']' expected", - "'}' expected", - "'\"' or \"'\" expected", - "'*/' expected", - "EOF expected", - "expression expected" - ] - tokToStr: array[TokKind, string] = [ - "invalid token", - "EOF", - "string literal", - "int literal", - "float literal", - "true", - "false", - "null", - "{", "}", "[", "]", ":", "," - ] - -proc open*(my: var JsonParser, input: Stream, filename: string) = - ## initializes the parser with an input stream. `Filename` is only used - ## for nice error messages. - lexbase.open(my, input) - my.filename = filename - my.state = @[stateStart] - my.kind = jsonError - my.a = "" - -proc close*(my: var JsonParser) {.inline.} = - ## closes the parser `my` and its associated input stream. - lexbase.close(my) - -proc str*(my: JsonParser): string {.inline.} = - ## returns the character data for the events: ``jsonInt``, ``jsonFloat``, - ## ``jsonString`` - assert(my.kind in {jsonInt, jsonFloat, jsonString}) - return my.a - -proc getInt*(my: JsonParser): BiggestInt {.inline.} = - ## returns the number for the event: ``jsonInt`` - assert(my.kind == jsonInt) - return parseBiggestInt(my.a) - -proc getFloat*(my: JsonParser): float {.inline.} = - ## returns the number for the event: ``jsonFloat`` - assert(my.kind == jsonFloat) - return parseFloat(my.a) - -proc kind*(my: JsonParser): JsonEventKind {.inline.} = - ## returns the current event type for the JSON parser - return my.kind - -proc getColumn*(my: JsonParser): int {.inline.} = - ## get the current column the parser has arrived at. - result = getColNumber(my, my.bufpos) - -proc getLine*(my: JsonParser): int {.inline.} = - ## get the current line the parser has arrived at. - result = my.lineNumber - -proc getFilename*(my: JsonParser): string {.inline.} = - ## get the filename of the file that the parser processes. - result = my.filename - -proc errorMsg*(my: JsonParser): string = - ## returns a helpful error message for the event ``jsonError`` - assert(my.kind == jsonError) - result = "$1($2, $3) Error: $4" % [ - my.filename, $getLine(my), $getColumn(my), errorMessages[my.err]] - -proc errorMsgExpected*(my: JsonParser, e: string): string = - ## returns an error message "`e` expected" in the same format as the - ## other error messages - result = "$1($2, $3) Error: $4" % [ - my.filename, $getLine(my), $getColumn(my), e & " expected"] - -proc handleHexChar(c: char, x: var int): bool = - result = true # Success - case c - of '0'..'9': x = (x shl 4) or (ord(c) - ord('0')) - of 'a'..'f': x = (x shl 4) or (ord(c) - ord('a') + 10) - of 'A'..'F': x = (x shl 4) or (ord(c) - ord('A') + 10) - else: result = false # error - -proc parseEscapedUTF16(buf: cstring, pos: var int): int = - result = 0 - #UTF-16 escape is always 4 bytes. - for _ in 0..3: - if handleHexChar(buf[pos], result): - inc(pos) - else: - return -1 - -proc parseString(my: var JsonParser): TokKind = - result = tkString - var pos = my.bufpos + 1 - var buf = my.buf - while true: - case buf[pos] - of '\0': - my.err = errQuoteExpected - result = tkError - break - of '"': - inc(pos) - break - of '\\': - case buf[pos+1] - of '\\', '"', '\'', '/': - add(my.a, buf[pos+1]) - inc(pos, 2) - of 'b': - add(my.a, '\b') - inc(pos, 2) - of 'f': - add(my.a, '\f') - inc(pos, 2) - of 'n': - add(my.a, '\L') - inc(pos, 2) - of 'r': - add(my.a, '\C') - inc(pos, 2) - of 't': - add(my.a, '\t') - inc(pos, 2) - of 'u': - inc(pos, 2) - var r = parseEscapedUTF16(buf, pos) - if r < 0: - my.err = errInvalidToken - break - # Deal with surrogates - if (r and 0xfc00) == 0xd800: - if buf[pos] & buf[pos+1] != "\\u": - my.err = errInvalidToken - break - inc(pos, 2) - var s = parseEscapedUTF16(buf, pos) - if (s and 0xfc00) == 0xdc00 and s > 0: - r = 0x10000 + (((r - 0xd800) shl 10) or (s - 0xdc00)) - else: - my.err = errInvalidToken - break - add(my.a, toUTF8(Rune(r))) - else: - # don't bother with the error - add(my.a, buf[pos]) - inc(pos) - of '\c': - pos = lexbase.handleCR(my, pos) - buf = my.buf - add(my.a, '\c') - of '\L': - pos = lexbase.handleLF(my, pos) - buf = my.buf - add(my.a, '\L') - else: - add(my.a, buf[pos]) - inc(pos) - my.bufpos = pos # store back - -proc skip(my: var JsonParser) = - var pos = my.bufpos - var buf = my.buf - while true: - case buf[pos] - of '/': - if buf[pos+1] == '/': - # skip line comment: - inc(pos, 2) - while true: - case buf[pos] - of '\0': - break - of '\c': - pos = lexbase.handleCR(my, pos) - buf = my.buf - break - of '\L': - pos = lexbase.handleLF(my, pos) - buf = my.buf - break - else: - inc(pos) - elif buf[pos+1] == '*': - # skip long comment: - inc(pos, 2) - while true: - case buf[pos] - of '\0': - my.err = errEOC_Expected - break - of '\c': - pos = lexbase.handleCR(my, pos) - buf = my.buf - of '\L': - pos = lexbase.handleLF(my, pos) - buf = my.buf - of '*': - inc(pos) - if buf[pos] == '/': - inc(pos) - break - else: - inc(pos) - else: - break - of ' ', '\t': - inc(pos) - of '\c': - pos = lexbase.handleCR(my, pos) - buf = my.buf - of '\L': - pos = lexbase.handleLF(my, pos) - buf = my.buf - else: - break - my.bufpos = pos - -proc parseNumber(my: var JsonParser) = - var pos = my.bufpos - var buf = my.buf - if buf[pos] == '-': - add(my.a, '-') - inc(pos) - if buf[pos] == '.': - add(my.a, "0.") - inc(pos) - else: - while buf[pos] in Digits: - add(my.a, buf[pos]) - inc(pos) - if buf[pos] == '.': - add(my.a, '.') - inc(pos) - # digits after the dot: - while buf[pos] in Digits: - add(my.a, buf[pos]) - inc(pos) - if buf[pos] in {'E', 'e'}: - add(my.a, buf[pos]) - inc(pos) - if buf[pos] in {'+', '-'}: - add(my.a, buf[pos]) - inc(pos) - while buf[pos] in Digits: - add(my.a, buf[pos]) - inc(pos) - my.bufpos = pos - -proc parseName(my: var JsonParser) = - var pos = my.bufpos - var buf = my.buf - if buf[pos] in IdentStartChars: - while buf[pos] in IdentChars: - add(my.a, buf[pos]) - inc(pos) - my.bufpos = pos - -proc getTok(my: var JsonParser): TokKind = - setLen(my.a, 0) - skip(my) # skip whitespace, comments - case my.buf[my.bufpos] - of '-', '.', '0'..'9': - parseNumber(my) - if {'.', 'e', 'E'} in my.a: - result = tkFloat - else: - result = tkInt - of '"': - result = parseString(my) - of '[': - inc(my.bufpos) - result = tkBracketLe - of '{': - inc(my.bufpos) - result = tkCurlyLe - of ']': - inc(my.bufpos) - result = tkBracketRi - of '}': - inc(my.bufpos) - result = tkCurlyRi - of ',': - inc(my.bufpos) - result = tkComma - of ':': - inc(my.bufpos) - result = tkColon - of '\0': - result = tkEof - of 'a'..'z', 'A'..'Z', '_': - parseName(my) - case my.a - of "null": result = tkNull - of "true": result = tkTrue - of "false": result = tkFalse - else: result = tkError - else: - inc(my.bufpos) - result = tkError - my.tok = result - -proc next*(my: var JsonParser) = - ## retrieves the first/next event. This controls the parser. - var tk = getTok(my) - var i = my.state.len-1 - # the following code is a state machine. If we had proper coroutines, - # the code could be much simpler. - case my.state[i] - of stateEof: - if tk == tkEof: - my.kind = jsonEof - else: - my.kind = jsonError - my.err = errEofExpected - of stateStart: - # tokens allowed? - case tk - of tkString, tkInt, tkFloat, tkTrue, tkFalse, tkNull: - my.state[i] = stateEof # expect EOF next! - my.kind = JsonEventKind(ord(tk)) - of tkBracketLe: - my.state.add(stateArray) # we expect any - my.kind = jsonArrayStart - of tkCurlyLe: - my.state.add(stateObject) - my.kind = jsonObjectStart - of tkEof: - my.kind = jsonEof - else: - my.kind = jsonError - my.err = errEofExpected - of stateObject: - case tk - of tkString, tkInt, tkFloat, tkTrue, tkFalse, tkNull: - my.state.add(stateExpectColon) - my.kind = JsonEventKind(ord(tk)) - of tkBracketLe: - my.state.add(stateExpectColon) - my.state.add(stateArray) - my.kind = jsonArrayStart - of tkCurlyLe: - my.state.add(stateExpectColon) - my.state.add(stateObject) - my.kind = jsonObjectStart - of tkCurlyRi: - my.kind = jsonObjectEnd - discard my.state.pop() - else: - my.kind = jsonError - my.err = errCurlyRiExpected - of stateArray: - case tk - of tkString, tkInt, tkFloat, tkTrue, tkFalse, tkNull: - my.state.add(stateExpectArrayComma) # expect value next! - my.kind = JsonEventKind(ord(tk)) - of tkBracketLe: - my.state.add(stateExpectArrayComma) - my.state.add(stateArray) - my.kind = jsonArrayStart - of tkCurlyLe: - my.state.add(stateExpectArrayComma) - my.state.add(stateObject) - my.kind = jsonObjectStart - of tkBracketRi: - my.kind = jsonArrayEnd - discard my.state.pop() - else: - my.kind = jsonError - my.err = errBracketRiExpected - of stateExpectArrayComma: - case tk - of tkComma: - discard my.state.pop() - next(my) - of tkBracketRi: - my.kind = jsonArrayEnd - discard my.state.pop() # pop stateExpectArrayComma - discard my.state.pop() # pop stateArray - else: - my.kind = jsonError - my.err = errBracketRiExpected - of stateExpectObjectComma: - case tk - of tkComma: - discard my.state.pop() - next(my) - of tkCurlyRi: - my.kind = jsonObjectEnd - discard my.state.pop() # pop stateExpectObjectComma - discard my.state.pop() # pop stateObject - else: - my.kind = jsonError - my.err = errCurlyRiExpected - of stateExpectColon: - case tk - of tkColon: - my.state[i] = stateExpectValue - next(my) - else: - my.kind = jsonError - my.err = errColonExpected - of stateExpectValue: - case tk - of tkString, tkInt, tkFloat, tkTrue, tkFalse, tkNull: - my.state[i] = stateExpectObjectComma - my.kind = JsonEventKind(ord(tk)) - of tkBracketLe: - my.state[i] = stateExpectObjectComma - my.state.add(stateArray) - my.kind = jsonArrayStart - of tkCurlyLe: - my.state[i] = stateExpectObjectComma - my.state.add(stateObject) - my.kind = jsonObjectStart - else: - my.kind = jsonError - my.err = errExprExpected - - -# ------------- higher level interface --------------------------------------- - -type JsonNodeKind* = enum ## possible JSON node types JNull, JBool, @@ -620,15 +132,6 @@ type of JArray: elems*: seq[JsonNode] - JsonParsingError* = object of ValueError ## is raised for a JSON error - -{.deprecated: [EJsonParsingError: JsonParsingError, TJsonNode: JsonNodeObj, - PJsonNode: JsonNode, TJsonNodeKind: JsonNodeKind].} - -proc raiseParseErr*(p: JsonParser, msg: string) {.noinline, noreturn.} = - ## raises an `EJsonParsingError` exception. - raise newException(JsonParsingError, errorMsgExpected(p, msg)) - proc newJString*(s: string): JsonNode = ## Creates a new `JString JsonNode`. new(result) @@ -695,8 +198,8 @@ proc getBiggestInt*(n: JsonNode, default: BiggestInt = 0): BiggestInt = if n.isNil or n.kind != JInt: return default else: return n.num -proc getNum*(n: JsonNode, default: BiggestInt = 0): BiggestInt {.deprecated.} = - ## Deprecated - use getInt or getBiggestInt instead +proc getNum*(n: JsonNode, default: BiggestInt = 0): BiggestInt {.deprecated: "use getInt or getBiggestInt instead".} = + ## **Deprecated since v0.18.2:** use ``getInt`` or ``getBiggestInt`` instead. getBiggestInt(n, default) proc getFloat*(n: JsonNode, default: float = 0.0): float = @@ -709,8 +212,8 @@ proc getFloat*(n: JsonNode, default: float = 0.0): float = of JInt: return float(n.num) else: return default -proc getFNum*(n: JsonNode, default: float = 0.0): float {.deprecated.} = - ## Deprecated - use getFloat instead +proc getFNum*(n: JsonNode, default: float = 0.0): float {.deprecated: "use getFloat instead".} = + ## **Deprecated since v0.18.2:** use ``getFloat`` instead. getFloat(n, default) proc getBool*(n: JsonNode, default: bool = false): bool = @@ -720,8 +223,8 @@ proc getBool*(n: JsonNode, default: bool = false): bool = if n.isNil or n.kind != JBool: return default else: return n.bval -proc getBVal*(n: JsonNode, default: bool = false): bool {.deprecated.} = - ## Deprecated - use getBVal instead +proc getBVal*(n: JsonNode, default: bool = false): bool {.deprecated: "use getBool instead".} = + ## **Deprecated since v0.18.2:** use ``getBool`` instead. getBool(n, default) proc getFields*(n: JsonNode, @@ -734,7 +237,7 @@ proc getFields*(n: JsonNode, else: return n.fields proc getElems*(n: JsonNode, default: seq[JsonNode] = @[]): seq[JsonNode] = - ## Retrieves the int value of a `JArray JsonNode`. + ## Retrieves the array of a `JArray JsonNode`. ## ## Returns ``default`` if ``n`` is not a ``JArray``, or if ``n`` is nil. if n.isNil or n.kind != JArray: return default @@ -753,7 +256,6 @@ proc add*(obj: JsonNode, key: string, val: JsonNode) = proc `%`*(s: string): JsonNode = ## Generic constructor for JSON data. Creates a new `JString JsonNode`. new(result) - if s.isNil: return result.kind = JString result.str = s @@ -825,21 +327,24 @@ proc toJson(x: NimNode): NimNode {.compiletime.} = result = newNimNode(nnkBracket) for i in 0 ..< x.len: result.add(toJson(x[i])) - result = newCall(bindSym"%", result) + result = newCall(bindSym("%", brOpen), result) of nnkTableConstr: # object if x.len == 0: return newCall(bindSym"newJObject") result = newNimNode(nnkTableConstr) for i in 0 ..< x.len: x[i].expectKind nnkExprColonExpr result.add newTree(nnkExprColonExpr, x[i][0], toJson(x[i][1])) - result = newCall(bindSym"%", result) + result = newCall(bindSym("%", brOpen), result) of nnkCurly: # empty object x.expectLen(0) result = newCall(bindSym"newJObject") of nnkNilLit: result = newCall(bindSym"newJNull") + of nnkPar: + if x.len == 1: result = toJson(x[0]) + else: result = newCall(bindSym("%", brOpen), x) else: - result = newCall(bindSym"%", x) + result = newCall(bindSym("%", brOpen), x) macro `%*`*(x: untyped): untyped = ## Convert an expression to a JsonNode directly, without having to specify @@ -946,8 +451,8 @@ proc contains*(node: JsonNode, val: JsonNode): bool = assert(node.kind == JArray) find(node.elems, val) >= 0 -proc existsKey*(node: JsonNode, key: string): bool {.deprecated.} = node.hasKey(key) - ## Deprecated for `hasKey` +proc existsKey*(node: JsonNode, key: string): bool {.deprecated: "use hasKey instead".} = node.hasKey(key) + ## **Deprecated:** use `hasKey` instead. proc `[]=`*(obj: JsonNode, key: string, val: JsonNode) {.inline.} = ## Sets a field from a `JObject`. @@ -958,12 +463,29 @@ proc `{}`*(node: JsonNode, keys: varargs[string]): JsonNode = ## Traverses the node and gets the given value. If any of the ## keys do not exist, returns ``nil``. Also returns ``nil`` if one of the ## intermediate data structures is not an object. + ## + ## This proc can be used to create tree structures on the + ## fly (sometimes called `autovivification`:idx:): + ## + ## .. code-block:: nim + ## myjson{"parent", "child", "grandchild"} = newJInt(1) + ## result = node for key in keys: if isNil(result) or result.kind != JObject: return nil result = result.fields.getOrDefault(key) +proc `{}`*(node: JsonNode, index: varargs[int]): JsonNode = + ## Traverses the node and gets the given value. If any of the + ## indexes do not exist, returns ``nil``. Also returns ``nil`` if one of the + ## intermediate data structures is not an array. + result = node + for i in index: + if isNil(result) or result.kind != JArray or i >= node.len: + return nil + result = result.elems[i] + proc getOrDefault*(node: JsonNode, key: string): JsonNode = ## Gets a field from a `node`. If `node` is nil or not an object or ## value at `key` does not exist, returns nil @@ -986,7 +508,7 @@ proc delete*(obj: JsonNode, key: string) = ## Deletes ``obj[key]``. assert(obj.kind == JObject) if not obj.fields.hasKey(key): - raise newException(IndexError, "key not in object") + raise newException(KeyError, "key not in object") obj.fields.del(key) proc copy*(p: JsonNode): JsonNode = @@ -1023,10 +545,9 @@ proc newIndent(curr, indent: int, ml: bool): int = proc nl(s: var string, ml: bool) = s.add(if ml: "\n" else: " ") -proc escapeJson*(s: string; result: var string) = - ## Converts a string `s` to its JSON representation. +proc escapeJsonUnquoted*(s: string; result: var string) = + ## Converts a string `s` to its JSON representation without quotes. ## Appends to ``result``. - result.add("\"") for c in s: case c of '\L': result.add("\\n") @@ -1035,12 +556,25 @@ proc escapeJson*(s: string; result: var string) = of '\t': result.add("\\t") of '\r': result.add("\\r") of '"': result.add("\\\"") + of '\0'..'\7': result.add("\\u000" & $ord(c)) + of '\14'..'\31': result.add("\\u00" & $ord(c)) of '\\': result.add("\\\\") else: result.add(c) + +proc escapeJsonUnquoted*(s: string): string = + ## Converts a string `s` to its JSON representation without quotes. + result = newStringOfCap(s.len + s.len shr 3) + escapeJsonUnquoted(s, result) + +proc escapeJson*(s: string; result: var string) = + ## Converts a string `s` to its JSON representation with quotes. + ## Appends to ``result``. + result.add("\"") + escapeJsonUnquoted(s, result) result.add("\"") proc escapeJson*(s: string): string = - ## Converts a string `s` to its JSON representation. + ## Converts a string `s` to its JSON representation with quotes. result = newStringOfCap(s.len + s.len shr 3) escapeJson(s, result) @@ -1180,10 +714,6 @@ iterator mpairs*(node: var JsonNode): tuple[key: string, val: var JsonNode] = for key, val in mpairs(node.fields): yield (key, val) -proc eat(p: var JsonParser, tok: TokKind) = - if p.tok == tok: discard getTok(p) - else: raiseParseErr(p, tokToStr[tok]) - proc parseJson(p: var JsonParser): JsonNode = ## Parses JSON from a JSON Parser `p`. case p.tok @@ -1239,10 +769,12 @@ when not defined(js): ## If `s` contains extra data, it will raise `JsonParsingError`. var p: JsonParser p.open(s, filename) - defer: p.close() - discard getTok(p) # read first token - result = p.parseJson() - eat(p, tkEof) # check if there is no extra data + try: + discard getTok(p) # read first token + result = p.parseJson() + eat(p, tkEof) # check if there is no extra data + finally: + p.close() proc parseJson*(buffer: string): JsonNode = ## Parses JSON from `buffer`. @@ -1260,7 +792,6 @@ else: from math import `mod` type JSObject = object - {.deprecated: [TJSObject: JSObject].} proc parseNativeJson(x: cstring): JSObject {.importc: "JSON.parse".} @@ -1482,6 +1013,13 @@ proc processElseBranch(recCaseNode, elseBranch, jsonNode, kindType, exprColonExpr.add(ifStmt) proc createConstructor(typeSym, jsonNode: NimNode): NimNode {.compileTime.} + +proc detectDistinctType(typeSym: NimNode): NimNode = + let + typeImpl = getTypeImpl(typeSym) + typeInst = getTypeInst(typeSym) + result = if typeImpl.typeKind == ntyDistinct: typeImpl else: typeInst + proc processObjField(field, jsonNode: NimNode): seq[NimNode] = ## Process a field from a ``RecList``. ## @@ -1500,8 +1038,8 @@ proc processObjField(field, jsonNode: NimNode): seq[NimNode] = # Add the field value. # -> jsonNode["`field`"] let indexedJsonNode = createJsonIndexer(jsonNode, $field) - exprColonExpr.add(createConstructor(getTypeInst(field), indexedJsonNode)) - + let typeNode = detectDistinctType(field) + exprColonExpr.add(createConstructor(typeNode, indexedJsonNode)) of nnkRecCase: # A "case" field that introduces a variant. let exprColonExpr = newNimNode(nnkExprColonExpr) @@ -1615,7 +1153,7 @@ proc processType(typeName: NimNode, obj: NimNode, result = quote do: ( verifyJsonKind(`jsonNode`, {JString, JNull}, astToStr(`jsonNode`)); - if `jsonNode`.kind == JNull: nil else: `jsonNode`.str + if `jsonNode`.kind == JNull: "" else: `jsonNode`.str ) of "biggestint": result = quote do: @@ -1687,7 +1225,7 @@ proc createConstructor(typeSym, jsonNode: NimNode): NimNode = result = quote do: ( - if `lenientJsonNode`.isNil: `workaround`[`optionGeneric`]() else: some[`optionGeneric`](`value`) + if `lenientJsonNode`.isNil or `jsonNode`.kind == JNull: `workaround`[`optionGeneric`]() else: some[`optionGeneric`](`value`) ) of "table", "orderedtable": let tableKeyType = typeSym[1] @@ -1726,7 +1264,7 @@ proc createConstructor(typeSym, jsonNode: NimNode): NimNode = let seqT = typeSym[1] let forLoopI = genSym(nskForVar, "i") let indexerNode = createJsonIndexer(jsonNode, forLoopI) - let constructorNode = createConstructor(seqT, indexerNode) + let constructorNode = createConstructor(detectDistinctType(seqT), indexerNode) # Create a statement expression containing a for loop. result = quote do: @@ -1762,17 +1300,35 @@ proc createConstructor(typeSym, jsonNode: NimNode): NimNode = # Handle all other types. let obj = getType(typeSym) - if obj.kind == nnkBracketExpr: + let typeNode = getTypeImpl(typeSym) + if typeNode.typeKind == ntyDistinct: + result = createConstructor(typeNode, jsonNode) + elif obj.kind == nnkBracketExpr: # When `Sym "Foo"` turns out to be a `ref object`. result = createConstructor(obj, jsonNode) else: result = processType(typeSym, obj, jsonNode, false) of nnkTupleTy: result = processType(typeSym, typeSym, jsonNode, false) - of nnkPar: + of nnkPar, nnkTupleConstr: # TODO: The fact that `jsonNode` here works to give a good line number # is weird. Specifying typeSym should work but doesn't. error("Use a named tuple instead of: " & $toStrLit(typeSym), jsonNode) + of nnkDistinctTy: + var baseType = typeSym + # solve nested distinct types + while baseType.typeKind == ntyDistinct: + let impl = getTypeImpl(baseType[0]) + if impl.typeKind != ntyDistinct: + baseType = baseType[0] + break + baseType = impl + let ret = createConstructor(baseType, jsonNode) + let typeInst = getTypeInst(typeSym) + result = quote do: + ( + `typeInst`(`ret`) + ) else: doAssert false, "Unable to create constructor for: " & $typeSym.kind @@ -1896,7 +1452,7 @@ macro to*(node: JsonNode, T: typedesc): untyped = ## doAssert data.person.age == 21 ## doAssert data.list == @[1, 2, 3, 4] - let typeNode = getTypeInst(T) + let typeNode = getTypeImpl(T) expectKind(typeNode, nnkBracketExpr) doAssert(($typeNode[0]).normalize == "typedesc") @@ -1974,22 +1530,23 @@ when isMainModule: doAssert parsedAgain["abc"].num == 5 # Bounds checking - try: - let a = testJson["a"][9] - doAssert(false, "EInvalidIndex not thrown") - except IndexError: - discard - try: - let a = testJson["a"][-1] - doAssert(false, "EInvalidIndex not thrown") - except IndexError: - discard - try: - doAssert(testJson["a"][0].num == 1, "Index doesn't correspond to its value") - except: - doAssert(false, "EInvalidIndex thrown for valid index") - - doAssert(testJson{"b"}.str=="asd", "Couldn't fetch a singly nested key with {}") + when compileOption("boundChecks"): + try: + let a = testJson["a"][9] + doAssert(false, "IndexError not thrown") + except IndexError: + discard + try: + let a = testJson["a"][-1] + doAssert(false, "IndexError not thrown") + except IndexError: + discard + try: + doAssert(testJson["a"][0].num == 1, "Index doesn't correspond to its value") + except: + doAssert(false, "IndexError thrown for valid index") + + doAssert(testJson{"b"}.getStr()=="asd", "Couldn't fetch a singly nested key with {}") doAssert(isNil(testJson{"nonexistent"}), "Non-existent keys should return nil") doAssert(isNil(testJson{"a", "b"}), "Indexing through a list should return nil") doAssert(isNil(testJson{"a", "b"}), "Indexing through a list should return nil") @@ -2060,7 +1617,10 @@ when isMainModule: var parsed2 = parseFile("tests/testdata/jsontest2.json") doAssert(parsed2{"repository", "description"}.str=="IRC Library for Haskell", "Couldn't fetch via multiply nested key using {}") + doAssert escapeJsonUnquoted("\10Foo🎃barÄ") == "\\nFoo🎃barÄ" + doAssert escapeJsonUnquoted("\0\7\20") == "\\u0000\\u0007\\u0020" # for #7887 doAssert escapeJson("\10Foo🎃barÄ") == "\"\\nFoo🎃barÄ\"" + doAssert escapeJson("\0\7\20") == "\"\\u0000\\u0007\\u0020\"" # for #7887 # Test with extra data when not defined(js): @@ -2079,5 +1639,3 @@ when isMainModule: # bug #6438 doAssert($ %*[] == "[]") doAssert($ %*{} == "{}") - - echo("Tests succeeded!") diff --git a/lib/pure/lenientops.nim b/lib/pure/lenientops.nim index b124a9512..cc7784c19 100644 --- a/lib/pure/lenientops.nim +++ b/lib/pure/lenientops.nim @@ -26,33 +26,33 @@ import typetraits -proc `+`*[I: SomeInteger, F: SomeReal](i: I, f: F): F {.noSideEffect, inline.} = +proc `+`*[I: SomeInteger, F: SomeFloat](i: I, f: F): F {.noSideEffect, inline.} = F(i) + f -proc `+`*[I: SomeInteger, F: SomeReal](f: F, i: I): F {.noSideEffect, inline.} = +proc `+`*[I: SomeInteger, F: SomeFloat](f: F, i: I): F {.noSideEffect, inline.} = f + F(i) -proc `-`*[I: SomeInteger, F: SomeReal](i: I, f: F): F {.noSideEffect, inline.} = +proc `-`*[I: SomeInteger, F: SomeFloat](i: I, f: F): F {.noSideEffect, inline.} = F(i) - f -proc `-`*[I: SomeInteger, F: SomeReal](f: F, i: I): F {.noSideEffect, inline.} = +proc `-`*[I: SomeInteger, F: SomeFloat](f: F, i: I): F {.noSideEffect, inline.} = f - F(i) -proc `*`*[I: SomeInteger, F: SomeReal](i: I, f: F): F {.noSideEffect, inline.} = +proc `*`*[I: SomeInteger, F: SomeFloat](i: I, f: F): F {.noSideEffect, inline.} = F(i) * f -proc `*`*[I: SomeInteger, F: SomeReal](f: F, i: I): F {.noSideEffect, inline.} = +proc `*`*[I: SomeInteger, F: SomeFloat](f: F, i: I): F {.noSideEffect, inline.} = f * F(i) -proc `/`*[I: SomeInteger, F: SomeReal](i: I, f: F): F {.noSideEffect, inline.} = +proc `/`*[I: SomeInteger, F: SomeFloat](i: I, f: F): F {.noSideEffect, inline.} = F(i) / f -proc `/`*[I: SomeInteger, F: SomeReal](f: F, i: I): F {.noSideEffect, inline.} = +proc `/`*[I: SomeInteger, F: SomeFloat](f: F, i: I): F {.noSideEffect, inline.} = f / F(i) -proc `<`*[I: SomeInteger, F: SomeReal](i: I, f: F): bool {.noSideEffect, inline.} = +proc `<`*[I: SomeInteger, F: SomeFloat](i: I, f: F): bool {.noSideEffect, inline.} = F(i) < f -proc `<`*[I: SomeInteger, F: SomeReal](f: F, i: I): bool {.noSideEffect, inline.} = +proc `<`*[I: SomeInteger, F: SomeFloat](f: F, i: I): bool {.noSideEffect, inline.} = f < F(i) -proc `<=`*[I: SomeInteger, F: SomeReal](i: I, f: F): bool {.noSideEffect, inline.} = +proc `<=`*[I: SomeInteger, F: SomeFloat](i: I, f: F): bool {.noSideEffect, inline.} = F(i) <= f -proc `<=`*[I: SomeInteger, F: SomeReal](f: F, i: I): bool {.noSideEffect, inline.} = +proc `<=`*[I: SomeInteger, F: SomeFloat](f: F, i: I): bool {.noSideEffect, inline.} = f <= F(i) # Note that we must not defined `>=` and `>`, because system.nim already has a diff --git a/lib/pure/lexbase.nim b/lib/pure/lexbase.nim index 15a390f0b..e38acd5ef 100644 --- a/lib/pure/lexbase.nim +++ b/lib/pure/lexbase.nim @@ -40,8 +40,6 @@ type offsetBase*: int # use ``offsetBase + bufpos`` to get the offset refillChars: set[char] -{.deprecated: [TBaseLexer: BaseLexer].} - const chrSize = sizeof(char) diff --git a/lib/pure/logging.nim b/lib/pure/logging.nim index 751fc0e8d..f2f5cac9e 100644 --- a/lib/pure/logging.nim +++ b/lib/pure/logging.nim @@ -96,10 +96,6 @@ when not defined(js): logFiles: int # how many log files already created, e.g. basename.1, basename.2... bufSize: int # size of output buffer (-1: use system defaults, 0: unbuffered, >0: fixed buffer size) - {.deprecated: [PFileLogger: FileLogger, PRollingFileLogger: RollingFileLogger].} - -{.deprecated: [TLevel: Level, PLogger: Logger, PConsoleLogger: ConsoleLogger].} - var level {.threadvar.}: Level ## global log filter handlers {.threadvar.}: seq[Logger] ## handlers with their own log levels @@ -107,14 +103,9 @@ var proc substituteLog*(frmt: string, level: Level, args: varargs[string, `$`]): string = ## Format a log message using the ``frmt`` format string, ``level`` and varargs. ## See the module documentation for the format string syntax. - const nilString = "nil" - var msgLen = 0 for arg in args: - if arg.isNil: - msgLen += nilString.len - else: - msgLen += arg.len + msgLen += arg.len result = newStringOfCap(frmt.len + msgLen + 20) var i = 0 while i < frmt.len: @@ -126,7 +117,7 @@ proc substituteLog*(frmt: string, level: Level, args: varargs[string, `$`]): str var v = "" let app = when defined(js): "" else: getAppFilename() while frmt[i] in IdentChars: - v.add(toLower(frmt[i])) + v.add(toLowerAscii(frmt[i])) inc(i) case v of "date": result.add(getDateStr()) @@ -141,10 +132,7 @@ proc substituteLog*(frmt: string, level: Level, args: varargs[string, `$`]): str of "levelname": result.add(LevelNames[level]) else: discard for arg in args: - if arg.isNil: - result.add(nilString) - else: - result.add(arg) + result.add(arg) method log*(logger: Logger, level: Level, args: varargs[string, `$`]) {. raises: [Exception], gcsafe, @@ -342,7 +330,6 @@ template fatal*(args: varargs[string, `$`]) = proc addHandler*(handler: Logger) = ## Adds ``handler`` to the list of handlers. - if handlers.isNil: handlers = @[] handlers.add(handler) proc getHandlers*(): seq[Logger] = diff --git a/lib/pure/marshal.nim b/lib/pure/marshal.nim index 6ee830786..b0bcfe535 100644 --- a/lib/pure/marshal.nim +++ b/lib/pure/marshal.nim @@ -98,8 +98,7 @@ proc storeAny(s: Stream, a: Any, stored: var IntSet) = of akProc, akPointer, akCString: s.write($a.getPointer.ptrToInt) of akString: var x = getString(a) - if isNil(x): s.write("null") - elif x.validateUtf8() == -1: s.write(escapeJson(x)) + if x.validateUtf8() == -1: s.write(escapeJson(x)) else: s.write("[") var i = 0 @@ -270,6 +269,8 @@ proc store*[T](s: Stream, data: T) = proc `$$`*[T](x: T): string = ## returns a string representation of `x`. + ## + ## Note: to serialize `x` to JSON use $(%x) from the ``json`` module var stored = initIntSet() var d: T shallowCopy(d, x) @@ -279,6 +280,17 @@ proc `$$`*[T](x: T): string = proc to*[T](data: string): T = ## reads data and transforms it to a ``T``. + runnableExamples: + type + Foo = object + id: int + bar: string + + let x = Foo(id: 1, bar: "baz") + # serialize + let y = ($$x) + # deserialize back to type 'Foo': + let z = y.to[:Foo] var tab = initTable[BiggestInt, pointer]() loadAny(newStringStream(data), toAny(result), tab) @@ -309,7 +321,6 @@ when not defined(testing) and isMainModule: Node = object next, prev: PNode data: string - {.deprecated: [TNode: Node].} proc buildList(): PNode = new(result) diff --git a/lib/pure/matchers.nim b/lib/pure/matchers.nim index 6366fee1a..97223ed01 100644 --- a/lib/pure/matchers.nim +++ b/lib/pure/matchers.nim @@ -12,7 +12,7 @@ ## **Warning:** This module is deprecated since version 0.14.0. {.deprecated.} -{.deadCodeElim: on.} +{.deadCodeElim: on.} # dce option deprecated {.push debugger:off .} # the user does not want to trace a part # of the standard library! @@ -29,21 +29,21 @@ proc validEmailAddress*(s: string): bool {.noSideEffect, chars = Letters + Digits + {'!','#','$','%','&', '\'','*','+','/','=','?','^','_','`','{','}','|','~','-','.'} var i = 0 - if s[i] notin chars or s[i] == '.': return false - while s[i] in chars: - if s[i] == '.' and s[i+1] == '.': return false + if i >= s.len or s[i] notin chars or s[i] == '.': return false + while i < s.len and s[i] in chars: + if i+1 < s.len and s[i] == '.' and s[i+1] == '.': return false inc(i) - if s[i] != '@': return false + if i >= s.len or s[i] != '@': return false var j = len(s)-1 - if s[j] notin Letters: return false + if j >= 0 and s[j] notin Letters: return false while j >= i and s[j] in Letters: dec(j) inc(i) # skip '@' - while s[i] in {'0'..'9', 'a'..'z', '-', '.'}: inc(i) - if s[i] != '\0': return false + while i < s.len and s[i] in {'0'..'9', 'a'..'z', '-', '.'}: inc(i) + if i != s.len: return false var x = substr(s, j+1) if len(x) == 2 and x[0] in Letters and x[1] in Letters: return true - case toLower(x) + case toLowerAscii(x) of "com", "org", "net", "gov", "mil", "biz", "info", "mobi", "name", "aero", "jobs", "museum": return true else: return false diff --git a/lib/pure/math.nim b/lib/pure/math.nim index cbd04a145..bc804eb86 100644 --- a/lib/pure/math.nim +++ b/lib/pure/math.nim @@ -21,6 +21,8 @@ include "system/inclrtl" {.push debugger:off .} # the user does not want to trace a part # of the standard library! +import bitops + proc binom*(n, k: int): int {.noSideEffect.} = ## Computes the binomial coefficient if k <= 0: return 1 @@ -29,15 +31,25 @@ proc binom*(n, k: int): int {.noSideEffect.} = for i in countup(2, k): result = (result * (n + 1 - i)) div i -proc fac*(n: int): int {.noSideEffect.} = +proc createFactTable[N: static[int]]: array[N, int] = + result[0] = 1 + for i in 1 ..< N: + result[i] = result[i - 1] * i + +proc fac*(n: int): int = ## Computes the faculty/factorial function. - result = 1 - for i in countup(2, n): - result = result * i + const factTable = + when sizeof(int) == 4: + createFactTable[13]() + else: + createFactTable[21]() + assert(n >= 0, $n & " must not be negative.") + assert(n < factTable.len, $n & " is too large to look up in the table") + factTable[n] {.push checks:off, line_dir:off, stack_trace:off.} -when defined(Posix) and not defined(haiku): +when defined(Posix): {.passl: "-lm".} const @@ -117,8 +129,14 @@ proc sum*[T](x: openArray[T]): T {.noSideEffect.} = ## If `x` is empty, 0 is returned. for i in items(x): result = result + i +proc prod*[T](x: openArray[T]): T {.noSideEffect.} = + ## Computes the product of the elements in ``x``. + ## If ``x`` is empty, 1 is returned. + result = 1.T + for i in items(x): result = result * i + {.push noSideEffect.} -when not defined(JS): +when not defined(JS): # C proc sqrt*(x: float32): float32 {.importc: "sqrtf", header: "<math.h>".} proc sqrt*(x: float64): float64 {.importc: "sqrt", header: "<math.h>".} ## Computes the square root of `x`. @@ -129,15 +147,45 @@ when not defined(JS): proc ln*(x: float32): float32 {.importc: "logf", header: "<math.h>".} proc ln*(x: float64): float64 {.importc: "log", header: "<math.h>".} ## Computes the natural log of `x` +else: # JS + proc sqrt*(x: float32): float32 {.importc: "Math.sqrt", nodecl.} + proc sqrt*(x: float64): float64 {.importc: "Math.sqrt", nodecl.} + + proc ln*(x: float32): float32 {.importc: "Math.log", nodecl.} + proc ln*(x: float64): float64 {.importc: "Math.log", nodecl.} + +proc log*[T: SomeFloat](x, base: T): T = + ## Computes the logarithm ``base`` of ``x`` + ln(x) / ln(base) + +when not defined(JS): # C proc log10*(x: float32): float32 {.importc: "log10f", header: "<math.h>".} proc log10*(x: float64): float64 {.importc: "log10", header: "<math.h>".} ## Computes the common logarithm (base 10) of `x` - proc log2*[T: float32|float64](x: T): T = return ln(x) / ln(2.0) - ## Computes the binary logarithm (base 2) of `x` proc exp*(x: float32): float32 {.importc: "expf", header: "<math.h>".} proc exp*(x: float64): float64 {.importc: "exp", header: "<math.h>".} ## Computes the exponential function of `x` (pow(E, x)) + proc sin*(x: float32): float32 {.importc: "sinf", header: "<math.h>".} + proc sin*(x: float64): float64 {.importc: "sin", header: "<math.h>".} + ## Computes the sine of `x` + proc cos*(x: float32): float32 {.importc: "cosf", header: "<math.h>".} + proc cos*(x: float64): float64 {.importc: "cos", header: "<math.h>".} + ## Computes the cosine of `x` + proc tan*(x: float32): float32 {.importc: "tanf", header: "<math.h>".} + proc tan*(x: float64): float64 {.importc: "tan", header: "<math.h>".} + ## Computes the tangent of `x` + + proc sinh*(x: float32): float32 {.importc: "sinhf", header: "<math.h>".} + proc sinh*(x: float64): float64 {.importc: "sinh", header: "<math.h>".} + ## Computes the hyperbolic sine of `x` + proc cosh*(x: float32): float32 {.importc: "coshf", header: "<math.h>".} + proc cosh*(x: float64): float64 {.importc: "cosh", header: "<math.h>".} + ## Computes the hyperbolic cosine of `x` + proc tanh*(x: float32): float32 {.importc: "tanhf", header: "<math.h>".} + proc tanh*(x: float64): float64 {.importc: "tanh", header: "<math.h>".} + ## Computes the hyperbolic tangent of `x` + proc arccos*(x: float32): float32 {.importc: "acosf", header: "<math.h>".} proc arccos*(x: float64): float64 {.importc: "acos", header: "<math.h>".} ## Computes the arc cosine of `x` @@ -154,52 +202,104 @@ when not defined(JS): ## results even when the resulting angle is near pi/2 or -pi/2 ## (`x` near 0). - proc cos*(x: float32): float32 {.importc: "cosf", header: "<math.h>".} - proc cos*(x: float64): float64 {.importc: "cos", header: "<math.h>".} - ## Computes the cosine of `x` - - proc cosh*(x: float32): float32 {.importc: "coshf", header: "<math.h>".} - proc cosh*(x: float64): float64 {.importc: "cosh", header: "<math.h>".} - ## Computes the hyperbolic cosine of `x` + proc arcsinh*(x: float32): float32 {.importc: "asinhf", header: "<math.h>".} + proc arcsinh*(x: float64): float64 {.importc: "asinh", header: "<math.h>".} + ## Computes the inverse hyperbolic sine of `x` + proc arccosh*(x: float32): float32 {.importc: "acoshf", header: "<math.h>".} + proc arccosh*(x: float64): float64 {.importc: "acosh", header: "<math.h>".} + ## Computes the inverse hyperbolic cosine of `x` + proc arctanh*(x: float32): float32 {.importc: "atanhf", header: "<math.h>".} + proc arctanh*(x: float64): float64 {.importc: "atanh", header: "<math.h>".} + ## Computes the inverse hyperbolic tangent of `x` + +else: # JS + proc log10*(x: float32): float32 {.importc: "Math.log10", nodecl.} + proc log10*(x: float64): float64 {.importc: "Math.log10", nodecl.} + proc log2*(x: float32): float32 {.importc: "Math.log2", nodecl.} + proc log2*(x: float64): float64 {.importc: "Math.log2", nodecl.} + proc exp*(x: float32): float32 {.importc: "Math.exp", nodecl.} + proc exp*(x: float64): float64 {.importc: "Math.exp", nodecl.} + proc sin*[T: float32|float64](x: T): T {.importc: "Math.sin", nodecl.} + proc cos*[T: float32|float64](x: T): T {.importc: "Math.cos", nodecl.} + proc tan*[T: float32|float64](x: T): T {.importc: "Math.tan", nodecl.} + + proc sinh*[T: float32|float64](x: T): T {.importc: "Math.sinh", nodecl.} + proc cosh*[T: float32|float64](x: T): T {.importc: "Math.cosh", nodecl.} + proc tanh*[T: float32|float64](x: T): T {.importc: "Math.tanh", nodecl.} + + proc arcsin*[T: float32|float64](x: T): T {.importc: "Math.asin", nodecl.} + proc arccos*[T: float32|float64](x: T): T {.importc: "Math.acos", nodecl.} + proc arctan*[T: float32|float64](x: T): T {.importc: "Math.atan", nodecl.} + proc arctan2*[T: float32|float64](y, x: T): T {.importC: "Math.atan2", nodecl.} + + proc arcsinh*[T: float32|float64](x: T): T {.importc: "Math.asinh", nodecl.} + proc arccosh*[T: float32|float64](x: T): T {.importc: "Math.acosh", nodecl.} + proc arctanh*[T: float32|float64](x: T): T {.importc: "Math.atanh", nodecl.} + +proc cot*[T: float32|float64](x: T): T = 1.0 / tan(x) + ## Computes the cotangent of `x` +proc sec*[T: float32|float64](x: T): T = 1.0 / cos(x) + ## Computes the secant of `x`. +proc csc*[T: float32|float64](x: T): T = 1.0 / sin(x) + ## Computes the cosecant of `x` + +proc coth*[T: float32|float64](x: T): T = 1.0 / tanh(x) + ## Computes the hyperbolic cotangent of `x` +proc sech*[T: float32|float64](x: T): T = 1.0 / cosh(x) + ## Computes the hyperbolic secant of `x` +proc csch*[T: float32|float64](x: T): T = 1.0 / sinh(x) + ## Computes the hyperbolic cosecant of `x` + +proc arccot*[T: float32|float64](x: T): T = arctan(1.0 / x) + ## Computes the inverse cotangent of `x` +proc arcsec*[T: float32|float64](x: T): T = arccos(1.0 / x) + ## Computes the inverse secant of `x` +proc arccsc*[T: float32|float64](x: T): T = arcsin(1.0 / x) + ## Computes the inverse cosecant of `x` + +proc arccoth*[T: float32|float64](x: T): T = arctanh(1.0 / x) + ## Computes the inverse hyperbolic cotangent of `x` +proc arcsech*[T: float32|float64](x: T): T = arccosh(1.0 / x) + ## Computes the inverse hyperbolic secant of `x` +proc arccsch*[T: float32|float64](x: T): T = arcsinh(1.0 / x) + ## Computes the inverse hyperbolic cosecant of `x` + +const windowsCC89 = defined(windows) and defined(bcc) + +when not defined(JS): # C proc hypot*(x, y: float32): float32 {.importc: "hypotf", header: "<math.h>".} proc hypot*(x, y: float64): float64 {.importc: "hypot", header: "<math.h>".} ## Computes the hypotenuse of a right-angle triangle with `x` and ## `y` as its base and height. Equivalent to ``sqrt(x*x + y*y)``. - proc sinh*(x: float32): float32 {.importc: "sinhf", header: "<math.h>".} - proc sinh*(x: float64): float64 {.importc: "sinh", header: "<math.h>".} - ## Computes the hyperbolic sine of `x` - proc sin*(x: float32): float32 {.importc: "sinf", header: "<math.h>".} - proc sin*(x: float64): float64 {.importc: "sin", header: "<math.h>".} - ## Computes the sine of `x` - - proc tan*(x: float32): float32 {.importc: "tanf", header: "<math.h>".} - proc tan*(x: float64): float64 {.importc: "tan", header: "<math.h>".} - ## Computes the tangent of `x` - proc tanh*(x: float32): float32 {.importc: "tanhf", header: "<math.h>".} - proc tanh*(x: float64): float64 {.importc: "tanh", header: "<math.h>".} - ## Computes the hyperbolic tangent of `x` - proc pow*(x, y: float32): float32 {.importc: "powf", header: "<math.h>".} proc pow*(x, y: float64): float64 {.importc: "pow", header: "<math.h>".} ## computes x to power raised of y. ## ## To compute power between integers, use `^` e.g. 2 ^ 6 - proc erf*(x: float32): float32 {.importc: "erff", header: "<math.h>".} - proc erf*(x: float64): float64 {.importc: "erf", header: "<math.h>".} - ## The error function - proc erfc*(x: float32): float32 {.importc: "erfcf", header: "<math.h>".} - proc erfc*(x: float64): float64 {.importc: "erfc", header: "<math.h>".} - ## The complementary error function - - proc lgamma*(x: float32): float32 {.importc: "lgammaf", header: "<math.h>".} - proc lgamma*(x: float64): float64 {.importc: "lgamma", header: "<math.h>".} - ## Natural log of the gamma function - proc tgamma*(x: float32): float32 {.importc: "tgammaf", header: "<math.h>".} - proc tgamma*(x: float64): float64 {.importc: "tgamma", header: "<math.h>".} - ## The gamma function + # TODO: add C89 version on windows + when not windowsCC89: + proc erf*(x: float32): float32 {.importc: "erff", header: "<math.h>".} + proc erf*(x: float64): float64 {.importc: "erf", header: "<math.h>".} + ## The error function + proc erfc*(x: float32): float32 {.importc: "erfcf", header: "<math.h>".} + proc erfc*(x: float64): float64 {.importc: "erfc", header: "<math.h>".} + ## The complementary error function + + proc gamma*(x: float32): float32 {.importc: "tgammaf", header: "<math.h>".} + proc gamma*(x: float64): float64 {.importc: "tgamma", header: "<math.h>".} + ## The gamma function + proc tgamma*(x: float32): float32 + {.deprecated: "use gamma instead", importc: "tgammaf", header: "<math.h>".} + proc tgamma*(x: float64): float64 + {.deprecated: "use gamma instead", importc: "tgamma", header: "<math.h>".} + ## The gamma function + ## **Deprecated since version 0.19.0**: Use ``gamma`` instead. + proc lgamma*(x: float32): float32 {.importc: "lgammaf", header: "<math.h>".} + proc lgamma*(x: float64): float64 {.importc: "lgamma", header: "<math.h>".} + ## Natural log of the gamma function proc floor*(x: float32): float32 {.importc: "floorf", header: "<math.h>".} proc floor*(x: float64): float64 {.importc: "floor", header: "<math.h>".} @@ -215,7 +315,7 @@ when not defined(JS): ## .. code-block:: nim ## echo ceil(-2.1) ## -2.0 - when defined(windows) and (defined(vcc) or defined(bcc)): + when windowsCC89: # MSVC 2010 don't have trunc/truncf # this implementation was inspired by Go-lang Math.Trunc proc truncImpl(f: float64): float64 = @@ -283,57 +383,31 @@ when not defined(JS): ## .. code-block:: nim ## echo trunc(PI) # 3.0 - proc fmod*(x, y: float32): float32 {.importc: "fmodf", header: "<math.h>".} - proc fmod*(x, y: float64): float64 {.importc: "fmod", header: "<math.h>".} + proc fmod*(x, y: float32): float32 {.deprecated, importc: "fmodf", header: "<math.h>".} + proc fmod*(x, y: float64): float64 {.deprecated, importc: "fmod", header: "<math.h>".} ## Computes the remainder of `x` divided by `y` ## ## .. code-block:: nim ## echo fmod(-2.5, 0.3) ## -0.1 -else: - proc trunc*(x: float32): float32 {.importc: "Math.trunc", nodecl.} - proc trunc*(x: float64): float64 {.importc: "Math.trunc", nodecl.} + proc `mod`*(x, y: float32): float32 {.importc: "fmodf", header: "<math.h>".} + proc `mod`*(x, y: float64): float64 {.importc: "fmod", header: "<math.h>".} + ## Computes the modulo operation for float operators. +else: # JS + proc hypot*[T: float32|float64](x, y: T): T = return sqrt(x*x + y*y) + proc pow*(x, y: float32): float32 {.importC: "Math.pow", nodecl.} + proc pow*(x, y: float64): float64 {.importc: "Math.pow", nodecl.} proc floor*(x: float32): float32 {.importc: "Math.floor", nodecl.} proc floor*(x: float64): float64 {.importc: "Math.floor", nodecl.} proc ceil*(x: float32): float32 {.importc: "Math.ceil", nodecl.} proc ceil*(x: float64): float64 {.importc: "Math.ceil", nodecl.} - - proc sqrt*(x: float32): float32 {.importc: "Math.sqrt", nodecl.} - proc sqrt*(x: float64): float64 {.importc: "Math.sqrt", nodecl.} - proc ln*(x: float32): float32 {.importc: "Math.log", nodecl.} - proc ln*(x: float64): float64 {.importc: "Math.log", nodecl.} - proc log10*[T: float32|float64](x: T): T = return ln(x) / ln(10.0) - proc log2*[T: float32|float64](x: T): T = return ln(x) / ln(2.0) - - proc exp*(x: float32): float32 {.importc: "Math.exp", nodecl.} - proc exp*(x: float64): float64 {.importc: "Math.exp", nodecl.} proc round0(x: float): float {.importc: "Math.round", nodecl.} + proc trunc*(x: float32): float32 {.importc: "Math.trunc", nodecl.} + proc trunc*(x: float64): float64 {.importc: "Math.trunc", nodecl.} - proc pow*(x, y: float32): float32 {.importC: "Math.pow", nodecl.} - proc pow*(x, y: float64): float64 {.importc: "Math.pow", nodecl.} - - proc arccos*(x: float32): float32 {.importc: "Math.acos", nodecl.} - proc arccos*(x: float64): float64 {.importc: "Math.acos", nodecl.} - proc arcsin*(x: float32): float32 {.importc: "Math.asin", nodecl.} - proc arcsin*(x: float64): float64 {.importc: "Math.asin", nodecl.} - proc arctan*(x: float32): float32 {.importc: "Math.atan", nodecl.} - proc arctan*(x: float64): float64 {.importc: "Math.atan", nodecl.} - proc arctan2*(y, x: float32): float32 {.importC: "Math.atan2", nodecl.} - proc arctan2*(y, x: float64): float64 {.importc: "Math.atan2", nodecl.} - - proc cos*(x: float32): float32 {.importc: "Math.cos", nodecl.} - proc cos*(x: float64): float64 {.importc: "Math.cos", nodecl.} - proc cosh*(x: float32): float32 = return (exp(x)+exp(-x))*0.5 - proc cosh*(x: float64): float64 = return (exp(x)+exp(-x))*0.5 - proc hypot*[T: float32|float64](x, y: T): T = return sqrt(x*x + y*y) - proc sinh*[T: float32|float64](x: T): T = return (exp(x)-exp(-x))*0.5 - proc sin*(x: float32): float32 {.importc: "Math.sin", nodecl.} - proc sin*(x: float64): float64 {.importc: "Math.sin", nodecl.} - proc tan*(x: float32): float32 {.importc: "Math.tan", nodecl.} - proc tan*(x: float64): float64 {.importc: "Math.tan", nodecl.} - proc tanh*[T: float32|float64](x: T): T = - var y = exp(2.0*x) - return (y-1.0)/(y+1.0) + proc `mod`*(x, y: float32): float32 {.importcpp: "# % #".} + proc `mod`*(x, y: float64): float64 {.importcpp: "# % #".} + ## Computes the modulo operation for float operators. proc round*[T: float32|float64](x: T, places: int = 0): T = ## Round a floating point number. @@ -350,6 +424,21 @@ proc round*[T: float32|float64](x: T, places: int = 0): T = var mult = pow(10.0, places.T) result = round0(x*mult)/mult +proc floorDiv*[T: SomeInteger](x, y: T): T = + ## Floor division is conceptually defined as ``floor(x / y)``. + ## This is different from the ``div`` operator, which is defined + ## as ``trunc(x / y)``. That is, ``div`` rounds towards ``0`` and ``floorDiv`` + ## rounds down. + result = x div y + let r = x mod y + if (r > 0 and y < 0) or (r < 0 and y > 0): result.dec 1 + +proc floorMod*[T: SomeNumber](x, y: T): T = + ## Floor modulus is conceptually defined as ``x - (floorDiv(x, y) * y). + ## This proc behaves the same as the ``%`` operator in python. + result = x mod y + if (result > 0 and y < 0) or (result < 0 and y > 0): result += y + when not defined(JS): proc c_frexp*(x: float32, exponent: var int32): float32 {. importc: "frexp", header: "<math.h>".} @@ -364,6 +453,28 @@ when not defined(JS): var exp: int32 result = c_frexp(x, exp) exponent = exp + + when windowsCC89: + # taken from Go-lang Math.Log2 + const ln2 = 0.693147180559945309417232121458176568075500134360255254120680009 + template log2Impl[T](x: T): T = + var exp: int32 + var frac = frexp(x, exp) + # Make sure exact powers of two give an exact answer. + # Don't depend on Log(0.5)*(1/Ln2)+exp being exactly exp-1. + if frac == 0.5: return T(exp - 1) + log10(frac)*(1/ln2) + T(exp) + + proc log2*(x: float32): float32 = log2Impl(x) + proc log2*(x: float64): float64 = log2Impl(x) + ## Log2 returns the binary logarithm of x. + ## The special cases are the same as for Log. + + else: + proc log2*(x: float32): float32 {.importc: "log2f", header: "<math.h>".} + proc log2*(x: float64): float64 {.importc: "log2", header: "<math.h>".} + ## Computes the binary logarithm (base 2) of `x` + else: proc frexp*[T: float32|float64](x: T, exponent: var int): T = if x == 0.0: @@ -414,21 +525,12 @@ proc sgn*[T: SomeNumber](x: T): int {.inline.} = ## `NaN`. ord(T(0) < x) - ord(x < T(0)) -proc `mod`*[T: float32|float64](x, y: T): T = - ## Computes the modulo operation for float operators. Equivalent - ## to ``x - y * floor(x/y)``. Note that the remainder will always - ## have the same sign as the divisor. - ## - ## .. code-block:: nim - ## echo (4.0 mod -3.1) # -2.2 - result = if y == 0.0: x else: x - y * (x/y).floor - {.pop.} {.pop.} proc `^`*[T](x: T, y: Natural): T = - ## Computes ``x`` to the power ``y`. ``x`` must be non-negative, use - ## `pow <#pow,float,float>` for negative exponents. + ## Computes ``x`` to the power ``y``. ``x`` must be non-negative, use + ## `pow <#pow,float,float>`_ for negative exponents. when compiles(y >= T(0)): assert y >= T(0) else: @@ -445,26 +547,53 @@ proc `^`*[T](x: T, y: Natural): T = x *= x proc gcd*[T](x, y: T): T = - ## Computes the greatest common divisor of ``x`` and ``y``. + ## Computes the greatest common (positive) divisor of ``x`` and ``y``. ## Note that for floats, the result cannot always be interpreted as ## "greatest decimal `z` such that ``z*N == x and z*M == y`` ## where N and M are positive integers." - var (x,y) = (x,y) + var (x, y) = (x, y) while y != 0: x = x mod y swap x, y abs x +proc gcd*(x, y: SomeInteger): SomeInteger = + ## Computes the greatest common (positive) divisor of ``x`` and ``y``. + ## Using binary GCD (aka Stein's) algorithm. + when x is SomeSignedInt: + var x = abs(x) + else: + var x = x + when y is SomeSignedInt: + var y = abs(y) + else: + var y = y + + if x == 0: + return y + if y == 0: + return x + + let shift = countTrailingZeroBits(x or y) + y = y shr countTrailingZeroBits(y) + while x != 0: + x = x shr countTrailingZeroBits(x) + if y > x: + swap y, x + x -= y + y shl shift + proc lcm*[T](x, y: T): T = ## Computes the least common multiple of ``x`` and ``y``. x div gcd(x, y) * y -when isMainModule and not defined(JS): +when isMainModule and not defined(JS) and not windowsCC89: # Check for no side effect annotation proc mySqrt(num: float): float {.noSideEffect.} = return sqrt(num) # check gamma function + assert(gamma(5.0) == 24.0) # 4! assert($tgamma(5.0) == $24.0) # 4! assert(lgamma(1.0) == 0.0) # ln(1.0) == 0.0 assert(erf(6.0) > erf(5.0)) @@ -474,6 +603,12 @@ when isMainModule: # Function for approximate comparison of floats proc `==~`(x, y: float): bool = (abs(x-y) < 1e-9) + block: # prod + doAssert prod([1, 2, 3, 4]) == 24 + doAssert prod([1.5, 3.4]) == 5.1 + let x: seq[float] = @[] + doAssert prod(x) == 1.0 + block: # round() tests # Round to 0 decimal places doAssert round(54.652) ==~ 55.0 @@ -550,3 +685,44 @@ when isMainModule: assert sgn(Inf) == 1 assert sgn(NaN) == 0 + block: # fac() tests + try: + discard fac(-1) + except AssertionError: + discard + + doAssert fac(0) == 1 + doAssert fac(1) == 1 + doAssert fac(2) == 2 + doAssert fac(3) == 6 + doAssert fac(4) == 24 + + block: # floorMod/floorDiv + doAssert floorDiv(8, 3) == 2 + doAssert floorMod(8, 3) == 2 + + doAssert floorDiv(8, -3) == -3 + doAssert floorMod(8, -3) == -1 + + doAssert floorDiv(-8, 3) == -3 + doAssert floorMod(-8, 3) == 1 + + doAssert floorDiv(-8, -3) == 2 + doAssert floorMod(-8, -3) == -2 + + doAssert floorMod(8.0, -3.0) ==~ -1.0 + doAssert floorMod(-8.5, 3.0) ==~ 0.5 + + block: # log + doAssert log(4.0, 3.0) == ln(4.0) / ln(3.0) + doAssert log2(8.0'f64) == 3.0'f64 + doAssert log2(4.0'f64) == 2.0'f64 + doAssert log2(2.0'f64) == 1.0'f64 + doAssert log2(1.0'f64) == 0.0'f64 + doAssert classify(log2(0.0'f64)) == fcNegInf + + doAssert log2(8.0'f32) == 3.0'f32 + doAssert log2(4.0'f32) == 2.0'f32 + doAssert log2(2.0'f32) == 1.0'f32 + doAssert log2(1.0'f32) == 0.0'f32 + doAssert classify(log2(0.0'f32)) == fcNegInf diff --git a/lib/pure/memfiles.nim b/lib/pure/memfiles.nim index 5c73381ff..9fccd08d4 100644 --- a/lib/pure/memfiles.nim +++ b/lib/pure/memfiles.nim @@ -22,7 +22,11 @@ elif defined(posix): else: {.error: "the memfiles module is not supported on your operating system!".} -import os +import os, streams + +proc newEIO(msg: string): ref IOError = + new(result) + result.msg = msg type MemFile* = object ## represents a memory mapped file @@ -38,19 +42,20 @@ type else: handle: cint -{.deprecated: [TMemFile: MemFile].} - proc mapMem*(m: var MemFile, mode: FileMode = fmRead, mappedSize = -1, offset = 0): pointer = ## returns a pointer to a mapped portion of MemFile `m` ## ## ``mappedSize`` of ``-1`` maps to the whole file, and ## ``offset`` must be multiples of the PAGE SIZE of your OS + if mode == fmAppend: + raise newEIO("The append mode is not supported.") + var readonly = mode == fmRead when defined(windows): result = mapViewOfFileEx( m.mapHandle, - if readonly: FILE_MAP_READ else: FILE_MAP_WRITE, + if readonly: FILE_MAP_READ else: FILE_MAP_READ or FILE_MAP_WRITE, int32(offset shr 32), int32(offset and 0xffffffff), if mappedSize == -1: 0 else: mappedSize, @@ -115,6 +120,9 @@ proc open*(filename: string, mode: FileMode = fmRead, ## mm_half = memfiles.open("/tmp/test.mmap", mode = fmReadWrite, mappedSize = 512) # The file can be resized only when write mode is used: + if mode == fmAppend: + raise newEIO("The append mode is not supported.") + assert newFileSize == -1 or mode != fmRead var readonly = mode == fmRead @@ -123,6 +131,10 @@ proc open*(filename: string, mode: FileMode = fmRead, result.size = 0 when defined(windows): + let desiredAccess = GENERIC_READ + let shareMode = FILE_SHARE_READ + let flags = FILE_FLAG_RANDOM_ACCESS + template fail(errCode: OSErrorCode, msg: untyped) = rollback() if result.fHandle != 0: discard closeHandle(result.fHandle) @@ -135,11 +147,11 @@ proc open*(filename: string, mode: FileMode = fmRead, winApiProc( filename, # GENERIC_ALL != (GENERIC_READ or GENERIC_WRITE) - if readonly: GENERIC_READ else: GENERIC_READ or GENERIC_WRITE, - FILE_SHARE_READ, + if readonly: desiredAccess else: desiredAccess or GENERIC_WRITE, + if readonly: shareMode else: shareMode or FILE_SHARE_WRITE, nil, if newFileSize != -1: CREATE_ALWAYS else: OPEN_EXISTING, - if readonly: FILE_ATTRIBUTE_READONLY else: FILE_ATTRIBUTE_TEMPORARY, + if readonly: FILE_ATTRIBUTE_READONLY or flags else: FILE_ATTRIBUTE_NORMAL or flags, 0) when useWinUnicode: @@ -174,7 +186,7 @@ proc open*(filename: string, mode: FileMode = fmRead, result.mem = mapViewOfFileEx( result.mapHandle, - if readonly: FILE_MAP_READ else: FILE_MAP_WRITE, + if readonly: FILE_MAP_READ else: FILE_MAP_READ or FILE_MAP_WRITE, int32(offset shr 32), int32(offset and 0xffffffff), if mappedSize == -1: 0 else: mappedSize, @@ -247,6 +259,28 @@ proc open*(filename: string, mode: FileMode = fmRead, if close(result.handle) == 0: result.handle = -1 +proc flush*(f: var MemFile; attempts: Natural = 3) = + ## Flushes `f`'s buffer for the number of attempts equal to `attempts`. + ## If were errors an exception `OSError` will be raised. + var res = false + var lastErr: OSErrorCode + when defined(windows): + for i in 1..attempts: + res = flushViewOfFile(f.mem, 0) != 0 + if res: + break + lastErr = osLastError() + if lastErr != ERROR_LOCK_VIOLATION.OSErrorCode: + raiseOSError(lastErr) + else: + for i in 1..attempts: + res = msync(f.mem, f.size, MS_SYNC or MS_INVALIDATE) == 0 + if res: + break + lastErr = osLastError() + if lastErr != EBUSY.OSErrorCode: + raiseOSError(lastErr, "error flushing mapping") + proc close*(f: var MemFile) = ## closes the memory mapped file `f`. All changes are written back to the ## file system, if `f` was opened with write access. @@ -288,14 +322,12 @@ type MemSlice* = object ## represent slice of a MemFile for iteration over deli proc `==`*(x, y: MemSlice): bool = ## Compare a pair of MemSlice for strict equality. - proc memcmp(a, b: pointer, n:int):int {.importc: "memcmp",header: "string.h".} - result = (x.size == y.size and memcmp(x.data, y.data, x.size) == 0) + result = (x.size == y.size and equalMem(x.data, y.data, x.size)) proc `$`*(ms: MemSlice): string {.inline.} = ## Return a Nim string built from a MemSlice. var buf = newString(ms.size) copyMem(addr(buf[0]), ms.data, ms.size) - buf[ms.size] = '\0' result = buf iterator memSlices*(mfile: MemFile, delim='\l', eat='\r'): MemSlice {.inline.} = @@ -364,9 +396,9 @@ iterator lines*(mfile: MemFile, buf: var TaintedString, delim='\l', eat='\r'): T ## echo line for ms in memSlices(mfile, delim, eat): - buf.setLen(ms.size) - copyMem(addr(buf[0]), ms.data, ms.size) - buf[ms.size] = '\0' + setLen(buf.string, ms.size) + if ms.size > 0: + copyMem(addr buf[0], ms.data, ms.size) yield buf iterator lines*(mfile: MemFile, delim='\l', eat='\r'): TaintedString {.inline.} = @@ -384,3 +416,68 @@ iterator lines*(mfile: MemFile, delim='\l', eat='\r'): TaintedString {.inline.} var buf = TaintedString(newStringOfCap(80)) for line in lines(mfile, buf, delim, eat): yield buf + +type + MemMapFileStream* = ref MemMapFileStreamObj ## a stream that encapsulates a `MemFile` + MemMapFileStreamObj* = object of Stream + mf: MemFile + mode: FileMode + pos: ByteAddress + +proc mmsClose(s: Stream) = + MemMapFileStream(s).pos = -1 + close(MemMapFileStream(s).mf) + +proc mmsFlush(s: Stream) = flush(MemMapFileStream(s).mf) + +proc mmsAtEnd(s: Stream): bool = (MemMapFileStream(s).pos >= MemMapFileStream(s).mf.size) or + (MemMapFileStream(s).pos < 0) + +proc mmsSetPosition(s: Stream, pos: int) = + if pos > MemMapFileStream(s).mf.size or pos < 0: + raise newEIO("cannot set pos in stream") + MemMapFileStream(s).pos = pos + +proc mmsGetPosition(s: Stream): int = MemMapFileStream(s).pos + +proc mmsPeekData(s: Stream, buffer: pointer, bufLen: int): int = + let startAddress = cast[ByteAddress](MemMapFileStream(s).mf.mem) + let p = cast[ByteAddress](MemMapFileStream(s).pos) + let l = min(bufLen, MemMapFileStream(s).mf.size - p) + moveMem(buffer, cast[pointer](startAddress + p), l) + result = l + +proc mmsReadData(s: Stream, buffer: pointer, bufLen: int): int = + result = mmsPeekData(s, buffer, bufLen) + inc(MemMapFileStream(s).pos, result) + +proc mmsWriteData(s: Stream, buffer: pointer, bufLen: int) = + if MemMapFileStream(s).mode == fmRead: + raise newEIO("cannot write to read-only stream") + let size = MemMapFileStream(s).mf.size + if MemMapFileStream(s).pos + bufLen > size: + raise newEIO("cannot write to stream") + let p = cast[ByteAddress](MemMapFileStream(s).mf.mem) + + cast[ByteAddress](MemMapFileStream(s).pos) + moveMem(cast[pointer](p), buffer, bufLen) + inc(MemMapFileStream(s).pos, bufLen) + +proc newMemMapFileStream*(filename: string, mode: FileMode = fmRead, fileSize: int = -1): + MemMapFileStream = + ## creates a new stream from the file named `filename` with the mode `mode`. + ## Raises ## `EOS` if the file cannot be opened. See the `system + ## <system.html>`_ module for a list of available FileMode enums. + ## ``fileSize`` can only be set if the file does not exist and is opened + ## with write access (e.g., with fmReadWrite). + var mf: MemFile = open(filename, mode, newFileSize = fileSize) + new(result) + result.mode = mode + result.mf = mf + result.closeImpl = mmsClose + result.atEndImpl = mmsAtEnd + result.setPositionImpl = mmsSetPosition + result.getPositionImpl = mmsGetPosition + result.readDataImpl = mmsReadData + result.peekDataImpl = mmsPeekData + result.writeDataImpl = mmsWriteData + result.flushImpl = mmsFlush diff --git a/lib/pure/mersenne.nim b/lib/pure/mersenne.nim index 6ac0c4e56..b2227f114 100644 --- a/lib/pure/mersenne.nim +++ b/lib/pure/mersenne.nim @@ -12,8 +12,6 @@ type mt: array[0..623, uint32] index: int -{.deprecated: [TMersenneTwister: MersenneTwister].} - proc newMersenneTwister*(seed: uint32): MersenneTwister = result.index = 0 result.mt[0] = seed diff --git a/lib/pure/mimetypes.nim b/lib/pure/mimetypes.nim index b397ef47b..8f5f3a183 100644 --- a/lib/pure/mimetypes.nim +++ b/lib/pure/mimetypes.nim @@ -13,8 +13,6 @@ type MimeDB* = object mimes: StringTableRef -{.deprecated: [TMimeDB: MimeDB].} - const mimes* = { "ez": "application/andrew-inset", "anx": "application/annodex", @@ -233,6 +231,7 @@ const mimes* = { "xcf": "application/x-xcf", "fig": "application/x-xfig", "xpi": "application/x-xpinstall", + "wasm": "application/wasm", "amr": "audio/amr", "awb": "audio/amr-wb", "amr": "audio/amr", diff --git a/lib/pure/nativesockets.nim b/lib/pure/nativesockets.nim index 280c4e927..d5fb0f89b 100644 --- a/lib/pure/nativesockets.nim +++ b/lib/pure/nativesockets.nim @@ -31,7 +31,7 @@ else: export Sockaddr_storage, Sockaddr_un, Sockaddr_un_path_length export SocketHandle, Sockaddr_in, Addrinfo, INADDR_ANY, SockAddr, SockLen, - Sockaddr_in6, + Sockaddr_in6, Sockaddr_storage, inet_ntoa, recv, `==`, connect, send, accept, recvfrom, sendto, freeAddrInfo @@ -85,9 +85,6 @@ type length*: int addrList*: seq[string] -{.deprecated: [TPort: Port, TDomain: Domain, TType: SockType, - TProtocol: Protocol, TServent: Servent, THostent: Hostent].} - when useWinVersion: let osInvalidSocket* = winlean.INVALID_SOCKET @@ -251,9 +248,10 @@ proc getAddrInfo*(address: string, port: Port, domain: Domain = AF_INET, hints.ai_socktype = toInt(sockType) hints.ai_protocol = toInt(protocol) # OpenBSD doesn't support AI_V4MAPPED and doesn't define the macro AI_V4MAPPED. - # FreeBSD doesn't support AI_V4MAPPED but defines the macro. + # FreeBSD, Haiku don't support AI_V4MAPPED but defines the macro. # https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=198092 - when not defined(freebsd) and not defined(openbsd) and not defined(netbsd) and not defined(android): + # https://dev.haiku-os.org/ticket/14323 + when not defined(freebsd) and not defined(openbsd) and not defined(netbsd) and not defined(android) and not defined(haiku): if domain == AF_INET6: hints.ai_flags = AI_V4MAPPED var gaiResult = getaddrinfo(address, $port, addr(hints), result) @@ -395,7 +393,15 @@ proc getHostByAddr*(ip: string): Hostent {.tags: [ReadIOEffect].} = result.addrtype = AF_INET6 else: raiseOSError(osLastError(), "unknown h_addrtype") - result.addrList = cstringArrayToSeq(s.h_addr_list) + if result.addrtype == AF_INET: + result.addrlist = @[] + var i = 0 + while not isNil(s.h_addrlist[i]): + var inaddr_ptr = cast[ptr InAddr](s.h_addr_list[i]) + result.addrlist.add($inet_ntoa(inaddr_ptr[])) + inc(i) + else: + result.addrList = cstringArrayToSeq(s.h_addr_list) result.length = int(s.h_length) proc getHostByName*(name: string): Hostent {.tags: [ReadIOEffect].} = @@ -416,7 +422,15 @@ proc getHostByName*(name: string): Hostent {.tags: [ReadIOEffect].} = result.addrtype = AF_INET6 else: raiseOSError(osLastError(), "unknown h_addrtype") - result.addrList = cstringArrayToSeq(s.h_addr_list) + if result.addrtype == AF_INET: + result.addrlist = @[] + var i = 0 + while not isNil(s.h_addrlist[i]): + var inaddr_ptr = cast[ptr InAddr](s.h_addr_list[i]) + result.addrlist.add($inet_ntoa(inaddr_ptr[])) + inc(i) + else: + result.addrList = cstringArrayToSeq(s.h_addr_list) result.length = int(s.h_length) proc getHostname*(): string {.tags: [ReadIOEffect].} = @@ -600,8 +614,12 @@ proc setBlocking*(s: SocketHandle, blocking: bool) = proc timeValFromMilliseconds(timeout = 500): Timeval = if timeout != -1: var seconds = timeout div 1000 - result.tv_sec = seconds.int32 - result.tv_usec = ((timeout - seconds * 1000) * 1000).int32 + when useWinVersion: + result.tv_sec = seconds.int32 + result.tv_usec = ((timeout - seconds * 1000) * 1000).int32 + else: + result.tv_sec = seconds.Time + result.tv_usec = ((timeout - seconds * 1000) * 1000).Suseconds proc createFdSet(fd: var TFdSet, s: seq[SocketHandle], m: var int) = FD_ZERO(fd) @@ -620,7 +638,7 @@ proc pruneSocketSet(s: var seq[SocketHandle], fd: var TFdSet) = inc(i) setLen(s, L) -proc select*(readfds: var seq[SocketHandle], timeout = 500): int {.deprecated.} = +proc select*(readfds: var seq[SocketHandle], timeout = 500): int {.deprecated: "use selectRead instead".} = ## When a socket in ``readfds`` is ready to be read from then a non-zero ## value will be returned specifying the count of the sockets which can be ## read from. The sockets which can be read from will also be removed diff --git a/lib/pure/net.nim b/lib/pure/net.nim index af9eea51a..a60137dab 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -41,7 +41,7 @@ ## immediately. ## ## .. code-block:: Nim -## var socket = newSocket() +## var socket = newSocket(AF_INET, SOCK_DGRAM, IPPROTO_UDP) ## socket.sendTo("192.168.0.1", Port(27960), "status\n") ## ## Creating a server @@ -58,17 +58,13 @@ ## You can then begin accepting connections using the ``accept`` procedure. ## ## .. code-block:: Nim -## var client = new Socket +## var client: Socket ## var address = "" ## while true: ## socket.acceptAddr(client, address) ## echo("Client connected from: ", address) -## -## **Note:** The ``client`` variable is initialised with ``new Socket`` **not** -## ``newSocket()``. The difference is that the latter creates a new file -## descriptor. -{.deadCodeElim: on.} +{.deadCodeElim: on.} # dce option deprecated import nativesockets, os, strutils, parseutils, times, sets, options export Port, `$`, `==` export Domain, SockType, Protocol @@ -110,9 +106,6 @@ when defineSsl: serverGetPskFunc: SslServerGetPskFunc clientGetPskFunc: SslClientGetPskFunc - {.deprecated: [ESSL: SSLError, TSSLCVerifyMode: SSLCVerifyMode, - TSSLProtVersion: SSLProtVersion, PSSLContext: SSLContext, - TSSLAcceptResult: SSLAcceptResult].} else: type SslContext* = void # TODO: Workaround #4797. @@ -159,10 +152,6 @@ type Peek, SafeDisconn ## Ensures disconnection exceptions (ECONNRESET, EPIPE etc) are not thrown. -{.deprecated: [TSocketFlags: SocketFlag, ETimeout: TimeoutError, - TReadLineResult: ReadLineResult, TSOBool: SOBool, PSocket: Socket, - TSocketImpl: SocketImpl].} - type IpAddressFamily* {.pure.} = enum ## Describes the type of an IP address IPv6, ## IPv6 address @@ -176,8 +165,6 @@ type of IpAddressFamily.IPv4: address_v4*: array[0..3, uint8] ## Contains the IP address in bytes in ## case of IPv4 -{.deprecated: [TIpAddress: IpAddress].} - proc socketError*(socket: Socket, err: int = -1, async = false, lastError = (-1).OSErrorCode): void {.gcsafe.} @@ -240,7 +227,7 @@ proc newSocket*(domain: Domain = AF_INET, sockType: SockType = SOCK_STREAM, raiseOSError(osLastError()) result = newSocket(fd, domain, sockType, protocol, buffered) -proc parseIPv4Address(address_str: string): IpAddress = +proc parseIPv4Address(addressStr: string): IpAddress = ## Parses IPv4 adresses ## Raises EInvalidValue on errors var @@ -250,15 +237,15 @@ proc parseIPv4Address(address_str: string): IpAddress = result.family = IpAddressFamily.IPv4 - for i in 0 .. high(address_str): - if address_str[i] in strutils.Digits: # Character is a number + for i in 0 .. high(addressStr): + if addressStr[i] in strutils.Digits: # Character is a number currentByte = currentByte * 10 + - cast[uint16](ord(address_str[i]) - ord('0')) + cast[uint16](ord(addressStr[i]) - ord('0')) if currentByte > 255'u16: raise newException(ValueError, "Invalid IP Address. Value is out of range") seperatorValid = true - elif address_str[i] == '.': # IPv4 address separator + elif addressStr[i] == '.': # IPv4 address separator if not seperatorValid or byteCount >= 3: raise newException(ValueError, "Invalid IP Address. The address consists of too many groups") @@ -274,11 +261,11 @@ proc parseIPv4Address(address_str: string): IpAddress = raise newException(ValueError, "Invalid IP Address") result.address_v4[byteCount] = cast[uint8](currentByte) -proc parseIPv6Address(address_str: string): IpAddress = +proc parseIPv6Address(addressStr: string): IpAddress = ## Parses IPv6 adresses ## Raises EInvalidValue on errors result.family = IpAddressFamily.IPv6 - if address_str.len < 2: + if addressStr.len < 2: raise newException(ValueError, "Invalid IP Address") var @@ -291,7 +278,7 @@ proc parseIPv6Address(address_str: string): IpAddress = v4StartPos = -1 byteCount = 0 - for i,c in address_str: + for i,c in addressStr: if c == ':': if not seperatorValid: raise newException(ValueError, @@ -302,7 +289,7 @@ proc parseIPv6Address(address_str: string): IpAddress = "Invalid IP Address. Address contains more than one \"::\" seperator") dualColonGroup = groupCount seperatorValid = false - elif i != 0 and i != high(address_str): + elif i != 0 and i != high(addressStr): if groupCount >= 8: raise newException(ValueError, "Invalid IP Address. The address consists of too many groups") @@ -312,11 +299,11 @@ proc parseIPv6Address(address_str: string): IpAddress = groupCount.inc() if dualColonGroup != -1: seperatorValid = false elif i == 0: # only valid if address starts with :: - if address_str[1] != ':': + if addressStr[1] != ':': raise newException(ValueError, "Invalid IP Address. Address may not start with \":\"") - else: # i == high(address_str) - only valid if address ends with :: - if address_str[high(address_str)-1] != ':': + else: # i == high(addressStr) - only valid if address ends with :: + if addressStr[high(addressStr)-1] != ':': raise newException(ValueError, "Invalid IP Address. Address may not end with \":\"") lastWasColon = true @@ -354,7 +341,7 @@ proc parseIPv6Address(address_str: string): IpAddress = result.address_v6[groupCount*2+1] = cast[uint8](currentShort and 0xFF) groupCount.inc() else: # Must parse IPv4 address - for i,c in address_str[v4StartPos..high(address_str)]: + for i,c in addressStr[v4StartPos..high(addressStr)]: if c in strutils.Digits: # Character is a number currentShort = currentShort * 10 + cast[uint32](ord(c) - ord('0')) if currentShort > 255'u32: @@ -395,25 +382,69 @@ proc parseIPv6Address(address_str: string): IpAddress = raise newException(ValueError, "Invalid IP Address. The address consists of too many groups") -proc parseIpAddress*(address_str: string): IpAddress = +proc parseIpAddress*(addressStr: string): IpAddress = ## Parses an IP address ## Raises EInvalidValue on error - if address_str == nil: - raise newException(ValueError, "IP Address string is nil") - if address_str.contains(':'): - return parseIPv6Address(address_str) + if addressStr.len == 0: + raise newException(ValueError, "IP Address string is empty") + if addressStr.contains(':'): + return parseIPv6Address(addressStr) else: - return parseIPv4Address(address_str) + return parseIPv4Address(addressStr) -proc isIpAddress*(address_str: string): bool {.tags: [].} = +proc isIpAddress*(addressStr: string): bool {.tags: [].} = ## Checks if a string is an IP address ## Returns true if it is, false otherwise try: - discard parseIpAddress(address_str) + discard parseIpAddress(addressStr) except ValueError: return false return true +proc toSockAddr*(address: IpAddress, port: Port, sa: var Sockaddr_storage, + sl: var Socklen) = + ## Converts `IpAddress` and `Port` to `SockAddr` and `Socklen` + let port = htons(uint16(port)) + case address.family + of IpAddressFamily.IPv4: + sl = sizeof(Sockaddr_in).Socklen + let s = cast[ptr Sockaddr_in](addr sa) + s.sin_family = type(s.sin_family)(toInt(AF_INET)) + s.sin_port = port + copyMem(addr s.sin_addr, unsafeAddr address.address_v4[0], + sizeof(s.sin_addr)) + of IpAddressFamily.IPv6: + sl = sizeof(Sockaddr_in6).Socklen + let s = cast[ptr Sockaddr_in6](addr sa) + s.sin6_family = type(s.sin6_family)(toInt(AF_INET6)) + s.sin6_port = port + copyMem(addr s.sin6_addr, unsafeAddr address.address_v6[0], + sizeof(s.sin6_addr)) + +proc fromSockAddrAux(sa: ptr Sockaddr_storage, sl: Socklen, + address: var IpAddress, port: var Port) = + if sa.ss_family.int == toInt(AF_INET) and sl == sizeof(Sockaddr_in).Socklen: + address = IpAddress(family: IpAddressFamily.IPv4) + let s = cast[ptr Sockaddr_in](sa) + copyMem(addr address.address_v4[0], addr s.sin_addr, + sizeof(address.address_v4)) + port = ntohs(s.sin_port).Port + elif sa.ss_family.int == toInt(AF_INET6) and + sl == sizeof(Sockaddr_in6).Socklen: + address = IpAddress(family: IpAddressFamily.IPv6) + let s = cast[ptr Sockaddr_in6](sa) + copyMem(addr address.address_v6[0], addr s.sin6_addr, + sizeof(address.address_v6)) + port = ntohs(s.sin6_port).Port + else: + raise newException(ValueError, "Neither IPv4 nor IPv6") + +proc fromSockAddr*(sa: Sockaddr_storage | SockAddr | Sockaddr_in | Sockaddr_in6, + sl: Socklen, address: var IpAddress, port: var Port) {.inline.} = + ## Converts `SockAddr` and `Socklen` to `IpAddress` and `Port`. Raises + ## `ObjectConversionError` in case of invalid `sa` and `sl` arguments. + fromSockAddrAux(cast[ptr Sockaddr_storage](unsafeAddr sa), sl, address, port) + when defineSsl: CRYPTO_malloc_init() doAssert SslLibraryInit() == 1 @@ -552,7 +583,7 @@ when defineSsl: proc pskClientCallback(ssl: SslPtr; hint: cstring; identity: cstring; max_identity_len: cuint; psk: ptr cuchar; max_psk_len: cuint): cuint {.cdecl.} = let ctx = SSLContext(context: ssl.SSL_get_SSL_CTX) - let hintString = if hint == nil: nil else: $hint + let hintString = if hint == nil: "" else: $hint let (identityString, pskString) = (ctx.clientGetPskFunc)(hintString) if psk.len.cuint > max_psk_len: return 0 @@ -622,7 +653,7 @@ when defineSsl: proc wrapConnectedSocket*(ctx: SSLContext, socket: Socket, handshake: SslHandshakeType, - hostname: string = nil) = + hostname: string = "") = ## Wraps a connected socket in an SSL context. This function effectively ## turns ``socket`` into an SSL socket. ## ``hostname`` should be specified so that the client knows which hostname @@ -636,7 +667,7 @@ when defineSsl: wrapSocket(ctx, socket) case handshake of handshakeAsClient: - if not hostname.isNil and not isIpAddress(hostname): + if hostname.len > 0 and not isIpAddress(hostname): # Discard result in case OpenSSL version doesn't support SNI, or we're # not using TLSv1+ discard SSL_set_tlsext_host_name(socket.sslHandle, hostname) @@ -754,16 +785,12 @@ proc acceptAddr*(server: Socket, client: var Socket, address: var string, ## The resulting client will inherit any properties of the server socket. For ## example: whether the socket is buffered or not. ## - ## **Note**: ``client`` must be initialised (with ``new``), this function - ## makes no effort to initialise the ``client`` variable. - ## ## The ``accept`` call may result in an error if the connecting socket ## disconnects during the duration of the ``accept``. If the ``SafeDisconn`` ## flag is specified then this error will not be raised and instead ## accept will be called again. - assert(client != nil) - assert client.fd.int <= 0, "Client socket needs to be initialised with " & - "`new`, not `newSocket`." + if client.isNil: + new(client) let ret = accept(server.fd) let sock = ret[0] @@ -775,6 +802,7 @@ proc acceptAddr*(server: Socket, client: var Socket, address: var string, else: address = ret[1] client.fd = sock + client.domain = getSockDomain(sock) client.isBuffered = server.isBuffered # Handle SSL. @@ -843,9 +871,6 @@ proc accept*(server: Socket, client: var Socket, ## Equivalent to ``acceptAddr`` but doesn't return the address, only the ## socket. ## - ## **Note**: ``client`` must be initialised (with ``new``), this function - ## makes no effort to initialise the ``client`` variable. - ## ## The ``accept`` call may result in an error if the connecting socket ## disconnects during the duration of the ``accept``. If the ``SafeDisconn`` ## flag is specified then this error will not be raised and instead @@ -932,7 +957,7 @@ when defined(posix) and not defined(nimdoc): raise newException(ValueError, "socket path too long") copyMem(addr result.sun_path, path.cstring, path.len + 1) -when defined(posix): +when defined(posix) or defined(nimdoc): proc connectUnix*(socket: Socket, path: string) = ## Connects to Unix socket on `path`. ## This only works on Unix-style systems: Mac OS X, BSD and Linux @@ -1120,7 +1145,7 @@ proc waitFor(socket: Socket, waited: var float, timeout, size: int, return 1 let sslPending = SSLPending(socket.sslHandle) if sslPending != 0: - return sslPending + return min(sslPending, size) var startTime = epochTime() let selRet = select(socket, timeout - int(waited * 1000.0)) @@ -1303,6 +1328,7 @@ proc recvFrom*(socket: Socket, data: var string, length: int, ## used. Therefore if ``socket`` contains something in its buffer this ## function will make no effort to return it. + assert(socket.protocol != IPPROTO_TCP, "Cannot `recvFrom` on a TCP socket") # TODO: Buffered sockets data.setLen(length) var sockAddress: Sockaddr_in @@ -1372,23 +1398,25 @@ proc trySend*(socket: Socket, data: string): bool {.tags: [WriteIOEffect].} = result = send(socket, cstring(data), data.len) == data.len proc sendTo*(socket: Socket, address: string, port: Port, data: pointer, - size: int, af: Domain = AF_INET, flags = 0'i32): int {. + size: int, af: Domain = AF_INET, flags = 0'i32) {. tags: [WriteIOEffect].} = ## This proc sends ``data`` to the specified ``address``, ## which may be an IP address or a hostname, if a hostname is specified ## this function will try each IP of that hostname. ## + ## If an error occurs an OSError exception will be raised. ## ## **Note:** You may wish to use the high-level version of this function ## which is defined below. ## ## **Note:** This proc is not available for SSL sockets. + assert(socket.protocol != IPPROTO_TCP, "Cannot `sendTo` on a TCP socket") assert(not socket.isClosed, "Cannot `sendTo` on a closed socket") - var aiList = getAddrInfo(address, port, af) - + var aiList = getAddrInfo(address, port, af, socket.sockType, socket.protocol) # try all possibilities: var success = false var it = aiList + var result = 0 while it != nil: result = sendto(socket.fd, data, size.cint, flags.cint, it.ai_addr, it.ai_addrlen.SockLen) @@ -1397,16 +1425,22 @@ proc sendTo*(socket: Socket, address: string, port: Port, data: pointer, break it = it.ai_next + let osError = osLastError() freeAddrInfo(aiList) + if not success: + raiseOSError(osError) + proc sendTo*(socket: Socket, address: string, port: Port, - data: string): int {.tags: [WriteIOEffect].} = + data: string) {.tags: [WriteIOEffect].} = ## This proc sends ``data`` to the specified ``address``, ## which may be an IP address or a hostname, if a hostname is specified ## this function will try each IP of that hostname. ## + ## If an error occurs an OSError exception will be raised. + ## ## This is the high-level version of the above ``sendTo`` function. - result = socket.sendTo(address, port, cstring(data), data.len) + socket.sendTo(address, port, cstring(data), data.len, socket.domain) proc isSsl*(socket: Socket): bool = @@ -1664,6 +1698,9 @@ proc connect*(socket: Socket, address: string, port = Port(0), if selectWrite(s, timeout) != 1: raise newException(TimeoutError, "Call to 'connect' timed out.") else: + let res = getSockOptInt(socket.fd, SOL_SOCKET, SO_ERROR) + if res != 0: + raiseOSError(OSErrorCode(res)) when defineSsl and not defined(nimdoc): if socket.isSSL: socket.fd.setBlocking(true) diff --git a/lib/pure/nimprof.nim b/lib/pure/nimprof.nim index 4289eb049..506c6bfaa 100644 --- a/lib/pure/nimprof.nim +++ b/lib/pure/nimprof.nim @@ -30,7 +30,6 @@ when not declared(system.StackTrace): type StackTrace = object lines: array[0..20, cstring] files: array[0..20, cstring] - {.deprecated: [TStackTrace: StackTrace].} proc `[]`*(st: StackTrace, i: int): cstring = st.lines[i] # We use a simple hash table of bounded size to keep track of the stack traces: @@ -39,7 +38,6 @@ type total: int st: StackTrace ProfileData = array[0..64*1024-1, ptr ProfileEntry] -{.deprecated: [TProfileEntry: ProfileEntry, TProfileData: ProfileData].} proc `==`(a, b: StackTrace): bool = for i in 0 .. high(a.lines): diff --git a/lib/pure/oids.nim b/lib/pure/oids.nim index 427a68964..d6369b5f9 100644 --- a/lib/pure/oids.nim +++ b/lib/pure/oids.nim @@ -23,11 +23,9 @@ type fuzz: int32 ## count: int32 ## -{.deprecated: [Toid: Oid].} - proc `==`*(oid1: Oid, oid2: Oid): bool = - ## Compare two Mongo Object IDs for equality - return (oid1.time == oid2.time) and (oid1.fuzz == oid2.fuzz) and (oid1.count == oid2.count) + ## Compare two Mongo Object IDs for equality + return (oid1.time == oid2.time) and (oid1.fuzz == oid2.fuzz) and (oid1.count == oid2.count) proc hexbyte*(hex: char): int = case hex @@ -71,7 +69,7 @@ proc genOid*(): Oid = proc rand(): cint {.importc: "rand", header: "<stdlib.h>", nodecl.} proc srand(seed: cint) {.importc: "srand", header: "<stdlib.h>", nodecl.} - var t = getTime().int32 + var t = getTime().toUnix.int32 var i = int32(atomicInc(incr)) diff --git a/lib/pure/options.nim b/lib/pure/options.nim index 6d2869bff..12e38d8b5 100644 --- a/lib/pure/options.nim +++ b/lib/pure/options.nim @@ -70,26 +70,55 @@ import typetraits type + SomePointer = ref | ptr | pointer + +type Option*[T] = object ## An optional type that stores its value and state separately in a boolean. - val: T - has: bool + when T is SomePointer: + val: T + else: + val: T + has: bool + UnpackError* = ref object of ValueError proc some*[T](val: T): Option[T] = ## Returns a ``Option`` that has this value. - result.has = true + when T is SomePointer: + assert val != nil + result.val = val + else: + result.has = true + result.val = val + +proc option*[T](val: T): Option[T] = + ## Can be used to convert a pointer type to an option type. It + ## converts ``nil`` to the none-option. result.val = val + when T isnot SomePointer: + result.has = true proc none*(T: typedesc): Option[T] = - ## Returns a ``Option`` for this type that has no value. - result.has = false + ## Returns an ``Option`` for this type that has no value. + # the default is the none type + discard -proc isSome*[T](self: Option[T]): bool = - self.has +proc none*[T]: Option[T] = + ## Alias for ``none(T)``. + none(T) -proc isNone*[T](self: Option[T]): bool = - not self.has +proc isSome*[T](self: Option[T]): bool {.inline.} = + when T is SomePointer: + self.val != nil + else: + self.has + +proc isNone*[T](self: Option[T]): bool {.inline.} = + when T is SomePointer: + self.val == nil + else: + not self.has proc unsafeGet*[T](self: Option[T]): T = ## Returns the value of a ``some``. Behavior is undefined for ``none``. @@ -100,32 +129,39 @@ proc get*[T](self: Option[T]): T = ## Returns contents of the Option. If it is none, then an exception is ## thrown. if self.isNone: - raise UnpackError(msg : "Can't obtain a value from a `none`") + raise UnpackError(msg: "Can't obtain a value from a `none`") self.val proc get*[T](self: Option[T], otherwise: T): T = ## Returns the contents of this option or `otherwise` if the option is none. - if self.has: + if self.isSome: self.val else: otherwise +proc get*[T](self: var Option[T]): var T = + ## Returns contents of the Option. If it is none, then an exception is + ## thrown. + if self.isNone: + raise UnpackError(msg: "Can't obtain a value from a `none`") + return self.val + proc map*[T](self: Option[T], callback: proc (input: T)) = ## Applies a callback to the value in this Option - if self.has: + if self.isSome: callback(self.val) proc map*[T, R](self: Option[T], callback: proc (input: T): R): Option[R] = ## Applies a callback to the value in this Option and returns an option - ## containing the new value. If this option is None, None will be returned. - if self.has: - some[R](callback(self.val)) + ## containing the new value. If this option is None, None will be returned + if self.isSome: + some[R]( callback(self.val) ) else: none(R) proc flatten*[A](self: Option[Option[A]]): Option[A] = ## Remove one level of structure in a nested Option. - if self.has: + if self.isSome: self.val else: none(A) @@ -142,7 +178,7 @@ proc filter*[T](self: Option[T], callback: proc (input: T): bool): Option[T] = ## Applies a callback to the value in this Option. If the callback returns ## `true`, the option is returned as a Some. If it returns false, it is ## returned as a None. - if self.has and not callback(self.val): + if self.isSome and not callback(self.val): none(T) else: self @@ -150,17 +186,19 @@ proc filter*[T](self: Option[T], callback: proc (input: T): bool): Option[T] = proc `==`*(a, b: Option): bool = ## Returns ``true`` if both ``Option``s are ``none``, ## or if they have equal values - (a.has and b.has and a.val == b.val) or (not a.has and not b.has) + (a.isSome and b.isSome and a.val == b.val) or (not a.isSome and not b.isSome) proc `$`*[T](self: Option[T]): string = ## Get the string representation of this option. If the option has a value, ## the result will be `Some(x)` where `x` is the string representation of the contained value. - ## If the option does not have a value, the result will be `None[T]` where `T` is the name of + ## If the option does not have a value, the result will be `None[T]` where `T` is the name of ## the type contained in the option. - if self.has: - "Some(" & $self.val & ")" + if self.isSome: + result = "Some(" + result.addQuoted self.val + result.add ")" else: - "None[" & T.name & "]" + result = "None[" & name(T) & "]" when isMainModule: import unittest, sequtils @@ -211,7 +249,7 @@ when isMainModule: check(stringNone.get("Correct") == "Correct") test "$": - check($(some("Correct")) == "Some(Correct)") + check($(some("Correct")) == "Some(\"Correct\")") check($(stringNone) == "None[string]") test "map with a void result": @@ -256,3 +294,30 @@ when isMainModule: check(some(1).flatMap(maybeToString).flatMap(maybeExclaim) == some("1!")) check(some(0).flatMap(maybeToString).flatMap(maybeExclaim) == none(string)) + + test "SomePointer": + var intref: ref int + check(option(intref).isNone) + intref.new + check(option(intref).isSome) + + let tmp = option(intref) + check(sizeof(tmp) == sizeof(ptr int)) + + test "none[T]": + check(none[int]().isNone) + check(none(int) == none[int]()) + + test "$ on typed with .name": + type Named = object + name: string + + let nobody = none(Named) + check($nobody == "None[Named]") + + test "$ on type with name()": + type Person = object + myname: string + + let noperson = none(Person) + check($noperson == "None[Person]") diff --git a/lib/pure/os.nim b/lib/pure/os.nim index f8936f549..2b3cf5142 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -10,7 +10,7 @@ ## This module contains basic operating system facilities like ## retrieving environment variables, reading command line arguments, ## working with directories, running shell commands, etc. -{.deadCodeElim: on.} +{.deadCodeElim: on.} # dce option deprecated {.push debugger: off.} @@ -23,6 +23,10 @@ when defined(windows): import winlean elif defined(posix): import posix + + proc toTime(ts: Timespec): times.Time {.inline.} = + result = initTime(ts.tv_sec.int64, ts.tv_nsec.int) + else: {.error: "OS module not ported to your operating system!".} @@ -70,7 +74,8 @@ when defined(windows): proc existsFile*(filename: string): bool {.rtl, extern: "nos$1", tags: [ReadDirEffect].} = - ## Returns true if the file exists, false otherwise. + ## Returns true if `filename` exists and is a regular file or symlink. + ## (directories, device files, named pipes and sockets return false) when defined(windows): when useWinUnicode: wrapUnary(a, getFileAttributesW, filename) @@ -139,6 +144,7 @@ proc findExe*(exe: string, followSymlinks: bool = true; ## is added the `ExeExts <#ExeExts>`_ file extensions if it has none. ## If the system supports symlinks it also resolves them until it ## meets the actual file. This behavior can be disabled if desired. + if exe.len == 0: return template checkCurrentDir() = for ext in extensions: result = addFileExt(exe, ext) @@ -149,6 +155,7 @@ proc findExe*(exe: string, followSymlinks: bool = true; checkCurrentDir() let path = string(getEnv("PATH")) for candidate in split(path, PathSep): + if candidate.len == 0: continue when defined(windows): var x = (if candidate[0] == '"' and candidate[^1] == '"': substr(candidate, 1, candidate.len-2) else: candidate) / @@ -183,12 +190,12 @@ proc getLastModificationTime*(file: string): times.Time {.rtl, extern: "nos$1".} when defined(posix): var res: Stat if stat(file, res) < 0'i32: raiseOSError(osLastError()) - return fromUnix(res.st_mtime.int64) + result = res.st_mtim.toTime else: var f: WIN32_FIND_DATA var h = findFirstFile(file, f) if h == -1'i32: raiseOSError(osLastError()) - result = fromUnix(winTimeToUnixTime(rdFileTime(f.ftLastWriteTime)).int64) + result = fromWinTime(rdFileTime(f.ftLastWriteTime)) findClose(h) proc getLastAccessTime*(file: string): times.Time {.rtl, extern: "nos$1".} = @@ -196,12 +203,12 @@ proc getLastAccessTime*(file: string): times.Time {.rtl, extern: "nos$1".} = when defined(posix): var res: Stat if stat(file, res) < 0'i32: raiseOSError(osLastError()) - return fromUnix(res.st_atime.int64) + result = res.st_atim.toTime else: var f: WIN32_FIND_DATA var h = findFirstFile(file, f) if h == -1'i32: raiseOSError(osLastError()) - result = fromUnix(winTimeToUnixTime(rdFileTime(f.ftLastAccessTime)).int64) + result = fromWinTime(rdFileTime(f.ftLastAccessTime)) findClose(h) proc getCreationTime*(file: string): times.Time {.rtl, extern: "nos$1".} = @@ -213,22 +220,25 @@ proc getCreationTime*(file: string): times.Time {.rtl, extern: "nos$1".} = when defined(posix): var res: Stat if stat(file, res) < 0'i32: raiseOSError(osLastError()) - return fromUnix(res.st_ctime.int64) + result = res.st_ctim.toTime else: var f: WIN32_FIND_DATA var h = findFirstFile(file, f) if h == -1'i32: raiseOSError(osLastError()) - result = fromUnix(winTimeToUnixTime(rdFileTime(f.ftCreationTime)).int64) + result = fromWinTime(rdFileTime(f.ftCreationTime)) findClose(h) proc fileNewer*(a, b: string): bool {.rtl, extern: "nos$1".} = ## Returns true if the file `a` is newer than file `b`, i.e. if `a`'s ## modification time is later than `b`'s. when defined(posix): - result = getLastModificationTime(a) - getLastModificationTime(b) >= 0 - # Posix's resolution sucks so, we use '>=' for posix. + # If we don't have access to nanosecond resolution, use '>=' + when not StatHasNanoseconds: + result = getLastModificationTime(a) >= getLastModificationTime(b) + else: + result = getLastModificationTime(a) > getLastModificationTime(b) else: - result = getLastModificationTime(a) - getLastModificationTime(b) > 0 + result = getLastModificationTime(a) > getLastModificationTime(b) proc getCurrentDir*(): string {.rtl, extern: "nos$1", tags: [].} = ## Returns the `current working directory`:idx:. @@ -286,10 +296,30 @@ proc setCurrentDir*(newDir: string) {.inline, tags: [].} = else: if chdir(newDir) != 0'i32: raiseOSError(osLastError()) +proc absolutePath*(path: string, root = getCurrentDir()): string = + ## Returns the absolute path of `path`, rooted at `root` (which must be absolute) + ## if `path` is absolute, return it, ignoring `root` + runnableExamples: + doAssert absolutePath("a") == getCurrentDir() / "a" + if isAbsolute(path): path + else: + if not root.isAbsolute: + raise newException(ValueError, "The specified root is not absolute: " & root) + joinPath(root, path) + +when isMainModule: + doAssertRaises(ValueError): discard absolutePath("a", "b") + doAssert absolutePath("a") == getCurrentDir() / "a" + doAssert absolutePath("a", "/b") == "/b" / "a" + when defined(Posix): + doAssert absolutePath("a", "/b/") == "/b" / "a" + doAssert absolutePath("a", "/b/c") == "/b/c" / "a" + doAssert absolutePath("/a", "b/") == "/a" + proc expandFilename*(filename: string): string {.rtl, extern: "nos$1", tags: [ReadDirEffect].} = - ## Returns the full (`absolute`:idx:) path of the file `filename`, - ## raises OSError in case of an error. + ## Returns the full (`absolute`:idx:) path of an existing file `filename`, + ## raises OSError in case of an error. Follows symlinks. when defined(windows): var bufsize = MAX_PATH.int32 when useWinUnicode: @@ -328,21 +358,63 @@ proc expandFilename*(filename: string): string {.rtl, extern: "nos$1", result = $r c_free(cast[pointer](r)) +proc normalizePath*(path: var string) {.rtl, extern: "nos$1", tags: [].} = + ## Normalize a path. + ## + ## Consecutive directory separators are collapsed, including an initial double slash. + ## + ## On relative paths, double dot (..) sequences are collapsed if possible. + ## On absolute paths they are always collapsed. + ## + ## Warning: URL-encoded and Unicode attempts at directory traversal are not detected. + ## Triple dot is not handled. + let isAbs = isAbsolute(path) + var stack: seq[string] = @[] + for p in split(path, {DirSep}): + case p + of "", ".": + continue + of "..": + if stack.len == 0: + if isAbs: + discard # collapse all double dots on absoluta paths + else: + stack.add(p) + elif stack[^1] == "..": + stack.add(p) + else: + discard stack.pop() + else: + stack.add(p) + + if isAbs: + path = DirSep & join(stack, $DirSep) + elif stack.len > 0: + path = join(stack, $DirSep) + else: + path = "." + +proc normalizedPath*(path: string): string {.rtl, extern: "nos$1", tags: [].} = + ## Returns a normalized path for the current OS. See `<#normalizePath>`_ + result = path + normalizePath(result) + when defined(Windows): - proc openHandle(path: string, followSymlink=true): Handle = + proc openHandle(path: string, followSymlink=true, writeAccess=false): Handle = var flags = FILE_FLAG_BACKUP_SEMANTICS or FILE_ATTRIBUTE_NORMAL if not followSymlink: flags = flags or FILE_FLAG_OPEN_REPARSE_POINT + let access = if writeAccess: GENERIC_WRITE else: 0'i32 when useWinUnicode: result = createFileW( - newWideCString(path), 0'i32, + newWideCString(path), access, FILE_SHARE_DELETE or FILE_SHARE_READ or FILE_SHARE_WRITE, nil, OPEN_EXISTING, flags, 0 ) else: result = createFileA( - path, 0'i32, + path, access, FILE_SHARE_DELETE or FILE_SHARE_READ or FILE_SHARE_WRITE, nil, OPEN_EXISTING, flags, 0 ) @@ -430,8 +502,6 @@ type fpOthersWrite, ## write access for others fpOthersRead ## read access for others -{.deprecated: [TFilePermission: FilePermission].} - proc getFilePermissions*(filename: string): set[FilePermission] {. rtl, extern: "nos$1", tags: [ReadDirEffect].} = ## retrieves file permissions for `filename`. `OSError` is raised in case of @@ -545,7 +615,10 @@ proc copyFile*(source, dest: string) {.rtl, extern: "nos$1", when not declared(ENOENT) and not defined(Windows): when NoFakeVars: - const ENOENT = cint(2) # 2 on most systems including Solaris + when not defined(haiku): + const ENOENT = cint(2) # 2 on most systems including Solaris + else: + const ENOENT = cint(-2147459069) else: var ENOENT {.importc, header: "<errno.h>".}: cint @@ -615,6 +688,7 @@ proc tryMoveFSObject(source, dest: string): bool = proc moveFile*(source, dest: string) {.rtl, extern: "nos$1", tags: [ReadIOEffect, WriteIOEffect].} = ## Moves a file from `source` to `dest`. If this fails, `OSError` is raised. + ## Can be used to `rename files`:idx: if not tryMoveFSObject(source, dest): when not defined(windows): # Fallback to copy & del @@ -728,8 +802,6 @@ type pcDir, ## path refers to a directory pcLinkToDir ## path refers to a symbolic link to a directory -{.deprecated: [TPathComponent: PathComponent].} - when defined(posix): proc getSymlinkFileKind(path: string): PathComponent = @@ -811,7 +883,8 @@ iterator walkDir*(dir: string; relative=false): tuple[kind: PathComponent, path: y = dir / y var k = pcFile - when defined(linux) or defined(macosx) or defined(bsd) or defined(genode): + when defined(linux) or defined(macosx) or + defined(bsd) or defined(genode) or defined(nintendoswitch): if x.d_type != DT_UNKNOWN: if x.d_type == DT_DIR: k = pcDir if x.d_type == DT_LNK: @@ -901,7 +974,15 @@ proc rawCreateDir(dir: string): bool = elif errno in {EEXIST, ENOSYS}: result = false else: - raiseOSError(osLastError()) + raiseOSError(osLastError(), dir) + elif defined(haiku): + let res = mkdir(dir, 0o777) + if res == 0'i32: + result = true + elif errno == EEXIST or errno == EROFS: + result = false + else: + raiseOSError(osLastError(), dir) elif defined(posix): let res = mkdir(dir, 0o777) if res == 0'i32: @@ -909,8 +990,8 @@ proc rawCreateDir(dir: string): bool = elif errno == EEXIST: result = false else: - echo res - raiseOSError(osLastError()) + #echo res + raiseOSError(osLastError(), dir) else: when useWinUnicode: wrapUnary(res, createDirectoryW, dir) @@ -922,9 +1003,10 @@ proc rawCreateDir(dir: string): bool = elif getLastError() == 183'i32: result = false else: - raiseOSError(osLastError()) + raiseOSError(osLastError(), dir) -proc existsOrCreateDir*(dir: string): bool = +proc existsOrCreateDir*(dir: string): bool {.rtl, extern: "nos$1", + tags: [WriteDirEffect, ReadDirEffect].} = ## Check if a `directory`:idx: `dir` exists, and create it otherwise. ## ## Does not create parent directories (fails if parent does not exist). @@ -934,7 +1016,7 @@ proc existsOrCreateDir*(dir: string): bool = if result: # path already exists - need to check that it is indeed a directory if not existsDir(dir): - raise newException(IOError, "Failed to create the directory") + raise newException(IOError, "Failed to create '" & dir & "'") proc createDir*(dir: string) {.rtl, extern: "nos$1", tags: [WriteDirEffect, ReadDirEffect].} = @@ -1057,18 +1139,17 @@ proc parseCmdLine*(c: string): seq[string] {. while true: setLen(a, 0) # eat all delimiting whitespace - while c[i] == ' ' or c[i] == '\t' or c[i] == '\l' or c[i] == '\r' : inc(i) + while i < c.len and c[i] in {' ', '\t', '\l', '\r'}: inc(i) + if i >= c.len: break when defined(windows): # parse a single argument according to the above rules: - if c[i] == '\0': break var inQuote = false - while true: + while i < c.len: case c[i] - of '\0': break of '\\': var j = i - while c[j] == '\\': inc(j) - if c[j] == '"': + while j < c.len and c[j] == '\\': inc(j) + if j < c.len and c[j] == '"': for k in 1..(j-i) div 2: a.add('\\') if (j-i) mod 2 == 0: i = j @@ -1081,7 +1162,7 @@ proc parseCmdLine*(c: string): seq[string] {. of '"': inc(i) if not inQuote: inQuote = true - elif c[i] == '"': + elif i < c.len and c[i] == '"': a.add(c[i]) inc(i) else: @@ -1099,13 +1180,12 @@ proc parseCmdLine*(c: string): seq[string] {. of '\'', '\"': var delim = c[i] inc(i) # skip ' or " - while c[i] != '\0' and c[i] != delim: + while i < c.len and c[i] != delim: add a, c[i] inc(i) - if c[i] != '\0': inc(i) - of '\0': break + if i < c.len: inc(i) else: - while c[i] > ' ': + while i < c.len and c[i] > ' ': add(a, c[i]) inc(i) add(result, a) @@ -1263,22 +1343,40 @@ elif defined(windows): # is always the same -- independent of the used C compiler. var ownArgv {.threadvar.}: seq[string] + ownParsedArgv {.threadvar.}: bool proc paramCount*(): int {.rtl, extern: "nos$1", tags: [ReadIOEffect].} = # Docstring in nimdoc block. - if isNil(ownArgv): ownArgv = parseCmdLine($getCommandLine()) + if not ownParsedArgv: + ownArgv = parseCmdLine($getCommandLine()) + ownParsedArgv = true result = ownArgv.len-1 proc paramStr*(i: int): TaintedString {.rtl, extern: "nos$1", tags: [ReadIOEffect].} = # Docstring in nimdoc block. - if isNil(ownArgv): ownArgv = parseCmdLine($getCommandLine()) + if not ownParsedArgv: + ownArgv = parseCmdLine($getCommandLine()) + ownParsedArgv = true if i < ownArgv.len and i >= 0: return TaintedString(ownArgv[i]) raise newException(IndexError, "invalid index") +elif defined(nintendoswitch): + proc paramStr*(i: int): TaintedString {.tags: [ReadIOEffect].} = + raise newException(OSError, "paramStr is not implemented on Nintendo Switch") + + proc paramCount*(): int {.tags: [ReadIOEffect].} = + raise newException(OSError, "paramCount is not implemented on Nintendo Switch") + +elif defined(genode): + proc paramStr*(i: int): TaintedString = + raise newException(OSError, "paramStr is not implemented on Genode") + + proc paramCount*(): int = + raise newException(OSError, "paramCount is not implemented on Genode") + elif not defined(createNimRtl) and - not(defined(posix) and appType == "lib") and - not defined(genode): + not(defined(posix) and appType == "lib"): # On Posix, there is no portable way to get the command line from a DLL. var cmdCount {.importc: "cmdCount".}: cint @@ -1432,26 +1530,14 @@ proc getAppFilename*(): string {.rtl, extern: "nos$1", tags: [ReadIOEffect].} = result = getApplAux("/proc/self/exe") elif defined(solaris): result = getApplAux("/proc/" & $getpid() & "/path/a.out") - elif defined(genode): - raiseOSError("POSIX command line not supported") + elif defined(genode) or defined(nintendoswitch): + raiseOSError(OSErrorCode(-1), "POSIX command line not supported") elif defined(freebsd) or defined(dragonfly): result = getApplFreebsd() # little heuristic that may work on other POSIX-like systems: if result.len == 0: result = getApplHeuristic() -proc getApplicationFilename*(): string {.rtl, extern: "nos$1", deprecated.} = - ## Returns the filename of the application's executable. - ## **Deprecated since version 0.8.12**: use ``getAppFilename`` - ## instead. - result = getAppFilename() - -proc getApplicationDir*(): string {.rtl, extern: "nos$1", deprecated.} = - ## Returns the directory of the application's executable. - ## **Deprecated since version 0.8.12**: use ``getAppDir`` - ## instead. - result = splitFile(getAppFilename()).dir - proc getAppDir*(): string {.rtl, extern: "nos$1", tags: [ReadIOEffect].} = ## Returns the directory of the application's executable. result = splitFile(getAppFilename()).dir @@ -1506,19 +1592,17 @@ type template rawToFormalFileInfo(rawInfo, path, formalInfo): untyped = ## Transforms the native file info structure into the one nim uses. - ## 'rawInfo' is either a 'TBY_HANDLE_FILE_INFORMATION' structure on Windows, + ## 'rawInfo' is either a 'BY_HANDLE_FILE_INFORMATION' structure on Windows, ## or a 'Stat' structure on posix when defined(Windows): - template toTime(e: FILETIME): untyped {.gensym.} = - fromUnix(winTimeToUnixTime(rdFileTime(e)).int64) # local templates default to bind semantics template merge(a, b): untyped = a or (b shl 32) formalInfo.id.device = rawInfo.dwVolumeSerialNumber formalInfo.id.file = merge(rawInfo.nFileIndexLow, rawInfo.nFileIndexHigh) formalInfo.size = merge(rawInfo.nFileSizeLow, rawInfo.nFileSizeHigh) formalInfo.linkCount = rawInfo.nNumberOfLinks - formalInfo.lastAccessTime = toTime(rawInfo.ftLastAccessTime) - formalInfo.lastWriteTime = toTime(rawInfo.ftLastWriteTime) - formalInfo.creationTime = toTime(rawInfo.ftCreationTime) + formalInfo.lastAccessTime = fromWinTime(rdFileTime(rawInfo.ftLastAccessTime)) + formalInfo.lastWriteTime = fromWinTime(rdFileTime(rawInfo.ftLastWriteTime)) + formalInfo.creationTime = fromWinTime(rdFileTime(rawInfo.ftCreationTime)) # Retrieve basic permissions if (rawInfo.dwFileAttributes and FILE_ATTRIBUTE_READONLY) != 0'i32: @@ -1534,7 +1618,6 @@ template rawToFormalFileInfo(rawInfo, path, formalInfo): untyped = if (rawInfo.dwFileAttributes and FILE_ATTRIBUTE_REPARSE_POINT) != 0'i32: formalInfo.kind = succ(result.kind) - else: template checkAndIncludeMode(rawMode, formalMode: untyped) = if (rawInfo.st_mode and rawMode) != 0'i32: @@ -1542,9 +1625,9 @@ template rawToFormalFileInfo(rawInfo, path, formalInfo): untyped = formalInfo.id = (rawInfo.st_dev, rawInfo.st_ino) formalInfo.size = rawInfo.st_size formalInfo.linkCount = rawInfo.st_Nlink.BiggestInt - formalInfo.lastAccessTime = fromUnix(rawInfo.st_atime.int64) - formalInfo.lastWriteTime = fromUnix(rawInfo.st_mtime.int64) - formalInfo.creationTime = fromUnix(rawInfo.st_ctime.int64) + formalInfo.lastAccessTime = rawInfo.st_atim.toTime + formalInfo.lastWriteTime = rawInfo.st_mtim.toTime + formalInfo.creationTime = rawInfo.st_ctim.toTime result.permissions = {} checkAndIncludeMode(S_IRUSR, fpUserRead) @@ -1652,3 +1735,20 @@ proc isHidden*(path: string): bool = result = (fileName[0] == '.') and (fileName[3] != '.') {.pop.} + +proc setLastModificationTime*(file: string, t: times.Time) = + ## Sets the `file`'s last modification time. `OSError` is raised in case of + ## an error. + when defined(posix): + let unixt = posix.Time(t.toUnix) + let micro = convert(Nanoseconds, Microseconds, t.nanosecond) + var timevals = [Timeval(tv_sec: unixt, tv_usec: micro), + Timeval(tv_sec: unixt, tv_usec: micro)] # [last access, last modification] + if utimes(file, timevals.addr) != 0: raiseOSError(osLastError()) + else: + let h = openHandle(path = file, writeAccess = true) + if h == INVALID_HANDLE_VALUE: raiseOSError(osLastError()) + var ft = t.toWinTime.toFILETIME + let res = setFileTime(h, nil, nil, ft.addr) + discard h.closeHandle + if res == 0'i32: raiseOSError(osLastError()) diff --git a/lib/pure/ospaths.nim b/lib/pure/ospaths.nim index 0d638abb9..bc6739dd3 100644 --- a/lib/pure/ospaths.nim +++ b/lib/pure/ospaths.nim @@ -29,11 +29,6 @@ type OSErrorCode* = distinct int32 ## Specifies an OS Error Code. -{.deprecated: [FReadEnv: ReadEnvEffect, FWriteEnv: WriteEnvEffect, - FReadDir: ReadDirEffect, - FWriteDir: WriteDirEffect, - TOSErrorCode: OSErrorCode -].} const doslikeFileSystem* = defined(windows) or defined(OS2) or defined(DOS) @@ -152,7 +147,7 @@ else: # UNIX-like operating system DirSep* = '/' AltSep* = DirSep PathSep* = ':' - FileSystemCaseSensitive* = true + FileSystemCaseSensitive* = when defined(macosx): false else: true ExeExt* = "" ScriptExt* = "" DynlibFormat* = when defined(macosx): "lib$1.dylib" else: "lib$1.so" @@ -191,12 +186,12 @@ proc joinPath*(head, tail: string): string {. if len(head) == 0: result = tail elif head[len(head)-1] in {DirSep, AltSep}: - if tail[0] in {DirSep, AltSep}: + if tail.len > 0 and tail[0] in {DirSep, AltSep}: result = head & substr(tail, 1) else: result = head & tail else: - if tail[0] in {DirSep, AltSep}: + if tail.len > 0 and tail[0] in {DirSep, AltSep}: result = head & tail else: result = head & DirSep & tail @@ -415,6 +410,11 @@ proc cmpPaths*(pathA, pathB: string): int {. ## | 0 iff pathA == pathB ## | < 0 iff pathA < pathB ## | > 0 iff pathA > pathB + runnableExamples: + when defined(macosx): + doAssert cmpPaths("foo", "Foo") == 0 + elif defined(posix): + doAssert cmpPaths("foo", "Foo") > 0 if FileSystemCaseSensitive: result = cmp(pathA, pathB) else: @@ -428,17 +428,52 @@ proc isAbsolute*(path: string): bool {.rtl, noSideEffect, extern: "nos$1".} = ## Checks whether a given `path` is absolute. ## ## On Windows, network paths are considered absolute too. + runnableExamples: + doAssert(not "".isAbsolute) + doAssert(not ".".isAbsolute) + when defined(posix): + doAssert "/".isAbsolute + doAssert(not "a/".isAbsolute) + + if len(path) == 0: return false + when doslikeFileSystem: var len = len(path) - result = (len > 0 and path[0] in {'/', '\\'}) or + result = (path[0] in {'/', '\\'}) or (len > 1 and path[0] in {'a'..'z', 'A'..'Z'} and path[1] == ':') elif defined(macos): - result = path.len > 0 and path[0] != ':' + # according to https://perldoc.perl.org/File/Spec/Mac.html `:a` is a relative path + result = path[0] != ':' elif defined(RISCOS): result = path[0] == '$' elif defined(posix): result = path[0] == '/' + +proc normalizePathEnd(path: var string, trailingSep = false) = + ## ensures ``path`` has exactly 0 or 1 trailing `DirSep`, depending on + ## ``trailingSep``, and taking care of edge cases: it preservers whether + ## a path is absolute or relative, and makes sure trailing sep is `DirSep`, + ## not `AltSep`. + if path.len == 0: return + var i = path.len + while i >= 1 and path[i-1] in {DirSep, AltSep}: dec(i) + if trailingSep: + # foo// => foo + path.setLen(i) + # foo => foo/ + path.add DirSep + elif i>0: + # foo// => foo + path.setLen(i) + else: + # // => / (empty case was already taken care of) + path = $DirSep + +proc normalizePathEnd(path: string, trailingSep = false): string = + result = path + result.normalizePathEnd(trailingSep) + proc unixToNativePath*(path: string, drive=""): string {. noSideEffect, rtl, extern: "nos$1".} = ## Converts an UNIX-like path to a native one. @@ -454,6 +489,9 @@ proc unixToNativePath*(path: string, drive=""): string {. when defined(unix): result = path else: + if path.len == 0: + return "" + var start: int if path[0] == '/': # an absolute path @@ -467,17 +505,17 @@ proc unixToNativePath*(path: string, drive=""): string {. else: result = $DirSep start = 1 - elif path[0] == '.' and path[1] == '/': + elif path[0] == '.' and (path.len == 1 or path[1] == '/'): # current directory result = $CurDir - start = 2 + start = when doslikeFileSystem: 1 else: 2 else: result = "" start = 0 var i = start while i < len(path): # ../../../ --> :::: - if path[i] == '.' and path[i+1] == '.' and path[i+2] == '/': + if i+2 < path.len and path[i] == '.' and path[i+1] == '.' and path[i+2] == '/': # parent directory when defined(macos): if result[high(result)] == ':': @@ -512,15 +550,17 @@ proc getConfigDir*(): string {.rtl, extern: "nos$1", ## Returns the config directory of the current user for applications. ## ## On non-Windows OSs, this proc conforms to the XDG Base Directory - ## spec. Thus, this proc returns the value of the XDG_CONFIG_DIR environment + ## spec. Thus, this proc returns the value of the XDG_CONFIG_HOME environment ## variable if it is set, and returns the default configuration directory, ## "~/.config/", otherwise. ## ## An OS-dependent trailing slash is always present at the end of the - ## returned string; `\\` on Windows and `/` on all other OSs. - when defined(windows): return string(getEnv("APPDATA")) & "\\" - elif getEnv("XDG_CONFIG_DIR"): return string(getEnv("XDG_CONFIG_DIR")) & "/" - else: return string(getEnv("HOME")) & "/.config/" + ## returned string; `\` on Windows and `/` on all other OSs. + when defined(windows): + result = getEnv("APPDATA").string + else: + result = getEnv("XDG_CONFIG_HOME", getEnv("HOME").string / ".config").string + result.normalizePathEnd(trailingSep = true) proc getTempDir*(): string {.rtl, extern: "nos$1", tags: [ReadEnvEffect, ReadIOEffect].} = @@ -540,25 +580,25 @@ proc getTempDir*(): string {.rtl, extern: "nos$1", proc expandTilde*(path: string): string {. tags: [ReadEnvEffect, ReadIOEffect].} = - ## Expands a path starting with ``~/`` to a full path. - ## - ## If `path` starts with the tilde character and is followed by `/` or `\\` - ## this proc will return the reminder of the path appended to the result of - ## the getHomeDir() proc, otherwise the input path will be returned without - ## modification. + ## Expands ``~`` or a path starting with ``~/`` to a full path, replacing + ## ``~`` with ``getHomeDir()`` (otherwise returns ``path`` unmodified). ## - ## The behaviour of this proc is the same on the Windows platform despite - ## not having this convention. Example: - ## - ## .. code-block:: nim - ## let configFile = expandTilde("~" / "appname.cfg") - ## echo configFile - ## # --> C:\Users\amber\appname.cfg - if len(path) > 1 and path[0] == '~' and (path[1] == '/' or path[1] == '\\'): + ## Windows: this is still supported despite Windows platform not having this + ## convention; also, both ``~/`` and ``~\`` are handled. + runnableExamples: + doAssert expandTilde("~" / "appname.cfg") == getHomeDir() / "appname.cfg" + if len(path) == 0 or path[0] != '~': + result = path + elif len(path) == 1: + result = getHomeDir() + elif (path[1] in {DirSep, AltSep}): result = getHomeDir() / path.substr(2) else: + # TODO: handle `~bob` and `~bob/` which means home of bob result = path +# TODO: consider whether quoteShellPosix, quoteShellWindows, quoteShell, quoteShellCommand +# belong in `strutils` instead; they are not specific to paths proc quoteShellWindows*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".} = ## Quote s, so it can be safely passed to Windows API. ## Based on Python's subprocess.list2cmdline @@ -602,7 +642,7 @@ proc quoteShellPosix*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".} else: return "'" & s.replace("'", "'\"'\"'") & "'" -when defined(windows) or defined(posix): +when defined(windows) or defined(posix) or defined(nintendoswitch): proc quoteShell*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".} = ## Quote ``s``, so it can be safely passed to shell. when defined(windows): @@ -610,6 +650,18 @@ when defined(windows) or defined(posix): else: return quoteShellPosix(s) + proc quoteShellCommand*(args: openArray[string]): string = + ## Concatenates and quotes shell arguments `args` + runnableExamples: + when defined(posix): + assert quoteShellCommand(["aaa", "", "c d"]) == "aaa '' 'c d'" + when defined(windows): + assert quoteShellCommand(["aaa", "", "c d"]) == "aaa \"\" \"c d\"" + # can't use `map` pending https://github.com/nim-lang/Nim/issues/8303 + for i in 0..<args.len: + if i > 0: result.add " " + result.add quoteShell(args[i]) + when isMainModule: assert quoteShellWindows("aaa") == "aaa" assert quoteShellWindows("aaa\"") == "aaa\\\"" @@ -622,3 +674,24 @@ when isMainModule: when defined(posix): assert quoteShell("") == "''" + + block normalizePathEndTest: + # handle edge cases correctly: shouldn't affect whether path is + # absolute/relative + doAssert "".normalizePathEnd(true) == "" + doAssert "".normalizePathEnd(false) == "" + doAssert "/".normalizePathEnd(true) == $DirSep + doAssert "/".normalizePathEnd(false) == $DirSep + + when defined(posix): + doAssert "//".normalizePathEnd(false) == "/" + doAssert "foo.bar//".normalizePathEnd == "foo.bar" + doAssert "bar//".normalizePathEnd(trailingSep = true) == "bar/" + when defined(Windows): + doAssert r"C:\foo\\".normalizePathEnd == r"C:\foo" + doAssert r"C:\foo".normalizePathEnd(trailingSep = true) == r"C:\foo\" + # this one is controversial: we could argue for returning `D:\` instead, + # but this is simplest. + doAssert r"D:\".normalizePathEnd == r"D:" + doAssert r"E:/".normalizePathEnd(trailingSep = true) == r"E:\" + doAssert "/".normalizePathEnd == r"\" diff --git a/lib/pure/osproc.nim b/lib/pure/osproc.nim index c1c727fc6..faeb01407 100644 --- a/lib/pure/osproc.nim +++ b/lib/pure/osproc.nim @@ -61,9 +61,6 @@ type Process* = ref ProcessObj ## represents an operating system process -{.deprecated: [TProcess: ProcessObj, PProcess: Process, - TProcessOption: ProcessOption].} - const poUseShell* {.deprecated.} = poUsePath ## Deprecated alias for poUsePath. @@ -373,17 +370,15 @@ template streamAccess(p) = when defined(Windows) and not defined(useNimRtl): # We need to implement a handle stream for Windows: type - PFileHandleStream = ref FileHandleStream - FileHandleStream = object of StreamObj + FileHandleStream = ref object of StreamObj handle: Handle atTheEnd: bool - {.deprecated: [TFileHandleStream: FileHandleStream].} proc hsClose(s: Stream) = discard # nothing to do here - proc hsAtEnd(s: Stream): bool = return PFileHandleStream(s).atTheEnd + proc hsAtEnd(s: Stream): bool = return FileHandleStream(s).atTheEnd proc hsReadData(s: Stream, buffer: pointer, bufLen: int): int = - var s = PFileHandleStream(s) + var s = FileHandleStream(s) if s.atTheEnd: return 0 var br: int32 var a = winlean.readFile(s.handle, buffer, bufLen.cint, addr br, nil) @@ -395,13 +390,13 @@ when defined(Windows) and not defined(useNimRtl): result = br proc hsWriteData(s: Stream, buffer: pointer, bufLen: int) = - var s = PFileHandleStream(s) + var s = FileHandleStream(s) var bytesWritten: int32 var a = winlean.writeFile(s.handle, buffer, bufLen.cint, addr bytesWritten, nil) if a == 0: raiseOSError(osLastError()) - proc newFileHandleStream(handle: Handle): PFileHandleStream = + proc newFileHandleStream(handle: Handle): FileHandleStream = new(result) result.handle = handle result.closeImpl = hsClose @@ -494,7 +489,7 @@ when defined(Windows) and not defined(useNimRtl): sa.nLength = sizeof(SECURITY_ATTRIBUTES).cint sa.lpSecurityDescriptor = nil sa.bInheritHandle = 1 - if createPipe(rdHandle, wrHandle, sa, 1024) == 0'i32: + if createPipe(rdHandle, wrHandle, sa, 0) == 0'i32: raiseOSError(osLastError()) proc fileClose(h: Handle) {.inline.} = @@ -524,6 +519,12 @@ when defined(Windows) and not defined(useNimRtl): he = ho else: createPipeHandles(he, si.hStdError) + if setHandleInformation(he, DWORD(1), DWORD(0)) == 0'i32: + raiseOsError(osLastError()) + if setHandleInformation(hi, DWORD(1), DWORD(0)) == 0'i32: + raiseOsError(osLastError()) + if setHandleInformation(ho, DWORD(1), DWORD(0)) == 0'i32: + raiseOsError(osLastError()) else: createAllPipeHandles(si, hi, ho, he, cast[int](result)) result.inHandle = FileHandle(hi) @@ -746,14 +747,14 @@ elif not defined(useNimRtl): copyMem(result[i], addr(x[0]), x.len+1) inc(i) - type StartProcessData = object - sysCommand: string - sysArgs: cstringArray - sysEnv: cstringArray - workingDir: cstring - pStdin, pStdout, pStderr, pErrorPipe: array[0..1, cint] - options: set[ProcessOption] - {.deprecated: [TStartProcessData: StartProcessData].} + type + StartProcessData = object + sysCommand: string + sysArgs: cstringArray + sysEnv: cstringArray + workingDir: cstring + pStdin, pStdout, pStderr, pErrorPipe: array[0..1, cint] + options: set[ProcessOption] const useProcessAuxSpawn = declared(posix_spawn) and not defined(useFork) and not defined(useClone) and not defined(linux) @@ -831,7 +832,7 @@ elif not defined(useNimRtl): # Parent process. Copy process information. if poEchoCmd in options: - echo(command, " ", join(args, " ")) + when declared(echo): echo(command, " ", join(args, " ")) result.id = pid result.exitFlag = false @@ -884,7 +885,7 @@ elif not defined(useNimRtl): chck posix_spawn_file_actions_addclose(fops, data.pStdout[readIdx]) chck posix_spawn_file_actions_adddup2(fops, data.pStdout[writeIdx], writeIdx) chck posix_spawn_file_actions_addclose(fops, data.pStderr[readIdx]) - if (poStdErrToStdOut in data.options): + if poStdErrToStdOut in data.options: chck posix_spawn_file_actions_adddup2(fops, data.pStdout[writeIdx], 2) else: chck posix_spawn_file_actions_adddup2(fops, data.pStderr[writeIdx], 2) @@ -995,13 +996,21 @@ elif not defined(useNimRtl): {.pop} proc close(p: Process) = - if p.inStream != nil: close(p.inStream) - if p.outStream != nil: close(p.outStream) - if p.errStream != nil: close(p.errStream) if poParentStreams notin p.options: - discard close(p.inHandle) - discard close(p.outHandle) - discard close(p.errHandle) + if p.inStream != nil: + close(p.inStream) + else: + discard close(p.inHandle) + + if p.outStream != nil: + close(p.outStream) + else: + discard close(p.outHandle) + + if p.errStream != nil: + close(p.errStream) + else: + discard close(p.errHandle) proc suspend(p: Process) = if kill(p.id, SIGSTOP) != 0'i32: raiseOsError(osLastError()) @@ -1275,7 +1284,7 @@ elif not defined(useNimRtl): proc select(readfds: var seq[Process], timeout = 500): int = var tv: Timeval - tv.tv_sec = 0 + tv.tv_sec = posix.Time(0) tv.tv_usec = timeout * 1000 var rd: TFdSet diff --git a/lib/pure/parsecfg.nim b/lib/pure/parsecfg.nim index 2a5dbc8f8..b991dd57f 100644 --- a/lib/pure/parsecfg.nim +++ b/lib/pure/parsecfg.nim @@ -17,12 +17,37 @@ ## ## .. include:: ../../doc/mytest.cfg ## :literal: -## The file ``examples/parsecfgex.nim`` demonstrates how to use the -## configuration file parser: -## -## .. code-block:: nim -## :file: ../../examples/parsecfgex.nim ## + +##[ Here is an example of how to use the configuration file parser: + +.. code-block:: nim + + import + os, parsecfg, strutils, streams + + var f = newFileStream(paramStr(1), fmRead) + if f != nil: + var p: CfgParser + open(p, f, paramStr(1)) + while true: + var e = next(p) + case e.kind + of cfgEof: break + of cfgSectionStart: ## a ``[section]`` has been parsed + echo("new section: " & e.section) + of cfgKeyValuePair: + echo("key-value-pair: " & e.key & ": " & e.value) + of cfgOption: + echo("command: " & e.key & ": " & e.value) + of cfgError: + echo(e.msg) + close(p) + else: + echo("cannot open: " & paramStr(1)) + +]## + ## Examples ## -------- ## @@ -125,9 +150,6 @@ type tok: Token filename: string -{.deprecated: [TCfgEventKind: CfgEventKind, TCfgEvent: CfgEvent, - TTokKind: TokKind, TToken: Token, TCfgParser: CfgParser].} - # implementation const diff --git a/lib/pure/parsecsv.nim b/lib/pure/parsecsv.nim index 358ce829d..796114d37 100644 --- a/lib/pure/parsecsv.nim +++ b/lib/pure/parsecsv.nim @@ -66,8 +66,6 @@ type CsvError* = object of IOError ## exception that is raised if ## a parsing error occurs -{.deprecated: [TCsvRow: CsvRow, TCsvParser: CsvParser, EInvalidCsv: CsvError].} - proc raiseEInvalidCsv(filename: string, line, col: int, msg: string) {.noreturn.} = var e: ref CsvError diff --git a/lib/pure/parsejson.nim b/lib/pure/parsejson.nim new file mode 100644 index 000000000..9c53af6a6 --- /dev/null +++ b/lib/pure/parsejson.nim @@ -0,0 +1,535 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2018 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module implements a json parser. It is used +## and exported by the ``json`` standard library +## module, but can also be used in its own right. + +import + strutils, lexbase, streams, unicode + +type + JsonEventKind* = enum ## enumeration of all events that may occur when parsing + jsonError, ## an error occurred during parsing + jsonEof, ## end of file reached + jsonString, ## a string literal + jsonInt, ## an integer literal + jsonFloat, ## a float literal + jsonTrue, ## the value ``true`` + jsonFalse, ## the value ``false`` + jsonNull, ## the value ``null`` + jsonObjectStart, ## start of an object: the ``{`` token + jsonObjectEnd, ## end of an object: the ``}`` token + jsonArrayStart, ## start of an array: the ``[`` token + jsonArrayEnd ## start of an array: the ``]`` token + + TokKind* = enum # must be synchronized with TJsonEventKind! + tkError, + tkEof, + tkString, + tkInt, + tkFloat, + tkTrue, + tkFalse, + tkNull, + tkCurlyLe, + tkCurlyRi, + tkBracketLe, + tkBracketRi, + tkColon, + tkComma + + JsonError* = enum ## enumeration that lists all errors that can occur + errNone, ## no error + errInvalidToken, ## invalid token + errStringExpected, ## string expected + errColonExpected, ## ``:`` expected + errCommaExpected, ## ``,`` expected + errBracketRiExpected, ## ``]`` expected + errCurlyRiExpected, ## ``}`` expected + errQuoteExpected, ## ``"`` or ``'`` expected + errEOC_Expected, ## ``*/`` expected + errEofExpected, ## EOF expected + errExprExpected ## expr expected + + ParserState = enum + stateEof, stateStart, stateObject, stateArray, stateExpectArrayComma, + stateExpectObjectComma, stateExpectColon, stateExpectValue + + JsonParser* = object of BaseLexer ## the parser object. + a*: string + tok*: TokKind + kind: JsonEventKind + err: JsonError + state: seq[ParserState] + filename: string + rawStringLiterals: bool + + JsonKindError* = object of ValueError ## raised by the ``to`` macro if the + ## JSON kind is incorrect. + JsonParsingError* = object of ValueError ## is raised for a JSON error + +const + errorMessages*: array[JsonError, string] = [ + "no error", + "invalid token", + "string expected", + "':' expected", + "',' expected", + "']' expected", + "'}' expected", + "'\"' or \"'\" expected", + "'*/' expected", + "EOF expected", + "expression expected" + ] + tokToStr: array[TokKind, string] = [ + "invalid token", + "EOF", + "string literal", + "int literal", + "float literal", + "true", + "false", + "null", + "{", "}", "[", "]", ":", "," + ] + +proc open*(my: var JsonParser, input: Stream, filename: string; + rawStringLiterals = false) = + ## initializes the parser with an input stream. `Filename` is only used + ## for nice error messages. If `rawStringLiterals` is true, string literals + ## are kepts with their surrounding quotes and escape sequences in them are + ## left untouched too. + lexbase.open(my, input) + my.filename = filename + my.state = @[stateStart] + my.kind = jsonError + my.a = "" + my.rawStringLiterals = rawStringLiterals + +proc close*(my: var JsonParser) {.inline.} = + ## closes the parser `my` and its associated input stream. + lexbase.close(my) + +proc str*(my: JsonParser): string {.inline.} = + ## returns the character data for the events: ``jsonInt``, ``jsonFloat``, + ## ``jsonString`` + assert(my.kind in {jsonInt, jsonFloat, jsonString}) + return my.a + +proc getInt*(my: JsonParser): BiggestInt {.inline.} = + ## returns the number for the event: ``jsonInt`` + assert(my.kind == jsonInt) + return parseBiggestInt(my.a) + +proc getFloat*(my: JsonParser): float {.inline.} = + ## returns the number for the event: ``jsonFloat`` + assert(my.kind == jsonFloat) + return parseFloat(my.a) + +proc kind*(my: JsonParser): JsonEventKind {.inline.} = + ## returns the current event type for the JSON parser + return my.kind + +proc getColumn*(my: JsonParser): int {.inline.} = + ## get the current column the parser has arrived at. + result = getColNumber(my, my.bufpos) + +proc getLine*(my: JsonParser): int {.inline.} = + ## get the current line the parser has arrived at. + result = my.lineNumber + +proc getFilename*(my: JsonParser): string {.inline.} = + ## get the filename of the file that the parser processes. + result = my.filename + +proc errorMsg*(my: JsonParser): string = + ## returns a helpful error message for the event ``jsonError`` + assert(my.kind == jsonError) + result = "$1($2, $3) Error: $4" % [ + my.filename, $getLine(my), $getColumn(my), errorMessages[my.err]] + +proc errorMsgExpected*(my: JsonParser, e: string): string = + ## returns an error message "`e` expected" in the same format as the + ## other error messages + result = "$1($2, $3) Error: $4" % [ + my.filename, $getLine(my), $getColumn(my), e & " expected"] + +proc handleHexChar(c: char, x: var int): bool = + result = true # Success + case c + of '0'..'9': x = (x shl 4) or (ord(c) - ord('0')) + of 'a'..'f': x = (x shl 4) or (ord(c) - ord('a') + 10) + of 'A'..'F': x = (x shl 4) or (ord(c) - ord('A') + 10) + else: result = false # error + +proc parseEscapedUTF16*(buf: cstring, pos: var int): int = + result = 0 + #UTF-16 escape is always 4 bytes. + for _ in 0..3: + if handleHexChar(buf[pos], result): + inc(pos) + else: + return -1 + +proc parseString(my: var JsonParser): TokKind = + result = tkString + var pos = my.bufpos + 1 + var buf = my.buf + if my.rawStringLiterals: + add(my.a, '"') + while true: + case buf[pos] + of '\0': + my.err = errQuoteExpected + result = tkError + break + of '"': + if my.rawStringLiterals: + add(my.a, '"') + inc(pos) + break + of '\\': + if my.rawStringLiterals: + add(my.a, '\\') + case buf[pos+1] + of '\\', '"', '\'', '/': + add(my.a, buf[pos+1]) + inc(pos, 2) + of 'b': + add(my.a, '\b') + inc(pos, 2) + of 'f': + add(my.a, '\f') + inc(pos, 2) + of 'n': + add(my.a, '\L') + inc(pos, 2) + of 'r': + add(my.a, '\C') + inc(pos, 2) + of 't': + add(my.a, '\t') + inc(pos, 2) + of 'u': + if my.rawStringLiterals: + add(my.a, 'u') + inc(pos, 2) + var pos2 = pos + var r = parseEscapedUTF16(buf, pos) + if r < 0: + my.err = errInvalidToken + break + # Deal with surrogates + if (r and 0xfc00) == 0xd800: + if buf[pos] != '\\' or buf[pos+1] != 'u': + my.err = errInvalidToken + break + inc(pos, 2) + var s = parseEscapedUTF16(buf, pos) + if (s and 0xfc00) == 0xdc00 and s > 0: + r = 0x10000 + (((r - 0xd800) shl 10) or (s - 0xdc00)) + else: + my.err = errInvalidToken + break + if my.rawStringLiterals: + let length = pos - pos2 + for i in 1 .. length: + if buf[pos2] in {'0'..'9', 'A'..'F', 'a'..'f'}: + add(my.a, buf[pos2]) + inc pos2 + else: + break + else: + add(my.a, toUTF8(Rune(r))) + else: + # don't bother with the error + add(my.a, buf[pos]) + inc(pos) + of '\c': + pos = lexbase.handleCR(my, pos) + buf = my.buf + add(my.a, '\c') + of '\L': + pos = lexbase.handleLF(my, pos) + buf = my.buf + add(my.a, '\L') + else: + add(my.a, buf[pos]) + inc(pos) + my.bufpos = pos # store back + +proc skip(my: var JsonParser) = + var pos = my.bufpos + var buf = my.buf + while true: + case buf[pos] + of '/': + if buf[pos+1] == '/': + # skip line comment: + inc(pos, 2) + while true: + case buf[pos] + of '\0': + break + of '\c': + pos = lexbase.handleCR(my, pos) + buf = my.buf + break + of '\L': + pos = lexbase.handleLF(my, pos) + buf = my.buf + break + else: + inc(pos) + elif buf[pos+1] == '*': + # skip long comment: + inc(pos, 2) + while true: + case buf[pos] + of '\0': + my.err = errEOC_Expected + break + of '\c': + pos = lexbase.handleCR(my, pos) + buf = my.buf + of '\L': + pos = lexbase.handleLF(my, pos) + buf = my.buf + of '*': + inc(pos) + if buf[pos] == '/': + inc(pos) + break + else: + inc(pos) + else: + break + of ' ', '\t': + inc(pos) + of '\c': + pos = lexbase.handleCR(my, pos) + buf = my.buf + of '\L': + pos = lexbase.handleLF(my, pos) + buf = my.buf + else: + break + my.bufpos = pos + +proc parseNumber(my: var JsonParser) = + var pos = my.bufpos + var buf = my.buf + if buf[pos] == '-': + add(my.a, '-') + inc(pos) + if buf[pos] == '.': + add(my.a, "0.") + inc(pos) + else: + while buf[pos] in Digits: + add(my.a, buf[pos]) + inc(pos) + if buf[pos] == '.': + add(my.a, '.') + inc(pos) + # digits after the dot: + while buf[pos] in Digits: + add(my.a, buf[pos]) + inc(pos) + if buf[pos] in {'E', 'e'}: + add(my.a, buf[pos]) + inc(pos) + if buf[pos] in {'+', '-'}: + add(my.a, buf[pos]) + inc(pos) + while buf[pos] in Digits: + add(my.a, buf[pos]) + inc(pos) + my.bufpos = pos + +proc parseName(my: var JsonParser) = + var pos = my.bufpos + var buf = my.buf + if buf[pos] in IdentStartChars: + while buf[pos] in IdentChars: + add(my.a, buf[pos]) + inc(pos) + my.bufpos = pos + +proc getTok*(my: var JsonParser): TokKind = + setLen(my.a, 0) + skip(my) # skip whitespace, comments + case my.buf[my.bufpos] + of '-', '.', '0'..'9': + parseNumber(my) + if {'.', 'e', 'E'} in my.a: + result = tkFloat + else: + result = tkInt + of '"': + result = parseString(my) + of '[': + inc(my.bufpos) + result = tkBracketLe + of '{': + inc(my.bufpos) + result = tkCurlyLe + of ']': + inc(my.bufpos) + result = tkBracketRi + of '}': + inc(my.bufpos) + result = tkCurlyRi + of ',': + inc(my.bufpos) + result = tkComma + of ':': + inc(my.bufpos) + result = tkColon + of '\0': + result = tkEof + of 'a'..'z', 'A'..'Z', '_': + parseName(my) + case my.a + of "null": result = tkNull + of "true": result = tkTrue + of "false": result = tkFalse + else: result = tkError + else: + inc(my.bufpos) + result = tkError + my.tok = result + + +proc next*(my: var JsonParser) = + ## retrieves the first/next event. This controls the parser. + var tk = getTok(my) + var i = my.state.len-1 + # the following code is a state machine. If we had proper coroutines, + # the code could be much simpler. + case my.state[i] + of stateEof: + if tk == tkEof: + my.kind = jsonEof + else: + my.kind = jsonError + my.err = errEofExpected + of stateStart: + # tokens allowed? + case tk + of tkString, tkInt, tkFloat, tkTrue, tkFalse, tkNull: + my.state[i] = stateEof # expect EOF next! + my.kind = JsonEventKind(ord(tk)) + of tkBracketLe: + my.state.add(stateArray) # we expect any + my.kind = jsonArrayStart + of tkCurlyLe: + my.state.add(stateObject) + my.kind = jsonObjectStart + of tkEof: + my.kind = jsonEof + else: + my.kind = jsonError + my.err = errEofExpected + of stateObject: + case tk + of tkString, tkInt, tkFloat, tkTrue, tkFalse, tkNull: + my.state.add(stateExpectColon) + my.kind = JsonEventKind(ord(tk)) + of tkBracketLe: + my.state.add(stateExpectColon) + my.state.add(stateArray) + my.kind = jsonArrayStart + of tkCurlyLe: + my.state.add(stateExpectColon) + my.state.add(stateObject) + my.kind = jsonObjectStart + of tkCurlyRi: + my.kind = jsonObjectEnd + discard my.state.pop() + else: + my.kind = jsonError + my.err = errCurlyRiExpected + of stateArray: + case tk + of tkString, tkInt, tkFloat, tkTrue, tkFalse, tkNull: + my.state.add(stateExpectArrayComma) # expect value next! + my.kind = JsonEventKind(ord(tk)) + of tkBracketLe: + my.state.add(stateExpectArrayComma) + my.state.add(stateArray) + my.kind = jsonArrayStart + of tkCurlyLe: + my.state.add(stateExpectArrayComma) + my.state.add(stateObject) + my.kind = jsonObjectStart + of tkBracketRi: + my.kind = jsonArrayEnd + discard my.state.pop() + else: + my.kind = jsonError + my.err = errBracketRiExpected + of stateExpectArrayComma: + case tk + of tkComma: + discard my.state.pop() + next(my) + of tkBracketRi: + my.kind = jsonArrayEnd + discard my.state.pop() # pop stateExpectArrayComma + discard my.state.pop() # pop stateArray + else: + my.kind = jsonError + my.err = errBracketRiExpected + of stateExpectObjectComma: + case tk + of tkComma: + discard my.state.pop() + next(my) + of tkCurlyRi: + my.kind = jsonObjectEnd + discard my.state.pop() # pop stateExpectObjectComma + discard my.state.pop() # pop stateObject + else: + my.kind = jsonError + my.err = errCurlyRiExpected + of stateExpectColon: + case tk + of tkColon: + my.state[i] = stateExpectValue + next(my) + else: + my.kind = jsonError + my.err = errColonExpected + of stateExpectValue: + case tk + of tkString, tkInt, tkFloat, tkTrue, tkFalse, tkNull: + my.state[i] = stateExpectObjectComma + my.kind = JsonEventKind(ord(tk)) + of tkBracketLe: + my.state[i] = stateExpectObjectComma + my.state.add(stateArray) + my.kind = jsonArrayStart + of tkCurlyLe: + my.state[i] = stateExpectObjectComma + my.state.add(stateObject) + my.kind = jsonObjectStart + else: + my.kind = jsonError + my.err = errExprExpected + +proc raiseParseErr*(p: JsonParser, msg: string) {.noinline, noreturn.} = + ## raises an `EJsonParsingError` exception. + raise newException(JsonParsingError, errorMsgExpected(p, msg)) + +proc eat*(p: var JsonParser, tok: TokKind) = + if p.tok == tok: discard getTok(p) + else: raiseParseErr(p, tokToStr[tok]) diff --git a/lib/pure/parseopt.nim b/lib/pure/parseopt.nim index 23568edb9..c91134738 100644 --- a/lib/pure/parseopt.nim +++ b/lib/pure/parseopt.nim @@ -11,11 +11,23 @@ ## It supports one convenience iterator over all command line options and some ## lower-level features. ## -## Supported syntax: +## Supported syntax with default empty ``shortNoVal``/``longNoVal``: ## ## 1. short options - ``-abcd``, where a, b, c, d are names ## 2. long option - ``--foo:bar``, ``--foo=bar`` or ``--foo`` ## 3. argument - everything else +## +## When ``shortNoVal``/``longNoVal`` are non-empty then the ':' and '=' above +## are still accepted, but become optional. Note that these option key sets +## must be updated along with the set of option keys taking no value, but +## keys which do take values need no special updates as their set evolves. +## +## When option values begin with ':' or '=' they need to be doubled up (as in +## ``--delim::``) or alternated (as in ``--delim=:``). +## +## The common ``--`` non-option argument delimiter appears as an empty string +## long option key. ``OptParser.cmd``, ``OptParser.pos``, and +## ``os.parseCmdLine`` may be used to complete parsing in that case. {.push debugger: off.} @@ -32,37 +44,41 @@ type cmdShortOption ## a short option ``-c`` detected OptParser* = object of RootObj ## this object implements the command line parser - cmd: string - pos: int + cmd*: string # cmd,pos exported so caller can catch "--" as.. + pos*: int # ..empty key or subcmd cmdArg & handle specially inShortState: bool + shortNoVal: set[char] + longNoVal: seq[string] + cmds: seq[string] + idx: int kind*: CmdLineKind ## the dected command line token key*, val*: TaintedString ## key and value pair; ``key`` is the option ## or the argument, ``value`` is not "" if ## the option was given a value -{.deprecated: [TCmdLineKind: CmdLineKind, TOptParser: OptParser].} - proc parseWord(s: string, i: int, w: var string, - delim: set[char] = {'\x09', ' ', '\0'}): int = + delim: set[char] = {'\t', ' '}): int = result = i - if s[result] == '\"': + if result < s.len and s[result] == '\"': inc(result) - while not (s[result] in {'\0', '\"'}): + while result < s.len: + if s[result] == '"': + inc result + break add(w, s[result]) inc(result) - if s[result] == '\"': inc(result) else: - while not (s[result] in delim): + while result < s.len and s[result] notin delim: add(w, s[result]) inc(result) when declared(os.paramCount): proc quote(s: string): string = - if find(s, {' ', '\t'}) >= 0 and s[0] != '"': + if find(s, {' ', '\t'}) >= 0 and s.len > 0 and s[0] != '"': if s[0] == '-': result = newStringOfCap(s.len) - var i = parseWord(s, 0, result, {'\0', ' ', '\x09', ':', '='}) - if s[i] in {':','='}: + var i = parseWord(s, 0, result, {' ', '\t', ':', '='}) + if i < s.len and s[i] in {':','='}: result.add s[i] inc i result.add '"' @@ -78,83 +94,157 @@ when declared(os.paramCount): # we cannot provide this for NimRtl creation on Posix, because we can't # access the command line arguments then! - proc initOptParser*(cmdline = ""): OptParser = + proc initOptParser*(cmdline = "", shortNoVal: set[char]={}, + longNoVal: seq[string] = @[]): OptParser = ## inits the option parser. If ``cmdline == ""``, the real command line - ## (as provided by the ``OS`` module) is taken. + ## (as provided by the ``OS`` module) is taken. If ``shortNoVal`` is + ## provided command users do not need to delimit short option keys and + ## values with a ':' or '='. If ``longNoVal`` is provided command users do + ## not need to delimit long option keys and values with a ':' or '=' + ## (though they still need at least a space). In both cases, ':' or '=' + ## may still be used if desired. They just become optional. result.pos = 0 + result.idx = 0 result.inShortState = false + result.shortNoVal = shortNoVal + result.longNoVal = longNoVal if cmdline != "": result.cmd = cmdline + result.cmds = parseCmdLine(cmdline) else: result.cmd = "" + result.cmds = newSeq[string](paramCount()) + for i in countup(1, paramCount()): + result.cmds[i-1] = paramStr(i).string + result.cmd.add quote(result.cmds[i-1]) + result.cmd.add ' ' + + result.kind = cmdEnd + result.key = TaintedString"" + result.val = TaintedString"" + + proc initOptParser*(cmdline: seq[TaintedString], shortNoVal: set[char]={}, + longNoVal: seq[string] = @[]): OptParser = + ## inits the option parser. If ``cmdline.len == 0``, the real command line + ## (as provided by the ``OS`` module) is taken. ``shortNoVal`` and + ## ``longNoVal`` behavior is the same as for ``initOptParser(string,...)``. + result.pos = 0 + result.idx = 0 + result.inShortState = false + result.shortNoVal = shortNoVal + result.longNoVal = longNoVal + result.cmd = "" + if cmdline.len != 0: + result.cmds = newSeq[string](cmdline.len) + for i in 0..<cmdline.len: + result.cmds[i] = cmdline[i].string + result.cmd.add quote(cmdline[i].string) + result.cmd.add ' ' + else: + result.cmds = newSeq[string](paramCount()) for i in countup(1, paramCount()): - result.cmd.add quote(paramStr(i).string) + result.cmds[i-1] = paramStr(i).string + result.cmd.add quote(result.cmds[i-1]) result.cmd.add ' ' result.kind = cmdEnd result.key = TaintedString"" result.val = TaintedString"" -proc handleShortOption(p: var OptParser) = +proc handleShortOption(p: var OptParser; cmd: string) = var i = p.pos p.kind = cmdShortOption - add(p.key.string, p.cmd[i]) + add(p.key.string, cmd[i]) inc(i) p.inShortState = true - while p.cmd[i] in {'\x09', ' '}: + while i < cmd.len and cmd[i] in {'\t', ' '}: inc(i) p.inShortState = false - if p.cmd[i] in {':', '='}: - inc(i) + if i < cmd.len and cmd[i] in {':', '='} or + card(p.shortNoVal) > 0 and p.key.string[0] notin p.shortNoVal: + if i < cmd.len and cmd[i] in {':', '='}: + inc(i) p.inShortState = false - while p.cmd[i] in {'\x09', ' '}: inc(i) - i = parseWord(p.cmd, i, p.val.string) - if p.cmd[i] == '\0': p.inShortState = false - p.pos = i + while i < cmd.len and cmd[i] in {'\t', ' '}: inc(i) + p.val = TaintedString substr(cmd, i) + p.pos = 0 + inc p.idx + else: + p.pos = i + if i >= cmd.len: + p.inShortState = false + p.pos = 0 + inc p.idx proc next*(p: var OptParser) {.rtl, extern: "npo$1".} = ## parses the first or next option; ``p.kind`` describes what token has been ## parsed. ``p.key`` and ``p.val`` are set accordingly. + if p.idx >= p.cmds.len: + p.kind = cmdEnd + return + var i = p.pos - while p.cmd[i] in {'\x09', ' '}: inc(i) + while i < p.cmds[p.idx].len and p.cmds[p.idx][i] in {'\t', ' '}: inc(i) p.pos = i setLen(p.key.string, 0) setLen(p.val.string, 0) if p.inShortState: - handleShortOption(p) - return - case p.cmd[i] - of '\0': - p.kind = cmdEnd - of '-': + p.inShortState = false + if i >= p.cmds[p.idx].len: + inc(p.idx) + p.pos = 0 + if p.idx >= p.cmds.len: + p.kind = cmdEnd + return + else: + handleShortOption(p, p.cmds[p.idx]) + return + + if i < p.cmds[p.idx].len and p.cmds[p.idx][i] == '-': inc(i) - if p.cmd[i] == '-': - p.kind = cmdLongoption + if i < p.cmds[p.idx].len and p.cmds[p.idx][i] == '-': + p.kind = cmdLongOption inc(i) - i = parseWord(p.cmd, i, p.key.string, {'\0', ' ', '\x09', ':', '='}) - while p.cmd[i] in {'\x09', ' '}: inc(i) - if p.cmd[i] in {':', '='}: + i = parseWord(p.cmds[p.idx], i, p.key.string, {' ', '\t', ':', '='}) + while i < p.cmds[p.idx].len and p.cmds[p.idx][i] in {'\t', ' '}: inc(i) + if i < p.cmds[p.idx].len and p.cmds[p.idx][i] in {':', '='}: inc(i) - while p.cmd[i] in {'\x09', ' '}: inc(i) - p.pos = parseWord(p.cmd, i, p.val.string) + while i < p.cmds[p.idx].len and p.cmds[p.idx][i] in {'\t', ' '}: inc(i) + # if we're at the end, use the next command line option: + if i >= p.cmds[p.idx].len and p.idx < p.cmds.len: + inc p.idx + i = 0 + p.val = TaintedString p.cmds[p.idx].substr(i) + elif len(p.longNoVal) > 0 and p.key.string notin p.longNoVal and p.idx+1 < p.cmds.len: + p.val = TaintedString p.cmds[p.idx+1] + inc p.idx else: - p.pos = i + p.val = TaintedString"" + inc p.idx + p.pos = 0 else: p.pos = i - handleShortOption(p) + handleShortOption(p, p.cmds[p.idx]) else: p.kind = cmdArgument - p.pos = parseWord(p.cmd, i, p.key.string) + p.key = TaintedString p.cmds[p.idx] + inc p.idx + p.pos = 0 -proc cmdLineRest*(p: OptParser): TaintedString {.rtl, extern: "npo$1".} = - ## retrieves the rest of the command line that has not been parsed yet. - result = strip(substr(p.cmd, p.pos, len(p.cmd) - 1)).TaintedString +when declared(os.paramCount): + proc cmdLineRest*(p: OptParser): TaintedString {.rtl, extern: "npo$1".} = + ## retrieves the rest of the command line that has not been parsed yet. + var res = "" + for i in p.idx..<p.cmds.len: + if i > p.idx: res.add ' ' + res.add quote(p.cmds[i]) + result = res.TaintedString iterator getopt*(p: var OptParser): tuple[kind: CmdLineKind, key, val: TaintedString] = ## This is an convenience iterator for iterating over the given OptParser object. ## Example: ## ## .. code-block:: nim - ## var p = initOptParser("--left --debug:3 -l=4 -r:2") + ## var p = initOptParser("--left --debug:3 -l -r:2") ## for kind, key, val in p.getopt(): ## case kind ## of cmdArgument: @@ -168,23 +258,37 @@ iterator getopt*(p: var OptParser): tuple[kind: CmdLineKind, key, val: TaintedSt ## # no filename has been given, so we show the help: ## writeHelp() p.pos = 0 + p.idx = 0 while true: next(p) if p.kind == cmdEnd: break yield (p.kind, p.key, p.val) when declared(initOptParser): - iterator getopt*(): tuple[kind: CmdLineKind, key, val: TaintedString] = - ## This is an convenience iterator for iterating over the command line arguments. - ## This create a new OptParser object. - ## See above for a more detailed example + iterator getopt*(cmdline: seq[TaintedString] = commandLineParams(), + shortNoVal: set[char]={}, longNoVal: seq[string] = @[]): + tuple[kind: CmdLineKind, key, val: TaintedString] = + ## This is an convenience iterator for iterating over command line arguments. + ## This creates a new OptParser. See the above ``getopt(var OptParser)`` + ## example for using default empty ``NoVal`` parameters. This example is + ## for the same option keys as that example but here option key-value + ## separators become optional for command users: ## ## .. code-block:: nim - ## for kind, key, val in getopt(): - ## # this will iterate over all arguments passed to the cmdline. - ## continue + ## for kind, key, val in getopt(shortNoVal = { 'l' }, + ## longNoVal = @[ "left" ]): + ## case kind + ## of cmdArgument: + ## filename = key + ## of cmdLongOption, cmdShortOption: + ## case key + ## of "help", "h": writeHelp() + ## of "version", "v": writeVersion() + ## of cmdEnd: assert(false) # cannot happen + ## if filename == "": + ## writeHelp() ## - var p = initOptParser() + var p = initOptParser(cmdline, shortNoVal=shortNoVal, longNoVal=longNoVal) while true: next(p) if p.kind == cmdEnd: break diff --git a/lib/pure/parseopt2.nim b/lib/pure/parseopt2.nim index a2ff9bf0c..51a70b6d1 100644 --- a/lib/pure/parseopt2.nim +++ b/lib/pure/parseopt2.nim @@ -17,6 +17,7 @@ ## 2. long option - ``--foo:bar``, ``--foo=bar`` or ``--foo`` ## 3. argument - everything else +{.deprecated: "Use the 'parseopt' module instead".} {.push debugger: off.} include "system/inclrtl" @@ -40,15 +41,13 @@ type ## or the argument, ``value`` is not "" if ## the option was given a value -{.deprecated: [TCmdLineKind: CmdLineKind, TOptParser: OptParser].} - proc initOptParser*(cmdline: seq[string]): OptParser {.rtl.} = ## Initalizes option parses with cmdline. cmdline should not contain ## argument 0 - program name. - ## If cmdline == nil default to current command line arguments. + ## If cmdline.len == 0 default to current command line arguments. result.remainingShortOptions = "" when not defined(createNimRtl): - if cmdline == nil: + if cmdline.len == 0: result.cmd = commandLineParams() return else: @@ -61,7 +60,7 @@ proc initOptParser*(cmdline: string): OptParser {.rtl, deprecated.} = ## and calls initOptParser(openarray[string]) ## Do not use. if cmdline == "": # backward compatibility - return initOptParser(seq[string](nil)) + return initOptParser(@[]) else: return initOptParser(cmdline.split) @@ -121,8 +120,6 @@ proc cmdLineRest*(p: OptParser): TaintedString {.rtl, extern: "npo2$1", deprecat type GetoptResult* = tuple[kind: CmdLineKind, key, val: TaintedString] -{.deprecated: [TGetoptResult: GetoptResult].} - iterator getopt*(p: var OptParser): GetoptResult = ## This is an convenience iterator for iterating over the given OptParser object. ## Example: diff --git a/lib/pure/parsesql.nim b/lib/pure/parsesql.nim index ae192ab9a..9aef43c1b 100644 --- a/lib/pure/parsesql.nim +++ b/lib/pure/parsesql.nim @@ -11,7 +11,7 @@ ## parser. It parses PostgreSQL syntax and the SQL ANSI standard. import - hashes, strutils, lexbase, streams + hashes, strutils, lexbase # ------------------- scanner ------------------------------------------------- @@ -45,8 +45,6 @@ type SqlLexer* = object of BaseLexer ## the parser object. filename: string -{.deprecated: [TToken: Token, TSqlLexer: SqlLexer].} - const tokKindToStr: array[TokKind, string] = [ "invalid", "[EOF]", "identifier", "quoted identifier", "string constant", @@ -62,10 +60,6 @@ const "count", ] -proc open(L: var SqlLexer, input: Stream, filename: string) = - lexbase.open(L, input) - L.filename = filename - proc close(L: var SqlLexer) = lexbase.close(L) @@ -496,6 +490,7 @@ type SqlNodeKind* = enum ## kind of SQL abstract syntax tree nkNone, nkIdent, + nkQuotedIdent, nkStringLit, nkBitStringLit, nkHexStringLit, @@ -551,13 +546,18 @@ type nkCreateIndexIfNotExists, nkEnumDef +const + LiteralNodes = { + nkIdent, nkQuotedIdent, nkStringLit, nkBitStringLit, nkHexStringLit, + nkIntegerLit, nkNumericLit + } + type SqlParseError* = object of ValueError ## Invalid SQL encountered SqlNode* = ref SqlNodeObj ## an SQL abstract syntax tree node SqlNodeObj* = object ## an SQL abstract syntax tree node case kind*: SqlNodeKind ## kind of syntax tree - of nkIdent, nkStringLit, nkBitStringLit, nkHexStringLit, - nkIntegerLit, nkNumericLit: + of LiteralNodes: strVal*: string ## AST leaf: the identifier, numeric literal ## string literal, etc. else: @@ -566,21 +566,26 @@ type SqlParser* = object of SqlLexer ## SQL parser object tok: Token + {.deprecated: [EInvalidSql: SqlParseError, PSqlNode: SqlNode, TSqlNode: SqlNodeObj, TSqlParser: SqlParser, TSqlNodeKind: SqlNodeKind].} -proc newNode(k: SqlNodeKind): SqlNode = +proc newNode*(k: SqlNodeKind): SqlNode = new(result) result.kind = k -proc newNode(k: SqlNodeKind, s: string): SqlNode = +proc newNode*(k: SqlNodeKind, s: string): SqlNode = new(result) result.kind = k result.strVal = s +proc newNode*(k: SqlNodeKind, sons: seq[SqlNode]): SqlNode = + new(result) + result.kind = k + result.sons = sons + proc len*(n: SqlNode): int = - if n.kind in {nkIdent, nkStringLit, nkBitStringLit, nkHexStringLit, - nkIntegerLit, nkNumericLit}: + if n.kind in LiteralNodes: result = 0 else: result = n.sons.len @@ -588,7 +593,6 @@ proc len*(n: SqlNode): int = proc `[]`*(n: SqlNode; i: int): SqlNode = n.sons[i] proc add*(father, n: SqlNode) = - if isNil(father.sons): father.sons = @[] add(father.sons, n) proc getTok(p: var SqlParser) = @@ -630,7 +634,7 @@ proc eat(p: var SqlParser, keyw: string) = if isKeyw(p, keyw): getTok(p) else: - sqlError(p, keyw.toUpper() & " expected") + sqlError(p, keyw.toUpperAscii() & " expected") proc opt(p: var SqlParser, kind: TokKind) = if p.tok.kind == kind: getTok(p) @@ -689,7 +693,10 @@ proc parseSelect(p: var SqlParser): SqlNode proc identOrLiteral(p: var SqlParser): SqlNode = case p.tok.kind - of tkIdentifier, tkQuotedIdentifier: + of tkQuotedIdentifier: + result = newNode(nkQuotedIdent, p.tok.literal) + getTok(p) + of tkIdentifier: result = newNode(nkIdent, p.tok.literal) getTok(p) of tkStringConstant, tkEscapeConstant, tkDollarQuotedConstant: @@ -713,11 +720,15 @@ proc identOrLiteral(p: var SqlParser): SqlNode = result.add(parseExpr(p)) eat(p, tkParRi) else: - sqlError(p, "expression expected") - getTok(p) # we must consume a token here to prevend endless loops! + if p.tok.literal == "*": + result = newNode(nkIdent, p.tok.literal) + getTok(p) + else: + sqlError(p, "expression expected") + getTok(p) # we must consume a token here to prevend endless loops! proc primary(p: var SqlParser): SqlNode = - if p.tok.kind == tkOperator or isKeyw(p, "not"): + if (p.tok.kind == tkOperator and (p.tok.literal == "+" or p.tok.literal == "-")) or isKeyw(p, "not"): result = newNode(nkPrefix) result.add(newNode(nkIdent, p.tok.literal)) getTok(p) @@ -762,7 +773,7 @@ proc lowestExprAux(p: var SqlParser, v: var SqlNode, limit: int): int = result = opPred while opPred > limit: node = newNode(nkInfix) - opNode = newNode(nkIdent, p.tok.literal.toLower()) + opNode = newNode(nkIdent, p.tok.literal.toLowerAscii()) getTok(p) result = lowestExprAux(p, v2, opPred) node.add(opNode) @@ -1078,11 +1089,23 @@ proc parseSelect(p: var SqlParser): SqlNode = if p.tok.kind != tkComma: break getTok(p) result.add(g) - if isKeyw(p, "limit"): + if isKeyw(p, "order"): getTok(p) - var l = newNode(nkLimit) - l.add(parseExpr(p)) - result.add(l) + eat(p, "by") + var n = newNode(nkOrder) + while true: + var e = parseExpr(p) + if isKeyw(p, "asc"): + getTok(p) # is default + elif isKeyw(p, "desc"): + getTok(p) + var x = newNode(nkDesc) + x.add(e) + e = x + n.add(e) + if p.tok.kind != tkComma: break + getTok(p) + result.add(n) if isKeyw(p, "having"): var h = newNode(nkHaving) while true: @@ -1099,22 +1122,6 @@ proc parseSelect(p: var SqlParser): SqlNode = elif isKeyw(p, "except"): result.add(newNode(nkExcept)) getTok(p) - if isKeyw(p, "order"): - getTok(p) - eat(p, "by") - var n = newNode(nkOrder) - while true: - var e = parseExpr(p) - if isKeyw(p, "asc"): getTok(p) # is default - elif isKeyw(p, "desc"): - getTok(p) - var x = newNode(nkDesc) - x.add(e) - e = x - n.add(e) - if p.tok.kind != tkComma: break - getTok(p) - result.add(n) if isKeyw(p, "join") or isKeyw(p, "inner") or isKeyw(p, "outer") or isKeyw(p, "cross"): var join = newNode(nkJoin) result.add(join) @@ -1122,12 +1129,17 @@ proc parseSelect(p: var SqlParser): SqlNode = join.add(newNode(nkIdent, "")) getTok(p) else: - join.add(newNode(nkIdent, p.tok.literal.toLower())) + join.add(newNode(nkIdent, p.tok.literal.toLowerAscii())) getTok(p) eat(p, "join") join.add(parseFromItem(p)) eat(p, "on") join.add(parseExpr(p)) + if isKeyw(p, "limit"): + getTok(p) + var l = newNode(nkLimit) + l.add(parseExpr(p)) + result.add(l) proc parseStmt(p: var SqlParser; parent: SqlNode) = if isKeyw(p, "create"): @@ -1161,14 +1173,6 @@ proc parseStmt(p: var SqlParser; parent: SqlNode) = else: sqlError(p, "SELECT, CREATE, UPDATE or DELETE expected") -proc open(p: var SqlParser, input: Stream, filename: string) = - ## opens the parser `p` and assigns the input stream `input` to it. - ## `filename` is only used for error messages. - open(SqlLexer(p), input, filename) - p.tok.kind = tkInvalid - p.tok.literal = "" - getTok(p) - proc parse(p: var SqlParser): SqlNode = ## parses the content of `p`'s input stream and returns the SQL AST. ## Syntax errors raise an `SqlParseError` exception. @@ -1183,24 +1187,6 @@ proc close(p: var SqlParser) = ## closes the parser `p`. The associated input stream is closed too. close(SqlLexer(p)) -proc parseSQL*(input: Stream, filename: string): SqlNode = - ## parses the SQL from `input` into an AST and returns the AST. - ## `filename` is only used for error messages. - ## Syntax errors raise an `SqlParseError` exception. - var p: SqlParser - open(p, input, filename) - try: - result = parse(p) - finally: - close(p) - -proc parseSQL*(input: string, filename=""): SqlNode = - ## parses the SQL from `input` into an AST and returns the AST. - ## `filename` is only used for error messages. - ## Syntax errors raise an `SqlParseError` exception. - parseSQL(newStringStream(input), "") - - type SqlWriter = object indent: int @@ -1218,12 +1204,12 @@ proc add(s: var SqlWriter, thing: string) = proc addKeyw(s: var SqlWriter, thing: string) = var keyw = thing if s.upperCase: - keyw = keyw.toUpper() + keyw = keyw.toUpperAscii() s.add(keyw) proc addIden(s: var SqlWriter, thing: string) = var iden = thing - if iden.toLower() in reservedKeywords: + if iden.toLowerAscii() in reservedKeywords: iden = '"' & iden & '"' s.add(iden) @@ -1251,15 +1237,20 @@ proc addMulti(s: var SqlWriter, n: SqlNode, sep = ',', prefix, suffix: char) = ra(n.sons[i], s) s.add(suffix) +proc quoted(s: string): string = + "\"" & replace(s, "\"", "\"\"") & "\"" + proc ra(n: SqlNode, s: var SqlWriter) = if n == nil: return case n.kind of nkNone: discard of nkIdent: - if allCharsInSet(n.strVal, {'\33'..'\127'}) and n.strVal.toLower() notin reservedKeywords: + if allCharsInSet(n.strVal, {'\33'..'\127'}): s.add(n.strVal) else: - s.add("\"" & replace(n.strVal, "\"", "\"\"") & "\"") + s.add(quoted(n.strVal)) + of nkQuotedIdent: + s.add(quoted(n.strVal)) of nkStringLit: s.add(escape(n.strVal, "'", "'")) of nkBitStringLit: @@ -1361,18 +1352,19 @@ proc ra(n: SqlNode, s: var SqlWriter) = s.addKeyw("select") if n.kind == nkSelectDistinct: s.addKeyw("distinct") - s.addMulti(n.sons[0]) - for i in 1 .. n.len-1: + for i in 0 ..< n.len: ra(n.sons[i], s) of nkSelectColumns: - assert(false) + for i, column in n.sons: + if i > 0: s.add(',') + ra(column, s) of nkSelectPair: ra(n.sons[0], s) if n.sons.len == 2: s.addKeyw("as") ra(n.sons[1], s) of nkFromItemPair: - if n.sons[0].kind == nkIdent: + if n.sons[0].kind in {nkIdent, nkQuotedIdent}: ra(n.sons[0], s) else: assert n.sons[0].kind == nkSelect @@ -1472,3 +1464,35 @@ proc renderSQL*(n: SqlNode, upperCase=false): string = proc `$`*(n: SqlNode): string = ## an alias for `renderSQL`. renderSQL(n) + +when not defined(js): + import streams + + proc open(L: var SqlLexer, input: Stream, filename: string) = + lexbase.open(L, input) + L.filename = filename + + proc open(p: var SqlParser, input: Stream, filename: string) = + ## opens the parser `p` and assigns the input stream `input` to it. + ## `filename` is only used for error messages. + open(SqlLexer(p), input, filename) + p.tok.kind = tkInvalid + p.tok.literal = "" + getTok(p) + + proc parseSQL*(input: Stream, filename: string): SqlNode = + ## parses the SQL from `input` into an AST and returns the AST. + ## `filename` is only used for error messages. + ## Syntax errors raise an `SqlParseError` exception. + var p: SqlParser + open(p, input, filename) + try: + result = parse(p) + finally: + close(p) + + proc parseSQL*(input: string, filename=""): SqlNode = + ## parses the SQL from `input` into an AST and returns the AST. + ## `filename` is only used for error messages. + ## Syntax errors raise an `SqlParseError` exception. + parseSQL(newStringStream(input), "") diff --git a/lib/pure/parseutils.nim b/lib/pure/parseutils.nim index 57387e62e..e633d8cf7 100644 --- a/lib/pure/parseutils.nim +++ b/lib/pure/parseutils.nim @@ -11,7 +11,7 @@ ## ## To unpack raw bytes look at the `streams <streams.html>`_ module. -{.deadCodeElim: on.} +{.deadCodeElim: on.} # dce option deprecated {.push debugger:off .} # the user does not want to trace a part # of the standard library! @@ -47,13 +47,15 @@ proc parseHex*(s: string, number: var int, start = 0; maxLen = 0): int {. ## discard parseHex("0x38", value) ## assert value == -200 ## - ## If 'maxLen==0' the length of the hexadecimal number has no - ## upper bound. Not more than ```maxLen`` characters are parsed. + ## If ``maxLen == 0`` the length of the hexadecimal number has no upper bound. + ## Else no more than ``start + maxLen`` characters are parsed, up to the + ## length of the string. var i = start var foundDigit = false - if s[i] == '0' and (s[i+1] == 'x' or s[i+1] == 'X'): inc(i, 2) - elif s[i] == '#': inc(i) - let last = if maxLen == 0: s.len else: i+maxLen + # get last index based on minimum `start + maxLen` or `s.len` + let last = min(s.len, if maxLen == 0: s.len else: i+maxLen) + if i+1 < last and s[i] == '0' and (s[i+1] in {'x', 'X'}): inc(i, 2) + elif i < last and s[i] == '#': inc(i) while i < last: case s[i] of '_': discard @@ -70,14 +72,20 @@ proc parseHex*(s: string, number: var int, start = 0; maxLen = 0): int {. inc(i) if foundDigit: result = i-start -proc parseOct*(s: string, number: var int, start = 0): int {. +proc parseOct*(s: string, number: var int, start = 0, maxLen = 0): int {. rtl, extern: "npuParseOct", noSideEffect.} = - ## parses an octal number and stores its value in ``number``. Returns + ## Parses an octal number and stores its value in ``number``. Returns ## the number of the parsed characters or 0 in case of an error. + ## + ## If ``maxLen == 0`` the length of the octal number has no upper bound. + ## Else no more than ``start + maxLen`` characters are parsed, up to the + ## length of the string. var i = start var foundDigit = false - if s[i] == '0' and (s[i+1] == 'o' or s[i+1] == 'O'): inc(i, 2) - while true: + # get last index based on minimum `start + maxLen` or `s.len` + let last = min(s.len, if maxLen == 0: s.len else: i+maxLen) + if i+1 < last and s[i] == '0' and (s[i+1] in {'o', 'O'}): inc(i, 2) + while i < last: case s[i] of '_': discard of '0'..'7': @@ -87,14 +95,20 @@ proc parseOct*(s: string, number: var int, start = 0): int {. inc(i) if foundDigit: result = i-start -proc parseBin*(s: string, number: var int, start = 0): int {. +proc parseBin*(s: string, number: var int, start = 0, maxLen = 0): int {. rtl, extern: "npuParseBin", noSideEffect.} = - ## parses an binary number and stores its value in ``number``. Returns + ## Parses an binary number and stores its value in ``number``. Returns ## the number of the parsed characters or 0 in case of an error. + ## + ## If ``maxLen == 0`` the length of the binary number has no upper bound. + ## Else no more than ``start + maxLen`` characters are parsed, up to the + ## length of the string. var i = start var foundDigit = false - if s[i] == '0' and (s[i+1] == 'b' or s[i+1] == 'B'): inc(i, 2) - while true: + # get last index based on minimum `start + maxLen` or `s.len` + let last = min(s.len, if maxLen == 0: s.len else: i+maxLen) + if i+1 < last and s[i] == '0' and (s[i+1] in {'b', 'B'}): inc(i, 2) + while i < last: case s[i] of '_': discard of '0'..'1': @@ -108,9 +122,9 @@ proc parseIdent*(s: string, ident: var string, start = 0): int = ## parses an identifier and stores it in ``ident``. Returns ## the number of the parsed characters or 0 in case of an error. var i = start - if s[i] in IdentStartChars: + if i < s.len and s[i] in IdentStartChars: inc(i) - while s[i] in IdentChars: inc(i) + while i < s.len and s[i] in IdentChars: inc(i) ident = substr(s, start, i-1) result = i-start @@ -119,11 +133,9 @@ proc parseIdent*(s: string, start = 0): string = ## Returns the parsed identifier or an empty string in case of an error. result = "" var i = start - - if s[i] in IdentStartChars: + if i < s.len and s[i] in IdentStartChars: inc(i) - while s[i] in IdentChars: inc(i) - + while i < s.len and s[i] in IdentChars: inc(i) result = substr(s, start, i-1) proc parseToken*(s: string, token: var string, validChars: set[char], @@ -134,24 +146,26 @@ proc parseToken*(s: string, token: var string, validChars: set[char], ## ## **Deprecated since version 0.8.12**: Use ``parseWhile`` instead. var i = start - while s[i] in validChars: inc(i) + while i < s.len and s[i] in validChars: inc(i) result = i-start token = substr(s, start, i-1) proc skipWhitespace*(s: string, start = 0): int {.inline.} = ## skips the whitespace starting at ``s[start]``. Returns the number of ## skipped characters. - while s[start+result] in Whitespace: inc(result) + while start+result < s.len and s[start+result] in Whitespace: inc(result) proc skip*(s, token: string, start = 0): int {.inline.} = ## skips the `token` starting at ``s[start]``. Returns the length of `token` ## or 0 if there was no `token` at ``s[start]``. - while result < token.len and s[result+start] == token[result]: inc(result) + while start+result < s.len and result < token.len and + s[result+start] == token[result]: + inc(result) if result != token.len: result = 0 proc skipIgnoreCase*(s, token: string, start = 0): int = ## same as `skip` but case is ignored for token matching. - while result < token.len and + while start+result < s.len and result < token.len and toLower(s[result+start]) == toLower(token[result]): inc(result) if result != token.len: result = 0 @@ -159,18 +173,18 @@ proc skipUntil*(s: string, until: set[char], start = 0): int {.inline.} = ## Skips all characters until one char from the set `until` is found ## or the end is reached. ## Returns number of characters skipped. - while s[result+start] notin until and s[result+start] != '\0': inc(result) + while start+result < s.len and s[result+start] notin until: inc(result) proc skipUntil*(s: string, until: char, start = 0): int {.inline.} = ## Skips all characters until the char `until` is found ## or the end is reached. ## Returns number of characters skipped. - while s[result+start] != until and s[result+start] != '\0': inc(result) + while start+result < s.len and s[result+start] != until: inc(result) proc skipWhile*(s: string, toSkip: set[char], start = 0): int {.inline.} = ## Skips all characters while one char from the set `token` is found. ## Returns number of characters skipped. - while s[result+start] in toSkip and s[result+start] != '\0': inc(result) + while start+result < s.len and s[result+start] in toSkip: inc(result) proc parseUntil*(s: string, token: var string, until: set[char], start = 0): int {.inline.} = @@ -197,6 +211,9 @@ proc parseUntil*(s: string, token: var string, until: string, ## parses a token and stores it in ``token``. Returns ## the number of the parsed characters or 0 in case of an error. A token ## consists of any character that comes before the `until` token. + if until.len == 0: + token.setLen(0) + return 0 var i = start while i < s.len: if s[i] == until[0]: @@ -214,7 +231,7 @@ proc parseWhile*(s: string, token: var string, validChars: set[char], ## the number of the parsed characters or 0 in case of an error. A token ## consists of the characters in `validChars`. var i = start - while s[i] in validChars: inc(i) + while i < s.len and s[i] in validChars: inc(i) result = i-start token = substr(s, start, i-1) @@ -231,16 +248,17 @@ proc rawParseInt(s: string, b: var BiggestInt, start = 0): int = var sign: BiggestInt = -1 i = start - if s[i] == '+': inc(i) - elif s[i] == '-': - inc(i) - sign = 1 - if s[i] in {'0'..'9'}: + if i < s.len: + if s[i] == '+': inc(i) + elif s[i] == '-': + inc(i) + sign = 1 + if i < s.len and s[i] in {'0'..'9'}: b = 0 - while s[i] in {'0'..'9'}: + while i < s.len and s[i] in {'0'..'9'}: b = b * 10 - (ord(s[i]) - ord('0')) inc(i) - while s[i] == '_': inc(i) # underscores are allowed and ignored + while i < s.len and s[i] == '_': inc(i) # underscores are allowed and ignored b = b * sign result = i - start {.pop.} # overflowChecks @@ -281,17 +299,17 @@ proc parseSaturatedNatural*(s: string, b: var int, start = 0): int = ## discard parseSaturatedNatural("848", res) ## doAssert res == 848 var i = start - if s[i] == '+': inc(i) - if s[i] in {'0'..'9'}: + if i < s.len and s[i] == '+': inc(i) + if i < s.len and s[i] in {'0'..'9'}: b = 0 - while s[i] in {'0'..'9'}: + while i < s.len and s[i] in {'0'..'9'}: let c = ord(s[i]) - ord('0') if b <= (high(int) - c) div 10: b = b * 10 + c else: b = high(int) inc(i) - while s[i] == '_': inc(i) # underscores are allowed and ignored + while i < s.len and s[i] == '_': inc(i) # underscores are allowed and ignored result = i - start # overflowChecks doesn't work with BiggestUInt @@ -300,16 +318,16 @@ proc rawParseUInt(s: string, b: var BiggestUInt, start = 0): int = res = 0.BiggestUInt prev = 0.BiggestUInt i = start - if s[i] == '+': inc(i) # Allow - if s[i] in {'0'..'9'}: + if i < s.len and s[i] == '+': inc(i) # Allow + if i < s.len and s[i] in {'0'..'9'}: b = 0 - while s[i] in {'0'..'9'}: + while i < s.len and s[i] in {'0'..'9'}: prev = res res = res * 10 + (ord(s[i]) - ord('0')).BiggestUInt if prev > res: return 0 # overflowChecks emulation inc(i) - while s[i] == '_': inc(i) # underscores are allowed and ignored + while i < s.len and s[i] == '_': inc(i) # underscores are allowed and ignored b = res result = i - start @@ -364,8 +382,6 @@ type ikVar, ## ``var`` part of the interpolated string ikExpr ## ``expr`` part of the interpolated string -{.deprecated: [TInterpolatedKind: InterpolatedKind].} - iterator interpolatedFragments*(s: string): tuple[kind: InterpolatedKind, value: string] = ## Tokenizes the string `s` into substrings for interpolation purposes. @@ -389,31 +405,31 @@ iterator interpolatedFragments*(s: string): tuple[kind: InterpolatedKind, var kind: InterpolatedKind while true: var j = i - if s[j] == '$': - if s[j+1] == '{': + if j < s.len and s[j] == '$': + if j+1 < s.len and s[j+1] == '{': inc j, 2 var nesting = 0 - while true: - case s[j] - of '{': inc nesting - of '}': - if nesting == 0: - inc j - break - dec nesting - of '\0': - raise newException(ValueError, - "Expected closing '}': " & substr(s, i, s.high)) - else: discard - inc j + block curlies: + while j < s.len: + case s[j] + of '{': inc nesting + of '}': + if nesting == 0: + inc j + break curlies + dec nesting + else: discard + inc j + raise newException(ValueError, + "Expected closing '}': " & substr(s, i, s.high)) inc i, 2 # skip ${ kind = ikExpr - elif s[j+1] in IdentStartChars: + elif j+1 < s.len and s[j+1] in IdentStartChars: inc j, 2 - while s[j] in IdentChars: inc(j) + while j < s.len and s[j] in IdentChars: inc(j) inc i # skip $ kind = ikVar - elif s[j+1] == '$': + elif j+1 < s.len and s[j+1] == '$': inc j, 2 inc i # skip $ kind = ikDollar diff --git a/lib/pure/parsexml.nim b/lib/pure/parsexml.nim index 978c9c516..d8d5a7a2d 100644 --- a/lib/pure/parsexml.nim +++ b/lib/pure/parsexml.nim @@ -26,27 +26,125 @@ ## creates. ## ## -## Example 1: Retrieve HTML title -## ============================== -## -## The file ``examples/htmltitle.nim`` demonstrates how to use the -## XML parser to accomplish a simple task: To determine the title of an HTML -## document. -## -## .. code-block:: nim -## :file: ../../examples/htmltitle.nim -## -## -## Example 2: Retrieve all HTML links -## ================================== -## -## The file ``examples/htmlrefs.nim`` demonstrates how to use the -## XML parser to accomplish another simple task: To determine all the links -## an HTML document contains. -## -## .. code-block:: nim -## :file: ../../examples/htmlrefs.nim -## + +##[ + +Example 1: Retrieve HTML title +============================== + +The file ``examples/htmltitle.nim`` demonstrates how to use the +XML parser to accomplish a simple task: To determine the title of an HTML +document. + +.. code-block:: nim + + # Example program to show the parsexml module + # This program reads an HTML file and writes its title to stdout. + # Errors and whitespace are ignored. + + import os, streams, parsexml, strutils + + if paramCount() < 1: + quit("Usage: htmltitle filename[.html]") + + var filename = addFileExt(paramStr(1), "html") + var s = newFileStream(filename, fmRead) + if s == nil: quit("cannot open the file " & filename) + var x: XmlParser + open(x, s, filename) + while true: + x.next() + case x.kind + of xmlElementStart: + if cmpIgnoreCase(x.elementName, "title") == 0: + var title = "" + x.next() # skip "<title>" + while x.kind == xmlCharData: + title.add(x.charData) + x.next() + if x.kind == xmlElementEnd and cmpIgnoreCase(x.elementName, "title") == 0: + echo("Title: " & title) + quit(0) # Success! + else: + echo(x.errorMsgExpected("/title")) + + of xmlEof: break # end of file reached + else: discard # ignore other events + + x.close() + quit("Could not determine title!") + +]## + +##[ + +Example 2: Retrieve all HTML links +================================== + +The file ``examples/htmlrefs.nim`` demonstrates how to use the +XML parser to accomplish another simple task: To determine all the links +an HTML document contains. + +.. code-block:: nim + + # Example program to show the new parsexml module + # This program reads an HTML file and writes all its used links to stdout. + # Errors and whitespace are ignored. + + import os, streams, parsexml, strutils + + proc `=?=` (a, b: string): bool = + # little trick: define our own comparator that ignores case + return cmpIgnoreCase(a, b) == 0 + + if paramCount() < 1: + quit("Usage: htmlrefs filename[.html]") + + var links = 0 # count the number of links + var filename = addFileExt(paramStr(1), "html") + var s = newFileStream(filename, fmRead) + if s == nil: quit("cannot open the file " & filename) + var x: XmlParser + open(x, s, filename) + next(x) # get first event + block mainLoop: + while true: + case x.kind + of xmlElementOpen: + # the <a href = "xyz"> tag we are interested in always has an attribute, + # thus we search for ``xmlElementOpen`` and not for ``xmlElementStart`` + if x.elementName =?= "a": + x.next() + if x.kind == xmlAttribute: + if x.attrKey =?= "href": + var link = x.attrValue + inc(links) + # skip until we have an ``xmlElementClose`` event + while true: + x.next() + case x.kind + of xmlEof: break mainLoop + of xmlElementClose: break + else: discard + x.next() # skip ``xmlElementClose`` + # now we have the description for the ``a`` element + var desc = "" + while x.kind == xmlCharData: + desc.add(x.charData) + x.next() + echo(desc & ": " & link) + else: + x.next() + of xmlEof: break # end of file reached + of xmlError: + echo(errorMsg(x)) + x.next() + else: x.next() # skip other events + + echo($links & " link(s) found!") + x.close() + +]## import hashes, strutils, lexbase, streams, unicode @@ -95,12 +193,10 @@ type kind: XmlEventKind err: XmlErrorKind state: ParserState + cIsEmpty: bool filename: string options: set[XmlParseOption] -{.deprecated: [TXmlParser: XmlParser, TXmlParseOptions: XmlParseOption, - TXmlError: XmlErrorKind, TXmlEventKind: XmlEventKind].} - const errorMessages: array[XmlErrorKind, string] = [ "no error", @@ -128,7 +224,8 @@ proc open*(my: var XmlParser, input: Stream, filename: string, my.kind = xmlError my.a = "" my.b = "" - my.c = nil + my.c = "" + my.cIsEmpty = true my.options = options proc close*(my: var XmlParser) {.inline.} = @@ -485,6 +582,7 @@ proc parseTag(my: var XmlParser) = my.kind = xmlElementOpen my.state = stateAttr my.c = my.a # save for later + my.cIsEmpty = false else: my.kind = xmlElementStart let slash = my.buf[my.bufpos] == '/' @@ -493,7 +591,8 @@ proc parseTag(my: var XmlParser) = if slash and my.buf[my.bufpos] == '>': inc(my.bufpos) my.state = stateEmptyElementTag - my.c = nil + my.c = "" + my.cIsEmpty = true elif my.buf[my.bufpos] == '>': inc(my.bufpos) else: @@ -681,7 +780,7 @@ proc next*(my: var XmlParser) = of stateEmptyElementTag: my.state = stateNormal my.kind = xmlElementEnd - if not my.c.isNil: + if not my.cIsEmpty: my.a = my.c of stateError: my.kind = xmlError diff --git a/lib/pure/pegs.nim b/lib/pure/pegs.nim index 5ae2d9182..3ee82917d 100644 --- a/lib/pure/pegs.nim +++ b/lib/pure/pegs.nim @@ -20,11 +20,11 @@ include "system/inclrtl" const useUnicode = true ## change this to deactivate proper UTF-8 support -import - strutils +import strutils, macros when useUnicode: import unicode + export unicode.`==` const InlineThreshold = 5 ## number of leaves; -1 to disable inlining @@ -32,7 +32,7 @@ const ## can be captured. More subpatterns cannot be captured! type - PegKind = enum + PegKind* = enum pkEmpty, pkAny, ## any character (.) pkAnyRune, ## any Unicode character (_) @@ -67,15 +67,15 @@ type pkRule, ## a <- b pkList, ## a, b pkStartAnchor ## ^ --> Internal DSL: startAnchor() - NonTerminalFlag = enum + NonTerminalFlag* = enum ntDeclared, ntUsed NonTerminalObj = object ## represents a non terminal symbol name: string ## the name of the symbol line: int ## line the symbol has been declared/used in col: int ## column the symbol has been declared/used in flags: set[NonTerminalFlag] ## the nonterminal's flags - rule: Node ## the rule that the symbol refers to - Node {.shallow.} = object + rule: Peg ## the rule that the symbol refers to + Peg* {.shallow.} = object ## type that represents a PEG case kind: PegKind of pkEmpty..pkWhitespace: nil of pkTerminal, pkTerminalIgnoreCase, pkTerminalIgnoreStyle: term: string @@ -83,12 +83,61 @@ type of pkCharChoice, pkGreedyRepSet: charChoice: ref set[char] of pkNonTerminal: nt: NonTerminal of pkBackRef..pkBackRefIgnoreStyle: index: range[0..MaxSubpatterns] - else: sons: seq[Node] + else: sons: seq[Peg] NonTerminal* = ref NonTerminalObj - Peg* = Node ## type that represents a PEG +proc kind*(p: Peg): PegKind = p.kind + ## Returns the *PegKind* of a given *Peg* object. -{.deprecated: [TPeg: Peg, TNode: Node].} +proc term*(p: Peg): string = p.term + ## Returns the *string* representation of a given *Peg* variant object + ## where present. + +proc ch*(p: Peg): char = p.ch + ## Returns the *char* representation of a given *Peg* variant object + ## where present. + +proc charChoice*(p: Peg): ref set[char] = p.charChoice + ## Returns the *charChoice* field of a given *Peg* variant object + ## where present. + +proc nt*(p: Peg): NonTerminal = p.nt + ## Returns the *NonTerminal* object of a given *Peg* variant object + ## where present. + +proc index*(p: Peg): range[0..MaxSubpatterns] = p.index + ## Returns the back-reference index of a captured sub-pattern in the + ## *Captures* object for a given *Peg* variant object where present. + +iterator items*(p: Peg): Peg {.inline.} = + ## Yields the child nodes of a *Peg* variant object where present. + for s in p.sons: + yield s + +iterator pairs*(p: Peg): (int, Peg) {.inline.} = + ## Yields the indices and child nodes of a *Peg* variant object where present. + for i in 0 ..< p.sons.len: + yield (i, p.sons[i]) + +proc name*(nt: NonTerminal): string = nt.name + ## Gets the name of the symbol represented by the parent *Peg* object variant + ## of a given *NonTerminal*. + +proc line*(nt: NonTerminal): int = nt.line + ## Gets the line number of the definition of the parent *Peg* object variant + ## of a given *NonTerminal*. + +proc col*(nt: NonTerminal): int = nt.col + ## Gets the column number of the definition of the parent *Peg* object variant + ## of a given *NonTerminal*. + +proc flags*(nt: NonTerminal): set[NonTerminalFlag] = nt.flags + ## Gets the *NonTerminalFlag*-typed flags field of the parent *Peg* variant + ## object of a given *NonTerminal*. + +proc rule*(nt: NonTerminal): Peg = nt.rule + ## Gets the *Peg* object representing the rule definition of the parent *Peg* + ## object variant of a given *NonTerminal*. proc term*(t: string): Peg {.nosideEffect, rtl, extern: "npegs$1Str".} = ## constructs a PEG from a terminal string @@ -525,215 +574,497 @@ when not useUnicode: proc isTitle(a: char): bool {.inline.} = return false proc isWhiteSpace(a: char): bool {.inline.} = return a in {' ', '\9'..'\13'} -proc rawMatch*(s: string, p: Peg, start: int, c: var Captures): int {. - nosideEffect, rtl, extern: "npegs$1".} = - ## low-level matching proc that implements the PEG interpreter. Use this - ## for maximum efficiency (every other PEG operation ends up calling this - ## proc). - ## Returns -1 if it does not match, else the length of the match - case p.kind - of pkEmpty: result = 0 # match of length 0 - of pkAny: - if s[start] != '\0': result = 1 - else: result = -1 - of pkAnyRune: - if s[start] != '\0': - result = runeLenAt(s, start) - else: - result = -1 - of pkLetter: - if s[start] != '\0': - var a: Rune - result = start - fastRuneAt(s, result, a) - if isAlpha(a): dec(result, start) +template matchOrParse(mopProc: untyped): typed = + # Used to make the main matcher proc *rawMatch* as well as event parser + # procs. For the former, *enter* and *leave* event handler code generators + # are provided which just return *discard*. + + proc mopProc(s: string, p: Peg, start: int, c: var Captures): int = + proc matchBackRef(s: string, p: Peg, start: int, c: var Captures): int = + # Parse handler code must run in an *of* clause of its own for each + # *PegKind*, so we encapsulate the identical clause body for + # *pkBackRef..pkBackRefIgnoreStyle* here. + if p.index >= c.ml: return -1 + var (a, b) = c.matches[p.index] + var n: Peg + n.kind = succ(pkTerminal, ord(p.kind)-ord(pkBackRef)) + n.term = s.substr(a, b) + mopProc(s, n, start, c) + + case p.kind + of pkEmpty: + enter(pkEmpty, s, p, start) + result = 0 # match of length 0 + leave(pkEmpty, s, p, start, result) + of pkAny: + enter(pkAny, s, p, start) + if start < s.len: result = 1 else: result = -1 - else: - result = -1 - of pkLower: - if s[start] != '\0': - var a: Rune - result = start - fastRuneAt(s, result, a) - if isLower(a): dec(result, start) + leave(pkAny, s, p, start, result) + of pkAnyRune: + enter(pkAnyRune, s, p, start) + if start < s.len: + result = runeLenAt(s, start) + else: + result = -1 + leave(pkAnyRune, s, p, start, result) + of pkLetter: + enter(pkLetter, s, p, start) + if start < s.len: + var a: Rune + result = start + fastRuneAt(s, result, a) + if isAlpha(a): dec(result, start) + else: result = -1 + else: + result = -1 + leave(pkLetter, s, p, start, result) + of pkLower: + enter(pkLower, s, p, start) + if start < s.len: + var a: Rune + result = start + fastRuneAt(s, result, a) + if isLower(a): dec(result, start) + else: result = -1 + else: + result = -1 + leave(pkLower, s, p, start, result) + of pkUpper: + enter(pkUpper, s, p, start) + if start < s.len: + var a: Rune + result = start + fastRuneAt(s, result, a) + if isUpper(a): dec(result, start) + else: result = -1 + else: + result = -1 + leave(pkUpper, s, p, start, result) + of pkTitle: + enter(pkTitle, s, p, start) + if start < s.len: + var a: Rune + result = start + fastRuneAt(s, result, a) + if isTitle(a): dec(result, start) + else: result = -1 + else: + result = -1 + leave(pkTitle, s, p, start, result) + of pkWhitespace: + enter(pkWhitespace, s, p, start) + if start < s.len: + var a: Rune + result = start + fastRuneAt(s, result, a) + if isWhiteSpace(a): dec(result, start) + else: result = -1 + else: + result = -1 + leave(pkWhitespace, s, p, start, result) + of pkGreedyAny: + enter(pkGreedyAny, s, p, start) + result = len(s) - start + leave(pkGreedyAny, s, p, start, result) + of pkNewLine: + enter(pkNewLine, s, p, start) + if start < s.len and s[start] == '\L': result = 1 + elif start < s.len and s[start] == '\C': + if start+1 < s.len and s[start+1] == '\L': result = 2 + else: result = 1 else: result = -1 - else: - result = -1 - of pkUpper: - if s[start] != '\0': - var a: Rune + leave(pkNewLine, s, p, start, result) + of pkTerminal: + enter(pkTerminal, s, p, start) + result = len(p.term) + for i in 0..result-1: + if start+i >= s.len or p.term[i] != s[start+i]: + result = -1 + break + leave(pkTerminal, s, p, start, result) + of pkTerminalIgnoreCase: + enter(pkTerminalIgnoreCase, s, p, start) + var + i = 0 + a, b: Rune result = start - fastRuneAt(s, result, a) - if isUpper(a): dec(result, start) - else: result = -1 - else: - result = -1 - of pkTitle: - if s[start] != '\0': - var a: Rune + while i < len(p.term): + if result >= s.len: + result = -1 + break + fastRuneAt(p.term, i, a) + fastRuneAt(s, result, b) + if toLower(a) != toLower(b): + result = -1 + break + dec(result, start) + leave(pkTerminalIgnoreCase, s, p, start, result) + of pkTerminalIgnoreStyle: + enter(pkTerminalIgnoreStyle, s, p, start) + var + i = 0 + a, b: Rune result = start - fastRuneAt(s, result, a) - if isTitle(a): dec(result, start) + while i < len(p.term): + while i < len(p.term): + fastRuneAt(p.term, i, a) + if a != Rune('_'): break + while result < s.len: + fastRuneAt(s, result, b) + if b != Rune('_'): break + if result >= s.len: + if i >= p.term.len: break + else: + result = -1 + break + elif toLower(a) != toLower(b): + result = -1 + break + dec(result, start) + leave(pkTerminalIgnoreStyle, s, p, start, result) + of pkChar: + enter(pkChar, s, p, start) + if start < s.len and p.ch == s[start]: result = 1 else: result = -1 - else: - result = -1 - of pkWhitespace: - if s[start] != '\0': - var a: Rune - result = start - fastRuneAt(s, result, a) - if isWhiteSpace(a): dec(result, start) + leave(pkChar, s, p, start, result) + of pkCharChoice: + enter(pkCharChoice, s, p, start) + if start < s.len and contains(p.charChoice[], s[start]): result = 1 else: result = -1 - else: + leave(pkCharChoice, s, p, start, result) + of pkNonTerminal: + enter(pkNonTerminal, s, p, start) + var oldMl = c.ml + when false: echo "enter: ", p.nt.name + result = mopProc(s, p.nt.rule, start, c) + when false: echo "leave: ", p.nt.name + if result < 0: c.ml = oldMl + leave(pkNonTerminal, s, p, start, result) + of pkSequence: + enter(pkSequence, s, p, start) + var oldMl = c.ml + result = 0 + for i in 0..high(p.sons): + var x = mopProc(s, p.sons[i], start+result, c) + if x < 0: + c.ml = oldMl + result = -1 + break + else: inc(result, x) + leave(pkSequence, s, p, start, result) + of pkOrderedChoice: + enter(pkOrderedChoice, s, p, start) + var oldMl = c.ml + for i in 0..high(p.sons): + result = mopProc(s, p.sons[i], start, c) + if result >= 0: break + c.ml = oldMl + leave(pkOrderedChoice, s, p, start, result) + of pkSearch: + enter(pkSearch, s, p, start) + var oldMl = c.ml + result = 0 + while start+result <= s.len: + var x = mopProc(s, p.sons[0], start+result, c) + if x >= 0: + inc(result, x) + leave(pkSearch, s, p, start, result) + return + inc(result) result = -1 - of pkGreedyAny: - result = len(s) - start - of pkNewLine: - if s[start] == '\L': result = 1 - elif s[start] == '\C': - if s[start+1] == '\L': result = 2 - else: result = 1 - else: result = -1 - of pkTerminal: - result = len(p.term) - for i in 0..result-1: - if p.term[i] != s[start+i]: - result = -1 - break - of pkTerminalIgnoreCase: - var - i = 0 - a, b: Rune - result = start - while i < len(p.term): - fastRuneAt(p.term, i, a) - fastRuneAt(s, result, b) - if toLower(a) != toLower(b): - result = -1 - break - dec(result, start) - of pkTerminalIgnoreStyle: - var - i = 0 - a, b: Rune - result = start - while i < len(p.term): - while true: - fastRuneAt(p.term, i, a) - if a != Rune('_'): break + c.ml = oldMl + leave(pkSearch, s, p, start, result) + of pkCapturedSearch: + enter(pkCapturedSearch, s, p, start) + var idx = c.ml # reserve a slot for the subpattern + inc(c.ml) + result = 0 + while start+result <= s.len: + var x = mopProc(s, p.sons[0], start+result, c) + if x >= 0: + if idx < MaxSubpatterns: + c.matches[idx] = (start, start+result-1) + #else: silently ignore the capture + inc(result, x) + leave(pkCapturedSearch, s, p, start, result) + return + inc(result) + result = -1 + c.ml = idx + leave(pkCapturedSearch, s, p, start, result) + of pkGreedyRep: + enter(pkGreedyRep, s, p, start) + result = 0 while true: - fastRuneAt(s, result, b) - if b != Rune('_'): break - if toLower(a) != toLower(b): - result = -1 - break - dec(result, start) - of pkChar: - if p.ch == s[start]: result = 1 - else: result = -1 - of pkCharChoice: - if contains(p.charChoice[], s[start]): result = 1 - else: result = -1 - of pkNonTerminal: - var oldMl = c.ml - when false: echo "enter: ", p.nt.name - result = rawMatch(s, p.nt.rule, start, c) - when false: echo "leave: ", p.nt.name - if result < 0: c.ml = oldMl - of pkSequence: - var oldMl = c.ml - result = 0 - for i in 0..high(p.sons): - var x = rawMatch(s, p.sons[i], start+result, c) - if x < 0: + var x = mopProc(s, p.sons[0], start+result, c) + # if x == 0, we have an endless loop; so the correct behaviour would be + # not to break. But endless loops can be easily introduced: + # ``(comment / \w*)*`` is such an example. Breaking for x == 0 does the + # expected thing in this case. + if x <= 0: break + inc(result, x) + leave(pkGreedyRep, s, p, start, result) + of pkGreedyRepChar: + enter(pkGreedyRepChar, s, p, start) + result = 0 + var ch = p.ch + while start+result < s.len and ch == s[start+result]: inc(result) + leave(pkGreedyRepChar, s, p, start, result) + of pkGreedyRepSet: + enter(pkGreedyRepSet, s, p, start) + result = 0 + while start+result < s.len and contains(p.charChoice[], s[start+result]): inc(result) + leave(pkGreedyRepSet, s, p, start, result) + of pkOption: + enter(pkOption, s, p, start) + result = max(0, mopProc(s, p.sons[0], start, c)) + leave(pkOption, s, p, start, result) + of pkAndPredicate: + enter(pkAndPredicate, s, p, start) + var oldMl = c.ml + result = mopProc(s, p.sons[0], start, c) + if result >= 0: result = 0 # do not consume anything + else: c.ml = oldMl + leave(pkAndPredicate, s, p, start, result) + of pkNotPredicate: + enter(pkNotPredicate, s, p, start) + var oldMl = c.ml + result = mopProc(s, p.sons[0], start, c) + if result < 0: result = 0 + else: c.ml = oldMl result = -1 - break - else: inc(result, x) - of pkOrderedChoice: - var oldMl = c.ml - for i in 0..high(p.sons): - result = rawMatch(s, p.sons[i], start, c) - if result >= 0: break - c.ml = oldMl - of pkSearch: - var oldMl = c.ml - result = 0 - while start+result <= s.len: - var x = rawMatch(s, p.sons[0], start+result, c) - if x >= 0: - inc(result, x) - return - inc(result) - result = -1 - c.ml = oldMl - of pkCapturedSearch: - var idx = c.ml # reserve a slot for the subpattern - inc(c.ml) - result = 0 - while start+result <= s.len: - var x = rawMatch(s, p.sons[0], start+result, c) - if x >= 0: + leave(pkNotPredicate, s, p, start, result) + of pkCapture: + enter(pkCapture, s, p, start) + var idx = c.ml # reserve a slot for the subpattern + inc(c.ml) + result = mopProc(s, p.sons[0], start, c) + if result >= 0: if idx < MaxSubpatterns: c.matches[idx] = (start, start+result-1) #else: silently ignore the capture - inc(result, x) - return - inc(result) - result = -1 - c.ml = idx - of pkGreedyRep: - result = 0 - while true: - var x = rawMatch(s, p.sons[0], start+result, c) - # if x == 0, we have an endless loop; so the correct behaviour would be - # not to break. But endless loops can be easily introduced: - # ``(comment / \w*)*`` is such an example. Breaking for x == 0 does the - # expected thing in this case. - if x <= 0: break - inc(result, x) - of pkGreedyRepChar: - result = 0 - var ch = p.ch - while ch == s[start+result]: inc(result) - of pkGreedyRepSet: - result = 0 - while contains(p.charChoice[], s[start+result]): inc(result) - of pkOption: - result = max(0, rawMatch(s, p.sons[0], start, c)) - of pkAndPredicate: - var oldMl = c.ml - result = rawMatch(s, p.sons[0], start, c) - if result >= 0: result = 0 # do not consume anything - else: c.ml = oldMl - of pkNotPredicate: - var oldMl = c.ml - result = rawMatch(s, p.sons[0], start, c) - if result < 0: result = 0 - else: - c.ml = oldMl - result = -1 - of pkCapture: - var idx = c.ml # reserve a slot for the subpattern - inc(c.ml) - result = rawMatch(s, p.sons[0], start, c) - if result >= 0: - if idx < MaxSubpatterns: - c.matches[idx] = (start, start+result-1) - #else: silently ignore the capture - else: - c.ml = idx - of pkBackRef..pkBackRefIgnoreStyle: - if p.index >= c.ml: return -1 - var (a, b) = c.matches[p.index] - var n: Peg - n.kind = succ(pkTerminal, ord(p.kind)-ord(pkBackRef)) - n.term = s.substr(a, b) - result = rawMatch(s, n, start, c) - of pkStartAnchor: - if c.origStart == start: result = 0 - else: result = -1 - of pkRule, pkList: assert false + else: + c.ml = idx + leave(pkCapture, s, p, start, result) + of pkBackRef: + enter(pkBackRef, s, p, start) + result = matchBackRef(s, p, start, c) + leave(pkBackRef, s, p, start, result) + of pkBackRefIgnoreCase: + enter(pkBackRefIgnoreCase, s, p, start) + result = matchBackRef(s, p, start, c) + leave(pkBackRefIgnoreCase, s, p, start, result) + of pkBackRefIgnoreStyle: + enter(pkBackRefIgnoreStyle, s, p, start) + result = matchBackRef(s, p, start, c) + leave(pkBackRefIgnoreStyle, s, p, start, result) + of pkStartAnchor: + enter(pkStartAnchor, s, p, start) + if c.origStart == start: result = 0 + else: result = -1 + leave(pkStartAnchor, s, p, start, result) + of pkRule, pkList: assert false + +proc rawMatch*(s: string, p: Peg, start: int, c: var Captures): int + {.noSideEffect, rtl, extern: "npegs$1".} = + ## low-level matching proc that implements the PEG interpreter. Use this + ## for maximum efficiency (every other PEG operation ends up calling this + ## proc). + ## Returns -1 if it does not match, else the length of the match + + # Set the handler generators to produce do-nothing handlers. + template enter(pk, s, p, start) = + discard + template leave(pk, s, p, start, length) = + discard + matchOrParse(matchIt) + result = matchIt(s, p, start, c) + +macro mkHandlerTplts(handlers: untyped): untyped = + # Transforms the handler spec in *handlers* into handler templates. + # The AST structure of *handlers[0]*: + # + # .. code-block:: + # StmtList + # Call + # Ident "pkNonTerminal" + # StmtList + # Call + # Ident "enter" + # StmtList + # <handler code block> + # Call + # Ident "leave" + # StmtList + # <handler code block> + # Call + # Ident "pkChar" + # StmtList + # Call + # Ident "leave" + # StmtList + # <handler code block> + # ... + proc mkEnter(hdName, body: NimNode): NimNode = + quote do: + template `hdName`(s, p, start) = + let s {.inject.} = s + let p {.inject.} = p + let start {.inject.} = start + `body` + + template mkLeave(hdPostf, body) {.dirty.} = + # this has to be dirty to be able to capture *result* as *length* in + # *leaveXX* calls. + template `leave hdPostf`(s, p, start, length) = + body + + result = newStmtList() + for topCall in handlers[0]: + if nnkCall != topCall.kind: + error("Call syntax expected.", topCall) + let pegKind = topCall[0] + if nnkIdent != pegKind.kind: + error("PegKind expected.", pegKind) + if 2 == topCall.len: + for hdDef in topCall[1]: + if nnkCall != hdDef.kind: + error("Call syntax expected.", hdDef) + if nnkIdent != hdDef[0].kind: + error("Handler identifier expected.", hdDef[0]) + if 2 == hdDef.len: + let hdPostf = substr(pegKind.strVal, 2) + case hdDef[0].strVal + of "enter": + result.add mkEnter(newIdentNode("enter" & hdPostf), hdDef[1]) + of "leave": + result.add getAst(mkLeave(ident(hdPostf), hdDef[1])) + else: + error( + "Unsupported handler identifier, expected 'enter' or 'leave'.", + hdDef[0] + ) + +template eventParser*(pegAst, handlers: untyped): (proc(s: string): int) = + ## Generates an interpreting event parser *proc* according to the specified + ## PEG AST and handler code blocks. The *proc* can be called with a string + ## to be parsed and will execute the handler code blocks whenever their + ## associated grammar element is matched. It returns -1 if the string does not + ## match, else the length of the total match. The following example code + ## evaluates an arithmetic expression defined by a simple PEG: + ## + ## .. code-block:: nim + ## import strutils, pegs + ## + ## let + ## pegAst = """ + ## Expr <- Sum + ## Sum <- Product (('+' / '-')Product)* + ## Product <- Value (('*' / '/')Value)* + ## Value <- [0-9]+ / '(' Expr ')' + ## """.peg + ## txt = "(5+3)/2-7*22" + ## + ## var + ## pStack: seq[string] = @[] + ## valStack: seq[float] = @[] + ## opStack = "" + ## let + ## parseArithExpr = pegAst.eventParser: + ## pkNonTerminal: + ## enter: + ## pStack.add p.nt.name + ## leave: + ## pStack.setLen pStack.high + ## if length > 0: + ## let matchStr = s.substr(start, start+length-1) + ## case p.nt.name + ## of "Value": + ## try: + ## valStack.add matchStr.parseFloat + ## echo valStack + ## except ValueError: + ## discard + ## of "Sum", "Product": + ## try: + ## let val = matchStr.parseFloat + ## except ValueError: + ## if valStack.len > 1 and opStack.len > 0: + ## valStack[^2] = case opStack[^1] + ## of '+': valStack[^2] + valStack[^1] + ## of '-': valStack[^2] - valStack[^1] + ## of '*': valStack[^2] * valStack[^1] + ## else: valStack[^2] / valStack[^1] + ## valStack.setLen valStack.high + ## echo valStack + ## opStack.setLen opStack.high + ## echo opStack + ## pkChar: + ## leave: + ## if length == 1 and "Value" != pStack[^1]: + ## let matchChar = s[start] + ## opStack.add matchChar + ## echo opStack + ## + ## let pLen = parseArithExpr(txt) + ## + ## The *handlers* parameter consists of code blocks for *PegKinds*, + ## which define the grammar elements of interest. Each block can contain + ## handler code to be executed when the parser enters and leaves text + ## matching the grammar element. An *enter* handler can access the specific + ## PEG AST node being matched as *p*, the entire parsed string as *s* + ## and the position of the matched text segment in *s* as *start*. A *leave* + ## handler can access *p*, *s*, *start* and also the length of the matched + ## text segment as *length*. For an unsuccessful match, the *enter* and + ## *leave* handlers will be executed, with *length* set to -1. + ## + ## Symbols declared in an *enter* handler can be made visible in the + ## corresponding *leave* handler by annotating them with an *inject* pragma. + proc rawParse(s: string, p: Peg, start: int, c: var Captures): int + {.genSym.} = + + # binding from *macros* + bind strVal + + mkHandlerTplts: + handlers + + macro enter(pegKind, s, pegNode, start: untyped): untyped = + # This is called by the matcher code in *matchOrParse* at the + # start of the code for a grammar element of kind *pegKind*. + # Expands to a call to the handler template if one was generated + # by *mkHandlerTplts*. + template mkDoEnter(hdPostf, s, pegNode, start) = + when declared(`enter hdPostf`): + `enter hdPostf`(s, pegNode, start): + else: + discard + let hdPostf = ident(substr(strVal(pegKind), 2)) + getAst(mkDoEnter(hdPostf, s, pegNode, start)) + + macro leave(pegKind, s, pegNode, start, length: untyped): untyped = + # Like *enter*, but called at the end of the matcher code for + # a grammar element of kind *pegKind*. + template mkDoLeave(hdPostf, s, pegNode, start, length) = + when declared(`leave hdPostf`): + `leave hdPostf`(s, pegNode, start, length): + else: + discard + let hdPostf = ident(substr(strVal(pegKind), 2)) + getAst(mkDoLeave(hdPostf, s, pegNode, start, length)) + + matchOrParse(parseIt) + parseIt(s, p, start, c) + + proc parser(s: string): int {.genSym.} = + # the proc to be returned + var + ms: array[MaxSubpatterns, (int, int)] + cs = Captures(matches: ms, ml: 0, origStart: 0) + rawParse(s, pegAst, 0, cs) + parser template fillMatches(s, caps, c) = for k in 0..c.ml-1: @@ -742,7 +1073,7 @@ template fillMatches(s, caps, c) = if startIdx != -1: caps[k] = substr(s, startIdx, endIdx) else: - caps[k] = nil + caps[k] = "" proc matchLen*(s: string, pattern: Peg, matches: var openArray[string], start = 0): int {.nosideEffect, rtl, extern: "npegs$1Capture".} = @@ -1006,14 +1337,18 @@ proc replace*(s: string, sub: Peg, cb: proc( inc(m) add(result, substr(s, i)) -proc transformFile*(infile, outfile: string, - subs: varargs[tuple[pattern: Peg, repl: string]]) {. - rtl, extern: "npegs$1".} = - ## reads in the file `infile`, performs a parallel replacement (calls - ## `parallelReplace`) and writes back to `outfile`. Raises ``EIO`` if an - ## error occurs. This is supposed to be used for quick scripting. - var x = readFile(infile).string - writeFile(outfile, x.parallelReplace(subs)) +when not defined(js): + proc transformFile*(infile, outfile: string, + subs: varargs[tuple[pattern: Peg, repl: string]]) {. + rtl, extern: "npegs$1".} = + ## reads in the file `infile`, performs a parallel replacement (calls + ## `parallelReplace`) and writes back to `outfile`. Raises ``EIO`` if an + ## error occurs. This is supposed to be used for quick scripting. + ## + ## **Note**: this proc does not exist while using the JS backend. + var x = readFile(infile).string + writeFile(outfile, x.parallelReplace(subs)) + iterator split*(s: string, sep: Peg): string = ## Splits the string `s` into substrings. @@ -1117,7 +1452,7 @@ proc handleCR(L: var PegLexer, pos: int): int = assert(L.buf[pos] == '\c') inc(L.lineNumber) result = pos+1 - if L.buf[result] == '\L': inc(result) + if result < L.buf.len and L.buf[result] == '\L': inc(result) L.lineStart = result proc handleLF(L: var PegLexer, pos: int): int = @@ -1213,12 +1548,13 @@ proc getEscapedChar(c: var PegLexer, tok: var Token) = proc skip(c: var PegLexer) = var pos = c.bufpos var buf = c.buf - while true: + while pos < c.buf.len: case buf[pos] of ' ', '\t': inc(pos) of '#': - while not (buf[pos] in {'\c', '\L', '\0'}): inc(pos) + while (pos < c.buf.len) and + not (buf[pos] in {'\c', '\L', '\0'}): inc(pos) of '\c': pos = handleCR(c, pos) buf = c.buf @@ -1234,7 +1570,7 @@ proc getString(c: var PegLexer, tok: var Token) = var pos = c.bufpos + 1 var buf = c.buf var quote = buf[pos-1] - while true: + while pos < c.buf.len: case buf[pos] of '\\': c.bufpos = pos @@ -1257,7 +1593,7 @@ proc getDollar(c: var PegLexer, tok: var Token) = if buf[pos] in {'0'..'9'}: tok.kind = tkBackref tok.index = 0 - while buf[pos] in {'0'..'9'}: + while pos < c.buf.len and buf[pos] in {'0'..'9'}: tok.index = tok.index * 10 + ord(buf[pos]) - ord('0') inc(pos) else: @@ -1273,11 +1609,11 @@ proc getCharSet(c: var PegLexer, tok: var Token) = if buf[pos] == '^': inc(pos) caret = true - while true: + while pos < c.buf.len: var ch: char case buf[pos] of ']': - inc(pos) + if pos < c.buf.len: inc(pos) break of '\\': c.bufpos = pos @@ -1292,11 +1628,14 @@ proc getCharSet(c: var PegLexer, tok: var Token) = inc(pos) incl(tok.charset, ch) if buf[pos] == '-': - if buf[pos+1] == ']': + if pos+1 < c.buf.len and buf[pos+1] == ']': incl(tok.charset, '-') inc(pos) else: - inc(pos) + if pos+1 < c.buf.len: + inc(pos) + else: + break var ch2: char case buf[pos] of '\\': @@ -1308,8 +1647,11 @@ proc getCharSet(c: var PegLexer, tok: var Token) = tok.kind = tkInvalid break else: - ch2 = buf[pos] - inc(pos) + if pos+1 < c.buf.len: + ch2 = buf[pos] + inc(pos) + else: + break for i in ord(ch)+1 .. ord(ch2): incl(tok.charset, chr(i)) c.bufpos = pos @@ -1318,15 +1660,15 @@ proc getCharSet(c: var PegLexer, tok: var Token) = proc getSymbol(c: var PegLexer, tok: var Token) = var pos = c.bufpos var buf = c.buf - while true: + while pos < c.buf.len: add(tok.literal, buf[pos]) inc(pos) - if buf[pos] notin strutils.IdentChars: break + if pos < buf.len and buf[pos] notin strutils.IdentChars: break c.bufpos = pos tok.kind = tkIdentifier proc getBuiltin(c: var PegLexer, tok: var Token) = - if c.buf[c.bufpos+1] in strutils.Letters: + if c.bufpos+1 < c.buf.len and c.buf[c.bufpos+1] in strutils.Letters: inc(c.bufpos) getSymbol(c, tok) tok.kind = tkBuiltin @@ -1339,10 +1681,12 @@ proc getTok(c: var PegLexer, tok: var Token) = tok.modifier = modNone setLen(tok.literal, 0) skip(c) + case c.buf[c.bufpos] of '{': inc(c.bufpos) - if c.buf[c.bufpos] == '@' and c.buf[c.bufpos+1] == '}': + if c.buf[c.bufpos] == '@' and c.bufpos+2 < c.buf.len and + c.buf[c.bufpos+1] == '}': tok.kind = tkCurlyAt inc(c.bufpos, 2) add(tok.literal, "{@}") @@ -1375,13 +1719,11 @@ proc getTok(c: var PegLexer, tok: var Token) = getBuiltin(c, tok) of '\'', '"': getString(c, tok) of '$': getDollar(c, tok) - of '\0': - tok.kind = tkEof - tok.literal = "[EOF]" of 'a'..'z', 'A'..'Z', '\128'..'\255': getSymbol(c, tok) if c.buf[c.bufpos] in {'\'', '"'} or - c.buf[c.bufpos] == '$' and c.buf[c.bufpos+1] in {'0'..'9'}: + c.buf[c.bufpos] == '$' and c.bufpos+1 < c.buf.len and + c.buf[c.bufpos+1] in {'0'..'9'}: case tok.literal of "i": tok.modifier = modIgnoreCase of "y": tok.modifier = modIgnoreStyle @@ -1402,7 +1744,7 @@ proc getTok(c: var PegLexer, tok: var Token) = inc(c.bufpos) add(tok.literal, '+') of '<': - if c.buf[c.bufpos+1] == '-': + if c.bufpos+2 < c.buf.len and c.buf[c.bufpos+1] == '-': inc(c.bufpos, 2) tok.kind = tkArrow add(tok.literal, "<-") @@ -1437,14 +1779,17 @@ proc getTok(c: var PegLexer, tok: var Token) = inc(c.bufpos) add(tok.literal, '^') else: + if c.bufpos >= c.buf.len: + tok.kind = tkEof + tok.literal = "[EOF]" add(tok.literal, c.buf[c.bufpos]) inc(c.bufpos) proc arrowIsNextTok(c: PegLexer): bool = # the only look ahead we need var pos = c.bufpos - while c.buf[pos] in {'\t', ' '}: inc(pos) - result = c.buf[pos] == '<' and c.buf[pos+1] == '-' + while pos < c.buf.len and c.buf[pos] in {'\t', ' '}: inc(pos) + result = c.buf[pos] == '<' and (pos+1 < c.buf.len) and c.buf[pos+1] == '-' # ----------------------------- parser ---------------------------------------- @@ -1467,7 +1812,7 @@ proc pegError(p: PegParser, msg: string, line = -1, col = -1) = proc getTok(p: var PegParser) = getTok(p, p.tok) - if p.tok.kind == tkInvalid: pegError(p, "invalid token") + if p.tok.kind == tkInvalid: pegError(p, "'" & p.tok.literal & "' is invalid token") proc eat(p: var PegParser, kind: TokKind) = if p.tok.kind == kind: getTok(p) @@ -1817,7 +2162,7 @@ when isMainModule: assert match("prefix/start", peg"^start$", 7) if "foo" =~ peg"{'a'}?.*": - assert matches[0] == nil + assert matches[0].len == 0 else: assert false if "foo" =~ peg"{''}.*": diff --git a/lib/pure/random.nim b/lib/pure/random.nim index 97ad12b99..e565fccf8 100644 --- a/lib/pure/random.nim +++ b/lib/pure/random.nim @@ -110,7 +110,7 @@ proc random*[T](a: openArray[T]): T {.deprecated.} = ## Use ``rand`` instead. result = a[random(a.low..a.len)] -proc rand*(r: var Rand; max: int): int {.benign.} = +proc rand*(r: var Rand; max: Natural): int {.benign.} = ## Returns a random number in the range 0..max. The sequence of ## random number is always the same, unless `randomize` is called ## which initializes the random number generator with a "random" @@ -128,7 +128,7 @@ proc rand*(max: int): int {.benign.} = ## number, i.e. a tickcount. rand(state, max) -proc rand*(r: var Rand; max: float): float {.benign.} = +proc rand*(r: var Rand; max: range[0.0 .. high(float)]): float {.benign.} = ## Returns a random number in the range 0..max. The sequence of ## random number is always the same, unless `randomize` is called ## which initializes the random number generator with a "random" @@ -191,8 +191,8 @@ when not defined(nimscript): proc randomize*() {.benign.} = ## Initializes the random number generator with a "random" ## number, i.e. a tickcount. Note: Does not work for NimScript. - let time = int64(times.epochTime() * 1_000_000_000) - randomize(time) + let now = times.getTime() + randomize(convert(Seconds, Nanoseconds, now.toUnix) + now.nanosecond) {.pop.} @@ -218,4 +218,17 @@ when isMainModule: doAssert rand(0) == 0 doAssert rand("a") == 'a' + when compileOption("rangeChecks"): + try: + discard rand(-1) + doAssert false + except RangeError: + discard + + try: + discard rand(-1.0) + doAssert false + except RangeError: + discard + main() diff --git a/lib/pure/rationals.nim b/lib/pure/rationals.nim index 7907b4e6c..3946cf85b 100644 --- a/lib/pure/rationals.nim +++ b/lib/pure/rationals.nim @@ -241,6 +241,33 @@ proc abs*[T](x: Rational[T]): Rational[T] = result.num = abs x.num result.den = abs x.den +proc `div`*[T: SomeInteger](x, y: Rational[T]): T = + ## Computes the rational truncated division. + (x.num * y.den) div (y.num * x.den) + +proc `mod`*[T: SomeInteger](x, y: Rational[T]): Rational[T] = + ## Computes the rational modulo by truncated division (remainder). + ## This is same as ``x - (x div y) * y``. + result = ((x.num * y.den) mod (y.num * x.den)) // (x.den * y.den) + reduce(result) + +proc floorDiv*[T: SomeInteger](x, y: Rational[T]): T = + ## Computes the rational floor division. + ## + ## Floor division is conceptually defined as ``floor(x / y)``. + ## This is different from the ``div`` operator, which is defined + ## as ``trunc(x / y)``. That is, ``div`` rounds towards ``0`` and ``floorDiv`` + ## rounds down. + floorDiv(x.num * y.den, y.num * x.den) + +proc floorMod*[T: SomeInteger](x, y: Rational[T]): Rational[T] = + ## Computes the rational modulo by floor division (modulo). + ## + ## This is same as ``x - floorDiv(x, y) * y``. + ## This proc behaves the same as the ``%`` operator in python. + result = floorMod(x.num * y.den, y.num * x.den) // (x.den * y.den) + reduce(result) + proc hash*[T](x: Rational[T]): Hash = ## Computes hash for rational `x` # reduce first so that hash(x) == hash(y) for x == y @@ -339,3 +366,12 @@ when isMainModule: assert toRational(0.33) == 33 // 100 assert toRational(0.22) == 11 // 50 assert toRational(10.0) == 10 // 1 + + assert (1//1) div (3//10) == 3 + assert (-1//1) div (3//10) == -3 + assert (3//10) mod (1//1) == 3//10 + assert (-3//10) mod (1//1) == -3//10 + assert floorDiv(1//1, 3//10) == 3 + assert floorDiv(-1//1, 3//10) == -4 + assert floorMod(3//10, 1//1) == 3//10 + assert floorMod(-3//10, 1//1) == 7//10 diff --git a/lib/pure/ropes.nim b/lib/pure/ropes.nim index 6ddd61afa..559afd279 100644 --- a/lib/pure/ropes.nim +++ b/lib/pure/ropes.nim @@ -19,7 +19,7 @@ include "system/inclrtl" import streams -{.deadCodeElim: on.} +{.deadCodeElim: on.} # dce option deprecated {.push debugger:off .} # the user does not want to trace a part # of the standard library! @@ -37,10 +37,6 @@ type length: int data: string # != nil if a leaf -{.deprecated: [PRope: Rope].} - -proc isConc(r: Rope): bool {.inline.} = return isNil(r.data) - # Note that the left and right pointers are not needed for leafs. # Leaves have relatively high memory overhead (~30 bytes on a 32 # bit machine) and we produce many of them. This is why we cache and @@ -131,7 +127,7 @@ proc insertInCache(s: string, tree: Rope): Rope = result.left = t t.right = nil -proc rope*(s: string = nil): Rope {.rtl, extern: "nro$1Str".} = +proc rope*(s: string = ""): Rope {.rtl, extern: "nro$1Str".} = ## Converts a string to a rope. if s.len == 0: result = nil @@ -173,16 +169,8 @@ proc `&`*(a, b: Rope): Rope {.rtl, extern: "nroConcRopeRope".} = else: result = newRope() result.length = a.length + b.length - when false: - # XXX rebalancing would be nice, but is too expensive. - result.left = a.left - var x = newRope() - x.left = a.right - x.right = b - result.right = x - else: - result.left = a - result.right = b + result.left = a + result.right = b proc `&`*(a: Rope, b: string): Rope {.rtl, extern: "nroConcRopeStr".} = ## the concatenation operator for ropes. @@ -211,11 +199,11 @@ proc `[]`*(r: Rope, i: int): char {.rtl, extern: "nroCharAt".} = var j = i if x == nil: return while true: - if not isConc(x): - if x.data.len <% j: return x.data[j] + if x != nil and x.data.len > 0: + if j < x.data.len: return x.data[j] return '\0' else: - if x.left.len >% j: + if x.left.length > j: x = x.left else: x = x.right @@ -227,11 +215,11 @@ iterator leaves*(r: Rope): string = var stack = @[r] while stack.len > 0: var it = stack.pop - while isConc(it): + while it.left != nil: + assert(it.right != nil) stack.add(it.right) it = it.left assert(it != nil) - assert(it.data != nil) yield it.data iterator items*(r: Rope): char = @@ -252,54 +240,6 @@ proc `$`*(r: Rope): string {.rtl, extern: "nroToString".}= result = newStringOfCap(r.len) for s in leaves(r): add(result, s) -when false: - # Format string caching seems reasonable: All leaves can be shared and format - # string parsing has to be done only once. A compiled format string is stored - # as a rope. A negative length is used for the index into the args array. - proc compiledArg(idx: int): Rope = - new(result) - result.length = -idx - - proc compileFrmt(frmt: string): Rope = - var i = 0 - var length = len(frmt) - result = nil - var num = 0 - while i < length: - if frmt[i] == '$': - inc(i) - case frmt[i] - of '$': - add(result, "$") - inc(i) - of '#': - inc(i) - add(result, compiledArg(num+1)) - inc(num) - of '0'..'9': - var j = 0 - while true: - j = j * 10 + ord(frmt[i]) - ord('0') - inc(i) - if frmt[i] notin {'0'..'9'}: break - add(s, compiledArg(j)) - of '{': - inc(i) - var j = 0 - while frmt[i] in {'0'..'9'}: - j = j * 10 + ord(frmt[i]) - ord('0') - inc(i) - if frmt[i] == '}': inc(i) - else: raise newException(EInvalidValue, "invalid format string") - add(s, compiledArg(j)) - else: raise newException(EInvalidValue, "invalid format string") - var start = i - while i < length: - if frmt[i] != '$': inc(i) - else: break - if i - 1 >= start: - add(result, substr(frmt, start, i-1)) - proc `%`*(frmt: string, args: openArray[Rope]): Rope {. rtl, extern: "nroFormat".} = ## `%` substitution operator for ropes. Does not support the ``$identifier`` diff --git a/lib/pure/scgi.nim b/lib/pure/scgi.nim index 1ff26954e..e36803823 100644 --- a/lib/pure/scgi.nim +++ b/lib/pure/scgi.nim @@ -94,9 +94,6 @@ type disp: Dispatcher AsyncScgiState* = ref AsyncScgiStateObj -{.deprecated: [EScgi: ScgiError, TScgiState: ScgiState, - PAsyncScgiState: AsyncScgiState].} - proc recvBuffer(s: var ScgiState, L: int) = if L > s.bufLen: s.bufLen = L diff --git a/lib/pure/selectors.nim b/lib/pure/selectors.nim index dda5658a2..e4c2b2124 100644 --- a/lib/pure/selectors.nim +++ b/lib/pure/selectors.nim @@ -261,6 +261,9 @@ else: param: int data: T + const + InvalidIdent = -1 + proc raiseIOSelectorsError[T](message: T) = var msg = "" when T is string: @@ -271,7 +274,7 @@ else: msg.add("Internal Error\n") var err = newException(IOSelectorsException, msg) raise err - + proc setNonBlocking(fd: cint) {.inline.} = setBlocking(fd.SocketHandle, false) @@ -302,6 +305,12 @@ else: if posix.sigprocmask(SIG_UNBLOCK, newmask, oldmask) == -1: raiseIOSelectorsError(osLastError()) + template clearKey[T](key: ptr SelectorKey[T]) = + var empty: T + key.ident = InvalidIdent + key.events = {} + key.data = empty + when defined(linux): include ioselects/ioselectors_epoll elif bsdPlatform: @@ -312,22 +321,24 @@ else: include ioselects/ioselectors_poll # need to replace it with event ports elif defined(genode): include ioselects/ioselectors_select # TODO: use the native VFS layer + elif defined(nintendoswitch): + include ioselects/ioselectors_select else: include ioselects/ioselectors_poll proc register*[T](s: Selector[T], fd: int | SocketHandle, - events: set[Event], data: T) {.deprecated.} = + events: set[Event], data: T) {.deprecated: "use registerHandle instead".} = ## **Deprecated since v0.18.0:** Use ``registerHandle`` instead. s.registerHandle(fd, events, data) -proc setEvent*(ev: SelectEvent) {.deprecated.} = +proc setEvent*(ev: SelectEvent) {.deprecated: "use trigger instead".} = ## Trigger event ``ev``. ## ## **Deprecated since v0.18.0:** Use ``trigger`` instead. ev.trigger() proc update*[T](s: Selector[T], fd: int | SocketHandle, - events: set[Event]) {.deprecated.} = + events: set[Event]) {.deprecated: "use updateHandle instead".} = ## Update file/socket descriptor ``fd``, registered in selector ## ``s`` with new events set ``event``. ## diff --git a/lib/pure/smtp.nim b/lib/pure/smtp.nim index 08e6c8112..d9b863a52 100644 --- a/lib/pure/smtp.nim +++ b/lib/pure/smtp.nim @@ -51,8 +51,6 @@ type Smtp* = SmtpBase[Socket] AsyncSmtp* = SmtpBase[AsyncSocket] -{.deprecated: [EInvalidReply: ReplyError, TMessage: Message, TSMTP: Smtp].} - proc debugSend(smtp: Smtp | AsyncSmtp, cmd: string) {.multisync.} = if smtp.debug: echo("C:" & cmd) @@ -121,8 +119,7 @@ proc newSmtp*(useSsl = false, debug=false, when compiledWithSsl: sslContext.wrapSocket(result.sock) else: - raise newException(SystemError, - "SMTP module compiled without SSL support") + {.error: "SMTP module compiled without SSL support".} proc newAsyncSmtp*(useSsl = false, debug=false, sslContext = defaultSslContext): AsyncSmtp = @@ -135,8 +132,7 @@ proc newAsyncSmtp*(useSsl = false, debug=false, when compiledWithSsl: sslContext.wrapSocket(result.sock) else: - raise newException(SystemError, - "SMTP module compiled without SSL support") + {.error: "SMTP module compiled without SSL support".} proc quitExcpt(smtp: AsyncSmtp, msg: string): Future[void] = var retFuture = newFuture[void]() diff --git a/lib/pure/stats.nim b/lib/pure/stats.nim index 964938133..ce32108c2 100644 --- a/lib/pure/stats.nim +++ b/lib/pure/stats.nim @@ -65,8 +65,6 @@ type y_stats*: RunningStat ## stats for second set of data s_xy: float ## accumulated data for combined xy -{.deprecated: [TFloatClass: FloatClass, TRunningStat: RunningStat].} - # ----------- RunningStat -------------------------- proc clear*(s: var RunningStat) = ## reset `s` diff --git a/lib/pure/streams.nim b/lib/pure/streams.nim index 354e07da3..09626136f 100644 --- a/lib/pure/streams.nim +++ b/lib/pure/streams.nim @@ -57,8 +57,6 @@ type tags: [WriteIOEffect], gcsafe.} flushImpl*: proc (s: Stream) {.nimcall, tags: [WriteIOEffect], gcsafe.} -{.deprecated: [PStream: Stream, TStream: StreamObj].} - proc flush*(s: Stream) = ## flushes the buffers that the stream `s` might use. if not isNil(s.flushImpl): s.flushImpl(s) @@ -76,48 +74,32 @@ proc atEnd*(s: Stream): bool = ## been read. result = s.atEndImpl(s) -proc atEnd*(s, unused: Stream): bool {.deprecated.} = - ## checks if more data can be read from `f`. Returns true if all data has - ## been read. - result = s.atEndImpl(s) - proc setPosition*(s: Stream, pos: int) = ## sets the position `pos` of the stream `s`. s.setPositionImpl(s, pos) -proc setPosition*(s, unused: Stream, pos: int) {.deprecated.} = - ## sets the position `pos` of the stream `s`. - s.setPositionImpl(s, pos) - proc getPosition*(s: Stream): int = ## retrieves the current position in the stream `s`. result = s.getPositionImpl(s) -proc getPosition*(s, unused: Stream): int {.deprecated.} = - ## retrieves the current position in the stream `s`. - result = s.getPositionImpl(s) - proc readData*(s: Stream, buffer: pointer, bufLen: int): int = ## low level proc that reads data into an untyped `buffer` of `bufLen` size. result = s.readDataImpl(s, buffer, bufLen) -proc readAll*(s: Stream): string = - ## Reads all available data. - const bufferSize = 1000 - result = newString(bufferSize) - var r = 0 - while true: - let readBytes = readData(s, addr(result[r]), bufferSize) - if readBytes < bufferSize: - setLen(result, r+readBytes) - break - inc r, bufferSize - setLen(result, r+bufferSize) - -proc readData*(s, unused: Stream, buffer: pointer, - bufLen: int): int {.deprecated.} = - ## low level proc that reads data into an untyped `buffer` of `bufLen` size. - result = s.readDataImpl(s, buffer, bufLen) +when not defined(js): + proc readAll*(s: Stream): string = + ## Reads all available data. + const bufferSize = 1024 + var buffer {.noinit.}: array[bufferSize, char] + while true: + let readBytes = readData(s, addr(buffer[0]), bufferSize) + if readBytes == 0: + break + let prevLen = result.len + result.setLen(prevLen + readBytes) + copyMem(addr(result[prevLen]), addr(buffer[0]), readBytes) + if readBytes < bufferSize: + break proc peekData*(s: Stream, buffer: pointer, bufLen: int): int = ## low level proc that reads data into an untyped `buffer` of `bufLen` size @@ -151,12 +133,12 @@ proc write*(s: Stream, x: string) = when nimvm: writeData(s, cstring(x), x.len) else: - if x.len > 0: writeData(s, unsafeAddr x[0], x.len) + if x.len > 0: writeData(s, cstring(x), x.len) -proc writeLn*(s: Stream, args: varargs[string, `$`]) {.deprecated.} = - ## **Deprecated since version 0.11.4:** Use **writeLine** instead. +proc write*(s: Stream, args: varargs[string, `$`]) = + ## writes one or more strings to the the stream. No length fields or + ## terminating zeros are written. for str in args: write(s, str) - write(s, "\n") proc writeLine*(s: Stream, args: varargs[string, `$`]) = ## writes one or more strings to the the stream `s` followed @@ -276,21 +258,21 @@ proc readStr*(s: Stream, length: int): TaintedString = ## reads a string of length `length` from the stream `s`. Raises `EIO` if ## an error occurred. result = newString(length).TaintedString - var L = readData(s, addr(string(result)[0]), length) + var L = readData(s, cstring(result), length) if L != length: setLen(result.string, L) proc peekStr*(s: Stream, length: int): TaintedString = ## peeks a string of length `length` from the stream `s`. Raises `EIO` if ## an error occurred. result = newString(length).TaintedString - var L = peekData(s, addr(string(result)[0]), length) + var L = peekData(s, cstring(result), length) if L != length: setLen(result.string, L) proc readLine*(s: Stream, line: var TaintedString): bool = ## reads a line of text from the stream `s` into `line`. `line` must not be ## ``nil``! May throw an IO exception. - ## A line of text may be delimited by ``CR``, ``LF`` or - ## ``CRLF``. The newline character(s) are not part of the returned string. + ## A line of text may be delimited by ```LF`` or ``CRLF``. + ## The newline character(s) are not part of the returned string. ## Returns ``false`` if the end of the file has been reached, ``true`` ## otherwise. If ``false`` is returned `line` contains no new data. line.string.setLen(0) @@ -321,6 +303,8 @@ proc readLine*(s: Stream): TaintedString = ## Reads a line from a stream `s`. Note: This is not very efficient. Raises ## `EIO` if an error occurred. result = TaintedString"" + if s.atEnd: + raise newEIO("cannot read from stream") while true: var c = readChar(s) if c == '\c': @@ -338,6 +322,13 @@ proc peekLine*(s: Stream): TaintedString = defer: setPosition(s, pos) result = readLine(s) +iterator lines*(s: Stream): TaintedString = + ## Iterates over every line in the stream. + ## The iteration is based on ``readLine``. + var line: TaintedString + while s.readLine(line): + yield line + when not defined(js): type @@ -346,8 +337,6 @@ when not defined(js): data*: string pos: int - {.deprecated: [PStringStream: StringStream, TStringStream: StringStreamObj].} - proc ssAtEnd(s: Stream): bool = var s = StringStream(s) return s.pos >= s.data.len @@ -388,7 +377,10 @@ when not defined(js): proc ssClose(s: Stream) = var s = StringStream(s) - s.data = nil + when defined(nimNoNilSeqs): + s.data = "" + else: + s.data = nil proc newStringStream*(s: string = ""): StringStream = ## creates a new stream from the string `s`. @@ -407,7 +399,6 @@ when not defined(js): FileStream* = ref FileStreamObj ## a stream that encapsulates a `File` FileStreamObj* = object of Stream f: File - {.deprecated: [PFileStream: FileStream, TFileStream: FileStreamObj].} proc fsClose(s: Stream) = if FileStream(s).f != nil: @@ -447,9 +438,19 @@ when not defined(js): ## creates a new stream from the file named `filename` with the mode `mode`. ## If the file cannot be opened, nil is returned. See the `system ## <system.html>`_ module for a list of available FileMode enums. + ## **This function returns nil in case of failure. To prevent unexpected + ## behavior and ensure proper error handling, use openFileStream instead.** var f: File if open(f, filename, mode, bufSize): result = newFileStream(f) + proc openFileStream*(filename: string, mode: FileMode = fmRead, bufSize: int = -1): FileStream = + ## creates a new stream from the file named `filename` with the mode `mode`. + ## If the file cannot be opened, an IO exception is raised. + var f: File + if open(f, filename, mode, bufSize): + return newFileStream(f) + else: + raise newEIO("cannot open file") when true: discard @@ -460,9 +461,6 @@ else: handle*: FileHandle pos: int - {.deprecated: [PFileHandleStream: FileHandleStream, - TFileHandleStream: FileHandleStreamObj].} - proc newEOS(msg: string): ref OSError = new(result) result.msg = msg diff --git a/lib/pure/strformat.nim b/lib/pure/strformat.nim index ba21f894f..f13eb5e8e 100644 --- a/lib/pure/strformat.nim +++ b/lib/pure/strformat.nim @@ -11,29 +11,78 @@ String `interpolation`:idx: / `format`:idx: inspired by Python's ``f``-strings. -Examples: +``fmt`` vs. ``&`` +================= + +You can use either ``fmt`` or the unary ``&`` operator for formatting. The +difference between them is subtle but important. + +The ``fmt"{expr}"`` syntax is more aesthetically pleasing, but it hides a small +gotcha. The string is a +`generalized raw string literal <manual.html#lexical-analysis-generalized-raw-string-literals>`_. +This has some surprising effects: + +.. code-block:: nim + + import strformat + let msg = "hello" + doAssert fmt"{msg}\n" == "hello\\n" + +Because the literal is a raw string literal, the ``\n`` is not interpreted as +an escape sequence. + +There are multiple ways to get around this, including the use of the ``&`` +operator: + +.. code-block:: nim + + import strformat + let msg = "hello" + + doAssert &"{msg}\n" == "hello\n" + + doAssert fmt"{msg}{'\n'}" == "hello\n" + doAssert fmt("{msg}\n") == "hello\n" + doAssert "{msg}\n".fmt == "hello\n" + +The choice of style is up to you. + +Formatting strings +================== .. code-block:: nim + import strformat + doAssert &"""{"abc":>4}""" == " abc" doAssert &"""{"abc":<4}""" == "abc " - doAssert &"{-12345:08}" == "-0012345" - doAssert &"{-1:3}" == " -1" - doAssert &"{-1:03}" == "-01" - doAssert &"{16:#X}" == "0x10" +Formatting floats +================= - doAssert &"{123.456}" == "123.456" - doAssert &"{123.456:>9.3f}" == " 123.456" - doAssert &"{123.456:9.3f}" == " 123.456" - doAssert &"{123.456:9.4f}" == " 123.4560" - doAssert &"{123.456:>9.0f}" == " 123." - doAssert &"{123.456:<9.4f}" == "123.4560 " +.. code-block:: nim - doAssert &"{123.456:e}" == "1.234560e+02" - doAssert &"{123.456:>13e}" == " 1.234560e+02" - doAssert &"{123.456:13e}" == " 1.234560e+02" + import strformat + doAssert fmt"{-12345:08}" == "-0012345" + doAssert fmt"{-1:3}" == " -1" + doAssert fmt"{-1:03}" == "-01" + doAssert fmt"{16:#X}" == "0x10" + + doAssert fmt"{123.456}" == "123.456" + doAssert fmt"{123.456:>9.3f}" == " 123.456" + doAssert fmt"{123.456:9.3f}" == " 123.456" + doAssert fmt"{123.456:9.4f}" == " 123.4560" + doAssert fmt"{123.456:>9.0f}" == " 123." + doAssert fmt"{123.456:<9.4f}" == "123.4560 " + + doAssert fmt"{123.456:e}" == "1.234560e+02" + doAssert fmt"{123.456:>13e}" == " 1.234560e+02" + doAssert fmt"{123.456:13e}" == " 1.234560e+02" + + +Implementation details +====================== An expression like ``&"{key} is {value:arg} {{z}}"`` is transformed into: @@ -86,8 +135,8 @@ For strings and numeric types the optional argument is a so-called "standard format specifier". -Standard format specifier -========================= +Standard format specifier for strings, integers and floats +========================================================== The general form of a standard format specifier is:: @@ -209,7 +258,9 @@ template callFormat(res, arg) {.dirty.} = # workaround in order to circumvent 'strutils.format' which matches # too but doesn't adhere to our protocol. res.add arg - elif compiles(format(arg, res)): + elif compiles(format(arg, res)) and + # Check if format returns void + not (compiles do: discard format(arg, res)): format(arg, res) elif compiles(format(arg)): res.add format(arg) @@ -228,133 +279,8 @@ template callFormatOption(res, arg, option) {.dirty.} = macro `&`*(pattern: string): untyped = ## For a specification of the ``&`` macro, see the module level documentation. - runnableExamples: - template check(actual, expected: string) = - doAssert actual == expected - - from strutils import toUpperAscii, repeat - - # Basic tests - let s = "string" - check &"{0} {s}", "0 string" - check &"{s[0..2].toUpperAscii}", "STR" - check &"{-10:04}", "-010" - check &"{-10:<04}", "-010" - check &"{-10:>04}", "-010" - check &"0x{10:02X}", "0x0A" - - check &"{10:#04X}", "0x0A" - - check &"""{"test":#>5}""", "#test" - check &"""{"test":>5}""", " test" - - check &"""{"test":#^7}""", "#test##" - - check &"""{"test": <5}""", "test " - check &"""{"test":<5}""", "test " - check &"{1f:.3f}", "1.000" - check &"Hello, {s}!", "Hello, string!" - - # Tests for identifers without parenthesis - check &"{s} works{s}", "string worksstring" - check &"{s:>7}", " string" - doAssert(not compiles(&"{s_works}")) # parsed as identifier `s_works` - - # Misc general tests - check &"{{}}", "{}" - check &"{0}%", "0%" - check &"{0}%asdf", "0%asdf" - check &("\n{\"\\n\"}\n"), "\n\n\n" - check &"""{"abc"}s""", "abcs" - - # String tests - check &"""{"abc"}""", "abc" - check &"""{"abc":>4}""", " abc" - check &"""{"abc":<4}""", "abc " - check &"""{"":>4}""", " " - check &"""{"":<4}""", " " - - # Int tests - check &"{12345}", "12345" - check &"{ - 12345}", "-12345" - check &"{12345:6}", " 12345" - check &"{12345:>6}", " 12345" - check &"{12345:4}", "12345" - check &"{12345:08}", "00012345" - check &"{-12345:08}", "-0012345" - check &"{0:0}", "0" - check &"{0:02}", "00" - check &"{-1:3}", " -1" - check &"{-1:03}", "-01" - check &"{10}", "10" - check &"{16:#X}", "0x10" - check &"{16:^#7X}", " 0x10 " - check &"{16:^+#7X}", " +0x10 " - - # Hex tests - check &"{0:x}", "0" - check &"{-0:x}", "0" - check &"{255:x}", "ff" - check &"{255:X}", "FF" - check &"{-255:x}", "-ff" - check &"{-255:X}", "-FF" - check &"{255:x} uNaffeCteD CaSe", "ff uNaffeCteD CaSe" - check &"{255:X} uNaffeCteD CaSe", "FF uNaffeCteD CaSe" - check &"{255:4x}", " ff" - check &"{255:04x}", "00ff" - check &"{-255:4x}", " -ff" - check &"{-255:04x}", "-0ff" - - # Float tests - check &"{123.456}", "123.456" - check &"{-123.456}", "-123.456" - check &"{123.456:.3f}", "123.456" - check &"{123.456:+.3f}", "+123.456" - check &"{-123.456:+.3f}", "-123.456" - check &"{-123.456:.3f}", "-123.456" - check &"{123.456:1g}", "123.456" - check &"{123.456:.1f}", "123.5" - check &"{123.456:.0f}", "123." - #check &"{123.456:.0f}", "123." - check &"{123.456:>9.3f}", " 123.456" - check &"{123.456:9.3f}", " 123.456" - check &"{123.456:>9.4f}", " 123.4560" - check &"{123.456:>9.0f}", " 123." - check &"{123.456:<9.4f}", "123.4560 " - - # Float (scientific) tests - check &"{123.456:e}", "1.234560e+02" - check &"{123.456:>13e}", " 1.234560e+02" - check &"{123.456:<13e}", "1.234560e+02 " - check &"{123.456:.1e}", "1.2e+02" - check &"{123.456:.2e}", "1.23e+02" - check &"{123.456:.3e}", "1.235e+02" - - # Note: times.format adheres to the format protocol. Test that this - # works: - import times - - var nullTime: DateTime - check &"{nullTime:yyyy-mm-dd}", "0000-00-00" - - # Unicode string tests - check &"""{"αβγ"}""", "αβγ" - check &"""{"αβγ":>5}""", " αβγ" - check &"""{"αβγ":<5}""", "αβγ " - check &"""a{"a"}α{"α"}€{"€"}𐍈{"𐍈"}""", "aaαα€€𐍈𐍈" - check &"""a{"a":2}α{"α":2}€{"€":2}𐍈{"𐍈":2}""", "aa αα €€ 𐍈𐍈 " - # Invalid unicode sequences should be handled as plain strings. - # Invalid examples taken from: https://stackoverflow.com/a/3886015/1804173 - let invalidUtf8 = [ - "\xc3\x28", "\xa0\xa1", - "\xe2\x28\xa1", "\xe2\x82\x28", - "\xf0\x28\x8c\xbc", "\xf0\x90\x28\xbc", "\xf0\x28\x8c\x28" - ] - for s in invalidUtf8: - check &"{s:>5}", repeat(" ", 5-s.len) & s - if pattern.kind notin {nnkStrLit..nnkTripleStrLit}: - error "& only works with string literals", pattern + error "string formatting (fmt(), &) only works with string literals", pattern let f = pattern.strVal var i = 0 let res = genSym(nskVar, "fmtRes") @@ -409,14 +335,6 @@ macro `&`*(pattern: string): untyped = template fmt*(pattern: string): untyped = ## An alias for ``&``. - ## **Examples:** - ## - ## .. code-block:: nim - ## import json - ## import strformat except `&` - ## - ## let example = "oh, look no conflicts anymore" - ## echo fmt"{example}" bind `&` &pattern @@ -459,7 +377,7 @@ type ## ``parseStandardFormatSpecifier`` returned. proc formatInt(n: SomeNumber; radix: int; spec: StandardFormatSpecifier): string = - ## Converts ``n`` to string. If ``n`` is `SomeReal`, it casts to `int64`. + ## Converts ``n`` to string. If ``n`` is `SomeFloat`, it casts to `int64`. ## Conversion is done using ``radix``. If result's length is lesser than ## ``minimumWidth``, it aligns result to the right or left (depending on ``a``) ## with ``fill`` char. @@ -587,8 +505,8 @@ proc format*(value: SomeInteger; specifier: string; res: var string) = " of 'x', 'X', 'b', 'd', 'o' but got: " & spec.typ) res.add formatInt(value, radix, spec) -proc format*(value: SomeReal; specifier: string; res: var string) = - ## Standard format implementation for ``SomeReal``. It makes little +proc format*(value: SomeFloat; specifier: string; res: var string) = + ## Standard format implementation for ``SomeFloat``. It makes little ## sense to call this directly, but it is required to exist ## by the ``&`` macro. let spec = parseStandardFormatSpecifier(specifier) @@ -608,8 +526,31 @@ proc format*(value: SomeReal; specifier: string; res: var string) = " of 'e', 'E', 'f', 'F', 'g', 'G' but got: " & spec.typ) var f = formatBiggestFloat(value, fmode, spec.precision) - if value >= 0.0 and spec.sign != '-': - f = spec.sign & f + var sign = false + if value >= 0.0: + if spec.sign != '-': + sign = true + if value == 0.0: + if 1.0 / value == Inf: + # only insert the sign if value != negZero + f.insert($spec.sign, 0) + else: + f.insert($spec.sign, 0) + else: + sign = true + + if spec.padWithZero: + var sign_str = "" + if sign: + sign_str = $f[0] + f = f[1..^1] + + let toFill = spec.minimumWidth - f.len - ord(sign) + if toFill > 0: + f = repeat('0', toFill) & f + if sign: + f = sign_str & f + # the default for numbers is right-alignment: let align = if spec.align == '\0': '>' else: spec.align let result = alignString(f, spec.minimumWidth, @@ -624,15 +565,149 @@ proc format*(value: string; specifier: string; res: var string) = ## sense to call this directly, but it is required to exist ## by the ``&`` macro. let spec = parseStandardFormatSpecifier(specifier) + var value = value case spec.typ of 's', '\0': discard else: raise newException(ValueError, "invalid type in format string for string, expected 's', but got " & spec.typ) + if spec.precision != -1: + if spec.precision < runelen(value): + setLen(value, runeOffset(value, spec.precision)) res.add alignString(value, spec.minimumWidth, spec.align, spec.fill) when isMainModule: + template check(actual, expected: string) = + doAssert actual == expected + + from strutils import toUpperAscii, repeat + + # Basic tests + let s = "string" + check &"{0} {s}", "0 string" + check &"{s[0..2].toUpperAscii}", "STR" + check &"{-10:04}", "-010" + check &"{-10:<04}", "-010" + check &"{-10:>04}", "-010" + check &"0x{10:02X}", "0x0A" + + check &"{10:#04X}", "0x0A" + + check &"""{"test":#>5}""", "#test" + check &"""{"test":>5}""", " test" + + check &"""{"test":#^7}""", "#test##" + + check &"""{"test": <5}""", "test " + check &"""{"test":<5}""", "test " + check &"{1f:.3f}", "1.000" + check &"Hello, {s}!", "Hello, string!" + + # Tests for identifers without parenthesis + check &"{s} works{s}", "string worksstring" + check &"{s:>7}", " string" + doAssert(not compiles(&"{s_works}")) # parsed as identifier `s_works` + + # Misc general tests + check &"{{}}", "{}" + check &"{0}%", "0%" + check &"{0}%asdf", "0%asdf" + check &("\n{\"\\n\"}\n"), "\n\n\n" + check &"""{"abc"}s""", "abcs" + + # String tests + check &"""{"abc"}""", "abc" + check &"""{"abc":>4}""", " abc" + check &"""{"abc":<4}""", "abc " + check &"""{"":>4}""", " " + check &"""{"":<4}""", " " + + # Int tests + check &"{12345}", "12345" + check &"{ - 12345}", "-12345" + check &"{12345:6}", " 12345" + check &"{12345:>6}", " 12345" + check &"{12345:4}", "12345" + check &"{12345:08}", "00012345" + check &"{-12345:08}", "-0012345" + check &"{0:0}", "0" + check &"{0:02}", "00" + check &"{-1:3}", " -1" + check &"{-1:03}", "-01" + check &"{10}", "10" + check &"{16:#X}", "0x10" + check &"{16:^#7X}", " 0x10 " + check &"{16:^+#7X}", " +0x10 " + + # Hex tests + check &"{0:x}", "0" + check &"{-0:x}", "0" + check &"{255:x}", "ff" + check &"{255:X}", "FF" + check &"{-255:x}", "-ff" + check &"{-255:X}", "-FF" + check &"{255:x} uNaffeCteD CaSe", "ff uNaffeCteD CaSe" + check &"{255:X} uNaffeCteD CaSe", "FF uNaffeCteD CaSe" + check &"{255:4x}", " ff" + check &"{255:04x}", "00ff" + check &"{-255:4x}", " -ff" + check &"{-255:04x}", "-0ff" + + # Float tests + check &"{123.456}", "123.456" + check &"{-123.456}", "-123.456" + check &"{123.456:.3f}", "123.456" + check &"{123.456:+.3f}", "+123.456" + check &"{-123.456:+.3f}", "-123.456" + check &"{-123.456:.3f}", "-123.456" + check &"{123.456:1g}", "123.456" + check &"{123.456:.1f}", "123.5" + check &"{123.456:.0f}", "123." + #check &"{123.456:.0f}", "123." + check &"{123.456:>9.3f}", " 123.456" + check &"{123.456:9.3f}", " 123.456" + check &"{123.456:>9.4f}", " 123.4560" + check &"{123.456:>9.0f}", " 123." + check &"{123.456:<9.4f}", "123.4560 " + + # Float (scientific) tests + check &"{123.456:e}", "1.234560e+02" + check &"{123.456:>13e}", " 1.234560e+02" + check &"{123.456:<13e}", "1.234560e+02 " + check &"{123.456:.1e}", "1.2e+02" + check &"{123.456:.2e}", "1.23e+02" + check &"{123.456:.3e}", "1.235e+02" + + # Note: times.format adheres to the format protocol. Test that this + # works: + import times + + var dt = initDateTime(01, mJan, 2000, 00, 00, 00) + check &"{dt:yyyy-MM-dd}", "2000-01-01" + + var tm = fromUnix(0) + discard &"{tm}" + + # Unicode string tests + check &"""{"αβγ"}""", "αβγ" + check &"""{"αβγ":>5}""", " αβγ" + check &"""{"αβγ":<5}""", "αβγ " + check &"""a{"a"}α{"α"}€{"€"}𐍈{"𐍈"}""", "aaαα€€𐍈𐍈" + check &"""a{"a":2}α{"α":2}€{"€":2}𐍈{"𐍈":2}""", "aa αα €€ 𐍈𐍈 " + # Invalid unicode sequences should be handled as plain strings. + # Invalid examples taken from: https://stackoverflow.com/a/3886015/1804173 + let invalidUtf8 = [ + "\xc3\x28", "\xa0\xa1", + "\xe2\x28\xa1", "\xe2\x82\x28", + "\xf0\x28\x8c\xbc", "\xf0\x90\x28\xbc", "\xf0\x28\x8c\x28" + ] + for s in invalidUtf8: + check &"{s:>5}", repeat(" ", 5-s.len) & s + + import json doAssert fmt"{'a'} {'b'}" == "a b" + + echo("All tests ok") diff --git a/lib/pure/strmisc.nim b/lib/pure/strmisc.nim index 89ef2fcd2..d1ff920c9 100644 --- a/lib/pure/strmisc.nim +++ b/lib/pure/strmisc.nim @@ -12,7 +12,7 @@ import strutils -{.deadCodeElim: on.} +{.deadCodeElim: on.} # dce option deprecated proc expandTabs*(s: string, tabSize: int = 8): string {.noSideEffect, procvar.} = diff --git a/lib/pure/strscans.nim b/lib/pure/strscans.nim index 2bd87837f..b17eee6ff 100644 --- a/lib/pure/strscans.nim +++ b/lib/pure/strscans.nim @@ -10,7 +10,7 @@ ##[ This module contains a `scanf`:idx: macro that can be used for extracting substrings from an input string. This is often easier than regular expressions. -Some examples as an apetizer: +Some examples as an appetizer: .. code-block:: nim # check if input string matches a triple of integers: @@ -87,7 +87,7 @@ which we then use in our scanf pattern to help us in the matching process: proc someSep(input: string; start: int; seps: set[char] = {':','-','.'}): int = # Note: The parameters and return value must match to what ``scanf`` requires result = 0 - while input[start+result] in seps: inc result + while start+result < input.len and input[start+result] in seps: inc result if scanf(input, "$w$[someSep]$w", key, value): ... @@ -231,7 +231,7 @@ is performed. var i = start var u = 0 while true: - if s[i] == '\0' or s[i] == unless: + if i >= s.len or s[i] == unless: return 0 elif s[i] == until[0]: u = 1 @@ -308,13 +308,18 @@ proc buildUserCall(x: string; args: varargs[NimNode]): NimNode = for i in 1..<y.len: result.add y[i] macro scanf*(input: string; pattern: static[string]; results: varargs[typed]): bool = - ## See top level documentation of his module of how ``scanf`` works. + ## See top level documentation of this module about how ``scanf`` works. template matchBind(parser) {.dirty.} = var resLen = genSym(nskLet, "resLen") conds.add newLetStmt(resLen, newCall(bindSym(parser), inp, results[i], idx)) conds.add resLen.notZero conds.add resLen + template at(s: string; i: int): char = (if i < s.len: s[i] else: '\0') + template matchError() = + error("type mismatch between pattern '$" & pattern[p] & "' (position: " & $p & ") and " & $getType(results[i]) & + " var '" & repr(results[i]) & "'") + var i = 0 var p = 0 var idx = genSym(nskVar, "idx") @@ -336,37 +341,37 @@ macro scanf*(input: string; pattern: static[string]; results: varargs[typed]): b if i < results.len and getType(results[i]).typeKind == ntyString: matchBind "parseIdent" else: - error("no string var given for $w") + matchError inc i of 'b': if i < results.len and getType(results[i]).typeKind == ntyInt: matchBind "parseBin" else: - error("no int var given for $b") + matchError inc i of 'o': if i < results.len and getType(results[i]).typeKind == ntyInt: matchBind "parseOct" else: - error("no int var given for $o") + matchError inc i of 'i': if i < results.len and getType(results[i]).typeKind == ntyInt: matchBind "parseInt" else: - error("no int var given for $i") + matchError inc i of 'h': if i < results.len and getType(results[i]).typeKind == ntyInt: matchBind "parseHex" else: - error("no int var given for $h") + matchError inc i of 'f': if i < results.len and getType(results[i]).typeKind == ntyFloat: matchBind "parseFloat" else: - error("no float var given for $f") + matchError inc i of 's': conds.add newCall(bindSym"inc", idx, newCall(bindSym"skipWhitespace", inp, idx)) @@ -390,14 +395,14 @@ macro scanf*(input: string; pattern: static[string]; results: varargs[typed]): b conds.add newCall(bindSym"!=", resLen, newLit min) conds.add resLen else: - error("no string var given for $" & pattern[p]) + matchError inc i of '{': inc p var nesting = 0 let start = p while true: - case pattern[p] + case pattern.at(p) of '{': inc nesting of '}': if nesting == 0: break @@ -412,14 +417,14 @@ macro scanf*(input: string; pattern: static[string]; results: varargs[typed]): b conds.add newCall(bindSym"!=", resLen, newLit 0) conds.add resLen else: - error("no var given for $" & expr) + error("no var given for $" & expr & " (position: " & $p & ")") inc i of '[': inc p var nesting = 0 let start = p while true: - case pattern[p] + case pattern.at(p) of '[': inc nesting of ']': if nesting == 0: break @@ -451,10 +456,12 @@ macro scanf*(input: string; pattern: static[string]; results: varargs[typed]): b template atom*(input: string; idx: int; c: char): bool = ## Used in scanp for the matching of atoms (usually chars). - input[idx] == c + idx < input.len and input[idx] == c template atom*(input: string; idx: int; s: set[char]): bool = - input[idx] in s + idx < input.len and input[idx] in s + +template hasNxt*(input: string; idx: int): bool = idx < input.len #template prepare*(input: string): int = 0 template success*(x: int): bool = x != 0 @@ -462,7 +469,7 @@ template success*(x: int): bool = x != 0 template nxt*(input: string; idx, step: int = 1) = inc(idx, step) macro scanp*(input, idx: typed; pattern: varargs[untyped]): bool = - ## ``scanp`` is currently undocumented. + ## See top level documentation of this module about how ``scanp`` works. type StmtTriple = tuple[init, cond, action: NimNode] template interf(x): untyped = bindSym(x, brForceOpen) @@ -508,8 +515,8 @@ macro scanp*(input, idx: typed; pattern: varargs[untyped]): bool = !!newCall(interf"nxt", input, idx, resLen)) of nnkCallKinds: # *{'A'..'Z'} !! s.add(!_) - template buildWhile(init, cond, action): untyped = - while true: + template buildWhile(input, idx, init, cond, action): untyped = + while hasNxt(input, idx): init if not cond: break action @@ -528,11 +535,11 @@ macro scanp*(input, idx: typed; pattern: varargs[untyped]): bool = !!newCall(interf"nxt", input, idx, it[2])) elif it.kind == nnkPrefix and it[0].eqIdent"*": let (init, cond, action) = atm(it[1], input, idx, attached) - result = (getAst(buildWhile(init, cond, action)), + result = (getAst(buildWhile(input, idx, init, cond, action)), newEmptyNode(), newEmptyNode()) elif it.kind == nnkPrefix and it[0].eqIdent"+": # x+ is the same as xx* - result = atm(newTree(nnkPar, it[1], newTree(nnkPrefix, ident"*", it[1])), + result = atm(newTree(nnkTupleConstr, it[1], newTree(nnkPrefix, ident"*", it[1])), input, idx, attached) elif it.kind == nnkPrefix and it[0].eqIdent"?": # optional. @@ -583,18 +590,18 @@ macro scanp*(input, idx: typed; pattern: varargs[untyped]): bool = result = (newEmptyNode(), newCall(interf"atom", input, idx, it), !!newCall(interf"nxt", input, idx)) of nnkCurlyExpr: if it.len == 3 and it[1].kind == nnkIntLit and it[2].kind == nnkIntLit: - var h = newTree(nnkPar, it[0]) + var h = newTree(nnkTupleConstr, it[0]) for count in 2i64 .. it[1].intVal: h.add(it[0]) for count in it[1].intVal .. it[2].intVal-1: h.add(newTree(nnkPrefix, ident"?", it[0])) result = atm(h, input, idx, attached) elif it.len == 2 and it[1].kind == nnkIntLit: - var h = newTree(nnkPar, it[0]) + var h = newTree(nnkTupleConstr, it[0]) for count in 2i64 .. it[1].intVal: h.add(it[0]) result = atm(h, input, idx, attached) else: error("invalid pattern") - of nnkPar: - if it.len == 1: + of nnkPar, nnkTupleConstr: + if it.len == 1 and it.kind == nnkPar: result = atm(it[0], input, idx, attached) else: # concatenation: @@ -621,7 +628,7 @@ macro scanp*(input, idx: typed; pattern: varargs[untyped]): bool = when isMainModule: proc twoDigits(input: string; x: var int; start: int): int = - if input[start] == '0' and input[start+1] == '0': + if start+1 < input.len and input[start] == '0' and input[start+1] == '0': result = 2 x = 13 else: @@ -629,10 +636,10 @@ when isMainModule: proc someSep(input: string; start: int; seps: set[char] = {';',',','-','.'}): int = result = 0 - while input[start+result] in seps: inc result + while start+result < input.len and input[start+result] in seps: inc result proc demangle(s: string; res: var string; start: int): int = - while s[result+start] in {'_', '@'}: inc result + while result+start < s.len and s[result+start] in {'_', '@'}: inc result res = "" while result+start < s.len and s[result+start] > ' ' and s[result+start] != '_': res.add s[result+start] @@ -652,7 +659,7 @@ when isMainModule: var info = "" if scanp(resp, idx, *`whites`, '#', *`digits`, +`whites`, ?("0x", *`hexdigits`, " in "), demangle($input, prc, $index), *`whites`, '(', * ~ ')', ')', - *`whites`, "at ", +(~{'\C', '\L', '\0'} -> info.add($_)) ): + *`whites`, "at ", +(~{'\C', '\L'} -> info.add($_)) ): result.add prc & " " & info else: break @@ -713,7 +720,7 @@ when isMainModule: "NimMainInner c:/users/anwender/projects/nim/lib/system.nim:2605", "NimMain c:/users/anwender/projects/nim/lib/system.nim:2613", "main c:/users/anwender/projects/nim/lib/system.nim:2620"] - doAssert parseGDB(gdbOut) == result + #doAssert parseGDB(gdbOut) == result # bug #6487 var count = 0 diff --git a/lib/pure/strtabs.nim b/lib/pure/strtabs.nim index 75c5e171d..d8a23286a 100644 --- a/lib/pure/strtabs.nim +++ b/lib/pure/strtabs.nim @@ -17,9 +17,7 @@ import when defined(js): {.pragma: rtlFunc.} - {.pragma: deprecatedGetFunc.} else: - {.pragma: deprecatedGetFunc, deprecatedGet.} {.pragma: rtlFunc, rtl.} import os include "system/inclrtl" @@ -29,7 +27,7 @@ type modeCaseSensitive, ## the table is case sensitive modeCaseInsensitive, ## the table is case insensitive modeStyleInsensitive ## the table is style insensitive - KeyValuePair = tuple[key, val: string] + KeyValuePair = tuple[key, val: string, hasValue: bool] KeyValuePairSeq = seq[KeyValuePair] StringTableObj* = object of RootObj counter: int @@ -38,9 +36,6 @@ type StringTableRef* = ref StringTableObj ## use this type to declare string tables -{.deprecated: [TStringTableMode: StringTableMode, - TStringTable: StringTableObj, PStringTable: StringTableRef].} - proc len*(t: StringTableRef): int {.rtlFunc, extern: "nst$1".} = ## returns the number of keys in `t`. result = t.counter @@ -48,19 +43,19 @@ proc len*(t: StringTableRef): int {.rtlFunc, extern: "nst$1".} = iterator pairs*(t: StringTableRef): tuple[key, value: string] = ## iterates over every (key, value) pair in the table `t`. for h in 0..high(t.data): - if not isNil(t.data[h].key): + if t.data[h].hasValue: yield (t.data[h].key, t.data[h].val) iterator keys*(t: StringTableRef): string = ## iterates over every key in the table `t`. for h in 0..high(t.data): - if not isNil(t.data[h].key): + if t.data[h].hasValue: yield t.data[h].key iterator values*(t: StringTableRef): string = ## iterates over every value in the table `t`. for h in 0..high(t.data): - if not isNil(t.data[h].key): + if t.data[h].hasValue: yield t.data[h].val type @@ -73,10 +68,6 @@ type useKey ## do not replace ``$key`` if it is not found ## in the table (or in the environment) -{.deprecated: [TFormatFlag: FormatFlag].} - -# implementation - const growthFactor = 2 startSize = 64 @@ -102,7 +93,7 @@ proc nextTry(h, maxHash: Hash): Hash {.inline.} = proc rawGet(t: StringTableRef, key: string): int = var h: Hash = myhash(t, key) and high(t.data) # start with real hash value - while not isNil(t.data[h].key): + while t.data[h].hasValue: if myCmp(t, t.data[h].key, key): return h h = nextTry(h, high(t.data)) @@ -118,17 +109,12 @@ template get(t: StringTableRef, key: string) = raise newException(KeyError, "key not found") proc `[]`*(t: StringTableRef, key: string): var string {. - rtlFunc, extern: "nstTake", deprecatedGetFunc.} = + rtlFunc, extern: "nstTake".} = ## retrieves the location at ``t[key]``. If `key` is not in `t`, the ## ``KeyError`` exception is raised. One can check with ``hasKey`` whether ## the key exists. get(t, key) -proc mget*(t: StringTableRef, key: string): var string {.deprecated.} = - ## retrieves the location at ``t[key]``. If `key` is not in `t`, the - ## ``KeyError`` exception is raised. Use ```[]``` instead. - get(t, key) - proc getOrDefault*(t: StringTableRef; key: string, default: string = ""): string = var index = rawGet(t, key) if index >= 0: result = t.data[index].val @@ -144,16 +130,17 @@ proc contains*(t: StringTableRef, key: string): bool = proc rawInsert(t: StringTableRef, data: var KeyValuePairSeq, key, val: string) = var h: Hash = myhash(t, key) and high(data) - while not isNil(data[h].key): + while data[h].hasValue: h = nextTry(h, high(data)) data[h].key = key data[h].val = val + data[h].hasValue = true proc enlarge(t: StringTableRef) = var n: KeyValuePairSeq newSeq(n, len(t.data) * growthFactor) for i in countup(0, high(t.data)): - if not isNil(t.data[i].key): rawInsert(t, n, t.data[i].key, t.data[i].val) + if t.data[i].hasValue: rawInsert(t, n, t.data[i].key, t.data[i].val) swap(t.data, n) proc `[]=`*(t: StringTableRef, key, val: string) {.rtlFunc, extern: "nstPut".} = @@ -192,14 +179,14 @@ proc newStringTable*(mode: StringTableMode): StringTableRef {. result.counter = 0 newSeq(result.data, startSize) -proc clear*(s: StringTableRef, mode: StringTableMode) = +proc clear*(s: StringTableRef, mode: StringTableMode) {. + rtlFunc, extern: "nst$1".} = ## resets a string table to be empty again. s.mode = mode s.counter = 0 s.data.setLen(startSize) for i in 0..<s.data.len: - if not isNil(s.data[i].key): - s.data[i].key = nil + s.data[i].hasValue = false proc newStringTable*(keyValuePairs: varargs[string], mode: StringTableMode): StringTableRef {. diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index 2ad006001..396f14972 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -17,7 +17,11 @@ import parseutils from math import pow, round, floor, log10 from algorithm import reverse -{.deadCodeElim: on.} +when defined(nimVmExportFixed): + from unicode import toLower, toUpper + export toLower, toUpper + +{.deadCodeElim: on.} # dce option deprecated {.push debugger:off .} # the user does not want to trace a part # of the standard library! @@ -106,6 +110,12 @@ proc isUpperAscii*(c: char): bool {.noSideEffect, procvar, ## This checks ASCII characters only. return c in {'A'..'Z'} +template isImpl(call) = + if s.len == 0: return false + result = true + for c in s: + if not call(c): return false + proc isAlphaAscii*(s: string): bool {.noSideEffect, procvar, rtl, extern: "nsuIsAlphaAsciiStr".} = ## Checks whether or not `s` is alphabetical. @@ -114,12 +124,7 @@ proc isAlphaAscii*(s: string): bool {.noSideEffect, procvar, ## Returns true if all characters in `s` are ## alphabetic and there is at least one character ## in `s`. - if s.len() == 0: - return false - - result = true - for c in s: - if not c.isAlphaAscii(): return false + isImpl isAlphaAscii proc isAlphaNumeric*(s: string): bool {.noSideEffect, procvar, rtl, extern: "nsuIsAlphaNumericStr".} = @@ -129,13 +134,7 @@ proc isAlphaNumeric*(s: string): bool {.noSideEffect, procvar, ## Returns true if all characters in `s` are ## alpanumeric and there is at least one character ## in `s`. - if s.len() == 0: - return false - - result = true - for c in s: - if not c.isAlphaNumeric(): - return false + isImpl isAlphaNumeric proc isDigit*(s: string): bool {.noSideEffect, procvar, rtl, extern: "nsuIsDigitStr".} = @@ -145,13 +144,7 @@ proc isDigit*(s: string): bool {.noSideEffect, procvar, ## Returns true if all characters in `s` are ## numeric and there is at least one character ## in `s`. - if s.len() == 0: - return false - - result = true - for c in s: - if not c.isDigit(): - return false + isImpl isDigit proc isSpaceAscii*(s: string): bool {.noSideEffect, procvar, rtl, extern: "nsuIsSpaceAsciiStr".} = @@ -159,43 +152,54 @@ proc isSpaceAscii*(s: string): bool {.noSideEffect, procvar, ## ## Returns true if all characters in `s` are whitespace ## characters and there is at least one character in `s`. - if s.len() == 0: - return false + isImpl isSpaceAscii - result = true +template isCaseImpl(s, charProc, skipNonAlpha) = + var hasAtleastOneAlphaChar = false + if s.len == 0: return false for c in s: - if not c.isSpaceAscii(): - return false + if skipNonAlpha: + var charIsAlpha = c.isAlphaAscii() + if not hasAtleastOneAlphaChar: + hasAtleastOneAlphaChar = charIsAlpha + if charIsAlpha and (not charProc(c)): + return false + else: + if not charProc(c): + return false + return if skipNonAlpha: hasAtleastOneAlphaChar else: true -proc isLowerAscii*(s: string): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsLowerAsciiStr".} = - ## Checks whether or not `s` contains all lower case characters. +proc isLowerAscii*(s: string, skipNonAlpha: bool): bool = + ## Checks whether ``s`` is lower case. ## ## This checks ASCII characters only. - ## Returns true if all characters in `s` are lower case - ## and there is at least one character in `s`. - if s.len() == 0: - return false - - for c in s: - if not c.isLowerAscii(): - return false - true + ## + ## If ``skipNonAlpha`` is true, returns true if all alphabetical + ## characters in ``s`` are lower case. Returns false if none of the + ## characters in ``s`` are alphabetical. + ## + ## If ``skipNonAlpha`` is false, returns true only if all characters + ## in ``s`` are alphabetical and lower case. + ## + ## For either value of ``skipNonAlpha``, returns false if ``s`` is + ## an empty string. + isCaseImpl(s, isLowerAscii, skipNonAlpha) -proc isUpperAscii*(s: string): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsUpperAsciiStr".} = - ## Checks whether or not `s` contains all upper case characters. +proc isUpperAscii*(s: string, skipNonAlpha: bool): bool = + ## Checks whether ``s`` is upper case. ## ## This checks ASCII characters only. - ## Returns true if all characters in `s` are upper case - ## and there is at least one character in `s`. - if s.len() == 0: - return false - - for c in s: - if not c.isUpperAscii(): - return false - true + ## + ## If ``skipNonAlpha`` is true, returns true if all alphabetical + ## characters in ``s`` are upper case. Returns false if none of the + ## characters in ``s`` are alphabetical. + ## + ## If ``skipNonAlpha`` is false, returns true only if all characters + ## in ``s`` are alphabetical and upper case. + ## + ## For either value of ``skipNonAlpha``, returns false if ``s`` is + ## an empty string. + isCaseImpl(s, isUpperAscii, skipNonAlpha) proc toLowerAscii*(c: char): char {.noSideEffect, procvar, rtl, extern: "nsuToLowerAsciiChar".} = @@ -209,6 +213,11 @@ proc toLowerAscii*(c: char): char {.noSideEffect, procvar, else: result = c +template toImpl(call) = + result = newString(len(s)) + for i in 0..len(s) - 1: + result[i] = call(s[i]) + proc toLowerAscii*(s: string): string {.noSideEffect, procvar, rtl, extern: "nsuToLowerAsciiStr".} = ## Converts `s` into lower case. @@ -216,9 +225,7 @@ proc toLowerAscii*(s: string): string {.noSideEffect, procvar, ## This works only for the letters ``A-Z``. See `unicode.toLower ## <unicode.html#toLower>`_ for a version that works for any Unicode ## character. - result = newString(len(s)) - for i in 0..len(s) - 1: - result[i] = toLowerAscii(s[i]) + toImpl toLowerAscii proc toUpperAscii*(c: char): char {.noSideEffect, procvar, rtl, extern: "nsuToUpperAsciiChar".} = @@ -239,154 +246,22 @@ proc toUpperAscii*(s: string): string {.noSideEffect, procvar, ## This works only for the letters ``A-Z``. See `unicode.toUpper ## <unicode.html#toUpper>`_ for a version that works for any Unicode ## character. - result = newString(len(s)) - for i in 0..len(s) - 1: - result[i] = toUpperAscii(s[i]) + toImpl toUpperAscii proc capitalizeAscii*(s: string): string {.noSideEffect, procvar, rtl, extern: "nsuCapitalizeAscii".} = ## Converts the first character of `s` into upper case. ## ## This works only for the letters ``A-Z``. - result = toUpperAscii(s[0]) & substr(s, 1) - -proc isSpace*(c: char): bool {.noSideEffect, procvar, - rtl, deprecated, extern: "nsuIsSpaceChar".}= - ## Checks whether or not `c` is a whitespace character. - ## - ## **Deprecated since version 0.15.0**: use ``isSpaceAscii`` instead. - isSpaceAscii(c) - -proc isLower*(c: char): bool {.noSideEffect, procvar, - rtl, deprecated, extern: "nsuIsLowerChar".}= - ## Checks whether or not `c` is a lower case character. - ## - ## This checks ASCII characters only. - ## - ## **Deprecated since version 0.15.0**: use ``isLowerAscii`` instead. - isLowerAscii(c) - -proc isUpper*(c: char): bool {.noSideEffect, procvar, - rtl, deprecated, extern: "nsuIsUpperChar".}= - ## Checks whether or not `c` is an upper case character. - ## - ## This checks ASCII characters only. - ## - ## **Deprecated since version 0.15.0**: use ``isUpperAscii`` instead. - isUpperAscii(c) - -proc isAlpha*(c: char): bool {.noSideEffect, procvar, - rtl, deprecated, extern: "nsuIsAlphaChar".}= - ## Checks whether or not `c` is alphabetical. - ## - ## This checks a-z, A-Z ASCII characters only. - ## - ## **Deprecated since version 0.15.0**: use ``isAlphaAscii`` instead. - isAlphaAscii(c) - -proc isAlpha*(s: string): bool {.noSideEffect, procvar, - rtl, deprecated, extern: "nsuIsAlphaStr".}= - ## Checks whether or not `s` is alphabetical. - ## - ## This checks a-z, A-Z ASCII characters only. - ## Returns true if all characters in `s` are - ## alphabetic and there is at least one character - ## in `s`. - ## - ## **Deprecated since version 0.15.0**: use ``isAlphaAscii`` instead. - isAlphaAscii(s) - -proc isSpace*(s: string): bool {.noSideEffect, procvar, - rtl, deprecated, extern: "nsuIsSpaceStr".}= - ## Checks whether or not `s` is completely whitespace. - ## - ## Returns true if all characters in `s` are whitespace - ## characters and there is at least one character in `s`. - ## - ## **Deprecated since version 0.15.0**: use ``isSpaceAscii`` instead. - isSpaceAscii(s) - -proc isLower*(s: string): bool {.noSideEffect, procvar, - rtl, deprecated, extern: "nsuIsLowerStr".}= - ## Checks whether or not `s` contains all lower case characters. - ## - ## This checks ASCII characters only. - ## Returns true if all characters in `s` are lower case - ## and there is at least one character in `s`. - ## - ## **Deprecated since version 0.15.0**: use ``isLowerAscii`` instead. - isLowerAscii(s) - -proc isUpper*(s: string): bool {.noSideEffect, procvar, - rtl, deprecated, extern: "nsuIsUpperStr".}= - ## Checks whether or not `s` contains all upper case characters. - ## - ## This checks ASCII characters only. - ## Returns true if all characters in `s` are upper case - ## and there is at least one character in `s`. - ## - ## **Deprecated since version 0.15.0**: use ``isUpperAscii`` instead. - isUpperAscii(s) - -proc toLower*(c: char): char {.noSideEffect, procvar, - rtl, deprecated, extern: "nsuToLowerChar".} = - ## Converts `c` into lower case. - ## - ## This works only for the letters ``A-Z``. See `unicode.toLower - ## <unicode.html#toLower>`_ for a version that works for any Unicode - ## character. - ## - ## **Deprecated since version 0.15.0**: use ``toLowerAscii`` instead. - toLowerAscii(c) - -proc toLower*(s: string): string {.noSideEffect, procvar, - rtl, deprecated, extern: "nsuToLowerStr".} = - ## Converts `s` into lower case. - ## - ## This works only for the letters ``A-Z``. See `unicode.toLower - ## <unicode.html#toLower>`_ for a version that works for any Unicode - ## character. - ## - ## **Deprecated since version 0.15.0**: use ``toLowerAscii`` instead. - toLowerAscii(s) - -proc toUpper*(c: char): char {.noSideEffect, procvar, - rtl, deprecated, extern: "nsuToUpperChar".} = - ## Converts `c` into upper case. - ## - ## This works only for the letters ``A-Z``. See `unicode.toUpper - ## <unicode.html#toUpper>`_ for a version that works for any Unicode - ## character. - ## - ## **Deprecated since version 0.15.0**: use ``toUpperAscii`` instead. - toUpperAscii(c) - -proc toUpper*(s: string): string {.noSideEffect, procvar, - rtl, deprecated, extern: "nsuToUpperStr".} = - ## Converts `s` into upper case. - ## - ## This works only for the letters ``A-Z``. See `unicode.toUpper - ## <unicode.html#toUpper>`_ for a version that works for any Unicode - ## character. - ## - ## **Deprecated since version 0.15.0**: use ``toUpperAscii`` instead. - toUpperAscii(s) - -proc capitalize*(s: string): string {.noSideEffect, procvar, - rtl, deprecated, extern: "nsuCapitalize".} = - ## Converts the first character of `s` into upper case. - ## - ## This works only for the letters ``A-Z``. - ## - ## **Deprecated since version 0.15.0**: use ``capitalizeAscii`` instead. - capitalizeAscii(s) + if s.len == 0: result = "" + else: result = toUpperAscii(s[0]) & substr(s, 1) proc normalize*(s: string): string {.noSideEffect, procvar, rtl, extern: "nsuNormalize".} = ## Normalizes the string `s`. ## - ## That means to convert it to lower case and remove any '_'. This is needed - ## for Nim identifiers for example. + ## That means to convert it to lower case and remove any '_'. This + ## should NOT be used to normalize Nim identifier names. result = newString(s.len) var j = 0 for i in 0..len(s) - 1: @@ -418,8 +293,10 @@ proc cmpIgnoreCase*(a, b: string): int {.noSideEffect, proc cmpIgnoreStyle*(a, b: string): int {.noSideEffect, rtl, extern: "nsuCmpIgnoreStyle", procvar.} = - ## Compares two strings normalized (i.e. case and - ## underscores do not matter). Returns: + ## Semantically the same as ``cmp(normalize(a), normalize(b))``. It + ## is just optimized to not allocate temporary strings. This should + ## NOT be used to compare Nim identifier names. use `macros.eqIdent` + ## for that. Returns: ## ## | 0 iff a == b ## | < 0 iff a < b @@ -427,28 +304,37 @@ proc cmpIgnoreStyle*(a, b: string): int {.noSideEffect, var i = 0 var j = 0 while true: - while a[i] == '_': inc(i) - while b[j] == '_': inc(j) # BUGFIX: typo - var aa = toLowerAscii(a[i]) - var bb = toLowerAscii(b[j]) + while i < a.len and a[i] == '_': inc i + while j < b.len and b[j] == '_': inc j + var aa = if i < a.len: toLowerAscii(a[i]) else: '\0' + var bb = if j < b.len: toLowerAscii(b[j]) else: '\0' result = ord(aa) - ord(bb) - if result != 0 or aa == '\0': break - inc(i) - inc(j) - + if result != 0: return result + # the characters are identical: + if i >= a.len: + # both cursors at the end: + if j >= b.len: return 0 + # not yet at the end of 'b': + return -1 + elif j >= b.len: + return 1 + inc i + inc j proc strip*(s: string, leading = true, trailing = true, chars: set[char] = Whitespace): string {.noSideEffect, rtl, extern: "nsuStrip".} = - ## Strips `chars` from `s` and returns the resulting string. + ## Strips leading or trailing `chars` from `s` and returns + ## the resulting string. ## ## If `leading` is true, leading `chars` are stripped. ## If `trailing` is true, trailing `chars` are stripped. + ## If both are false, the string is returned unchanged. var first = 0 last = len(s)-1 if leading: - while s[first] in chars: inc(first) + while first <= last and s[first] in chars: inc(first) if trailing: while last >= 0 and s[last] in chars: dec(last) result = substr(s, first, last) @@ -464,15 +350,14 @@ proc toOctal*(c: char): string {.noSideEffect, rtl, extern: "nsuToOctal".} = result[i] = chr(val mod 8 + ord('0')) val = val div 8 -proc isNilOrEmpty*(s: string): bool {.noSideEffect, procvar, rtl, extern: "nsuIsNilOrEmpty".} = +proc isNilOrEmpty*(s: string): bool {.noSideEffect, procvar, rtl, + extern: "nsuIsNilOrEmpty", + deprecated: "use 'x.len == 0' instead".} = ## Checks if `s` is nil or empty. result = len(s) == 0 proc isNilOrWhitespace*(s: string): bool {.noSideEffect, procvar, rtl, extern: "nsuIsNilOrWhitespace".} = ## Checks if `s` is nil or consists entirely of whitespace characters. - if len(s) == 0: - return true - result = true for c in s: if not c.isSpaceAscii(): @@ -483,7 +368,6 @@ proc substrEq(s: string, pos: int, substr: string): bool = var length = substr.len while i < length and s[pos+i] == substr[i]: inc i - return i == length # --------- Private templates for different split separators ----------- @@ -517,7 +401,7 @@ template oldSplit(s, seps, maxsplit) = var splits = maxsplit assert(not ('\0' in seps)) while last < len(s): - while s[last] in seps: inc(last) + while last < len(s) and s[last] in seps: inc(last) var first = last while last < len(s) and s[last] notin seps: inc(last) if first <= last-1: @@ -568,10 +452,7 @@ iterator split*(s: string, seps: set[char] = Whitespace, ## "08" ## "08.398990" ## - when defined(nimOldSplit): - oldSplit(s, seps, maxsplit) - else: - splitCommon(s, seps, maxsplit, 1) + splitCommon(s, seps, maxsplit, 1) iterator splitWhitespace*(s: string, maxsplit: int = -1): string = ## Splits the string ``s`` at whitespace stripping leading and trailing @@ -657,7 +538,6 @@ iterator split*(s: string, sep: string, maxsplit: int = -1): string = ## "is" ## "corrupted" ## - splitCommon(s, sep, maxsplit, sep.len) template rsplitCommon(s, sep, maxsplit, sepLen) = @@ -667,29 +547,21 @@ template rsplitCommon(s, sep, maxsplit, sepLen) = first = last splits = maxsplit startPos = 0 - # go to -1 in order to get separators at the beginning while first >= -1: while first >= 0 and not stringHasSep(s, first, sep): dec(first) - if splits == 0: # No more splits means set first to the beginning first = -1 - if first == -1: startPos = 0 else: startPos = first + sepLen - yield substr(s, startPos, last) - - if splits == 0: - break - + if splits == 0: break dec(splits) dec(first) - last = first iterator rsplit*(s: string, seps: set[char] = Whitespace, @@ -709,7 +581,6 @@ iterator rsplit*(s: string, seps: set[char] = Whitespace, ## "foo" ## ## Substrings are separated from the right by the set of chars `seps` - rsplitCommon(s, seps, maxsplit, 1) iterator rsplit*(s: string, sep: char, @@ -750,12 +621,13 @@ iterator rsplit*(s: string, sep: string, maxsplit: int = -1, ## Substrings are separated from the right by the string `sep` rsplitCommon(s, sep, maxsplit, sep.len) -iterator splitLines*(s: string): string = +iterator splitLines*(s: string, keepEol = false): string = ## Splits the string `s` into its containing lines. ## ## Every `character literal <manual.html#character-literals>`_ newline ## combination (CR, LF, CR-LF) is supported. The result strings contain no - ## trailing ``\n``. + ## trailing end of line characters unless parameter ``keepEol`` is set to + ## ``true``. ## ## Example: ## @@ -775,22 +647,30 @@ iterator splitLines*(s: string): string = ## "" var first = 0 var last = 0 + var eolpos = 0 while true: - while s[last] notin {'\0', '\c', '\l'}: inc(last) - yield substr(s, first, last-1) - # skip newlines: - if s[last] == '\l': inc(last) - elif s[last] == '\c': - inc(last) + while last < s.len and s[last] notin {'\c', '\l'}: inc(last) + + eolpos = last + if last < s.len: if s[last] == '\l': inc(last) - else: break # was '\0' + elif s[last] == '\c': + inc(last) + if last < s.len and s[last] == '\l': inc(last) + + yield substr(s, first, if keepEol: last-1 else: eolpos-1) + + # no eol characters consumed means that the string is over + if eolpos == last: + break + first = last -proc splitLines*(s: string): seq[string] {.noSideEffect, +proc splitLines*(s: string, keepEol = false): seq[string] {.noSideEffect, rtl, extern: "nsuSplitLines".} = ## The same as the `splitLines <#splitLines.i,string>`_ iterator, but is a ## proc that returns a sequence of substrings. - accumulateResult(splitLines(s)) + accumulateResult(splitLines(s, keepEol=keepEol)) proc countLines*(s: string): int {.noSideEffect, rtl, extern: "nsuCountLines".} = @@ -808,7 +688,7 @@ proc countLines*(s: string): int {.noSideEffect, while i < s.len: case s[i] of '\c': - if s[i+1] == '\l': inc i + if i+1 < s.len and s[i+1] == '\l': inc i inc result of '\l': inc result else: discard @@ -940,7 +820,7 @@ proc toHex*(x: BiggestInt, len: Positive): string {.noSideEffect, # handle negative overflow if n == 0 and x < 0: n = -1 -proc toHex*[T](x: T): string = +proc toHex*[T: SomeInteger](x: T): string = ## Shortcut for ``toHex(x, T.sizeOf * 2)`` toHex(BiggestInt(x), T.sizeOf * 2) @@ -974,7 +854,7 @@ proc parseInt*(s: string): int {.noSideEffect, procvar, ## Parses a decimal integer value contained in `s`. ## ## If `s` is not a valid integer, `ValueError` is raised. - var L = parseutils.parseInt(s, result, 0) + let L = parseutils.parseInt(s, result, 0) if L != s.len or L == 0: raise newException(ValueError, "invalid integer: " & s) @@ -983,7 +863,7 @@ proc parseBiggestInt*(s: string): BiggestInt {.noSideEffect, procvar, ## Parses a decimal integer value contained in `s`. ## ## If `s` is not a valid integer, `ValueError` is raised. - var L = parseutils.parseBiggestInt(s, result, 0) + let L = parseutils.parseBiggestInt(s, result, 0) if L != s.len or L == 0: raise newException(ValueError, "invalid integer: " & s) @@ -992,7 +872,7 @@ proc parseUInt*(s: string): uint {.noSideEffect, procvar, ## Parses a decimal unsigned integer value contained in `s`. ## ## If `s` is not a valid integer, `ValueError` is raised. - var L = parseutils.parseUInt(s, result, 0) + let L = parseutils.parseUInt(s, result, 0) if L != s.len or L == 0: raise newException(ValueError, "invalid unsigned integer: " & s) @@ -1001,7 +881,7 @@ proc parseBiggestUInt*(s: string): BiggestUInt {.noSideEffect, procvar, ## Parses a decimal unsigned integer value contained in `s`. ## ## If `s` is not a valid integer, `ValueError` is raised. - var L = parseutils.parseBiggestUInt(s, result, 0) + let L = parseutils.parseBiggestUInt(s, result, 0) if L != s.len or L == 0: raise newException(ValueError, "invalid unsigned integer: " & s) @@ -1010,34 +890,42 @@ proc parseFloat*(s: string): float {.noSideEffect, procvar, ## Parses a decimal floating point value contained in `s`. If `s` is not ## a valid floating point number, `ValueError` is raised. ``NAN``, ## ``INF``, ``-INF`` are also supported (case insensitive comparison). - var L = parseutils.parseFloat(s, result, 0) + let L = parseutils.parseFloat(s, result, 0) if L != s.len or L == 0: raise newException(ValueError, "invalid float: " & s) +proc parseBinInt*(s: string): int {.noSideEffect, procvar, + rtl, extern: "nsuParseBinInt".} = + ## Parses a binary integer value contained in `s`. + ## + ## If `s` is not a valid binary integer, `ValueError` is raised. `s` can have + ## one of the following optional prefixes: ``0b``, ``0B``. Underscores within + ## `s` are ignored. + let L = parseutils.parseBin(s, result, 0) + if L != s.len or L == 0: + raise newException(ValueError, "invalid binary integer: " & s) + +proc parseOctInt*(s: string): int {.noSideEffect, + rtl, extern: "nsuParseOctInt".} = + ## Parses an octal integer value contained in `s`. + ## + ## If `s` is not a valid oct integer, `ValueError` is raised. `s` can have one + ## of the following optional prefixes: ``0o``, ``0O``. Underscores within + ## `s` are ignored. + let L = parseutils.parseOct(s, result, 0) + if L != s.len or L == 0: + raise newException(ValueError, "invalid oct integer: " & s) + proc parseHexInt*(s: string): int {.noSideEffect, procvar, rtl, extern: "nsuParseHexInt".} = ## Parses a hexadecimal integer value contained in `s`. ## - ## If `s` is not a valid integer, `ValueError` is raised. `s` can have one + ## If `s` is not a valid hex integer, `ValueError` is raised. `s` can have one ## of the following optional prefixes: ``0x``, ``0X``, ``#``. Underscores ## within `s` are ignored. - var i = 0 - if s[i] == '0' and (s[i+1] == 'x' or s[i+1] == 'X'): inc(i, 2) - elif s[i] == '#': inc(i) - while true: - case s[i] - of '_': inc(i) - of '0'..'9': - result = result shl 4 or (ord(s[i]) - ord('0')) - inc(i) - of 'a'..'f': - result = result shl 4 or (ord(s[i]) - ord('a') + 10) - inc(i) - of 'A'..'F': - result = result shl 4 or (ord(s[i]) - ord('A') + 10) - inc(i) - of '\0': break - else: raise newException(ValueError, "invalid integer: " & s) + let L = parseutils.parseHex(s, result, 0) + if L != s.len or L == 0: + raise newException(ValueError, "invalid hex integer: " & s) proc generateHexCharToValueMap(): string = ## Generate a string to map a hex digit to uint value @@ -1145,14 +1033,6 @@ template spaces*(n: Natural): string = repeat(' ', n) ## echo text1 & spaces(max(0, width - text1.len)) & "|" ## echo text2 & spaces(max(0, width - text2.len)) & "|" -proc repeatChar*(count: Natural, c: char = ' '): string {.deprecated.} = - ## deprecated: use repeat() or spaces() - repeat(c, count) - -proc repeatStr*(count: Natural, s: string): string {.deprecated.} = - ## deprecated: use repeat(string, count) or string.repeat(count) - repeat(s, count) - proc align*(s: string, count: Natural, padding = ' '): string {. noSideEffect, rtl, extern: "nsuAlignString".} = ## Aligns a string `s` with `padding`, so that it is of length `count`. @@ -1223,7 +1103,7 @@ iterator tokenize*(s: string, seps: set[char] = Whitespace): tuple[ var i = 0 while true: var j = i - var isSep = s[j] in seps + var isSep = j < s.len and s[j] in seps while j < s.len and (s[j] in seps) == isSep: inc(j) if j > i: yield (substr(s, i, j-1), isSep) @@ -1248,7 +1128,7 @@ proc wordWrap*(s: string, maxLineWidth = 80, if len(word) > spaceLeft: if splitLongWords and len(word) > maxLineWidth: result.add(substr(word, 0, spaceLeft-1)) - var w = spaceLeft+1 + var w = spaceLeft var wordLeft = len(word) - spaceLeft while wordLeft > 0: result.add(newLine) @@ -1294,7 +1174,7 @@ proc unindent*(s: string, count: Natural, padding: string = " "): string var indentCount = 0 for j in 0..<count.int: indentCount.inc - if line[j .. j + padding.len-1] != padding: + if j + padding.len-1 >= line.len or line[j .. j + padding.len-1] != padding: indentCount = j break result.add(line[indentCount*padding.len .. ^1]) @@ -1322,13 +1202,13 @@ proc startsWith*(s, prefix: string): bool {.noSideEffect, ## If ``prefix == ""`` true is returned. var i = 0 while true: - if prefix[i] == '\0': return true - if s[i] != prefix[i]: return false + if i >= prefix.len: return true + if i >= s.len or s[i] != prefix[i]: return false inc(i) proc startsWith*(s: string, prefix: char): bool {.noSideEffect, inline.} = ## Returns true iff ``s`` starts with ``prefix``. - result = s[0] == prefix + result = s.len > 0 and s[0] == prefix proc endsWith*(s, suffix: string): bool {.noSideEffect, rtl, extern: "nsuEndsWith".} = @@ -1340,11 +1220,11 @@ proc endsWith*(s, suffix: string): bool {.noSideEffect, while i+j <% s.len: if s[i+j] != suffix[i]: return false inc(i) - if suffix[i] == '\0': return true + if i >= suffix.len: return true proc endsWith*(s: string, suffix: char): bool {.noSideEffect, inline.} = ## Returns true iff ``s`` ends with ``suffix``. - result = s[s.high] == suffix + result = s.len > 0 and s[s.high] == suffix proc continuesWith*(s, substr: string, start: Natural): bool {.noSideEffect, rtl, extern: "nsuContinuesWith".} = @@ -1353,8 +1233,8 @@ proc continuesWith*(s, substr: string, start: Natural): bool {.noSideEffect, ## If ``substr == ""`` true is returned. var i = 0 while true: - if substr[i] == '\0': return true - if s[i+start] != substr[i]: return false + if i >= substr.len: return true + if i+start >= s.len or s[i+start] != substr[i]: return false inc(i) proc addSep*(dest: var string, sep = ", ", startLen: Natural = 0) @@ -1430,21 +1310,20 @@ proc initSkipTable*(a: var SkipTable, sub: string) {.noSideEffect, rtl, extern: "nsuInitSkipTable".} = ## Preprocess table `a` for `sub`. let m = len(sub) - let m1 = m + 1 var i = 0 while i <= 0xff-7: - a[chr(i + 0)] = m1 - a[chr(i + 1)] = m1 - a[chr(i + 2)] = m1 - a[chr(i + 3)] = m1 - a[chr(i + 4)] = m1 - a[chr(i + 5)] = m1 - a[chr(i + 6)] = m1 - a[chr(i + 7)] = m1 + a[chr(i + 0)] = m + a[chr(i + 1)] = m + a[chr(i + 2)] = m + a[chr(i + 3)] = m + a[chr(i + 4)] = m + a[chr(i + 5)] = m + a[chr(i + 6)] = m + a[chr(i + 7)] = m i += 8 - for i in 0..m-1: - a[sub[i]] = m-i + for i in 0 ..< m - 1: + a[sub[i]] = m - 1 - i proc find*(a: SkipTable, s, sub: string, start: Natural = 0, last: Natural = 0): int {.noSideEffect, rtl, extern: "nsuFindStrA".} = @@ -1452,18 +1331,29 @@ proc find*(a: SkipTable, s, sub: string, start: Natural = 0, last: Natural = 0): ## If `last` is unspecified, it defaults to `s.high`. ## ## Searching is case-sensitive. If `sub` is not in `s`, -1 is returned. + let last = if last==0: s.high else: last - m = len(sub) - n = last + 1 - # search: - var j = start - while j <= n - m: - block match: - for k in 0..m-1: - if sub[k] != s[k+j]: break match - return j - inc(j, a[s[j+m]]) + sLen = last - start + 1 + subLast = sub.len - 1 + + if subLast == -1: + # this was an empty needle string, + # we count this as match in the first possible position: + return start + + # This is an implementation of the Boyer-Moore Horspool algorithms + # https://en.wikipedia.org/wiki/Boyer%E2%80%93Moore%E2%80%93Horspool_algorithm + var skip = start + + while last - skip >= subLast: + var i = subLast + while s[skip + i] == sub[i]: + if i == 0: + return skip + dec i + inc skip, a[s[skip + subLast]] + return -1 when not (defined(js) or defined(nimdoc) or defined(nimscript)): @@ -1485,9 +1375,11 @@ proc find*(s: string, sub: char, start: Natural = 0, last: Natural = 0): int {.n if sub == s[i]: return i else: when hasCStringBuiltin: - let found = c_memchr(s[start].unsafeAddr, sub, last-start+1) - if not found.isNil: - return cast[ByteAddress](found) -% cast[ByteAddress](s.cstring) + let L = last-start+1 + if L > 0: + let found = c_memchr(s[start].unsafeAddr, sub, L) + if not found.isNil: + return cast[ByteAddress](found) -% cast[ByteAddress](s.cstring) else: for i in start..last: if sub == s[i]: return i @@ -1499,12 +1391,8 @@ proc find*(s, sub: string, start: Natural = 0, last: Natural = 0): int {.noSideE ## If `last` is unspecified, it defaults to `s.high`. ## ## Searching is case-sensitive. If `sub` is not in `s`, -1 is returned. - if sub.len > s.len: - return -1 - - if sub.len == 1: - return find(s, sub[0], start, last) - + if sub.len > s.len: return -1 + if sub.len == 1: return find(s, sub[0], start, last) var a {.noinit.}: SkipTable initSkipTable(a, sub) result = find(a, s, sub, start, last) @@ -1561,18 +1449,14 @@ proc center*(s: string, width: int, fillChar: char = ' '): string {. ## ## The original string is returned if `width` is less than or equal ## to `s.len`. - if width <= s.len: - return s - + if width <= s.len: return s result = newString(width) - # Left padding will be one fillChar # smaller if there are an odd number # of characters let charsLeft = (width - s.len) leftPadding = charsLeft div 2 - for i in 0 ..< width: if i >= leftPadding and i < leftPadding + s.len: # we are where the string should be located @@ -1590,27 +1474,22 @@ proc count*(s: string, sub: string, overlapping: bool = false): int {. var i = 0 while true: i = s.find(sub, i) - if i < 0: - break - if overlapping: - inc i - else: - i += sub.len + if i < 0: break + if overlapping: inc i + else: i += sub.len inc result proc count*(s: string, sub: char): int {.noSideEffect, rtl, extern: "nsuCountChar".} = ## Count the occurrences of the character `sub` in the string `s`. for c in s: - if c == sub: - inc result + if c == sub: inc result proc count*(s: string, subs: set[char]): int {.noSideEffect, rtl, extern: "nsuCountCharSet".} = ## Count the occurrences of the group of character `subs` in the string `s`. for c in s: - if c in subs: - inc result + if c in subs: inc result proc quoteIfContainsWhite*(s: string): string {.deprecated.} = ## Returns ``'"' & s & '"'`` if `s` contains a space and does not @@ -1618,10 +1497,8 @@ proc quoteIfContainsWhite*(s: string): string {.deprecated.} = ## ## **DEPRECATED** as it was confused for shell quoting function. For this ## application use `osproc.quoteShell <osproc.html#quoteShell>`_. - if find(s, {' ', '\t'}) >= 0 and s[0] != '"': - result = '"' & s & '"' - else: - result = s + if find(s, {' ', '\t'}) >= 0 and s[0] != '"': result = '"' & s & '"' + else: result = s proc contains*(s: string, c: char): bool {.noSideEffect.} = ## Same as ``find(s, c) >= 0``. @@ -1638,19 +1515,41 @@ proc contains*(s: string, chars: set[char]): bool {.noSideEffect.} = proc replace*(s, sub: string, by = ""): string {.noSideEffect, rtl, extern: "nsuReplaceStr".} = ## Replaces `sub` in `s` by the string `by`. - var a {.noinit.}: SkipTable result = "" - initSkipTable(a, sub) - let last = s.high - var i = 0 - while true: - var j = find(a, s, sub, i, last) - if j < 0: break - add result, substr(s, i, j - 1) + let subLen = sub.len + if subLen == 0: + for c in s: + add result, by + add result, c add result, by - i = j + len(sub) - # copy the rest: - add result, substr(s, i) + return + elif subLen == 1: + # when the pattern is a single char, we use a faster + # char-based search that doesn't need a skip table: + let c = sub[0] + let last = s.high + var i = 0 + while true: + let j = find(s, c, i, last) + if j < 0: break + add result, substr(s, i, j - 1) + add result, by + i = j + subLen + # copy the rest: + add result, substr(s, i) + else: + var a {.noinit.}: SkipTable + initSkipTable(a, sub) + let last = s.high + var i = 0 + while true: + let j = find(a, s, sub, i, last) + if j < 0: break + add result, substr(s, i, j - 1) + add result, by + i = j + subLen + # copy the rest: + add result, substr(s, i) proc replace*(s: string, sub, by: char): string {.noSideEffect, rtl, extern: "nsuReplaceChar".} = @@ -1671,12 +1570,14 @@ proc replaceWord*(s, sub: string, by = ""): string {.noSideEffect, ## Each occurrence of `sub` has to be surrounded by word boundaries ## (comparable to ``\\w`` in regular expressions), otherwise it is not ## replaced. + if sub.len == 0: return s const wordChars = {'a'..'z', 'A'..'Z', '0'..'9', '_', '\128'..'\255'} var a {.noinit.}: SkipTable result = "" initSkipTable(a, sub) var i = 0 let last = s.high + let sublen = max(sub.len, 1) while true: var j = find(a, s, sub, i, last) if j < 0: break @@ -1685,7 +1586,7 @@ proc replaceWord*(s, sub: string, by = ""): string {.noSideEffect, (j+sub.len >= s.len or s[j+sub.len] notin wordChars): add result, substr(s, i, j - 1) add result, by - i = j + len(sub) + i = j + sublen else: add result, substr(s, i, j) i = j + 1 @@ -1696,9 +1597,8 @@ proc multiReplace*(s: string, replacements: varargs[(string, string)]): string { ## Same as replace, but specialized for doing multiple replacements in a single ## pass through the input string. ## - ## Calling replace multiple times after each other is inefficient and result in too many allocations - ## follwed by immediate deallocations as portions of the string gets replaced. - ## multiReplace performs all replacements in a single pass. + ## multiReplace performs all replacements in a single pass, this means it can be used + ## to swap the occurences of "a" and "b", for instance. ## ## If the resulting string is not longer than the original input string, only a single ## memory allocation is required. @@ -1737,24 +1637,6 @@ proc delete*(s: var string, first, last: int) {.noSideEffect, inc(j) setLen(s, newLen) -proc parseOctInt*(s: string): int {.noSideEffect, - rtl, extern: "nsuParseOctInt".} = - ## Parses an octal integer value contained in `s`. - ## - ## If `s` is not a valid integer, `ValueError` is raised. `s` can have one - ## of the following optional prefixes: ``0o``, ``0O``. Underscores within - ## `s` are ignored. - var i = 0 - if s[i] == '0' and (s[i+1] == 'o' or s[i+1] == 'O'): inc(i, 2) - while true: - case s[i] - of '_': inc(i) - of '0'..'7': - result = result shl 3 or (ord(s[i]) - ord('0')) - inc(i) - of '\0': break - else: raise newException(ValueError, "invalid integer: " & s) - proc toOct*(x: BiggestInt, len: Positive): string {.noSideEffect, rtl, extern: "nsuToOct".} = ## Converts `x` into its octal representation. @@ -1819,7 +1701,14 @@ proc escape*(s: string, prefix = "\"", suffix = "\""): string {.noSideEffect, result = newStringOfCap(s.len + s.len shr 2) result.add(prefix) for c in items(s): - result.addEscapedChar(c) + case c + of '\0'..'\31', '\127'..'\255': + add(result, "\\x") + add(result, toHex(ord(c), 2)) + of '\\': add(result, "\\\\") + of '\'': add(result, "\\'") + of '\"': add(result, "\\\"") + else: add(result, c) add(result, suffix) proc unescape*(s: string, prefix = "\"", suffix = "\""): string {.noSideEffect, @@ -1835,11 +1724,13 @@ proc unescape*(s: string, prefix = "\"", suffix = "\""): string {.noSideEffect, var i = prefix.len if not s.startsWith(prefix): raise newException(ValueError, - "String does not start with a prefix of: " & prefix) + "String does not start with: " & prefix) while true: - if i == s.len-suffix.len: break - case s[i] - of '\\': + if i >= s.len-suffix.len: break + if s[i] == '\\': + if i+1 >= s.len: + result.add('\\') + break case s[i+1]: of 'x': inc i, 2 @@ -1853,15 +1744,15 @@ proc unescape*(s: string, prefix = "\"", suffix = "\""): string {.noSideEffect, result.add('\'') of '\"': result.add('\"') - else: result.add("\\" & s[i+1]) - inc(i) - of '\0': break + else: + result.add("\\" & s[i+1]) + inc(i, 2) else: result.add(s[i]) - inc(i) + inc(i) if not s.endsWith(suffix): raise newException(ValueError, - "String does not end with a suffix of: " & suffix) + "String does not end in: " & suffix) proc validIdentifier*(s: string): bool {.noSideEffect, rtl, extern: "nsuValidIdentifier".} = @@ -1871,7 +1762,7 @@ proc validIdentifier*(s: string): bool {.noSideEffect, ## and is followed by any number of characters of the set `IdentChars`. runnableExamples: doAssert "abc_def08".validIdentifier - if s[0] in IdentStartChars: + if s.len > 0 and s[0] in IdentStartChars: for i in 1..s.len-1: if s[i] notin IdentChars: return false return true @@ -1890,7 +1781,7 @@ proc editDistance*(a, b: string): int {.noSideEffect, # strip common prefix: var s = 0 - while a[s] == b[s] and a[s] != '\0': + while s < len1 and a[s] == b[s]: inc(s) dec(len1) dec(len2) @@ -1963,8 +1854,6 @@ proc editDistance*(a, b: string): int {.noSideEffect, if x > c3: x = c3 row[p] = x result = row[e] - #dealloc(row) - # floating point formating: when not defined(js): @@ -1994,6 +1883,10 @@ proc formatBiggestFloat*(f: BiggestFloat, format: FloatFormatMode = ffDefault, ## ## If ``precision == -1``, it tries to format it nicely. when defined(js): + var precision = precision + if precision == -1: + # use the same default precision as c_sprintf + precision = 6 var res: cstring case format of ffDefault: @@ -2003,6 +1896,9 @@ proc formatBiggestFloat*(f: BiggestFloat, format: FloatFormatMode = ffDefault, of ffScientific: {.emit: "`res` = `f`.toExponential(`precision`);".} result = $res + if 1.0 / f == -Inf: + # JavaScript removes the "-" from negative Zero, add it back here + result = "-" & $res for i in 0 ..< result.len: # Depending on the locale either dot or comma is produced, # but nothing else is possible: @@ -2073,7 +1969,7 @@ proc trimZeros*(x: var string) {.noSideEffect.} = var spl: seq[string] if x.contains('.') or x.contains(','): if x.contains('e'): - spl= x.split('e') + spl = x.split('e') x = spl[0] while x[x.high] == '0': x.setLen(x.len-1) @@ -2142,12 +2038,13 @@ proc formatEng*(f: BiggestFloat, precision: range[0..32] = 10, trim: bool = true, siPrefix: bool = false, - unit: string = nil, - decimalSep = '.'): string {.noSideEffect.} = + unit: string = "", + decimalSep = '.', + useUnitSpace = false): string {.noSideEffect.} = ## Converts a floating point value `f` to a string using engineering notation. ## ## Numbers in of the range -1000.0<f<1000.0 will be formatted without an - ## exponent. Numbers outside of this range will be formatted as a + ## exponent. Numbers outside of this range will be formatted as a ## significand in the range -1000.0<f<1000.0 and an exponent that will always ## be an integer multiple of 3, corresponding with the SI prefix scale k, M, ## G, T etc for numbers with an absolute value greater than 1 and m, μ, n, p @@ -2155,7 +2052,7 @@ proc formatEng*(f: BiggestFloat, ## ## The default configuration (`trim=true` and `precision=10`) shows the ## **shortest** form that precisely (up to a maximum of 10 decimal places) - ## displays the value. For example, 4.100000 will be displayed as 4.1 (which + ## displays the value. For example, 4.100000 will be displayed as 4.1 (which ## is mathematically identical) whereas 4.1000003 will be displayed as ## 4.1000003. ## @@ -2175,15 +2072,15 @@ proc formatEng*(f: BiggestFloat, ## formatEng(-52731234, 2) == "-52.73e6" ## ## If `siPrefix` is set to true, the number will be displayed with the SI - ## prefix corresponding to the exponent. For example 4100 will be displayed - ## as "4.1 k" instead of "4.1e3". Note that `u` is used for micro- in place - ## of the greek letter mu (μ) as per ISO 2955. Numbers with an absolute + ## prefix corresponding to the exponent. For example 4100 will be displayed + ## as "4.1 k" instead of "4.1e3". Note that `u` is used for micro- in place + ## of the greek letter mu (μ) as per ISO 2955. Numbers with an absolute ## value outside of the range 1e-18<f<1000e18 (1a<f<1000E) will be displayed ## with an exponent rather than an SI prefix, regardless of whether ## `siPrefix` is true. ## - ## If `unit` is not nil, the provided unit will be appended to the string - ## (with a space as required by the SI standard). This behaviour is slightly + ## If `useUnitSpace` is true, the provided unit will be appended to the string + ## (with a space as required by the SI standard). This behaviour is slightly ## different to appending the unit to the result as the location of the space ## is altered depending on whether there is an exponent. ## @@ -2197,7 +2094,7 @@ proc formatEng*(f: BiggestFloat, ## formatEng(4100, siPrefix=true, unit="") == "4.1 k" ## formatEng(4100) == "4.1e3" ## formatEng(4100, unit="V") == "4.1e3 V" - ## formatEng(4100, unit="") == "4.1e3 " # Space with unit="" + ## formatEng(4100, unit="", useUnitSpace=true) == "4.1e3 " # Space with useUnitSpace=true ## ## `decimalSep` is used as the decimal separator. var @@ -2265,10 +2162,9 @@ proc formatEng*(f: BiggestFloat, if p != ' ': suffix = " " & p exponent = 0 # Exponent replaced by SI prefix - if suffix == "" and unit != nil: + if suffix == "" and useUnitSpace: suffix = " " - if unit != nil: - suffix &= unit + suffix &= unit if exponent != 0: result &= "e" & $exponent result &= suffix @@ -2291,11 +2187,10 @@ proc addf*(s: var string, formatstr: string, a: varargs[string, `$`]) {. var i = 0 var num = 0 while i < len(formatstr): - if formatstr[i] == '$': - case formatstr[i+1] # again we use the fact that strings - # are zero-terminated here + if formatstr[i] == '$' and i+1 < len(formatstr): + case formatstr[i+1] of '#': - if num >% a.high: invalidFormatString() + if num > a.high: invalidFormatString() add s, a[num] inc i, 2 inc num @@ -2307,11 +2202,11 @@ proc addf*(s: var string, formatstr: string, a: varargs[string, `$`]) {. inc(i) # skip $ var negative = formatstr[i] == '-' if negative: inc i - while formatstr[i] in Digits: + while i < formatstr.len and formatstr[i] in Digits: j = j * 10 + ord(formatstr[i]) - ord('0') inc(i) let idx = if not negative: j-1 else: a.len-j - if idx >% a.high: invalidFormatString() + if idx < 0 or idx > a.high: invalidFormatString() add s, a[idx] of '{': var j = i+2 @@ -2319,7 +2214,7 @@ proc addf*(s: var string, formatstr: string, a: varargs[string, `$`]) {. var negative = formatstr[j] == '-' if negative: inc j var isNumber = 0 - while formatstr[j] notin {'\0', '}'}: + while j < formatstr.len and formatstr[j] notin {'\0', '}'}: if formatstr[j] in Digits: k = k * 10 + ord(formatstr[j]) - ord('0') if isNumber == 0: isNumber = 1 @@ -2328,7 +2223,7 @@ proc addf*(s: var string, formatstr: string, a: varargs[string, `$`]) {. inc(j) if isNumber == 1: let idx = if not negative: k-1 else: a.len-k - if idx >% a.high: invalidFormatString() + if idx < 0 or idx > a.high: invalidFormatString() add s, a[idx] else: var x = findNormalized(substr(formatstr, i+2, j-1), a) @@ -2337,7 +2232,7 @@ proc addf*(s: var string, formatstr: string, a: varargs[string, `$`]) {. i = j+1 of 'a'..'z', 'A'..'Z', '\128'..'\255', '_': var j = i+1 - while formatstr[j] in PatternChars: inc(j) + while j < formatstr.len and formatstr[j] in PatternChars: inc(j) var x = findNormalized(substr(formatstr, i+1, j-1), a) if x >= 0 and x < high(a): add s, a[x+1] else: invalidFormatString() @@ -2496,234 +2391,260 @@ proc removePrefix*(s: var string, prefix: string) {. s.delete(0, prefix.len - 1) when isMainModule: - doAssert align("abc", 4) == " abc" - doAssert align("a", 0) == "a" - doAssert align("1232", 6) == " 1232" - doAssert align("1232", 6, '#') == "##1232" - - doAssert alignLeft("abc", 4) == "abc " - doAssert alignLeft("a", 0) == "a" - doAssert alignLeft("1232", 6) == "1232 " - doAssert alignLeft("1232", 6, '#') == "1232##" - - let - inp = """ this is a long text -- muchlongerthan10chars and here - it goes""" - outp = " this is a\nlong text\n--\nmuchlongerthan10chars\nand here\nit goes" - doAssert wordWrap(inp, 10, false) == outp - - doAssert formatBiggestFloat(1234.567, ffDecimal, -1) == "1234.567000" - doAssert formatBiggestFloat(1234.567, ffDecimal, 0) == "1235." - doAssert formatBiggestFloat(1234.567, ffDecimal, 1) == "1234.6" - doAssert formatBiggestFloat(0.00000000001, ffDecimal, 11) == "0.00000000001" - doAssert formatBiggestFloat(0.00000000001, ffScientific, 1, ',') in - ["1,0e-11", "1,0e-011"] - # bug #6589 - doAssert formatFloat(123.456, ffScientific, precision = -1) == "1.234560e+02" - - doAssert "$# $3 $# $#" % ["a", "b", "c"] == "a c b c" - doAssert "${1}12 ${-1}$2" % ["a", "b"] == "a12 bb" - - block: # formatSize tests - doAssert formatSize((1'i64 shl 31) + (300'i64 shl 20)) == "2.293GiB" - doAssert formatSize((2.234*1024*1024).int) == "2.234MiB" - doAssert formatSize(4096) == "4KiB" - doAssert formatSize(4096, prefix=bpColloquial, includeSpace=true) == "4 kB" - doAssert formatSize(4096, includeSpace=true) == "4 KiB" - doAssert formatSize(5_378_934, prefix=bpColloquial, decimalSep=',') == "5,13MB" - - doAssert "$animal eats $food." % ["animal", "The cat", "food", "fish"] == - "The cat eats fish." - - doAssert "-ld a-ldz -ld".replaceWord("-ld") == " a-ldz " - doAssert "-lda-ldz -ld abc".replaceWord("-ld") == "-lda-ldz abc" - - type MyEnum = enum enA, enB, enC, enuD, enE - doAssert parseEnum[MyEnum]("enu_D") == enuD - - doAssert parseEnum("invalid enum value", enC) == enC - - doAssert center("foo", 13) == " foo " - doAssert center("foo", 0) == "foo" - doAssert center("foo", 3, fillChar = 'a') == "foo" - doAssert center("foo", 10, fillChar = '\t') == "\t\t\tfoo\t\t\t\t" - - doAssert count("foofoofoo", "foofoo") == 1 - doAssert count("foofoofoo", "foofoo", overlapping = true) == 2 - doAssert count("foofoofoo", 'f') == 3 - doAssert count("foofoofoobar", {'f','b'}) == 4 - - doAssert strip(" foofoofoo ") == "foofoofoo" - doAssert strip("sfoofoofoos", chars = {'s'}) == "foofoofoo" - doAssert strip("barfoofoofoobar", chars = {'b', 'a', 'r'}) == "foofoofoo" - doAssert strip("stripme but don't strip this stripme", - chars = {'s', 't', 'r', 'i', 'p', 'm', 'e'}) == - " but don't strip this " - doAssert strip("sfoofoofoos", leading = false, chars = {'s'}) == "sfoofoofoo" - doAssert strip("sfoofoofoos", trailing = false, chars = {'s'}) == "foofoofoos" - - doAssert " foo\n bar".indent(4, "Q") == "QQQQ foo\nQQQQ bar" - - doAssert "abba".multiReplace(("a", "b"), ("b", "a")) == "baab" - doAssert "Hello World.".multiReplace(("ello", "ELLO"), ("World.", "PEOPLE!")) == "HELLO PEOPLE!" - doAssert "aaaa".multiReplace(("a", "aa"), ("aa", "bb")) == "aaaaaaaa" - - doAssert isAlphaAscii('r') - doAssert isAlphaAscii('A') - doAssert(not isAlphaAscii('$')) - - doAssert isAlphaAscii("Rasp") - doAssert isAlphaAscii("Args") - doAssert(not isAlphaAscii("$Tomato")) - - doAssert isAlphaNumeric('3') - doAssert isAlphaNumeric('R') - doAssert(not isAlphaNumeric('!')) - - doAssert isAlphaNumeric("34ABc") - doAssert isAlphaNumeric("Rad") - doAssert isAlphaNumeric("1234") - doAssert(not isAlphaNumeric("@nose")) - - doAssert isDigit('3') - doAssert(not isDigit('a')) - doAssert(not isDigit('%')) - - doAssert isDigit("12533") - doAssert(not isDigit("12.33")) - doAssert(not isDigit("A45b")) - - doAssert isSpaceAscii('\t') - doAssert isSpaceAscii('\l') - doAssert(not isSpaceAscii('A')) - - doAssert isSpaceAscii("\t\l \v\r\f") - doAssert isSpaceAscii(" ") - doAssert(not isSpaceAscii("ABc \td")) - - doAssert(isNilOrEmpty("")) - doAssert(isNilOrEmpty(nil)) - doAssert(not isNilOrEmpty("test")) - doAssert(not isNilOrEmpty(" ")) - - doAssert(isNilOrWhitespace("")) - doAssert(isNilOrWhitespace(nil)) - doAssert(isNilOrWhitespace(" ")) - doAssert(isNilOrWhitespace("\t\l \v\r\f")) - doAssert(not isNilOrWhitespace("ABc \td")) - - doAssert isLowerAscii('a') - doAssert isLowerAscii('z') - doAssert(not isLowerAscii('A')) - doAssert(not isLowerAscii('5')) - doAssert(not isLowerAscii('&')) - - doAssert isLowerAscii("abcd") - doAssert(not isLowerAscii("abCD")) - doAssert(not isLowerAscii("33aa")) - - doAssert isUpperAscii('A') - doAssert(not isUpperAscii('b')) - doAssert(not isUpperAscii('5')) - doAssert(not isUpperAscii('%')) - - doAssert isUpperAscii("ABC") - doAssert(not isUpperAscii("AAcc")) - doAssert(not isUpperAscii("A#$")) - - doAssert rsplit("foo bar", seps=Whitespace) == @["foo", "bar"] - doAssert rsplit(" foo bar", seps=Whitespace, maxsplit=1) == @[" foo", "bar"] - doAssert rsplit(" foo bar ", seps=Whitespace, maxsplit=1) == @[" foo bar", ""] - doAssert rsplit(":foo:bar", sep=':') == @["", "foo", "bar"] - doAssert rsplit(":foo:bar", sep=':', maxsplit=2) == @["", "foo", "bar"] - doAssert rsplit(":foo:bar", sep=':', maxsplit=3) == @["", "foo", "bar"] - doAssert rsplit("foothebar", sep="the") == @["foo", "bar"] - - doAssert(unescape(r"\x013", "", "") == "\x013") - - doAssert join(["foo", "bar", "baz"]) == "foobarbaz" - doAssert join(@["foo", "bar", "baz"], ", ") == "foo, bar, baz" - doAssert join([1, 2, 3]) == "123" - doAssert join(@[1, 2, 3], ", ") == "1, 2, 3" - - doAssert """~~!!foo + proc nonStaticTests = + doAssert formatBiggestFloat(1234.567, ffDecimal, -1) == "1234.567000" + when not defined(js): + doAssert formatBiggestFloat(1234.567, ffDecimal, 0) == "1235." # <=== bug 8242 + doAssert formatBiggestFloat(1234.567, ffDecimal, 1) == "1234.6" + doAssert formatBiggestFloat(0.00000000001, ffDecimal, 11) == "0.00000000001" + doAssert formatBiggestFloat(0.00000000001, ffScientific, 1, ',') in + ["1,0e-11", "1,0e-011"] + # bug #6589 + when not defined(js): + doAssert formatFloat(123.456, ffScientific, precision = -1) == "1.234560e+02" + + doAssert "$# $3 $# $#" % ["a", "b", "c"] == "a c b c" + doAssert "${1}12 ${-1}$2" % ["a", "b"] == "a12 bb" + + block: # formatSize tests + when not defined(js): + doAssert formatSize((1'i64 shl 31) + (300'i64 shl 20)) == "2.293GiB" # <=== bug #8231 + doAssert formatSize((2.234*1024*1024).int) == "2.234MiB" + doAssert formatSize(4096) == "4KiB" + doAssert formatSize(4096, prefix=bpColloquial, includeSpace=true) == "4 kB" + doAssert formatSize(4096, includeSpace=true) == "4 KiB" + doAssert formatSize(5_378_934, prefix=bpColloquial, decimalSep=',') == "5,13MB" + + block: # formatEng tests + doAssert formatEng(0, 2, trim=false) == "0.00" + doAssert formatEng(0, 2) == "0" + doAssert formatEng(53, 2, trim=false) == "53.00" + doAssert formatEng(0.053, 2, trim=false) == "53.00e-3" + doAssert formatEng(0.053, 4, trim=false) == "53.0000e-3" + doAssert formatEng(0.053, 4, trim=true) == "53e-3" + doAssert formatEng(0.053, 0) == "53e-3" + doAssert formatEng(52731234) == "52.731234e6" + doAssert formatEng(-52731234) == "-52.731234e6" + doAssert formatEng(52731234, 1) == "52.7e6" + doAssert formatEng(-52731234, 1) == "-52.7e6" + doAssert formatEng(52731234, 1, decimalSep=',') == "52,7e6" + doAssert formatEng(-52731234, 1, decimalSep=',') == "-52,7e6" + + doAssert formatEng(4100, siPrefix=true, unit="V") == "4.1 kV" + doAssert formatEng(4.1, siPrefix=true, unit="V", useUnitSpace=true) == "4.1 V" + doAssert formatEng(4.1, siPrefix=true) == "4.1" # Note lack of space + doAssert formatEng(4100, siPrefix=true) == "4.1 k" + doAssert formatEng(4.1, siPrefix=true, unit="", useUnitSpace=true) == "4.1 " # Includes space + doAssert formatEng(4100, siPrefix=true, unit="") == "4.1 k" + doAssert formatEng(4100) == "4.1e3" + doAssert formatEng(4100, unit="V", useUnitSpace=true) == "4.1e3 V" + doAssert formatEng(4100, unit="", useUnitSpace=true) == "4.1e3 " + # Don't use SI prefix as number is too big + doAssert formatEng(3.1e22, siPrefix=true, unit="a", useUnitSpace=true) == "31e21 a" + # Don't use SI prefix as number is too small + doAssert formatEng(3.1e-25, siPrefix=true, unit="A", useUnitSpace=true) == "310e-27 A" + + proc staticTests = + doAssert align("abc", 4) == " abc" + doAssert align("a", 0) == "a" + doAssert align("1232", 6) == " 1232" + doAssert align("1232", 6, '#') == "##1232" + + doAssert alignLeft("abc", 4) == "abc " + doAssert alignLeft("a", 0) == "a" + doAssert alignLeft("1232", 6) == "1232 " + doAssert alignLeft("1232", 6, '#') == "1232##" + + let + inp = """ this is a long text -- muchlongerthan10chars and here + it goes""" + outp = " this is a\nlong text\n--\nmuchlongerthan10chars\nand here\nit goes" + doAssert wordWrap(inp, 10, false) == outp + + let + longInp = """ThisIsOneVeryLongStringWhichWeWillSplitIntoEightSeparatePartsNow""" + longOutp = "ThisIsOn\neVeryLon\ngStringW\nhichWeWi\nllSplitI\nntoEight\nSeparate\nPartsNow" + doAssert wordWrap(longInp, 8, true) == longOutp + + doAssert "$animal eats $food." % ["animal", "The cat", "food", "fish"] == + "The cat eats fish." + + doAssert "-ld a-ldz -ld".replaceWord("-ld") == " a-ldz " + doAssert "-lda-ldz -ld abc".replaceWord("-ld") == "-lda-ldz abc" + + doAssert "-lda-ldz -ld abc".replaceWord("") == "-lda-ldz -ld abc" + doAssert "oo".replace("", "abc") == "abcoabcoabc" + + type MyEnum = enum enA, enB, enC, enuD, enE + doAssert parseEnum[MyEnum]("enu_D") == enuD + + doAssert parseEnum("invalid enum value", enC) == enC + + doAssert center("foo", 13) == " foo " + doAssert center("foo", 0) == "foo" + doAssert center("foo", 3, fillChar = 'a') == "foo" + doAssert center("foo", 10, fillChar = '\t') == "\t\t\tfoo\t\t\t\t" + + doAssert count("foofoofoo", "foofoo") == 1 + doAssert count("foofoofoo", "foofoo", overlapping = true) == 2 + doAssert count("foofoofoo", 'f') == 3 + doAssert count("foofoofoobar", {'f','b'}) == 4 + + doAssert strip(" foofoofoo ") == "foofoofoo" + doAssert strip("sfoofoofoos", chars = {'s'}) == "foofoofoo" + doAssert strip("barfoofoofoobar", chars = {'b', 'a', 'r'}) == "foofoofoo" + doAssert strip("stripme but don't strip this stripme", + chars = {'s', 't', 'r', 'i', 'p', 'm', 'e'}) == + " but don't strip this " + doAssert strip("sfoofoofoos", leading = false, chars = {'s'}) == "sfoofoofoo" + doAssert strip("sfoofoofoos", trailing = false, chars = {'s'}) == "foofoofoos" + + doAssert " foo\n bar".indent(4, "Q") == "QQQQ foo\nQQQQ bar" + + doAssert "abba".multiReplace(("a", "b"), ("b", "a")) == "baab" + doAssert "Hello World.".multiReplace(("ello", "ELLO"), ("World.", "PEOPLE!")) == "HELLO PEOPLE!" + doAssert "aaaa".multiReplace(("a", "aa"), ("aa", "bb")) == "aaaaaaaa" + + doAssert isAlphaAscii('r') + doAssert isAlphaAscii('A') + doAssert(not isAlphaAscii('$')) + + doAssert isAlphaAscii("Rasp") + doAssert isAlphaAscii("Args") + doAssert(not isAlphaAscii("$Tomato")) + + doAssert isAlphaNumeric('3') + doAssert isAlphaNumeric('R') + doAssert(not isAlphaNumeric('!')) + + doAssert isAlphaNumeric("34ABc") + doAssert isAlphaNumeric("Rad") + doAssert isAlphaNumeric("1234") + doAssert(not isAlphaNumeric("@nose")) + + doAssert isDigit('3') + doAssert(not isDigit('a')) + doAssert(not isDigit('%')) + + doAssert isDigit("12533") + doAssert(not isDigit("12.33")) + doAssert(not isDigit("A45b")) + + doAssert isSpaceAscii('\t') + doAssert isSpaceAscii('\l') + doAssert(not isSpaceAscii('A')) + + doAssert isSpaceAscii("\t\l \v\r\f") + doAssert isSpaceAscii(" ") + doAssert(not isSpaceAscii("ABc \td")) + + doAssert(isNilOrWhitespace("")) + doAssert(isNilOrWhitespace(" ")) + doAssert(isNilOrWhitespace("\t\l \v\r\f")) + doAssert(not isNilOrWhitespace("ABc \td")) + + doAssert isLowerAscii('a') + doAssert isLowerAscii('z') + doAssert(not isLowerAscii('A')) + doAssert(not isLowerAscii('5')) + doAssert(not isLowerAscii('&')) + doAssert(not isLowerAscii(' ')) + + doAssert isLowerAscii("abcd", false) + doAssert(not isLowerAscii("33aa", false)) + doAssert(not isLowerAscii("a b", false)) + + doAssert(not isLowerAscii("abCD", true)) + doAssert isLowerAscii("33aa", true) + doAssert isLowerAscii("a b", true) + doAssert isLowerAscii("1, 2, 3 go!", true) + doAssert(not isLowerAscii(" ", true)) + doAssert(not isLowerAscii("(*&#@(^#$ ", true)) # None of the string chars are alphabets + + doAssert isUpperAscii('A') + doAssert(not isUpperAscii('b')) + doAssert(not isUpperAscii('5')) + doAssert(not isUpperAscii('%')) + + doAssert isUpperAscii("ABC", false) + doAssert(not isUpperAscii("A#$", false)) + doAssert(not isUpperAscii("A B", false)) + + doAssert(not isUpperAscii("AAcc", true)) + doAssert isUpperAscii("A#$", true) + doAssert isUpperAscii("A B", true) + doAssert isUpperAscii("1, 2, 3 GO!", true) + doAssert(not isUpperAscii(" ", true)) + doAssert(not isUpperAscii("(*&#@(^#$ ", true)) # None of the string chars are alphabets + + doAssert rsplit("foo bar", seps=Whitespace) == @["foo", "bar"] + doAssert rsplit(" foo bar", seps=Whitespace, maxsplit=1) == @[" foo", "bar"] + doAssert rsplit(" foo bar ", seps=Whitespace, maxsplit=1) == @[" foo bar", ""] + doAssert rsplit(":foo:bar", sep=':') == @["", "foo", "bar"] + doAssert rsplit(":foo:bar", sep=':', maxsplit=2) == @["", "foo", "bar"] + doAssert rsplit(":foo:bar", sep=':', maxsplit=3) == @["", "foo", "bar"] + doAssert rsplit("foothebar", sep="the") == @["foo", "bar"] + + doAssert(unescape(r"\x013", "", "") == "\x013") + + doAssert join(["foo", "bar", "baz"]) == "foobarbaz" + doAssert join(@["foo", "bar", "baz"], ", ") == "foo, bar, baz" + doAssert join([1, 2, 3]) == "123" + doAssert join(@[1, 2, 3], ", ") == "1, 2, 3" + + doAssert """~~!!foo ~~!!bar ~~!!baz""".unindent(2, "~~!!") == "foo\nbar\nbaz" - doAssert """~~!!foo + doAssert """~~!!foo ~~!!bar ~~!!baz""".unindent(2, "~~!!aa") == "~~!!foo\n~~!!bar\n~~!!baz" - doAssert """~~foo + doAssert """~~foo ~~ bar ~~ baz""".unindent(4, "~") == "foo\n bar\n baz" - doAssert """foo + doAssert """foo bar baz """.unindent(4) == "foo\nbar\nbaz\n" - doAssert """foo + doAssert """foo bar baz """.unindent(2) == "foo\n bar\n baz\n" - doAssert """foo + doAssert """foo bar baz """.unindent(100) == "foo\nbar\nbaz\n" - doAssert """foo + doAssert """foo foo bar """.unindent() == "foo\nfoo\nbar\n" - let s = " this is an example " - let s2 = ":this;is;an:example;;" - - doAssert s.split() == @["", "this", "is", "an", "example", "", ""] - doAssert s2.split(seps={':', ';'}) == @["", "this", "is", "an", "example", "", ""] - doAssert s.split(maxsplit=4) == @["", "this", "is", "an", "example "] - doAssert s.split(' ', maxsplit=1) == @["", "this is an example "] - doAssert s.split(" ", maxsplit=4) == @["", "this", "is", "an", "example "] - - doAssert s.splitWhitespace() == @["this", "is", "an", "example"] - doAssert s.splitWhitespace(maxsplit=1) == @["this", "is an example "] - doAssert s.splitWhitespace(maxsplit=2) == @["this", "is", "an example "] - doAssert s.splitWhitespace(maxsplit=3) == @["this", "is", "an", "example "] - doAssert s.splitWhitespace(maxsplit=4) == @["this", "is", "an", "example"] - - block: # formatEng tests - doAssert formatEng(0, 2, trim=false) == "0.00" - doAssert formatEng(0, 2) == "0" - doAssert formatEng(53, 2, trim=false) == "53.00" - doAssert formatEng(0.053, 2, trim=false) == "53.00e-3" - doAssert formatEng(0.053, 4, trim=false) == "53.0000e-3" - doAssert formatEng(0.053, 4, trim=true) == "53e-3" - doAssert formatEng(0.053, 0) == "53e-3" - doAssert formatEng(52731234) == "52.731234e6" - doAssert formatEng(-52731234) == "-52.731234e6" - doAssert formatEng(52731234, 1) == "52.7e6" - doAssert formatEng(-52731234, 1) == "-52.7e6" - doAssert formatEng(52731234, 1, decimalSep=',') == "52,7e6" - doAssert formatEng(-52731234, 1, decimalSep=',') == "-52,7e6" - - doAssert formatEng(4100, siPrefix=true, unit="V") == "4.1 kV" - doAssert formatEng(4.1, siPrefix=true, unit="V") == "4.1 V" - doAssert formatEng(4.1, siPrefix=true) == "4.1" # Note lack of space - doAssert formatEng(4100, siPrefix=true) == "4.1 k" - doAssert formatEng(4.1, siPrefix=true, unit="") == "4.1 " # Includes space - doAssert formatEng(4100, siPrefix=true, unit="") == "4.1 k" - doAssert formatEng(4100) == "4.1e3" - doAssert formatEng(4100, unit="V") == "4.1e3 V" - doAssert formatEng(4100, unit="") == "4.1e3 " # Space with unit="" - # Don't use SI prefix as number is too big - doAssert formatEng(3.1e22, siPrefix=true, unit="a") == "31e21 a" - # Don't use SI prefix as number is too small - doAssert formatEng(3.1e-25, siPrefix=true, unit="A") == "310e-27 A" - - block: # startsWith / endsWith char tests - var s = "abcdef" - doAssert s.startsWith('a') - doAssert s.startsWith('b') == false - doAssert s.endsWith('f') - doAssert s.endsWith('a') == false - doAssert s.endsWith('\0') == false - - #echo("strutils tests passed") + let s = " this is an example " + let s2 = ":this;is;an:example;;" + + doAssert s.split() == @["", "this", "is", "an", "example", "", ""] + doAssert s2.split(seps={':', ';'}) == @["", "this", "is", "an", "example", "", ""] + doAssert s.split(maxsplit=4) == @["", "this", "is", "an", "example "] + doAssert s.split(' ', maxsplit=1) == @["", "this is an example "] + doAssert s.split(" ", maxsplit=4) == @["", "this", "is", "an", "example "] + + doAssert s.splitWhitespace() == @["this", "is", "an", "example"] + doAssert s.splitWhitespace(maxsplit=1) == @["this", "is an example "] + doAssert s.splitWhitespace(maxsplit=2) == @["this", "is", "an example "] + doAssert s.splitWhitespace(maxsplit=3) == @["this", "is", "an", "example "] + doAssert s.splitWhitespace(maxsplit=4) == @["this", "is", "an", "example"] + + block: # startsWith / endsWith char tests + var s = "abcdef" + doAssert s.startsWith('a') + doAssert s.startsWith('b') == false + doAssert s.endsWith('f') + doAssert s.endsWith('a') == false + doAssert s.endsWith('\0') == false + + #echo("strutils tests passed") + + nonStaticTests() + staticTests() + static: staticTests() diff --git a/lib/pure/subexes.nim b/lib/pure/subexes.nim index 9d807abd4..8149c72cc 100644 --- a/lib/pure/subexes.nim +++ b/lib/pure/subexes.nim @@ -31,8 +31,6 @@ type SubexError* = object of ValueError ## exception that is raised for ## an invalid subex -{.deprecated: [EInvalidSubex: SubexError].} - proc raiseInvalidFormat(msg: string) {.noinline.} = raise newException(SubexError, "invalid format string: " & msg) @@ -44,7 +42,6 @@ type else: f: cstring num, i, lineLen: int -{.deprecated: [TFormatParser: FormatParser].} template call(x: untyped): untyped = p.i = i diff --git a/lib/pure/sugar.nim b/lib/pure/sugar.nim new file mode 100644 index 000000000..8ded552d9 --- /dev/null +++ b/lib/pure/sugar.nim @@ -0,0 +1,238 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2015 Dominik Picheta +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module implements nice syntactic sugar based on Nim's +## macro system. + +import macros + +proc createProcType(p, b: NimNode): NimNode {.compileTime.} = + #echo treeRepr(p) + #echo treeRepr(b) + result = newNimNode(nnkProcTy) + var formalParams = newNimNode(nnkFormalParams) + + formalParams.add b + + case p.kind + of nnkPar, nnkTupleConstr: + for i in 0 ..< p.len: + let ident = p[i] + var identDefs = newNimNode(nnkIdentDefs) + case ident.kind + of nnkExprColonExpr: + identDefs.add ident[0] + identDefs.add ident[1] + else: + identDefs.add newIdentNode("i" & $i) + identDefs.add(ident) + identDefs.add newEmptyNode() + formalParams.add identDefs + else: + var identDefs = newNimNode(nnkIdentDefs) + identDefs.add newIdentNode("i0") + identDefs.add(p) + identDefs.add newEmptyNode() + formalParams.add identDefs + + result.add formalParams + result.add newEmptyNode() + #echo(treeRepr(result)) + #echo(result.toStrLit()) + +macro `=>`*(p, b: untyped): untyped = + ## Syntax sugar for anonymous procedures. + ## + ## .. code-block:: nim + ## + ## proc passTwoAndTwo(f: (int, int) -> int): int = + ## f(2, 2) + ## + ## passTwoAndTwo((x, y) => x + y) # 4 + + #echo treeRepr(p) + #echo(treeRepr(b)) + var params: seq[NimNode] = @[newIdentNode("auto")] + + case p.kind + of nnkPar, nnkTupleConstr: + for c in children(p): + var identDefs = newNimNode(nnkIdentDefs) + case c.kind + of nnkExprColonExpr: + identDefs.add(c[0]) + identDefs.add(c[1]) + identDefs.add(newEmptyNode()) + of nnkIdent: + identDefs.add(c) + identDefs.add(newIdentNode("auto")) + identDefs.add(newEmptyNode()) + of nnkInfix: + if c[0].kind == nnkIdent and c[0].ident == !"->": + var procTy = createProcType(c[1], c[2]) + params[0] = procTy[0][0] + for i in 1 ..< procTy[0].len: + params.add(procTy[0][i]) + else: + error("Expected proc type (->) got (" & $c[0].ident & ").") + break + else: + echo treeRepr c + error("Incorrect procedure parameter list.") + params.add(identDefs) + of nnkIdent: + var identDefs = newNimNode(nnkIdentDefs) + identDefs.add(p) + identDefs.add(newIdentNode("auto")) + identDefs.add(newEmptyNode()) + params.add(identDefs) + of nnkInfix: + if p[0].kind == nnkIdent and p[0].ident == !"->": + var procTy = createProcType(p[1], p[2]) + params[0] = procTy[0][0] + for i in 1 ..< procTy[0].len: + params.add(procTy[0][i]) + else: + error("Expected proc type (->) got (" & $p[0].ident & ").") + else: + error("Incorrect procedure parameter list.") + result = newProc(params = params, body = b, procType = nnkLambda) + #echo(result.treeRepr) + #echo(result.toStrLit()) + #return result # TODO: Bug? + +macro `->`*(p, b: untyped): untyped = + ## Syntax sugar for procedure types. + ## + ## .. code-block:: nim + ## + ## proc pass2(f: (float, float) -> float): float = + ## f(2, 2) + ## + ## # is the same as: + ## + ## proc pass2(f: proc (x, y: float): float): float = + ## f(2, 2) + + result = createProcType(p, b) + +type ListComprehension = object +var lc*: ListComprehension + +macro `[]`*(lc: ListComprehension, comp, typ: untyped): untyped = + ## List comprehension, returns a sequence. `comp` is the actual list + ## comprehension, for example ``x | (x <- 1..10, x mod 2 == 0)``. `typ` is + ## the type that will be stored inside the result seq. + ## + ## .. code-block:: nim + ## + ## echo lc[x | (x <- 1..10, x mod 2 == 0), int] + ## + ## const n = 20 + ## echo lc[(x,y,z) | (x <- 1..n, y <- x..n, z <- y..n, x*x + y*y == z*z), + ## tuple[a,b,c: int]] + + expectLen(comp, 3) + expectKind(comp, nnkInfix) + expectKind(comp[0], nnkIdent) + assert($comp[0].ident == "|") + + result = newCall( + newDotExpr( + newIdentNode("result"), + newIdentNode("add")), + comp[1]) + + for i in countdown(comp[2].len-1, 0): + let x = comp[2][i] + expectMinLen(x, 1) + if x[0].kind == nnkIdent and $x[0].ident == "<-": + expectLen(x, 3) + result = newNimNode(nnkForStmt).add(x[1], x[2], result) + else: + result = newIfStmt((x, result)) + + result = newNimNode(nnkCall).add( + newNimNode(nnkPar).add( + newNimNode(nnkLambda).add( + newEmptyNode(), + newEmptyNode(), + newEmptyNode(), + newNimNode(nnkFormalParams).add( + newNimNode(nnkBracketExpr).add( + newIdentNode("seq"), + typ)), + newEmptyNode(), + newEmptyNode(), + newStmtList( + newAssignment( + newIdentNode("result"), + newNimNode(nnkPrefix).add( + newIdentNode("@"), + newNimNode(nnkBracket))), + result)))) + + +macro dump*(x: typed): untyped = + ## Dumps the content of an expression, useful for debugging. + ## It accepts any expression and prints a textual representation + ## of the tree representing the expression - as it would appear in + ## source code - together with the value of the expression. + ## + ## As an example, + ## + ## .. code-block:: nim + ## let + ## x = 10 + ## y = 20 + ## dump(x + y) + ## + ## will print ``x + y = 30``. + let s = x.toStrLit + let r = quote do: + debugEcho `s`, " = ", `x` + return r + +# TODO: consider exporting this in macros.nim +proc freshIdentNodes(ast: NimNode): NimNode = + # Replace NimIdent and NimSym by a fresh ident node + # see also https://github.com/nim-lang/Nim/pull/8531#issuecomment-410436458 + proc inspect(node: NimNode): NimNode = + case node.kind: + of nnkIdent, nnkSym: + result = ident($node) + of nnkEmpty, nnkLiterals: + result = node + else: + result = node.kind.newTree() + for child in node: + result.add inspect(child) + result = inspect(ast) + +macro distinctBase*(T: typedesc): untyped = + ## reverses ``type T = distinct A``; works recursively. + runnableExamples: + type T = distinct int + doAssert distinctBase(T) is int + doAssert: not compiles(distinctBase(int)) + type T2 = distinct T + doAssert distinctBase(T2) is int + + let typeNode = getTypeImpl(T) + expectKind(typeNode, nnkBracketExpr) + if typeNode[0].typeKind != ntyTypeDesc: + error "expected typeDesc, got " & $typeNode[0] + var typeSym = typeNode[1] + typeSym = getTypeImpl(typeSym) + if typeSym.typeKind != ntyDistinct: + error "type is not distinct" + typeSym = typeSym[0] + while typeSym.typeKind == ntyDistinct: + typeSym = getTypeImpl(typeSym)[0] + typeSym.freshIdentNodes diff --git a/lib/pure/terminal.nim b/lib/pure/terminal.nim index 4f2f73ba7..2e138b27e 100644 --- a/lib/pure/terminal.nim +++ b/lib/pure/terminal.nim @@ -19,32 +19,36 @@ import macros import strformat from strutils import toLowerAscii -import colors +import colors, tables -const - hasThreadSupport = compileOption("threads") +when defined(windows): + import winlean -when not hasThreadSupport: - import tables - var - colorsFGCache = initTable[Color, string]() - colorsBGCache = initTable[Color, string]() - when not defined(windows): - var - styleCache = initTable[int, string]() +type + PTerminal = ref object + trueColorIsSupported: bool + trueColorIsEnabled: bool + fgSetColor: bool + when defined(windows): + hStdout: Handle + hStderr: Handle + oldStdoutAttr: int16 + oldStderrAttr: int16 -var - trueColorIsSupported: bool - trueColorIsEnabled: bool - fgSetColor: bool +var gTerm {.threadvar.}: PTerminal + +proc newTerminal(): PTerminal + +proc getTerminal(): PTerminal {.inline.} = + if isNil(gTerm): + gTerm = newTerminal() + result = gTerm const fgPrefix = "\x1b[38;2;" bgPrefix = "\x1b[48;2;" - -when not defined(windows): - const - stylePrefix = "\e[" + ansiResetCode* = "\e[0m" + stylePrefix = "\e[" when defined(windows): import winlean, os @@ -160,23 +164,6 @@ when defined(windows): proc setConsoleMode(hConsoleHandle: Handle, dwMode: DWORD): WINBOOL{. stdcall, dynlib: "kernel32", importc: "SetConsoleMode".} - var - hStdout: Handle # = createFile("CONOUT$", GENERIC_WRITE, 0, nil, - # OPEN_ALWAYS, 0, 0) - hStderr: Handle - - block: - var hStdoutTemp = getStdHandle(STD_OUTPUT_HANDLE) - if duplicateHandle(getCurrentProcess(), hStdoutTemp, getCurrentProcess(), - addr(hStdout), 0, 1, DUPLICATE_SAME_ACCESS) == 0: - when defined(consoleapp): - raiseOSError(osLastError()) - var hStderrTemp = getStdHandle(STD_ERROR_HANDLE) - if duplicateHandle(getCurrentProcess(), hStderrTemp, getCurrentProcess(), - addr(hStderr), 0, 1, DUPLICATE_SAME_ACCESS) == 0: - when defined(consoleapp): - raiseOSError(osLastError()) - proc getCursorPos(h: Handle): tuple [x,y: int] = var c: CONSOLESCREENBUFFERINFO if getConsoleScreenBufferInfo(h, addr(c)) == 0: @@ -197,12 +184,23 @@ when defined(windows): return c.wAttributes return 0x70'i16 # ERROR: return white background, black text - var - oldStdoutAttr = getAttributes(hStdout) - oldStderrAttr = getAttributes(hStderr) + proc initTerminal(term: PTerminal) = + var hStdoutTemp = getStdHandle(STD_OUTPUT_HANDLE) + if duplicateHandle(getCurrentProcess(), hStdoutTemp, getCurrentProcess(), + addr(term.hStdout), 0, 1, DUPLICATE_SAME_ACCESS) == 0: + when defined(consoleapp): + raiseOSError(osLastError()) + var hStderrTemp = getStdHandle(STD_ERROR_HANDLE) + if duplicateHandle(getCurrentProcess(), hStderrTemp, getCurrentProcess(), + addr(term.hStderr), 0, 1, DUPLICATE_SAME_ACCESS) == 0: + when defined(consoleapp): + raiseOSError(osLastError()) + term.oldStdoutAttr = getAttributes(term.hStdout) + term.oldStderrAttr = getAttributes(term.hStderr) template conHandle(f: File): Handle = - if f == stderr: hStderr else: hStdout + let term = getTerminal() + if f == stderr: term.hStderr else: term.hStdout else: import termios, posix, os, parseutils @@ -310,7 +308,7 @@ proc setCursorPos*(f: File, x, y: int) = let h = conHandle(f) setCursorPos(h, x, y) else: - f.write(fmt"{stylePrefix}{y};{x}f") + f.write(fmt"{stylePrefix}{y+1};{x+1}f") proc setCursorXPos*(f: File, x: int) = ## Sets the terminal's cursor to the x position. @@ -325,7 +323,7 @@ proc setCursorXPos*(f: File, x: int) = if setConsoleCursorPosition(h, origin) == 0: raiseOSError(osLastError()) else: - f.write(fmt"{stylePrefix}{x}G") + f.write(fmt"{stylePrefix}{x+1}G") when defined(windows): proc setCursorYPos*(f: File, y: int) = @@ -463,39 +461,43 @@ proc eraseScreen*(f: File) = proc resetAttributes*(f: File) = ## Resets all attributes. when defined(windows): + let term = getTerminal() if f == stderr: - discard setConsoleTextAttribute(hStderr, oldStderrAttr) + discard setConsoleTextAttribute(term.hStderr, term.oldStderrAttr) else: - discard setConsoleTextAttribute(hStdout, oldStdoutAttr) + discard setConsoleTextAttribute(term.hStdout, term.oldStdoutAttr) else: - f.write("\e[0m") + f.write(ansiResetCode) type - Style* = enum ## different styles for text output + Style* = enum ## different styles for text output styleBright = 1, ## bright text styleDim, ## dim text - styleUnknown, ## unknown - styleUnderscore = 4, ## underscored text + styleItalic, ## italic (or reverse on terminals not supporting) + styleUnderscore, ## underscored text styleBlink, ## blinking/bold text - styleReverse = 7, ## unknown - styleHidden ## hidden text + styleBlinkRapid, ## rapid blinking/bold text (not widely supported) + styleReverse, ## reverse + styleHidden, ## hidden text + styleStrikethrough ## strikethrough {.deprecated: [TStyle: Style].} +{.deprecated: [styleUnknown: styleItalic].} when not defined(windows): var gFG {.threadvar.}: int gBG {.threadvar.}: int - proc getStyleStr(style: int): string = - when hasThreadSupport: - result = fmt"{stylePrefix}{style}m" - else: - if styleCache.hasKey(style): - result = styleCache[style] - else: - result = fmt"{stylePrefix}{style}m" - styleCache[style] = result +proc ansiStyleCode*(style: int): string = + result = fmt"{stylePrefix}{style}m" + +template ansiStyleCode*(style: Style): string = + ansiStyleCode(style.int) + +# The styleCache can be skipped when `style` is known at compile-time +template ansiStyleCode*(style: static[Style]): string = + (static(stylePrefix & $style.int & "m")) proc setStyle*(f: File, style: set[Style]) = ## Sets the terminal style. @@ -510,23 +512,24 @@ proc setStyle*(f: File, style: set[Style]) = discard setConsoleTextAttribute(h, old or a) else: for s in items(style): - f.write(getStyleStr(ord(s))) + f.write(ansiStyleCode(s)) proc writeStyled*(txt: string, style: set[Style] = {styleBright}) = ## Writes the text `txt` in a given `style` to stdout. when defined(windows): - var old = getAttributes(hStdout) + let term = getTerminal() + var old = getAttributes(term.hStdout) stdout.setStyle(style) stdout.write(txt) - discard setConsoleTextAttribute(hStdout, old) + discard setConsoleTextAttribute(term.hStdout, old) else: stdout.setStyle(style) stdout.write(txt) stdout.resetAttributes() if gFG != 0: - stdout.write(getStyleStr(gFG)) + stdout.write(ansiStyleCode(gFG)) if gBG != 0: - stdout.write(getStyleStr(gBG)) + stdout.write(ansiStyleCode(gBG)) type ForegroundColor* = enum ## terminal's foreground colors @@ -537,7 +540,9 @@ type fgBlue, ## blue fgMagenta, ## magenta fgCyan, ## cyan - fgWhite ## white + fgWhite, ## white + fg8Bit, ## 256-color (not supported, see ``enableTrueColors`` instead.) + fgDefault ## default terminal foreground color BackgroundColor* = enum ## terminal's background colors bgBlack = 40, ## black @@ -547,92 +552,112 @@ type bgBlue, ## blue bgMagenta, ## magenta bgCyan, ## cyan - bgWhite ## white + bgWhite, ## white + bg8Bit, ## 256-color (not supported, see ``enableTrueColors`` instead.) + bgDefault ## default terminal background color {.deprecated: [TForegroundColor: ForegroundColor, TBackgroundColor: BackgroundColor].} +when defined(windows): + var defaultForegroundColor, defaultBackgroundColor: int16 = 0xFFFF'i16 # Default to an invalid value 0xFFFF + proc setForegroundColor*(f: File, fg: ForegroundColor, bright=false) = ## Sets the terminal's foreground color. when defined(windows): let h = conHandle(f) var old = getAttributes(h) and not FOREGROUND_RGB - if bright: - old = old or FOREGROUND_INTENSITY + if defaultForegroundColor == 0xFFFF'i16: + defaultForegroundColor = old + old = if bright: old or FOREGROUND_INTENSITY + else: old and not(FOREGROUND_INTENSITY) const lookup: array[ForegroundColor, int] = [ - 0, + 0, # ForegroundColor enum with ordinal 30 (FOREGROUND_RED), (FOREGROUND_GREEN), (FOREGROUND_RED or FOREGROUND_GREEN), (FOREGROUND_BLUE), (FOREGROUND_RED or FOREGROUND_BLUE), (FOREGROUND_BLUE or FOREGROUND_GREEN), - (FOREGROUND_BLUE or FOREGROUND_GREEN or FOREGROUND_RED)] - discard setConsoleTextAttribute(h, toU16(old or lookup[fg])) + (FOREGROUND_BLUE or FOREGROUND_GREEN or FOREGROUND_RED), + 0, # fg8Bit not supported, see ``enableTrueColors`` instead. + 0] # unused + if fg == fgDefault: + discard setConsoleTextAttribute(h, toU16(old or defaultForegroundColor)) + else: + discard setConsoleTextAttribute(h, toU16(old or lookup[fg])) else: gFG = ord(fg) if bright: inc(gFG, 60) - f.write(getStyleStr(gFG)) + f.write(ansiStyleCode(gFG)) proc setBackgroundColor*(f: File, bg: BackgroundColor, bright=false) = ## Sets the terminal's background color. when defined(windows): let h = conHandle(f) var old = getAttributes(h) and not BACKGROUND_RGB - if bright: - old = old or BACKGROUND_INTENSITY + if defaultBackgroundColor == 0xFFFF'i16: + defaultBackgroundColor = old + old = if bright: old or BACKGROUND_INTENSITY + else: old and not(BACKGROUND_INTENSITY) const lookup: array[BackgroundColor, int] = [ - 0, + 0, # BackgroundColor enum with ordinal 40 (BACKGROUND_RED), (BACKGROUND_GREEN), (BACKGROUND_RED or BACKGROUND_GREEN), (BACKGROUND_BLUE), (BACKGROUND_RED or BACKGROUND_BLUE), (BACKGROUND_BLUE or BACKGROUND_GREEN), - (BACKGROUND_BLUE or BACKGROUND_GREEN or BACKGROUND_RED)] - discard setConsoleTextAttribute(h, toU16(old or lookup[bg])) + (BACKGROUND_BLUE or BACKGROUND_GREEN or BACKGROUND_RED), + 0, # bg8Bit not supported, see ``enableTrueColors`` instead. + 0] # unused + if bg == bgDefault: + discard setConsoleTextAttribute(h, toU16(old or defaultBackgroundColor)) + else: + discard setConsoleTextAttribute(h, toU16(old or lookup[bg])) else: gBG = ord(bg) if bright: inc(gBG, 60) - f.write(getStyleStr(gBG)) + f.write(ansiStyleCode(gBG)) +proc ansiForegroundColorCode*(fg: ForegroundColor, bright=false): string = + var style = ord(fg) + if bright: inc(style, 60) + return ansiStyleCode(style) -proc getFGColorStr(color: Color): string = - when hasThreadSupport: - let rgb = extractRGB(color) - result = fmt"{fgPrefix}{rgb.r};{rgb.g};{rgb.b}m" - else: - if colorsFGCache.hasKey(color): - result = colorsFGCache[color] - else: - let rgb = extractRGB(color) - result = fmt"{fgPrefix}{rgb.r};{rgb.g};{rgb.b}m" - colorsFGCache[color] = result - -proc getBGColorStr(color: Color): string = - when hasThreadSupport: - let rgb = extractRGB(color) - result = fmt"{bgPrefix}{rgb.r};{rgb.g};{rgb.b}m" - else: - if colorsBGCache.hasKey(color): - result = colorsBGCache[color] - else: - let rgb = extractRGB(color) - result = fmt"{bgPrefix}{rgb.r};{rgb.g};{rgb.b}m" - colorsFGCache[color] = result +template ansiForegroundColorCode*(fg: static[ForegroundColor], + bright: static[bool] = false): string = + ansiStyleCode(fg.int + bright.int * 60) + +proc ansiForegroundColorCode*(color: Color): string = + let rgb = extractRGB(color) + result = fmt"{fgPrefix}{rgb.r};{rgb.g};{rgb.b}m" + +template ansiForegroundColorCode*(color: static[Color]): string = + const rgb = extractRGB(color) + (static(fmt"{fgPrefix}{rgb.r};{rgb.g};{rgb.b}m")) + +proc ansiBackgroundColorCode*(color: Color): string = + let rgb = extractRGB(color) + result = fmt"{bgPrefix}{rgb.r};{rgb.g};{rgb.b}m" + +template ansiBackgroundColorCode*(color: static[Color]): string = + const rgb = extractRGB(color) + (static(fmt"{bgPrefix}{rgb.r};{rgb.g};{rgb.b}m")) proc setForegroundColor*(f: File, color: Color) = ## Sets the terminal's foreground true color. - if trueColorIsEnabled: - f.write(getFGColorStr(color)) + if getTerminal().trueColorIsEnabled: + f.write(ansiForegroundColorCode(color)) proc setBackgroundColor*(f: File, color: Color) = ## Sets the terminal's background true color. - if trueColorIsEnabled: - f.write(getBGColorStr(color)) + if getTerminal().trueColorIsEnabled: + f.write(ansiBackgroundColorCode(color)) proc setTrueColor(f: File, color: Color) = - if fgSetColor: + let term = getTerminal() + if term.fgSetColor: setForegroundColor(f, color) else: setBackgroundColor(f, color) @@ -671,8 +696,8 @@ template styledEchoProcessArg(f: File, cmd: TerminalCmd) = when cmd == bgColor: fgSetColor = false -macro styledWriteLine*(f: File, m: varargs[typed]): untyped = - ## Similar to ``writeLine``, but treating terminal style arguments specially. +macro styledWrite*(f: File, m: varargs[typed]): untyped = + ## Similar to ``write``, but treating terminal style arguments specially. ## When some argument is ``Style``, ``set[Style]``, ``ForegroundColor``, ## ``BackgroundColor`` or ``TerminalCmd`` then it is not sent directly to ## ``f``, but instead corresponding terminal style proc is called. @@ -681,20 +706,19 @@ macro styledWriteLine*(f: File, m: varargs[typed]): untyped = ## ## .. code-block:: nim ## - ## proc error(msg: string) = - ## styledWriteLine(stderr, fgRed, "Error: ", resetStyle, msg) + ## stdout.styledWrite(fgRed, "red text ") + ## stdout.styledWrite(fgGreen, "green text") ## - let m = callsite() var reset = false result = newNimNode(nnkStmtList) - for i in countup(2, m.len - 1): + for i in countup(0, m.len - 1): let item = m[i] case item.kind of nnkStrLit..nnkTripleStrLit: if i == m.len - 1: - # optimize if string literal is last, just call writeLine - result.add(newCall(bindSym"writeLine", f, item)) + # optimize if string literal is last, just call write + result.add(newCall(bindSym"write", f, item)) if reset: result.add(newCall(bindSym"resetAttributes", f)) return else: @@ -703,16 +727,24 @@ macro styledWriteLine*(f: File, m: varargs[typed]): untyped = else: result.add(newCall(bindSym"styledEchoProcessArg", f, item)) reset = true - - result.add(newCall(bindSym"write", f, newStrLitNode("\n"))) if reset: result.add(newCall(bindSym"resetAttributes", f)) -macro styledEcho*(args: varargs[untyped]): untyped = +template styledWriteLine*(f: File, args: varargs[untyped]) = + ## Calls ``styledWrite`` and appends a newline at the end. + ## + ## Example: + ## + ## .. code-block:: nim + ## + ## proc error(msg: string) = + ## styledWriteLine(stderr, fgRed, "Error: ", resetStyle, msg) + ## + styledWrite(f, args) + write(f, "\n") + +template styledEcho*(args: varargs[untyped]) = ## Echoes styles arguments to stdout using ``styledWriteLine``. - result = newCall(bindSym"styledWriteLine") - result.add(bindSym"stdout") - for arg in children(args): - result.add(arg) + stdout.styledWriteLine(args) proc getch*(): char = ## Read a single character from the terminal, blocking until it is entered. @@ -759,6 +791,10 @@ when defined(windows): x = runeLenAt(password.string, i) inc i, x password.string.setLen(max(password.len - x, 0)) + of chr(0x0): + # modifier key - ignore - for details see + # https://github.com/nim-lang/Nim/issues/7764 + continue else: password.string.add(toUTF8(c.Rune)) stdout.write "\n" @@ -815,63 +851,102 @@ proc resetAttributes*() {.noconv.} = ## ``system.addQuitProc(resetAttributes)``. resetAttributes(stdout) -when not defined(testing) and isMainModule: - #system.addQuitProc(resetAttributes) - write(stdout, "never mind") - stdout.eraseLine() - stdout.styledWriteLine("styled text ", {styleBright, styleBlink, styleUnderscore}) - stdout.setBackGroundColor(bgCyan, true) - stdout.setForeGroundColor(fgBlue) - stdout.writeLine("ordinary text") - stdout.resetAttributes() - proc isTrueColorSupported*(): bool = ## Returns true if a terminal supports true color. - return trueColorIsSupported + return getTerminal().trueColorIsSupported when defined(windows): import os proc enableTrueColors*() = ## Enable true color. + var term = getTerminal() when defined(windows): var ver: OSVERSIONINFO ver.dwOSVersionInfoSize = sizeof(ver).DWORD let res = getVersionExW(addr ver) if res == 0: - trueColorIsSupported = false + term.trueColorIsSupported = false else: - trueColorIsSupported = ver.dwMajorVersion > 10 or + term.trueColorIsSupported = ver.dwMajorVersion > 10 or (ver.dwMajorVersion == 10 and (ver.dwMinorVersion > 0 or (ver.dwMinorVersion == 0 and ver.dwBuildNumber >= 10586))) - if not trueColorIsSupported: - trueColorIsSupported = getEnv("ANSICON_DEF").len > 0 + if not term.trueColorIsSupported: + term.trueColorIsSupported = getEnv("ANSICON_DEF").len > 0 - if trueColorIsSupported: + if term.trueColorIsSupported: if getEnv("ANSICON_DEF").len == 0: var mode: DWORD = 0 if getConsoleMode(getStdHandle(STD_OUTPUT_HANDLE), addr(mode)) != 0: mode = mode or ENABLE_VIRTUAL_TERMINAL_PROCESSING if setConsoleMode(getStdHandle(STD_OUTPUT_HANDLE), mode) != 0: - trueColorIsEnabled = true + term.trueColorIsEnabled = true else: - trueColorIsEnabled = false + term.trueColorIsEnabled = false else: - trueColorIsEnabled = true + term.trueColorIsEnabled = true else: - trueColorIsSupported = string(getEnv("COLORTERM")).toLowerAscii() in ["truecolor", "24bit"] - trueColorIsEnabled = trueColorIsSupported + term.trueColorIsSupported = string(getEnv("COLORTERM")).toLowerAscii() in ["truecolor", "24bit"] + term.trueColorIsEnabled = term.trueColorIsSupported proc disableTrueColors*() = ## Disable true color. + var term = getTerminal() when defined(windows): - if trueColorIsSupported: + if term.trueColorIsSupported: if getEnv("ANSICON_DEF").len == 0: var mode: DWORD = 0 if getConsoleMode(getStdHandle(STD_OUTPUT_HANDLE), addr(mode)) != 0: mode = mode and not ENABLE_VIRTUAL_TERMINAL_PROCESSING discard setConsoleMode(getStdHandle(STD_OUTPUT_HANDLE), mode) - trueColorIsEnabled = false + term.trueColorIsEnabled = false else: - trueColorIsEnabled = false + term.trueColorIsEnabled = false + +proc newTerminal(): PTerminal = + new result + when defined(windows): + initTerminal(result) + +when not defined(testing) and isMainModule: + assert ansiStyleCode(styleBright) == "\e[1m" + assert ansiStyleCode(styleStrikethrough) == "\e[9m" + #system.addQuitProc(resetAttributes) + write(stdout, "never mind") + stdout.eraseLine() + stdout.styledWriteLine({styleBright, styleBlink, styleUnderscore}, "styled text ") + stdout.styledWriteLine("italic text ", {styleItalic}) + stdout.setBackGroundColor(bgCyan, true) + stdout.setForeGroundColor(fgBlue) + stdout.write("blue text in cyan background") + stdout.resetAttributes() + echo "" + stdout.writeLine("ordinary text") + echo "more ordinary text" + styledEcho styleBright, fgGreen, "[PASS]", resetStyle, fgGreen, " Yay!" + echo "ordinary text again" + styledEcho styleBright, fgRed, "[FAIL]", resetStyle, fgRed, " Nay :(" + echo "ordinary text again" + setForeGroundColor(fgGreen) + echo "green text" + echo "more green text" + setForeGroundColor(fgBlue) + echo "blue text" + resetAttributes() + echo "ordinary text" + + stdout.styledWriteLine(fgRed, "red text ") + stdout.styledWriteLine(fgWhite, bgRed, "white text in red background") + stdout.styledWriteLine(" ordinary text ") + stdout.styledWriteLine(fgGreen, "green text") + + stdout.styledWrite(fgRed, "red text ") + stdout.styledWrite(fgWhite, bgRed, "white text in red background") + stdout.styledWrite(" ordinary text ") + stdout.styledWrite(fgGreen, "green text") + echo "" + echo "ordinary text" + stdout.styledWriteLine(fgRed, "red text ", styleBright, "bold red", fgDefault, " bold text") + stdout.styledWriteLine(bgYellow, "text in yellow bg", styleBright, " bold text in yellow bg", bgDefault, " bold text") + echo "ordinary text" diff --git a/lib/pure/times.nim b/lib/pure/times.nim index 6c1e1fe87..a7ccbf6ee 100644 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -1,49 +1,166 @@ # # # Nim's Runtime Library -# (c) Copyright 2015 Andreas Rumpf +# (c) Copyright 2017 Nim contributors # # See the file "copying.txt", included in this # distribution, for details about the copyright. # +##[ + This module contains routines and types for dealing with time using a proleptic Gregorian calendar. + It's also available for the `JavaScript target <backends.html#the-javascript-target>`_. + + Although the types use nanosecond time resolution, the underlying resolution used by ``getTime()`` + depends on the platform and backend (JS is limited to millisecond precision). + + Examples: + + .. code-block:: nim + + import times, os + let time = cpuTime() + + sleep(100) # replace this with something to be timed + echo "Time taken: ",cpuTime() - time + + echo "My formatted time: ", format(now(), "d MMMM yyyy HH:mm") + echo "Using predefined formats: ", getClockStr(), " ", getDateStr() + + echo "cpuTime() float value: ", cpuTime() + echo "An hour from now : ", now() + 1.hours + echo "An hour from (UTC) now: ", getTime().utc + initDuration(hours = 1) + + Parsing and Formatting Dates + ---------------------------- + + The ``DateTime`` type can be parsed and formatted using the different + ``parse`` and ``format`` procedures. + + .. code-block:: nim + + let dt = parse("2000-01-01", "yyyy-MM-dd") + echo dt.format("yyyy-MM-dd") + + The different format patterns that are supported are documented below. + + ============= ================================================================================= ================================================ + Pattern Description Example + ============= ================================================================================= ================================================ + ``d`` Numeric value representing the day of the month, | ``1/04/2012 -> 1`` + it will be either one or two digits long. | ``21/04/2012 -> 21`` + ``dd`` Same as above, but is always two digits. | ``1/04/2012 -> 01`` + | ``21/04/2012 -> 21`` + ``ddd`` Three letter string which indicates the day of the week. | ``Saturday -> Sat`` + | ``Monday -> Mon`` + ``dddd`` Full string for the day of the week. | ``Saturday -> Saturday`` + | ``Monday -> Monday`` + ``h`` The hours in one digit if possible. Ranging from 1-12. | ``5pm -> 5`` + | ``2am -> 2`` + ``hh`` The hours in two digits always. If the hour is one digit 0 is prepended. | ``5pm -> 05`` + | ``11am -> 11`` + ``H`` The hours in one digit if possible, ranging from 0-23. | ``5pm -> 17`` + | ``2am -> 2`` + ``HH`` The hours in two digits always. 0 is prepended if the hour is one digit. | ``5pm -> 17`` + | ``2am -> 02`` + ``m`` The minutes in 1 digit if possible. | ``5:30 -> 30`` + | ``2:01 -> 1`` + ``mm`` Same as above but always 2 digits, 0 is prepended if the minute is one digit. | ``5:30 -> 30`` + | ``2:01 -> 01`` + ``M`` The month in one digit if possible. | ``September -> 9`` + | ``December -> 12`` + ``MM`` The month in two digits always. 0 is prepended. | ``September -> 09`` + | ``December -> 12`` + ``MMM`` Abbreviated three-letter form of the month. | ``September -> Sep`` + | ``December -> Dec`` + ``MMMM`` Full month string, properly capitalized. | ``September -> September`` + ``s`` Seconds as one digit if possible. | ``00:00:06 -> 6`` + ``ss`` Same as above but always two digits. 0 is prepended. | ``00:00:06 -> 06`` + ``t`` ``A`` when time is in the AM. ``P`` when time is in the PM. | ``5pm -> P`` + | ``2am -> A`` + ``tt`` Same as above, but ``AM`` and ``PM`` instead of ``A`` and ``P`` respectively. | ``5pm -> PM`` + | ``2am -> AM`` + ``yy`` The last two digits of the year. When parsing, the current century is assumed. | ``2012 AD -> 12`` + ``yyyy`` The year, padded to atleast four digits. | ``2012 AD -> 2012`` + Is always positive, even when the year is BC. | ``24 AD -> 0024`` + When the year is more than four digits, '+' is prepended. | ``24 BC -> 00024`` + | ``12345 AD -> +12345`` + ``YYYY`` The year without any padding. | ``2012 AD -> 2012`` + Is always positive, even when the year is BC. | ``24 AD -> 24`` + | ``24 BC -> 24`` + | ``12345 AD -> 12345`` + ``uuuu`` The year, padded to atleast four digits. Will be negative when the year is BC. | ``2012 AD -> 2012`` + When the year is more than four digits, '+' is prepended unless the year is BC. | ``24 AD -> 0024`` + | ``24 BC -> -0023`` + | ``12345 AD -> +12345`` + ``UUUU`` The year without any padding. Will be negative when the year is BC. | ``2012 AD -> 2012`` + | ``24 AD -> 24`` + | ``24 BC -> -23`` + | ``12345 AD -> 12345`` + ``z`` Displays the timezone offset from UTC. | ``GMT+7 -> +7`` + | ``GMT-5 -> -5`` + ``zz`` Same as above but with leading 0. | ``GMT+7 -> +07`` + | ``GMT-5 -> -05`` + ``zzz`` Same as above but with ``:mm`` where *mm* represents minutes. | ``GMT+7 -> +07:00`` + | ``GMT-5 -> -05:00`` + ``zzzz`` Same as above but with ``:ss`` where *ss* represents seconds. | ``GMT+7 -> +07:00:00`` + | ``GMT-5 -> -05:00:00`` + ``g`` Era: AD or BC | ``300 AD -> AD`` + | ``300 BC -> BC`` + ``fff`` Milliseconds display | ``1000000 nanoseconds -> 1`` + ``ffffff`` Microseconds display | ``1000000 nanoseconds -> 1000`` + ``fffffffff`` Nanoseconds display | ``1000000 nanoseconds -> 1000000`` + ============= ================================================================================= ================================================ + + Other strings can be inserted by putting them in ``''``. For example + ``hh'->'mm`` will give ``01->56``. The following characters can be + inserted without quoting them: ``:`` ``-`` ``(`` ``)`` ``/`` ``[`` ``]`` + ``,``. A literal ``'`` can be specified with ``''``. + + However you don't need to necessarily separate format patterns, a + unambiguous format string like ``yyyyMMddhhmmss`` is valid too (although + only for years in the range 1..9999). +]## -## This module contains routines and types for dealing with time. -## This module is available for the `JavaScript target -## <backends.html#the-javascript-target>`_. The proleptic Gregorian calendar is the only calendar supported. -## -## Examples: -## -## .. code-block:: nim -## -## import times, os -## let time = cpuTime() -## -## sleep(100) # replace this with something to be timed -## echo "Time taken: ",cpuTime() - time -## -## echo "My formatted time: ", format(now(), "d MMMM yyyy HH:mm") -## echo "Using predefined formats: ", getClockStr(), " ", getDateStr() -## -## echo "epochTime() float value: ", epochTime() -## echo "cpuTime() float value: ", cpuTime() -## echo "An hour from now : ", now() + 1.hours -## echo "An hour from (UTC) now: ", getTime().utc + initInterval(0,0,0,1) {.push debugger:off.} # the user does not want to trace a part # of the standard library! import - strutils, parseutils + strutils, parseutils, algorithm, math, options, strformat include "system/inclrtl" +# This is really bad, but overflow checks are broken badly for +# ints on the JS backend. See #6752. +when defined(JS): + {.push overflowChecks: off.} + proc `*`(a, b: int64): int64 = + system.`* `(a, b) + proc `*`(a, b: int): int = + system.`* `(a, b) + proc `+`(a, b: int64): int64 = + system.`+ `(a, b) + proc `+`(a, b: int): int = + system.`+ `(a, b) + proc `-`(a, b: int64): int64 = + system.`- `(a, b) + proc `-`(a, b: int): int = + system.`- `(a, b) + proc inc(a: var int, b: int) = + system.inc(a, b) + proc inc(a: var int64, b: int) = + system.inc(a, b) + {.pop.} + when defined(posix): import posix type CTime = posix.Time - proc posix_gettimeofday(tp: var Timeval, unused: pointer = nil) {. + var CLOCK_REALTIME {.importc: "CLOCK_REALTIME", header: "<time.h>".}: Clockid + + proc gettimeofday(tp: var Timeval, unused: pointer = nil) {. importc: "gettimeofday", header: "<sys/time.h>".} when not defined(freebsd) and not defined(netbsd) and not defined(openbsd): @@ -53,34 +170,49 @@ when defined(posix): elif defined(windows): import winlean - # newest version of Visual C++ defines time_t to be of 64 bits - type CTime {.importc: "time_t", header: "<time.h>".} = distinct int64 + when defined(i386) and defined(gcc): + type CTime {.importc: "time_t", header: "<time.h>".} = distinct int32 + else: + # newest version of Visual C++ defines time_t to be of 64 bits + type CTime {.importc: "time_t", header: "<time.h>".} = distinct int64 # visual c's c runtime exposes these under a different name var timezone {.importc: "_timezone", header: "<time.h>".}: int type Month* = enum ## Represents a month. Note that the enum starts at ``1``, so ``ord(month)`` will give ## the month number in the range ``[1..12]``. - mJan = 1, mFeb, mMar, mApr, mMay, mJun, mJul, mAug, mSep, mOct, mNov, mDec + mJan = (1, "January") + mFeb = "February" + mMar = "March" + mApr = "April" + mMay = "May" + mJun = "June" + mJul = "July" + mAug = "August" + mSep = "September" + mOct = "October" + mNov = "November" + mDec = "December" WeekDay* = enum ## Represents a weekday. - dMon, dTue, dWed, dThu, dFri, dSat, dSun + dMon = "Monday" + dTue = "Tuesday" + dWed = "Wednesday" + dThu = "Thursday" + dFri = "Friday" + dSat = "Saturday" + dSun = "Sunday" MonthdayRange* = range[1..31] HourRange* = range[0..23] MinuteRange* = range[0..59] SecondRange* = range[0..60] YeardayRange* = range[0..365] + NanosecondRange* = range[0..999_999_999] - TimeImpl = int64 - - Time* = distinct TimeImpl ## Represents a point in time. - ## This is currently implemented as a ``int64`` representing - ## seconds since ``1970-01-01T00:00:00Z``, but don't - ## rely on this knowledge because it might change - ## in the future to allow for higher precision. - ## Use the procs ``toUnix`` and ``fromUnix`` to - ## work with unix timestamps instead. + Time* = object ## Represents a point in time. + seconds: int64 + nanosecond: NanosecondRange DateTime* = object of RootObj ## Represents a time in different parts. ## Although this type can represent leap @@ -89,6 +221,8 @@ type ## but the ``DateTime``'s returned by ## procedures in this module will never have ## a leap second. + nanosecond*: NanosecondRange ## The number of nanoseconds after the second, + ## in the range 0 to 999_999_999. second*: SecondRange ## The number of seconds after the minute, ## normally in the range 0 to 59, but can ## be up to 60 to allow for a leap second. @@ -111,33 +245,58 @@ type ## of the one in a formatted offset string like ``+01:00`` ## (which would be parsed into the UTC offset ``-3600``). - TimeInterval* = object ## Represents a duration of time. Can be used to add and subtract - ## from a ``DateTime`` or ``Time``. - ## Note that a ``TimeInterval`` doesn't represent a fixed duration of time, + TimeInterval* = object ## Represents a non-fixed duration of time. Can be used to add and subtract + ## non-fixed time units from a ``DateTime`` or ``Time``. + ## ``TimeInterval`` doesn't represent a fixed duration of time, ## since the duration of some units depend on the context (e.g a year ## can be either 365 or 366 days long). The non-fixed time units are years, ## months and days. + + nanoseconds*: int ## The number of nanoseconds + microseconds*: int ## The number of microseconds milliseconds*: int ## The number of milliseconds - seconds*: int ## The number of seconds - minutes*: int ## The number of minutes - hours*: int ## The number of hours - days*: int ## The number of days - months*: int ## The number of months - years*: int ## The number of years - - Timezone* = object ## Timezone interface for supporting ``DateTime``'s of arbritary timezones. - ## The ``times`` module only supplies implementations for the systems local time and UTC. - ## The members ``zoneInfoFromUtc`` and ``zoneInfoFromTz`` should not be accessed directly - ## and are only exported so that ``Timezone`` can be implemented by other modules. - zoneInfoFromUtc*: proc (time: Time): ZonedTime {.tags: [], raises: [], benign.} - zoneInfoFromTz*: proc (adjTime: Time): ZonedTime {.tags: [], raises: [], benign.} - name*: string ## The name of the timezone, f.ex 'Europe/Stockholm' or 'Etc/UTC'. Used for checking equality. - ## Se also: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones - ZonedTime* = object ## Represents a zooned instant in time that is not associated with any calendar. - ## This type is only used for implementing timezones. - adjTime*: Time ## Time adjusted to a timezone. - utcOffset*: int - isDst*: bool + seconds*: int ## The number of seconds + minutes*: int ## The number of minutes + hours*: int ## The number of hours + days*: int ## The number of days + weeks*: int ## The number of weeks + months*: int ## The number of months + years*: int ## The number of years + + Duration* = object ## Represents a fixed duration of time. + ## Uses the same time resolution as ``Time``. + ## This type should be prefered over ``TimeInterval`` unless + ## non-static time units is needed. + seconds: int64 + nanosecond: NanosecondRange + + TimeUnit* = enum ## Different units of time. + Nanoseconds, Microseconds, Milliseconds, Seconds, Minutes, Hours, Days, Weeks, Months, Years + + FixedTimeUnit* = range[Nanoseconds..Weeks] ## Subrange of ``TimeUnit`` that only includes units of fixed duration. + ## These are the units that can be represented by a ``Duration``. + + Timezone* = ref object ## \ + ## Timezone interface for supporting ``DateTime``'s of arbritary + ## timezones. The ``times`` module only supplies implementations for the + ## systems local time and UTC. + zonedTimeFromTimeImpl: proc (x: Time): ZonedTime + {.tags: [], raises: [], benign.} + zonedTimeFromAdjTimeImpl: proc (x: Time): ZonedTime + {.tags: [], raises: [], benign.} + name: string + + ZonedTime* = object ## Represents a point in time with an associated + ## UTC offset and DST flag. This type is only used for + ## implementing timezones. + time*: Time ## The point in time being represented. + utcOffset*: int ## The offset in seconds west of UTC, + ## including any offset due to DST. + isDst*: bool ## Determines whether DST is in effect. + + DurationParts* = array[FixedTimeUnit, int64] # Array of Duration parts starts + TimeIntervalParts* = array[TimeUnit, int] # Array of Duration parts starts + TimesMutableTypes = DateTime | Time | Duration | TimeInterval {.deprecated: [TMonth: Month, TWeekDay: WeekDay, TTime: Time, TTimeInterval: TimeInterval, TTimeInfo: DateTime, TimeInfo: DateTime].} @@ -147,14 +306,136 @@ const secondsInHour = 60*60 secondsInDay = 60*60*24 minutesInHour = 60 + rateDiff = 10000000'i64 # 100 nsecs + # The number of hectonanoseconds between 1601/01/01 (windows epoch) + # and 1970/01/01 (unix epoch). + epochDiff = 116444736000000000'i64 + +const unitWeights: array[FixedTimeUnit, int64] = [ + 1'i64, + 1000, + 1_000_000, + 1e9.int64, + secondsInMin * 1e9.int64, + secondsInHour * 1e9.int64, + secondsInDay * 1e9.int64, + 7 * secondsInDay * 1e9.int64, +] + +proc convert*[T: SomeInteger](unitFrom, unitTo: FixedTimeUnit, quantity: T): T {.inline.} = + ## Convert a quantity of some duration unit to another duration unit. + runnableExamples: + doAssert convert(Days, Hours, 2) == 48 + doAssert convert(Days, Weeks, 13) == 1 # Truncated + doAssert convert(Seconds, Milliseconds, -1) == -1000 + if unitFrom < unitTo: + (quantity div (unitWeights[unitTo] div unitWeights[unitFrom])).T + else: + ((unitWeights[unitFrom] div unitWeights[unitTo]) * quantity).T + +proc normalize[T: Duration|Time](seconds, nanoseconds: int64): T = + ## Normalize a (seconds, nanoseconds) pair and return it as either + ## a ``Duration`` or ``Time``. A normalized ``Duration|Time`` has a + ## positive nanosecond part in the range ``NanosecondRange``. + result.seconds = seconds + convert(Nanoseconds, Seconds, nanoseconds) + var nanosecond = nanoseconds mod convert(Seconds, Nanoseconds, 1) + if nanosecond < 0: + nanosecond += convert(Seconds, Nanoseconds, 1) + result.seconds -= 1 + result.nanosecond = nanosecond.int + +# Forward declarations +proc utcTzInfo(time: Time): ZonedTime {.tags: [], raises: [], benign .} +proc localZonedTimeFromTime(time: Time): ZonedTime {.tags: [], raises: [], benign .} +proc localZonedTimeFromAdjTime(adjTime: Time): ZonedTime {.tags: [], raises: [], benign .} +proc initTime*(unix: int64, nanosecond: NanosecondRange): Time + {.tags: [], raises: [], benign noSideEffect.} + +proc initDuration*(nanoseconds, microseconds, milliseconds, + seconds, minutes, hours, days, weeks: int64 = 0): Duration + {.tags: [], raises: [], benign noSideEffect.} + +proc nanosecond*(time: Time): NanosecondRange = + ## Get the fractional part of a ``Time`` as the number + ## of nanoseconds of the second. + time.nanosecond + + +proc weeks*(dur: Duration): int64 {.inline.} = + ## Number of whole weeks represented by the duration. + convert(Seconds, Weeks, dur.seconds) + +proc days*(dur: Duration): int64 {.inline.} = + ## Number of whole days represented by the duration. + convert(Seconds, Days, dur.seconds) + +proc minutes*(dur: Duration): int64 {.inline.} = + ## Number of whole minutes represented by the duration. + convert(Seconds, Minutes, dur.seconds) + +proc hours*(dur: Duration): int64 {.inline.} = + ## Number of whole hours represented by the duration. + convert(Seconds, Hours, dur.seconds) + +proc seconds*(dur: Duration): int64 {.inline.} = + ## Number of whole seconds represented by the duration. + dur.seconds + +proc milliseconds*(dur: Duration): int {.inline.} = + ## Number of whole milliseconds represented by the **fractional** + ## part of the duration. + runnableExamples: + let dur = initDuration(seconds = 1, milliseconds = 1) + doAssert dur.milliseconds == 1 + convert(Nanoseconds, Milliseconds, dur.nanosecond) + +proc microseconds*(dur: Duration): int {.inline.} = + ## Number of whole microseconds represented by the **fractional** + ## part of the duration. + runnableExamples: + let dur = initDuration(seconds = 1, microseconds = 1) + doAssert dur.microseconds == 1 + convert(Nanoseconds, Microseconds, dur.nanosecond) + +proc nanoseconds*(dur: Duration): int {.inline.} = + ## Number of whole nanoseconds represented by the **fractional** + ## part of the duration. + runnableExamples: + let dur = initDuration(seconds = 1, nanoseconds = 1) + doAssert dur.nanoseconds == 1 + dur.nanosecond + +proc fractional*(dur: Duration): Duration {.inline.} = + ## The fractional part of duration, as a duration. + runnableExamples: + let dur = initDuration(seconds = 1, nanoseconds = 5) + doAssert dur.fractional == initDuration(nanoseconds = 5) + initDuration(nanoseconds = dur.nanosecond) + proc fromUnix*(unix: int64): Time {.benign, tags: [], raises: [], noSideEffect.} = ## Convert a unix timestamp (seconds since ``1970-01-01T00:00:00Z``) to a ``Time``. - Time(unix) + runnableExamples: + doAssert $fromUnix(0).utc == "1970-01-01T00:00:00Z" + initTime(unix, 0) proc toUnix*(t: Time): int64 {.benign, tags: [], raises: [], noSideEffect.} = ## Convert ``t`` to a unix timestamp (seconds since ``1970-01-01T00:00:00Z``). - t.int64 + runnableExamples: + doAssert fromUnix(0).toUnix() == 0 + t.seconds + +proc fromWinTime*(win: int64): Time = + ## Convert a Windows file time (100-nanosecond intervals since ``1601-01-01T00:00:00Z``) + ## to a ``Time``. + const hnsecsPerSec = convert(Seconds, Nanoseconds, 1) div 100 + let nanos = floorMod(win, hnsecsPerSec) * 100 + let seconds = floorDiv(win - epochDiff, hnsecsPerSec) + result = initTime(seconds, nanos) + +proc toWinTime*(t: Time): int64 = + ## Convert ``t`` to a Windows file time (100-nanosecond intervals since ``1601-01-01T00:00:00Z``). + result = t.seconds * rateDiff + epochDiff + t.nanosecond div 100 proc isLeapYear*(year: int): bool = ## Returns true if ``year`` is a leap year. @@ -174,7 +455,7 @@ proc getDaysInYear*(year: int): int = proc assertValidDate(monthday: MonthdayRange, month: Month, year: int) {.inline.} = assert monthday <= getDaysInMonth(month, year), - $year & "-" & $ord(month) & "-" & $monthday & " is not a valid date" + $year & "-" & intToStr(ord(month), 2) & "-" & $monthday & " is not a valid date" proc toEpochDay(monthday: MonthdayRange, month: Month, year: int): int64 = ## Get the epoch day from a year/month/day date. @@ -209,7 +490,7 @@ proc fromEpochDay(epochday: int64): tuple[monthday: MonthdayRange, month: Month, proc getDayOfYear*(monthday: MonthdayRange, month: Month, year: int): YeardayRange {.tags: [], raises: [], benign .} = ## Returns the day of the year. - ## Equivalent with ``initDateTime(day, month, year).yearday``. + ## Equivalent with ``initDateTime(monthday, month, year, 0, 0, 0).yearday``. assertValidDate monthday, month, year const daysUntilMonth: array[Month, int] = [0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334] const daysUntilMonthLeap: array[Month, int] = [0, 31, 60, 91, 121, 152, 182, 213, 244, 274, 305, 335] @@ -221,63 +502,268 @@ proc getDayOfYear*(monthday: MonthdayRange, month: Month, year: int): YeardayRan proc getDayOfWeek*(monthday: MonthdayRange, month: Month, year: int): WeekDay {.tags: [], raises: [], benign .} = ## Returns the day of the week enum from day, month and year. - ## Equivalent with ``initDateTime(day, month, year).weekday``. + ## Equivalent with ``initDateTime(monthday, month, year, 0, 0, 0).weekday``. assertValidDate monthday, month, year # 1970-01-01 is a Thursday, we adjust to the previous Monday let days = toEpochday(monthday, month, year) - 3 - let weeks = (if days >= 0: days else: days - 6) div 7 + let weeks = floorDiv(days, 7) let wd = days - weeks * 7 # The value of d is 0 for a Sunday, 1 for a Monday, 2 for a Tuesday, etc. # so we must correct for the WeekDay type. result = if wd == 0: dSun else: WeekDay(wd - 1) -# Forward declarations -proc utcZoneInfoFromUtc(time: Time): ZonedTime {.tags: [], raises: [], benign .} -proc utcZoneInfoFromTz(adjTime: Time): ZonedTime {.tags: [], raises: [], benign .} -proc localZoneInfoFromUtc(time: Time): ZonedTime {.tags: [], raises: [], benign .} -proc localZoneInfoFromTz(adjTime: Time): ZonedTime {.tags: [], raises: [], benign .} - -proc `-`*(a, b: Time): int64 {. - rtl, extern: "ntDiffTime", tags: [], raises: [], noSideEffect, benign, deprecated.} = - ## Computes the difference of two calendar times. Result is in seconds. - ## This is deprecated because it will need to change when sub second time resolution is implemented. - ## Use ``a.toUnix - b.toUnix`` instead. + +{. pragma: operator, rtl, noSideEffect, benign .} + +template subImpl[T: Duration|Time](a: Duration|Time, b: Duration|Time): T = + normalize[T](a.seconds - b.seconds, a.nanosecond - b.nanosecond) + +template addImpl[T: Duration|Time](a: Duration|Time, b: Duration|Time): T = + normalize[T](a.seconds + b.seconds, a.nanosecond + b.nanosecond) + +template ltImpl(a: Duration|Time, b: Duration|Time): bool = + a.seconds < b.seconds or ( + a.seconds == b.seconds and a.nanosecond < b.nanosecond) + +template lqImpl(a: Duration|Time, b: Duration|Time): bool = + a.seconds < b.seconds or ( + a.seconds == b.seconds and a.nanosecond <= b.nanosecond) + +template eqImpl(a: Duration|Time, b: Duration|Time): bool = + a.seconds == b.seconds and a.nanosecond == b.nanosecond + +proc initDuration*(nanoseconds, microseconds, milliseconds, + seconds, minutes, hours, days, weeks: int64 = 0): Duration = + runnableExamples: + let dur = initDuration(seconds = 1, milliseconds = 1) + doAssert dur.milliseconds == 1 + doAssert dur.seconds == 1 + + let seconds = convert(Weeks, Seconds, weeks) + + convert(Days, Seconds, days) + + convert(Minutes, Seconds, minutes) + + convert(Hours, Seconds, hours) + + convert(Seconds, Seconds, seconds) + + convert(Milliseconds, Seconds, milliseconds) + + convert(Microseconds, Seconds, microseconds) + + convert(Nanoseconds, Seconds, nanoseconds) + let nanoseconds = (convert(Milliseconds, Nanoseconds, milliseconds mod 1000) + + convert(Microseconds, Nanoseconds, microseconds mod 1_000_000) + + nanoseconds mod 1_000_000_000).int + # Nanoseconds might be negative so we must normalize. + result = normalize[Duration](seconds, nanoseconds) + +const DurationZero* = initDuration() ## \ + ## Zero value for durations. Useful for comparisons. ## ## .. code-block:: nim - ## let a = fromSeconds(1_000_000_000) - ## let b = fromSeconds(1_500_000_000) - ## echo initInterval(seconds=int(b - a)) - ## # (milliseconds: 0, seconds: 20, minutes: 53, hours: 0, days: 5787, months: 0, years: 0) - a.toUnix - b.toUnix - -proc `<`*(a, b: Time): bool {. - rtl, extern: "ntLtTime", tags: [], raises: [], noSideEffect, borrow.} + ## + ## doAssert initDuration(seconds = 1) > DurationZero + ## doAssert initDuration(seconds = 0) == DurationZero + +proc toParts*(dur: Duration): DurationParts = + ## Converts a duration into an array consisting of fixed time units. + ## + ## Each value in the array gives information about a specific unit of + ## time, for example ``result[Days]`` gives a count of days. + ## + ## This procedure is useful for converting ``Duration`` values to strings. + runnableExamples: + var dp = toParts(initDuration(weeks=2, days=1)) + doAssert dp[Days] == 1 + doAssert dp[Weeks] == 2 + dp = toParts(initDuration(days = -1)) + doAssert dp[Days] == -1 + + var remS = dur.seconds + var remNs = dur.nanosecond.int + + # Ensure the same sign for seconds and nanoseconds + if remS < 0 and remNs != 0: + remNs -= convert(Seconds, Nanoseconds, 1) + remS.inc 1 + + for unit in countdown(Weeks, Seconds): + let quantity = convert(Seconds, unit, remS) + remS = remS mod convert(unit, Seconds, 1) + + result[unit] = quantity + + for unit in countdown(Milliseconds, Nanoseconds): + let quantity = convert(Nanoseconds, unit, remNs) + remNs = remNs mod convert(unit, Nanoseconds, 1) + + result[unit] = quantity + +proc stringifyUnit(value: int | int64, unit: TimeUnit): string = + ## Stringify time unit with it's name, lowercased + let strUnit = $unit + result = "" + result.add($value) + result.add(" ") + if abs(value) != 1: + result.add(strUnit.toLowerAscii()) + else: + result.add(strUnit[0..^2].toLowerAscii()) + +proc humanizeParts(parts: seq[string]): string = + ## Make date string parts human-readable + result = "" + if parts.len == 0: + result.add "0 nanoseconds" + elif parts.len == 1: + result = parts[0] + elif parts.len == 2: + result = parts[0] & " and " & parts[1] + else: + for i in 0..high(parts)-1: + result.add parts[i] & ", " + result.add "and " & parts[high(parts)] + +proc `$`*(dur: Duration): string = + ## Human friendly string representation of ``Duration``. + runnableExamples: + doAssert $initDuration(seconds = 2) == "2 seconds" + doAssert $initDuration(weeks = 1, days = 2) == "1 week and 2 days" + doAssert $initDuration(hours = 1, minutes = 2, seconds = 3) == "1 hour, 2 minutes, and 3 seconds" + doAssert $initDuration(milliseconds = -1500) == "-1 second and -500 milliseconds" + var parts = newSeq[string]() + var numParts = toParts(dur) + + for unit in countdown(Weeks, Nanoseconds): + let quantity = numParts[unit] + if quantity != 0.int64: + parts.add(stringifyUnit(quantity, unit)) + + result = humanizeParts(parts) + +proc `+`*(a, b: Duration): Duration {.operator.} = + ## Add two durations together. + runnableExamples: + doAssert initDuration(seconds = 1) + initDuration(days = 1) == + initDuration(seconds = 1, days = 1) + addImpl[Duration](a, b) + +proc `-`*(a, b: Duration): Duration {.operator.} = + ## Subtract a duration from another. + runnableExamples: + doAssert initDuration(seconds = 1, days = 1) - initDuration(seconds = 1) == + initDuration(days = 1) + subImpl[Duration](a, b) + +proc `-`*(a: Duration): Duration {.operator.} = + ## Reverse a duration. + runnableExamples: + doAssert -initDuration(seconds = 1) == initDuration(seconds = -1) + normalize[Duration](-a.seconds, -a.nanosecond) + +proc `<`*(a, b: Duration): bool {.operator.} = + ## Note that a duration can be negative, + ## so even if ``a < b`` is true ``a`` might + ## represent a larger absolute duration. + ## Use ``abs(a) < abs(b)`` to compare the absolute + ## duration. + runnableExamples: + doAssert initDuration(seconds = 1) < initDuration(seconds = 2) + doAssert initDuration(seconds = -2) < initDuration(seconds = 1) + ltImpl(a, b) + +proc `<=`*(a, b: Duration): bool {.operator.} = + lqImpl(a, b) + +proc `==`*(a, b: Duration): bool {.operator.} = + eqImpl(a, b) + +proc `*`*(a: int64, b: Duration): Duration {.operator} = + ## Multiply a duration by some scalar. + runnableExamples: + doAssert 5 * initDuration(seconds = 1) == initDuration(seconds = 5) + normalize[Duration](a * b.seconds, a * b.nanosecond) + +proc `*`*(a: Duration, b: int64): Duration {.operator} = + ## Multiply a duration by some scalar. + runnableExamples: + doAssert initDuration(seconds = 1) * 5 == initDuration(seconds = 5) + b * a + +proc `div`*(a: Duration, b: int64): Duration {.operator} = + ## Integer division for durations. + runnableExamples: + doAssert initDuration(seconds = 3) div 2 == initDuration(milliseconds = 1500) + doAssert initDuration(nanoseconds = 3) div 2 == initDuration(nanoseconds = 1) + let carryOver = convert(Seconds, Nanoseconds, a.seconds mod b) + normalize[Duration](a.seconds div b, (a.nanosecond + carryOver) div b) + +proc initTime*(unix: int64, nanosecond: NanosecondRange): Time = + ## Create a ``Time`` from a unix timestamp and a nanosecond part. + result.seconds = unix + result.nanosecond = nanosecond + +proc `-`*(a, b: Time): Duration {.operator, extern: "ntDiffTime".} = + ## Computes the duration between two points in time. + subImpl[Duration](a, b) + +proc `+`*(a: Time, b: Duration): Time {.operator, extern: "ntAddTime".} = + ## Add a duration of time to a ``Time``. + runnableExamples: + doAssert (fromUnix(0) + initDuration(seconds = 1)) == fromUnix(1) + addImpl[Time](a, b) + +proc `-`*(a: Time, b: Duration): Time {.operator, extern: "ntSubTime".} = + ## Subtracts a duration of time from a ``Time``. + runnableExamples: + doAssert (fromUnix(0) - initDuration(seconds = 1)) == fromUnix(-1) + subImpl[Time](a, b) + +proc `<`*(a, b: Time): bool {.operator, extern: "ntLtTime".} = ## Returns true iff ``a < b``, that is iff a happened before b. + ltImpl(a, b) -proc `<=` * (a, b: Time): bool {. - rtl, extern: "ntLeTime", tags: [], raises: [], noSideEffect, borrow.} +proc `<=` * (a, b: Time): bool {.operator, extern: "ntLeTime".} = ## Returns true iff ``a <= b``. + lqImpl(a, b) -proc `==`*(a, b: Time): bool {. - rtl, extern: "ntEqTime", tags: [], raises: [], noSideEffect, borrow.} +proc `==`*(a, b: Time): bool {.operator, extern: "ntEqTime".} = ## Returns true if ``a == b``, that is if both times represent the same point in time. + eqImpl(a, b) + +proc high*(typ: typedesc[Time]): Time = + initTime(high(int64), high(NanosecondRange)) + +proc low*(typ: typedesc[Time]): Time = + initTime(low(int64), 0) + +proc high*(typ: typedesc[Duration]): Duration = + ## Get the longest representable duration. + initDuration(seconds = high(int64), nanoseconds = high(NanosecondRange)) + +proc low*(typ: typedesc[Duration]): Duration = + ## Get the longest representable duration of negative direction. + initDuration(seconds = low(int64)) + +proc abs*(a: Duration): Duration = + runnableExamples: + doAssert initDuration(milliseconds = -1500).abs == + initDuration(milliseconds = 1500) + initDuration(seconds = abs(a.seconds), nanoseconds = -a.nanosecond) proc toTime*(dt: DateTime): Time {.tags: [], raises: [], benign.} = ## Converts a broken-down time structure to ## calendar time representation. let epochDay = toEpochday(dt.monthday, dt.month, dt.year) - result = Time(epochDay * secondsInDay) - result.inc dt.hour * secondsInHour - result.inc dt.minute * 60 - result.inc dt.second - # The code above ignores the UTC offset of `timeInfo`, - # so we need to compensate for that here. - result.inc dt.utcOffset + var seconds = epochDay * secondsInDay + seconds.inc dt.hour * secondsInHour + seconds.inc dt.minute * 60 + seconds.inc dt.second + seconds.inc dt.utcOffset + result = initTime(seconds, dt.nanosecond) proc initDateTime(zt: ZonedTime, zone: Timezone): DateTime = - let adjTime = zt.adjTime.int64 - let epochday = (if adjTime >= 0: adjTime else: adjTime - (secondsInDay - 1)) div secondsInDay - var rem = zt.adjTime.int64 - epochday * secondsInDay + ## Create a new ``DateTime`` using ``ZonedTime`` in the specified timezone. + let adjTime = zt.time - initDuration(seconds = zt.utcOffset) + let s = adjTime.seconds + let epochday = floorDiv(s, secondsInDay) + var rem = s - epochday * secondsInDay let hour = rem div secondsInHour rem = rem - hour * secondsInHour let minute = rem div secondsInMin @@ -293,6 +779,7 @@ proc initDateTime(zt: ZonedTime, zone: Timezone): DateTime = hour: hour, minute: minute, second: second, + nanosecond: zt.time.nanosecond, weekday: getDayOfWeek(d, m, y), yearday: getDayOfYear(d, m, y), isDst: zt.isDst, @@ -300,14 +787,55 @@ proc initDateTime(zt: ZonedTime, zone: Timezone): DateTime = utcOffset: zt.utcOffset ) -proc inZone*(time: Time, zone: Timezone): DateTime {.tags: [], raises: [], benign.} = - ## Break down ``time`` into a ``DateTime`` using ``zone`` as the timezone. - let zoneInfo = zone.zoneInfoFromUtc(time) - result = initDateTime(zoneInfo, zone) +proc newTimezone*( + name: string, + zonedTimeFromTimeImpl: proc (time: Time): ZonedTime {.tags: [], raises: [], benign.}, + zonedTimeFromAdjTimeImpl: proc (adjTime: Time): ZonedTime {.tags: [], raises: [], benign.} + ): Timezone = + ## Create a new ``Timezone``. + ## + ## ``zonedTimeFromTimeImpl`` and ``zonedTimeFromAdjTimeImpl`` is used + ## as the underlying implementations for ``zonedTimeFromTime`` and + ## ``zonedTimeFromAdjTime``. + ## + ## If possible, the name parameter should match the name used in the + ## tz database. If the timezone doesn't exist in the tz database, or if the + ## timezone name is unknown, then any string that describes the timezone + ## unambiguously can be used. Note that the timezones name is used for + ## checking equality! + runnableExamples: + proc utcTzInfo(time: Time): ZonedTime = + ZonedTime(utcOffset: 0, isDst: false, time: time) + let utc = newTimezone("Etc/UTC", utcTzInfo, utcTzInfo) + Timezone( + name: name, + zonedTimeFromTimeImpl: zonedTimeFromTimeImpl, + zonedTimeFromAdjTimeImpl: zonedTimeFromAdjTimeImpl + ) -proc inZone*(dt: DateTime, zone: Timezone): DateTime {.tags: [], raises: [], benign.} = - ## Convert ``dt`` into a ``DateTime`` using ``zone`` as the timezone. - dt.toTime.inZone(zone) +proc name*(zone: Timezone): string = + ## The name of the timezone. + ## + ## If possible, the name will be the name used in the tz database. + ## If the timezone doesn't exist in the tz database, or if the timezone + ## name is unknown, then any string that describes the timezone + ## unambiguously might be used. For example, the string "LOCAL" is used + ## for the systems local timezone. + ## + ## See also: https://en.wikipedia.org/wiki/Tz_database + zone.name + +proc zonedTimeFromTime*(zone: Timezone, time: Time): ZonedTime = + ## Returns the ``ZonedTime`` for some point in time. + zone.zonedTimeFromTimeImpl(time) + +proc zonedTimeFromAdjTime*(zone: TimeZone, adjTime: Time): ZonedTime = + ## Returns the ``ZonedTime`` for some local time. + ## + ## Note that the ``Time`` argument does not represent a point in time, it + ## represent a local time! E.g if ``adjTime`` is ``fromUnix(0)``, it should be + ## interpreted as 1970-01-01T00:00:00 in the ``zone`` timezone, not in UTC. + zone.zonedTimeFromAdjTimeImpl(adjTime) proc `$`*(zone: Timezone): string = ## Returns the name of the timezone. @@ -315,14 +843,27 @@ proc `$`*(zone: Timezone): string = proc `==`*(zone1, zone2: Timezone): bool = ## Two ``Timezone``'s are considered equal if their name is equal. + runnableExamples: + doAssert local() == local() + doAssert local() != utc() zone1.name == zone2.name +proc inZone*(time: Time, zone: Timezone): DateTime {.tags: [], raises: [], benign.} = + ## Convert ``time`` into a ``DateTime`` using ``zone`` as the timezone. + result = initDateTime(zone.zonedTimeFromTime(time), zone) + +proc inZone*(dt: DateTime, zone: Timezone): DateTime {.tags: [], raises: [], benign.} = + ## Returns a ``DateTime`` representing the same point in time as ``dt`` but + ## using ``zone`` as the timezone. + dt.toTime.inZone(zone) + proc toAdjTime(dt: DateTime): Time = let epochDay = toEpochday(dt.monthday, dt.month, dt.year) - result = Time(epochDay * secondsInDay) - result.inc dt.hour * secondsInHour - result.inc dt.minute * secondsInMin - result.inc dt.second + var seconds = epochDay * secondsInDay + seconds.inc dt.hour * secondsInHour + seconds.inc dt.minute * secondsInMin + seconds.inc dt.second + result = initTime(seconds, dt.nanosecond) when defined(JS): type JsDate = object @@ -350,15 +891,15 @@ when defined(JS): proc getYear(js: JsDate): int {.tags: [], raises: [], benign, importcpp.} proc setFullYear(js: JsDate, year: int): void {.tags: [], raises: [], benign, importcpp.} - proc localZoneInfoFromUtc(time: Time): ZonedTime = - let jsDate = newDate(time.float * 1000) + proc localZonedTimeFromTime(time: Time): ZonedTime = + let jsDate = newDate(time.seconds.float * 1000) let offset = jsDate.getTimezoneOffset() * secondsInMin - result.adjTime = Time(time.int64 - offset) + result.time = time result.utcOffset = offset result.isDst = false - proc localZoneInfoFromTz(adjTime: Time): ZonedTime = - let utcDate = newDate(adjTime.float * 1000) + proc localZonedTimeFromAdjTime(adjTime: Time): ZonedTime = + let utcDate = newDate(adjTime.seconds.float * 1000) let localDate = newDate(utcDate.getUTCFullYear(), utcDate.getUTCMonth(), utcDate.getUTCDate(), utcDate.getUTCHours(), utcDate.getUTCMinutes(), utcDate.getUTCSeconds(), 0) @@ -368,8 +909,8 @@ when defined(JS): if utcDate.getUTCFullYear() in 0 .. 99: localDate.setFullYear(utcDate.getUTCFullYear()) - result.adjTime = adjTime result.utcOffset = localDate.getTimezoneOffset() * secondsInMin + result.time = adjTime + initDuration(seconds = result.utcOffset) result.isDst = false else: @@ -399,7 +940,7 @@ else: weekday {.importc: "tm_wday".}, yearday {.importc: "tm_yday".}, isdst {.importc: "tm_isdst".}: cint - when defined(linux) and defined(amd64): + when defined(linux) and defined(amd64) or defined(haiku): gmtoff {.importc: "tm_gmtoff".}: clong zone {.importc: "tm_zone".}: cstring type @@ -407,82 +948,88 @@ else: proc localtime(timer: ptr CTime): StructTmPtr {. importc: "localtime", header: "<time.h>", tags: [].} - proc toAdjTime(tm: StructTm): Time = + proc toAdjUnix(tm: StructTm): int64 = let epochDay = toEpochday(tm.monthday, (tm.month + 1).Month, tm.year.int + 1900) - result = Time(epochDay * secondsInDay) + result = epochDay * secondsInDay result.inc tm.hour * secondsInHour result.inc tm.minute * 60 result.inc tm.second - proc getStructTm(time: Time | int64): StructTm = - let timei64 = time.int64 - var a = - if timei64 < low(CTime): - CTime(low(CTime)) - elif timei64 > high(CTime): - CTime(high(CTime)) - else: - CTime(timei64) - result = localtime(addr(a))[] - - proc localZoneInfoFromUtc(time: Time): ZonedTime = - let tm = getStructTm(time) - let adjTime = tm.toAdjTime - result.adjTime = adjTime - result.utcOffset = (time.toUnix - adjTime.toUnix).int - result.isDst = tm.isdst > 0 - - proc localZoneInfoFromTz(adjTime: Time): ZonedTime = - var adjTimei64 = adjTime.int64 - let past = adjTimei64 - secondsInDay - var tm = getStructTm(past) - let pastOffset = past - tm.toAdjTime.int64 - - let future = adjTimei64 + secondsInDay - tm = getStructTm(future) - let futureOffset = future - tm.toAdjTime.int64 + proc getLocalOffsetAndDst(unix: int64): tuple[offset: int, dst: bool] = + # Windows can't handle unix < 0, so we fall back to unix = 0. + # FIXME: This should be improved by falling back to the WinAPI instead. + when defined(windows): + if unix < 0: + var a = 0.CTime + let tmPtr = localtime(addr(a)) + if not tmPtr.isNil: + let tm = tmPtr[] + return ((0 - tm.toAdjUnix).int, false) + return (0, false) + + var a = unix.CTime + let tmPtr = localtime(addr(a)) + if not tmPtr.isNil: + let tm = tmPtr[] + return ((unix - tm.toAdjUnix).int, tm.isdst > 0) + return (0, false) + + proc localZonedTimeFromTime(time: Time): ZonedTime = + let (offset, dst) = getLocalOffsetAndDst(time.seconds) + result.time = time + result.utcOffset = offset + result.isDst = dst + + proc localZonedTimeFromAdjTime(adjTime: Time): ZonedTime = + var adjUnix = adjTime.seconds + let past = adjUnix - secondsInDay + let (pastOffset, _) = getLocalOffsetAndDst(past) + + let future = adjUnix + secondsInDay + let (futureOffset, _) = getLocalOffsetAndDst(future) var utcOffset: int if pastOffset == futureOffset: utcOffset = pastOffset.int else: if pastOffset > futureOffset: - adjTimei64 -= secondsInHour + adjUnix -= secondsInHour - adjTimei64 += pastOffset - utcOffset = (adjTimei64 - getStructTm(adjTimei64).toAdjTime.int64).int + adjUnix += pastOffset + utcOffset = getLocalOffsetAndDst(adjUnix).offset # This extra roundtrip is needed to normalize any impossible datetimes # as a result of offset changes (normally due to dst) - let utcTime = adjTime.int64 + utcOffset - tm = getStructTm(utcTime) - result.adjTime = tm.toAdjTime - result.utcOffset = (utcTime - result.adjTime.int64).int - result.isDst = tm.isdst > 0 + let utcUnix = adjTime.seconds + utcOffset + let (finalOffset, dst) = getLocalOffsetAndDst(utcUnix) + result.time = initTime(utcUnix, adjTime.nanosecond) + result.utcOffset = finalOffset + result.isDst = dst -proc utcZoneInfoFromUtc(time: Time): ZonedTime = - result.adjTime = time - result.utcOffset = 0 - result.isDst = false +proc utcTzInfo(time: Time): ZonedTime = + ZonedTime(utcOffset: 0, isDst: false, time: time) -proc utcZoneInfoFromTz(adjTime: Time): ZonedTime = - utcZoneInfoFromUtc(adjTime) # adjTime == time since we are in UTC +var utcInstance {.threadvar.}: Timezone +var localInstance {.threadvar.}: Timezone proc utc*(): TimeZone = ## Get the ``Timezone`` implementation for the UTC timezone. - ## - ## .. code-block:: nim - ## doAssert now().utc.timezone == utc() - ## doAssert utc().name == "Etc/UTC" - Timezone(zoneInfoFromUtc: utcZoneInfoFromUtc, zoneInfoFromTz: utcZoneInfoFromTz, name: "Etc/UTC") + runnableExamples: + doAssert now().utc.timezone == utc() + doAssert utc().name == "Etc/UTC" + if utcInstance.isNil: + utcInstance = newTimezone("Etc/UTC", utcTzInfo, utcTzInfo) + result = utcInstance proc local*(): TimeZone = ## Get the ``Timezone`` implementation for the local timezone. - ## - ## .. code-block:: nim - ## doAssert now().timezone == local() - ## doAssert local().name == "LOCAL" - Timezone(zoneInfoFromUtc: localZoneInfoFromUtc, zoneInfoFromTz: localZoneInfoFromTz, name: "LOCAL") + runnableExamples: + doAssert now().timezone == local() + doAssert local().name == "LOCAL" + if localInstance.isNil: + localInstance = newTimezone("LOCAL", localZonedTimeFromTime, + localZonedTimeFromAdjTime) + result = localInstance proc utc*(dt: DateTime): DateTime = ## Shorthand for ``dt.inZone(utc())``. @@ -500,9 +1047,27 @@ proc local*(t: Time): DateTime = ## Shorthand for ``t.inZone(local())``. t.inZone(local()) -proc getTime*(): Time {.tags: [TimeEffect], benign.} - ## Gets the current time as a ``Time`` with second resolution. Use epochTime for higher - ## resolution. +proc getTime*(): Time {.tags: [TimeEffect], benign.} = + ## Gets the current time as a ``Time`` with nanosecond resolution. + when defined(JS): + let millis = newDate().getTime() + let seconds = convert(Milliseconds, Seconds, millis) + let nanos = convert(Milliseconds, Nanoseconds, + millis mod convert(Seconds, Milliseconds, 1).int) + result = initTime(seconds, nanos) + # I'm not entirely certain if freebsd needs to use `gettimeofday`. + elif defined(macosx) or defined(freebsd): + var a: Timeval + gettimeofday(a) + result = initTime(a.tv_sec.int64, convert(Microseconds, Nanoseconds, a.tv_usec.int)) + elif defined(posix): + var ts: Timespec + discard clock_gettime(CLOCK_REALTIME, ts) + result = initTime(ts.tv_sec.int64, ts.tv_nsec.int) + elif defined(windows): + var f: FILETIME + getSystemTimeAsFileTime(f) + result = fromWinTime(rdFileTime(f)) proc now*(): DateTime {.tags: [TimeEffect], benign.} = ## Get the current time as a ``DateTime`` in the local timezone. @@ -510,51 +1075,57 @@ proc now*(): DateTime {.tags: [TimeEffect], benign.} = ## Shorthand for ``getTime().local``. getTime().local -proc initInterval*(milliseconds, seconds, minutes, hours, days, months, - years: int = 0): TimeInterval = +proc initTimeInterval*(nanoseconds, microseconds, milliseconds, + seconds, minutes, hours, + days, weeks, months, years: int = 0): TimeInterval = ## Creates a new ``TimeInterval``. ## ## You can also use the convenience procedures called ``milliseconds``, ## ``seconds``, ``minutes``, ``hours``, ``days``, ``months``, and ``years``. ## - ## Example: - ## - ## .. code-block:: nim - ## - ## let day = initInterval(hours=24) - ## let dt = initDateTime(01, mJan, 2000, 12, 00, 00, utc()) - ## doAssert $(dt + day) == "2000-01-02T12-00-00+00:00" + runnableExamples: + let day = initTimeInterval(hours=24) + let dt = initDateTime(01, mJan, 2000, 12, 00, 00, utc()) + doAssert $(dt + day) == "2000-01-02T12:00:00Z" + result.nanoseconds = nanoseconds + result.microseconds = microseconds result.milliseconds = milliseconds result.seconds = seconds result.minutes = minutes result.hours = hours result.days = days + result.weeks = weeks result.months = months result.years = years proc `+`*(ti1, ti2: TimeInterval): TimeInterval = ## Adds two ``TimeInterval`` objects together. + result.nanoseconds = ti1.nanoseconds + ti2.nanoseconds + result.microseconds = ti1.microseconds + ti2.microseconds result.milliseconds = ti1.milliseconds + ti2.milliseconds result.seconds = ti1.seconds + ti2.seconds result.minutes = ti1.minutes + ti2.minutes result.hours = ti1.hours + ti2.hours result.days = ti1.days + ti2.days + result.weeks = ti1.weeks + ti2.weeks result.months = ti1.months + ti2.months result.years = ti1.years + ti2.years proc `-`*(ti: TimeInterval): TimeInterval = ## Reverses a time interval - ## - ## .. code-block:: nim - ## - ## let day = -initInterval(hours=24) - ## echo day # -> (milliseconds: 0, seconds: 0, minutes: 0, hours: -24, days: 0, months: 0, years: 0) + runnableExamples: + let day = -initTimeInterval(hours=24) + doAssert day.hours == -24 + result = TimeInterval( + nanoseconds: -ti.nanoseconds, + microseconds: -ti.microseconds, milliseconds: -ti.milliseconds, seconds: -ti.seconds, minutes: -ti.minutes, hours: -ti.hours, days: -ti.days, + weeks: -ti.weeks, months: -ti.months, years: -ti.years ) @@ -562,48 +1133,179 @@ proc `-`*(ti: TimeInterval): TimeInterval = proc `-`*(ti1, ti2: TimeInterval): TimeInterval = ## Subtracts TimeInterval ``ti1`` from ``ti2``. ## - ## Time components are compared one-by-one, see output: - ## - ## .. code-block:: nim - ## let a = fromUnix(1_000_000_000) - ## let b = fromUnix(1_500_000_000) - ## echo b.toTimeInterval - a.toTimeInterval - ## # (milliseconds: 0, seconds: -40, minutes: -6, hours: 1, days: 5, months: -2, years: 16) + ## Time components are subtracted one-by-one, see output: + runnableExamples: + let ti1 = initTimeInterval(hours=24) + let ti2 = initTimeInterval(hours=4) + doAssert (ti1 - ti2) == initTimeInterval(hours=20) + result = ti1 + (-ti2) -proc evaluateInterval(dt: DateTime, interval: TimeInterval): tuple[adjDiff, absDiff: int64] = - ## Evaluates how many seconds the interval is worth - ## in the context of ``dt``. - ## The result in split into an adjusted diff and an absolute diff. +proc getDateStr*(): string {.rtl, extern: "nt$1", tags: [TimeEffect].} = + ## Gets the current date as a string of the format ``YYYY-MM-DD``. + var ti = now() + result = $ti.year & '-' & intToStr(ord(ti.month), 2) & + '-' & intToStr(ti.monthday, 2) - var anew = dt - var newinterv = interval +proc getClockStr*(): string {.rtl, extern: "nt$1", tags: [TimeEffect].} = + ## Gets the current clock time as a string of the format ``HH:MM:SS``. + var ti = now() + result = intToStr(ti.hour, 2) & ':' & intToStr(ti.minute, 2) & + ':' & intToStr(ti.second, 2) + +proc toParts* (ti: TimeInterval): TimeIntervalParts = + ## Converts a `TimeInterval` into an array consisting of its time units, + ## starting with nanoseconds and ending with years + ## + ## This procedure is useful for converting ``TimeInterval`` values to strings. + ## E.g. then you need to implement custom interval printing + runnableExamples: + var tp = toParts(initTimeInterval(years=1, nanoseconds=123)) + doAssert tp[Years] == 1 + doAssert tp[Nanoseconds] == 123 + + var index = 0 + for name, value in fieldPairs(ti): + result[index.TimeUnit()] = value + index += 1 + +proc `$`*(ti: TimeInterval): string = + ## Get string representation of `TimeInterval` + runnableExamples: + doAssert $initTimeInterval(years=1, nanoseconds=123) == "1 year and 123 nanoseconds" + doAssert $initTimeInterval() == "0 nanoseconds" + + var parts: seq[string] = @[] + var tiParts = toParts(ti) + for unit in countdown(Years, Nanoseconds): + if tiParts[unit] != 0: + parts.add(stringifyUnit(tiParts[unit], unit)) + + result = humanizeParts(parts) + +proc nanoseconds*(nanos: int): TimeInterval {.inline.} = + ## TimeInterval of ``nanos`` nanoseconds. + initTimeInterval(nanoseconds = nanos) + +proc microseconds*(micros: int): TimeInterval {.inline.} = + ## TimeInterval of ``micros`` microseconds. + initTimeInterval(microseconds = micros) + +proc milliseconds*(ms: int): TimeInterval {.inline.} = + ## TimeInterval of ``ms`` milliseconds. + initTimeInterval(milliseconds = ms) + +proc seconds*(s: int): TimeInterval {.inline.} = + ## TimeInterval of ``s`` seconds. + ## + ## ``echo getTime() + 5.second`` + initTimeInterval(seconds = s) + +proc minutes*(m: int): TimeInterval {.inline.} = + ## TimeInterval of ``m`` minutes. + ## + ## ``echo getTime() + 5.minutes`` + initTimeInterval(minutes = m) + +proc hours*(h: int): TimeInterval {.inline.} = + ## TimeInterval of ``h`` hours. + ## + ## ``echo getTime() + 2.hours`` + initTimeInterval(hours = h) - newinterv.months += interval.years * 12 - var curMonth = anew.month +proc days*(d: int): TimeInterval {.inline.} = + ## TimeInterval of ``d`` days. + ## + ## ``echo getTime() + 2.days`` + initTimeInterval(days = d) + +proc weeks*(w: int): TimeInterval {.inline.} = + ## TimeInterval of ``w`` weeks. + ## + ## ``echo getTime() + 2.weeks`` + initTimeInterval(weeks = w) + +proc months*(m: int): TimeInterval {.inline.} = + ## TimeInterval of ``m`` months. + ## + ## ``echo getTime() + 2.months`` + initTimeInterval(months = m) + +proc years*(y: int): TimeInterval {.inline.} = + ## TimeInterval of ``y`` years. + ## + ## ``echo getTime() + 2.years`` + initTimeInterval(years = y) + +proc evaluateInterval(dt: DateTime, interval: TimeInterval): tuple[adjDur, absDur: Duration] = + ## Evaluates how many nanoseconds the interval is worth + ## in the context of ``dt``. + ## The result in split into an adjusted diff and an absolute diff. + var months = interval.years * 12 + interval.months + var curYear = dt.year + var curMonth = dt.month # Subtracting - if newinterv.months < 0: - for mth in countDown(-1 * newinterv.months, 1): + if months < 0: + for mth in countDown(-1 * months, 1): if curMonth == mJan: curMonth = mDec - anew.year.dec() + curYear.dec else: curMonth.dec() - result.adjDiff -= getDaysInMonth(curMonth, anew.year) * secondsInDay + let days = getDaysInMonth(curMonth, curYear) + result.adjDur = result.adjDur - initDuration(days = days) # Adding else: - for mth in 1 .. newinterv.months: - result.adjDiff += getDaysInMonth(curMonth, anew.year) * secondsInDay + for mth in 1 .. months: + let days = getDaysInMonth(curMonth, curYear) + result.adjDur = result.adjDur + initDuration(days = days) if curMonth == mDec: curMonth = mJan - anew.year.inc() + curYear.inc else: curMonth.inc() - result.adjDiff += newinterv.days * secondsInDay - result.absDiff += newinterv.hours * secondsInHour - result.absDiff += newinterv.minutes * secondsInMin - result.absDiff += newinterv.seconds - result.absDiff += newinterv.milliseconds div 1000 + + result.adjDur = result.adjDur + initDuration( + days = interval.days, + weeks = interval.weeks) + result.absDur = initDuration( + nanoseconds = interval.nanoseconds, + microseconds = interval.microseconds, + milliseconds = interval.milliseconds, + seconds = interval.seconds, + minutes = interval.minutes, + hours = interval.hours) + + +proc initDateTime*(monthday: MonthdayRange, month: Month, year: int, + hour: HourRange, minute: MinuteRange, second: SecondRange, + nanosecond: NanosecondRange, zone: Timezone = local()): DateTime = + ## Create a new ``DateTime`` in the specified timezone. + runnableExamples: + let dt1 = initDateTime(30, mMar, 2017, 00, 00, 00, 00, utc()) + doAssert $dt1 == "2017-03-30T00:00:00Z" + + assertValidDate monthday, month, year + let dt = DateTime( + monthday: monthday, + year: year, + month: month, + hour: hour, + minute: minute, + second: second, + nanosecond: nanosecond + ) + result = initDateTime(zone.zonedTimeFromAdjTime(dt.toAdjTime), zone) + +proc initDateTime*(monthday: MonthdayRange, month: Month, year: int, + hour: HourRange, minute: MinuteRange, second: SecondRange, + zone: Timezone = local()): DateTime = + ## Create a new ``DateTime`` in the specified timezone. + runnableExamples: + let dt1 = initDateTime(30, mMar, 2017, 00, 00, 00, utc()) + doAssert $dt1 == "2017-03-30T00:00:00Z" + initDateTime(monthday, month, year, hour, minute, second, 0, zone) + proc `+`*(dt: DateTime, interval: TimeInterval): DateTime = ## Adds ``interval`` to ``dt``. Components from ``interval`` are added @@ -615,618 +1317,970 @@ proc `+`*(dt: DateTime, interval: TimeInterval): DateTime = ## set to the number of days overflowed. So adding one month to `31 October` will result in `31 November`, ## which will overflow and result in `1 December`. ## - ## .. code-block:: nim - ## let dt = initDateTime(30, mMar, 2017, 00, 00, 00, utc()) - ## doAssert $(dt + 1.months) == "2017-04-30T00:00:00+00:00" - ## # This is correct and happens due to monthday overflow. - ## doAssert $(dt - 1.months) == "2017-03-02T00:00:00+00:00" - let (adjDiff, absDiff) = evaluateInterval(dt, interval) - - if adjDiff.int64 != 0: - let zInfo = dt.timezone.zoneInfoFromTz(Time(dt.toAdjTime.int64 + adjDiff)) - - if absDiff != 0: - let time = Time(zInfo.adjTime.int64 + zInfo.utcOffset + absDiff) - result = initDateTime(dt.timezone.zoneInfoFromUtc(time), dt.timezone) + runnableExamples: + let dt = initDateTime(30, mMar, 2017, 00, 00, 00, utc()) + doAssert $(dt + 1.months) == "2017-04-30T00:00:00Z" + # This is correct and happens due to monthday overflow. + doAssert $(dt - 1.months) == "2017-03-02T00:00:00Z" + let (adjDur, absDur) = evaluateInterval(dt, interval) + + if adjDur != DurationZero: + var zt = dt.timezone.zonedTimeFromAdjTime(dt.toAdjTime + adjDur) + if absDur != DurationZero: + zt = dt.timezone.zonedTimeFromTime(zt.time + absDur) + result = initDateTime(zt, dt.timezone) else: - result = initDateTime(zInfo, dt.timezone) + result = initDateTime(zt, dt.timezone) else: - result = initDateTime(dt.timezone.zoneInfoFromUtc(Time(dt.toTime.int64 + absDiff)), dt.timezone) + var zt = dt.timezone.zonedTimeFromTime(dt.toTime + absDur) + result = initDateTime(zt, dt.timezone) proc `-`*(dt: DateTime, interval: TimeInterval): DateTime = ## Subtract ``interval`` from ``dt``. Components from ``interval`` are subtracted ## in the order of their size, i.e first the ``years`` component, then the ``months`` ## component and so on. The returned ``DateTime`` will have the same timezone as the input. + runnableExamples: + let dt = initDateTime(30, mMar, 2017, 00, 00, 00, utc()) + doAssert $(dt - 5.days) == "2017-03-25T00:00:00Z" + dt + (-interval) -proc getDateStr*(): string {.rtl, extern: "nt$1", tags: [TimeEffect].} = - ## Gets the current date as a string of the format ``YYYY-MM-DD``. - var ti = now() - result = $ti.year & '-' & intToStr(ord(ti.month), 2) & - '-' & intToStr(ti.monthday, 2) +proc `+`*(dt: DateTime, dur: Duration): DateTime = + runnableExamples: + let dt = initDateTime(30, mMar, 2017, 00, 00, 00, utc()) + let dur = initDuration(hours = 5) + doAssert $(dt + dur) == "2017-03-30T05:00:00Z" -proc getClockStr*(): string {.rtl, extern: "nt$1", tags: [TimeEffect].} = - ## Gets the current clock time as a string of the format ``HH:MM:SS``. - var ti = now() - result = intToStr(ti.hour, 2) & ':' & intToStr(ti.minute, 2) & - ':' & intToStr(ti.second, 2) + (dt.toTime + dur).inZone(dt.timezone) -proc `$`*(day: WeekDay): string = - ## Stringify operator for ``WeekDay``. - const lookup: array[WeekDay, string] = ["Monday", "Tuesday", "Wednesday", - "Thursday", "Friday", "Saturday", "Sunday"] - return lookup[day] +proc `-`*(dt: DateTime, dur: Duration): DateTime = + runnableExamples: + let dt = initDateTime(30, mMar, 2017, 00, 00, 00, utc()) + let dur = initDuration(days = 5) + doAssert $(dt - dur) == "2017-03-25T00:00:00Z" -proc `$`*(m: Month): string = - ## Stringify operator for ``Month``. - const lookup: array[Month, string] = ["January", "February", "March", - "April", "May", "June", "July", "August", "September", "October", - "November", "December"] - return lookup[m] + (dt.toTime - dur).inZone(dt.timezone) -proc milliseconds*(ms: int): TimeInterval {.inline.} = - ## TimeInterval of `ms` milliseconds - ## - ## Note: not all time procedures have millisecond resolution - initInterval(milliseconds = ms) +proc `-`*(dt1, dt2: DateTime): Duration = + ## Compute the duration between ``dt1`` and ``dt2``. + runnableExamples: + let dt1 = initDateTime(30, mMar, 2017, 00, 00, 00, utc()) + let dt2 = initDateTime(25, mMar, 2017, 00, 00, 00, utc()) -proc seconds*(s: int): TimeInterval {.inline.} = - ## TimeInterval of `s` seconds - ## - ## ``echo getTime() + 5.second`` - initInterval(seconds = s) + doAssert dt1 - dt2 == initDuration(days = 5) -proc minutes*(m: int): TimeInterval {.inline.} = - ## TimeInterval of `m` minutes - ## - ## ``echo getTime() + 5.minutes`` - initInterval(minutes = m) + dt1.toTime - dt2.toTime -proc hours*(h: int): TimeInterval {.inline.} = - ## TimeInterval of `h` hours - ## - ## ``echo getTime() + 2.hours`` - initInterval(hours = h) +proc `<`*(a, b: DateTime): bool = + ## Returns true iff ``a < b``, that is iff a happened before b. + return a.toTime < b.toTime -proc days*(d: int): TimeInterval {.inline.} = - ## TimeInterval of `d` days - ## - ## ``echo getTime() + 2.days`` - initInterval(days = d) +proc `<=` * (a, b: DateTime): bool = + ## Returns true iff ``a <= b``. + return a.toTime <= b.toTime -proc months*(m: int): TimeInterval {.inline.} = - ## TimeInterval of `m` months - ## - ## ``echo getTime() + 2.months`` - initInterval(months = m) +proc `==`*(a, b: DateTime): bool = + ## Returns true if ``a == b``, that is if both dates represent the same point in time. + return a.toTime == b.toTime -proc years*(y: int): TimeInterval {.inline.} = - ## TimeInterval of `y` years - ## - ## ``echo getTime() + 2.years`` - initInterval(years = y) -proc `+=`*(time: var Time, interval: TimeInterval) = - ## Modifies `time` by adding `interval`. - time = toTime(time.local + interval) +proc isStaticInterval(interval: TimeInterval): bool = + interval.years == 0 and interval.months == 0 and + interval.days == 0 and interval.weeks == 0 -proc `+`*(time: Time, interval: TimeInterval): Time = - ## Adds `interval` to `time` - ## by converting to a ``DateTime`` in the local timezone, - ## adding the interval, and converting back to ``Time``. +proc evaluateStaticInterval(interval: TimeInterval): Duration = + assert interval.isStaticInterval + initDuration(nanoseconds = interval.nanoseconds, + microseconds = interval.microseconds, + milliseconds = interval.milliseconds, + seconds = interval.seconds, + minutes = interval.minutes, + hours = interval.hours) + +proc between*(startDt, endDt: DateTime): TimeInterval = + ## Evaluate difference between two dates in ``TimeInterval`` format, so, it + ## will be relative. ## - ## ``echo getTime() + 1.day`` - result = toTime(time.local + interval) + ## **Warning:** It's not recommended to use ``between`` for ``DateTime's`` in + ## different ``TimeZone's``. + ## ``a + between(a, b) == b`` is only guaranteed when ``a`` and ``b`` are in UTC. + runnableExamples: + var a = initDateTime(year = 2018, month = Month(3), monthday = 25, + hour = 0, minute = 59, second = 59, nanosecond = 1, + zone = utc()).local + var b = initDateTime(year = 2018, month = Month(3), monthday = 25, + hour = 1, minute = 1, second = 1, nanosecond = 0, + zone = utc()).local + doAssert between(a, b) == initTimeInterval( + nanoseconds=999, milliseconds=999, microseconds=999, seconds=1, minutes=1) + + a = parse("2018-01-09T00:00:00+00:00", "yyyy-MM-dd'T'HH:mm:sszzz", utc()) + b = parse("2018-01-10T23:00:00-02:00", "yyyy-MM-dd'T'HH:mm:sszzz") + doAssert between(a, b) == initTimeInterval(hours=1, days=2) + ## Though, here correct answer should be 1 day 25 hours (cause this day in + ## this tz is actually 26 hours). That's why operating different TZ is + ## discouraged + + var startDt = startDt.utc() + var endDt = endDt.utc() + + if endDt == startDt: + return initTimeInterval() + elif endDt < startDt: + return -between(endDt, startDt) + + var coeffs: array[FixedTimeUnit, int64] = unitWeights + var timeParts: array[FixedTimeUnit, int] + for unit in Nanoseconds..Weeks: + timeParts[unit] = 0 + + for unit in Seconds..Days: + coeffs[unit] = coeffs[unit] div unitWeights[Seconds] + + var startTimepart = initTime( + nanosecond = startDt.nanosecond, + unix = startDt.hour * coeffs[Hours] + startDt.minute * coeffs[Minutes] + + startDt.second + ) + var endTimepart = initTime( + nanosecond = endDt.nanosecond, + unix = endDt.hour * coeffs[Hours] + endDt.minute * coeffs[Minutes] + + endDt.second + ) + # We wand timeParts for Seconds..Hours be positive, so we'll borrow one day + if endTimepart < startTimepart: + timeParts[Days] = -1 + + let diffTime = endTimepart - startTimepart + timeParts[Seconds] = diffTime.seconds.int() + #Nanoseconds - preliminary count + timeParts[Nanoseconds] = diffTime.nanoseconds + for unit in countdown(Milliseconds, Microseconds): + timeParts[unit] += timeParts[Nanoseconds] div coeffs[unit].int() + timeParts[Nanoseconds] -= timeParts[unit] * coeffs[unit].int() + + #Counting Seconds .. Hours - final, Days - preliminary + for unit in countdown(Days, Minutes): + timeParts[unit] += timeParts[Seconds] div coeffs[unit].int() + # Here is accounted the borrowed day + timeParts[Seconds] -= timeParts[unit] * coeffs[unit].int() + + # Set Nanoseconds .. Hours in result + result.nanoseconds = timeParts[Nanoseconds] + result.microseconds = timeParts[Microseconds] + result.milliseconds = timeParts[Milliseconds] + result.seconds = timeParts[Seconds] + result.minutes = timeParts[Minutes] + result.hours = timeParts[Hours] + + #Days + if endDt.monthday.int + timeParts[Days] < startDt.monthday.int(): + if endDt.month > 1.Month: + endDt.month -= 1.Month + else: + endDt.month = 12.Month + endDt.year -= 1 + timeParts[Days] += endDt.monthday.int() + getDaysInMonth( + endDt.month, endDt.year) - startDt.monthday.int() + else: + timeParts[Days] += endDt.monthday.int() - + startDt.monthday.int() -proc `-=`*(time: var Time, interval: TimeInterval) = - ## Modifies `time` by subtracting `interval`. - time = toTime(time.local - interval) + result.days = timeParts[Days] -proc `-`*(time: Time, interval: TimeInterval): Time = - ## Subtracts `interval` from Time `time`. - ## - ## ``echo getTime() - 1.day`` - result = toTime(time.local - interval) + #Months + if endDt.month < startDt.month: + result.months = endDt.month.int() + 12 - startDt.month.int() + endDt.year -= 1 + else: + result.months = endDt.month.int() - + startDt.month.int() -proc formatToken(dt: DateTime, token: string, buf: var string) = - ## Helper of the format proc to parse individual tokens. - ## - ## Pass the found token in the user input string, and the buffer where the - ## final string is being built. This has to be a var value because certain - ## formatting tokens require modifying the previous characters. - case token - of "d": - buf.add($dt.monthday) - of "dd": - if dt.monthday < 10: - buf.add("0") - buf.add($dt.monthday) - of "ddd": - buf.add(($dt.weekday)[0 .. 2]) - of "dddd": - buf.add($dt.weekday) - of "h": - buf.add($(if dt.hour > 12: dt.hour - 12 else: dt.hour)) - of "hh": - let amerHour = if dt.hour > 12: dt.hour - 12 else: dt.hour - if amerHour < 10: - buf.add('0') - buf.add($amerHour) - of "H": - buf.add($dt.hour) - of "HH": - if dt.hour < 10: - buf.add('0') - buf.add($dt.hour) - of "m": - buf.add($dt.minute) - of "mm": - if dt.minute < 10: - buf.add('0') - buf.add($dt.minute) - of "M": - buf.add($ord(dt.month)) - of "MM": - if dt.month < mOct: - buf.add('0') - buf.add($ord(dt.month)) - of "MMM": - buf.add(($dt.month)[0..2]) - of "MMMM": - buf.add($dt.month) - of "s": - buf.add($dt.second) - of "ss": - if dt.second < 10: - buf.add('0') - buf.add($dt.second) - of "t": - if dt.hour >= 12: - buf.add('P') - else: buf.add('A') - of "tt": - if dt.hour >= 12: - buf.add("PM") - else: buf.add("AM") - of "y": - var fr = ($dt.year).len()-1 - if fr < 0: fr = 0 - buf.add(($dt.year)[fr .. ($dt.year).len()-1]) - of "yy": - var fr = ($dt.year).len()-2 - if fr < 0: fr = 0 - var fyear = ($dt.year)[fr .. ($dt.year).len()-1] - if fyear.len != 2: fyear = repeat('0', 2-fyear.len()) & fyear - buf.add(fyear) - of "yyy": - var fr = ($dt.year).len()-3 - if fr < 0: fr = 0 - var fyear = ($dt.year)[fr .. ($dt.year).len()-1] - if fyear.len != 3: fyear = repeat('0', 3-fyear.len()) & fyear - buf.add(fyear) - of "yyyy": - var fr = ($dt.year).len()-4 - if fr < 0: fr = 0 - var fyear = ($dt.year)[fr .. ($dt.year).len()-1] - if fyear.len != 4: fyear = repeat('0', 4-fyear.len()) & fyear - buf.add(fyear) - of "yyyyy": - var fr = ($dt.year).len()-5 - if fr < 0: fr = 0 - var fyear = ($dt.year)[fr .. ($dt.year).len()-1] - if fyear.len != 5: fyear = repeat('0', 5-fyear.len()) & fyear - buf.add(fyear) - of "z": - let - nonDstTz = dt.utcOffset - hours = abs(nonDstTz) div secondsInHour - if nonDstTz <= 0: buf.add('+') - else: buf.add('-') - buf.add($hours) - of "zz": - let - nonDstTz = dt.utcOffset - hours = abs(nonDstTz) div secondsInHour - if nonDstTz <= 0: buf.add('+') - else: buf.add('-') - if hours < 10: buf.add('0') - buf.add($hours) - of "zzz": - let - nonDstTz = dt.utcOffset - hours = abs(nonDstTz) div secondsInHour - minutes = (abs(nonDstTz) div secondsInMin) mod minutesInHour - if nonDstTz <= 0: buf.add('+') - else: buf.add('-') - if hours < 10: buf.add('0') - buf.add($hours) - buf.add(':') - if minutes < 10: buf.add('0') - buf.add($minutes) - of "": - discard + # Years + result.years = endDt.year - startDt.year + +proc `+`*(time: Time, interval: TimeInterval): Time = + ## Adds `interval` to `time`. + ## If `interval` contains any years, months, weeks or days the operation + ## is performed in the local timezone. + runnableExamples: + let tm = fromUnix(0) + doAssert tm + 5.seconds == fromUnix(5) + + if interval.isStaticInterval: + time + evaluateStaticInterval(interval) else: - raise newException(ValueError, "Invalid format string: " & token) + toTime(time.local + interval) +proc `-`*(time: Time, interval: TimeInterval): Time = + ## Subtracts `interval` from Time `time`. + ## If `interval` contains any years, months, weeks or days the operation + ## is performed in the local timezone. + runnableExamples: + let tm = fromUnix(5) + doAssert tm - 5.seconds == fromUnix(0) + + if interval.isStaticInterval: + time - evaluateStaticInterval(interval) + else: + toTime(time.local - interval) + +proc `+=`*[T, U: TimesMutableTypes](a: var T, b: U) = + ## Modify ``a`` in place by adding ``b``. + runnableExamples: + var tm = fromUnix(0) + tm += initDuration(seconds = 1) + doAssert tm == fromUnix(1) + a = a + b + +proc `-=`*[T, U: TimesMutableTypes](a: var T, b: U) = + ## Modify ``a`` in place by subtracting ``b``. + runnableExamples: + var tm = fromUnix(5) + tm -= initDuration(seconds = 5) + doAssert tm == fromUnix(0) + a = a - b + +proc `*=`*[T: TimesMutableTypes, U](a: var T, b: U) = + # Mutable type is often multiplied by number + runnableExamples: + var dur = initDuration(seconds = 1) + dur *= 5 + doAssert dur == initDuration(seconds = 5) + + a = a * b -proc format*(dt: DateTime, f: string): string {.tags: [].}= - ## This procedure formats `dt` as specified by `f`. The following format - ## specifiers are available: - ## - ## ========== ================================================================================= ================================================ - ## Specifier Description Example - ## ========== ================================================================================= ================================================ - ## d Numeric value of the day of the month, it will be one or two digits long. ``1/04/2012 -> 1``, ``21/04/2012 -> 21`` - ## dd Same as above, but always two digits. ``1/04/2012 -> 01``, ``21/04/2012 -> 21`` - ## ddd Three letter string which indicates the day of the week. ``Saturday -> Sat``, ``Monday -> Mon`` - ## dddd Full string for the day of the week. ``Saturday -> Saturday``, ``Monday -> Monday`` - ## h The hours in one digit if possible. Ranging from 0-12. ``5pm -> 5``, ``2am -> 2`` - ## hh The hours in two digits always. If the hour is one digit 0 is prepended. ``5pm -> 05``, ``11am -> 11`` - ## H The hours in one digit if possible, randing from 0-24. ``5pm -> 17``, ``2am -> 2`` - ## HH The hours in two digits always. 0 is prepended if the hour is one digit. ``5pm -> 17``, ``2am -> 02`` - ## m The minutes in 1 digit if possible. ``5:30 -> 30``, ``2:01 -> 1`` - ## mm Same as above but always 2 digits, 0 is prepended if the minute is one digit. ``5:30 -> 30``, ``2:01 -> 01`` - ## M The month in one digit if possible. ``September -> 9``, ``December -> 12`` - ## MM The month in two digits always. 0 is prepended. ``September -> 09``, ``December -> 12`` - ## MMM Abbreviated three-letter form of the month. ``September -> Sep``, ``December -> Dec`` - ## MMMM Full month string, properly capitalized. ``September -> September`` - ## s Seconds as one digit if possible. ``00:00:06 -> 6`` - ## ss Same as above but always two digits. 0 is prepended. ``00:00:06 -> 06`` - ## t ``A`` when time is in the AM. ``P`` when time is in the PM. - ## tt Same as above, but ``AM`` and ``PM`` instead of ``A`` and ``P`` respectively. - ## y(yyyy) This displays the year to different digits. You most likely only want 2 or 4 'y's - ## yy Displays the year to two digits. ``2012 -> 12`` - ## yyyy Displays the year to four digits. ``2012 -> 2012`` - ## z Displays the timezone offset from UTC. ``GMT+7 -> +7``, ``GMT-5 -> -5`` - ## zz Same as above but with leading 0. ``GMT+7 -> +07``, ``GMT-5 -> -05`` - ## zzz Same as above but with ``:mm`` where *mm* represents minutes. ``GMT+7 -> +07:00``, ``GMT-5 -> -05:00`` - ## ========== ================================================================================= ================================================ - ## - ## Other strings can be inserted by putting them in ``''``. For example - ## ``hh'->'mm`` will give ``01->56``. The following characters can be - ## inserted without quoting them: ``:`` ``-`` ``(`` ``)`` ``/`` ``[`` ``]`` - ## ``,``. However you don't need to necessarily separate format specifiers, a - ## unambiguous format string like ``yyyyMMddhhmmss`` is valid too. +# +# Parse & format implementation +# - result = "" +type + AmPm = enum + apUnknown, apAm, apPm + + Era = enum + eraUnknown, eraAd, eraBc + + ParsedTime = object + amPm: AmPm + era: Era + year: Option[int] + month: Option[int] + monthday: Option[int] + utcOffset: Option[int] + + # '0' as default for these work fine + # so no need for `Option`. + hour: int + minute: int + second: int + nanosecond: int + + FormatTokenKind = enum + tkPattern, tkLiteral + + FormatPattern {.pure.} = enum + d, dd, ddd, dddd + h, hh, H, HH + m, mm, M, MM, MMM, MMMM + s, ss + fff, ffffff, fffffffff + t, tt + y, yy, yyy, yyyy, yyyyy + YYYY + uuuu + UUUU + z, zz, zzz, zzzz + g + + # This is a special value used to mark literal format values. + # See the doc comment for ``TimeFormat.patterns``. + Lit + + TimeFormat* = object ## Represents a format for parsing and printing + ## time types. + patterns: seq[byte] ## \ + ## Contains the patterns encoded as bytes. + ## Literal values are encoded in a special way. + ## They start with ``Lit.byte``, then the length of the literal, then the + ## raw char values of the literal. For example, the literal `foo` would + ## be encoded as ``@[Lit.byte, 3.byte, 'f'.byte, 'o'.byte, 'o'.byte]``. + formatStr: string + +const FormatLiterals = { ' ', '-', '/', ':', '(', ')', '[', ']', ',' } + +proc `$`*(f: TimeFormat): string = + ## Returns the format string that was used to construct ``f``. + runnableExamples: + let f = initTimeFormat("yyyy-MM-dd") + doAssert $f == "yyyy-MM-dd" + f.formatStr + +proc raiseParseException(f: TimeFormat, input: string, msg: string) = + raise newException(ValueError, + &"Failed to parse '{input}' with format '{f}'. {msg}") + +iterator tokens(f: string): tuple[kind: FormatTokenKind, token: string] = var i = 0 - var currentF = "" - while true: - case f[i] - of ' ', '-', '/', ':', '\'', '\0', '(', ')', '[', ']', ',': - formatToken(dt, currentF, result) + var currToken = "" - currentF = "" - if f[i] == '\0': break + template yieldCurrToken() = + if currToken.len != 0: + yield (tkPattern, currToken) + currToken = "" - if f[i] == '\'': + while i < f.len: + case f[i] + of '\'': + yieldCurrToken() + if i.succ < f.len and f[i.succ] == '\'': + yield (tkLiteral, "'") + i.inc 2 + else: + var token = "" inc(i) # Skip ' - while f[i] != '\'' and f.len-1 > i: - result.add(f[i]) - inc(i) - else: result.add(f[i]) - + while i < f.len and f[i] != '\'': + token.add f[i] + i.inc + + if i > f.high: + raise newException(ValueError, + &"Unclosed ' in time format string. " & + "For a literal ', use ''.") + i.inc + yield (tkLiteral, token) + of FormatLiterals: + yieldCurrToken() + yield (tkLiteral, $f[i]) + i.inc else: # Check if the letter being added matches previous accumulated buffer. - if currentF.len < 1 or currentF[high(currentF)] == f[i]: - currentF.add(f[i]) + if currToken.len == 0 or currToken[0] == f[i]: + currToken.add(f[i]) + i.inc else: - formatToken(dt, currentF, result) - dec(i) # Move position back to re-process the character separately. - currentF = "" - - inc(i) - -proc `$`*(dt: DateTime): string {.tags: [], raises: [], benign.} = - ## Converts a `DateTime` object to a string representation. - ## It uses the format ``yyyy-MM-dd'T'HH-mm-sszzz``. - try: - result = format(dt, "yyyy-MM-dd'T'HH:mm:sszzz") # todo: optimize this - except ValueError: assert false # cannot happen because format string is valid - -proc `$`*(time: Time): string {.tags: [], raises: [], benign.} = - ## converts a `Time` value to a string representation. It will use the local - ## time zone and use the format ``yyyy-MM-dd'T'HH-mm-sszzz``. - $time.local - -{.pop.} - -proc parseToken(dt: var DateTime; token, value: string; j: var int) = - ## Helper of the parse proc to parse individual tokens. - - # Overwrite system.`[]` to raise a ValueError on index out of bounds. - proc `[]`[T, U](s: string, x: HSlice[T, U]): string = - if x.a >= s.len or x.b >= s.len: - raise newException(ValueError, "Value is missing required tokens, got: " & - s) - return system.`[]`(s, x) - - var sv: int - case token - of "d": - var pd = parseInt(value[j..j+1], sv) - dt.monthday = sv - j += pd - of "dd": - dt.monthday = value[j..j+1].parseInt() - j += 2 - of "ddd": - case value[j..j+2].toLowerAscii() - of "sun": dt.weekday = dSun - of "mon": dt.weekday = dMon - of "tue": dt.weekday = dTue - of "wed": dt.weekday = dWed - of "thu": dt.weekday = dThu - of "fri": dt.weekday = dFri - of "sat": dt.weekday = dSat + yield (tkPattern, currToken) + currToken = $f[i] + i.inc + + yieldCurrToken() + +proc stringToPattern(str: string): FormatPattern = + case str + of "d": result = d + of "dd": result = dd + of "ddd": result = ddd + of "dddd": result = dddd + of "h": result = h + of "hh": result = hh + of "H": result = H + of "HH": result = HH + of "m": result = m + of "mm": result = mm + of "M": result = M + of "MM": result = MM + of "MMM": result = MMM + of "MMMM": result = MMMM + of "s": result = s + of "ss": result = ss + of "fff": result = fff + of "ffffff": result = ffffff + of "fffffffff": result = fffffffff + of "t": result = t + of "tt": result = tt + of "y": result = y + of "yy": result = yy + of "yyy": result = yyy + of "yyyy": result = yyyy + of "yyyyy": result = yyyyy + of "YYYY": result = YYYY + of "uuuu": result = uuuu + of "UUUU": result = UUUU + of "z": result = z + of "zz": result = zz + of "zzz": result = zzz + of "zzzz": result = zzzz + of "g": result = g + else: raise newException(ValueError, &"'{str}' is not a valid pattern") + +proc initTimeFormat*(format: string): TimeFormat = + ## Construct a new time format for parsing & formatting time types. + ## + ## See `Parsing and formatting dates`_ for documentation of the + ## ``format`` argument. + runnableExamples: + let f = initTimeFormat("yyyy-MM-dd") + doAssert "2000-01-01" == "2000-01-01".parse(f).format(f) + result.formatStr = format + result.patterns = @[] + for kind, token in format.tokens: + case kind + of tkLiteral: + case token + else: + result.patterns.add(FormatPattern.Lit.byte) + if token.len > 255: + raise newException(ValueError, + "Format literal is to long:" & token) + result.patterns.add(token.len.byte) + for c in token: + result.patterns.add(c.byte) + of tkPattern: + result.patterns.add(stringToPattern(token).byte) + +proc formatPattern(dt: DateTime, pattern: FormatPattern, result: var string) = + template yearOfEra(dt: DateTime): int = + if dt.year <= 0: abs(dt.year) + 1 else: dt.year + + case pattern + of d: + result.add $dt.monthday + of dd: + result.add dt.monthday.intToStr(2) + of ddd: + result.add ($dt.weekday)[0..2] + of dddd: + result.add $dt.weekday + of h: + result.add( + if dt.hour == 0: "12" + elif dt.hour > 12: $(dt.hour - 12) + else: $dt.hour + ) + of hh: + result.add( + if dt.hour == 0: "12" + elif dt.hour > 12: (dt.hour - 12).intToStr(2) + else: dt.hour.intToStr(2) + ) + of H: + result.add $dt.hour + of HH: + result.add dt.hour.intToStr(2) + of m: + result.add $dt.minute + of mm: + result.add dt.minute.intToStr(2) + of M: + result.add $ord(dt.month) + of MM: + result.add ord(dt.month).intToStr(2) + of MMM: + result.add ($dt.month)[0..2] + of MMMM: + result.add $dt.month + of s: + result.add $dt.second + of ss: + result.add dt.second.intToStr(2) + of fff: + result.add(intToStr(convert(Nanoseconds, Milliseconds, dt.nanosecond), 3)) + of ffffff: + result.add(intToStr(convert(Nanoseconds, Microseconds, dt.nanosecond), 6)) + of fffffffff: + result.add(intToStr(dt.nanosecond, 9)) + of t: + result.add if dt.hour >= 12: "P" else: "A" + of tt: + result.add if dt.hour >= 12: "PM" else: "AM" + of y: # Deprecated + result.add $(dt.yearOfEra mod 10) + of yy: + result.add (dt.yearOfEra mod 100).intToStr(2) + of yyy: # Deprecated + result.add (dt.yearOfEra mod 1000).intToStr(3) + of yyyy: + let year = dt.yearOfEra + if year < 10000: + result.add year.intToStr(4) + else: + result.add '+' & $year + of yyyyy: # Deprecated + result.add (dt.yearOfEra mod 100_000).intToStr(5) + of YYYY: + if dt.year < 1: + result.add $(abs(dt.year) + 1) else: - raise newException(ValueError, - "Couldn't parse day of week (ddd), got: " & value[j..j+2]) - j += 3 - of "dddd": - if value.len >= j+6 and value[j..j+5].cmpIgnoreCase("sunday") == 0: - dt.weekday = dSun - j += 6 - elif value.len >= j+6 and value[j..j+5].cmpIgnoreCase("monday") == 0: - dt.weekday = dMon - j += 6 - elif value.len >= j+7 and value[j..j+6].cmpIgnoreCase("tuesday") == 0: - dt.weekday = dTue - j += 7 - elif value.len >= j+9 and value[j..j+8].cmpIgnoreCase("wednesday") == 0: - dt.weekday = dWed - j += 9 - elif value.len >= j+8 and value[j..j+7].cmpIgnoreCase("thursday") == 0: - dt.weekday = dThu - j += 8 - elif value.len >= j+6 and value[j..j+5].cmpIgnoreCase("friday") == 0: - dt.weekday = dFri - j += 6 - elif value.len >= j+8 and value[j..j+7].cmpIgnoreCase("saturday") == 0: - dt.weekday = dSat - j += 8 + result.add $dt.year + of uuuu: + let year = dt.year + if year < 10000 or year < 0: + result.add year.intToStr(4) else: - raise newException(ValueError, - "Couldn't parse day of week (dddd), got: " & value) - of "h", "H": - var pd = parseInt(value[j..j+1], sv) - dt.hour = sv - j += pd - of "hh", "HH": - dt.hour = value[j..j+1].parseInt() - j += 2 - of "m": - var pd = parseInt(value[j..j+1], sv) - dt.minute = sv - j += pd - of "mm": - dt.minute = value[j..j+1].parseInt() - j += 2 - of "M": - var pd = parseInt(value[j..j+1], sv) - dt.month = sv.Month - j += pd - of "MM": - var month = value[j..j+1].parseInt() - j += 2 - dt.month = month.Month - of "MMM": - case value[j..j+2].toLowerAscii(): - of "jan": dt.month = mJan - of "feb": dt.month = mFeb - of "mar": dt.month = mMar - of "apr": dt.month = mApr - of "may": dt.month = mMay - of "jun": dt.month = mJun - of "jul": dt.month = mJul - of "aug": dt.month = mAug - of "sep": dt.month = mSep - of "oct": dt.month = mOct - of "nov": dt.month = mNov - of "dec": dt.month = mDec + result.add '+' & $year + of UUUU: + result.add $dt.year + of z, zz, zzz, zzzz: + if dt.timezone.name == "Etc/UTC": + result.add 'Z' + else: + result.add if -dt.utcOffset >= 0: '+' else: '-' + let absOffset = abs(dt.utcOffset) + case pattern: + of z: + result.add $(absOffset div 3600) + of zz: + result.add (absOffset div 3600).intToStr(2) + of zzz: + let h = (absOffset div 3600).intToStr(2) + let m = ((absOffset div 60) mod 60).intToStr(2) + result.add h & ":" & m + of zzzz: + let absOffset = abs(dt.utcOffset) + let h = (absOffset div 3600).intToStr(2) + let m = ((absOffset div 60) mod 60).intToStr(2) + let s = (absOffset mod 60).intToStr(2) + result.add h & ":" & m & ":" & s + else: assert false + of g: + result.add if dt.year < 1: "BC" else: "AD" + of Lit: assert false # Can't happen + +proc parsePattern(input: string, pattern: FormatPattern, i: var int, + parsed: var ParsedTime): bool = + template takeInt(allowedWidth: Slice[int]): int = + var sv: int + let max = i + allowedWidth.b - 1 + var pd = + if max > input.high: + parseInt(input, sv, i) + else: + parseInt(input[i..max], sv) + if pd notin allowedWidth: + return false + i.inc pd + sv + + template contains[T](t: typedesc[T], i: int): bool = + i in low(t)..high(t) + + result = true + + case pattern + of d: + parsed.monthday = some(takeInt(1..2)) + result = parsed.monthday.get() in MonthdayRange + of dd: + parsed.monthday = some(takeInt(2..2)) + result = parsed.monthday.get() in MonthdayRange + of ddd: + result = input.substr(i, i+2).toLowerAscii() in [ + "sun", "mon", "tue", "wed", "thu", "fri", "sat"] + if result: + i.inc 3 + of dddd: + if input.substr(i, i+5).cmpIgnoreCase("sunday") == 0: + i.inc 6 + elif input.substr(i, i+5).cmpIgnoreCase("monday") == 0: + i.inc 6 + elif input.substr(i, i+6).cmpIgnoreCase("tuesday") == 0: + i.inc 7 + elif input.substr(i, i+8).cmpIgnoreCase("wednesday") == 0: + i.inc 9 + elif input.substr(i, i+7).cmpIgnoreCase("thursday") == 0: + i.inc 8 + elif input.substr(i, i+5).cmpIgnoreCase("friday") == 0: + i.inc 6 + elif input.substr(i, i+7).cmpIgnoreCase("saturday") == 0: + i.inc 8 else: - raise newException(ValueError, - "Couldn't parse month (MMM), got: " & value) - j += 3 - of "MMMM": - if value.len >= j+7 and value[j..j+6].cmpIgnoreCase("january") == 0: - dt.month = mJan - j += 7 - elif value.len >= j+8 and value[j..j+7].cmpIgnoreCase("february") == 0: - dt.month = mFeb - j += 8 - elif value.len >= j+5 and value[j..j+4].cmpIgnoreCase("march") == 0: - dt.month = mMar - j += 5 - elif value.len >= j+5 and value[j..j+4].cmpIgnoreCase("april") == 0: - dt.month = mApr - j += 5 - elif value.len >= j+3 and value[j..j+2].cmpIgnoreCase("may") == 0: - dt.month = mMay - j += 3 - elif value.len >= j+4 and value[j..j+3].cmpIgnoreCase("june") == 0: - dt.month = mJun - j += 4 - elif value.len >= j+4 and value[j..j+3].cmpIgnoreCase("july") == 0: - dt.month = mJul - j += 4 - elif value.len >= j+6 and value[j..j+5].cmpIgnoreCase("august") == 0: - dt.month = mAug - j += 6 - elif value.len >= j+9 and value[j..j+8].cmpIgnoreCase("september") == 0: - dt.month = mSep - j += 9 - elif value.len >= j+7 and value[j..j+6].cmpIgnoreCase("october") == 0: - dt.month = mOct - j += 7 - elif value.len >= j+8 and value[j..j+7].cmpIgnoreCase("november") == 0: - dt.month = mNov - j += 8 - elif value.len >= j+8 and value[j..j+7].cmpIgnoreCase("december") == 0: - dt.month = mDec - j += 8 + result = false + of h, H: + parsed.hour = takeInt(1..2) + result = parsed.hour in HourRange + of hh, HH: + parsed.hour = takeInt(2..2) + result = parsed.hour in HourRange + of m: + parsed.minute = takeInt(1..2) + result = parsed.hour in MinuteRange + of mm: + parsed.minute = takeInt(2..2) + result = parsed.hour in MinuteRange + of M: + let month = takeInt(1..2) + result = month in 1..12 + parsed.month = some(month) + of MM: + let month = takeInt(2..2) + result = month in 1..12 + parsed.month = some(month) + of MMM: + case input.substr(i, i+2).toLowerAscii() + of "jan": parsed.month = some(1) + of "feb": parsed.month = some(2) + of "mar": parsed.month = some(3) + of "apr": parsed.month = some(4) + of "may": parsed.month = some(5) + of "jun": parsed.month = some(6) + of "jul": parsed.month = some(7) + of "aug": parsed.month = some(8) + of "sep": parsed.month = some(9) + of "oct": parsed.month = some(10) + of "nov": parsed.month = some(11) + of "dec": parsed.month = some(12) else: - raise newException(ValueError, - "Couldn't parse month (MMMM), got: " & value) - of "s": - var pd = parseInt(value[j..j+1], sv) - dt.second = sv - j += pd - of "ss": - dt.second = value[j..j+1].parseInt() - j += 2 - of "t": - if value[j] == 'P' and dt.hour > 0 and dt.hour < 12: - dt.hour += 12 - j += 1 - of "tt": - if value[j..j+1] == "PM" and dt.hour > 0 and dt.hour < 12: - dt.hour += 12 - j += 2 - of "yy": + result = false + if result: + i.inc 3 + of MMMM: + if input.substr(i, i+6).cmpIgnoreCase("january") == 0: + parsed.month = some(1) + i.inc 7 + elif input.substr(i, i+7).cmpIgnoreCase("february") == 0: + parsed.month = some(2) + i.inc 8 + elif input.substr(i, i+4).cmpIgnoreCase("march") == 0: + parsed.month = some(3) + i.inc 5 + elif input.substr(i, i+4).cmpIgnoreCase("april") == 0: + parsed.month = some(4) + i.inc 5 + elif input.substr(i, i+2).cmpIgnoreCase("may") == 0: + parsed.month = some(5) + i.inc 3 + elif input.substr(i, i+3).cmpIgnoreCase("june") == 0: + parsed.month = some(6) + i.inc 4 + elif input.substr(i, i+3).cmpIgnoreCase("july") == 0: + parsed.month = some(7) + i.inc 4 + elif input.substr(i, i+5).cmpIgnoreCase("august") == 0: + parsed.month = some(8) + i.inc 6 + elif input.substr(i, i+8).cmpIgnoreCase("september") == 0: + parsed.month = some(9) + i.inc 9 + elif input.substr(i, i+6).cmpIgnoreCase("october") == 0: + parsed.month = some(10) + i.inc 7 + elif input.substr(i, i+7).cmpIgnoreCase("november") == 0: + parsed.month = some(11) + i.inc 8 + elif input.substr(i, i+7).cmpIgnoreCase("december") == 0: + parsed.month = some(12) + i.inc 8 + else: + result = false + of s: + parsed.second = takeInt(1..2) + of ss: + parsed.second = takeInt(2..2) + of fff, ffffff, fffffffff: + let len = ($pattern).len + let v = takeInt(len..len) + parsed.nanosecond = v * 10^(9 - len) + result = parsed.nanosecond in NanosecondRange + of t: + case input[i]: + of 'P': + parsed.amPm = apPm + of 'A': + parsed.amPm = apAm + else: + result = false + i.inc 1 + of tt: + if input.substr(i, i+1).cmpIgnoreCase("AM") == 0: + parsed.amPm = apAM + i.inc 2 + elif input.substr(i, i+1).cmpIgnoreCase("PM") == 0: + parsed.amPm = apPm + i.inc 2 + else: + result = false + of yy: # Assumes current century - var year = value[j..j+1].parseInt() + var year = takeInt(2..2) var thisCen = now().year div 100 - dt.year = thisCen*100 + year - j += 2 - of "yyyy": - dt.year = value[j..j+3].parseInt() - j += 4 - of "z": - dt.isDst = false - if value[j] == '+': - dt.utcOffset = 0 - parseInt($value[j+1]) * secondsInHour - elif value[j] == '-': - dt.utcOffset = parseInt($value[j+1]) * secondsInHour - elif value[j] == 'Z': - dt.utcOffset = 0 - j += 1 - return + parsed.year = some(thisCen*100 + year) + result = year > 0 + of yyyy: + let year = + if input[i] in { '+', '-' }: + takeInt(4..high(int)) + else: + takeInt(4..4) + result = year > 0 + parsed.year = some(year) + of YYYY: + let year = takeInt(1..high(int)) + parsed.year = some(year) + result = year > 0 + of uuuu: + let year = + if input[i] in { '+', '-' }: + takeInt(4..high(int)) + else: + takeInt(4..4) + parsed.year = some(year) + of UUUU: + parsed.year = some(takeInt(1..high(int))) + of z, zz, zzz, zzzz: + case input[i] + of '+', '-': + let sign = if input[i] == '-': 1 else: -1 + i.inc + var offset = 0 + case pattern + of z: + offset = takeInt(1..2) * -3600 + of zz: + offset = takeInt(2..2) * -3600 + of zzz: + offset.inc takeInt(2..2) * 3600 + if input[i] != ':': + return false + i.inc + offset.inc takeInt(2..2) * 60 + of zzzz: + offset.inc takeInt(2..2) * 3600 + if input[i] != ':': + return false + i.inc + offset.inc takeInt(2..2) * 60 + if input[i] != ':': + return false + i.inc + offset.inc takeInt(2..2) + else: assert false + parsed.utcOffset = some(offset * sign) + of 'Z': + parsed.utcOffset = some(0) + i.inc else: - raise newException(ValueError, - "Couldn't parse timezone offset (z), got: " & value[j]) - j += 2 - of "zz": - dt.isDst = false - if value[j] == '+': - dt.utcOffset = 0 - value[j+1..j+2].parseInt() * secondsInHour - elif value[j] == '-': - dt.utcOffset = value[j+1..j+2].parseInt() * secondsInHour - elif value[j] == 'Z': - dt.utcOffset = 0 - j += 1 - return + result = false + of g: + if input.substr(i, i+1).cmpIgnoreCase("BC") == 0: + parsed.era = eraBc + i.inc 2 + elif input.substr(i, i+1).cmpIgnoreCase("AD") == 0: + parsed.era = eraAd + i.inc 2 else: - raise newException(ValueError, - "Couldn't parse timezone offset (zz), got: " & value[j]) - j += 3 - of "zzz": - dt.isDst = false - var factor = 0 - if value[j] == '+': factor = -1 - elif value[j] == '-': factor = 1 - elif value[j] == 'Z': - dt.utcOffset = 0 - j += 1 - return + result = false + of y, yyy, yyyyy: + raise newException(ValueError, + &"The pattern '{pattern}' is only valid for formatting") + of Lit: assert false # Can't happen + +proc toDateTime(p: ParsedTime, zone: Timezone, f: TimeFormat, + input: string): DateTime = + var month = mJan + var year: int + var monthday: int + # `now()` is an expensive call, so we avoid it when possible + (year, month, monthday) = + if p.year.isNone or p.month.isNone or p.monthday.isNone: + let n = now() + (p.year.get(n.year), + p.month.get(n.month.int).Month, + p.monthday.get(n.monthday)) else: - raise newException(ValueError, - "Couldn't parse timezone offset (zzz), got: " & value[j]) - dt.utcOffset = factor * value[j+1..j+2].parseInt() * secondsInHour - j += 4 - dt.utcOffset += factor * value[j..j+1].parseInt() * 60 - j += 2 + (p.year.get(), p.month.get().Month, p.monthday.get()) + + year = + case p.era + of eraUnknown: + year + of eraBc: + if year < 1: + raiseParseException(f, input, + "Expected year to be positive " & + "(use 'UUUU' or 'uuuu' for negative years).") + -year + 1 + of eraAd: + if year < 1: + raiseParseException(f, input, + "Expected year to be positive " & + "(use 'UUUU' or 'uuuu' for negative years).") + year + + let hour = + case p.amPm + of apUnknown: + p.hour + of apAm: + if p.hour notin 1..12: + raiseParseException(f, input, + "AM/PM time must be in the interval 1..12") + if p.hour == 12: 0 else: p.hour + of apPm: + if p.hour notin 1..12: + raiseParseException(f, input, + "AM/PM time must be in the interval 1..12") + if p.hour == 12: p.hour else: p.hour + 12 + let minute = p.minute + let second = p.second + let nanosecond = p.nanosecond + + if monthday > getDaysInMonth(month, year): + raiseParseException(f, input, + $year & "-" & ord(month).intToStr(2) & + "-" & $monthday & " is not a valid date") + + result = DateTime( + year: year, month: month, monthday: monthday, + hour: hour, minute: minute, second: second, nanosecond: nanosecond + ) + + if p.utcOffset.isNone: + # No timezone parsed - assume timezone is `zone` + result = initDateTime(zone.zonedTimeFromAdjTime(result.toAdjTime), zone) else: - # Ignore the token and move forward in the value string by the same length - j += token.len + # Otherwise convert to `zone` + result.utcOffset = p.utcOffset.get() + result = result.toTime.inZone(zone) + +proc format*(dt: DateTime, f: TimeFormat): string {.raises: [].} = + ## Format ``dt`` using the format specified by ``f``. + runnableExamples: + let f = initTimeFormat("yyyy-MM-dd") + let dt = initDateTime(01, mJan, 2000, 00, 00, 00, utc()) + doAssert "2000-01-01" == dt.format(f) + var idx = 0 + while idx <= f.patterns.high: + case f.patterns[idx].FormatPattern + of Lit: + idx.inc + let len = f.patterns[idx] + for i in 1'u8..len: + idx.inc + result.add f.patterns[idx].char + idx.inc + else: + formatPattern(dt, f.patterns[idx].FormatPattern, result = result) + idx.inc -proc parse*(value, layout: string, zone: Timezone = local()): DateTime = - ## This procedure parses a date/time string using the standard format - ## identifiers as listed below. The procedure defaults information not provided - ## in the format string from the running program (month, year, etc). - ## - ## The return value will always be in the `zone` timezone. If no UTC offset was - ## parsed, then the input will be assumed to be specified in the `zone` timezone - ## already, so no timezone conversion will be done in that case. +proc format*(dt: DateTime, f: string): string = + ## Shorthand for constructing a ``TimeFormat`` and using it to format ``dt``. ## - ## ========== ================================================================================= ================================================ - ## Specifier Description Example - ## ========== ================================================================================= ================================================ - ## d Numeric value of the day of the month, it will be one or two digits long. ``1/04/2012 -> 1``, ``21/04/2012 -> 21`` - ## dd Same as above, but always two digits. ``1/04/2012 -> 01``, ``21/04/2012 -> 21`` - ## ddd Three letter string which indicates the day of the week. ``Saturday -> Sat``, ``Monday -> Mon`` - ## dddd Full string for the day of the week. ``Saturday -> Saturday``, ``Monday -> Monday`` - ## h The hours in one digit if possible. Ranging from 0-12. ``5pm -> 5``, ``2am -> 2`` - ## hh The hours in two digits always. If the hour is one digit 0 is prepended. ``5pm -> 05``, ``11am -> 11`` - ## H The hours in one digit if possible, randing from 0-24. ``5pm -> 17``, ``2am -> 2`` - ## HH The hours in two digits always. 0 is prepended if the hour is one digit. ``5pm -> 17``, ``2am -> 02`` - ## m The minutes in 1 digit if possible. ``5:30 -> 30``, ``2:01 -> 1`` - ## mm Same as above but always 2 digits, 0 is prepended if the minute is one digit. ``5:30 -> 30``, ``2:01 -> 01`` - ## M The month in one digit if possible. ``September -> 9``, ``December -> 12`` - ## MM The month in two digits always. 0 is prepended. ``September -> 09``, ``December -> 12`` - ## MMM Abbreviated three-letter form of the month. ``September -> Sep``, ``December -> Dec`` - ## MMMM Full month string, properly capitalized. ``September -> September`` - ## s Seconds as one digit if possible. ``00:00:06 -> 6`` - ## ss Same as above but always two digits. 0 is prepended. ``00:00:06 -> 06`` - ## t ``A`` when time is in the AM. ``P`` when time is in the PM. - ## tt Same as above, but ``AM`` and ``PM`` instead of ``A`` and ``P`` respectively. - ## yy Displays the year to two digits. ``2012 -> 12`` - ## yyyy Displays the year to four digits. ``2012 -> 2012`` - ## z Displays the timezone offset from UTC. ``Z`` is parsed as ``+0`` ``GMT+7 -> +7``, ``GMT-5 -> -5`` - ## zz Same as above but with leading 0. ``GMT+7 -> +07``, ``GMT-5 -> -05`` - ## zzz Same as above but with ``:mm`` where *mm* represents minutes. ``GMT+7 -> +07:00``, ``GMT-5 -> -05:00`` - ## ========== ================================================================================= ================================================ + ## See `Parsing and formatting dates`_ for documentation of the + ## ``format`` argument. + runnableExamples: + let dt = initDateTime(01, mJan, 2000, 00, 00, 00, utc()) + doAssert "2000-01-01" == format(dt, "yyyy-MM-dd") + let dtFormat = initTimeFormat(f) + result = dt.format(dtFormat) + +proc format*(dt: DateTime, f: static[string]): string {.raises: [].} = + ## Overload that validates ``format`` at compile time. + const f2 = initTimeFormat(f) + result = dt.format(f2) + +proc format*(time: Time, f: string, zone: Timezone = local()): string {.tags: [].} = + ## Shorthand for constructing a ``TimeFormat`` and using it to format + ## ``time``. Will use the timezone specified by ``zone``. ## - ## Other strings can be inserted by putting them in ``''``. For example - ## ``hh'->'mm`` will give ``01->56``. The following characters can be - ## inserted without quoting them: ``:`` ``-`` ``(`` ``)`` ``/`` ``[`` ``]`` - ## ``,``. However you don't need to necessarily separate format specifiers, a - ## unambiguous format string like ``yyyyMMddhhmmss`` is valid too. - var i = 0 # pointer for format string - var j = 0 # pointer for value string - var token = "" - # Assumes current day of month, month and year, but time is reset to 00:00:00. Weekday will be reset after parsing. - var dt = now() - dt.hour = 0 - dt.minute = 0 - dt.second = 0 - dt.isDst = true # using this is flag for checking whether a timezone has \ - # been read (because DST is always false when a tz is parsed) - while true: - case layout[i] - of ' ', '-', '/', ':', '\'', '\0', '(', ')', '[', ']', ',': - if token.len > 0: - parseToken(dt, token, value, j) - # Reset token - token = "" - # Break if at end of line - if layout[i] == '\0': break - # Skip separator and everything between single quotes - # These are literals in both the layout and the value string - if layout[i] == '\'': - inc(i) - while layout[i] != '\'' and layout.len-1 > i: - inc(i) - inc(j) - inc(i) - else: - inc(i) - inc(j) + ## See `Parsing and formatting dates`_ for documentation of the + ## ``f`` argument. + runnableExamples: + var dt = initDateTime(01, mJan, 1970, 00, 00, 00, utc()) + var tm = dt.toTime() + doAssert format(tm, "yyyy-MM-dd'T'HH:mm:ss", utc()) == "1970-01-01T00:00:00" + time.inZone(zone).format(f) + +proc format*(time: Time, f: static[string], + zone: Timezone = local()): string {.tags: [].} = + ## Overload that validates ``f`` at compile time. + const f2 = initTimeFormat(f) + result = time.inZone(zone).format(f2) + +proc parse*(input: string, f: TimeFormat, zone: Timezone = local()): DateTime = + ## Parses ``input`` as a ``DateTime`` using the format specified by ``f``. + ## If no UTC offset was parsed, then ``input`` is assumed to be specified in + ## the ``zone`` timezone. If a UTC offset was parsed, the result will be + ## converted to the ``zone`` timezone. + runnableExamples: + let f = initTimeFormat("yyyy-MM-dd") + let dt = initDateTime(01, mJan, 2000, 00, 00, 00, utc()) + doAssert dt == "2000-01-01".parse(f, utc()) + var inpIdx = 0 # Input index + var patIdx = 0 # Pattern index + var parsed: ParsedTime + while inpIdx <= input.high and patIdx <= f.patterns.high: + let pattern = f.patterns[patIdx].FormatPattern + case pattern + of Lit: + patIdx.inc + let len = f.patterns[patIdx] + patIdx.inc + for _ in 1'u8..len: + if input[inpIdx] != f.patterns[patIdx].char: + raiseParseException(f, input, + "Unexpected character: " & input[inpIdx]) + inpIdx.inc + patIdx.inc else: - # Check if the letter being added matches previous accumulated buffer. - if token.len < 1 or token[high(token)] == layout[i]: - token.add(layout[i]) - inc(i) - else: - parseToken(dt, token, value, j) - token = "" + if not parsePattern(input, pattern, inpIdx, parsed): + raiseParseException(f, input, &"Failed on pattern '{pattern}'") + patIdx.inc - if dt.isDst: - # No timezone parsed - assume timezone is `zone` - result = initDateTime(zone.zoneInfoFromTz(dt.toAdjTime), zone) - else: - # Otherwise convert to `zone` - result = dt.toTime.inZone(zone) + if inpIdx <= input.high: + raiseParseException(f, input, + "Parsing ended but there was still input remaining") + + if patIdx <= f.patterns.high: + raiseParseException(f, input, + "Parsing ended but there was still patterns remaining") + + result = toDateTime(parsed, zone, f, input) + +proc parse*(input, f: string, tz: Timezone = local()): DateTime = + ## Shorthand for constructing a ``TimeFormat`` and using it to parse + ## ``input`` as a ``DateTime``. + ## + ## See `Parsing and formatting dates`_ for documentation of the + ## ``f`` argument. + runnableExamples: + let dt = initDateTime(01, mJan, 2000, 00, 00, 00, utc()) + doAssert dt == parse("2000-01-01", "yyyy-MM-dd", utc()) + let dtFormat = initTimeFormat(f) + result = input.parse(dtFormat, tz) + +proc parse*(input: string, f: static[string], zone: Timezone = local()): DateTime = + ## Overload that validates ``f`` at compile time. + const f2 = initTimeFormat(f) + result = input.parse(f2, zone) + +proc parseTime*(input, f: string, zone: Timezone): Time = + ## Shorthand for constructing a ``TimeFormat`` and using it to parse + ## ``input`` as a ``DateTime``, then converting it a ``Time``. + ## + ## See `Parsing and formatting dates`_ for documentation of the + ## ``format`` argument. + runnableExamples: + let tStr = "1970-01-01T00:00:00+00:00" + doAssert parseTime(tStr, "yyyy-MM-dd'T'HH:mm:sszzz", utc()) == fromUnix(0) + parse(input, f, zone).toTime() + +proc parseTime*(input: string, f: static[string], zone: Timezone): Time = + ## Overload that validates ``format`` at compile time. + const f2 = initTimeFormat(f) + result = input.parse(f2, zone).toTime() + +# +# End of parse & format implementation +# + +proc `$`*(dt: DateTime): string {.tags: [], raises: [], benign.} = + ## Converts a `DateTime` object to a string representation. + ## It uses the format ``yyyy-MM-dd'T'HH-mm-sszzz``. + runnableExamples: + let dt = initDateTime(01, mJan, 2000, 12, 00, 00, utc()) + doAssert $dt == "2000-01-01T12:00:00Z" + result = format(dt, "yyyy-MM-dd'T'HH:mm:sszzz") + +proc `$`*(time: Time): string {.tags: [], raises: [], benign.} = + ## converts a `Time` value to a string representation. It will use the local + ## time zone and use the format ``yyyy-MM-dd'T'HH-mm-sszzz``. + runnableExamples: + let dt = initDateTime(01, mJan, 1970, 00, 00, 00, local()) + let tm = dt.toTime() + doAssert $tm == "1970-01-01T00:00:00" & format(dt, "zzz") + $time.local + +{.pop.} proc countLeapYears*(yearSpan: int): int = ## Returns the number of leap years spanned by a given number of years. @@ -1255,93 +2309,53 @@ proc countYearsAndDays*(daySpan: int): tuple[years: int, days: int] = proc toTimeInterval*(time: Time): TimeInterval = ## Converts a Time to a TimeInterval. ## - ## To be used when diffing times. - ## - ## .. code-block:: nim - ## let a = fromSeconds(1_000_000_000) - ## let b = fromSeconds(1_500_000_000) - ## echo a, " ", b # real dates - ## echo a.toTimeInterval # meaningless value, don't use it by itself - ## echo b.toTimeInterval - a.toTimeInterval - ## # (milliseconds: 0, seconds: -40, minutes: -6, hours: 1, days: 5, months: -2, years: 16) - # Milliseconds not available from Time + ## To be used when diffing times. Consider using `between` instead. + runnableExamples: + let a = fromUnix(10) + let b = fromUnix(1_500_000_000) + let ti = b.toTimeInterval() - a.toTimeInterval() + doAssert a + ti == b var dt = time.local - initInterval(0, dt.second, dt.minute, dt.hour, dt.monthday, dt.month.ord - 1, dt.year) - -proc initDateTime*(monthday: MonthdayRange, month: Month, year: int, - hour: HourRange, minute: MinuteRange, second: SecondRange, zone: Timezone = local()): DateTime = - ## Create a new ``DateTime`` in the specified timezone. - assertValidDate monthday, month, year - doAssert monthday <= getDaysInMonth(month, year), "Invalid date: " & $month & " " & $monthday & ", " & $year - let dt = DateTime( - monthday: monthday, - year: year, - month: month, - hour: hour, - minute: minute, - second: second - ) - result = initDateTime(zone.zoneInfoFromTz(dt.toAdjTime), zone) + initTimeInterval(dt.nanosecond, 0, 0, dt.second, dt.minute, dt.hour, + dt.monthday, 0, dt.month.ord - 1, dt.year) when not defined(JS): - proc epochTime*(): float {.rtl, extern: "nt$1", tags: [TimeEffect].} - ## gets time after the UNIX epoch (1970) in seconds. It is a float - ## because sub-second resolution is likely to be supported (depending - ## on the hardware/OS). - - proc cpuTime*(): float {.rtl, extern: "nt$1", tags: [TimeEffect].} - ## gets time spent that the CPU spent to run the current process in - ## seconds. This may be more useful for benchmarking than ``epochTime``. - ## However, it may measure the real time instead (depending on the OS). - ## The value of the result has no meaning. - ## To generate useful timing values, take the difference between - ## the results of two ``cpuTime`` calls: - ## - ## .. code-block:: nim - ## var t0 = cpuTime() - ## doWork() - ## echo "CPU time [s] ", cpuTime() - t0 - -when defined(JS): - proc getTime(): Time = - (newDate().getTime() div 1000).Time - - proc epochTime*(): float {.tags: [TimeEffect].} = - newDate().getTime() / 1000 - -else: type Clock {.importc: "clock_t".} = distinct int - proc timec(timer: ptr CTime): CTime {. - importc: "time", header: "<time.h>", tags: [].} - proc getClock(): Clock {.importc: "clock", header: "<time.h>", tags: [TimeEffect].} var clocksPerSec {.importc: "CLOCKS_PER_SEC", nodecl.}: int - proc getTime(): Time = - timec(nil).Time - - const - epochDiff = 116444736000000000'i64 - rateDiff = 10000000'i64 # 100 nsecs - - proc unixTimeToWinTime*(time: CTime): int64 = - ## converts a UNIX `Time` (``time_t``) to a Windows file time - result = int64(time) * rateDiff + epochDiff - - proc winTimeToUnixTime*(time: int64): CTime = - ## converts a Windows time to a UNIX `Time` (``time_t``) - result = CTime((time - epochDiff) div rateDiff) - when not defined(useNimRtl): - proc epochTime(): float = + proc cpuTime*(): float {.rtl, extern: "nt$1", tags: [TimeEffect].} = + ## gets time spent that the CPU spent to run the current process in + ## seconds. This may be more useful for benchmarking than ``epochTime``. + ## However, it may measure the real time instead (depending on the OS). + ## The value of the result has no meaning. + ## To generate useful timing values, take the difference between + ## the results of two ``cpuTime`` calls: + runnableExamples: + var t0 = cpuTime() + # some useless work here (calculate fibonacci) + var fib = @[0, 1, 1] + for i in 1..10: + fib.add(fib[^1] + fib[^2]) + echo "CPU time [s] ", cpuTime() - t0 + echo "Fib is [s] ", fib + result = toFloat(int(getClock())) / toFloat(clocksPerSec) + + proc epochTime*(): float {.rtl, extern: "nt$1", tags: [TimeEffect].} = + ## gets time after the UNIX epoch (1970) in seconds. It is a float + ## because sub-second resolution is likely to be supported (depending + ## on the hardware/OS). + ## + ## ``getTime`` should generally be prefered over this proc. when defined(posix): var a: Timeval - posix_gettimeofday(a) - result = toFloat(a.tv_sec) + toFloat(a.tv_usec)*0.00_0001 + gettimeofday(a) + result = toBiggestFloat(a.tv_sec.int64) + toFloat(a.tv_usec)*0.00_0001 elif defined(windows): var f: winlean.FILETIME getSystemTimeAsFileTime(f) @@ -1352,30 +2366,50 @@ else: else: {.error: "unknown OS".} - proc cpuTime(): float = - result = toFloat(int(getClock())) / toFloat(clocksPerSec) +when defined(JS): + proc epochTime*(): float {.tags: [TimeEffect].} = + newDate().getTime() / 1000 # Deprecated procs +when not defined(JS): + proc unixTimeToWinTime*(time: CTime): int64 {.deprecated: "Use toWinTime instead".} = + ## Converts a UNIX `Time` (``time_t``) to a Windows file time + ## + ## **Deprecated:** use ``toWinTime`` instead. + result = int64(time) * rateDiff + epochDiff + + proc winTimeToUnixTime*(time: int64): CTime {.deprecated: "Use fromWinTime instead".} = + ## Converts a Windows time to a UNIX `Time` (``time_t``) + ## + ## **Deprecated:** use ``fromWinTime`` instead. + result = CTime((time - epochDiff) div rateDiff) + +proc initInterval*(seconds, minutes, hours, days, months, + years: int = 0): TimeInterval {.deprecated.} = + ## **Deprecated since v0.18.0:** use ``initTimeInterval`` instead. + initTimeInterval(0, 0, 0, seconds, minutes, hours, days, 0, months, years) + proc fromSeconds*(since1970: float): Time {.tags: [], raises: [], benign, deprecated.} = ## Takes a float which contains the number of seconds since the unix epoch and ## returns a time object. ## ## **Deprecated since v0.18.0:** use ``fromUnix`` instead - Time(since1970) + let nanos = ((since1970 - since1970.int64.float) * convert(Seconds, Nanoseconds, 1).float).int + initTime(since1970.int64, nanos) proc fromSeconds*(since1970: int64): Time {.tags: [], raises: [], benign, deprecated.} = ## Takes an int which contains the number of seconds since the unix epoch and ## returns a time object. ## ## **Deprecated since v0.18.0:** use ``fromUnix`` instead - Time(since1970) + fromUnix(since1970) proc toSeconds*(time: Time): float {.tags: [], raises: [], benign, deprecated.} = ## Returns the time in seconds since the unix epoch. ## ## **Deprecated since v0.18.0:** use ``toUnix`` instead - float(time) + time.seconds.float + time.nanosecond / convert(Seconds, Nanoseconds, 1) proc getLocalTime*(time: Time): DateTime {.tags: [], raises: [], benign, deprecated.} = ## Converts the calendar time `time` to broken-time representation, @@ -1386,7 +2420,7 @@ proc getLocalTime*(time: Time): DateTime {.tags: [], raises: [], benign, depreca proc getGMTime*(time: Time): DateTime {.tags: [], raises: [], benign, deprecated.} = ## Converts the calendar time `time` to broken-down time representation, - ## expressed in Coordinated Universal Time (UTC). + ## expressed in Coordinated Universal Time (UTC). ## ## **Deprecated since v0.18.0:** use ``utc`` instead time.utc @@ -1399,7 +2433,8 @@ proc getTimezone*(): int {.tags: [TimeEffect], raises: [], benign, deprecated.} when defined(JS): return newDate().getTimezoneOffset() * 60 elif defined(freebsd) or defined(netbsd) or defined(openbsd): - var a = timec(nil) + var a: CTime + discard time(a) let lt = localtime(addr(a)) # BSD stores in `gmtoff` offset east of UTC in seconds, # but posix systems using west of UTC in seconds @@ -1410,79 +2445,41 @@ proc getTimezone*(): int {.tags: [TimeEffect], raises: [], benign, deprecated.} proc timeInfoToTime*(dt: DateTime): Time {.tags: [], benign, deprecated.} = ## Converts a broken-down time structure to calendar time representation. ## - ## **Warning:** This procedure is deprecated since version 0.14.0. - ## Use ``toTime`` instead. + ## **Deprecated since v0.14.0:** use ``toTime`` instead. dt.toTime when defined(JS): - var startMilsecs = getTime() + var start = getTime() proc getStartMilsecs*(): int {.deprecated, tags: [TimeEffect], benign.} = - ## get the milliseconds from the start of the program. **Deprecated since - ## version 0.8.10.** Use ``epochTime`` or ``cpuTime`` instead. - when defined(JS): - ## get the milliseconds from the start of the program - return int(getTime() - startMilsecs) + let dur = getTime() - start + result = (convert(Seconds, Milliseconds, dur.seconds) + + convert(Nanoseconds, Milliseconds, dur.nanosecond)).int else: proc getStartMilsecs*(): int {.deprecated, tags: [TimeEffect], benign.} = + ## get the milliseconds from the start of the program. + ## + ## **Deprecated since v0.8.10:** use ``epochTime`` or ``cpuTime`` instead. when defined(macosx): result = toInt(toFloat(int(getClock())) / (toFloat(clocksPerSec) / 1000.0)) else: result = int(getClock()) div (clocksPerSec div 1000) -proc miliseconds*(t: TimeInterval): int {.deprecated.} = - t.milliseconds - proc timeToTimeInterval*(t: Time): TimeInterval {.deprecated.} = ## Converts a Time to a TimeInterval. ## - ## **Warning:** This procedure is deprecated since version 0.14.0. - ## Use ``toTimeInterval`` instead. + ## **Deprecated since v0.14.0:** use ``toTimeInterval`` instead. # Milliseconds not available from Time t.toTimeInterval() -proc timeToTimeInfo*(t: Time): DateTime {.deprecated.} = - ## Converts a Time to DateTime. - ## - ## **Warning:** This procedure is deprecated since version 0.14.0. - ## Use ``inZone`` instead. - const epochStartYear = 1970 - - let - secs = t.toSeconds().int - daysSinceEpoch = secs div secondsInDay - (yearsSinceEpoch, daysRemaining) = countYearsAndDays(daysSinceEpoch) - daySeconds = secs mod secondsInDay - - y = yearsSinceEpoch + epochStartYear - - var - mon = mJan - days = daysRemaining - daysInMonth = getDaysInMonth(mon, y) - - # calculate month and day remainder - while days > daysInMonth and mon <= mDec: - days -= daysInMonth - mon.inc - daysInMonth = getDaysInMonth(mon, y) - - let - yd = daysRemaining - m = mon # month is zero indexed enum - md = days - # NB: month is zero indexed but dayOfWeek expects 1 indexed. - wd = getDayOfWeek(days, mon, y).Weekday - h = daySeconds div secondsInHour + 1 - mi = (daySeconds mod secondsInHour) div secondsInMin - s = daySeconds mod secondsInMin - result = DateTime(year: y, yearday: yd, month: m, monthday: md, weekday: wd, hour: h, minute: mi, second: s) - proc getDayOfWeek*(day, month, year: int): WeekDay {.tags: [], raises: [], benign, deprecated.} = + ## **Deprecated since v0.18.0:** use + ## ``getDayOfWeek(monthday: MonthdayRange; month: Month; year: int)`` instead. getDayOfWeek(day, month.Month, year) proc getDayOfWeekJulian*(day, month, year: int): WeekDay {.deprecated.} = ## Returns the day of the week enum from day, month and year, ## according to the Julian calendar. + ## **Deprecated since v0.18.0** # Day & month start from one. let a = (14 - month) div 12 @@ -1490,3 +2487,23 @@ proc getDayOfWeekJulian*(day, month, year: int): WeekDay {.deprecated.} = m = month + (12*a) - 2 d = (5 + day + y + (y div 4) + (31*m) div 12) mod 7 result = d.WeekDay + +proc adjTime*(zt: ZonedTime): Time + {.deprecated: "Use zt.time instead".} = + ## **Deprecated since v0.19.0:** use the ``time`` field instead. + zt.time - initDuration(seconds = zt.utcOffset) + +proc `adjTime=`*(zt: var ZonedTime, adjTime: Time) + {.deprecated: "Use zt.time instead".} = + ## **Deprecated since v0.19.0:** use the ``time`` field instead. + zt.time = adjTime + initDuration(seconds = zt.utcOffset) + +proc zoneInfoFromUtc*(zone: Timezone, time: Time): ZonedTime + {.deprecated: "Use zonedTimeFromTime instead".} = + ## **Deprecated since v0.19.0:** use ``zonedTimeFromTime`` instead. + zone.zonedTimeFromTime(time) + +proc zoneInfoFromTz*(zone: Timezone, adjTime: Time): ZonedTime + {.deprecated: "Use zonedTimeFromAdjTime instead".} = + ## **Deprecated since v0.19.0:** use the ``zonedTimeFromAdjTime`` instead. + zone.zonedTimeFromAdjTime(adjTime) \ No newline at end of file diff --git a/lib/pure/typetraits.nim b/lib/pure/typetraits.nim index 2047abda4..e9579e824 100644 --- a/lib/pure/typetraits.nim +++ b/lib/pure/typetraits.nim @@ -59,9 +59,4 @@ proc supportsCopyMem*(t: typedesc): bool {.magic: "TypeTrait".} when isMainModule: - # echo type(42) - import streams - var ss = newStringStream() - ss.write($type(42)) # needs `$` - ss.setPosition(0) - doAssert ss.readAll() == "int" + doAssert $type(42) == "int" diff --git a/lib/pure/unicode.nim b/lib/pure/unicode.nim index 257c620f7..978f569ac 100644 --- a/lib/pure/unicode.nim +++ b/lib/pure/unicode.nim @@ -9,7 +9,7 @@ ## This module provides support to handle the Unicode UTF-8 encoding. -{.deadCodeElim: on.} +{.deadCodeElim: on.} # dce option deprecated include "system/inclrtl" @@ -1392,7 +1392,7 @@ proc isCombining*(c: Rune): bool {.rtl, extern: "nuc$1", procvar.} = (c >= 0xfe20 and c <= 0xfe2f)) template runeCheck(s, runeProc) = - ## Common code for rune.isLower, rune.isUpper, etc + ## Common code for isAlpha and isSpace. result = if len(s) == 0: false else: true var @@ -1403,16 +1403,6 @@ template runeCheck(s, runeProc) = fastRuneAt(s, i, rune, doInc=true) result = runeProc(rune) and result -proc isUpper*(s: string): bool {.noSideEffect, procvar, - rtl, extern: "nuc$1Str".} = - ## Returns true iff `s` contains all upper case unicode characters. - runeCheck(s, isUpper) - -proc isLower*(s: string): bool {.noSideEffect, procvar, - rtl, extern: "nuc$1Str".} = - ## Returns true iff `s` contains all lower case unicode characters. - runeCheck(s, isLower) - proc isAlpha*(s: string): bool {.noSideEffect, procvar, rtl, extern: "nuc$1Str".} = ## Returns true iff `s` contains all alphabetic unicode characters. @@ -1423,6 +1413,56 @@ proc isSpace*(s: string): bool {.noSideEffect, procvar, ## Returns true iff `s` contains all whitespace unicode characters. runeCheck(s, isWhiteSpace) +template runeCaseCheck(s, runeProc, skipNonAlpha) = + ## Common code for rune.isLower and rune.isUpper. + if len(s) == 0: return false + + var + i = 0 + rune: Rune + hasAtleastOneAlphaRune = false + + while i < len(s): + fastRuneAt(s, i, rune, doInc=true) + if skipNonAlpha: + var runeIsAlpha = isAlpha(rune) + if not hasAtleastOneAlphaRune: + hasAtleastOneAlphaRune = runeIsAlpha + if runeIsAlpha and (not runeProc(rune)): + return false + else: + if not runeProc(rune): + return false + return if skipNonAlpha: hasAtleastOneAlphaRune else: true + +proc isLower*(s: string, skipNonAlpha: bool): bool = + ## Checks whether ``s`` is lower case. + ## + ## If ``skipNonAlpha`` is true, returns true if all alphabetical + ## runes in ``s`` are lower case. Returns false if none of the + ## runes in ``s`` are alphabetical. + ## + ## If ``skipNonAlpha`` is false, returns true only if all runes in + ## ``s`` are alphabetical and lower case. + ## + ## For either value of ``skipNonAlpha``, returns false if ``s`` is + ## an empty string. + runeCaseCheck(s, isLower, skipNonAlpha) + +proc isUpper*(s: string, skipNonAlpha: bool): bool = + ## Checks whether ``s`` is upper case. + ## + ## If ``skipNonAlpha`` is true, returns true if all alphabetical + ## runes in ``s`` are upper case. Returns false if none of the + ## runes in ``s`` are alphabetical. + ## + ## If ``skipNonAlpha`` is false, returns true only if all runes in + ## ``s`` are alphabetical and upper case. + ## + ## For either value of ``skipNonAlpha``, returns false if ``s`` is + ## an empty string. + runeCaseCheck(s, isUpper, skipNonAlpha) + template convertRune(s, runeProc) = ## Convert runes in `s` using `runeProc` as the converter. result = newString(len(s)) @@ -1755,25 +1795,39 @@ when isMainModule: doAssert(not isSpace("")) doAssert(not isSpace("ΑΓc \td")) - doAssert isLower("a") - doAssert isLower("γ") - doAssert(not isLower("Γ")) - doAssert(not isLower("4")) - doAssert(not isLower("")) - - doAssert isLower("abcdγ") - doAssert(not isLower("abCDΓ")) - doAssert(not isLower("33aaΓ")) - - doAssert isUpper("Γ") - doAssert(not isUpper("b")) - doAssert(not isUpper("α")) - doAssert(not isUpper("✓")) - doAssert(not isUpper("")) - - doAssert isUpper("ΑΒΓ") - doAssert(not isUpper("AAccβ")) - doAssert(not isUpper("A#$β")) + doAssert(not isLower(' '.Rune)) + + doAssert isLower("a", false) + doAssert isLower("γ", true) + doAssert(not isLower("Γ", false)) + doAssert(not isLower("4", true)) + doAssert(not isLower("", false)) + doAssert isLower("abcdγ", false) + doAssert(not isLower("33aaΓ", false)) + doAssert(not isLower("a b", false)) + + doAssert(not isLower("abCDΓ", true)) + doAssert isLower("a b", true) + doAssert isLower("1, 2, 3 go!", true) + doAssert(not isLower(" ", true)) + doAssert(not isLower("(*&#@(^#$✓ ", true)) # None of the string runes are alphabets + + doAssert(not isUpper(' '.Rune)) + + doAssert isUpper("Γ", false) + doAssert(not isUpper("α", false)) + doAssert(not isUpper("", false)) + doAssert isUpper("ΑΒΓ", false) + doAssert(not isUpper("A#$β", false)) + doAssert(not isUpper("A B", false)) + + doAssert(not isUpper("b", true)) + doAssert(not isUpper("✓", true)) + doAssert(not isUpper("AAccβ", true)) + doAssert isUpper("A B", true) + doAssert isUpper("1, 2, 3 GO!", true) + doAssert(not isUpper(" ", true)) + doAssert(not isUpper("(*&#@(^#$✓ ", true)) # None of the string runes are alphabets doAssert toUpper("Γ") == "Γ" doAssert toUpper("b") == "B" @@ -1826,8 +1880,8 @@ when isMainModule: doAssert(runeSubStr(s, 17, 1) == "€") # echo runeStrAtPos(s, 18) # index error - doAssert(runeSubStr(s, 0) == "Hänsel ««: 10,00€") - doAssert(runeSubStr(s, -18) == "Hänsel ««: 10,00€") + doAssert(runeSubStr(s, 0) == "Hänsel ««: 10,00€") + doAssert(runeSubStr(s, -18) == "Hänsel ««: 10,00€") doAssert(runeSubStr(s, 10) == ": 10,00€") doAssert(runeSubStr(s, 18) == "") doAssert(runeSubStr(s, 0, 10) == "Hänsel ««") @@ -1840,7 +1894,7 @@ when isMainModule: doAssert(runeSubStr(s, -6, 5) == "10,00") doAssert(runeSubStr(s, -6, -1) == "10,00") - doAssert(runeSubStr(s, 0, 100) == "Hänsel ««: 10,00€") - doAssert(runeSubStr(s, -100, 100) == "Hänsel ««: 10,00€") + doAssert(runeSubStr(s, 0, 100) == "Hänsel ««: 10,00€") + doAssert(runeSubStr(s, -100, 100) == "Hänsel ««: 10,00€") doAssert(runeSubStr(s, 0, -100) == "") doAssert(runeSubStr(s, 100, -100) == "") diff --git a/lib/pure/unidecode/gen.py b/lib/pure/unidecode/gen.py index 8da0136ff..f0647ea6c 100644 --- a/lib/pure/unidecode/gen.py +++ b/lib/pure/unidecode/gen.py @@ -1,26 +1,30 @@ -#! usr/bin/env python +#! usr/bin/env python3 # -*- coding: utf-8 -*- # Generates the unidecode.dat module # (c) 2010 Andreas Rumpf from unidecode import unidecode +try: + import warnings + warnings.simplefilter("ignore") +except ImportError: + pass -def main2(): - data = [] - for x in xrange(128, 0xffff + 1): - u = eval("u'\u%04x'" % x) - - val = unidecode(u) - data.append(val) - - - f = open("unidecode.dat", "wb+") - for d in data: - f.write("%s\n" % d) - f.close() +def main2(): + f = open("unidecode.dat", "wb+") + for x in range(128, 0xffff + 1): + u = eval("u'\\u%04x'" % x) + val = unidecode(u) -main2() + # f.write("%x | " % x) + if x==0x2028: # U+2028 = LINE SEPARATOR + val = "" + elif x==0x2029: # U+2028 = PARAGRAPH SEPARATOR + val = "" + f.write("%s\n" % val) + f.close() +main2() \ No newline at end of file diff --git a/lib/pure/unidecode/unidecode.dat b/lib/pure/unidecode/unidecode.dat index 9dff0a4a9..5f4c075d8 100644 --- a/lib/pure/unidecode/unidecode.dat +++ b/lib/pure/unidecode/unidecode.dat @@ -58,9 +58,9 @@ P 1 o >> -1/4 -1/2 -3/4 + 1/4 + 1/2 + 3/4 ? A A @@ -91,7 +91,7 @@ U U U U -U +Y Th ss a @@ -177,7 +177,7 @@ i I i IJ - +ij J j K @@ -368,7 +368,7 @@ ZH zh j DZ -D +Dz dz G g @@ -414,8 +414,8 @@ Y y H h -[?] -[?] +N +d OU ou Z @@ -434,34 +434,34 @@ O o Y y +l +n +t +j +db +qp +A +C +c +L +T +s +z [?] [?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] +B +U +^ +E +e +J +j +q +q +R +r +Y +y a a a @@ -503,13 +503,13 @@ o OE O F -R -R -R -R r r -R +r +r +r +r +r R R s @@ -519,12 +519,12 @@ S S t t -U +u U v ^ -W -Y +w +y Y z z @@ -556,9 +556,9 @@ ls lz WW ]] -[?] -[?] -k +h +h +h h j r @@ -737,19 +737,19 @@ V -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] +a +e +i +o +u +c +d +h +m +r +t +v +x [?] [?] [?] @@ -1287,7 +1287,7 @@ o f ew [?] -. +: - [?] [?] @@ -1340,9 +1340,9 @@ o u ' +- - - +| : @@ -7402,41 +7402,41 @@ bh +b +d +f +m +n +p +r +r +s +t +z +g +p +b +d +f +g +k +l +m +n +p +r +s - - - - - - - - - - - - - - - - - - - - - - - - - - - +v +x +z @@ -7708,7 +7708,7 @@ a S [?] [?] -[?] +Ss [?] A a @@ -8109,9 +8109,6 @@ _ - - - %0 %00 @@ -8136,19 +8133,23 @@ _ / -[ ]- -[?] +?? ?! !? 7 PP (] [) +* [?] [?] [?] +% +~ [?] [?] [?] +'''' [?] [?] [?] @@ -8156,12 +8157,8 @@ PP [?] [?] [?] -[?] -[?] -[?] -[?] -[?] -[?] + + [?] [?] [?] @@ -8178,7 +8175,7 @@ PP 0 - +i 4 @@ -8209,19 +8206,19 @@ n ( ) [?] +a +e +o +x [?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] +h +k +l +m +n +p +s +t [?] [?] [?] @@ -8237,26 +8234,26 @@ Rs W NS D -EU +EUR K T Dr -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] +Pf +P +G +A +UAH +C| +L +Sm +T +Rs +L +M +m +R +l +BTC [?] [?] [?] @@ -8294,6 +8291,7 @@ Dr [?] + [?] [?] [?] @@ -8319,63 +8317,67 @@ Dr [?] [?] [?] -[?] - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + a/c + a/s +C + c/o + c/u +g +H +H +H +h +I +I +L +l +N +No. +P +Q +R +R +R +(sm) +TEL +(tm) +Z +Z +K +A +B +C +e +e +E +F +F +M +o +i +FAX @@ -8385,25 +8387,20 @@ Dr [?] [?] [?] +D +d +e +i +j [?] [?] [?] [?] +F [?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] + 1/7 + 1/9 + 1/10 1/3 2/3 1/5 @@ -8458,7 +8455,7 @@ D) [?] [?] [?] -[?] + 0/3 [?] [?] [?] @@ -8595,8 +8592,12 @@ V [?] [?] [?] +- [?] [?] +/ +\ +* [?] [?] [?] @@ -8608,6 +8609,7 @@ V [?] [?] [?] +| [?] [?] [?] @@ -8626,11 +8628,13 @@ V [?] [?] [?] +: [?] [?] [?] [?] [?] +~ [?] [?] [?] @@ -8670,17 +8674,10 @@ V [?] [?] [?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] +<= +>= +<= +>= [?] [?] [?] @@ -8836,6 +8833,7 @@ V [?] [?] [?] +^ [?] [?] [?] @@ -8873,9 +8871,8 @@ V [?] [?] [?] -[?] -[?] -[?] +< +> [?] [?] [?] @@ -9185,166 +9182,166 @@ V [?] [?] [?] - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] - +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +(1) +(2) +(3) +(4) +(5) +(6) +(7) +(8) +(9) +(10) +(11) +(12) +(13) +(14) +(15) +(16) +(17) +(18) +(19) +(20) +1. +2. +3. +4. +5. +6. +7. +8. +9. +10. +11. +12. +13. +14. +15. +16. +17. +18. +19. +20. +(a) +(b) +(c) +(d) +(e) +(f) +(g) +(h) +(i) +(j) +(k) +(l) +(m) +(n) +(o) +(p) +(q) +(r) +(s) +(t) +(u) +(v) +(w) +(x) +(y) +(z) +A +B +C +D +E +F +G +H +I +J +K +L +M +N +O +P +Q +R +S +T +U +V +W +X +Y +Z +a +b +c +d +e +f +g +h +i +j +k +l +m +n +o +p +q +r +s +t +u +v +w +x +y +z +0 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +0 - - | @@ -9712,7 +9709,7 @@ O - +# [?] @@ -9906,6 +9903,7 @@ O +* @@ -9944,8 +9942,7 @@ O - - +| @@ -9955,7 +9952,7 @@ O [?] [?] - +! @@ -10087,10 +10084,10 @@ O [?] [?] [?] +[ [?] -[?] -[?] -[?] +< +> [?] [?] [?] @@ -10500,6 +10497,8 @@ y +{ +} @@ -10739,6 +10738,9 @@ y +::= +== +=== @@ -11228,27 +11230,22 @@ y +L +l +L +P +R +a +t +H +h +K +k +Z +z - - - - - - - - - - - - - - - - - - - - +M +A @@ -12754,21 +12751,21 @@ H [?] [?] [?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] +21 +22 +23 +24 +25 +26 +27 +28 +29 +30 +31 +32 +33 +34 +35 (g) (n) (d) @@ -12850,21 +12847,21 @@ KIS (Zi) (Xie) (Ye) -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] +36 +37 +38 +39 +40 +41 +42 +43 +44 +45 +46 +47 +48 +49 +50 1M 2M 3M @@ -12877,10 +12874,10 @@ KIS 10M 11M 12M -[?] -[?] -[?] -[?] +Hg +erg +eV +LTD a i u @@ -13042,16 +13039,16 @@ watt 22h 23h 24h -HPA +hPa da AU bar oV pc -[?] -[?] -[?] -[?] +dm +dm^2 +dm^3 +IU Heisei Syouwa Taisyou @@ -13092,7 +13089,7 @@ mm^2 cm^2 m^2 km^2 -mm^4 +mm^3 cm^3 m^3 km^3 @@ -13184,7 +13181,7 @@ Wb 29d 30d 31d - +gal @@ -19841,7 +19838,7 @@ Wb [?] [?] -[?] +Yi Ding Kao Qi diff --git a/lib/pure/unidecode/unidecode.nim b/lib/pure/unidecode/unidecode.nim index 9d8843f06..e0b8d3946 100644 --- a/lib/pure/unidecode/unidecode.nim +++ b/lib/pure/unidecode/unidecode.nim @@ -22,14 +22,14 @@ ## strictly one-way transformation. However a human reader will probably ## still be able to guess what original string was meant from the context. ## -## This module needs the data file "unidecode.dat" to work: You can either -## ship this file with your application and initialize this module with the -## `loadUnidecodeTable` proc or you can define the ``embedUnidecodeTable`` -## symbol to embed the file as a resource into your application. +## This module needs the data file "unidecode.dat" to work: This file is +## embedded as a resource into your application by default. But you an also +## define the symbol ``--define:noUnidecodeTable`` during compile time and +## use the `loadUnidecodeTable` proc to initialize this module. import unicode -when defined(embedUnidecodeTable): +when not defined(noUnidecodeTable): import strutils const translationTable = splitLines(slurp"unidecode/unidecode.dat") @@ -38,11 +38,11 @@ else: var translationTable: seq[string] proc loadUnidecodeTable*(datafile = "unidecode.dat") = - ## loads the datafile that `unidecode` to work. Unless this module is - ## compiled with the ``embedUnidecodeTable`` symbol defined, this needs - ## to be called by the main thread before any thread can make a call - ## to `unidecode`. - when not defined(embedUnidecodeTable): + ## loads the datafile that `unidecode` to work. This is only required if + ## the module was compiled with the ``--define:noUnidecodeTable`` switch. + ## This needs to be called by the main thread before any thread can make a + ## call to `unidecode`. + when defined(noUnidecodeTable): newSeq(translationTable, 0xffff) var i = 0 for line in lines(datafile): @@ -61,7 +61,6 @@ proc unidecode*(s: string): string = ## ## Results in: "Bei Jing" ## - assert(not isNil(translationTable)) result = "" for r in runes(s): var c = int(r) @@ -69,6 +68,6 @@ proc unidecode*(s: string): string = elif c <% translationTable.len: add(result, translationTable[c-128]) when isMainModule: - loadUnidecodeTable("lib/pure/unidecode/unidecode.dat") - assert unidecode("Äußerst") == "Ausserst" - + #loadUnidecodeTable("lib/pure/unidecode/unidecode.dat") + doAssert unidecode("Äußerst") == "Ausserst" + doAssert unidecode("北京") == "Bei Jing " diff --git a/lib/pure/unittest.nim b/lib/pure/unittest.nim index fbce087ff..757bf4745 100644 --- a/lib/pure/unittest.nim +++ b/lib/pure/unittest.nim @@ -121,9 +121,16 @@ type ConsoleOutputFormatter* = ref object of OutputFormatter colorOutput: bool ## Have test results printed in color. - ## Default is true for the non-js target - ## unless, the environment variable - ## ``NIMTEST_NO_COLOR`` is set. + ## Default is true for the non-js target, + ## for which ``stdout`` is a tty. + ## Setting the environment variable + ## ``NIMTEST_COLOR`` to ``always`` or + ## ``never`` changes the default for the + ## non-js target to true or false respectively. + ## The deprecated environment variable + ## ``NIMTEST_NO_COLOR``, when set, + ## changes the defualt to true, if + ## ``NIMTEST_COLOR`` is undefined. outputLevel: OutputLevel ## Set the verbosity of test results. ## Default is ``PRINT_ALL``, unless @@ -150,6 +157,7 @@ var checkpoints {.threadvar.}: seq[string] formatters {.threadvar.}: seq[OutputFormatter] testsFilters {.threadvar.}: HashSet[string] + disabledParamFiltering {.threadvar.}: bool when declared(stdout): abortOnError = existsEnv("NIMTEST_ABORT_ON_ERROR") @@ -168,10 +176,7 @@ method suiteEnded*(formatter: OutputFormatter) {.base, gcsafe.} = discard proc addOutputFormatter*(formatter: OutputFormatter) = - if formatters == nil: - formatters = @[formatter] - else: - formatters.add(formatter) + formatters.add(formatter) proc newConsoleOutputFormatter*(outputLevel: OutputLevel = PRINT_ALL, colorOutput = true): ConsoleOutputFormatter = @@ -185,7 +190,15 @@ proc defaultConsoleFormatter*(): ConsoleOutputFormatter = # Reading settings # On a terminal this branch is executed var envOutLvl = os.getEnv("NIMTEST_OUTPUT_LVL").string - var colorOutput = not existsEnv("NIMTEST_NO_COLOR") + var colorOutput = isatty(stdout) + if existsEnv("NIMTEST_COLOR"): + let colorEnv = getenv("NIMTEST_COLOR") + if colorEnv == "never": + colorOutput = false + elif colorEnv == "always": + colorOutput = true + elif existsEnv("NIMTEST_NO_COLOR"): + colorOutput = false var outputLevel = PRINT_ALL if envOutLvl.len > 0: for opt in countup(low(OutputLevel), high(OutputLevel)): @@ -209,7 +222,7 @@ method testStarted*(formatter: ConsoleOutputFormatter, testName: string) = formatter.isInTest = true method failureOccurred*(formatter: ConsoleOutputFormatter, checkpoints: seq[string], stackTrace: string) = - if stackTrace != nil: + if stackTrace.len > 0: echo stackTrace let prefix = if formatter.isInSuite: " " else: "" for msg in items(checkpoints): @@ -220,7 +233,7 @@ method testEnded*(formatter: ConsoleOutputFormatter, testResult: TestResult) = if formatter.outputLevel != PRINT_NONE and (formatter.outputLevel == PRINT_ALL or testResult.status == FAILED): - let prefix = if testResult.suiteName != nil: " " else: "" + let prefix = if testResult.suiteName.len > 0: " " else: "" template rawPrint() = echo(prefix, "[", $testResult.status, "] ", testResult.testName) when not defined(ECMAScript): if formatter.colorOutput and not defined(ECMAScript): @@ -285,7 +298,7 @@ method failureOccurred*(formatter: JUnitOutputFormatter, checkpoints: seq[string ## ``stackTrace`` is provided only if the failure occurred due to an exception. ## ``checkpoints`` is never ``nil``. formatter.testErrors.add(checkpoints) - if stackTrace != nil: + if stackTrace.len > 0: formatter.testStackTrace = stackTrace method testEnded*(formatter: JUnitOutputFormatter, testResult: TestResult) = @@ -376,10 +389,10 @@ proc shouldRun(currentSuiteName, testName: string): bool = return false proc ensureInitialized() = - if formatters == nil: + if formatters.len == 0: formatters = @[OutputFormatter(defaultConsoleFormatter())] - if not testsFilters.isValid: + if not disabledParamFiltering and not testsFilters.isValid: testsFilters.init() when declared(paramCount): # Read tests to run from the command line. @@ -446,6 +459,8 @@ template suite*(name, body) {.dirty.} = finally: suiteEnded() +template exceptionTypeName(e: typed): string = $e.name + template test*(name, body) {.dirty.} = ## Define a single test case identified by `name`. ## @@ -460,7 +475,7 @@ template test*(name, body) {.dirty.} = ## .. code-block:: ## ## [OK] roses are red - bind shouldRun, checkpoints, formatters, ensureInitialized, testEnded + bind shouldRun, checkpoints, formatters, ensureInitialized, testEnded, exceptionTypeName ensureInitialized() @@ -479,15 +494,17 @@ template test*(name, body) {.dirty.} = except: when not defined(js): - checkpoint("Unhandled exception: " & getCurrentExceptionMsg()) - var stackTrace {.inject.} = getCurrentException().getStackTrace() + let e = getCurrentException() + let eTypeDesc = "[" & exceptionTypeName(e) & "]" + checkpoint("Unhandled exception: " & getCurrentExceptionMsg() & " " & eTypeDesc) + var stackTrace {.inject.} = e.getStackTrace() fail() finally: if testStatusIMPL == FAILED: programResult += 1 let testResult = TestResult( - suiteName: when declared(testSuiteName): testSuiteName else: nil, + suiteName: when declared(testSuiteName): testSuiteName else: "", testName: name, status: testStatusIMPL ) @@ -505,8 +522,6 @@ proc checkpoint*(msg: string) = ## checkpoint("Checkpoint B") ## ## outputs "Checkpoint A" once it fails. - if checkpoints == nil: - checkpoints = @[] checkpoints.add(msg) # TODO: add support for something like SCOPED_TRACE from Google Test @@ -537,7 +552,7 @@ template fail* = when declared(stackTrace): formatter.failureOccurred(checkpoints, stackTrace) else: - formatter.failureOccurred(checkpoints, nil) + formatter.failureOccurred(checkpoints, "") when not defined(ECMAScript): if abortOnError: quit(programResult) @@ -701,3 +716,7 @@ macro expect*(exceptions: varargs[typed], body: untyped): untyped = errorTypes.add(exp[i]) result = getAst(expectBody(errorTypes, exp.lineinfo, body)) + +proc disableParamFiltering* = + ## disables filtering tests with the command line params + disabledParamFiltering = true diff --git a/lib/pure/uri.nim b/lib/pure/uri.nim index d2d11253a..dd8040928 100644 --- a/lib/pure/uri.nim +++ b/lib/pure/uri.nim @@ -18,14 +18,12 @@ type hostname*, port*, path*, query*, anchor*: string opaque*: bool -{.deprecated: [TUrl: Url, TUri: Uri].} - {.push warning[deprecated]: off.} -proc `$`*(url: Url): string {.deprecated.} = +proc `$`*(url: Url): string {.deprecated: "use Uri instead".} = ## **Deprecated since 0.9.6**: Use ``Uri`` instead. return string(url) -proc `/`*(a, b: Url): Url {.deprecated.} = +proc `/`*(a, b: Url): Url {.deprecated: "use Uri instead".} = ## Joins two URLs together, separating them with / if needed. ## ## **Deprecated since 0.9.6**: Use ``Uri`` instead. @@ -40,39 +38,50 @@ proc `/`*(a, b: Url): Url {.deprecated.} = urlS.add(bs) result = Url(urlS) -proc add*(url: var Url, a: Url) {.deprecated.} = +proc add*(url: var Url, a: Url) {.deprecated: "use Uri instead".} = ## Appends url to url. ## ## **Deprecated since 0.9.6**: Use ``Uri`` instead. url = url / a {.pop.} -proc encodeUrl*(s: string): string = - ## Encodes a value to be HTTP safe: This means that characters in the set - ## ``{'A'..'Z', 'a'..'z', '0'..'9', '_'}`` are carried over to the result, - ## a space is converted to ``'+'`` and every other character is encoded as - ## ``'%xx'`` where ``xx`` denotes its hexadecimal value. +proc encodeUrl*(s: string, usePlus=true): string = + ## Encodes a URL according to RFC3986. + ## + ## This means that characters in the set + ## ``{'a'..'z', 'A'..'Z', '0'..'9', '-', '.', '_', '~'}`` are + ## carried over to the result. + ## All other characters are encoded as ``''%xx'`` where ``xx`` + ## denotes its hexadecimal value. + ## + ## As a special rule, when the value of ``usePlus`` is true, + ## spaces are encoded as ``'+'`` instead of ``'%20'``. result = newStringOfCap(s.len + s.len shr 2) # assume 12% non-alnum-chars - for i in 0..s.len-1: - case s[i] - of 'a'..'z', 'A'..'Z', '0'..'9', '_': add(result, s[i]) - of ' ': add(result, '+') + let fromSpace = if usePlus: "+" else: "%20" + for c in s: + case c + of 'a'..'z', 'A'..'Z', '0'..'9', '-', '.', '_', '~': add(result, c) + of ' ': add(result, fromSpace) else: add(result, '%') - add(result, toHex(ord(s[i]), 2)) - -proc decodeUrl*(s: string): string = - ## Decodes a value from its HTTP representation: This means that a ``'+'`` - ## is converted to a space, ``'%xx'`` (where ``xx`` denotes a hexadecimal - ## value) is converted to the character with ordinal number ``xx``, and + add(result, toHex(ord(c), 2)) + +proc decodeUrl*(s: string, decodePlus=true): string = + ## Decodes a URL according to RFC3986. + ## + ## This means that any ``'%xx'`` (where ``xx`` denotes a hexadecimal + ## value) are converted to the character with ordinal number ``xx``, ## and every other character is carried over. + ## + ## As a special rule, when the value of ``decodePlus`` is true, ``'+'`` + ## characters are converted to a space. proc handleHexChar(c: char, x: var int) {.inline.} = case c of '0'..'9': x = (x shl 4) or (ord(c) - ord('0')) of 'a'..'f': x = (x shl 4) or (ord(c) - ord('a') + 10) of 'A'..'F': x = (x shl 4) or (ord(c) - ord('A') + 10) else: assert(false) - + result = newString(s.len) var i = 0 var j = 0 @@ -84,7 +93,11 @@ proc decodeUrl*(s: string): string = handleHexChar(s[i+2], x) inc(i, 2) result[j] = chr(x) - of '+': result[j] = ' ' + of '+': + if decodePlus: + result[j] = ' ' + else: + result[j] = s[i] else: result[j] = s[i] inc(i) inc(j) @@ -94,7 +107,7 @@ proc parseAuthority(authority: string, result: var Uri) = var i = 0 var inPort = false var inIPv6 = false - while true: + while i < authority.len: case authority[i] of '@': swap result.password, result.port @@ -111,7 +124,6 @@ proc parseAuthority(authority: string, result: var Uri) = inIPv6 = true of ']': inIPv6 = false - of '\0': break else: if inPort: result.port.add(authority[i]) @@ -128,11 +140,11 @@ proc parsePath(uri: string, i: var int, result: var Uri) = parseAuthority(result.path, result) result.path.setLen(0) - if uri[i] == '?': + if i < uri.len and uri[i] == '?': i.inc # Skip '?' i.inc parseUntil(uri, result.query, {'#'}, i) - if uri[i] == '#': + if i < uri.len and uri[i] == '#': i.inc # Skip '#' i.inc parseUntil(uri, result.anchor, {}, i) @@ -156,7 +168,7 @@ proc parseUri*(uri: string, result: var Uri) = # Check if this is a reference URI (relative URI) let doubleSlash = uri.len > 1 and uri[1] == '/' - if uri[i] == '/': + if i < uri.len and uri[i] == '/': # Make sure ``uri`` doesn't begin with '//'. if not doubleSlash: parsePath(uri, i, result) @@ -164,7 +176,7 @@ proc parseUri*(uri: string, result: var Uri) = # Scheme i.inc parseWhile(uri, result.scheme, Letters + Digits + {'+', '-', '.'}, i) - if uri[i] != ':' and not doubleSlash: + if (i >= uri.len or uri[i] != ':') and not doubleSlash: # Assume this is a reference URI (relative URI) i = 0 result.scheme.setLen(0) @@ -174,7 +186,7 @@ proc parseUri*(uri: string, result: var Uri) = i.inc # Skip ':' # Authority - if uri[i] == '/' and uri[i+1] == '/': + if i+1 < uri.len and uri[i] == '/' and uri[i+1] == '/': i.inc(2) # Skip // var authority = "" i.inc parseUntil(uri, authority, {'/', '?', '#'}, i) @@ -197,13 +209,13 @@ proc removeDotSegments(path: string): string = let endsWithSlash = path[path.len-1] == '/' var i = 0 var currentSegment = "" - while true: + while i < path.len: case path[i] of '/': collection.add(currentSegment) currentSegment = "" of '.': - if path[i+1] == '.' and path[i+2] == '/': + if i+2 < path.len and path[i+1] == '.' and path[i+2] == '/': if collection.len > 0: discard collection.pop() i.inc 3 @@ -212,13 +224,11 @@ proc removeDotSegments(path: string): string = i.inc 2 continue currentSegment.add path[i] - of '\0': - if currentSegment != "": - collection.add currentSegment - break else: currentSegment.add path[i] i.inc + if currentSegment != "": + collection.add currentSegment result = collection.join("/") if endsWithSlash: result.add '/' @@ -320,18 +330,18 @@ proc `/`*(x: Uri, path: string): Uri = result = x if result.path.len == 0: - if path[0] != '/': + if path.len == 0 or path[0] != '/': result.path = "/" result.path.add(path) return - if result.path[result.path.len-1] == '/': - if path[0] == '/': + if result.path.len > 0 and result.path[result.path.len-1] == '/': + if path.len > 0 and path[0] == '/': result.path.add(path[1 .. path.len-1]) else: result.path.add(path) else: - if path[0] != '/': + if path.len == 0 or path[0] != '/': result.path.add '/' result.path.add(path) @@ -373,7 +383,10 @@ when isMainModule: const test1 = "abc\L+def xyz" doAssert encodeUrl(test1) == "abc%0A%2Bdef+xyz" doAssert decodeUrl(encodeUrl(test1)) == test1 - + doAssert encodeUrl(test1, false) == "abc%0A%2Bdef%20xyz" + doAssert decodeUrl(encodeUrl(test1, false), false) == test1 + doAssert decodeUrl(encodeUrl(test1)) == test1 + block: let str = "http://localhost" let test = parseUri(str) @@ -464,7 +477,7 @@ when isMainModule: doAssert test.hostname == "github.com" doAssert test.port == "dom96" doAssert test.path == "/packages" - + block: let str = "file:///foo/bar/baz.txt" let test = parseUri(str) diff --git a/lib/pure/xmldom.nim b/lib/pure/xmldom.nim index 3c891c81b..82f88a996 100644 --- a/lib/pure/xmldom.nim +++ b/lib/pure/xmldom.nim @@ -172,34 +172,30 @@ proc documentElement*(doc: PDocument): PElement = proc findNodes(nl: PNode, name: string): seq[PNode] = # Made for getElementsByTagName var r: seq[PNode] = @[] - if isNil(nl.childNodes): return @[] - if nl.childNodes.len() == 0: return @[] + if nl.childNodes.len == 0: return @[] for i in items(nl.childNodes): if i.fNodeType == ElementNode: if i.fNodeName == name or name == "*": r.add(i) - if not isNil(i.childNodes): - if i.childNodes.len() != 0: - r.add(findNodes(i, name)) + if i.childNodes.len() != 0: + r.add(findNodes(i, name)) return r proc findNodesNS(nl: PNode, namespaceURI: string, localName: string): seq[PNode] = # Made for getElementsByTagNameNS var r: seq[PNode] = @[] - if isNil(nl.childNodes): return @[] - if nl.childNodes.len() == 0: return @[] + if nl.childNodes.len == 0: return @[] for i in items(nl.childNodes): if i.fNodeType == ElementNode: if (i.fNamespaceURI == namespaceURI or namespaceURI == "*") and (i.fLocalName == localName or localName == "*"): r.add(i) - if not isNil(i.childNodes): - if i.childNodes.len() != 0: - r.add(findNodesNS(i, namespaceURI, localName)) + if i.childNodes.len != 0: + r.add(findNodesNS(i, namespaceURI, localName)) return r @@ -217,9 +213,9 @@ proc createAttribute*(doc: PDocument, name: string): PAttr = new(attrNode) attrNode.fName = name attrNode.fNodeName = name - attrNode.fLocalName = nil - attrNode.prefix = nil - attrNode.fNamespaceURI = nil + attrNode.fLocalName = "" + attrNode.prefix = "" + attrNode.fNamespaceURI = "" attrNode.value = "" attrNode.fSpecified = false return attrNode @@ -232,10 +228,10 @@ proc createAttributeNS*(doc: PDocument, namespaceURI: string, qualifiedName: str raise newException(EInvalidCharacterErr, "Invalid character") # Exceptions if qualifiedName.contains(':'): - let qfnamespaces = qualifiedName.toLower().split(':') - if isNil(namespaceURI): - raise newException(ENamespaceErr, "When qualifiedName contains a prefix namespaceURI cannot be nil") - elif qfnamespaces[0] == "xml" and + let qfnamespaces = qualifiedName.toLowerAscii().split(':') + if namespaceURI.len == 0: + raise newException(ENamespaceErr, "When qualifiedName contains a prefix namespaceURI cannot be empty") + elif qfnamespaces[0] == "xml" and namespaceURI != "http://www.w3.org/XML/1998/namespace" and qfnamespaces[1] notin stdattrnames: raise newException(ENamespaceErr, @@ -254,7 +250,7 @@ proc createAttributeNS*(doc: PDocument, namespaceURI: string, qualifiedName: str attrNode.prefix = qualifiedName.split(':')[0] attrNode.fLocalName = qualifiedName.split(':')[1] else: - attrNode.prefix = nil + attrNode.prefix = "" attrNode.fLocalName = qualifiedName attrNode.value = "" @@ -298,9 +294,9 @@ proc createElement*(doc: PDocument, tagName: string): PElement = new(elNode) elNode.fTagName = tagName elNode.fNodeName = tagName - elNode.fLocalName = nil - elNode.prefix = nil - elNode.fNamespaceURI = nil + elNode.fLocalName = "" + elNode.prefix = "" + elNode.fNamespaceURI = "" elNode.childNodes = @[] elNode.attributes = @[] @@ -311,10 +307,10 @@ proc createElement*(doc: PDocument, tagName: string): PElement = proc createElementNS*(doc: PDocument, namespaceURI: string, qualifiedName: string): PElement = ## Creates an element of the given qualified name and namespace URI. if qualifiedName.contains(':'): - let qfnamespaces = qualifiedName.toLower().split(':') - if isNil(namespaceURI): - raise newException(ENamespaceErr, "When qualifiedName contains a prefix namespaceURI cannot be nil") - elif qfnamespaces[0] == "xml" and + let qfnamespaces = qualifiedName.toLowerAscii().split(':') + if namespaceURI.len == 0: + raise newException(ENamespaceErr, "When qualifiedName contains a prefix namespaceURI cannot be empty") + elif qfnamespaces[0] == "xml" and namespaceURI != "http://www.w3.org/XML/1998/namespace" and qfnamespaces[1] notin stdattrnames: raise newException(ENamespaceErr, @@ -332,7 +328,7 @@ proc createElementNS*(doc: PDocument, namespaceURI: string, qualifiedName: strin elNode.prefix = qualifiedName.split(':')[0] elNode.fLocalName = qualifiedName.split(':')[1] else: - elNode.prefix = nil + elNode.prefix = "" elNode.fLocalName = qualifiedName elNode.fNamespaceURI = namespaceURI elNode.childNodes = @[] @@ -453,7 +449,7 @@ proc importNode*(doc: PDocument, importedNode: PNode, deep: bool): PNode = proc firstChild*(n: PNode): PNode = ## Returns this node's first child - if not isNil(n.childNodes) and n.childNodes.len() > 0: + if n.childNodes.len > 0: return n.childNodes[0] else: return nil @@ -461,8 +457,8 @@ proc firstChild*(n: PNode): PNode = proc lastChild*(n: PNode): PNode = ## Returns this node's last child - if not isNil(n.childNodes) and n.childNodes.len() > 0: - return n.childNodes[n.childNodes.len() - 1] + if n.childNodes.len > 0: + return n.childNodes[n.childNodes.len - 1] else: return nil @@ -482,7 +478,7 @@ proc `namespaceURI=`*(n: PNode, value: string) = proc nextSibling*(n: PNode): PNode = ## Returns the next sibling of this node - if isNil(n.fParentNode) or isNil(n.fParentNode.childNodes): + if isNil(n.fParentNode): return nil var nLow: int = low(n.fParentNode.childNodes) var nHigh: int = high(n.fParentNode.childNodes) @@ -514,7 +510,7 @@ proc parentNode*(n: PNode): PNode = proc previousSibling*(n: PNode): PNode = ## Returns the previous sibling of this node - if isNil(n.fParentNode) or isNil(n.fParentNode.childNodes): + if isNil(n.fParentNode): return nil var nLow: int = low(n.fParentNode.childNodes) var nHigh: int = high(n.fParentNode.childNodes) @@ -531,15 +527,15 @@ proc `prefix=`*(n: PNode, value: string) = if illegalChars in value: raise newException(EInvalidCharacterErr, "Invalid character") - if isNil(n.fNamespaceURI): - raise newException(ENamespaceErr, "namespaceURI cannot be nil") - elif value.toLower() == "xml" and n.fNamespaceURI != "http://www.w3.org/XML/1998/namespace": + if n.fNamespaceURI.len == 0: + raise newException(ENamespaceErr, "namespaceURI cannot be empty") + elif value.toLowerAscii() == "xml" and n.fNamespaceURI != "http://www.w3.org/XML/1998/namespace": raise newException(ENamespaceErr, "When the namespace prefix is \"xml\" namespaceURI has to be \"http://www.w3.org/XML/1998/namespace\"") - elif value.toLower() == "xmlns" and n.fNamespaceURI != "http://www.w3.org/2000/xmlns/": + elif value.toLowerAscii() == "xmlns" and n.fNamespaceURI != "http://www.w3.org/2000/xmlns/": raise newException(ENamespaceErr, "When the namespace prefix is \"xmlns\" namespaceURI has to be \"http://www.w3.org/2000/xmlns/\"") - elif value.toLower() == "xmlns" and n.fNodeType == AttributeNode: + elif value.toLowerAscii() == "xmlns" and n.fNodeType == AttributeNode: raise newException(ENamespaceErr, "An AttributeNode cannot have a prefix of \"xmlns\"") n.fNodeName = value & ":" & n.fLocalName @@ -557,10 +553,9 @@ proc appendChild*(n: PNode, newChild: PNode) = ## If the newChild is already in the tree, it is first removed. # Check if n contains newChild - if not isNil(n.childNodes): - for i in low(n.childNodes)..high(n.childNodes): - if n.childNodes[i] == newChild: - raise newException(EHierarchyRequestErr, "The node to append is already in this nodes children.") + for i in low(n.childNodes)..high(n.childNodes): + if n.childNodes[i] == newChild: + raise newException(EHierarchyRequestErr, "The node to append is already in this nodes children.") # Check if newChild is from this nodes document if n.fOwnerDocument != newChild.fOwnerDocument: @@ -572,7 +567,8 @@ proc appendChild*(n: PNode, newChild: PNode) = if n.nodeType in childlessObjects: raise newException(ENoModificationAllowedErr, "Cannot append children to a childless node") - if isNil(n.childNodes): n.childNodes = @[] + when not defined(nimNoNilSeqs): + if isNil(n.childNodes): n.childNodes = @[] newChild.fParentNode = n for i in low(n.childNodes)..high(n.childNodes): @@ -597,10 +593,10 @@ proc cloneNode*(n: PNode, deep: bool): PNode = newNode = PElement(n) # Import the childNodes var tmp: seq[PNode] = n.childNodes - n.childNodes = @[] - if deep and not isNil(tmp): - for i in low(tmp.len())..high(tmp.len()): - n.childNodes.add(cloneNode(tmp[i], deep)) + newNode.childNodes = @[] + if deep: + for i in low(n.childNodes)..high(n.childNodes): + newNode.childNodes.add(cloneNode(n.childNodes[i], deep)) return newNode else: var newNode: PNode @@ -610,11 +606,11 @@ proc cloneNode*(n: PNode, deep: bool): PNode = proc hasAttributes*(n: PNode): bool = ## Returns whether this node (if it is an element) has any attributes. - return not isNil(n.attributes) and n.attributes.len() > 0 + return n.attributes.len > 0 proc hasChildNodes*(n: PNode): bool = ## Returns whether this node has any children. - return not isNil(n.childNodes) and n.childNodes.len() > 0 + return n.childNodes.len > 0 proc insertBefore*(n: PNode, newChild: PNode, refChild: PNode): PNode = ## Inserts the node ``newChild`` before the existing child node ``refChild``. @@ -624,9 +620,6 @@ proc insertBefore*(n: PNode, newChild: PNode, refChild: PNode): PNode = if n.fOwnerDocument != newChild.fOwnerDocument: raise newException(EWrongDocumentErr, "This node belongs to a different document, use importNode.") - if isNil(n.childNodes): - n.childNodes = @[] - for i in low(n.childNodes)..high(n.childNodes): if n.childNodes[i] == refChild: n.childNodes.insert(newChild, i - 1) @@ -641,7 +634,7 @@ proc isSupported*(n: PNode, feature: string, version: string): bool = proc isEmpty(s: string): bool = - if isNil(s) or s == "": + if s == "": return true for i in items(s): if i != ' ': @@ -655,7 +648,7 @@ proc normalize*(n: PNode) = var newChildNodes: seq[PNode] = @[] while true: - if isNil(n.childNodes) or i >= n.childNodes.len: + if i >= n.childNodes.len: break if n.childNodes[i].nodeType == TextNode: @@ -679,12 +672,11 @@ proc normalize*(n: PNode) = proc removeChild*(n: PNode, oldChild: PNode): PNode = ## Removes the child node indicated by ``oldChild`` from the list of children, and returns it. - if not isNil(n.childNodes): - for i in low(n.childNodes)..high(n.childNodes): - if n.childNodes[i] == oldChild: - result = n.childNodes[i] - n.childNodes.delete(i) - return + for i in low(n.childNodes)..high(n.childNodes): + if n.childNodes[i] == oldChild: + result = n.childNodes[i] + n.childNodes.delete(i) + return raise newException(ENotFoundErr, "Node not found") @@ -695,12 +687,11 @@ proc replaceChild*(n: PNode, newChild: PNode, oldChild: PNode): PNode = if n.fOwnerDocument != newChild.fOwnerDocument: raise newException(EWrongDocumentErr, "This node belongs to a different document, use importNode.") - if not isNil(n.childNodes): - for i in low(n.childNodes)..high(n.childNodes): - if n.childNodes[i] == oldChild: - result = n.childNodes[i] - n.childNodes[i] = newChild - return + for i in low(n.childNodes)..high(n.childNodes): + if n.childNodes[i] == oldChild: + result = n.childNodes[i] + n.childNodes[i] = newChild + return raise newException(ENotFoundErr, "Node not found") @@ -764,11 +755,10 @@ proc removeNamedItemNS*(nList: var seq[PNode], namespaceURI: string, localName: proc setNamedItem*(nList: var seq[PNode], arg: PNode): PNode = ## Adds ``arg`` as a ``Node`` to the ``NList`` ## If a node with the same name is already present in this map, it is replaced by the new one. - if not isNil(nList): - if nList.len() > 0: - #Check if newChild is from this nodes document - if nList[0].fOwnerDocument != arg.fOwnerDocument: - raise newException(EWrongDocumentErr, "This node belongs to a different document, use importNode.") + if nList.len > 0: + #Check if newChild is from this nodes document + if nList[0].fOwnerDocument != arg.fOwnerDocument: + raise newException(EWrongDocumentErr, "This node belongs to a different document, use importNode.") #Exceptions End var item: PNode = nList.getNamedItem(arg.nodeName()) @@ -788,11 +778,10 @@ proc setNamedItem*(nList: var seq[PNode], arg: PNode): PNode = proc setNamedItem*(nList: var seq[PAttr], arg: PAttr): PAttr = ## Adds ``arg`` as a ``Node`` to the ``NList`` ## If a node with the same name is already present in this map, it is replaced by the new one. - if not isNil(nList): - if nList.len() > 0: - # Check if newChild is from this nodes document - if nList[0].fOwnerDocument != arg.fOwnerDocument: - raise newException(EWrongDocumentErr, "This node belongs to a different document, use importNode.") + if nList.len > 0: + # Check if newChild is from this nodes document + if nList[0].fOwnerDocument != arg.fOwnerDocument: + raise newException(EWrongDocumentErr, "This node belongs to a different document, use importNode.") if not isNil(arg.fOwnerElement): raise newException(EInuseAttributeErr, "This attribute is in use by another element, use cloneNode") @@ -814,11 +803,10 @@ proc setNamedItem*(nList: var seq[PAttr], arg: PAttr): PAttr = proc setNamedItemNS*(nList: var seq[PNode], arg: PNode): PNode = ## Adds a node using its ``namespaceURI`` and ``localName`` - if not isNil(nList): - if nList.len() > 0: - # Check if newChild is from this nodes document - if nList[0].fOwnerDocument != arg.fOwnerDocument: - raise newException(EWrongDocumentErr, "This node belongs to a different document, use importNode.") + if nList.len > 0: + # Check if newChild is from this nodes document + if nList[0].fOwnerDocument != arg.fOwnerDocument: + raise newException(EWrongDocumentErr, "This node belongs to a different document, use importNode.") #Exceptions end var item: PNode = nList.getNamedItemNS(arg.namespaceURI(), arg.localName()) @@ -837,11 +825,10 @@ proc setNamedItemNS*(nList: var seq[PNode], arg: PNode): PNode = proc setNamedItemNS*(nList: var seq[PAttr], arg: PAttr): PAttr = ## Adds a node using its ``namespaceURI`` and ``localName`` - if not isNil(nList): - if nList.len() > 0: - # Check if newChild is from this nodes document - if nList[0].fOwnerDocument != arg.fOwnerDocument: - raise newException(EWrongDocumentErr, "This node belongs to a different document, use importNode.") + if nList.len > 0: + # Check if newChild is from this nodes document + if nList[0].fOwnerDocument != arg.fOwnerDocument: + raise newException(EWrongDocumentErr, "This node belongs to a different document, use importNode.") if not isNil(arg.fOwnerElement): raise newException(EInuseAttributeErr, "This attribute is in use by another element, use cloneNode") @@ -868,17 +855,14 @@ proc setNamedItemNS*(nList: var seq[PAttr], arg: PAttr): PAttr = # Attributes proc name*(a: PAttr): string = ## Returns the name of the Attribute - return a.fName proc specified*(a: PAttr): bool = ## Specifies whether this attribute was specified in the original document - return a.fSpecified proc ownerElement*(a: PAttr): PElement = ## Returns this Attributes owner element - return a.fOwnerElement # Element @@ -886,41 +870,32 @@ proc ownerElement*(a: PAttr): PElement = proc tagName*(el: PElement): string = ## Returns the Element Tag Name - return el.fTagName # Procedures proc getAttribute*(el: PNode, name: string): string = ## Retrieves an attribute value by ``name`` - if isNil(el.attributes): - return nil var attribute = el.attributes.getNamedItem(name) if not isNil(attribute): return attribute.value else: - return nil + return "" proc getAttributeNS*(el: PNode, namespaceURI: string, localName: string): string = ## Retrieves an attribute value by ``localName`` and ``namespaceURI`` - if isNil(el.attributes): - return nil var attribute = el.attributes.getNamedItemNS(namespaceURI, localName) if not isNil(attribute): return attribute.value else: - return nil + return "" proc getAttributeNode*(el: PElement, name: string): PAttr = ## Retrieves an attribute node by ``name`` ## To retrieve an attribute node by qualified name and namespace URI, use the `getAttributeNodeNS` method - if isNil(el.attributes): - return nil return el.attributes.getNamedItem(name) proc getAttributeNodeNS*(el: PElement, namespaceURI: string, localName: string): PAttr = ## Retrieves an `Attr` node by ``localName`` and ``namespaceURI`` - if isNil(el.attributes): - return nil return el.attributes.getNamedItemNS(namespaceURI, localName) proc getElementsByTagName*(el: PElement, name: string): seq[PNode] = @@ -938,41 +913,34 @@ proc getElementsByTagNameNS*(el: PElement, namespaceURI: string, localName: stri proc hasAttribute*(el: PElement, name: string): bool = ## Returns ``true`` when an attribute with a given ``name`` is specified ## on this element , ``false`` otherwise. - if isNil(el.attributes): - return false return not isNil(el.attributes.getNamedItem(name)) proc hasAttributeNS*(el: PElement, namespaceURI: string, localName: string): bool = ## Returns ``true`` when an attribute with a given ``localName`` and ## ``namespaceURI`` is specified on this element , ``false`` otherwise - if isNil(el.attributes): - return false return not isNil(el.attributes.getNamedItemNS(namespaceURI, localName)) proc removeAttribute*(el: PElement, name: string) = ## Removes an attribute by ``name`` - if not isNil(el.attributes): - for i in low(el.attributes)..high(el.attributes): - if el.attributes[i].fName == name: - el.attributes.delete(i) + for i in low(el.attributes)..high(el.attributes): + if el.attributes[i].fName == name: + el.attributes.delete(i) proc removeAttributeNS*(el: PElement, namespaceURI: string, localName: string) = ## Removes an attribute by ``localName`` and ``namespaceURI`` - if not isNil(el.attributes): - for i in low(el.attributes)..high(el.attributes): - if el.attributes[i].fNamespaceURI == namespaceURI and - el.attributes[i].fLocalName == localName: - el.attributes.delete(i) + for i in low(el.attributes)..high(el.attributes): + if el.attributes[i].fNamespaceURI == namespaceURI and + el.attributes[i].fLocalName == localName: + el.attributes.delete(i) proc removeAttributeNode*(el: PElement, oldAttr: PAttr): PAttr = ## Removes the specified attribute node ## If the attribute node cannot be found raises ``ENotFoundErr`` - if not isNil(el.attributes): - for i in low(el.attributes)..high(el.attributes): - if el.attributes[i] == oldAttr: - result = el.attributes[i] - el.attributes.delete(i) - return + for i in low(el.attributes)..high(el.attributes): + if el.attributes[i] == oldAttr: + result = el.attributes[i] + el.attributes.delete(i) + return raise newException(ENotFoundErr, "oldAttr is not a member of el's Attributes") @@ -991,7 +959,6 @@ proc setAttributeNode*(el: PElement, newAttr: PAttr): PAttr = "This attribute is in use by another element, use cloneNode") # Exceptions end - if isNil(el.attributes): el.attributes = @[] return el.attributes.setNamedItem(newAttr) proc setAttributeNodeNS*(el: PElement, newAttr: PAttr): PAttr = @@ -1009,7 +976,6 @@ proc setAttributeNodeNS*(el: PElement, newAttr: PAttr): PAttr = "This attribute is in use by another element, use cloneNode") # Exceptions end - if isNil(el.attributes): el.attributes = @[] return el.attributes.setNamedItemNS(newAttr) proc setAttribute*(el: PElement, name: string, value: string) = @@ -1057,9 +1023,9 @@ proc splitData*(textNode: PText, offset: int): PText = var left: string = textNode.data.substr(0, offset) textNode.data = left - var right: string = textNode.data.substr(offset, textNode.data.len()) + var right: string = textNode.data.substr(offset, textNode.data.len) - if not isNil(textNode.fParentNode) and not isNil(textNode.fParentNode.childNodes): + if not isNil(textNode.fParentNode) and textNode.fParentNode.childNodes.len > 0: for i in low(textNode.fParentNode.childNodes)..high(textNode.fParentNode.childNodes): if textNode.fParentNode.childNodes[i] == textNode: var newNode: PText = textNode.fOwnerDocument.createTextNode(right) @@ -1069,17 +1035,15 @@ proc splitData*(textNode: PText, offset: int): PText = var newNode: PText = textNode.fOwnerDocument.createTextNode(right) return newNode - # ProcessingInstruction proc target*(pi: PProcessingInstruction): string = ## Returns the Processing Instructions target return pi.fTarget - -# --Other stuff-- -# Writer -proc addEscaped(s: string): string = +proc escapeXml*(s: string; result: var string) = + ## Prepares a string for insertion into a XML document + ## by escaping the XML special characters. result = "" for c in items(s): case c @@ -1089,13 +1053,21 @@ proc addEscaped(s: string): string = of '"': result.add(""") else: result.add(c) +proc escapeXml*(s: string): string = + ## Prepares a string for insertion into a XML document + ## by escaping the XML special characters. + result = newStringOfCap(s.len + s.len shr 4) + escapeXml(s, result) + +# --Other stuff-- +# Writer + proc nodeToXml(n: PNode, indent: int = 0): string = result = spaces(indent) & "<" & n.nodeName - if not isNil(n.attributes): - for i in items(n.attributes): - result.add(" " & i.name & "=\"" & addEscaped(i.value) & "\"") + for i in items(n.attributes): + result.add(" " & i.name & "=\"" & escapeXml(i.value) & "\"") - if isNil(n.childNodes) or n.childNodes.len() == 0: + if n.childNodes.len == 0: result.add("/>") # No idea why this doesn't need a \n :O else: # End the beginning of this tag @@ -1106,7 +1078,7 @@ proc nodeToXml(n: PNode, indent: int = 0): string = result.add(nodeToXml(i, indent + 2)) of TextNode: result.add(spaces(indent * 2)) - result.add(addEscaped(i.nodeValue)) + result.add(escapeXml(i.nodeValue)) of CDataSectionNode: result.add(spaces(indent * 2)) result.add("<![CDATA[" & i.nodeValue & "]]>") @@ -1127,3 +1099,4 @@ proc `$`*(doc: PDocument): string = ## Converts a PDocument object into a string representation of it's XML result = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n" result.add(nodeToXml(doc.documentElement)) + \ No newline at end of file diff --git a/lib/pure/xmldomparser.nim b/lib/pure/xmldomparser.nim index 7c7f7b99c..8d995102e 100644 --- a/lib/pure/xmldomparser.nim +++ b/lib/pure/xmldomparser.nim @@ -119,7 +119,7 @@ proc loadXMLStream*(stream: Stream): PDocument = ## a ``PDocument`` var x: XmlParser - open(x, stream, nil, {reportComments}) + open(x, stream, "", {reportComments}) var xmlDoc: PDocument var dom: PDOMImplementation = getDOM() @@ -161,7 +161,7 @@ when not defined(testing) and isMainModule: #echo(xml.getElementsByTagName("bla:test")[0].namespaceURI) #echo(xml.getElementsByTagName("test")[0].namespaceURI) for i in items(xml.getElementsByTagName("*")): - if i.namespaceURI != nil: + if i.namespaceURI.len > 0: echo(i.nodeName, "=", i.namespaceURI) diff --git a/lib/pure/xmlparser.nim b/lib/pure/xmlparser.nim index 22bd259b7..597b80eb5 100644 --- a/lib/pure/xmlparser.nim +++ b/lib/pure/xmlparser.nim @@ -12,11 +12,9 @@ import streams, parsexml, strtabs, xmltree type - XmlError* = object of ValueError ## exception that is raised - ## for invalid XML - errors*: seq[string] ## all detected parsing errors - -{.deprecated: [EInvalidXml: XmlError].} + XmlError* = object of ValueError ## Exception that is raised + ## for invalid XML. + errors*: seq[string] ## All detected parsing errors. proc raiseInvalidXml(errors: seq[string]) = var e: ref XmlError @@ -102,8 +100,8 @@ proc parse(x: var XmlParser, errors: var seq[string]): XmlNode = proc parseXml*(s: Stream, filename: string, errors: var seq[string]): XmlNode = - ## parses the XML from stream `s` and returns a ``PXmlNode``. Every - ## occurred parsing error is added to the `errors` sequence. + ## Parses the XML from stream ``s`` and returns a ``XmlNode``. Every + ## occurred parsing error is added to the ``errors`` sequence. var x: XmlParser open(x, s, filename, {reportComments}) while true: @@ -121,15 +119,20 @@ proc parseXml*(s: Stream, filename: string, close(x) proc parseXml*(s: Stream): XmlNode = - ## parses the XTML from stream `s` and returns a ``PXmlNode``. All parsing - ## errors are turned into an ``EInvalidXML`` exception. + ## Parses the XML from stream ``s`` and returns a ``XmlNode``. All parsing + ## errors are turned into an ``XmlError`` exception. var errors: seq[string] = @[] - result = parseXml(s, "unknown_html_doc", errors) + result = parseXml(s, "unknown_xml_doc", errors) if errors.len > 0: raiseInvalidXml(errors) +proc parseXml*(str: string): XmlNode = + ## Parses the XML from string ``str`` and returns a ``XmlNode``. All parsing + ## errors are turned into an ``XmlError`` exception. + parseXml(newStringStream(str)) + proc loadXml*(path: string, errors: var seq[string]): XmlNode = ## Loads and parses XML from file specified by ``path``, and returns - ## a ``PXmlNode``. Every occurred parsing error is added to the `errors` + ## a ``XmlNode``. Every occurred parsing error is added to the ``errors`` ## sequence. var s = newFileStream(path, fmRead) if s == nil: raise newException(IOError, "Unable to read file: " & path) @@ -137,7 +140,7 @@ proc loadXml*(path: string, errors: var seq[string]): XmlNode = proc loadXml*(path: string): XmlNode = ## Loads and parses XML from file specified by ``path``, and returns - ## a ``PXmlNode``. All parsing errors are turned into an ``EInvalidXML`` + ## a ``XmlNode``. All parsing errors are turned into an ``XmlError`` ## exception. var errors: seq[string] = @[] result = loadXml(path, errors) diff --git a/lib/pure/xmltree.nim b/lib/pure/xmltree.nim index 45696c80c..d536cfed0 100644 --- a/lib/pure/xmltree.nim +++ b/lib/pure/xmltree.nim @@ -9,12 +9,12 @@ ## A simple XML tree. More efficient and simpler than the DOM. -import macros, strtabs +import macros, strtabs, strutils type - XmlNode* = ref XmlNodeObj ## an XML tree consists of ``PXmlNode``'s. + XmlNode* = ref XmlNodeObj ## an XML tree consists of ``XmlNode``'s. - XmlNodeKind* = enum ## different kinds of ``PXmlNode``'s + XmlNodeKind* = enum ## different kinds of ``XmlNode``'s xnText, ## a text element xnElement, ## an element with 0 or more children xnCData, ## a CDATA node @@ -33,9 +33,6 @@ type fAttr: XmlAttributes fClientData: int ## for other clients -{.deprecated: [PXmlNode: XmlNode, TXmlNodeKind: XmlNodeKind, PXmlAttributes: - XmlAttributes, TXmlNode: XmlNodeObj].} - proc newXmlNode(kind: XmlNodeKind): XmlNode = ## creates a new ``XmlNode``. new(result) @@ -155,11 +152,6 @@ proc `[]`* (n: var XmlNode, i: int): var XmlNode {.inline.} = assert n.k == xnElement result = n.s[i] -proc mget*(n: var XmlNode, i: int): var XmlNode {.inline, deprecated.} = - ## returns the `i`'th child of `n` so that it can be modified. Use ```[]``` - ## instead. - n[i] - iterator items*(n: XmlNode): XmlNode {.inline.} = ## iterates over any child of `n`. assert n.k == xnElement @@ -225,8 +217,9 @@ proc escape*(s: string): string = result = newStringOfCap(s.len) addEscaped(result, s) -proc addIndent(result: var string, indent: int) = - result.add("\n") +proc addIndent(result: var string, indent: int, addNewLines: bool) = + if addNewLines: + result.add("\n") for i in 1..indent: result.add(' ') proc noWhitespace(n: XmlNode): bool = @@ -235,7 +228,8 @@ proc noWhitespace(n: XmlNode): bool = for i in 0..n.len-1: if n[i].kind in {xnText, xnEntity}: return true -proc add*(result: var string, n: XmlNode, indent = 0, indWidth = 2) = +proc add*(result: var string, n: XmlNode, indent = 0, indWidth = 2, + addNewLines=true) = ## adds the textual representation of `n` to `result`. proc addEscapedAttr(result: var string, s: string) = @@ -268,14 +262,15 @@ proc add*(result: var string, n: XmlNode, indent = 0, indWidth = 2) = # for mixed leaves, we cannot output whitespace for readability, # because this would be wrong. For example: ``a<b>b</b>`` is # different from ``a <b>b</b>``. - for i in 0..n.len-1: result.add(n[i], indent+indWidth, indWidth) + for i in 0..n.len-1: + result.add(n[i], indent+indWidth, indWidth, addNewLines) else: for i in 0..n.len-1: - result.addIndent(indent+indWidth) - result.add(n[i], indent+indWidth, indWidth) - result.addIndent(indent) + result.addIndent(indent+indWidth, addNewLines) + result.add(n[i], indent+indWidth, indWidth, addNewLines) + result.addIndent(indent, addNewLines) else: - result.add(n[0], indent+indWidth, indWidth) + result.add(n[0], indent+indWidth, indWidth, addNewLines) result.add("</") result.add(n.fTag) result.add(">") @@ -315,18 +310,19 @@ proc newXmlTree*(tag: string, children: openArray[XmlNode], for i in 0..children.len-1: result.s[i] = children[i] result.fAttr = attributes -proc xmlConstructor(e: NimNode): NimNode {.compileTime.} = - expectLen(e, 2) - var a = e[1] +proc xmlConstructor(a: NimNode): NimNode {.compileTime.} = if a.kind == nnkCall: result = newCall("newXmlTree", toStrLit(a[0])) var attrs = newNimNode(nnkBracket, a) - var newStringTabCall = newCall("newStringTable", attrs, - newIdentNode("modeCaseSensitive")) + var newStringTabCall = newCall(bindSym"newStringTable", attrs, + bindSym"modeCaseSensitive") var elements = newNimNode(nnkBracket, a) for i in 1..a.len-1: if a[i].kind == nnkExprEqExpr: - attrs.add(toStrLit(a[i][0])) + # In order to support attributes like `data-lang` we have to + # replace whitespace because `toStrLit` gives `data - lang`. + let attrName = toStrLit(a[i][0]).strVal.replace(" ", "") + attrs.add(newStrLitNode(attrName)) attrs.add(a[i][1]) #echo repr(attrs) else: @@ -348,7 +344,6 @@ macro `<>`*(x: untyped): untyped = ## ## <a href="http://nim-lang.org">Nim rules.</a> ## - let x = callsite() result = xmlConstructor(x) proc child*(n: XmlNode, name: string): XmlNode = @@ -382,7 +377,6 @@ proc findAll*(n: XmlNode, tag: string, result: var seq[XmlNode]) = ## findAll(html, "img", tags) ## for imgTag in tags: ## process(imgTag) - assert isNil(result) == false assert n.k == xnElement for child in n.items(): if child.k != xnElement: |