diff options
Diffstat (limited to 'lib/pure')
113 files changed, 11459 insertions, 13265 deletions
diff --git a/lib/pure/algorithm.nim b/lib/pure/algorithm.nim index 0410d65ee..b12ed7cdd 100644 --- a/lib/pure/algorithm.nim +++ b/lib/pure/algorithm.nim @@ -44,6 +44,10 @@ runnableExamples: import std/private/since +when defined(nimPreviewSlimSystem): + import std/assertions + + type SortOrder* = enum Descending, Ascending @@ -131,43 +135,29 @@ proc reverse*[T](a: var openArray[T]) = # the max is needed, since a.high is -1 if a is empty reverse(a, 0, max(0, a.high)) -proc reversed*[T](a: openArray[T], first: Natural, last: int): seq[T] = - ## Returns the reverse of the slice `a[first..last]`. - ## - ## If an invalid range is passed, it raises `IndexDefect`. +proc reversed*[T](a: openArray[T]): seq[T] {.inline.} = + ## Returns the elements of `a` in reverse order. ## ## **See also:** - ## * `reverse proc<#reverse,openArray[T],Natural,Natural>`_ reverse a slice ## * `reverse proc<#reverse,openArray[T]>`_ runnableExamples: - let - a = [1, 2, 3, 4, 5, 6] - b = a.reversed(1, 3) - assert b == @[4, 3, 2] - assert last >= first - 1 - var i = last - first - var x = first.int - result = newSeq[T](i + 1) - while i >= 0: - result[i] = a[x] - dec(i) - inc(x) + assert [10, 11, 12].reversed == @[12, 11, 10] + assert seq[string].default.reversed == @[] + let n = a.len + result.setLen(n) + for i in 0..<n: result[i] = a[n - (i + 1)] -proc reversed*[T](a: openArray[T]): seq[T] = - ## Returns the reverse of the container `a`. - ## - ## **See also:** - ## * `reverse proc<#reverse,openArray[T],Natural,Natural>`_ reverse a slice - ## * `reverse proc<#reverse,openArray[T]>`_ - runnableExamples: - let - a = [1, 2, 3, 4, 5, 6] - b = reversed(a) - assert b == @[6, 5, 4, 3, 2, 1] - reversed(a, 0, a.high) +proc reversed*[T](a: openArray[T], first: Natural, last: int): seq[T] + {.inline, deprecated: "use: `reversed(toOpenArray(a, first, last))`".} = + reversed(toOpenArray(a, first, last)) + +when defined(nimHasEffectsOf): + {.experimental: "strictEffects".} +else: + {.pragma: effectsOf.} proc binarySearch*[T, K](a: openArray[T], key: K, - cmp: proc (x: T, y: K): int {.closure.}): int = + cmp: proc (x: T, y: K): int {.closure.}): int {.effectsOf: cmp.} = ## Binary search for `key` in `a`. Return the index of `key` or -1 if not found. ## Assumes that `a` is sorted according to `cmp`. ## @@ -229,7 +219,7 @@ const onlySafeCode = true proc lowerBound*[T, K](a: openArray[T], key: K, - cmp: proc(x: T, k: K): int {.closure.}): int = + cmp: proc(x: T, k: K): int {.closure.}): int {.effectsOf: cmp.} = ## Returns the index of the first element in `a` that is not less than ## (i.e. greater or equal to) `key`, or last if no such element is found. ## In other words if you have a sorted sequence and you call @@ -279,7 +269,7 @@ proc lowerBound*[T](a: openArray[T], key: T): int = lowerBound(a, key, cmp[T]) ## * `upperBound proc<#upperBound,openArray[T],T>`_ proc upperBound*[T, K](a: openArray[T], key: K, - cmp: proc(x: T, k: K): int {.closure.}): int = + cmp: proc(x: T, k: K): int {.closure.}): int {.effectsOf: cmp.} = ## Returns the index of the first element in `a` that is greater than ## `key`, or last if no such element is found. ## In other words if you have a sorted sequence and you call @@ -336,12 +326,12 @@ template `<-`(a, b) = else: copyMem(addr(a), addr(b), sizeof(T)) -proc merge[T](a, b: var openArray[T], lo, m, hi: int, - cmp: proc (x, y: T): int {.closure.}, order: SortOrder) = +proc mergeAlt[T](a, b: var openArray[T], lo, m, hi: int, + cmp: proc (x, y: T): int {.closure.}, order: SortOrder) {.effectsOf: cmp.} = # Optimization: If max(left) <= min(right) there is nothing to do! # 1 2 3 4 ## 5 6 7 8 # -> O(n) for sorted arrays. - # On random data this saves up to 40% of merge calls. + # On random data this saves up to 40% of mergeAlt calls. if cmp(a[m], a[m+1]) * order <= 0: return var j = lo # copy a[j..m] into b: @@ -377,7 +367,7 @@ proc merge[T](a, b: var openArray[T], lo, m, hi: int, func sort*[T](a: var openArray[T], cmp: proc (x, y: T): int {.closure.}, - order = SortOrder.Ascending) = + order = SortOrder.Ascending) {.effectsOf: cmp.} = ## Default Nim sort (an implementation of merge sort). The sorting ## is guaranteed to be stable (that is, equal elements stay in the same order) ## and the worst case is guaranteed to be O(n log n). @@ -389,22 +379,22 @@ func sort*[T](a: var openArray[T], ## `cmp`, you may use `system.cmp` or instead call the overloaded ## version of `sort`, which uses `system.cmp`. ## - ## .. code-block:: nim - ## - ## sort(myIntArray, system.cmp[int]) - ## # do not use cmp[string] here as we want to use the specialized - ## # overload: - ## sort(myStrArray, system.cmp) + ## ```nim + ## sort(myIntArray, system.cmp[int]) + ## # do not use cmp[string] here as we want to use the specialized + ## # overload: + ## sort(myStrArray, system.cmp) + ## ``` ## ## You can inline adhoc comparison procs with the `do notation - ## <manual_experimental.html#do-notation>`_. Example: - ## - ## .. code-block:: nim + ## <manual.html#procedures-do-notation>`_. Example: ## + ## ```nim ## people.sort do (x, y: Person) -> int: ## result = cmp(x.surname, y.surname) ## if result == 0: ## result = cmp(x.name, y.name) + ## ``` ## ## **See also:** ## * `sort proc<#sort,openArray[T]>`_ @@ -424,7 +414,7 @@ func sort*[T](a: var openArray[T], while s < n: var m = n-1-s while m >= 0: - merge(a, b, max(m-s+1, 0), m, m+s, cmp, order) + mergeAlt(a, b, max(m-s+1, 0), m, m+s, cmp, order) dec(m, s*2) s = s*2 @@ -439,7 +429,7 @@ proc sort*[T](a: var openArray[T], order = SortOrder.Ascending) = sort[T](a, ## * `sortedByIt template<#sortedByIt.t,untyped,untyped>`_ proc sorted*[T](a: openArray[T], cmp: proc(x, y: T): int {.closure.}, - order = SortOrder.Ascending): seq[T] = + order = SortOrder.Ascending): seq[T] {.effectsOf: cmp.} = ## Returns `a` sorted by `cmp` in the specified `order`. ## ## **See also:** @@ -516,7 +506,7 @@ template sortedByIt*(seq1, op: untyped): untyped = func isSorted*[T](a: openArray[T], cmp: proc(x, y: T): int {.closure.}, - order = SortOrder.Ascending): bool = + order = SortOrder.Ascending): bool {.effectsOf: cmp.} = ## Checks to see whether `a` is already sorted in `order` ## using `cmp` for the comparison. The parameters are identical ## to `sort`. Requires O(n) time. @@ -564,7 +554,7 @@ proc isSorted*[T](a: openArray[T], order = SortOrder.Ascending): bool = proc merge*[T]( result: var seq[T], x, y: openArray[T], cmp: proc(x, y: T): int {.closure.} -) {.since: (1, 5, 1).} = +) {.since: (1, 5, 1), effectsOf: cmp.} = ## Merges two sorted `openArray`. `x` and `y` are assumed to be sorted. ## If you do not wish to provide your own `cmp`, ## you may use `system.cmp` or instead call the overloaded @@ -657,7 +647,7 @@ proc product*[T](x: openArray[seq[T]]): seq[seq[T]] = ## Produces the Cartesian product of the array. ## Every element of the result is a combination of one element from each seq in `x`, ## with the ith element coming from `x[i]`. - ## + ## ## .. warning:: complexity may explode. runnableExamples: assert product(@[@[1], @[2]]) == @[@[1, 2]] @@ -835,10 +825,10 @@ proc rotateLeft*[T](arg: var openArray[T]; slice: HSlice[int, int]; ## If an invalid range (`HSlice`) is passed, it raises `IndexDefect`. ## ## `slice` - ## The indices of the element range that should be rotated. + ## : The indices of the element range that should be rotated. ## ## `dist` - ## The distance in amount of elements that the data should be rotated. + ## : The distance in amount of elements that the data should be rotated. ## Can be negative, can be any number. ## ## **See also:** @@ -886,10 +876,10 @@ proc rotatedLeft*[T](arg: openArray[T]; slice: HSlice[int, int], ## If an invalid range (`HSlice`) is passed, it raises `IndexDefect`. ## ## `slice` - ## The indices of the element range that should be rotated. + ## : The indices of the element range that should be rotated. ## ## `dist` - ## The distance in amount of elements that the data should be rotated. + ## : The distance in amount of elements that the data should be rotated. ## Can be negative, can be any number. ## ## **See also:** diff --git a/lib/pure/async.nim b/lib/pure/async.nim index 97b29f81d..e4d8d41c3 100644 --- a/lib/pure/async.nim +++ b/lib/pure/async.nim @@ -1,6 +1,9 @@ +## Exports [asyncmacro](asyncmacro.html) and [asyncfutures](asyncfutures.html) for native backends, +## and [asyncjs](asyncjs.html) on the JS backend. + when defined(js): - import asyncjs - export asyncjs + import std/asyncjs + export asyncjs else: - import asyncmacro, asyncfutures - export asyncmacro, asyncfutures + import std/[asyncmacro, asyncfutures] + export asyncmacro, asyncfutures diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index 55a20270d..126db7a7f 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -41,13 +41,13 @@ ## requested amount of data is read **or** an exception occurs. ## ## Code to read some data from a socket may look something like this: -## -## .. code-block::nim -## var future = socket.recv(100) -## future.addCallback( -## proc () = -## echo(future.read) -## ) +## ```Nim +## var future = socket.recv(100) +## future.addCallback( +## proc () = +## echo(future.read) +## ) +## ``` ## ## All asynchronous functions returning a `Future` will not block. They ## will not however return immediately. An asynchronous function will have @@ -61,8 +61,8 @@ ## callback on this future which will be called once the future completes. ## All the callback does is write the data stored in the future to `stdout`. ## The `read` function is used for this and it checks whether the future -## completes with an error for you (if it did it will simply raise the -## error), if there is no error however it returns the value of the future. +## completes with an error for you (if it did, it will simply raise the +## error), if there is no error, however, it returns the value of the future. ## ## Asynchronous procedures ## ======================= @@ -98,40 +98,71 @@ ## `await`. The following section shows different ways that you can handle ## exceptions in async procs. ## +## .. caution:: +## Procedures marked {.async.} do not support mutable parameters such +## as `var int`. References such as `ref int` should be used instead. +## ## Handling Exceptions ## ------------------- ## -## The most reliable way to handle exceptions is to use `yield` on a future -## then check the future's `failed` property. For example: +## You can handle exceptions in the same way as in ordinary Nim code; +## by using the try statement: ## -## .. code-block:: Nim -## var future = sock.recv(100) -## yield future -## if future.failed: -## # Handle exception +## ```Nim +## try: +## let data = await sock.recv(100) +## echo("Received ", data) +## except: +## # Handle exception +## ``` ## -## The `async` procedures also offer limited support for the try statement. -## -## .. code-block:: Nim -## try: -## let data = await sock.recv(100) -## echo("Received ", data) -## except: -## # Handle exception +## An alternative approach to handling exceptions is to use `yield` on a future +## then check the future's `failed` property. For example: ## -## Unfortunately the semantics of the try statement may not always be correct, -## and occasionally the compilation may fail altogether. -## As such it is better to use the former style when possible. +## ```Nim +## var future = sock.recv(100) +## yield future +## if future.failed: +## # Handle exception +## ``` ## ## ## Discarding futures ## ================== ## -## Futures should **never** be discarded. This is because they may contain -## errors. If you do not care for the result of a Future then you should -## use the `asyncCheck` procedure instead of the `discard` keyword. Note -## however that this does not wait for completion, and you should use -## `waitFor` for that purpose. +## Futures should **never** be discarded directly because they may contain +## errors. If you do not care for the result of a Future then you should use +## the `asyncCheck` procedure instead of the `discard` keyword. Note that this +## does not wait for completion, and you should use `waitFor` or `await` for that purpose. +## +## .. note:: `await` also checks if the future fails, so you can safely discard +## its result. +## +## Handling futures +## ================ +## +## There are many different operations that apply to a future. +## The three primary high-level operations are `asyncCheck`, +## `waitFor`, and `await`. +## +## * `asyncCheck`: Raises an exception if the future fails. It neither waits +## for the future to finish nor returns the result of the future. +## * `waitFor`: Polls the event loop and blocks the current thread until the +## future finishes. This is often used to call an async procedure from a +## synchronous context and should never be used in an `async` proc. +## * `await`: Pauses execution in the current async procedure until the future +## finishes. While the current procedure is paused, other async procedures will +## continue running. Should be used instead of `waitFor` in an async +## procedure. +## +## Here is a handy quick reference chart showing their high-level differences: +## ============== ===================== ======================= +## Procedure Context Blocking +## ============== ===================== ======================= +## `asyncCheck` non-async and async non-blocking +## `waitFor` non-async blocks current thread +## `await` async suspends current proc +## ============== ===================== ======================= ## ## Examples ## ======== @@ -165,19 +196,49 @@ ## ================ ## ## * The effect system (`raises: []`) does not work with async procedures. +## * Mutable parameters are not supported by async procedures. +## +## +## Multiple async backend support +## ============================== +## +## Thanks to its powerful macro support, Nim allows ``async``/``await`` to be +## implemented in libraries with only minimal support from the language - as +## such, multiple ``async`` libraries exist, including ``asyncdispatch`` and +## ``chronos``, and more may come to be developed in the future. +## +## Libraries built on top of async/await may wish to support multiple async +## backends - the best way to do so is to create separate modules for each backend +## that may be imported side-by-side. +## +## An alternative way is to select backend using a global compile flag - this +## method makes it difficult to compose applications that use both backends as may +## happen with transitive dependencies, but may be appropriate in some cases - +## libraries choosing this path should call the flag `asyncBackend`, allowing +## applications to choose the backend with `-d:asyncBackend=<backend_name>`. +## +## Known `async` backends include: +## +## * `-d:asyncBackend=none`: disable `async` support completely +## * `-d:asyncBackend=asyncdispatch`: https://nim-lang.org/docs/asyncdispatch.html +## * `-d:asyncBackend=chronos`: https://github.com/status-im/nim-chronos/ +## +## ``none`` can be used when a library supports both a synchronous and +## asynchronous API, to disable the latter. + +import std/[os, tables, strutils, times, heapqueue, options, asyncstreams] +import std/[math, monotimes] +import std/asyncfutures except callSoon -import os, tables, strutils, times, heapqueue, options, asyncstreams -import options, math, std/monotimes -import asyncfutures except callSoon +import std/[nativesockets, net, deques] -import nativesockets, net, deques +when defined(nimPreviewSlimSystem): + import std/[assertions, syncio] export Port, SocketFlag export asyncfutures except callSoon export asyncstreams -#{.injectStmt: newGcInvariant().} - # TODO: Check if yielded future is nil and throw a more meaningful exception type @@ -220,6 +281,8 @@ proc adjustTimeout( result = max(nextTimer.get(), 0) result = min(pollTimeout, result) +proc runOnce(timeout: int): bool {.gcsafe.} + proc callSoon*(cbproc: proc () {.gcsafe.}) {.gcsafe.} ## Schedule `cbproc` to be called as soon as possible. ## The callback is called when control returns to the event loop. @@ -239,7 +302,7 @@ template implementSetInheritable() {.dirty.} = fd.FileHandle.setInheritable(inheritable) when defined(windows) or defined(nimdoc): - import winlean, sets, hashes + import std/[winlean, sets, hashes] type CompletionKey = ULONG_PTR @@ -329,7 +392,7 @@ when defined(windows) or defined(nimdoc): let p = getGlobalDispatcher() p.handles.len != 0 or p.timers.len != 0 or p.callbacks.len != 0 - proc runOnce(timeout = 500): bool = + proc runOnce(timeout: int): bool = let p = getGlobalDispatcher() if p.handles.len == 0 and p.timers.len == 0 and p.callbacks.len == 0: raise newException(ValueError, @@ -470,7 +533,7 @@ when defined(windows) or defined(nimdoc): if flags.isDisconnectionError(errcode): retFuture.complete("") else: - retFuture.fail(newException(OSError, osErrorMsg(errcode))) + retFuture.fail(newOSError(errcode)) if dataBuf.buf != nil: dealloc dataBuf.buf dataBuf.buf = nil @@ -488,7 +551,7 @@ when defined(windows) or defined(nimdoc): if flags.isDisconnectionError(err): retFuture.complete("") else: - retFuture.fail(newException(OSError, osErrorMsg(err))) + retFuture.fail(newOSError(err)) elif ret == 0: # Request completed immediately. if bytesReceived != 0: @@ -540,7 +603,7 @@ when defined(windows) or defined(nimdoc): if flags.isDisconnectionError(errcode): retFuture.complete(0) else: - retFuture.fail(newException(OSError, osErrorMsg(errcode))) + retFuture.fail(newOSError(errcode)) if dataBuf.buf != nil: dataBuf.buf = nil ) @@ -556,7 +619,7 @@ when defined(windows) or defined(nimdoc): if flags.isDisconnectionError(err): retFuture.complete(0) else: - retFuture.fail(newException(OSError, osErrorMsg(err))) + retFuture.fail(newOSError(err)) elif ret == 0: # Request completed immediately. if bytesReceived != 0: @@ -604,7 +667,7 @@ when defined(windows) or defined(nimdoc): if flags.isDisconnectionError(err): retFuture.complete() else: - retFuture.fail(newException(OSError, osErrorMsg(err))) + retFuture.fail(newOSError(err)) else: retFuture.complete() # We don't deallocate `ol` here because even though this completed @@ -639,7 +702,7 @@ when defined(windows) or defined(nimdoc): if errcode == OSErrorCode(-1): retFuture.complete() else: - retFuture.fail(newException(OSError, osErrorMsg(errcode))) + retFuture.fail(newOSError(errcode)) ) let ret = WSASendTo(socket.SocketHandle, addr dataBuf, 1, addr bytesSent, @@ -649,7 +712,7 @@ when defined(windows) or defined(nimdoc): let err = osLastError() if err.int32 != ERROR_IO_PENDING: GC_unref(ol) - retFuture.fail(newException(OSError, osErrorMsg(err))) + retFuture.fail(newOSError(err)) else: retFuture.complete() # We don't deallocate `ol` here because even though this completed @@ -683,7 +746,7 @@ when defined(windows) or defined(nimdoc): else: # datagram sockets don't have disconnection, # so we can just raise an exception - retFuture.fail(newException(OSError, osErrorMsg(errcode))) + retFuture.fail(newOSError(errcode)) ) let res = WSARecvFrom(socket.SocketHandle, addr dataBuf, 1, @@ -694,7 +757,7 @@ when defined(windows) or defined(nimdoc): let err = osLastError() if err.int32 != ERROR_IO_PENDING: GC_unref(ol) - retFuture.fail(newException(OSError, osErrorMsg(err))) + retFuture.fail(newOSError(err)) else: # Request completed immediately. if bytesReceived != 0: @@ -707,7 +770,7 @@ when defined(windows) or defined(nimdoc): proc acceptAddr*(socket: AsyncFD, flags = {SocketFlag.SafeDisconn}, inheritable = defined(nimInheritHandles)): - owned(Future[tuple[address: string, client: AsyncFD]]) = + owned(Future[tuple[address: string, client: AsyncFD]]) {.gcsafe.} = ## Accepts a new connection. Returns a future containing the client socket ## corresponding to that connection and the remote address of the client. ## The future will complete when the connection is successfully accepted. @@ -745,7 +808,7 @@ when defined(windows) or defined(nimdoc): else: retFuture.complete(newAcceptFut.read) else: - retFuture.fail(newException(OSError, osErrorMsg(errcode))) + retFuture.fail(newOSError(errcode)) template completeAccept() {.dirty.} = var listenSock = socket @@ -774,7 +837,7 @@ when defined(windows) or defined(nimdoc): var ol = newCustom() ol.data = CompletionData(fd: socket, cb: - proc (fd: AsyncFD, bytesCount: DWORD, errcode: OSErrorCode) = + proc (fd: AsyncFD, bytesCount: DWORD, errcode: OSErrorCode) {.gcsafe.} = if not retFuture.finished: if errcode == OSErrorCode(-1): completeAccept() @@ -1024,7 +1087,7 @@ when defined(windows) or defined(nimdoc): raiseOSError(osLastError()) var pcd = cast[PostCallbackDataPtr](allocShared0(sizeof(PostCallbackData))) - var flags = WT_EXECUTEINWAITTHREAD.DWORD + var flags = WT_EXECUTEINWAITTHREAD.DWORD or WT_EXECUTEONLYONCE.DWORD proc proccb(fd: AsyncFD, bytesCount: DWORD, errcode: OSErrorCode) = closeWaitable(hProcess) @@ -1103,11 +1166,14 @@ when defined(windows) or defined(nimdoc): initAll() else: - import selectors - from posix import EINTR, EAGAIN, EINPROGRESS, EWOULDBLOCK, MSG_PEEK, + import std/selectors + from std/posix import EINTR, EAGAIN, EINPROGRESS, EWOULDBLOCK, MSG_PEEK, MSG_NOSIGNAL when declared(posix.accept4): - from posix import accept4, SOCK_CLOEXEC + from std/posix import accept4, SOCK_CLOEXEC + when defined(genode): + import genode/env # get the implicit Genode env + import genode/signals const InitCallbackListSize = 4 # initial size of callbacks sequence, @@ -1126,6 +1192,8 @@ else: PDispatcher* = ref object of PDispatcherBase selector: Selector[AsyncData] + when defined(genode): + signalHandler: SignalHandler proc `==`*(x, y: AsyncFD): bool {.borrow.} proc `==`*(x, y: AsyncEvent): bool {.borrow.} @@ -1141,9 +1209,22 @@ else: result.selector = newSelector[AsyncData]() result.timers.clear() result.callbacks = initDeque[proc () {.closure, gcsafe.}](InitDelayedCallbackListSize) + when defined(genode): + let entrypoint = ep(cast[GenodeEnv](runtimeEnv)) + result.signalHandler = newSignalHandler(entrypoint): + discard runOnce(0) var gDisp{.threadvar.}: owned PDispatcher ## Global dispatcher + when defined(nuttx): + import std/exitprocs + + proc cleanDispatcher() {.noconv.} = + gDisp = nil + + proc addFinalyzer() = + addExitProc(cleanDispatcher) + proc setGlobalDispatcher*(disp: owned PDispatcher) = if not gDisp.isNil: assert gDisp.callbacks.len == 0 @@ -1153,6 +1234,8 @@ else: proc getGlobalDispatcher*(): PDispatcher = if gDisp.isNil: setGlobalDispatcher(newDispatcher()) + when defined(nuttx): + addFinalyzer() result = gDisp proc getIoHandler*(disp: PDispatcher): Selector[AsyncData] = @@ -1310,10 +1393,11 @@ else: ValueError, "Expecting async operations to stop when fd has closed." ) - - proc runOnce(timeout = 500): bool = + proc runOnce(timeout: int): bool = let p = getGlobalDispatcher() if p.selector.isEmpty() and p.timers.len == 0 and p.callbacks.len == 0: + when defined(genode): + if timeout == 0: return raise newException(ValueError, "No handles or timers registered in dispatcher.") @@ -1384,7 +1468,7 @@ else: if flags.isDisconnectionError(lastError): retFuture.complete("") else: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) + retFuture.fail(newOSError(lastError)) else: result = false # We still want this callback to be called. elif res == 0: @@ -1413,7 +1497,7 @@ else: if flags.isDisconnectionError(lastError): retFuture.complete(0) else: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) + retFuture.fail(newOSError(lastError)) else: result = false # We still want this callback to be called. else: @@ -1479,7 +1563,7 @@ else: let lastError = osLastError() if lastError.int32 != EINTR and lastError.int32 != EWOULDBLOCK and lastError.int32 != EAGAIN: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) + retFuture.fail(newOSError(lastError)) else: result = false # We still want this callback to be called. else: @@ -1505,7 +1589,7 @@ else: let lastError = osLastError() if lastError.int32 != EINTR and lastError.int32 != EWOULDBLOCK and lastError.int32 != EAGAIN: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) + retFuture.fail(newOSError(lastError)) else: result = false else: @@ -1518,7 +1602,7 @@ else: owned(Future[tuple[address: string, client: AsyncFD]]) = var retFuture = newFuture[tuple[address: string, client: AsyncFD]]("acceptAddr") - proc cb(sock: AsyncFD): bool = + proc cb(sock: AsyncFD): bool {.gcsafe.} = result = true var sockAddress: Sockaddr_storage var addrLen = sizeof(sockAddress).SockLen @@ -1546,7 +1630,7 @@ else: if flags.isDisconnectionError(lastError): return false else: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) + retFuture.fail(newOSError(lastError)) else: try: let address = getAddrString(cast[ptr SockAddr](addr sockAddress)) @@ -1675,9 +1759,11 @@ when defined(windows) or defined(nimdoc): proc (fd: AsyncFD, bytesCount: DWORD, errcode: OSErrorCode) = if not retFuture.finished: if errcode == OSErrorCode(-1): + const SO_UPDATE_CONNECT_CONTEXT = 0x7010 + socket.SocketHandle.setSockOptInt(SOL_SOCKET, SO_UPDATE_CONNECT_CONTEXT, 1) # 15022 retFuture.complete() else: - retFuture.fail(newException(OSError, osErrorMsg(errcode))) + retFuture.fail(newOSError(errcode)) ) let ret = connectEx(socket.SocketHandle, addrInfo.ai_addr, @@ -1695,7 +1781,7 @@ when defined(windows) or defined(nimdoc): # With ERROR_IO_PENDING `ol` will be deallocated in `poll`, # and the future will be completed/failed there, too. GC_unref(ol) - retFuture.fail(newException(OSError, osErrorMsg(lastError))) + retFuture.fail(newOSError(lastError)) else: proc doConnect(socket: AsyncFD, addrInfo: ptr AddrInfo): owned(Future[void]) = let retFuture = newFuture[void]("doConnect") @@ -1712,7 +1798,7 @@ else: # interrupted, keep waiting return false else: - retFuture.fail(newException(OSError, osErrorMsg(OSErrorCode(ret)))) + retFuture.fail(newOSError(OSErrorCode(ret))) return true let ret = connect(socket.SocketHandle, @@ -1726,7 +1812,7 @@ else: if lastError.int32 == EINTR or lastError.int32 == EINPROGRESS: addWrite(socket, cb) else: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) + retFuture.fail(newOSError(lastError)) template asyncAddrInfoLoop(addrInfo: ptr AddrInfo, fd: untyped, protocol: Protocol = IPPROTO_RAW) = @@ -1765,7 +1851,7 @@ template asyncAddrInfoLoop(addrInfo: ptr AddrInfo, fd: untyped, curAddrInfo = curAddrInfo.ai_next if curAddrInfo == nil: - freeaddrinfo(addrInfo) + freeAddrInfo(addrInfo) when shouldCreateFd: closeUnusedFds() if lastException != nil: @@ -1781,7 +1867,7 @@ template asyncAddrInfoLoop(addrInfo: ptr AddrInfo, fd: untyped, try: curFd = createAsyncNativeSocket(domain, sockType, protocol) except: - freeaddrinfo(addrInfo) + freeAddrInfo(addrInfo) closeUnusedFds() raise getCurrentException() when defined(windows): @@ -1791,7 +1877,7 @@ template asyncAddrInfoLoop(addrInfo: ptr AddrInfo, fd: untyped, doConnect(curFd, curAddrInfo).callback = tryNextAddrInfo curAddrInfo = curAddrInfo.ai_next else: - freeaddrinfo(addrInfo) + freeAddrInfo(addrInfo) when shouldCreateFd: closeUnusedFds(ord(domain)) retFuture.complete(curFd) @@ -1908,7 +1994,8 @@ proc send*(socket: AsyncFD, data: string, return retFuture # -- Await Macro -include asyncmacro +import std/asyncmacro +export asyncmacro proc readAll*(future: FutureStream[string]): owned(Future[string]) {.async.} = ## Returns a future that will complete when all the string data from the @@ -1945,17 +2032,34 @@ proc activeDescriptors*(): int {.inline.} = result = getGlobalDispatcher().selector.count when defined(posix): - import posix + import std/posix -when defined(linux) or defined(windows) or defined(macosx) or defined(bsd): +when defined(linux) or defined(windows) or defined(macosx) or defined(bsd) or + defined(solaris) or defined(zephyr) or defined(freertos) or defined(nuttx) or defined(haiku): proc maxDescriptors*(): int {.raises: OSError.} = ## Returns the maximum number of active file descriptors for the current ## process. This involves a system call. For now `maxDescriptors` is - ## supported on the following OSes: Windows, Linux, OSX, BSD. + ## supported on the following OSes: Windows, Linux, OSX, BSD, Solaris. when defined(windows): result = 16_700_000 + elif defined(zephyr) or defined(freertos): + result = FD_MAX else: var fdLim: RLimit if getrlimit(RLIMIT_NOFILE, fdLim) < 0: raiseOSError(osLastError()) result = int(fdLim.rlim_cur) - 1 + +when defined(genode): + proc scheduleCallbacks*(): bool {.discardable.} = + ## *Genode only.* + ## Schedule callback processing and return immediately. + ## Returns `false` if there is nothing to schedule. + ## RPC servers should call this to dispatch `callSoon` + ## bodies after retiring an RPC to its client. + ## This is effectively a non-blocking `poll(…)` and is + ## equivalent to scheduling a momentary no-op timeout + ## but faster and with less overhead. + let dis = getGlobalDispatcher() + result = dis.callbacks.len > 0 + if result: submit(dis.signalHandler.cap) diff --git a/lib/pure/asyncfile.nim b/lib/pure/asyncfile.nim index c51b338e3..0f6504342 100644 --- a/lib/pure/asyncfile.nim +++ b/lib/pure/asyncfile.nim @@ -9,27 +9,33 @@ ## This module implements asynchronous file reading and writing. ## -## .. code-block:: Nim -## import std/[asyncfile, asyncdispatch, os] +## ```Nim +## import std/[asyncfile, asyncdispatch, os] ## -## proc main() {.async.} = -## var file = openAsync(getTempDir() / "foobar.txt", fmReadWrite) -## await file.write("test") -## file.setFilePos(0) -## let data = await file.readAll() -## doAssert data == "test" -## file.close() +## proc main() {.async.} = +## var file = openAsync(getTempDir() / "foobar.txt", fmReadWrite) +## await file.write("test") +## file.setFilePos(0) +## let data = await file.readAll() +## doAssert data == "test" +## file.close() ## -## waitFor main() +## waitFor main() +## ``` -import asyncdispatch, os +import std/[asyncdispatch, os] + +when defined(nimPreviewSlimSystem): + import std/[assertions, syncio] + when defined(windows) or defined(nimdoc): + import std/widestrs # TODO: Fix duplication introduced by PR #4683. when defined(windows) or defined(nimdoc): - import winlean + import std/winlean else: - import posix + import std/posix type AsyncFile* = ref object @@ -96,14 +102,9 @@ proc openAsync*(filename: string, mode = fmRead): AsyncFile = let flags = FILE_FLAG_OVERLAPPED or FILE_ATTRIBUTE_NORMAL let desiredAccess = getDesiredAccess(mode) let creationDisposition = getCreationDisposition(mode, filename) - when useWinUnicode: - let fd = createFileW(newWideCString(filename), desiredAccess, - FILE_SHARE_READ, - nil, creationDisposition, flags, 0) - else: - let fd = createFileA(filename, desiredAccess, - FILE_SHARE_READ, - nil, creationDisposition, flags, 0) + let fd = createFileW(newWideCString(filename), desiredAccess, + FILE_SHARE_READ, + nil, creationDisposition, flags, 0) if fd == INVALID_HANDLE_VALUE: raiseOSError(osLastError()) @@ -145,7 +146,7 @@ proc readBuffer*(f: AsyncFile, buf: pointer, size: int): Future[int] = if errcode.int32 == ERROR_HANDLE_EOF: retFuture.complete(0) else: - retFuture.fail(newException(OSError, osErrorMsg(errcode))) + retFuture.fail(newOSError(errcode)) ) ol.offset = DWORD(f.offset and 0xffffffff) ol.offsetHigh = DWORD(f.offset shr 32) @@ -161,7 +162,7 @@ proc readBuffer*(f: AsyncFile, buf: pointer, size: int): Future[int] = # This happens in Windows Server 2003 retFuture.complete(0) else: - retFuture.fail(newException(OSError, osErrorMsg(err))) + retFuture.fail(newOSError(err)) else: # Request completed immediately. var bytesRead: DWORD @@ -172,7 +173,7 @@ proc readBuffer*(f: AsyncFile, buf: pointer, size: int): Future[int] = if err.int32 == ERROR_HANDLE_EOF: retFuture.complete(0) else: - retFuture.fail(newException(OSError, osErrorMsg(osLastError()))) + retFuture.fail(newOSError(osLastError())) else: assert bytesRead > 0 assert bytesRead <= size @@ -185,7 +186,7 @@ proc readBuffer*(f: AsyncFile, buf: pointer, size: int): Future[int] = if res < 0: let lastError = osLastError() if lastError.int32 != EAGAIN: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) + retFuture.fail(newOSError(lastError)) else: result = false # We still want this callback to be called. elif res == 0: @@ -202,10 +203,11 @@ proc readBuffer*(f: AsyncFile, buf: pointer, size: int): Future[int] = proc read*(f: AsyncFile, size: int): Future[string] = ## Read `size` bytes from the specified file asynchronously starting at - ## the current position of the file pointer. + ## the current position of the file pointer. `size` should be greater than zero. ## ## If the file pointer is past the end of the file then an empty string is ## returned. + assert size > 0 var retFuture = newFuture[string]("asyncfile.read") when defined(windows) or defined(nimdoc): @@ -226,7 +228,7 @@ proc read*(f: AsyncFile, size: int): Future[string] = if errcode.int32 == ERROR_HANDLE_EOF: retFuture.complete("") else: - retFuture.fail(newException(OSError, osErrorMsg(errcode))) + retFuture.fail(newOSError(errcode)) if buffer != nil: dealloc buffer buffer = nil @@ -249,7 +251,7 @@ proc read*(f: AsyncFile, size: int): Future[string] = # This happens in Windows Server 2003 retFuture.complete("") else: - retFuture.fail(newException(OSError, osErrorMsg(err))) + retFuture.fail(newOSError(err)) else: # Request completed immediately. var bytesRead: DWORD @@ -260,7 +262,7 @@ proc read*(f: AsyncFile, size: int): Future[string] = if err.int32 == ERROR_HANDLE_EOF: retFuture.complete("") else: - retFuture.fail(newException(OSError, osErrorMsg(osLastError()))) + retFuture.fail(newOSError(osLastError())) else: assert bytesRead > 0 assert bytesRead <= size @@ -277,7 +279,7 @@ proc read*(f: AsyncFile, size: int): Future[string] = if res < 0: let lastError = osLastError() if lastError.int32 != EAGAIN: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) + retFuture.fail(newOSError(lastError)) else: result = false # We still want this callback to be called. elif res == 0: @@ -299,6 +301,8 @@ proc readLine*(f: AsyncFile): Future[string] {.async.} = result = "" while true: var c = await read(f, 1) + if c.len == 0: + break if c[0] == '\c': c = await read(f, 1) break @@ -346,7 +350,7 @@ proc writeBuffer*(f: AsyncFile, buf: pointer, size: int): Future[void] = assert bytesCount == size.int32 retFuture.complete() else: - retFuture.fail(newException(OSError, osErrorMsg(errcode))) + retFuture.fail(newOSError(errcode)) ) # passing -1 here should work according to MSDN, but doesn't. For more # information see @@ -363,14 +367,14 @@ proc writeBuffer*(f: AsyncFile, buf: pointer, size: int): Future[void] = let err = osLastError() if err.int32 != ERROR_IO_PENDING: GC_unref(ol) - retFuture.fail(newException(OSError, osErrorMsg(err))) + retFuture.fail(newOSError(err)) else: # Request completed immediately. var bytesWritten: DWORD let overlappedRes = getOverlappedResult(f.fd.Handle, cast[POVERLAPPED](ol), bytesWritten, false.WINBOOL) if not overlappedRes.bool: - retFuture.fail(newException(OSError, osErrorMsg(osLastError()))) + retFuture.fail(newOSError(osLastError())) else: assert bytesWritten == size.int32 retFuture.complete() @@ -385,7 +389,7 @@ proc writeBuffer*(f: AsyncFile, buf: pointer, size: int): Future[void] = if res < 0: let lastError = osLastError() if lastError.int32 != EAGAIN: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) + retFuture.fail(newOSError(lastError)) else: result = false # We still want this callback to be called. else: @@ -419,7 +423,7 @@ proc write*(f: AsyncFile, data: string): Future[void] = assert bytesCount == data.len.int32 retFuture.complete() else: - retFuture.fail(newException(OSError, osErrorMsg(errcode))) + retFuture.fail(newOSError(errcode)) if buffer != nil: dealloc buffer buffer = nil @@ -438,14 +442,14 @@ proc write*(f: AsyncFile, data: string): Future[void] = dealloc buffer buffer = nil GC_unref(ol) - retFuture.fail(newException(OSError, osErrorMsg(err))) + retFuture.fail(newOSError(err)) else: # Request completed immediately. var bytesWritten: DWORD let overlappedRes = getOverlappedResult(f.fd.Handle, cast[POVERLAPPED](ol), bytesWritten, false.WINBOOL) if not overlappedRes.bool: - retFuture.fail(newException(OSError, osErrorMsg(osLastError()))) + retFuture.fail(newOSError(osLastError())) else: assert bytesWritten == data.len.int32 retFuture.complete() @@ -466,7 +470,7 @@ proc write*(f: AsyncFile, data: string): Future[void] = if res < 0: let lastError = osLastError() if lastError.int32 != EAGAIN: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) + retFuture.fail(newOSError(lastError)) else: result = false # We still want this callback to be called. else: diff --git a/lib/pure/asyncftpclient.nim b/lib/pure/asyncftpclient.nim deleted file mode 100644 index 0ee45785d..000000000 --- a/lib/pure/asyncftpclient.nim +++ /dev/null @@ -1,448 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Dominik Picheta -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## This module implements an asynchronous FTP client. It allows you to connect -## to an FTP server and perform operations on it such as for example: -## -## * The upload of new files. -## * The removal of existing files. -## * Download of files. -## * Changing of files' permissions. -## * Navigation through the FTP server's directories. -## -## Connecting to an FTP server -## =========================== -## -## In order to begin any sort of transfer of files you must first -## connect to an FTP server. You can do so with the `connect` procedure. -## -## .. code-block::nim -## import std/[asyncdispatch, asyncftpclient] -## proc main() {.async.} = -## var ftp = newAsyncFtpClient("example.com", user = "test", pass = "test") -## await ftp.connect() -## echo("Connected") -## waitFor(main()) -## -## A new `main` async procedure must be declared to allow the use of the -## `await` keyword. The connection will complete asynchronously and the -## client will be connected after the `await ftp.connect()` call. -## -## Uploading a new file -## ==================== -## -## After a connection is made you can use the `store` procedure to upload -## a new file to the FTP server. Make sure to check you are in the correct -## working directory before you do so with the `pwd` procedure, you can also -## instead specify an absolute path. -## -## .. code-block::nim -## import std/[asyncdispatch, asyncftpclient] -## proc main() {.async.} = -## var ftp = newAsyncFtpClient("example.com", user = "test", pass = "test") -## await ftp.connect() -## let currentDir = await ftp.pwd() -## assert currentDir == "/home/user/" -## await ftp.store("file.txt", "file.txt") -## echo("File finished uploading") -## waitFor(main()) -## -## Checking the progress of a file transfer -## ======================================== -## -## The progress of either a file upload or a file download can be checked -## by specifying a `onProgressChanged` procedure to the `store` or -## `retrFile` procedures. -## -## Procs that take an `onProgressChanged` callback will call this every -## `progressInterval` milliseconds. -## -## .. code-block::nim -## import std/[asyncdispatch, asyncftpclient] -## -## proc onProgressChanged(total, progress: BiggestInt, -## speed: float) {.async.} = -## echo("Uploaded ", progress, " of ", total, " bytes") -## echo("Current speed: ", speed, " kb/s") -## -## proc main() {.async.} = -## var ftp = newAsyncFtpClient("example.com", user = "test", pass = "test", progressInterval = 500) -## await ftp.connect() -## await ftp.store("file.txt", "/home/user/file.txt", onProgressChanged) -## echo("File finished uploading") -## waitFor(main()) - - -import asyncdispatch, asyncnet, nativesockets, strutils, parseutils, os, times -from net import BufferSize - -type - AsyncFtpClient* = ref object - csock*: AsyncSocket - dsock*: AsyncSocket - user*, pass*: string - address*: string - port*: Port - progressInterval: int - jobInProgress*: bool - job*: FtpJob - dsockConnected*: bool - - FtpJobType* = enum - JRetrText, JRetr, JStore - - FtpJob = ref object - prc: proc (ftp: AsyncFtpClient, async: bool): bool {.nimcall, gcsafe.} - case typ*: FtpJobType - of JRetrText: - lines: string - of JRetr, JStore: - file: File - filename: string - total: BiggestInt # In bytes. - progress: BiggestInt # In bytes. - oneSecond: BiggestInt # Bytes transferred in one second. - lastProgressReport: float # Time - toStore: string # Data left to upload (Only used with async) - - FtpEventType* = enum - EvTransferProgress, EvLines, EvRetr, EvStore - - FtpEvent* = object ## Event - filename*: string - case typ*: FtpEventType - of EvLines: - lines*: string ## Lines that have been transferred. - of EvRetr, EvStore: ## Retr/Store operation finished. - nil - of EvTransferProgress: - bytesTotal*: BiggestInt ## Bytes total. - bytesFinished*: BiggestInt ## Bytes transferred. - speed*: BiggestInt ## Speed in bytes/s - currentJob*: FtpJobType ## The current job being performed. - - ReplyError* = object of IOError - - ProgressChangedProc* = - proc (total, progress: BiggestInt, speed: float): - Future[void] {.closure, gcsafe.} - -const multiLineLimit = 10000 - -proc expectReply(ftp: AsyncFtpClient): Future[string] {.async.} = - var line = await ftp.csock.recvLine() - result = line - var count = 0 - while line.len > 3 and line[3] == '-': - ## Multi-line reply. - line = await ftp.csock.recvLine() - result.add("\n" & line) - count.inc() - if count >= multiLineLimit: - raise newException(ReplyError, "Reached maximum multi-line reply count.") - -proc send*(ftp: AsyncFtpClient, m: string): Future[string] {.async.} = - ## Send a message to the server, and wait for a primary reply. - ## `\c\L` is added for you. - ## - ## You need to make sure that the message `m` doesn't contain any newline - ## characters. Failing to do so will raise `AssertionDefect`. - ## - ## **Note:** The server may return multiple lines of coded replies. - doAssert(not m.contains({'\c', '\L'}), "message shouldn't contain any newline characters") - await ftp.csock.send(m & "\c\L") - return await ftp.expectReply() - -proc assertReply(received: string, expected: varargs[string]) = - for i in items(expected): - if received.startsWith(i): return - raise newException(ReplyError, - "Expected reply '$1' got: $2" % - [expected.join("' or '"), received]) - -proc pasv(ftp: AsyncFtpClient) {.async.} = - ## Negotiate a data connection. - ftp.dsock = newAsyncSocket() - - var pasvMsg = (await ftp.send("PASV")).strip - assertReply(pasvMsg, "227") - var betweenParens = captureBetween(pasvMsg, '(', ')') - var nums = betweenParens.split(',') - var ip = nums[0 .. ^3] - var port = nums[^2 .. ^1] - var properPort = port[0].parseInt()*256+port[1].parseInt() - await ftp.dsock.connect(ip.join("."), Port(properPort)) - ftp.dsockConnected = true - -proc normalizePathSep(path: string): string = - return replace(path, '\\', '/') - -proc connect*(ftp: AsyncFtpClient) {.async.} = - ## Connect to the FTP server specified by `ftp`. - await ftp.csock.connect(ftp.address, ftp.port) - - var reply = await ftp.expectReply() - if reply.startsWith("120"): - # 120 Service ready in nnn minutes. - # We wait until we receive 220. - reply = await ftp.expectReply() - - # Handle 220 messages from the server - assertReply(reply, "220") - - if ftp.user != "": - assertReply(await(ftp.send("USER " & ftp.user)), "230", "331") - - if ftp.pass != "": - assertReply(await(ftp.send("PASS " & ftp.pass)), "230") - -proc pwd*(ftp: AsyncFtpClient): Future[string] {.async.} = - ## Returns the current working directory. - let wd = await ftp.send("PWD") - assertReply wd, "257" - return wd.captureBetween('"') # " - -proc cd*(ftp: AsyncFtpClient, dir: string) {.async.} = - ## Changes the current directory on the remote FTP server to `dir`. - assertReply(await(ftp.send("CWD " & dir.normalizePathSep)), "250") - -proc cdup*(ftp: AsyncFtpClient) {.async.} = - ## Changes the current directory to the parent of the current directory. - assertReply(await(ftp.send("CDUP")), "200") - -proc getLines(ftp: AsyncFtpClient): Future[string] {.async.} = - ## Downloads text data in ASCII mode - result = "" - assert ftp.dsockConnected - while ftp.dsockConnected: - let r = await ftp.dsock.recvLine() - if r == "": - ftp.dsockConnected = false - else: - result.add(r & "\n") - - assertReply(await(ftp.expectReply()), "226") - -proc listDirs*(ftp: AsyncFtpClient, dir = ""): Future[seq[string]] {.async.} = - ## Returns a list of filenames in the given directory. If `dir` is "", - ## the current directory is used. If `async` is true, this - ## function will return immediately and it will be your job to - ## use asyncdispatch's `poll` to progress this operation. - await ftp.pasv() - - assertReply(await(ftp.send("NLST " & dir.normalizePathSep)), ["125", "150"]) - - result = splitLines(await ftp.getLines()) - -proc fileExists*(ftp: AsyncFtpClient, file: string): Future[bool] {.async.} = - ## Determines whether `file` exists. - var files = await ftp.listDirs() - for f in items(files): - if f.normalizePathSep == file.normalizePathSep: return true - -proc createDir*(ftp: AsyncFtpClient, dir: string, recursive = false){.async.} = - ## Creates a directory `dir`. If `recursive` is true, the topmost - ## subdirectory of `dir` will be created first, following the secondmost... - ## etc. this allows you to give a full path as the `dir` without worrying - ## about subdirectories not existing. - if not recursive: - assertReply(await(ftp.send("MKD " & dir.normalizePathSep)), "257") - else: - var reply = "" - var previousDirs = "" - for p in split(dir, {os.DirSep, os.AltSep}): - if p != "": - previousDirs.add(p) - reply = await ftp.send("MKD " & previousDirs) - previousDirs.add('/') - assertReply reply, "257" - -proc chmod*(ftp: AsyncFtpClient, path: string, - permissions: set[FilePermission]) {.async.} = - ## Changes permission of `path` to `permissions`. - var userOctal = 0 - var groupOctal = 0 - var otherOctal = 0 - for i in items(permissions): - case i - of fpUserExec: userOctal.inc(1) - of fpUserWrite: userOctal.inc(2) - of fpUserRead: userOctal.inc(4) - of fpGroupExec: groupOctal.inc(1) - of fpGroupWrite: groupOctal.inc(2) - of fpGroupRead: groupOctal.inc(4) - of fpOthersExec: otherOctal.inc(1) - of fpOthersWrite: otherOctal.inc(2) - of fpOthersRead: otherOctal.inc(4) - - var perm = $userOctal & $groupOctal & $otherOctal - assertReply(await(ftp.send("SITE CHMOD " & perm & - " " & path.normalizePathSep)), "200") - -proc list*(ftp: AsyncFtpClient, dir = ""): Future[string] {.async.} = - ## Lists all files in `dir`. If `dir` is `""`, uses the current - ## working directory. - await ftp.pasv() - - let reply = await ftp.send("LIST" & " " & dir.normalizePathSep) - assertReply(reply, ["125", "150"]) - - result = await ftp.getLines() - -proc retrText*(ftp: AsyncFtpClient, file: string): Future[string] {.async.} = - ## Retrieves `file`. File must be ASCII text. - await ftp.pasv() - let reply = await ftp.send("RETR " & file.normalizePathSep) - assertReply(reply, ["125", "150"]) - - result = await ftp.getLines() - -proc getFile(ftp: AsyncFtpClient, file: File, total: BiggestInt, - onProgressChanged: ProgressChangedProc) {.async.} = - assert ftp.dsockConnected - var progress = 0 - var progressInSecond = 0 - var countdownFut = sleepAsync(ftp.progressInterval) - var dataFut = ftp.dsock.recv(BufferSize) - while ftp.dsockConnected: - await dataFut or countdownFut - if countdownFut.finished: - asyncCheck onProgressChanged(total, progress, - progressInSecond.float) - progressInSecond = 0 - countdownFut = sleepAsync(ftp.progressInterval) - - if dataFut.finished: - let data = dataFut.read - if data != "": - progress.inc(data.len) - progressInSecond.inc(data.len) - file.write(data) - dataFut = ftp.dsock.recv(BufferSize) - else: - ftp.dsockConnected = false - - assertReply(await(ftp.expectReply()), "226") - -proc defaultOnProgressChanged*(total, progress: BiggestInt, - speed: float): Future[void] {.nimcall, gcsafe.} = - ## Default FTP `onProgressChanged` handler. Does nothing. - result = newFuture[void]() - #echo(total, " ", progress, " ", speed) - result.complete() - -proc retrFile*(ftp: AsyncFtpClient, file, dest: string, - onProgressChanged: ProgressChangedProc = defaultOnProgressChanged) {.async.} = - ## Downloads `file` and saves it to `dest`. - ## The `EvRetr` event is passed to the specified `handleEvent` function - ## when the download is finished. The event's `filename` field will be equal - ## to `file`. - var destFile = open(dest, mode = fmWrite) - await ftp.pasv() - var reply = await ftp.send("RETR " & file.normalizePathSep) - assertReply reply, ["125", "150"] - if {'(', ')'} notin reply: - raise newException(ReplyError, "Reply has no file size.") - var fileSize: BiggestInt - if reply.captureBetween('(', ')').parseBiggestInt(fileSize) == 0: - raise newException(ReplyError, "Reply has no file size.") - - await getFile(ftp, destFile, fileSize, onProgressChanged) - destFile.close() - -proc doUpload(ftp: AsyncFtpClient, file: File, - onProgressChanged: ProgressChangedProc) {.async.} = - assert ftp.dsockConnected - - let total = file.getFileSize() - var data = newString(4000) - var progress = 0 - var progressInSecond = 0 - var countdownFut = sleepAsync(ftp.progressInterval) - var sendFut: Future[void] = nil - while ftp.dsockConnected: - if sendFut == nil or sendFut.finished: - # TODO: Async file reading. - let len = file.readBuffer(addr(data[0]), 4000) - setLen(data, len) - if len == 0: - # File finished uploading. - ftp.dsock.close() - ftp.dsockConnected = false - - assertReply(await(ftp.expectReply()), "226") - else: - progress.inc(len) - progressInSecond.inc(len) - sendFut = ftp.dsock.send(data) - - if countdownFut.finished: - asyncCheck onProgressChanged(total, progress, progressInSecond.float) - progressInSecond = 0 - countdownFut = sleepAsync(ftp.progressInterval) - - await countdownFut or sendFut - -proc store*(ftp: AsyncFtpClient, file, dest: string, - onProgressChanged: ProgressChangedProc = defaultOnProgressChanged) {.async.} = - ## Uploads `file` to `dest` on the remote FTP server. Usage of this - ## function asynchronously is recommended to view the progress of - ## the download. - ## The `EvStore` event is passed to the specified `handleEvent` function - ## when the upload is finished, and the `filename` field will be - ## equal to `file`. - var destFile = open(file) - await ftp.pasv() - - let reply = await ftp.send("STOR " & dest.normalizePathSep) - assertReply reply, ["125", "150"] - - await doUpload(ftp, destFile, onProgressChanged) - -proc rename*(ftp: AsyncFtpClient, nameFrom: string, nameTo: string) {.async.} = - ## Rename a file or directory on the remote FTP Server from current name - ## `name_from` to new name `name_to` - assertReply(await ftp.send("RNFR " & nameFrom), "350") - assertReply(await ftp.send("RNTO " & nameTo), "250") - -proc removeFile*(ftp: AsyncFtpClient, filename: string) {.async.} = - ## Delete a file `filename` on the remote FTP server - assertReply(await ftp.send("DELE " & filename), "250") - -proc removeDir*(ftp: AsyncFtpClient, dir: string) {.async.} = - ## Delete a directory `dir` on the remote FTP server - assertReply(await ftp.send("RMD " & dir), "250") - -proc newAsyncFtpClient*(address: string, port = Port(21), - user, pass = "", progressInterval: int = 1000): AsyncFtpClient = - ## Creates a new `AsyncFtpClient` object. - new result - result.user = user - result.pass = pass - result.address = address - result.port = port - result.progressInterval = progressInterval - result.dsockConnected = false - result.csock = newAsyncSocket() - -when not defined(testing) and isMainModule: - var ftp = newAsyncFtpClient("example.com", user = "test", pass = "test") - proc main(ftp: AsyncFtpClient) {.async.} = - await ftp.connect() - echo await ftp.pwd() - echo await ftp.listDirs() - await ftp.store("payload.jpg", "payload.jpg") - await ftp.retrFile("payload.jpg", "payload2.jpg") - await ftp.rename("payload.jpg", "payload_renamed.jpg") - await ftp.store("payload.jpg", "payload_remove.jpg") - await ftp.removeFile("payload_remove.jpg") - await ftp.createDir("deleteme") - await ftp.removeDir("deleteme") - echo("Finished") - - waitFor main(ftp) diff --git a/lib/pure/asyncfutures.nim b/lib/pure/asyncfutures.nim index c0c5c3f07..29ebf8f89 100644 --- a/lib/pure/asyncfutures.nim +++ b/lib/pure/asyncfutures.nim @@ -7,10 +7,14 @@ # distribution, for details about the copyright. # -import os, tables, strutils, times, heapqueue, options, deques, cstrutils +import std/[os, tables, strutils, times, heapqueue, options, deques, cstrutils, typetraits] import system/stacktraces +when defined(nimPreviewSlimSystem): + import std/objectdollar # for StackTraceEntry + import std/assertions + # TODO: This shouldn't need to be included, but should ideally be exported. type CallbackFunc = proc () {.closure, gcsafe.} @@ -25,7 +29,7 @@ type finished: bool error*: ref Exception ## Stored exception errorStackTrace*: string - when not defined(release): + when not defined(release) or defined(futureLogging): stackTrace: seq[StackTraceEntry] ## For debugging purposes only. id: int fromProc: string @@ -47,7 +51,7 @@ const NimAsyncContinueSuffix* = "NimAsyncContinue" ## For internal usage. Do not use. when isFutureLoggingEnabled: - import hashes + import std/hashes type FutureInfo* = object stackTrace*: seq[StackTraceEntry] @@ -93,7 +97,7 @@ proc setCallSoonProc*(p: (proc(cbproc: proc ()) {.gcsafe.})) = ## Change current implementation of `callSoon`. This is normally called when dispatcher from `asyncdispatcher` is initialized. callSoonProc = p -proc callSoon*(cbproc: proc ()) = +proc callSoon*(cbproc: proc () {.gcsafe.}) = ## Call `cbproc` "soon". ## ## If async dispatcher is running, `cbproc` will be executed during next dispatcher tick. @@ -189,24 +193,22 @@ proc add(callbacks: var CallbackList, function: CallbackFunc) = last = last.next last.next = newCallback -proc complete*[T](future: Future[T], val: T) = - ## Completes `future` with value `val`. +proc completeImpl[T, U](future: Future[T], val: sink U, isVoid: static bool) = #assert(not future.finished, "Future already finished, cannot finish twice.") checkFinished(future) assert(future.error == nil) - future.value = val + when not isVoid: + future.value = val future.finished = true future.callbacks.call() when isFutureLoggingEnabled: logFutureFinish(future) -proc complete*(future: Future[void]) = - ## Completes a void `future`. - #assert(not future.finished, "Future already finished, cannot finish twice.") - checkFinished(future) - assert(future.error == nil) - future.finished = true - future.callbacks.call() - when isFutureLoggingEnabled: logFutureFinish(future) +proc complete*[T](future: Future[T], val: sink T) = + ## Completes `future` with value `val`. + completeImpl(future, val, false) + +proc complete*(future: Future[void], val = Future[void].default) = + completeImpl(future, (), true) proc complete*[T](future: FutureVar[T]) = ## Completes a `FutureVar`. @@ -217,7 +219,7 @@ proc complete*[T](future: FutureVar[T]) = fut.callbacks.call() when isFutureLoggingEnabled: logFutureFinish(Future[T](future)) -proc complete*[T](future: FutureVar[T], val: T) = +proc complete*[T](future: FutureVar[T], val: sink T) = ## Completes a `FutureVar` with value `val`. ## ## Any previously stored value will be overwritten. @@ -227,7 +229,7 @@ proc complete*[T](future: FutureVar[T], val: T) = fut.finished = true fut.value = val fut.callbacks.call() - when isFutureLoggingEnabled: logFutureFinish(future) + when isFutureLoggingEnabled: logFutureFinish(fut) proc fail*[T](future: Future[T], error: ref Exception) = ## Completes `future` with `error`. @@ -280,19 +282,33 @@ proc `callback=`*[T](future: Future[T], ## If future has already completed then `cb` will be called immediately. future.callback = proc () = cb(future) +template getFilenameProcname(entry: StackTraceEntry): (string, string) = + when compiles(entry.filenameStr) and compiles(entry.procnameStr): + # We can't rely on "entry.filename" and "entry.procname" still being valid + # cstring pointers, because the "string.data" buffers they pointed to might + # be already garbage collected (this entry being a non-shallow copy, + # "entry.filename" no longer points to "entry.filenameStr.data", but to the + # buffer of the original object). + (entry.filenameStr, entry.procnameStr) + else: + ($entry.filename, $entry.procname) + proc getHint(entry: StackTraceEntry): string = ## We try to provide some hints about stack trace entries that the user ## may not be familiar with, in particular calls inside the stdlib. + + let (filename, procname) = getFilenameProcname(entry) + result = "" - if entry.procname == cstring"processPendingCallbacks": - if cmpIgnoreStyle(entry.filename, "asyncdispatch.nim") == 0: + if procname == "processPendingCallbacks": + if cmpIgnoreStyle(filename, "asyncdispatch.nim") == 0: return "Executes pending callbacks" - elif entry.procname == cstring"poll": - if cmpIgnoreStyle(entry.filename, "asyncdispatch.nim") == 0: + elif procname == "poll": + if cmpIgnoreStyle(filename, "asyncdispatch.nim") == 0: return "Processes asynchronous completion events" - if entry.procname.endsWith(NimAsyncContinueSuffix): - if cmpIgnoreStyle(entry.filename, "asyncmacro.nim") == 0: + if procname.endsWith(NimAsyncContinueSuffix): + if cmpIgnoreStyle(filename, "asyncmacro.nim") == 0: return "Resumes an async procedure" proc `$`*(stackTraceEntries: seq[StackTraceEntry]): string = @@ -305,33 +321,29 @@ proc `$`*(stackTraceEntries: seq[StackTraceEntry]): string = # Find longest filename & line number combo for alignment purposes. var longestLeft = 0 for entry in entries: - if entry.procname.isNil: continue + let (filename, procname) = getFilenameProcname(entry) - let left = $entry.filename & $entry.line - if left.len > longestLeft: - longestLeft = left.len + if procname == "": continue + + let leftLen = filename.len + len($entry.line) + if leftLen > longestLeft: + longestLeft = leftLen - var indent = 2 # Format the entries. for entry in entries: - if entry.procname.isNil: - if entry.line == reraisedFromBegin: - result.add(spaces(indent) & "#[\n") - indent.inc(2) - elif entry.line == reraisedFromEnd: - indent.dec(2) - result.add(spaces(indent) & "]#\n") - continue - - let left = "$#($#)" % [$entry.filename, $entry.line] - result.add((spaces(indent) & "$#$# $#\n") % [ + let (filename, procname) = getFilenameProcname(entry) + + if procname == "" and entry.line == reraisedFromBegin: + break + + let left = "$#($#)" % [filename, $entry.line] + result.add((spaces(2) & "$# $#\n") % [ left, - spaces(longestLeft - left.len + 2), - $entry.procname + procname ]) let hint = getHint(entry) if hint.len > 0: - result.add(spaces(indent+2) & "## " & hint & "\n") + result.add(spaces(4) & "## " & hint & "\n") proc injectStacktrace[T](future: Future[T]) = when not defined(release): @@ -351,34 +363,38 @@ proc injectStacktrace[T](future: Future[T]) = newMsg.add($entries) newMsg.add("Exception message: " & exceptionMsg & "\n") - newMsg.add("Exception type:") # # For debugging purposes + # newMsg.add("Exception type:") # for entry in getStackTraceEntries(future.error): # newMsg.add "\n" & $entry future.error.msg = newMsg -proc read*[T](future: Future[T] | FutureVar[T]): T = - ## Retrieves the value of `future`. Future must be finished otherwise - ## this function will fail with a `ValueError` exception. - ## - ## If the result of the future is an error then that error will be raised. - {.push hint[ConvFromXtoItselfNotNeeded]: off.} +template readImpl(future, T) = when future is Future[T]: - let fut = future + let fut {.cursor.} = future else: - let fut = Future[T](future) - {.pop.} + let fut {.cursor.} = Future[T](future) if fut.finished: if fut.error != nil: injectStacktrace(fut) raise fut.error when T isnot void: - result = fut.value + result = distinctBase(future).value else: # TODO: Make a custom exception type for this? raise newException(ValueError, "Future still in progress.") +proc read*[T](future: Future[T] | FutureVar[T]): lent T = + ## Retrieves the value of `future`. Future must be finished otherwise + ## this function will fail with a `ValueError` exception. + ## + ## If the result of the future is an error then that error will be raised. + readImpl(future, T) + +proc read*(future: Future[void] | FutureVar[void]) = + readImpl(future, void) + proc readError*[T](future: Future[T]): ref Exception = ## Retrieves the exception stored in `future`. ## diff --git a/lib/pure/asynchttpserver.nim b/lib/pure/asynchttpserver.nim index 38be4ceac..39e945d5e 100644 --- a/lib/pure/asynchttpserver.nim +++ b/lib/pure/asynchttpserver.nim @@ -15,20 +15,20 @@ ## instead of allowing users to connect directly to this server. runnableExamples("-r:off"): - # This example will create an HTTP server on port 8080. The server will - # respond to all requests with a `200 OK` response code and "Hello World" + # This example will create an HTTP server on an automatically chosen port. + # It will respond to all requests with a `200 OK` response code and "Hello World" # as the response body. import std/asyncdispatch proc main {.async.} = - const port = 8080 var server = newAsyncHttpServer() proc cb(req: Request) {.async.} = echo (req.reqMethod, req.url, req.headers) let headers = {"Content-type": "text/plain; charset=utf-8"} await req.respond(Http200, "Hello World", headers.newHttpHeaders()) - echo "test this with: curl localhost:" & $port & "/" - server.listen(Port(port)) + server.listen(Port(0)) # or Port(8080) to hardcode the standard HTTP port. + let port = server.getPort + echo "test this with: curl localhost:" & $port.uint16 & "/" while true: if server.shouldAcceptRequest(): await server.acceptRequest(cb) @@ -39,10 +39,14 @@ runnableExamples("-r:off"): waitFor main() -import asyncnet, asyncdispatch, parseutils, uri, strutils -import httpcore +import std/[asyncnet, asyncdispatch, parseutils, uri, strutils] +import std/httpcore +from std/nativesockets import getLocalAddr, Domain, AF_INET, AF_INET6 import std/private/since +when defined(nimPreviewSlimSystem): + import std/assertions + export httpcore except parseHeader const @@ -70,21 +74,18 @@ type maxBody: int ## The maximum content-length that will be read for the body. maxFDs: int -func getSocket*(a: AsyncHttpServer): AsyncSocket {.since: (1, 5, 1).} = - ## Returns the `AsyncHttpServer`s internal `AsyncSocket` instance. - ## - ## Useful for identifying what port the AsyncHttpServer is bound to, if it - ## was chosen automatically. +proc getPort*(self: AsyncHttpServer): Port {.since: (1, 5, 1).} = + ## Returns the port `self` was bound to. + ## + ## Useful for identifying what port `self` is bound to, if it + ## was chosen automatically, for example via `listen(Port(0))`. runnableExamples: - from std/asyncdispatch import Port - from std/asyncnet import getFd - from std/nativesockets import getLocalAddr, AF_INET + from std/nativesockets import Port let server = newAsyncHttpServer() - server.listen(Port(0)) # Socket is not bound until this point - let port = getLocalAddr(server.getSocket.getFd, AF_INET)[1] - doAssert uint16(port) > 0 + server.listen(Port(0)) + assert server.getPort.uint16 > 0 server.close() - a.socket + result = getLocalAddr(self.socket)[1] proc newAsyncHttpServer*(reuseAddr = true, reusePort = false, maxBody = 8388608): AsyncHttpServer = @@ -109,16 +110,16 @@ proc respond*(req: Request, code: HttpCode, content: string, ## This procedure will **not** close the client socket. ## ## Example: - ## - ## .. code-block::nim - ## import std/json - ## proc handler(req: Request) {.async.} = - ## if req.url.path == "/hello-world": - ## let msg = %* {"message": "Hello World"} - ## let headers = newHttpHeaders([("Content-Type","application/json")]) - ## await req.respond(Http200, $msg, headers) - ## else: - ## await req.respond(Http404, "Not Found") + ## ```Nim + ## import std/json + ## proc handler(req: Request) {.async.} = + ## if req.url.path == "/hello-world": + ## let msg = %* {"message": "Hello World"} + ## let headers = newHttpHeaders([("Content-Type","application/json")]) + ## await req.respond(Http200, $msg, headers) + ## else: + ## await req.respond(Http404, "Not Found") + ## ``` var msg = "HTTP/1.1 " & $code & "\c\L" if headers != nil: @@ -157,7 +158,7 @@ proc parseProtocol(protocol: string): tuple[orig: string, major, minor: int] = proc sendStatus(client: AsyncSocket, status: string): Future[void] = client.send("HTTP/1.1 " & status & "\c\L\c\L") -func hasChunkedEncoding(request: Request): bool = +func hasChunkedEncoding(request: Request): bool = ## Searches for a chunked transfer encoding const transferEncoding = "Transfer-Encoding" @@ -172,7 +173,7 @@ proc processRequest( server: AsyncHttpServer, req: FutureVar[Request], client: AsyncSocket, - address: string, + address: sink string, lineFut: FutureVar[string], callback: proc (request: Request): Future[void] {.closure, gcsafe.}, ): Future[bool] {.async.} = @@ -186,7 +187,10 @@ proc processRequest( # \n request.headers.clear() request.body = "" - request.hostname.shallowCopy(address) + when defined(gcArc) or defined(gcOrc) or defined(gcAtomicArc): + request.hostname = address + else: + request.hostname.shallowCopy(address) assert client != nil request.client = client @@ -296,7 +300,7 @@ proc processRequest( while true: lineFut.mget.setLen(0) lineFut.clean() - + # The encoding format alternates between specifying a number of bytes to read # and the data to be read, of the previously specified size if sizeOrData mod 2 == 0: @@ -364,7 +368,9 @@ proc processClient(server: AsyncHttpServer, client: AsyncSocket, address: string let retry = await processRequest( server, request, client, address, lineFut, callback ) - if not retry: break + if not retry: + client.close() + break const nimMaxDescriptorsFallback* {.intdefine.} = 16_000 ## fallback value for \ @@ -372,17 +378,18 @@ const ## This can be set on the command line during compilation ## via `-d:nimMaxDescriptorsFallback=N` -proc listen*(server: AsyncHttpServer; port: Port; address = "") = +proc listen*(server: AsyncHttpServer; port: Port; address = ""; domain = AF_INET) = ## Listen to the given port and address. when declared(maxDescriptors): server.maxFDs = try: maxDescriptors() except: nimMaxDescriptorsFallback else: server.maxFDs = nimMaxDescriptorsFallback - server.socket = newAsyncSocket() + server.socket = newAsyncSocket(domain) if server.reuseAddr: server.socket.setSockOpt(OptReuseAddr, true) - if server.reusePort: - server.socket.setSockOpt(OptReusePort, true) + when not defined(nuttx): + if server.reusePort: + server.socket.setSockOpt(OptReusePort, true) server.socket.bindAddr(port, address) server.socket.listen() @@ -404,7 +411,8 @@ proc acceptRequest*(server: AsyncHttpServer, proc serve*(server: AsyncHttpServer, port: Port, callback: proc (request: Request): Future[void] {.closure, gcsafe.}, address = ""; - assumedDescriptorsPerRequest = -1) {.async.} = + assumedDescriptorsPerRequest = -1; + domain = AF_INET) {.async.} = ## Starts the process of listening for incoming HTTP connections on the ## specified address and port. ## @@ -417,7 +425,7 @@ proc serve*(server: AsyncHttpServer, port: Port, ## ## You should prefer to call `acceptRequest` instead with a custom server ## loop so that you're in control over the error handling and logging. - listen server, port, address + listen server, port, address, domain while true: if shouldAcceptRequest(server, assumedDescriptorsPerRequest): var (address, client) = await server.socket.acceptAddr() diff --git a/lib/pure/asyncmacro.nim b/lib/pure/asyncmacro.nim index eecec4278..d4e72c28a 100644 --- a/lib/pure/asyncmacro.nim +++ b/lib/pure/asyncmacro.nim @@ -7,10 +7,14 @@ # distribution, for details about the copyright. # -## `asyncdispatch` module depends on the `asyncmacro` module to work properly. +## Implements the `async` and `multisync` macros for `asyncdispatch`. -import macros, strutils, asyncfutures +import std/[macros, strutils, asyncfutures] +type + Context = ref object + inTry: int + hasRet: bool # TODO: Ref https://github.com/nim-lang/Nim/issues/5617 # TODO: Add more line infos @@ -21,9 +25,8 @@ proc newCallWithLineInfo(fromNode: NimNode; theProc: NimNode, args: varargs[NimN template createCb(retFutureSym, iteratorNameSym, strName, identName, futureVarCompletions: untyped) = bind finished - var nameIterVar = iteratorNameSym - proc identName {.closure.} = + proc identName {.closure, stackTrace: off.} = try: if not nameIterVar.finished: var next = nameIterVar() @@ -35,14 +38,11 @@ template createCb(retFutureSym, iteratorNameSym, if next == nil: if not retFutureSym.finished: - let msg = "Async procedure ($1) yielded `nil`, are you await'ing a " & - "`nil` Future?" + let msg = "Async procedure ($1) yielded `nil`, are you await'ing a `nil` Future?" raise newException(AssertionDefect, msg % strName) else: {.gcsafe.}: - {.push hint[ConvFromXtoItselfNotNeeded]: off.} next.addCallback cast[proc() {.closure, gcsafe.}](identName) - {.pop.} except: futureVarCompletions if retFutureSym.finished: @@ -68,10 +68,7 @@ proc createFutureVarCompletions(futureVarIdents: seq[NimNode], fromNode: NimNode ) ) -proc processBody(node, retFutureSym: NimNode, - subTypeIsVoid: bool, - futureVarIdents: seq[NimNode]): NimNode = - #echo(node.treeRepr) +proc processBody(ctx: Context; node, needsCompletionSym, retFutureSym: NimNode, futureVarIdents: seq[NimNode]): NimNode = result = node case node.kind of nnkReturnStmt: @@ -80,29 +77,53 @@ proc processBody(node, retFutureSym: NimNode, # As I've painfully found out, the order here really DOES matter. result.add createFutureVarCompletions(futureVarIdents, node) + ctx.hasRet = true if node[0].kind == nnkEmpty: - if not subTypeIsVoid: - result.add newCall(newIdentNode("complete"), retFutureSym, - newIdentNode("result")) + if ctx.inTry == 0: + result.add newCallWithLineInfo(node, newIdentNode("complete"), retFutureSym, newIdentNode("result")) else: - result.add newCall(newIdentNode("complete"), retFutureSym) + result.add newAssignment(needsCompletionSym, newLit(true)) else: - let x = node[0].processBody(retFutureSym, subTypeIsVoid, - futureVarIdents) + let x = processBody(ctx, node[0], needsCompletionSym, retFutureSym, futureVarIdents) if x.kind == nnkYieldStmt: result.add x + elif ctx.inTry == 0: + result.add newCallWithLineInfo(node, newIdentNode("complete"), retFutureSym, x) else: - result.add newCall(newIdentNode("complete"), retFutureSym, x) + result.add newAssignment(newIdentNode("result"), x) + result.add newAssignment(needsCompletionSym, newLit(true)) result.add newNimNode(nnkReturnStmt, node).add(newNilLit()) return # Don't process the children of this return stmt of RoutineNodes-{nnkTemplateDef}: # skip all the nested procedure definitions return - else: discard - - for i in 0 ..< result.len: - result[i] = processBody(result[i], retFutureSym, subTypeIsVoid, - futureVarIdents) + of nnkTryStmt: + if result[^1].kind == nnkFinally: + inc ctx.inTry + result[0] = processBody(ctx, result[0], needsCompletionSym, retFutureSym, futureVarIdents) + dec ctx.inTry + for i in 1 ..< result.len: + result[i] = processBody(ctx, result[i], needsCompletionSym, retFutureSym, futureVarIdents) + if ctx.inTry == 0 and ctx.hasRet: + let finallyNode = copyNimNode(result[^1]) + let stmtNode = newNimNode(nnkStmtList) + for child in result[^1]: + stmtNode.add child + stmtNode.add newIfStmt( + ( needsCompletionSym, + newCallWithLineInfo(node, newIdentNode("complete"), retFutureSym, + newIdentNode("result") + ) + ) + ) + finallyNode.add stmtNode + result[^1] = finallyNode + else: + for i in 0 ..< result.len: + result[i] = processBody(ctx, result[i], needsCompletionSym, retFutureSym, futureVarIdents) + else: + for i in 0 ..< result.len: + result[i] = processBody(ctx, result[i], needsCompletionSym, retFutureSym, futureVarIdents) # echo result.repr @@ -139,9 +160,21 @@ template await*(f: typed): untyped {.used.} = error "await expects Future[T], got " & $typeof(f) template await*[T](f: Future[T]): auto {.used.} = - var internalTmpFuture: FutureBase = f - yield internalTmpFuture - (cast[typeof(f)](internalTmpFuture)).read() + when not defined(nimHasTemplateRedefinitionPragma): + {.pragma: redefine.} + template yieldFuture {.redefine.} = yield FutureBase() + + when compiles(yieldFuture): + var internalTmpFuture: FutureBase = f + yield internalTmpFuture + (cast[typeof(f)](internalTmpFuture)).read() + else: + macro errorAsync(futureError: Future[T]) = + error( + "Can only 'await' inside a proc marked as 'async'. Use " & + "'waitFor' when calling an 'async' proc in a non-async scope instead", + futureError) + errorAsync(f) proc asyncSingleProc(prc: NimNode): NimNode = ## This macro transforms a single procedure into a closure iterator. @@ -149,9 +182,13 @@ proc asyncSingleProc(prc: NimNode): NimNode = if prc.kind == nnkProcTy: result = prc if prc[0][0].kind == nnkEmpty: - result[0][0] = parseExpr("Future[void]") + result[0][0] = quote do: Future[void] return result + if prc.kind in RoutineNodes and prc.name.kind != nnkEmpty: + # Only non anonymous functions need/can have stack trace disabled + prc.addPragma(nnkExprColonExpr.newTree(ident"stackTrace", ident"off")) + if prc.kind notin {nnkProcDef, nnkLambda, nnkMethodDef, nnkDo}: error("Cannot transform this node kind into an async proc." & " proc/method definition or lambda node expected.", prc) @@ -183,11 +220,7 @@ proc asyncSingleProc(prc: NimNode): NimNode = else: verifyReturnType(repr(returnType), returnType) - let subtypeIsVoid = returnType.kind == nnkEmpty or - (baseType.kind == nnkIdent and returnType[1].eqIdent("void")) - let futureVarIdents = getFutureVarIdents(prc.params) - var outerProcBody = newNimNode(nnkStmtList, prc.body) # Extract the documentation comment from the original procedure declaration. @@ -214,43 +247,42 @@ proc asyncSingleProc(prc: NimNode): NimNode = # -> {.pop.} # -> <proc_body> # -> complete(retFuture, result) - var iteratorNameSym = genSym(nskIterator, $prcName & "Iter") - var procBody = prc.body.processBody(retFutureSym, subtypeIsVoid, - futureVarIdents) + var iteratorNameSym = genSym(nskIterator, $prcName & " (Async)") + var needsCompletionSym = genSym(nskVar, "needsCompletion") + var ctx = Context() + var procBody = processBody(ctx, prc.body, needsCompletionSym, retFutureSym, futureVarIdents) # don't do anything with forward bodies (empty) if procBody.kind != nnkEmpty: # fix #13899, defer should not escape its original scope - procBody = newStmtList(newTree(nnkBlockStmt, newEmptyNode(), procBody)) - + let blockStmt = newStmtList(newTree(nnkBlockStmt, newEmptyNode(), procBody)) + procBody = newStmtList() + let resultIdent = ident"result" + procBody.add quote do: + # Check whether there is an implicit return + when typeof(`blockStmt`) is void: + `blockStmt` + else: + `resultIdent` = `blockStmt` procBody.add(createFutureVarCompletions(futureVarIdents, nil)) + procBody.insert(0): quote do: + {.push warning[resultshadowed]: off.} + when `subRetType` isnot void: + var `resultIdent`: `subRetType` + else: + var `resultIdent`: Future[void] + {.pop.} - if not subtypeIsVoid: - procBody.insert(0, newNimNode(nnkPragma).add(newIdentNode("push"), - newNimNode(nnkExprColonExpr).add(newNimNode(nnkBracketExpr).add( - newIdentNode("warning"), newIdentNode("resultshadowed")), - newIdentNode("off")))) # -> {.push warning[resultshadowed]: off.} - - procBody.insert(1, newNimNode(nnkVarSection, prc.body).add( - newIdentDefs(newIdentNode("result"), baseType))) # -> var result: T - - procBody.insert(2, newNimNode(nnkPragma).add( - newIdentNode("pop"))) # -> {.pop.}) - - procBody.add( - newCall(newIdentNode("complete"), - retFutureSym, newIdentNode("result"))) # -> complete(retFuture, result) - else: - # -> complete(retFuture) - procBody.add(newCall(newIdentNode("complete"), retFutureSym)) + var `needsCompletionSym` = false + procBody.add quote do: + complete(`retFutureSym`, `resultIdent`) - var closureIterator = newProc(iteratorNameSym, [parseExpr("owned(FutureBase)")], + var closureIterator = newProc(iteratorNameSym, [quote do: owned(FutureBase)], procBody, nnkIteratorDef) closureIterator.pragma = newNimNode(nnkPragma, lineInfoFrom = prc.body) closureIterator.addPragma(newIdentNode("closure")) # If proc has an explicit gcsafe pragma, we add it to iterator as well. - if prc.pragma.findChild(it.kind in {nnkSym, nnkIdent} and $it == - "gcsafe") != nil: + if prc.pragma.findChild(it.kind in {nnkSym, nnkIdent} and $it == "gcsafe") != nil: closureIterator.addPragma(newIdentNode("gcsafe")) outerProcBody.add(closureIterator) @@ -259,21 +291,20 @@ proc asyncSingleProc(prc: NimNode): NimNode = # friendlier stack traces: var cbName = genSym(nskProc, prcName & NimAsyncContinueSuffix) var procCb = getAst createCb(retFutureSym, iteratorNameSym, - newStrLitNode(prcName), - cbName, - createFutureVarCompletions(futureVarIdents, nil)) + newStrLitNode(prcName), + cbName, + createFutureVarCompletions(futureVarIdents, nil) + ) outerProcBody.add procCb # -> return retFuture outerProcBody.add newNimNode(nnkReturnStmt, prc.body[^1]).add(retFutureSym) result = prc - - if subtypeIsVoid: - # Add discardable pragma. - if returnType.kind == nnkEmpty: - # Add Future[void] - result.params[0] = parseExpr("owned(Future[void])") + # Add discardable pragma. + if returnType.kind == nnkEmpty: + # xxx consider removing `owned`? it's inconsistent with non-void case + result.params[0] = quote do: owned(Future[void]) # based on the yglukhov's patch to chronos: https://github.com/status-im/nim-chronos/pull/47 if procBody.kind != nnkEmpty: @@ -281,10 +312,6 @@ proc asyncSingleProc(prc: NimNode): NimNode = `outerProcBody` result.body = body2 - #echo(treeRepr(result)) - #if prcName == "recvLineInto": - # echo(toStrLit(result)) - macro async*(prc: untyped): untyped = ## Macro which processes async procedures into the appropriate ## iterators and yield statements. @@ -300,8 +327,8 @@ macro async*(prc: untyped): untyped = proc splitParamType(paramType: NimNode, async: bool): NimNode = result = paramType if paramType.kind == nnkInfix and paramType[0].strVal in ["|", "or"]: - let firstAsync = "async" in paramType[1].strVal.normalize - let secondAsync = "async" in paramType[2].strVal.normalize + let firstAsync = "async" in paramType[1].toStrLit().strVal.normalize + let secondAsync = "async" in paramType[2].toStrLit().strVal.normalize if firstAsync: result = paramType[if async: 1 else: 2] @@ -354,9 +381,3 @@ macro multisync*(prc: untyped): untyped = result = newStmtList() result.add(asyncSingleProc(asyncPrc)) result.add(sync) - # echo result.repr - -# overload for await as a fallback handler, based on the yglukhov's patch to chronos: https://github.com/status-im/nim-chronos/pull/47 -# template await*(f: typed): untyped = - # static: - # error "await only available within {.async.}" diff --git a/lib/pure/asyncnet.nim b/lib/pure/asyncnet.nim index f36c427de..ee07e599e 100644 --- a/lib/pure/asyncnet.nim +++ b/lib/pure/asyncnet.nim @@ -65,8 +65,7 @@ ## ## The following example demonstrates a simple chat server. ## -## .. code-block::nim -## +## ```Nim ## import std/[asyncnet, asyncdispatch] ## ## var clients {.threadvar.}: seq[AsyncSocket] @@ -93,19 +92,25 @@ ## ## asyncCheck serve() ## runForever() -## +## ``` import std/private/since -import asyncdispatch, nativesockets, net, os + +when defined(nimPreviewSlimSystem): + import std/[assertions, syncio] + +import std/[asyncdispatch, nativesockets, net, os] export SOBool # TODO: Remove duplication introduced by PR #4683. const defineSsl = defined(ssl) or defined(nimdoc) +const useNimNetLite = defined(nimNetLite) or defined(freertos) or defined(zephyr) or + defined(nuttx) when defineSsl: - import openssl + import std/openssl type # TODO: I would prefer to just do: @@ -178,11 +183,12 @@ proc getLocalAddr*(socket: AsyncSocket): (string, Port) = ## This is high-level interface for `getsockname`:idx:. getLocalAddr(socket.fd, socket.domain) -proc getPeerAddr*(socket: AsyncSocket): (string, Port) = - ## Get the socket's peer address and port number. - ## - ## This is high-level interface for `getpeername`:idx:. - getPeerAddr(socket.fd, socket.domain) +when not useNimNetLite: + proc getPeerAddr*(socket: AsyncSocket): (string, Port) = + ## Get the socket's peer address and port number. + ## + ## This is high-level interface for `getpeername`:idx:. + getPeerAddr(socket.fd, socket.domain) proc newAsyncSocket*(domain, sockType, protocol: cint, buffered = true, @@ -224,7 +230,7 @@ when defineSsl: let len = bioCtrlPending(socket.bioOut) if len > 0: var data = newString(len) - let read = bioRead(socket.bioOut, addr data[0], len) + let read = bioRead(socket.bioOut, cast[cstring](addr data[0]), len) assert read != 0 if read < 0: raiseSSLError() @@ -242,7 +248,7 @@ when defineSsl: var data = await recv(socket.fd.AsyncFD, BufferSize, flags) let length = len(data) if length > 0: - let ret = bioWrite(socket.bioIn, addr data[0], length.cint) + let ret = bioWrite(socket.bioIn, cast[cstring](addr data[0]), length.cint) if ret < 0: raiseSSLError() elif length == 0: @@ -259,14 +265,17 @@ when defineSsl: ErrClearError() # Call the desired operation. opResult = op - + let err = + if opResult < 0: + getSslError(socket, opResult.cint) + else: + SSL_ERROR_NONE # Send any remaining pending SSL data. await sendPendingSslData(socket, flags) # If the operation failed, try to see if SSL has some data to read # or write. if opResult < 0: - let err = getSslError(socket, opResult.cint) let fut = appeaseSsl(socket, flags, err.cint) yield fut if not fut.read(): @@ -397,7 +406,8 @@ proc recv*(socket: AsyncSocket, size: int, ## to be read then the future will complete with a value of `""`. if socket.isBuffered: result = newString(size) - shallow(result) + when not defined(nimSeqsV2): + shallow(result) let originalBufPos = socket.currPos if socket.bufLen == 0: @@ -453,7 +463,7 @@ proc send*(socket: AsyncSocket, data: string, when defineSsl: var copy = data sslLoop(socket, flags, - sslWrite(socket.sslHandle, addr copy[0], copy.len.cint)) + sslWrite(socket.sslHandle, cast[cstring](addr copy[0]), copy.len.cint)) await sendPendingSslData(socket, flags) else: await send(socket.fd.AsyncFD, data, flags) @@ -646,11 +656,16 @@ proc bindAddr*(socket: AsyncSocket, port = Port(0), address = "") {. var aiList = getAddrInfo(realaddr, port, socket.domain) if bindAddr(socket.fd, aiList.ai_addr, aiList.ai_addrlen.SockLen) < 0'i32: - freeaddrinfo(aiList) + freeAddrInfo(aiList) raiseOSError(osLastError()) - freeaddrinfo(aiList) + freeAddrInfo(aiList) + +proc hasDataBuffered*(s: AsyncSocket): bool {.since: (1, 5).} = + ## Determines whether an AsyncSocket has data buffered. + # xxx dedup with std/net + s.isBuffered and s.bufLen > 0 and s.currPos != s.bufLen -when defined(posix): +when defined(posix) and not useNimNetLite: proc connectUnix*(socket: AsyncSocket, path: string): owned(Future[void]) = ## Binds Unix socket to `path`. @@ -667,12 +682,12 @@ when defined(posix): elif ret == EINTR: return false else: - retFuture.fail(newException(OSError, osErrorMsg(OSErrorCode(ret)))) + retFuture.fail(newOSError(OSErrorCode(ret))) return true var socketAddr = makeUnixAddr(path) let ret = socket.fd.connect(cast[ptr SockAddr](addr socketAddr), - (sizeof(socketAddr.sun_family) + path.len).SockLen) + (offsetOf(socketAddr, sun_path) + path.len + 1).SockLen) if ret == 0: # Request to connect completed immediately. retFuture.complete() @@ -681,7 +696,7 @@ when defined(posix): if lastError.int32 == EINTR or lastError.int32 == EINPROGRESS: addWrite(AsyncFD(socket.fd), cb) else: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) + retFuture.fail(newOSError(lastError)) proc bindUnix*(socket: AsyncSocket, path: string) {. tags: [ReadIOEffect].} = @@ -690,7 +705,7 @@ when defined(posix): when not defined(nimdoc): var socketAddr = makeUnixAddr(path) if socket.fd.bindAddr(cast[ptr SockAddr](addr socketAddr), - (sizeof(socketAddr.sun_family) + path.len).SockLen) != 0'i32: + (offsetOf(socketAddr, sun_path) + path.len + 1).SockLen) != 0'i32: raiseOSError(osLastError()) elif defined(nimdoc): @@ -731,6 +746,11 @@ proc close*(socket: AsyncSocket) = raiseSSLError() when defineSsl: + proc sslHandle*(self: AsyncSocket): SslPtr = + ## Retrieve the ssl pointer of `socket`. + ## Useful for interfacing with `openssl`. + self.sslHandle + proc wrapSocket*(ctx: SslContext, socket: AsyncSocket) = ## Wraps a socket in an SSL context. This function effectively turns ## `socket` into an SSL socket. @@ -847,7 +867,7 @@ proc sendTo*(socket: AsyncSocket, address: string, port: Port, data: string, it = it.ai_next - freeaddrinfo(aiList) + freeAddrInfo(aiList) if not success: if lastException != nil: diff --git a/lib/pure/asyncstreams.nim b/lib/pure/asyncstreams.nim index 083c6f0ea..c97b98d55 100644 --- a/lib/pure/asyncstreams.nim +++ b/lib/pure/asyncstreams.nim @@ -9,9 +9,12 @@ ## Unstable API. -import asyncfutures +import std/asyncfutures -import deques +when defined(nimPreviewSlimSystem): + import std/assertions + +import std/deques type FutureStream*[T] = ref object ## Special future that acts as diff --git a/lib/pure/base64.nim b/lib/pure/base64.nim index bf196b54d..591d22cc0 100644 --- a/lib/pure/base64.nim +++ b/lib/pure/base64.nim @@ -16,50 +16,48 @@ ## Each Base64 digit represents exactly 6 bits of data. Three 8-bit ## bytes (i.e., a total of 24 bits) can therefore be represented by ## four 6-bit Base64 digits. -## -## Basic usage -## =========== -## + +##[ +# Basic usage ## Encoding data -## ------------- -## -## .. code-block::nim -## import std/base64 -## let encoded = encode("Hello World") -## assert encoded == "SGVsbG8gV29ybGQ=" +]## + +runnableExamples: + let encoded = encode("Hello World") + assert encoded == "SGVsbG8gV29ybGQ=" + ## ## Apart from strings you can also encode lists of integers or characters: ## -## .. code-block::nim -## import std/base64 -## let encodedInts = encode([1,2,3]) -## assert encodedInts == "AQID" -## let encodedChars = encode(['h','e','y']) -## assert encodedChars == "aGV5" -## -## + +runnableExamples: + let encodedInts = encode([1'u8,2,3]) + assert encodedInts == "AQID" + let encodedChars = encode(['h','e','y']) + assert encodedChars == "aGV5" + +##[ ## Decoding data -## ------------- -## -## .. code-block::nim -## import std/base64 -## let decoded = decode("SGVsbG8gV29ybGQ=") -## assert decoded == "Hello World" -## +]## + +runnableExamples: + let decoded = decode("SGVsbG8gV29ybGQ=") + assert decoded == "Hello World" + +##[ ## URL Safe Base64 -## --------------- -## -## .. code-block::nim -## import std/base64 -## doAssert encode("c\xf7>", safe = true) == "Y_c-" -## doAssert encode("c\xf7>", safe = false) == "Y/c+" -## +]## + +runnableExamples: + assert encode("c\xf7>", safe = true) == "Y_c-" + assert encode("c\xf7>", safe = false) == "Y/c+" + ## See also ## ======== ## ## * `hashes module<hashes.html>`_ for efficient computations of hash values for diverse Nim types -## * `md5 module<md5.html>`_ implements the MD5 checksum algorithm -## * `sha1 module<sha1.html>`_ implements a sha1 encoder and decoder +## * `md5 module<md5.html>`_ for the MD5 checksum algorithm +## * `sha1 module<sha1.html>`_ for the SHA-1 checksum algorithm template cbBase(a, b): untyped = [ 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', @@ -68,15 +66,11 @@ template cbBase(a, b): untyped = [ 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', a, b] -let +const cb64 = cbBase('+', '/') cb64safe = cbBase('-', '_') const - cb64VM = cbBase('+', '/') - cb64safeVM = cbBase('-', '_') - -const invalidChar = 255 template encodeSize(size: int): int = (size * 4 div 3) + 6 @@ -86,10 +80,13 @@ template encodeInternal(s, alphabet: typed): untyped = result.setLen(encodeSize(s.len)) + let + padding = s.len mod 3 + inputEnds = s.len - padding + var inputIndex = 0 outputIndex = 0 - inputEnds = s.len - s.len mod 3 n: uint32 b: uint32 @@ -115,7 +112,6 @@ template encodeInternal(s, alphabet: typed): untyped = outputChar(n shr 6) outputChar(n shr 0) - var padding = s.len mod 3 if padding == 1: inputByte(b shl 16) outputChar(n shr 18) @@ -134,21 +130,14 @@ template encodeInternal(s, alphabet: typed): untyped = result.setLen(outputIndex) template encodeImpl() {.dirty.} = - when nimVM: - block: - let lookupTableVM = if safe: cb64safeVM else: cb64VM - encodeInternal(s, lookupTableVM) + if safe: + encodeInternal(s, cb64safe) else: - block: - let lookupTable = if safe: unsafeAddr(cb64safe) else: unsafeAddr(cb64) - encodeInternal(s, lookupTable) + encodeInternal(s, cb64) -proc encode*[T: SomeInteger|char](s: openArray[T], safe = false): string = +proc encode*[T: byte|char](s: openArray[T], safe = false): string = ## Encodes `s` into base64 representation. ## - ## This procedure encodes an openarray (array or sequence) of either integers - ## or characters. - ## ## If `safe` is `true` then it will encode using the ## URL-Safe and Filesystem-safe standard alphabet characters, ## which substitutes `-` instead of `+` and `_` instead of `/`. @@ -156,18 +145,24 @@ proc encode*[T: SomeInteger|char](s: openArray[T], safe = false): string = ## * https://tools.ietf.org/html/rfc4648#page-7 ## ## **See also:** - ## * `encode proc<#encode,string>`_ for encoding a string ## * `decode proc<#decode,string>`_ for decoding a string runnableExamples: + assert encode("Hello World") == "SGVsbG8gV29ybGQ=" assert encode(['n', 'i', 'm']) == "bmlt" assert encode(@['n', 'i', 'm']) == "bmlt" - assert encode([1, 2, 3, 4, 5]) == "AQIDBAU=" + assert encode([1'u8, 2, 3, 4, 5]) == "AQIDBAU=" encodeImpl() -proc encode*(s: string, safe = false): string = - ## Encodes `s` into base64 representation. +proc encode*[T: SomeInteger and not byte](s: openArray[T], safe = false): string + {.deprecated: "use `byte` or `char` instead".} = + encodeImpl() + +proc encodeMime*(s: string, lineLen = 75.Positive, newLine = "\r\n", + safe = false): string = + ## Encodes `s` into base64 representation as lines. + ## Used in email MIME format, use `lineLen` and `newline`. ## - ## This procedure encodes a string. + ## This procedure encodes a string according to MIME spec. ## ## If `safe` is `true` then it will encode using the ## URL-Safe and Filesystem-safe standard alphabet characters, @@ -176,28 +171,29 @@ proc encode*(s: string, safe = false): string = ## * https://tools.ietf.org/html/rfc4648#page-7 ## ## **See also:** - ## * `encode proc<#encode,openArray[T]>`_ for encoding an openarray - ## * `decode proc<#decode,string>`_ for decoding a string - runnableExamples: - assert encode("Hello World") == "SGVsbG8gV29ybGQ=" - encodeImpl() - -proc encodeMime*(s: string, lineLen = 75, newLine = "\r\n"): string = - ## Encodes `s` into base64 representation as lines. - ## Used in email MIME format, use `lineLen` and `newline`. - ## - ## This procedure encodes a string according to MIME spec. - ## - ## **See also:** - ## * `encode proc<#encode,string>`_ for encoding a string + ## * `encode proc<#encode,openArray[T]>`_ for encoding an openArray ## * `decode proc<#decode,string>`_ for decoding a string runnableExamples: assert encodeMime("Hello World", 4, "\n") == "SGVs\nbG8g\nV29y\nbGQ=" - result = newStringOfCap(encodeSize(s.len)) - for i, c in encode(s): - if i != 0 and (i mod lineLen == 0): - result.add(newLine) - result.add(c) + template cpy(l, src, idx) = + b = l + while i < b: + result[i] = src[idx] + inc i + inc idx + + if s.len == 0: return + let e = encode(s, safe) + if e.len <= lineLen or newLine.len == 0: + return e + result = newString(e.len + newLine.len * ((e.len div lineLen) - int(e.len mod lineLen == 0))) + var i, j, k, b: int + let nd = e.len - lineLen + while j < nd: + cpy(i + lineLen, e, j) + cpy(i + newLine.len, newLine, k) + k = 0 + cpy(result.len, e, j) proc initDecodeTable*(): array[256, char] = # computes a decode table at compile time @@ -220,7 +216,6 @@ proc decode*(s: string): string = ## ## **See also:** ## * `encode proc<#encode,openArray[T]>`_ for encoding an openarray - ## * `encode proc<#encode,string>`_ for encoding a string runnableExamples: assert decode("SGVsbG8gV29ybGQ=") == "Hello World" assert decode(" SGVsbG8gV29ybGQ=") == "Hello World" @@ -249,7 +244,7 @@ proc decode*(s: string): string = inputLen = s.len inputEnds = 0 # strip trailing characters - while s[inputLen - 1] in {'\n', '\r', ' ', '='}: + while inputLen > 0 and s[inputLen - 1] in {'\n', '\r', ' ', '='}: dec inputLen # hot loop: read 4 characters at at time inputEnds = inputLen - 4 diff --git a/lib/pure/bitops.nim b/lib/pure/bitops.nim index 377602e75..0d3351ee5 100644 --- a/lib/pure/bitops.nim +++ b/lib/pure/bitops.nim @@ -25,11 +25,9 @@ ## At this time only `fastLog2`, `firstSetBit`, `countLeadingZeroBits` and `countTrailingZeroBits` ## may return undefined and/or platform dependent values if given invalid input. -import macros +import std/macros import std/private/since -from std/private/vmutils import forwardImpl, toUnsigned - - +from std/private/bitops_utils import forwardImpl, castToUnsigned func bitnot*[T: SomeInteger](x: T): T {.magic: "BitnotI".} ## Computes the `bitwise complement` of the integer `x`. @@ -65,6 +63,12 @@ macro bitxor*[T: SomeInteger](x, y: T; z: varargs[T]): T = type BitsRange*[T] = range[0..sizeof(T)*8-1] ## A range with all bit positions for type `T`. +template typeMasked[T: SomeInteger](x: T): T = + when defined(js): + T(x and ((0xffffffff_ffffffff'u shr (64 - sizeof(T) * 8)))) + else: + x + func bitsliced*[T: SomeInteger](v: T; slice: Slice[int]): T {.inline, since: (1, 3).} = ## Returns an extracted (and shifted) slice of bits from `v`. runnableExamples: @@ -74,8 +78,8 @@ func bitsliced*[T: SomeInteger](v: T; slice: Slice[int]): T {.inline, since: (1, let upmost = sizeof(T) * 8 - 1 - uv = when v is SomeUnsignedInt: v else: v.toUnsigned - (uv shl (upmost - slice.b) shr (upmost - slice.b + slice.a)).T + uv = v.castToUnsigned + ((uv shl (upmost - slice.b)).typeMasked shr (upmost - slice.b + slice.a)).T proc bitslice*[T: SomeInteger](v: var T; slice: Slice[int]) {.inline, since: (1, 3).} = ## Mutates `v` into an extracted (and shifted) slice of bits from `v`. @@ -86,8 +90,8 @@ proc bitslice*[T: SomeInteger](v: var T; slice: Slice[int]) {.inline, since: (1, let upmost = sizeof(T) * 8 - 1 - uv = when v is SomeUnsignedInt: v else: v.toUnsigned - v = (uv shl (upmost - slice.b) shr (upmost - slice.b + slice.a)).T + uv = v.castToUnsigned + v = ((uv shl (upmost - slice.b)).typeMasked shr (upmost - slice.b + slice.a)).T func toMask*[T: SomeInteger](slice: Slice[int]): T {.inline, since: (1, 3).} = ## Creates a bitmask based on a slice of bits. @@ -97,11 +101,8 @@ func toMask*[T: SomeInteger](slice: Slice[int]): T {.inline, since: (1, 3).} = let upmost = sizeof(T) * 8 - 1 - bitmask = when T is SomeUnsignedInt: - bitnot(0.T) - else: - bitnot(0.T).toUnsigned - (bitmask shl (upmost - slice.b + slice.a) shr (upmost - slice.b)).T + bitmask = bitnot(0.T).castToUnsigned + ((bitmask shl (upmost - slice.b + slice.a)).typeMasked shr (upmost - slice.b)).T proc masked*[T: SomeInteger](v, mask :T): T {.inline, since: (1, 3).} = ## Returns `v`, with only the `1` bits from `mask` matching those of @@ -413,7 +414,6 @@ func fastlog2Nim(x: uint64): int {.inline.} = import system/countbits_impl -const arch64 = sizeof(int) == 8 const useBuiltinsRotate = (defined(amd64) or defined(i386)) and (defined(gcc) or defined(clang) or defined(vcc) or (defined(icl) and not defined(cpp))) and useBuiltins @@ -451,33 +451,33 @@ when useGCC_builtins: elif useVCC_builtins: # Search the mask data from most significant bit (MSB) to least significant bit (LSB) for a set bit (1). - func bitScanReverse(index: ptr culong, mask: culong): cuchar {. + func bitScanReverse(index: ptr culong, mask: culong): uint8 {. importc: "_BitScanReverse", header: "<intrin.h>".} - func bitScanReverse64(index: ptr culong, mask: uint64): cuchar {. + func bitScanReverse64(index: ptr culong, mask: uint64): uint8 {. importc: "_BitScanReverse64", header: "<intrin.h>".} # Search the mask data from least significant bit (LSB) to the most significant bit (MSB) for a set bit (1). - func bitScanForward(index: ptr culong, mask: culong): cuchar {. + func bitScanForward(index: ptr culong, mask: culong): uint8 {. importc: "_BitScanForward", header: "<intrin.h>".} - func bitScanForward64(index: ptr culong, mask: uint64): cuchar {. + func bitScanForward64(index: ptr culong, mask: uint64): uint8 {. importc: "_BitScanForward64", header: "<intrin.h>".} template vcc_scan_impl(fnc: untyped; v: untyped): int = - var index: culong + var index {.inject.}: culong = 0 discard fnc(index.addr, v) index.int elif useICC_builtins: # Returns the number of trailing 0-bits in x, starting at the least significant bit position. If x is 0, the result is undefined. - func bitScanForward(p: ptr uint32, b: uint32): cuchar {. + func bitScanForward(p: ptr uint32, b: uint32): uint8 {. importc: "_BitScanForward", header: "<immintrin.h>".} - func bitScanForward64(p: ptr uint32, b: uint64): cuchar {. + func bitScanForward64(p: ptr uint32, b: uint64): uint8 {. importc: "_BitScanForward64", header: "<immintrin.h>".} # Returns the number of leading 0-bits in x, starting at the most significant bit position. If x is 0, the result is undefined. - func bitScanReverse(p: ptr uint32, b: uint32): cuchar {. + func bitScanReverse(p: ptr uint32, b: uint32): uint8 {. importc: "_BitScanReverse", header: "<immintrin.h>".} - func bitScanReverse64(p: ptr uint32, b: uint64): cuchar {. + func bitScanReverse64(p: ptr uint32, b: uint64): uint8 {. importc: "_BitScanReverse64", header: "<immintrin.h>".} template icc_scan_impl(fnc: untyped; v: untyped): int = @@ -508,8 +508,7 @@ func parityBits*(x: SomeInteger): int {.inline.} = # Can be used a base if creating ASM version. # https://stackoverflow.com/questions/21617970/how-to-check-if-value-has-even-parity-of-bits-or-odd - when x is SomeSignedInt: - let x = x.toUnsigned + let x = x.castToUnsigned when nimvm: result = forwardImpl(parityImpl, x) else: @@ -532,8 +531,7 @@ func firstSetBit*(x: SomeInteger): int {.inline.} = doAssert firstSetBit(0b0000_1111'u8) == 1 # GCC builtin 'builtin_ffs' already handle zero input. - when x is SomeSignedInt: - let x = x.toUnsigned + let x = x.castToUnsigned when nimvm: when noUndefined: if x == 0: @@ -575,8 +573,7 @@ func fastLog2*(x: SomeInteger): int {.inline.} = doAssert fastLog2(0b0000_1000'u8) == 3 doAssert fastLog2(0b0000_1111'u8) == 3 - when x is SomeSignedInt: - let x = x.toUnsigned + let x = x.castToUnsigned when noUndefined: if x == 0: return -1 @@ -618,8 +615,7 @@ func countLeadingZeroBits*(x: SomeInteger): int {.inline.} = doAssert countLeadingZeroBits(0b0000_1000'u8) == 4 doAssert countLeadingZeroBits(0b0000_1111'u8) == 4 - when x is SomeSignedInt: - let x = x.toUnsigned + let x = x.castToUnsigned when noUndefined: if x == 0: return 0 @@ -647,8 +643,7 @@ func countTrailingZeroBits*(x: SomeInteger): int {.inline.} = doAssert countTrailingZeroBits(0b0000_1000'u8) == 3 doAssert countTrailingZeroBits(0b0000_1111'u8) == 0 - when x is SomeSignedInt: - let x = x.toUnsigned + let x = x.castToUnsigned when noUndefined: if x == 0: return 0 @@ -665,7 +660,7 @@ when useBuiltinsRotate: when defined(gcc): # GCC was tested until version 4.8.1 and intrinsics were present. Not tested # in previous versions. - func builtin_rotl8(value: cuchar, shift: cint): cuchar + func builtin_rotl8(value: uint8, shift: cint): uint8 {.importc: "__rolb", header: "<x86intrin.h>".} func builtin_rotl16(value: cushort, shift: cint): cushort {.importc: "__rolw", header: "<x86intrin.h>".} @@ -675,7 +670,7 @@ when useBuiltinsRotate: func builtin_rotl64(value: culonglong, shift: cint): culonglong {.importc: "__rolq", header: "<x86intrin.h>".} - func builtin_rotr8(value: cuchar, shift: cint): cuchar + func builtin_rotr8(value: uint8, shift: cint): uint8 {.importc: "__rorb", header: "<x86intrin.h>".} func builtin_rotr16(value: cushort, shift: cint): cushort {.importc: "__rorw", header: "<x86intrin.h>".} @@ -691,7 +686,7 @@ when useBuiltinsRotate: # https://releases.llvm.org/8.0.0/tools/clang/docs/ReleaseNotes.html#non-comprehensive-list-of-changes-in-this-release # https://releases.llvm.org/8.0.0/tools/clang/docs/LanguageExtensions.html#builtin-rotateleft # source for correct declarations: https://github.com/llvm/llvm-project/blob/main/clang/include/clang/Basic/Builtins.def - func builtin_rotl8(value: cuchar, shift: cuchar): cuchar + func builtin_rotl8(value: uint8, shift: uint8): uint8 {.importc: "__builtin_rotateleft8", nodecl.} func builtin_rotl16(value: cushort, shift: cushort): cushort {.importc: "__builtin_rotateleft16", nodecl.} @@ -701,7 +696,7 @@ when useBuiltinsRotate: func builtin_rotl64(value: culonglong, shift: culonglong): culonglong {.importc: "__builtin_rotateleft64", nodecl.} - func builtin_rotr8(value: cuchar, shift: cuchar): cuchar + func builtin_rotr8(value: uint8, shift: uint8): uint8 {.importc: "__builtin_rotateright8", nodecl.} func builtin_rotr16(value: cushort, shift: cushort): cushort {.importc: "__builtin_rotateright16", nodecl.} @@ -717,9 +712,9 @@ when useBuiltinsRotate: # https://docs.microsoft.com/en-us/cpp/intrinsics/rotl8-rotl16?view=msvc-160 # https://docs.microsoft.com/en-us/cpp/intrinsics/rotr8-rotr16?view=msvc-160 # https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/rotl-rotl64-rotr-rotr64?view=msvc-160 - func builtin_rotl8(value: cuchar, shift: cuchar): cuchar + func builtin_rotl8(value: uint8, shift: uint8): uint8 {.importc: "_rotl8", header: "<intrin.h>".} - func builtin_rotl16(value: cushort, shift: cuchar): cushort + func builtin_rotl16(value: cushort, shift: uint8): cushort {.importc: "_rotl16", header: "<intrin.h>".} func builtin_rotl32(value: cuint, shift: cint): cuint {.importc: "_rotl", header: "<stdlib.h>".} @@ -727,9 +722,9 @@ when useBuiltinsRotate: func builtin_rotl64(value: culonglong, shift: cint): culonglong {.importc: "_rotl64", header: "<stdlib.h>".} - func builtin_rotr8(value: cuchar, shift: cuchar): cuchar + func builtin_rotr8(value: uint8, shift: uint8): uint8 {.importc: "_rotr8", header: "<intrin.h>".} - func builtin_rotr16(value: cushort, shift: cuchar): cushort + func builtin_rotr16(value: cushort, shift: uint8): cushort {.importc: "_rotr16", header: "<intrin.h>".} func builtin_rotr32(value: cuint, shift: cint): cuint {.importc: "_rotr", header: "<stdlib.h>".} @@ -739,7 +734,7 @@ when useBuiltinsRotate: elif defined(icl): # Tested on Intel(R) C++ Intel(R) 64 Compiler Classic Version 2021.1.2 Build # 20201208_000000 x64 and x86. Not tested in previous versions. - func builtin_rotl8(value: cuchar, shift: cint): cuchar + func builtin_rotl8(value: uint8, shift: cint): uint8 {.importc: "__rolb", header: "<immintrin.h>".} func builtin_rotl16(value: cushort, shift: cint): cushort {.importc: "__rolw", header: "<immintrin.h>".} @@ -749,7 +744,7 @@ when useBuiltinsRotate: func builtin_rotl64(value: culonglong, shift: cint): culonglong {.importc: "__rolq", header: "<immintrin.h>".} - func builtin_rotr8(value: cuchar, shift: cint): cuchar + func builtin_rotr8(value: uint8, shift: cint): uint8 {.importc: "__rorb", header: "<immintrin.h>".} func builtin_rotr16(value: cushort, shift: cint): cushort {.importc: "__rorw", header: "<immintrin.h>".} @@ -772,11 +767,13 @@ func rotr[T: SomeUnsignedInt](value: T, rot: int32): T {.inline.} = let rot = rot and mask (value shr rot) or (value shl ((-rot) and mask)) -func shiftTypeToImpl(size: static int, shift: int): auto {.inline.} = +func shiftTypeTo(size: static int, shift: int): auto {.inline.} = + ## Returns the `shift` for the rotation according to the compiler and the + ## `size`. when (defined(vcc) and (size in [4, 8])) or defined(gcc) or defined(icl): cint(shift) elif (defined(vcc) and (size in [1, 2])) or (defined(clang) and size == 1): - cuchar(shift) + uint8(shift) elif defined(clang): when size == 2: cushort(shift) @@ -785,118 +782,59 @@ func shiftTypeToImpl(size: static int, shift: int): auto {.inline.} = elif size == 8: culonglong(shift) -template shiftTypeTo[T](value: T, shift: int): auto = - ## Returns the `shift` for the rotation according to the compiler and the size - ## of the` value`. - shiftTypeToImpl(sizeof(value), shift) - -func rotateLeftBits*(value: uint8, shift: range[0..8]): uint8 {.inline.} = - ## Left-rotate bits in a 8-bits value. +func rotateLeftBits*[T: SomeUnsignedInt](value: T, shift: range[0..(sizeof(T) * 8)]): T {.inline.} = + ## Left-rotate bits in a `value`. runnableExamples: doAssert rotateLeftBits(0b0110_1001'u8, 4) == 0b1001_0110'u8 - - when nimvm: - rotl(value, shift.int32) - else: - when useBuiltinsRotate: - builtin_rotl8(value.cuchar, shiftTypeTo(value, shift)).uint8 - else: - rotl(value, shift.int32) - -func rotateLeftBits*(value: uint16, shift: range[0..16]): uint16 {.inline.} = - ## Left-rotate bits in a 16-bits value. - runnableExamples: doAssert rotateLeftBits(0b00111100_11000011'u16, 8) == 0b11000011_00111100'u16 - - when nimvm: - rotl(value, shift.int32) - else: - when useBuiltinsRotate: - builtin_rotl16(value.cushort, shiftTypeTo(value, shift)).uint16 - else: - rotl(value, shift.int32) - -func rotateLeftBits*(value: uint32, shift: range[0..32]): uint32 {.inline.} = - ## Left-rotate bits in a 32-bits value. - runnableExamples: doAssert rotateLeftBits(0b0000111111110000_1111000000001111'u32, 16) == 0b1111000000001111_0000111111110000'u32 - - when nimvm: - rotl(value, shift.int32) - else: - when useBuiltinsRotate: - builtin_rotl32(value.cuint, shiftTypeTo(value, shift)).uint32 - else: - rotl(value, shift.int32) - -func rotateLeftBits*(value: uint64, shift: range[0..64]): uint64 {.inline.} = - ## Left-rotate bits in a 64-bits value. - runnableExamples: doAssert rotateLeftBits(0b00000000111111111111111100000000_11111111000000000000000011111111'u64, 32) == 0b11111111000000000000000011111111_00000000111111111111111100000000'u64 - when nimvm: rotl(value, shift.int32) else: - when useBuiltinsRotate and defined(amd64): - builtin_rotl64(value.culonglong, shiftTypeTo(value, shift)).uint64 + when useBuiltinsRotate: + const size = sizeof(T) + when size == 1: + builtin_rotl8(value.uint8, shiftTypeTo(size, shift)).T + elif size == 2: + builtin_rotl16(value.cushort, shiftTypeTo(size, shift)).T + elif size == 4: + builtin_rotl32(value.cuint, shiftTypeTo(size, shift)).T + elif size == 8 and arch64: + builtin_rotl64(value.culonglong, shiftTypeTo(size, shift)).T + else: + rotl(value, shift.int32) else: rotl(value, shift.int32) -func rotateRightBits*(value: uint8, shift: range[0..8]): uint8 {.inline.} = - ## Right-rotate bits in a 8-bits value. +func rotateRightBits*[T: SomeUnsignedInt](value: T, shift: range[0..(sizeof(T) * 8)]): T {.inline.} = + ## Right-rotate bits in a `value`. runnableExamples: doAssert rotateRightBits(0b0110_1001'u8, 4) == 0b1001_0110'u8 - - when nimvm: - rotr(value, shift.int32) - else: - when useBuiltinsRotate: - builtin_rotr8(value.cuchar, shiftTypeTo(value, shift)).uint8 - else: - rotr(value, shift.int32) - -func rotateRightBits*(value: uint16, shift: range[0..16]): uint16 {.inline.} = - ## Right-rotate bits in a 16-bits value. - runnableExamples: doAssert rotateRightBits(0b00111100_11000011'u16, 8) == 0b11000011_00111100'u16 - - when nimvm: - rotr(value, shift.int32) - else: - when useBuiltinsRotate: - builtin_rotr16(value.cushort, shiftTypeTo(value, shift)).uint16 - else: - rotr(value, shift.int32) - -func rotateRightBits*(value: uint32, shift: range[0..32]): uint32 {.inline.} = - ## Right-rotate bits in a 32-bits value. - runnableExamples: doAssert rotateRightBits(0b0000111111110000_1111000000001111'u32, 16) == 0b1111000000001111_0000111111110000'u32 - - when nimvm: - rotr(value, shift.int32) - else: - when useBuiltinsRotate: - builtin_rotr32(value.cuint, shiftTypeTo(value, shift)).uint32 - else: - rotr(value, shift.int32) - -func rotateRightBits*(value: uint64, shift: range[0..64]): uint64 {.inline.} = - ## Right-rotate bits in a 64-bits value. - runnableExamples: doAssert rotateRightBits(0b00000000111111111111111100000000_11111111000000000000000011111111'u64, 32) == 0b11111111000000000000000011111111_00000000111111111111111100000000'u64 - when nimvm: rotr(value, shift.int32) else: - when useBuiltinsRotate and defined(amd64): - builtin_rotr64(value.culonglong, shiftTypeTo(value, shift)).uint64 + when useBuiltinsRotate: + const size = sizeof(T) + when size == 1: + builtin_rotr8(value.uint8, shiftTypeTo(size, shift)).T + elif size == 2: + builtin_rotr16(value.cushort, shiftTypeTo(size, shift)).T + elif size == 4: + builtin_rotr32(value.cuint, shiftTypeTo(size, shift)).T + elif size == 8 and arch64: + builtin_rotr64(value.culonglong, shiftTypeTo(size, shift)).T + else: + rotr(value, shift.int32) else: rotr(value, shift.int32) diff --git a/lib/pure/browsers.nim b/lib/pure/browsers.nim index 08f5208d2..59e2078df 100644 --- a/lib/pure/browsers.nim +++ b/lib/pure/browsers.nim @@ -12,17 +12,22 @@ ## ## Unstable API. -import std/private/since +import std/private/since # used by the deprecated `openDefaultBrowser()` -import strutils +import std/strutils + +when defined(nimPreviewSlimSystem): + import std/assertions when defined(windows): - import winlean - from os import absolutePath + import std/winlean + when defined(nimPreviewSlimSystem): + import std/widestrs + from std/os import absolutePath else: - import os + import std/os when not defined(osx): - import osproc + import std/osproc const osOpenCmd* = when defined(macos) or defined(macosx) or defined(windows): "open" else: "xdg-open" ## \ @@ -35,15 +40,17 @@ proc prepare(s: string): string = else: result = "file://" & absolutePath(s) -proc openDefaultBrowserImpl(url: string) = +proc openDefaultBrowserRaw(url: string) = + ## note the url argument should be alreadly prepared, i.e. the url is passed "AS IS" + when defined(windows): var o = newWideCString(osOpenCmd) - var u = newWideCString(prepare url) + var u = newWideCString(url) discard shellExecuteW(0'i32, o, u, nil, nil, SW_SHOWNORMAL) elif defined(macosx): - discard execShellCmd(osOpenCmd & " " & quoteShell(prepare url)) + discard execShellCmd(osOpenCmd & " " & quoteShell(url)) else: - var u = quoteShell(prepare url) + var u = quoteShell(url) if execShellCmd(osOpenCmd & " " & u) == 0: return for b in getEnv("BROWSER").split(PathSep): try: @@ -64,26 +71,42 @@ proc openDefaultBrowser*(url: string) = ## ## This proc doesn't raise an exception on error, beware. ## - ## .. code-block:: nim + ## ```nim ## block: openDefaultBrowser("https://nim-lang.org") + ## ``` doAssert url.len > 0, "URL must not be empty string" - openDefaultBrowserImpl(url) + openDefaultBrowserRaw(url) -proc openDefaultBrowser*() {.since: (1, 1).} = - ## Opens the user's default browser without any `url` (blank page). This does not block. - ## Implements IETF RFC-6694 Section 3, "about:blank" must be reserved for a blank page. +proc openDefaultBrowser*() {.since: (1, 1), deprecated: + "not implemented, please open with a specific url instead".} = + ## Intends to open the user's default browser without any `url` (blank page). + ## This does not block. + ## Intends to implement IETF RFC-6694 Section 3, + ## ("about:blank" is reserved for a blank page). ## - ## Under Windows, `ShellExecute` is used. Under Mac OS X the `open` - ## command is used. Under Unix, it is checked if `xdg-open` exists and - ## used if it does. Otherwise the environment variable `BROWSER` is - ## used to determine the default browser to use. + ## Beware that this intended behavior is **not** implemented and + ## considered not worthy to implement here. + ## + ## The following describes the behavior of current implementation: + ## + ## - Under Windows, this will only cause a pop-up dialog \ + ## asking the assocated application with `about` \ + ## (as Windows simply treats `about:` as a protocol like `http`). + ## - Under Mac OS X the `open "about:blank"` command is used. + ## - Under Unix, it is checked if `xdg-open` exists and used \ + ## if it does and open the application assocated with `text/html` mime \ + ## (not `x-scheme-handler/http`, so maybe html-viewer \ + ## other than your default browser is opened). \ + ## Otherwise the environment variable `BROWSER` is used \ + ## to determine the default browser to use. ## ## This proc doesn't raise an exception on error, beware. ## + ## ```nim + ## block: openDefaultBrowser() + ## ``` + ## ## **See also:** ## ## * https://tools.ietf.org/html/rfc6694#section-3 - ## - ## .. code-block:: nim - ## block: openDefaultBrowser() - openDefaultBrowserImpl("http:about:blank") # See IETF RFC-6694 Section 3. + openDefaultBrowserRaw("about:blank") # See IETF RFC-6694 Section 3. diff --git a/lib/pure/cgi.nim b/lib/pure/cgi.nim index fd5089dbb..034f224ac 100644 --- a/lib/pure/cgi.nim +++ b/lib/pure/cgi.nim @@ -9,29 +9,32 @@ ## This module implements helper procs for CGI applications. Example: ## -## .. code-block:: Nim +## ```Nim +## import std/[strtabs, cgi] ## -## import std/[strtabs, cgi] -## -## # Fill the values when debugging: -## when debug: -## setTestData("name", "Klaus", "password", "123456") -## # read the data into `myData` -## var myData = readData() -## # check that the data's variable names are "name" or "password" -## validateData(myData, "name", "password") -## # start generating content: -## writeContentType() -## # generate content: -## write(stdout, "<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.01//EN\">\n") -## write(stdout, "<html><head><title>Test</title></head><body>\n") -## writeLine(stdout, "your name: " & myData["name"]) -## writeLine(stdout, "your password: " & myData["password"]) -## writeLine(stdout, "</body></html>") +## # Fill the values when debugging: +## when debug: +## setTestData("name", "Klaus", "password", "123456") +## # read the data into `myData` +## var myData = readData() +## # check that the data's variable names are "name" or "password" +## validateData(myData, "name", "password") +## # start generating content: +## writeContentType() +## # generate content: +## write(stdout, "<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.01//EN\">\n") +## write(stdout, "<html><head><title>Test</title></head><body>\n") +## writeLine(stdout, "your name: " & myData["name"]) +## writeLine(stdout, "your password: " & myData["password"]) +## writeLine(stdout, "</body></html>") +## ``` import std/[strutils, os, strtabs, cookies, uri] export uri.encodeUrl, uri.decodeUrl +when defined(nimPreviewSlimSystem): + import std/syncio + proc addXmlChar(dest: var string, c: char) {.inline.} = case c @@ -249,9 +252,9 @@ proc setTestData*(keysvalues: varargs[string]) = ## Fills the appropriate environment variables to test your CGI application. ## This can only simulate the 'GET' request method. `keysvalues` should ## provide embedded (name, value)-pairs. Example: - ## - ## .. code-block:: Nim - ## setTestData("name", "Hanz", "password", "12345") + ## ```Nim + ## setTestData("name", "Hanz", "password", "12345") + ## ``` putEnv("REQUEST_METHOD", "GET") var i = 0 var query = "" @@ -266,9 +269,9 @@ proc setTestData*(keysvalues: varargs[string]) = proc writeContentType*() = ## Calls this before starting to send your HTML data to `stdout`. This ## implements this part of the CGI protocol: - ## - ## .. code-block:: Nim - ## write(stdout, "Content-type: text/html\n\n") + ## ```Nim + ## write(stdout, "Content-type: text/html\n\n") + ## ``` write(stdout, "Content-type: text/html\n\n") proc resetForStacktrace() = diff --git a/lib/pure/collections/critbits.nim b/lib/pure/collections/critbits.nim index e12984995..24257dacb 100644 --- a/lib/pure/collections/critbits.nim +++ b/lib/pure/collections/critbits.nim @@ -36,6 +36,9 @@ runnableExamples: import std/private/since +when defined(nimPreviewSlimSystem): + import std/assertions + type NodeObj[T] {.acyclic.} = object byte: int ## byte index of the difference @@ -197,7 +200,7 @@ proc missingOrExcl*[T](c: var CritBitTree[T], key: string): bool = discard exclImpl(c, key) result = c.count == oldCount -proc containsOrIncl*[T](c: var CritBitTree[T], key: string, val: T): bool = +proc containsOrIncl*[T](c: var CritBitTree[T], key: string, val: sink T): bool = ## Returns true if `c` contains the given `key`. If the key does not exist, ## `c[key] = val` is performed. ## @@ -270,7 +273,7 @@ proc incl*(c: var CritBitTree[void], key: string) = discard rawInsert(c, key) -proc incl*[T](c: var CritBitTree[T], key: string, val: T) = +proc incl*[T](c: var CritBitTree[T], key: string, val: sink T) = ## Inserts `key` with value `val` into `c`. ## ## **See also:** @@ -284,7 +287,7 @@ proc incl*[T](c: var CritBitTree[T], key: string, val: T) = var n = rawInsert(c, key) n.val = val -proc `[]=`*[T](c: var CritBitTree[T], key: string, val: T) = +proc `[]=`*[T](c: var CritBitTree[T], key: string, val: sink T) = ## Alias for `incl <#incl,CritBitTree[T],string,T>`_. ## ## **See also:** @@ -300,7 +303,7 @@ template get[T](c: CritBitTree[T], key: string): T = n.val -func `[]`*[T](c: CritBitTree[T], key: string): T {.inline.} = +func `[]`*[T](c: CritBitTree[T], key: string): lent T {.inline.} = ## Retrieves the value at `c[key]`. If `key` is not in `t`, the ## `KeyError` exception is raised. One can check with `hasKey` whether ## the key exists. @@ -342,7 +345,7 @@ iterator keys*[T](c: CritBitTree[T]): string = for x in leaves(c.root): yield x.key -iterator values*[T](c: CritBitTree[T]): T = +iterator values*[T](c: CritBitTree[T]): lent T = ## Yields all values of `c` in the lexicographical order of the ## corresponding keys. ## @@ -415,7 +418,7 @@ iterator keysWithPrefix*[T](c: CritBitTree[T], prefix: string): string = let top = allprefixedAux(c, prefix) for x in leaves(top): yield x.key -iterator valuesWithPrefix*[T](c: CritBitTree[T], prefix: string): T = +iterator valuesWithPrefix*[T](c: CritBitTree[T], prefix: string): lent T = ## Yields all values of `c` starting with `prefix` of the ## corresponding keys. ## @@ -518,7 +521,7 @@ func commonPrefixLen*[T](c: CritBitTree[T]): int {.inline, since((1, 3)).} = else: c.root.byte else: 0 -proc toCritBitTree*[T](pairs: openArray[(string, T)]): CritBitTree[T] {.since: (1, 3).} = +proc toCritBitTree*[T](pairs: sink openArray[(string, T)]): CritBitTree[T] {.since: (1, 3).} = ## Creates a new `CritBitTree` that contains the given `pairs`. runnableExamples: doAssert {"a": "0", "b": "1", "c": "2"}.toCritBitTree is CritBitTree[string] @@ -526,7 +529,7 @@ proc toCritBitTree*[T](pairs: openArray[(string, T)]): CritBitTree[T] {.since: ( for item in pairs: result.incl item[0], item[1] -proc toCritBitTree*(items: openArray[string]): CritBitTree[void] {.since: (1, 3).} = +proc toCritBitTree*(items: sink openArray[string]): CritBitTree[void] {.since: (1, 3).} = ## Creates a new `CritBitTree` that contains the given `items`. runnableExamples: doAssert ["a", "b", "c"].toCritBitTree is CritBitTree[void] diff --git a/lib/pure/collections/deques.nim b/lib/pure/collections/deques.nim index c8bbb1cc6..d2b0099f2 100644 --- a/lib/pure/collections/deques.nim +++ b/lib/pure/collections/deques.nim @@ -10,8 +10,9 @@ ## An implementation of a `deque`:idx: (double-ended queue). ## The underlying implementation uses a `seq`. ## -## Note that none of the procs that get an individual value from the deque should be used -## on an empty deque. +## .. note:: None of the procs that get an individual value from the deque should be used +## on an empty deque. +## ## If compiled with the `boundChecks` option, those procs will raise an `IndexDefect` ## on such access. This should not be relied upon, as `-d:danger` or `--checks:off` will ## disable those checks and then the procs may return garbage or crash the program. @@ -46,11 +47,10 @@ runnableExamples: ## See also ## ======== ## * `lists module <lists.html>`_ for singly and doubly linked lists and rings -## * `channels module <channels_builtin.html>`_ for inter-thread communication import std/private/since -import math +import std/[assertions, hashes, math] type Deque*[T] = object @@ -59,20 +59,28 @@ type ## To initialize an empty deque, ## use the `initDeque proc <#initDeque,int>`_. data: seq[T] - head, tail, count, mask: int + + # `head` and `tail` are masked only when accessing an element of `data` + # so that `tail - head == data.len` when the deque is full. + # They are uint so that incrementing/decrementing them doesn't cause + # over/underflow. You can get a number of items with `tail - head` + # even if `tail` or `head` is wraps around and `tail < head`, because + # `tail - head == (uint.high + 1 + tail) - head` when `tail < head`. + head, tail: uint const defaultInitialSize* = 4 template initImpl(result: typed, initialSize: int) = let correctSize = nextPowerOfTwo(initialSize) - result.mask = correctSize - 1 newSeq(result.data, correctSize) template checkIfInitialized(deq: typed) = - when compiles(defaultInitialSize): - if deq.mask == 0: - initImpl(deq, defaultInitialSize) + if deq.data.len == 0: + initImpl(deq, defaultInitialSize) + +func mask[T](deq: Deque[T]): uint {.inline.} = + uint(deq.data.len) - 1 proc initDeque*[T](initialSize: int = defaultInitialSize): Deque[T] = ## Creates a new empty deque. @@ -86,41 +94,27 @@ proc initDeque*[T](initialSize: int = defaultInitialSize): Deque[T] = ## * `toDeque proc <#toDeque,openArray[T]>`_ result.initImpl(initialSize) -proc toDeque*[T](x: openArray[T]): Deque[T] {.since: (1, 3).} = - ## Creates a new deque that contains the elements of `x` (in the same order). - ## - ## **See also:** - ## * `initDeque proc <#initDeque,int>`_ - runnableExamples: - let a = toDeque([7, 8, 9]) - assert len(a) == 3 - assert $a == "[7, 8, 9]" - - result.initImpl(x.len) - for item in items(x): - result.addLast(item) - -proc len*[T](deq: Deque[T]): int {.inline.} = +func len*[T](deq: Deque[T]): int {.inline.} = ## Returns the number of elements of `deq`. - result = deq.count + int(deq.tail - deq.head) template emptyCheck(deq) = # Bounds check for the regular deque access. when compileOption("boundChecks"): - if unlikely(deq.count < 1): + if unlikely(deq.len < 1): raise newException(IndexDefect, "Empty deque.") template xBoundsCheck(deq, i) = # Bounds check for the array like accesses. when compileOption("boundChecks"): # `-d:danger` or `--checks:off` should disable this. - if unlikely(i >= deq.count): # x < deq.low is taken care by the Natural parameter + if unlikely(i >= deq.len): # x < deq.low is taken care by the Natural parameter raise newException(IndexDefect, - "Out of bounds: " & $i & " > " & $(deq.count - 1)) + "Out of bounds: " & $i & " > " & $(deq.len - 1)) if unlikely(i < 0): # when used with BackwardsIndex raise newException(IndexDefect, "Out of bounds: " & $i & " < 0") -proc `[]`*[T](deq: Deque[T], i: Natural): T {.inline.} = +proc `[]`*[T](deq: Deque[T], i: Natural): lent T {.inline.} = ## Accesses the `i`-th element of `deq`. runnableExamples: let a = [10, 20, 30, 40, 50].toDeque @@ -129,7 +123,7 @@ proc `[]`*[T](deq: Deque[T], i: Natural): T {.inline.} = doAssertRaises(IndexDefect, echo a[8]) xBoundsCheck(deq, i) - return deq.data[(deq.head + i) and deq.mask] + return deq.data[(deq.head + i.uint) and deq.mask] proc `[]`*[T](deq: var Deque[T], i: Natural): var T {.inline.} = ## Accesses the `i`-th element of `deq` and returns a mutable @@ -140,9 +134,9 @@ proc `[]`*[T](deq: var Deque[T], i: Natural): var T {.inline.} = assert a[0] == 11 xBoundsCheck(deq, i) - return deq.data[(deq.head + i) and deq.mask] + return deq.data[(deq.head + i.uint) and deq.mask] -proc `[]=`*[T](deq: var Deque[T], i: Natural, val: T) {.inline.} = +proc `[]=`*[T](deq: var Deque[T], i: Natural, val: sink T) {.inline.} = ## Sets the `i`-th element of `deq` to `val`. runnableExamples: var a = [10, 20, 30, 40, 50].toDeque @@ -152,9 +146,9 @@ proc `[]=`*[T](deq: var Deque[T], i: Natural, val: T) {.inline.} = checkIfInitialized(deq) xBoundsCheck(deq, i) - deq.data[(deq.head + i) and deq.mask] = val + deq.data[(deq.head + i.uint) and deq.mask] = val -proc `[]`*[T](deq: Deque[T], i: BackwardsIndex): T {.inline.} = +proc `[]`*[T](deq: Deque[T], i: BackwardsIndex): lent T {.inline.} = ## Accesses the backwards indexed `i`-th element. ## ## `deq[^1]` is the last element. @@ -180,7 +174,7 @@ proc `[]`*[T](deq: var Deque[T], i: BackwardsIndex): var T {.inline.} = xBoundsCheck(deq, deq.len - int(i)) return deq[deq.len - int(i)] -proc `[]=`*[T](deq: var Deque[T], i: BackwardsIndex, x: T) {.inline.} = +proc `[]=`*[T](deq: var Deque[T], i: BackwardsIndex, x: sink T) {.inline.} = ## Sets the backwards indexed `i`-th element of `deq` to `x`. ## ## `deq[^1]` is the last element. @@ -194,27 +188,25 @@ proc `[]=`*[T](deq: var Deque[T], i: BackwardsIndex, x: T) {.inline.} = xBoundsCheck(deq, deq.len - int(i)) deq[deq.len - int(i)] = x -iterator items*[T](deq: Deque[T]): T = +iterator items*[T](deq: Deque[T]): lent T = ## Yields every element of `deq`. ## ## **See also:** - ## * `mitems iterator <#mitems,Deque[T]>`_ + ## * `mitems iterator <#mitems.i,Deque[T]>`_ runnableExamples: from std/sequtils import toSeq let a = [10, 20, 30, 40, 50].toDeque assert toSeq(a.items) == @[10, 20, 30, 40, 50] - var i = deq.head - for c in 0 ..< deq.count: - yield deq.data[i] - i = (i + 1) and deq.mask + for c in 0 ..< deq.len: + yield deq.data[(deq.head + c.uint) and deq.mask] iterator mitems*[T](deq: var Deque[T]): var T = ## Yields every element of `deq`, which can be modified. ## ## **See also:** - ## * `items iterator <#items,Deque[T]>`_ + ## * `items iterator <#items.i,Deque[T]>`_ runnableExamples: var a = [10, 20, 30, 40, 50].toDeque assert $a == "[10, 20, 30, 40, 50]" @@ -222,10 +214,8 @@ iterator mitems*[T](deq: var Deque[T]): var T = x = 5 * x - 1 assert $a == "[49, 99, 149, 199, 249]" - var i = deq.head - for c in 0 ..< deq.count: - yield deq.data[i] - i = (i + 1) and deq.mask + for c in 0 ..< deq.len: + yield deq.data[(deq.head + c.uint) and deq.mask] iterator pairs*[T](deq: Deque[T]): tuple[key: int, val: T] = ## Yields every `(position, value)`-pair of `deq`. @@ -235,10 +225,8 @@ iterator pairs*[T](deq: Deque[T]): tuple[key: int, val: T] = let a = [10, 20, 30].toDeque assert toSeq(a.pairs) == @[(0, 10), (1, 20), (2, 30)] - var i = deq.head - for c in 0 ..< deq.count: - yield (c, deq.data[i]) - i = (i + 1) and deq.mask + for c in 0 ..< deq.len: + yield (c, deq.data[(deq.head + c.uint) and deq.mask]) proc contains*[T](deq: Deque[T], item: T): bool {.inline.} = ## Returns true if `item` is in `deq` or false if not found. @@ -257,24 +245,24 @@ proc contains*[T](deq: Deque[T], item: T): bool {.inline.} = proc expandIfNeeded[T](deq: var Deque[T]) = checkIfInitialized(deq) - var cap = deq.mask + 1 - if unlikely(deq.count >= cap): + let cap = deq.data.len + assert deq.len <= cap + if unlikely(deq.len == cap): var n = newSeq[T](cap * 2) var i = 0 for x in mitems(deq): - when nimVM: n[i] = x # workaround for VM bug + when nimvm: n[i] = x # workaround for VM bug else: n[i] = move(x) inc i deq.data = move(n) - deq.mask = cap * 2 - 1 - deq.tail = deq.count + deq.tail = cap.uint deq.head = 0 -proc addFirst*[T](deq: var Deque[T], item: T) = +proc addFirst*[T](deq: var Deque[T], item: sink T) = ## Adds an `item` to the beginning of `deq`. ## ## **See also:** - ## * `addLast proc <#addLast,Deque[T],T>`_ + ## * `addLast proc <#addLast,Deque[T],sinkT>`_ runnableExamples: var a = initDeque[int]() for i in 1 .. 5: @@ -282,15 +270,14 @@ proc addFirst*[T](deq: var Deque[T], item: T) = assert $a == "[50, 40, 30, 20, 10]" expandIfNeeded(deq) - inc deq.count - deq.head = (deq.head - 1) and deq.mask - deq.data[deq.head] = item + dec deq.head + deq.data[deq.head and deq.mask] = item -proc addLast*[T](deq: var Deque[T], item: T) = +proc addLast*[T](deq: var Deque[T], item: sink T) = ## Adds an `item` to the end of `deq`. ## ## **See also:** - ## * `addFirst proc <#addFirst,Deque[T],T>`_ + ## * `addFirst proc <#addFirst,Deque[T],sinkT>`_ runnableExamples: var a = initDeque[int]() for i in 1 .. 5: @@ -298,11 +285,24 @@ proc addLast*[T](deq: var Deque[T], item: T) = assert $a == "[10, 20, 30, 40, 50]" expandIfNeeded(deq) - inc deq.count - deq.data[deq.tail] = item - deq.tail = (deq.tail + 1) and deq.mask + deq.data[deq.tail and deq.mask] = item + inc deq.tail -proc peekFirst*[T](deq: Deque[T]): T {.inline.} = +proc toDeque*[T](x: openArray[T]): Deque[T] {.since: (1, 3).} = + ## Creates a new deque that contains the elements of `x` (in the same order). + ## + ## **See also:** + ## * `initDeque proc <#initDeque,int>`_ + runnableExamples: + let a = toDeque([7, 8, 9]) + assert len(a) == 3 + assert $a == "[7, 8, 9]" + + result.initImpl(x.len) + for item in items(x): + result.addLast(item) + +proc peekFirst*[T](deq: Deque[T]): lent T {.inline.} = ## Returns the first element of `deq`, but does not remove it from the deque. ## ## **See also:** @@ -315,9 +315,9 @@ proc peekFirst*[T](deq: Deque[T]): T {.inline.} = assert len(a) == 5 emptyCheck(deq) - result = deq.data[deq.head] + result = deq.data[deq.head and deq.mask] -proc peekLast*[T](deq: Deque[T]): T {.inline.} = +proc peekLast*[T](deq: Deque[T]): lent T {.inline.} = ## Returns the last element of `deq`, but does not remove it from the deque. ## ## **See also:** @@ -345,7 +345,7 @@ proc peekFirst*[T](deq: var Deque[T]): var T {.inline, since: (1, 3).} = assert $a == "[99, 20, 30, 40, 50]" emptyCheck(deq) - result = deq.data[deq.head] + result = deq.data[deq.head and deq.mask] proc peekLast*[T](deq: var Deque[T]): var T {.inline, since: (1, 3).} = ## Returns a mutable reference to the last element of `deq`, @@ -378,10 +378,8 @@ proc popFirst*[T](deq: var Deque[T]): T {.inline, discardable.} = assert $a == "[20, 30, 40, 50]" emptyCheck(deq) - dec deq.count - result = deq.data[deq.head] - destroy(deq.data[deq.head]) - deq.head = (deq.head + 1) and deq.mask + result = move deq.data[deq.head and deq.mask] + inc deq.head proc popLast*[T](deq: var Deque[T]): T {.inline, discardable.} = ## Removes and returns the last element of the `deq`. @@ -396,10 +394,8 @@ proc popLast*[T](deq: var Deque[T]): T {.inline, discardable.} = assert $a == "[10, 20, 30, 40]" emptyCheck(deq) - dec deq.count - deq.tail = (deq.tail - 1) and deq.mask - result = deq.data[deq.tail] - destroy(deq.data[deq.tail]) + dec deq.tail + result = move deq.data[deq.tail and deq.mask] proc clear*[T](deq: var Deque[T]) {.inline.} = ## Resets the deque so that it is empty. @@ -413,7 +409,6 @@ proc clear*[T](deq: var Deque[T]) {.inline.} = assert len(a) == 0 for el in mitems(deq): destroy(el) - deq.count = 0 deq.tail = deq.head proc shrink*[T](deq: var Deque[T], fromFirst = 0, fromLast = 0) = @@ -433,19 +428,17 @@ proc shrink*[T](deq: var Deque[T], fromFirst = 0, fromLast = 0) = a.shrink(fromFirst = 2, fromLast = 1) assert $a == "[30, 40]" - if fromFirst + fromLast > deq.count: + if fromFirst + fromLast > deq.len: clear(deq) return for i in 0 ..< fromFirst: - destroy(deq.data[deq.head]) - deq.head = (deq.head + 1) and deq.mask + destroy(deq.data[deq.head and deq.mask]) + inc deq.head for i in 0 ..< fromLast: - destroy(deq.data[deq.tail]) - deq.tail = (deq.tail - 1) and deq.mask - - dec deq.count, fromFirst + fromLast + destroy(deq.data[(deq.tail - 1) and deq.mask]) + dec deq.tail proc `$`*[T](deq: Deque[T]): string = ## Turns a deque into its string representation. @@ -458,3 +451,30 @@ proc `$`*[T](deq: Deque[T]): string = if result.len > 1: result.add(", ") result.addQuoted(x) result.add("]") + +func `==`*[T](deq1, deq2: Deque[T]): bool = + ## The `==` operator for Deque. + ## Returns `true` if both deques contains the same values in the same order. + runnableExamples: + var a, b = initDeque[int]() + a.addFirst(2) + a.addFirst(1) + b.addLast(1) + b.addLast(2) + doAssert a == b + + if deq1.len != deq2.len: + return false + + for i in 0 ..< deq1.len: + if deq1.data[(deq1.head + i.uint) and deq1.mask] != deq2.data[(deq2.head + i.uint) and deq2.mask]: + return false + + true + +func hash*[T](deq: Deque[T]): Hash = + ## Hashing of Deque. + var h: Hash = 0 + for x in deq: + h = h !& hash(x) + !$h diff --git a/lib/pure/collections/hashcommon.nim b/lib/pure/collections/hashcommon.nim index 030446176..17785c8c7 100644 --- a/lib/pure/collections/hashcommon.nim +++ b/lib/pure/collections/hashcommon.nim @@ -10,6 +10,11 @@ # An `include` file which contains common code for # hash sets and tables. +when defined(nimPreviewSlimSystem): + import std/assertions + +import std / outparams + const growthFactor = 2 @@ -33,16 +38,6 @@ proc slotsNeeded(count: Natural): int {.inline.} = # Make sure to synchronize with `mustRehash` above result = nextPowerOfTwo(count * 3 div 2 + 4) -proc rightSize*(count: Natural): int {.inline, deprecated: "Deprecated since 1.4.0".} = - ## **Deprecated since Nim v1.4.0**, it is not needed anymore - ## because picking the correct size is done internally. - ## - ## Return the value of `initialSize` to support `count` items. - ## - ## If more items are expected to be added, simply add that - ## expected extra amount to the parameter before calling this. - result = count - template rawGetKnownHCImpl() {.dirty.} = if t.dataLen == 0: return -1 @@ -63,7 +58,10 @@ proc rawGetKnownHC[X, A](t: X, key: A, hc: Hash): int {.inline.} = template genHashImpl(key, hc: typed) = hc = hash(key) if hc == 0: # This almost never taken branch should be very predictable. - hc = 314159265 # Value doesn't matter; Any non-zero favorite is fine. + when sizeof(int) < 4: + hc = 31415 # Value doesn't matter; Any non-zero favorite is fine <= 16-bit. + else: + hc = 314159265 # Value doesn't matter; Any non-zero favorite is fine. template genHash(key: typed): Hash = var res: Hash @@ -74,5 +72,5 @@ template rawGetImpl() {.dirty.} = genHashImpl(key, hc) rawGetKnownHCImpl() -proc rawGet[X, A](t: X, key: A, hc: var Hash): int {.inline.} = +proc rawGet[X, A](t: X, key: A, hc: var Hash): int {.inline, outParamsAt: [3].} = rawGetImpl() diff --git a/lib/pure/collections/heapqueue.nim b/lib/pure/collections/heapqueue.nim index 89e532951..96f9b4430 100644 --- a/lib/pure/collections/heapqueue.nim +++ b/lib/pure/collections/heapqueue.nim @@ -47,6 +47,9 @@ runnableExamples: import std/private/since +when defined(nimPreviewSlimSystem): + import std/assertions + type HeapQueue*[T] = object ## A heap queue, commonly known as a priority queue. data: seq[T] @@ -59,7 +62,7 @@ proc initHeapQueue*[T](): HeapQueue[T] = ## ## **See also:** ## * `toHeapQueue proc <#toHeapQueue,openArray[T]>`_ - discard + result = default(HeapQueue[T]) proc len*[T](heap: HeapQueue[T]): int {.inline.} = ## Returns the number of elements of `heap`. @@ -73,6 +76,13 @@ proc `[]`*[T](heap: HeapQueue[T], i: Natural): lent T {.inline.} = ## Accesses the i-th element of `heap`. heap.data[i] +iterator items*[T](heap: HeapQueue[T]): lent T {.inline, since: (2, 1, 1).} = + ## Iterates over each item of `heap`. + let L = len(heap) + for i in 0 .. high(heap.data): + yield heap.data[i] + assert(len(heap) == L, "the length of the HeapQueue changed while iterating over it") + proc heapCmp[T](x, y: T): bool {.inline.} = x < y proc siftup[T](heap: var HeapQueue[T], startpos, p: int) = @@ -178,6 +188,11 @@ proc find*[T](heap: HeapQueue[T], x: T): int {.since: (1, 3).} = for i in 0 ..< heap.len: if heap[i] == x: return i +proc contains*[T](heap: HeapQueue[T], x: T): bool {.since: (2, 1, 1).} = + ## Returns true if `x` is in `heap` or false if not found. This is a shortcut + ## for `find(heap, x) >= 0`. + result = find(heap, x) >= 0 + proc del*[T](heap: var HeapQueue[T], index: Natural) = ## Removes the element at `index` from `heap`, maintaining the heap invariant. runnableExamples: diff --git a/lib/pure/collections/intsets.nim b/lib/pure/collections/intsets.nim index 05e5addcc..765a23e97 100644 --- a/lib/pure/collections/intsets.nim +++ b/lib/pure/collections/intsets.nim @@ -8,7 +8,7 @@ # ## Specialization of the generic `packedsets module <packedsets.html>`_ -## for ordinal sparse sets. +## (see its documentation for more examples) for ordinal sparse sets. import std/private/since import std/packedsets diff --git a/lib/pure/collections/lists.nim b/lib/pure/collections/lists.nim index c1fcb0743..6b88747ef 100644 --- a/lib/pure/collections/lists.nim +++ b/lib/pure/collections/lists.nim @@ -19,15 +19,15 @@ ## ## ## Lists runnableExamples: - var l = initDoublyLinkedList[int]() + var list = initDoublyLinkedList[int]() let a = newDoublyLinkedNode[int](3) b = newDoublyLinkedNode[int](7) c = newDoublyLinkedNode[int](9) - l.add(a) - l.add(b) - l.prepend(c) + list.add(a) + list.add(b) + list.prepend(c) assert a.next == b assert a.prev == c @@ -38,15 +38,15 @@ runnableExamples: ## ## Rings runnableExamples: - var l = initSinglyLinkedRing[int]() + var ring = initSinglyLinkedRing[int]() let a = newSinglyLinkedNode[int](3) b = newSinglyLinkedNode[int](7) c = newSinglyLinkedNode[int](9) - l.add(a) - l.add(b) - l.prepend(c) + ring.add(a) + ring.add(b) + ring.prepend(c) assert c.next == a assert a.next == b @@ -56,19 +56,18 @@ runnableExamples: ## # See also ## * `deques module <deques.html>`_ for double-ended queues -## * `sharedlist module <sharedlist.html>`_ for shared singly-linked lists import std/private/since -when not defined(nimHasCursor): - {.pragma: cursor.} +when defined(nimPreviewSlimSystem): + import std/assertions type DoublyLinkedNodeObj*[T] = object ## A node of a doubly linked list. ## ## It consists of a `value` field, and pointers to `next` and `prev`. - next*: <//>(DoublyLinkedNode[T]) + next*: DoublyLinkedNode[T] prev* {.cursor.}: DoublyLinkedNode[T] value*: T DoublyLinkedNode*[T] = ref DoublyLinkedNodeObj[T] @@ -77,23 +76,23 @@ type ## A node of a singly linked list. ## ## It consists of a `value` field, and a pointer to `next`. - next*: <//>(SinglyLinkedNode[T]) + next*: SinglyLinkedNode[T] value*: T SinglyLinkedNode*[T] = ref SinglyLinkedNodeObj[T] SinglyLinkedList*[T] = object ## A singly linked list. - head*: <//>(SinglyLinkedNode[T]) + head*: SinglyLinkedNode[T] tail* {.cursor.}: SinglyLinkedNode[T] DoublyLinkedList*[T] = object ## A doubly linked list. - head*: <//>(DoublyLinkedNode[T]) + head*: DoublyLinkedNode[T] tail* {.cursor.}: DoublyLinkedNode[T] SinglyLinkedRing*[T] = object ## A singly linked ring. - head*: <//>(SinglyLinkedNode[T]) + head*: SinglyLinkedNode[T] tail* {.cursor.}: SinglyLinkedNode[T] DoublyLinkedRing*[T] = object @@ -148,7 +147,7 @@ proc initDoublyLinkedRing*[T](): DoublyLinkedRing[T] = discard -proc newDoublyLinkedNode*[T](value: T): <//>(DoublyLinkedNode[T]) = +proc newDoublyLinkedNode*[T](value: T): DoublyLinkedNode[T] = ## Creates a new doubly linked node with the given `value`. runnableExamples: let n = newDoublyLinkedNode[int](5) @@ -157,7 +156,7 @@ proc newDoublyLinkedNode*[T](value: T): <//>(DoublyLinkedNode[T]) = new(result) result.value = value -proc newSinglyLinkedNode*[T](value: T): <//>(SinglyLinkedNode[T]) = +proc newSinglyLinkedNode*[T](value: T): SinglyLinkedNode[T] = ## Creates a new singly linked node with the given `value`. runnableExamples: let n = newSinglyLinkedNode[int](5) @@ -166,44 +165,22 @@ proc newSinglyLinkedNode*[T](value: T): <//>(SinglyLinkedNode[T]) = new(result) result.value = value -func toSinglyLinkedList*[T](elems: openArray[T]): SinglyLinkedList[T] {.since: (1, 5, 1).} = - ## Creates a new `SinglyLinkedList` from the members of `elems`. - runnableExamples: - from std/sequtils import toSeq - let a = [1, 2, 3, 4, 5].toSinglyLinkedList - assert a.toSeq == [1, 2, 3, 4, 5] - - result = initSinglyLinkedList[T]() - for elem in elems.items: - result.add(elem) - -func toDoublyLinkedList*[T](elems: openArray[T]): DoublyLinkedList[T] {.since: (1, 5, 1).} = - ## Creates a new `DoublyLinkedList` from the members of `elems`. - runnableExamples: - from std/sequtils import toSeq - let a = [1, 2, 3, 4, 5].toDoublyLinkedList - assert a.toSeq == [1, 2, 3, 4, 5] - - result = initDoublyLinkedList[T]() - for elem in elems.items: - result.add(elem) - template itemsListImpl() {.dirty.} = - var it = list.head + var it {.cursor.} = L.head while it != nil: yield it.value it = it.next template itemsRingImpl() {.dirty.} = - var it = ring.head + var it {.cursor.} = L.head if it != nil: while true: yield it.value it = it.next - if it == ring.head: break + if it == L.head: break -iterator items*[T](list: SomeLinkedList[T]): T = - ## Yields every value of `list`. +iterator items*[T](L: SomeLinkedList[T]): T = + ## Yields every value of `L`. ## ## **See also:** ## * `mitems iterator <#mitems.i,SomeLinkedList[T]>`_ @@ -218,8 +195,8 @@ iterator items*[T](list: SomeLinkedList[T]): T = itemsListImpl() -iterator items*[T](ring: SomeLinkedRing[T]): T = - ## Yields every value of `ring`. +iterator items*[T](L: SomeLinkedRing[T]): T = + ## Yields every value of `L`. ## ## **See also:** ## * `mitems iterator <#mitems.i,SomeLinkedRing[T]>`_ @@ -234,8 +211,8 @@ iterator items*[T](ring: SomeLinkedRing[T]): T = itemsRingImpl() -iterator mitems*[T](list: var SomeLinkedList[T]): var T = - ## Yields every value of `list` so that you can modify it. +iterator mitems*[T](L: var SomeLinkedList[T]): var T = + ## Yields every value of `L` so that you can modify it. ## ## **See also:** ## * `items iterator <#items.i,SomeLinkedList[T]>`_ @@ -251,8 +228,8 @@ iterator mitems*[T](list: var SomeLinkedList[T]): var T = itemsListImpl() -iterator mitems*[T](ring: var SomeLinkedRing[T]): var T = - ## Yields every value of `ring` so that you can modify it. +iterator mitems*[T](L: var SomeLinkedRing[T]): var T = + ## Yields every value of `L` so that you can modify it. ## ## **See also:** ## * `items iterator <#items.i,SomeLinkedRing[T]>`_ @@ -268,7 +245,7 @@ iterator mitems*[T](ring: var SomeLinkedRing[T]): var T = itemsRingImpl() -iterator nodes*[T](list: SomeLinkedList[T]): SomeLinkedNode[T] = +iterator nodes*[T](L: SomeLinkedList[T]): SomeLinkedNode[T] = ## Iterates over every node of `x`. Removing the current node from the ## list during traversal is supported. ## @@ -287,13 +264,13 @@ iterator nodes*[T](list: SomeLinkedList[T]): SomeLinkedNode[T] = x.value = 5 * x.value - 1 assert $a == "[49, 99, 199, 249]" - var it = list.head + var it {.cursor.} = L.head while it != nil: let nxt = it.next yield it it = nxt -iterator nodes*[T](ring: SomeLinkedRing[T]): SomeLinkedNode[T] = +iterator nodes*[T](L: SomeLinkedRing[T]): SomeLinkedNode[T] = ## Iterates over every node of `x`. Removing the current node from the ## list during traversal is supported. ## @@ -312,27 +289,27 @@ iterator nodes*[T](ring: SomeLinkedRing[T]): SomeLinkedNode[T] = x.value = 5 * x.value - 1 assert $a == "[49, 99, 199, 249]" - var it = ring.head + var it {.cursor.} = L.head if it != nil: while true: let nxt = it.next yield it it = nxt - if it == ring.head: break + if it == L.head: break -proc `$`*[T](l: SomeLinkedCollection[T]): string = +proc `$`*[T](L: SomeLinkedCollection[T]): string = ## Turns a list into its string representation for logging and printing. runnableExamples: let a = [1, 2, 3, 4].toSinglyLinkedList assert $a == "[1, 2, 3, 4]" result = "[" - for x in nodes(l): + for x in nodes(L): if result.len > 1: result.add(", ") result.addQuoted(x.value) result.add("]") -proc find*[T](l: SomeLinkedCollection[T], value: T): SomeLinkedNode[T] = +proc find*[T](L: SomeLinkedCollection[T], value: T): SomeLinkedNode[T] = ## Searches in the list for a value. Returns `nil` if the value does not ## exist. ## @@ -343,10 +320,10 @@ proc find*[T](l: SomeLinkedCollection[T], value: T): SomeLinkedNode[T] = assert a.find(9).value == 9 assert a.find(1) == nil - for x in nodes(l): + for x in nodes(L): if x.value == value: return x -proc contains*[T](l: SomeLinkedCollection[T], value: T): bool {.inline.} = +proc contains*[T](L: SomeLinkedCollection[T], value: T): bool {.inline.} = ## Searches in the list for a value. Returns `false` if the value does not ## exist, `true` otherwise. This allows the usage of the `in` and `notin` ## operators. @@ -360,7 +337,7 @@ proc contains*[T](l: SomeLinkedCollection[T], value: T): bool {.inline.} = assert(not a.contains(1)) assert 2 notin a - result = find(l, value) != nil + result = find(L, value) != nil proc prepend*[T: SomeLinkedList](a: var T, b: T) {.since: (1, 5, 1).} = ## Prepends a shallow copy of `b` to the beginning of `a`. @@ -407,12 +384,10 @@ proc prependMoved*[T: SomeLinkedList](a, b: var T) {.since: (1, 5, 1).} = assert s == [0, 1, 0, 1, 0, 1] b.addMoved(a) - when defined(js): # XXX: swap broken in js; bug #16771 - (b, a) = (a, b) - else: swap a, b + swap a, b -proc add*[T](list: var SinglyLinkedList[T], n: SinglyLinkedNode[T]) {.inline.} = - ## Appends (adds to the end) a node `n` to `list`. Efficiency: O(1). +proc add*[T](L: var SinglyLinkedList[T], n: SinglyLinkedNode[T]) {.inline.} = + ## Appends (adds to the end) a node `n` to `L`. Efficiency: O(1). ## ## **See also:** ## * `add proc <#add,SinglyLinkedList[T],T>`_ for appending a value @@ -426,14 +401,14 @@ proc add*[T](list: var SinglyLinkedList[T], n: SinglyLinkedNode[T]) {.inline.} = assert a.contains(9) n.next = nil - if list.tail != nil: - assert(list.tail.next == nil) - list.tail.next = n - list.tail = n - if list.head == nil: list.head = n + if L.tail != nil: + assert(L.tail.next == nil) + L.tail.next = n + L.tail = n + if L.head == nil: L.head = n -proc add*[T](list: var SinglyLinkedList[T], value: T) {.inline.} = - ## Appends (adds to the end) a value to `list`. Efficiency: O(1). +proc add*[T](L: var SinglyLinkedList[T], value: T) {.inline.} = + ## Appends (adds to the end) a value to `L`. Efficiency: O(1). ## ## **See also:** ## * `add proc <#add,SinglyLinkedList[T],T>`_ for appending a value @@ -446,11 +421,11 @@ proc add*[T](list: var SinglyLinkedList[T], value: T) {.inline.} = a.add(8) assert a.contains(9) - add(list, newSinglyLinkedNode(value)) + add(L, newSinglyLinkedNode(value)) -proc prepend*[T](list: var SinglyLinkedList[T], +proc prepend*[T](L: var SinglyLinkedList[T], n: SinglyLinkedNode[T]) {.inline.} = - ## Prepends (adds to the beginning) a node to `list`. Efficiency: O(1). + ## Prepends (adds to the beginning) a node to `L`. Efficiency: O(1). ## ## **See also:** ## * `add proc <#add,SinglyLinkedList[T],SinglyLinkedNode[T]>`_ @@ -463,12 +438,12 @@ proc prepend*[T](list: var SinglyLinkedList[T], a.prepend(n) assert a.contains(9) - n.next = list.head - list.head = n - if list.tail == nil: list.tail = n + n.next = L.head + L.head = n + if L.tail == nil: L.tail = n -proc prepend*[T](list: var SinglyLinkedList[T], value: T) {.inline.} = - ## Prepends (adds to the beginning) a node to `list`. Efficiency: O(1). +proc prepend*[T](L: var SinglyLinkedList[T], value: T) {.inline.} = + ## Prepends (adds to the beginning) a node to `L`. Efficiency: O(1). ## ## **See also:** ## * `add proc <#add,SinglyLinkedList[T],SinglyLinkedNode[T]>`_ @@ -482,7 +457,7 @@ proc prepend*[T](list: var SinglyLinkedList[T], value: T) {.inline.} = a.prepend(8) assert a.contains(9) - prepend(list, newSinglyLinkedNode(value)) + prepend(L, newSinglyLinkedNode(value)) func copy*[T](a: SinglyLinkedList[T]): SinglyLinkedList[T] {.since: (1, 5, 1).} = ## Creates a shallow copy of `a`. @@ -531,17 +506,18 @@ proc addMoved*[T](a, b: var SinglyLinkedList[T]) {.since: (1, 5, 1).} = ci assert s == [0, 1, 0, 1, 0, 1] - if a.tail != nil: - a.tail.next = b.head - a.tail = b.tail - if a.head == nil: - a.head = b.head + if b.head != nil: + if a.head == nil: + a.head = b.head + else: + a.tail.next = b.head + a.tail = b.tail if a.addr != b.addr: b.head = nil b.tail = nil -proc add*[T](list: var DoublyLinkedList[T], n: DoublyLinkedNode[T]) = - ## Appends (adds to the end) a node `n` to `list`. Efficiency: O(1). +proc add*[T](L: var DoublyLinkedList[T], n: DoublyLinkedNode[T]) = + ## Appends (adds to the end) a node `n` to `L`. Efficiency: O(1). ## ## **See also:** ## * `add proc <#add,DoublyLinkedList[T],T>`_ for appending a value @@ -557,15 +533,15 @@ proc add*[T](list: var DoublyLinkedList[T], n: DoublyLinkedNode[T]) = assert a.contains(9) n.next = nil - n.prev = list.tail - if list.tail != nil: - assert(list.tail.next == nil) - list.tail.next = n - list.tail = n - if list.head == nil: list.head = n + n.prev = L.tail + if L.tail != nil: + assert(L.tail.next == nil) + L.tail.next = n + L.tail = n + if L.head == nil: L.head = n -proc add*[T](list: var DoublyLinkedList[T], value: T) = - ## Appends (adds to the end) a value to `list`. Efficiency: O(1). +proc add*[T](L: var DoublyLinkedList[T], value: T) = + ## Appends (adds to the end) a value to `L`. Efficiency: O(1). ## ## **See also:** ## * `add proc <#add,DoublyLinkedList[T],DoublyLinkedNode[T]>`_ @@ -581,10 +557,10 @@ proc add*[T](list: var DoublyLinkedList[T], value: T) = a.add(8) assert a.contains(9) - add(list, newDoublyLinkedNode(value)) + add(L, newDoublyLinkedNode(value)) -proc prepend*[T](list: var DoublyLinkedList[T], n: DoublyLinkedNode[T]) = - ## Prepends (adds to the beginning) a node `n` to `list`. Efficiency: O(1). +proc prepend*[T](L: var DoublyLinkedList[T], n: DoublyLinkedNode[T]) = + ## Prepends (adds to the beginning) a node `n` to `L`. Efficiency: O(1). ## ## **See also:** ## * `add proc <#add,DoublyLinkedList[T],DoublyLinkedNode[T]>`_ @@ -600,15 +576,15 @@ proc prepend*[T](list: var DoublyLinkedList[T], n: DoublyLinkedNode[T]) = assert a.contains(9) n.prev = nil - n.next = list.head - if list.head != nil: - assert(list.head.prev == nil) - list.head.prev = n - list.head = n - if list.tail == nil: list.tail = n + n.next = L.head + if L.head != nil: + assert(L.head.prev == nil) + L.head.prev = n + L.head = n + if L.tail == nil: L.tail = n -proc prepend*[T](list: var DoublyLinkedList[T], value: T) = - ## Prepends (adds to the beginning) a value to `list`. Efficiency: O(1). +proc prepend*[T](L: var DoublyLinkedList[T], value: T) = + ## Prepends (adds to the beginning) a value to `L`. Efficiency: O(1). ## ## **See also:** ## * `add proc <#add,DoublyLinkedList[T],DoublyLinkedNode[T]>`_ @@ -624,7 +600,7 @@ proc prepend*[T](list: var DoublyLinkedList[T], value: T) = a.prepend(8) assert a.contains(9) - prepend(list, newDoublyLinkedNode(value)) + prepend(L, newDoublyLinkedNode(value)) func copy*[T](a: DoublyLinkedList[T]): DoublyLinkedList[T] {.since: (1, 5, 1).} = ## Creates a shallow copy of `a`. @@ -675,12 +651,12 @@ proc addMoved*[T](a, b: var DoublyLinkedList[T]) {.since: (1, 5, 1).} = assert s == [0, 1, 0, 1, 0, 1] if b.head != nil: - b.head.prev = a.tail - if a.tail != nil: - a.tail.next = b.head - a.tail = b.tail - if a.head == nil: - a.head = b.head + if a.head == nil: + a.head = b.head + else: + b.head.prev = a.tail + a.tail.next = b.head + a.tail = b.tail if a.addr != b.addr: b.head = nil b.tail = nil @@ -705,9 +681,9 @@ proc add*[T: SomeLinkedList](a: var T, b: T) {.since: (1, 5, 1).} = var tmp = b.copy a.addMoved(tmp) -proc remove*[T](list: var SinglyLinkedList[T], n: SinglyLinkedNode[T]): bool {.discardable.} = - ## Removes a node `n` from `list`. - ## Returns `true` if `n` was found in `list`. +proc remove*[T](L: var SinglyLinkedList[T], n: SinglyLinkedNode[T]): bool {.discardable.} = + ## Removes a node `n` from `L`. + ## Returns `true` if `n` was found in `L`. ## Efficiency: O(n); the list is traversed until `n` is found. ## Attempting to remove an element not contained in the list is a no-op. ## When the list is cyclic, the cycle is preserved after removal. @@ -728,22 +704,24 @@ proc remove*[T](list: var SinglyLinkedList[T], n: SinglyLinkedNode[T]): bool {.d ai assert s == [2, 2, 2, 2] - if n == list.head: - list.head = n.next - if list.tail.next == n: - list.tail.next = list.head # restore cycle + if n == L.head: + L.head = n.next + if L.tail.next == n: + L.tail.next = L.head # restore cycle else: - var prev = list.head + var prev {.cursor.} = L.head while prev.next != n and prev.next != nil: prev = prev.next if prev.next == nil: return false prev.next = n.next + if L.tail == n: + L.tail = prev # update tail if we removed the last node true -proc remove*[T](list: var DoublyLinkedList[T], n: DoublyLinkedNode[T]) = - ## Removes a node `n` from `list`. Efficiency: O(1). - ## This function assumes, for the sake of efficiency, that `n` is contained in `list`, +proc remove*[T](L: var DoublyLinkedList[T], n: DoublyLinkedNode[T]) = + ## Removes a node `n` from `L`. Efficiency: O(1). + ## This function assumes, for the sake of efficiency, that `n` is contained in `L`, ## otherwise the effects are undefined. ## When the list is cyclic, the cycle is preserved after removal. runnableExamples: @@ -763,15 +741,15 @@ proc remove*[T](list: var DoublyLinkedList[T], n: DoublyLinkedNode[T]) = ai assert s == [2, 2, 2, 2] - if n == list.tail: list.tail = n.prev - if n == list.head: list.head = n.next + if n == L.tail: L.tail = n.prev + if n == L.head: L.head = n.next if n.next != nil: n.next.prev = n.prev if n.prev != nil: n.prev.next = n.next -proc add*[T](ring: var SinglyLinkedRing[T], n: SinglyLinkedNode[T]) = - ## Appends (adds to the end) a node `n` to `ring`. Efficiency: O(1). +proc add*[T](L: var SinglyLinkedRing[T], n: SinglyLinkedNode[T]) = + ## Appends (adds to the end) a node `n` to `L`. Efficiency: O(1). ## ## **See also:** ## * `add proc <#add,SinglyLinkedRing[T],T>`_ for appending a value @@ -784,17 +762,17 @@ proc add*[T](ring: var SinglyLinkedRing[T], n: SinglyLinkedNode[T]) = a.add(n) assert a.contains(9) - if ring.head != nil: - n.next = ring.head - assert(ring.tail != nil) - ring.tail.next = n + if L.head != nil: + n.next = L.head + assert(L.tail != nil) + L.tail.next = n else: n.next = n - ring.head = n - ring.tail = n + L.head = n + L.tail = n -proc add*[T](ring: var SinglyLinkedRing[T], value: T) = - ## Appends (adds to the end) a value to `ring`. Efficiency: O(1). +proc add*[T](L: var SinglyLinkedRing[T], value: T) = + ## Appends (adds to the end) a value to `L`. Efficiency: O(1). ## ## **See also:** ## * `add proc <#add,SinglyLinkedRing[T],SinglyLinkedNode[T]>`_ @@ -808,10 +786,10 @@ proc add*[T](ring: var SinglyLinkedRing[T], value: T) = a.add(8) assert a.contains(9) - add(ring, newSinglyLinkedNode(value)) + add(L, newSinglyLinkedNode(value)) -proc prepend*[T](ring: var SinglyLinkedRing[T], n: SinglyLinkedNode[T]) = - ## Prepends (adds to the beginning) a node `n` to `ring`. Efficiency: O(1). +proc prepend*[T](L: var SinglyLinkedRing[T], n: SinglyLinkedNode[T]) = + ## Prepends (adds to the beginning) a node `n` to `L`. Efficiency: O(1). ## ## **See also:** ## * `add proc <#add,SinglyLinkedRing[T],SinglyLinkedNode[T]>`_ @@ -824,17 +802,17 @@ proc prepend*[T](ring: var SinglyLinkedRing[T], n: SinglyLinkedNode[T]) = a.prepend(n) assert a.contains(9) - if ring.head != nil: - n.next = ring.head - assert(ring.tail != nil) - ring.tail.next = n + if L.head != nil: + n.next = L.head + assert(L.tail != nil) + L.tail.next = n else: n.next = n - ring.tail = n - ring.head = n + L.tail = n + L.head = n -proc prepend*[T](ring: var SinglyLinkedRing[T], value: T) = - ## Prepends (adds to the beginning) a value to `ring`. Efficiency: O(1). +proc prepend*[T](L: var SinglyLinkedRing[T], value: T) = + ## Prepends (adds to the beginning) a value to `L`. Efficiency: O(1). ## ## **See also:** ## * `add proc <#add,SinglyLinkedRing[T],SinglyLinkedNode[T]>`_ @@ -848,12 +826,12 @@ proc prepend*[T](ring: var SinglyLinkedRing[T], value: T) = a.prepend(8) assert a.contains(9) - prepend(ring, newSinglyLinkedNode(value)) + prepend(L, newSinglyLinkedNode(value)) -proc add*[T](ring: var DoublyLinkedRing[T], n: DoublyLinkedNode[T]) = - ## Appends (adds to the end) a node `n` to `ring`. Efficiency: O(1). +proc add*[T](L: var DoublyLinkedRing[T], n: DoublyLinkedNode[T]) = + ## Appends (adds to the end) a node `n` to `L`. Efficiency: O(1). ## ## **See also:** ## * `add proc <#add,DoublyLinkedRing[T],T>`_ for appending a value @@ -868,18 +846,18 @@ proc add*[T](ring: var DoublyLinkedRing[T], n: DoublyLinkedNode[T]) = a.add(n) assert a.contains(9) - if ring.head != nil: - n.next = ring.head - n.prev = ring.head.prev - ring.head.prev.next = n - ring.head.prev = n + if L.head != nil: + n.next = L.head + n.prev = L.head.prev + L.head.prev.next = n + L.head.prev = n else: n.prev = n n.next = n - ring.head = n + L.head = n -proc add*[T](ring: var DoublyLinkedRing[T], value: T) = - ## Appends (adds to the end) a value to `ring`. Efficiency: O(1). +proc add*[T](L: var DoublyLinkedRing[T], value: T) = + ## Appends (adds to the end) a value to `L`. Efficiency: O(1). ## ## **See also:** ## * `add proc <#add,DoublyLinkedRing[T],DoublyLinkedNode[T]>`_ @@ -895,10 +873,10 @@ proc add*[T](ring: var DoublyLinkedRing[T], value: T) = a.add(8) assert a.contains(9) - add(ring, newDoublyLinkedNode(value)) + add(L, newDoublyLinkedNode(value)) -proc prepend*[T](ring: var DoublyLinkedRing[T], n: DoublyLinkedNode[T]) = - ## Prepends (adds to the beginning) a node `n` to `ring`. Efficiency: O(1). +proc prepend*[T](L: var DoublyLinkedRing[T], n: DoublyLinkedNode[T]) = + ## Prepends (adds to the beginning) a node `n` to `L`. Efficiency: O(1). ## ## **See also:** ## * `add proc <#add,DoublyLinkedRing[T],DoublyLinkedNode[T]>`_ @@ -913,18 +891,18 @@ proc prepend*[T](ring: var DoublyLinkedRing[T], n: DoublyLinkedNode[T]) = a.prepend(n) assert a.contains(9) - if ring.head != nil: - n.next = ring.head - n.prev = ring.head.prev - ring.head.prev.next = n - ring.head.prev = n + if L.head != nil: + n.next = L.head + n.prev = L.head.prev + L.head.prev.next = n + L.head.prev = n else: n.prev = n n.next = n - ring.head = n + L.head = n -proc prepend*[T](ring: var DoublyLinkedRing[T], value: T) = - ## Prepends (adds to the beginning) a value to `ring`. Efficiency: O(1). +proc prepend*[T](L: var DoublyLinkedRing[T], value: T) = + ## Prepends (adds to the beginning) a value to `L`. Efficiency: O(1). ## ## **See also:** ## * `add proc <#add,DoublyLinkedRing[T],DoublyLinkedNode[T]>`_ @@ -940,11 +918,11 @@ proc prepend*[T](ring: var DoublyLinkedRing[T], value: T) = a.prepend(8) assert a.contains(9) - prepend(ring, newDoublyLinkedNode(value)) + prepend(L, newDoublyLinkedNode(value)) -proc remove*[T](ring: var DoublyLinkedRing[T], n: DoublyLinkedNode[T]) = - ## Removes `n` from `ring`. Efficiency: O(1). - ## This function assumes, for the sake of efficiency, that `n` is contained in `ring`, +proc remove*[T](L: var DoublyLinkedRing[T], n: DoublyLinkedNode[T]) = + ## Removes `n` from `L`. Efficiency: O(1). + ## This function assumes, for the sake of efficiency, that `n` is contained in `L`, ## otherwise the effects are undefined. runnableExamples: var a = initDoublyLinkedRing[int]() @@ -956,13 +934,13 @@ proc remove*[T](ring: var DoublyLinkedRing[T], n: DoublyLinkedNode[T]) = n.next.prev = n.prev n.prev.next = n.next - if n == ring.head: - let p = ring.head.prev - if p == ring.head: + if n == L.head: + let p = L.head.prev + if p == L.head: # only one element left: - ring.head = nil + L.head = nil else: - ring.head = p + L.head = p proc append*[T](a: var (SinglyLinkedList[T] | SinglyLinkedRing[T]), b: SinglyLinkedList[T] | SinglyLinkedNode[T] | T) = @@ -991,3 +969,47 @@ proc appendMoved*[T: SomeLinkedList](a, b: var T) {.since: (1, 5, 1).} = ## * `addMoved proc <#addMoved,SinglyLinkedList[T],SinglyLinkedList[T]>`_ ## * `addMoved proc <#addMoved,DoublyLinkedList[T],DoublyLinkedList[T]>`_ a.addMoved(b) + +func toSinglyLinkedList*[T](elems: openArray[T]): SinglyLinkedList[T] {.since: (1, 5, 1).} = + ## Creates a new `SinglyLinkedList` from the members of `elems`. + runnableExamples: + from std/sequtils import toSeq + let a = [1, 2, 3, 4, 5].toSinglyLinkedList + assert a.toSeq == [1, 2, 3, 4, 5] + + result = initSinglyLinkedList[T]() + for elem in elems.items: + result.add(elem) + +func toSinglyLinkedRing*[T](elems: openArray[T]): SinglyLinkedRing[T] = + ## Creates a new `SinglyLinkedRing` from the members of `elems`. + runnableExamples: + from std/sequtils import toSeq + let a = [1, 2, 3, 4, 5].toSinglyLinkedRing + assert a.toSeq == [1, 2, 3, 4, 5] + + result = initSinglyLinkedRing[T]() + for elem in elems.items: + result.add(elem) + +func toDoublyLinkedList*[T](elems: openArray[T]): DoublyLinkedList[T] {.since: (1, 5, 1).} = + ## Creates a new `DoublyLinkedList` from the members of `elems`. + runnableExamples: + from std/sequtils import toSeq + let a = [1, 2, 3, 4, 5].toDoublyLinkedList + assert a.toSeq == [1, 2, 3, 4, 5] + + result = initDoublyLinkedList[T]() + for elem in elems.items: + result.add(elem) + +func toDoublyLinkedRing*[T](elems: openArray[T]): DoublyLinkedRing[T] = + ## Creates a new `DoublyLinkedRing` from the members of `elems`. + runnableExamples: + from std/sequtils import toSeq + let a = [1, 2, 3, 4, 5].toDoublyLinkedRing + assert a.toSeq == [1, 2, 3, 4, 5] + + result = initDoublyLinkedRing[T]() + for elem in elems.items: + result.add(elem) diff --git a/lib/pure/collections/sequtils.nim b/lib/pure/collections/sequtils.nim index e937b8394..3c0d8dc0e 100644 --- a/lib/pure/collections/sequtils.nim +++ b/lib/pure/collections/sequtils.nim @@ -82,7 +82,17 @@ runnableExamples: import std/private/since -import macros +import std/macros +from std/typetraits import supportsCopyMem + +when defined(nimPreviewSlimSystem): + import std/assertions + + +when defined(nimHasEffectsOf): + {.experimental: "strictEffects".} +else: + {.pragma: effectsOf.} macro evalOnceAs(expAlias, exp: untyped, letAssigneable: static[bool]): untyped = @@ -131,6 +141,22 @@ func concat*[T](seqs: varargs[seq[T]]): seq[T] = result[i] = itm inc(i) +func addUnique*[T](s: var seq[T], x: sink T) = + ## Adds `x` to the container `s` if it is not already present. + ## Uses `==` to check if the item is already present. + runnableExamples: + var a = @[1, 2, 3] + a.addUnique(4) + a.addUnique(4) + assert a == @[1, 2, 3, 4] + + for i in 0..high(s): + if s[i] == x: return + when declared(ensureMove): + s.add ensureMove(x) + else: + s.add x + func count*[T](s: openArray[T], x: T): int = ## Returns the number of occurrences of the item `x` in the container `s`. ## @@ -164,7 +190,7 @@ func cycle*[T](s: openArray[T], n: Natural): seq[T] = result[o] = e inc o -func repeat*[T](x: T, n: Natural): seq[T] = +proc repeat*[T](x: T, n: Natural): seq[T] = ## Returns a new sequence with the item `x` repeated `n` times. ## `n` must be a non-negative number (zero or more). ## @@ -239,9 +265,18 @@ func maxIndex*[T](s: openArray[T]): int {.since: (1, 1).} = for i in 1..high(s): if s[i] > s[result]: result = i +func minmax*[T](x: openArray[T]): (T, T) = + ## The minimum and maximum values of `x`. `T` needs to have a `<` operator. + var l = x[0] + var h = x[0] + for i in 1..high(x): + if x[i] < l: l = x[i] + if h < x[i]: h = x[i] + result = (l, h) + template zipImpl(s1, s2, retType: untyped): untyped = - func zip*[S, T](s1: openArray[S], s2: openArray[T]): retType = + proc zip*[S, T](s1: openArray[S], s2: openArray[T]): retType = ## Returns a new sequence with a combination of the two input containers. ## ## The input containers can be of different types. @@ -284,7 +319,7 @@ when (NimMajor, NimMinor) <= (1, 0): else: zipImpl(s1, s2, seq[(S, T)]) -func unzip*[S, T](s: openArray[(S, T)]): (seq[S], seq[T]) {.since: (1, 1).} = +proc unzip*[S, T](s: openArray[(S, T)]): (seq[S], seq[T]) {.since: (1, 1).} = ## Returns a tuple of two sequences split out from a sequence of 2-field tuples. runnableExamples: let @@ -293,8 +328,7 @@ func unzip*[S, T](s: openArray[(S, T)]): (seq[S], seq[T]) {.since: (1, 1).} = unzipped2 = @['a', 'b', 'c'] assert zipped.unzip() == (unzipped1, unzipped2) assert zip(unzipped1, unzipped2).unzip() == (unzipped1, unzipped2) - result[0] = newSeq[S](s.len) - result[1] = newSeq[T](s.len) + result = (newSeq[S](s.len), newSeq[T](s.len)) for i in 0..<s.len: result[0][i] = s[i][0] result[1][i] = s[i][1] @@ -356,7 +390,7 @@ func distribute*[T](s: seq[T], num: Positive, spread = true): seq[seq[T]] = first = last proc map*[T, S](s: openArray[T], op: proc (x: T): S {.closure.}): - seq[S]{.inline.} = + seq[S] {.inline, effectsOf: op.} = ## Returns a new sequence with the results of the `op` proc applied to every ## item in the container `s`. ## @@ -382,7 +416,7 @@ proc map*[T, S](s: openArray[T], op: proc (x: T): S {.closure.}): result[i] = op(s[i]) proc apply*[T](s: var openArray[T], op: proc (x: var T) {.closure.}) - {.inline.} = + {.inline, effectsOf: op.} = ## Applies `op` to every item in `s`, modifying it directly. ## ## Note that the container `s` must be declared as a `var`, @@ -401,7 +435,7 @@ proc apply*[T](s: var openArray[T], op: proc (x: var T) {.closure.}) for i in 0 ..< s.len: op(s[i]) proc apply*[T](s: var openArray[T], op: proc (x: T): T {.closure.}) - {.inline.} = + {.inline, effectsOf: op.} = ## Applies `op` to every item in `s` modifying it directly. ## ## Note that the container `s` must be declared as a `var` @@ -420,7 +454,7 @@ proc apply*[T](s: var openArray[T], op: proc (x: T): T {.closure.}) for i in 0 ..< s.len: s[i] = op(s[i]) -proc apply*[T](s: openArray[T], op: proc (x: T) {.closure.}) {.inline, since: (1, 3).} = +proc apply*[T](s: openArray[T], op: proc (x: T) {.closure.}) {.inline, since: (1, 3), effectsOf: op.} = ## Same as `apply` but for a proc that does not return anything ## and does not mutate `s` directly. runnableExamples: @@ -429,7 +463,7 @@ proc apply*[T](s: openArray[T], op: proc (x: T) {.closure.}) {.inline, since: (1 assert message == "01234" for i in 0 ..< s.len: op(s[i]) -iterator filter*[T](s: openArray[T], pred: proc(x: T): bool {.closure.}): T = +iterator filter*[T](s: openArray[T], pred: proc(x: T): bool {.closure.}): T {.effectsOf: pred.} = ## Iterates through a container `s` and yields every item that fulfills the ## predicate `pred` (a function that returns a `bool`). ## @@ -453,7 +487,7 @@ iterator filter*[T](s: openArray[T], pred: proc(x: T): bool {.closure.}): T = yield s[i] proc filter*[T](s: openArray[T], pred: proc(x: T): bool {.closure.}): seq[T] - {.inline.} = + {.inline, effectsOf: pred.} = ## Returns a new sequence with all the items of `s` that fulfill the ## predicate `pred` (a function that returns a `bool`). ## @@ -480,7 +514,7 @@ proc filter*[T](s: openArray[T], pred: proc(x: T): bool {.closure.}): seq[T] result.add(s[i]) proc keepIf*[T](s: var seq[T], pred: proc(x: T): bool {.closure.}) - {.inline.} = + {.inline, effectsOf: pred.} = ## Keeps the items in the passed sequence `s` if they fulfill the ## predicate `pred` (a function that returns a `bool`). ## @@ -509,12 +543,51 @@ proc keepIf*[T](s: var seq[T], pred: proc(x: T): bool {.closure.}) inc(pos) setLen(s, pos) -func delete*[T](s: var seq[T]; first, last: Natural) = +func delete*[T](s: var seq[T]; slice: Slice[int]) = + ## Deletes the items `s[slice]`, raising `IndexDefect` if the slice contains + ## elements out of range. + ## + ## This operation moves all elements after `s[slice]` in linear time. + runnableExamples: + var a = @[10, 11, 12, 13, 14] + doAssertRaises(IndexDefect): a.delete(4..5) + assert a == @[10, 11, 12, 13, 14] + a.delete(4..4) + assert a == @[10, 11, 12, 13] + a.delete(1..2) + assert a == @[10, 13] + a.delete(1..<1) # empty slice + assert a == @[10, 13] + when compileOption("boundChecks"): + if not (slice.a < s.len and slice.a >= 0 and slice.b < s.len): + raise newException(IndexDefect, $(slice: slice, len: s.len)) + if slice.b >= slice.a: + template defaultImpl = + var i = slice.a + var j = slice.b + 1 + var newLen = s.len - j + i + while i < newLen: + when defined(gcDestructors): + s[i] = move(s[j]) + else: + s[i].shallowCopy(s[j]) + inc(i) + inc(j) + setLen(s, newLen) + when nimvm: defaultImpl() + else: + when defined(js): + let n = slice.b - slice.a + 1 + let first = slice.a + {.emit: "`s`.splice(`first`, `n`);".} + else: + defaultImpl() + +func delete*[T](s: var seq[T]; first, last: Natural) {.deprecated: "use `delete(s, first..last)`".} = ## Deletes the items of a sequence `s` at positions `first..last` ## (including both ends of the range). ## This modifies `s` itself, it does not return a copy. - ## - runnableExamples: + runnableExamples("--warning:deprecated:off"): let outcome = @[1, 1, 1, 1, 1, 1, 1, 1] var dest = @[1, 1, 1, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1] dest.delete(3, 8) @@ -646,7 +719,7 @@ since (1, 1): if pred: result += 1 result -proc all*[T](s: openArray[T], pred: proc(x: T): bool {.closure.}): bool = +proc all*[T](s: openArray[T], pred: proc(x: T): bool {.closure.}): bool {.effectsOf: pred.} = ## Iterates through a container and checks if every item fulfills the ## predicate. ## @@ -688,7 +761,7 @@ template allIt*(s, pred: untyped): bool = break result -proc any*[T](s: openArray[T], pred: proc(x: T): bool {.closure.}): bool = +proc any*[T](s: openArray[T], pred: proc(x: T): bool {.closure.}): bool {.effectsOf: pred.} = ## Iterates through a container and checks if at least one item ## fulfills the predicate. ## @@ -743,7 +816,7 @@ template toSeq1(s: not iterator): untyped = i += 1 result else: - var result: seq[OutType] = @[] + var result: seq[OutType]# = @[] for it in s: result.add(it) result @@ -760,7 +833,7 @@ template toSeq2(iter: iterator): untyped = result else: type OutType = typeof(iter2()) - var result: seq[OutType] = @[] + var result: seq[OutType]# = @[] when compiles(iter2()): evalOnceAs(iter4, iter, false) let iter3 = iter4() @@ -866,7 +939,7 @@ template foldl*(sequence, operation, first): untyped = ## ## The `operation` parameter should be an expression which uses the variables ## `a` and `b` for each step of the fold. The `first` parameter is the - ## start value (the first `a`) and therefor defines the type of the result. + ## start value (the first `a`) and therefore defines the type of the result. ## ## **See also:** ## * `foldr template<#foldr.t,untyped,untyped>`_ @@ -972,7 +1045,7 @@ template mapIt*(s: typed, op: untyped): untyped = i += 1 result else: - var result: seq[OutType] = @[] + var result: seq[OutType]# = @[] # use `items` to avoid https://github.com/nim-lang/Nim/issues/12639 for it {.inject.} in items(s): result.add(op) @@ -1014,27 +1087,31 @@ template applyIt*(varSeq, op: untyped) = template newSeqWith*(len: int, init: untyped): untyped = - ## Creates a new sequence of length `len`, calling `init` to initialize - ## each value of the sequence. - ## - ## Useful for creating "2D" sequences - sequences containing other sequences - ## or to populate fields of the created sequence. + ## Creates a new `seq` of length `len`, calling `init` to initialize + ## each value of the seq. ## + ## Useful for creating "2D" seqs - seqs containing other seqs + ## or to populate fields of the created seq. runnableExamples: - ## Creates a sequence containing 5 bool sequences, each of length of 3. + ## Creates a seq containing 5 bool seqs, each of length of 3. var seq2D = newSeqWith(5, newSeq[bool](3)) assert seq2D.len == 5 assert seq2D[0].len == 3 assert seq2D[4][2] == false - ## Creates a sequence of 20 random numbers from 1 to 10 - import random - var seqRand = newSeqWith(20, rand(10)) - - var result = newSeq[typeof(init)](len) - for i in 0 ..< len: + ## Creates a seq with random numbers + import std/random + var seqRand = newSeqWith(20, rand(1.0)) + assert seqRand[0] != seqRand[1] + type T = typeof(init) + let newLen = len + when supportsCopyMem(T) and declared(newSeqUninit): + var result = newSeqUninit[T](newLen) + else: # TODO: use `newSeqUnsafe` when that's available + var result = newSeq[T](newLen) + for i in 0 ..< newLen: result[i] = init - result + move(result) # refs bug #7295 func mapLitsImpl(constructor: NimNode; op: NimNode; nested: bool; filter = nnkLiterals): NimNode = diff --git a/lib/pure/collections/setimpl.nim b/lib/pure/collections/setimpl.nim index 7ebd22760..360a075d6 100644 --- a/lib/pure/collections/setimpl.nim +++ b/lib/pure/collections/setimpl.nim @@ -62,6 +62,7 @@ template containsOrInclImpl() {.dirty.} = if index >= 0: result = true else: + result = false if mustRehash(s): enlarge(s) index = rawGetKnownHC(s, key, hc) @@ -86,7 +87,9 @@ proc exclImpl[A](s: var HashSet[A], key: A): bool {.inline.} = var j = i # The correctness of this depends on (h+1) in nextTry, var r = j # though may be adaptable to other simple sequences. s.data[i].hcode = 0 # mark current EMPTY - s.data[i].key = default(typeof(s.data[i].key)) + {.push warning[UnsafeDefault]:off.} + reset(s.data[i].key) + {.pop.} doWhile((i >= r and r > j) or (r > j and j > i) or (j > i and i >= r)): i = (i + 1) and msk # increment mod table size if isEmpty(s.data[i].hcode): # end of collision cluster; So all done diff --git a/lib/pure/collections/sets.nim b/lib/pure/collections/sets.nim index 3f91d9030..af13135aa 100644 --- a/lib/pure/collections/sets.nim +++ b/lib/pure/collections/sets.nim @@ -25,7 +25,9 @@ ## `difference <#difference,HashSet[A],HashSet[A]>`_, and ## `symmetric difference <#symmetricDifference,HashSet[A],HashSet[A]>`_ ## -## .. code-block:: +## **Examples:** +## +## ```Nim ## echo toHashSet([9, 5, 1]) # {9, 1, 5} ## echo toOrderedSet([9, 5, 1]) # {9, 5, 1} ## @@ -37,7 +39,7 @@ ## echo s1 - s2 # {1, 9} ## echo s1 * s2 # {5} ## echo s1 -+- s2 # {9, 1, 3, 7} -## +## ``` ## ## Note: The data types declared here have *value semantics*: This means ## that `=` performs a copy of the set. @@ -48,7 +50,10 @@ import - hashes, math + std/[hashes, math] + +when not defined(nimHasEffectsOf): + {.pragma: effectsOf.} {.pragma: myShallow.} # For "integer-like A" that are too big for intsets/bit-vectors to be practical, @@ -61,7 +66,7 @@ type HashSet*[A] {.myShallow.} = object ## \ ## A generic hash set. ## - ## Use `init proc <#init,HashSet[A]>`_ or `initHashSet proc <#initHashSet,int>`_ + ## Use `init proc <#init,HashSet[A]>`_ or `initHashSet proc <#initHashSet>`_ ## before calling other procs on it. data: KeyValuePairSeq[A] counter: int @@ -125,7 +130,7 @@ proc initHashSet*[A](initialSize = defaultInitialSize): HashSet[A] = var a = initHashSet[int]() a.incl(3) assert len(a) == 1 - + result = default(HashSet[A]) result.init(initialSize) proc `[]`*[A](s: var HashSet[A], key: A): var A = @@ -246,7 +251,7 @@ iterator items*[A](s: HashSet[A]): A = ## If you need a sequence with the elements you can use `sequtils.toSeq ## template <sequtils.html#toSeq.t,untyped>`_. ## - ## .. code-block:: + ## ```Nim ## type ## pair = tuple[a, b: int] ## var @@ -259,6 +264,7 @@ iterator items*[A](s: HashSet[A]): A = ## assert a.len == 2 ## echo b ## # --> {(a: 1, b: 3), (a: 0, b: 4)} + ## ``` let length = s.len for h in 0 .. high(s.data): if isFilled(s.data[h].hcode): @@ -344,7 +350,7 @@ proc missingOrExcl*[A](s: var HashSet[A], key: A): bool = proc pop*[A](s: var HashSet[A]): A = ## Removes and returns an arbitrary element from the set `s`. ## - ## Raises KeyError if the set `s` is empty. + ## Raises `KeyError` if the set `s` is empty. ## ## See also: ## * `clear proc <#clear,HashSet[A]>`_ @@ -376,7 +382,9 @@ proc clear*[A](s: var HashSet[A]) = s.counter = 0 for i in 0 ..< s.data.len: s.data[i].hcode = 0 - s.data[i].key = default(typeof(s.data[i].key)) + {.push warning[UnsafeDefault]:off.} + reset(s.data[i].key) + {.pop.} proc union*[A](s1, s2: HashSet[A]): HashSet[A] = @@ -422,7 +430,7 @@ proc intersection*[A](s1, s2: HashSet[A]): HashSet[A] = assert c == toHashSet(["b"]) result = initHashSet[A](max(min(s1.data.len, s2.data.len), 2)) - + # iterate over the elements of the smaller set if s1.data.len < s2.data.len: for item in s1: @@ -430,7 +438,7 @@ proc intersection*[A](s1, s2: HashSet[A]): HashSet[A] = else: for item in s2: if item in s1: incl(result, item) - + proc difference*[A](s1, s2: HashSet[A]): HashSet[A] = ## Returns the difference of the sets `s1` and `s2`. @@ -556,7 +564,7 @@ proc `==`*[A](s, t: HashSet[A]): bool = s.counter == t.counter and s <= t -proc map*[A, B](data: HashSet[A], op: proc (x: A): B {.closure.}): HashSet[B] = +proc map*[A, B](data: HashSet[A], op: proc (x: A): B {.closure.}): HashSet[B] {.effectsOf: op.} = ## Returns a new set after applying `op` proc on each of the elements of ##`data` set. ## @@ -583,12 +591,12 @@ proc `$`*[A](s: HashSet[A]): string = ## any moment and values are not escaped. ## ## **Examples:** - ## - ## .. code-block:: + ## ```Nim ## echo toHashSet([2, 4, 5]) ## # --> {2, 4, 5} ## echo toHashSet(["no", "esc'aping", "is \" provided"]) ## # --> {no, esc'aping, is " provided} + ## ``` dollarImpl() @@ -603,12 +611,10 @@ proc isValid*[A](s: HashSet[A]): bool {.deprecated: ## Returns `true` if the set has been initialized (with `initHashSet proc ## <#initHashSet>`_ or `init proc <#init,HashSet[A]>`_). ## - ## **Examples:** - ## - ## .. code-block :: - ## proc savePreferences(options: HashSet[string]) = - ## assert options.isValid, "Pass an initialized set!" - ## # Do stuff here, may crash in release builds! + runnableExamples: + proc savePreferences(options: HashSet[string]) = + assert options.isValid, "Pass an initialized set!" + # Do stuff here, may crash in release builds! result = s.data.len > 0 @@ -812,7 +818,9 @@ proc clear*[A](s: var OrderedSet[A]) = for i in 0 ..< s.data.len: s.data[i].hcode = 0 s.data[i].next = 0 - s.data[i].key = default(typeof(s.data[i].key)) + {.push warning[UnsafeDefault]:off.} + reset(s.data[i].key) + {.pop.} proc len*[A](s: OrderedSet[A]): int {.inline.} = ## Returns the number of elements in `s`. @@ -873,12 +881,12 @@ proc `$`*[A](s: OrderedSet[A]): string = ## any moment and values are not escaped. ## ## **Examples:** - ## - ## .. code-block:: + ## ```Nim ## echo toOrderedSet([2, 4, 5]) ## # --> {2, 4, 5} ## echo toOrderedSet(["no", "esc'aping", "is \" provided"]) ## # --> {no, esc'aping, is " provided} + ## ``` dollarImpl() @@ -889,7 +897,7 @@ iterator items*[A](s: OrderedSet[A]): A = ## If you need a sequence with the elements you can use `sequtils.toSeq ## template <sequtils.html#toSeq.t,untyped>`_. ## - ## .. code-block:: + ## ```Nim ## var a = initOrderedSet[int]() ## for value in [9, 2, 1, 5, 1, 8, 4, 2]: ## a.incl(value) @@ -901,6 +909,7 @@ iterator items*[A](s: OrderedSet[A]): A = ## # --> Got 5 ## # --> Got 8 ## # --> Got 4 + ## ``` let length = s.len forAllOrderedPairs: yield s.data[h].key diff --git a/lib/pure/collections/sharedlist.nim b/lib/pure/collections/sharedlist.nim index c72477675..ec8f1cd86 100644 --- a/lib/pure/collections/sharedlist.nim +++ b/lib/pure/collections/sharedlist.nim @@ -11,10 +11,12 @@ ## ## Unstable API. +{.deprecated.} + {.push stackTrace: off.} import - locks + std/locks const ElemsPerNode = 100 @@ -35,8 +37,10 @@ template withLock(t, x: untyped) = release(t.lock) proc iterAndMutate*[A](x: var SharedList[A]; action: proc(x: A): bool) = - ## iterates over the list. If 'action' returns true, the + ## Iterates over the list. If `action` returns true, the ## current item is removed from the list. + ## + ## .. warning:: It may not preserve the element order after some modifications. withLock(x): var n = x.head while n != nil: @@ -47,8 +51,9 @@ proc iterAndMutate*[A](x: var SharedList[A]; action: proc(x: A): bool) = if action(n.d[i]): acquire(x.lock) let t = x.tail + dec t.dataLen # TODO considering t.dataLen == 0, + # probably the module should be refactored using doubly linked lists n.d[i] = t.d[t.dataLen] - dec t.dataLen else: acquire(x.lock) inc i diff --git a/lib/pure/collections/sharedtables.nim b/lib/pure/collections/sharedtables.nim index 4ac3befb6..b474ecd31 100644 --- a/lib/pure/collections/sharedtables.nim +++ b/lib/pure/collections/sharedtables.nim @@ -14,8 +14,10 @@ ## ## Unstable API. +{.deprecated.} + import - hashes, math, locks + std/[hashes, math, locks] type KeyValuePair[A, B] = tuple[hcode: Hash, key: A, val: B] @@ -60,17 +62,27 @@ template withLock(t, x: untyped) = template withValue*[A, B](t: var SharedTable[A, B], key: A, value, body: untyped) = - ## retrieves the value at `t[key]`. + ## Retrieves the value at `t[key]`. ## `value` can be modified in the scope of the `withValue` call. - ## - ## .. code-block:: nim - ## - ## sharedTable.withValue(key, value) do: - ## # block is executed only if `key` in `t` - ## # value is threadsafe in block - ## value.name = "username" - ## value.uid = 1000 - ## + runnableExamples: + var table: SharedTable[string, string] + init(table) + + table["a"] = "x" + table["b"] = "y" + table["c"] = "z" + + table.withValue("a", value): + assert value[] == "x" + + table.withValue("b", value): + value[] = "modified" + + table.withValue("b", value): + assert value[] == "modified" + + table.withValue("nonexistent", value): + assert false # not called acquire(t.lock) try: var hc: Hash @@ -84,20 +96,33 @@ template withValue*[A, B](t: var SharedTable[A, B], key: A, template withValue*[A, B](t: var SharedTable[A, B], key: A, value, body1, body2: untyped) = - ## retrieves the value at `t[key]`. + ## Retrieves the value at `t[key]`. ## `value` can be modified in the scope of the `withValue` call. - ## - ## .. code-block:: nim - ## - ## sharedTable.withValue(key, value) do: - ## # block is executed only if `key` in `t` - ## # value is threadsafe in block - ## value.name = "username" - ## value.uid = 1000 - ## do: - ## # block is executed when `key` not in `t` - ## raise newException(KeyError, "Key not found") - ## + runnableExamples: + var table: SharedTable[string, string] + init(table) + + table["a"] = "x" + table["b"] = "y" + table["c"] = "z" + + + table.withValue("a", value): + value[] = "m" + + var flag = false + table.withValue("d", value): + discard value + doAssert false + do: # if "d" notin table + flag = true + + if flag: + table["d"] = "n" + + assert table.mget("a") == "m" + assert table.mget("d") == "n" + acquire(t.lock) try: var hc: Hash @@ -112,7 +137,7 @@ template withValue*[A, B](t: var SharedTable[A, B], key: A, release(t.lock) proc mget*[A, B](t: var SharedTable[A, B], key: A): var B = - ## retrieves the value at `t[key]`. The value can be modified. + ## Retrieves the value at `t[key]`. The value can be modified. ## If `key` is not in `t`, the `KeyError` exception is raised. withLock t: var hc: Hash @@ -126,7 +151,7 @@ proc mget*[A, B](t: var SharedTable[A, B], key: A): var B = raise newException(KeyError, "key not found") proc mgetOrPut*[A, B](t: var SharedTable[A, B], key: A, val: B): var B = - ## retrieves value at `t[key]` or puts `val` if not present, either way + ## Retrieves value at `t[key]` or puts `val` if not present, either way ## returning a value which can be modified. **Note**: This is inherently ## unsafe in the context of multi-threading since it returns a pointer ## to `B`. @@ -134,7 +159,7 @@ proc mgetOrPut*[A, B](t: var SharedTable[A, B], key: A, val: B): var B = mgetOrPutImpl(enlarge) proc hasKeyOrPut*[A, B](t: var SharedTable[A, B], key: A, val: B): bool = - ## returns true if `key` is in the table, otherwise inserts `value`. + ## Returns true if `key` is in the table, otherwise inserts `value`. withLock t: hasKeyOrPutImpl(enlarge) @@ -166,8 +191,7 @@ proc withKey*[A, B](t: var SharedTable[A, B], key: A, ## ## Example usage: ## - ## .. code-block:: nim - ## + ## ```nim ## # If value exists, decrement it. ## # If it becomes zero or less, delete the key ## t.withKey(1'i64) do (k: int64, v: var int, pairExists: var bool): @@ -175,6 +199,7 @@ proc withKey*[A, B](t: var SharedTable[A, B], key: A, ## dec v ## if v <= 0: ## pairExists = false + ## ``` withLock t: var hc: Hash var index = rawGet(t, key, hc) @@ -191,28 +216,28 @@ proc withKey*[A, B](t: var SharedTable[A, B], key: A, st_maybeRehashPutImpl(enlarge) proc `[]=`*[A, B](t: var SharedTable[A, B], key: A, val: B) = - ## puts a (key, value)-pair into `t`. + ## Puts a (key, value)-pair into `t`. withLock t: putImpl(enlarge) proc add*[A, B](t: var SharedTable[A, B], key: A, val: B) = - ## puts a new (key, value)-pair into `t` even if `t[key]` already exists. + ## Puts a new (key, value)-pair into `t` even if `t[key]` already exists. ## This can introduce duplicate keys into the table! withLock t: addImpl(enlarge) proc del*[A, B](t: var SharedTable[A, B], key: A) = - ## deletes `key` from hash table `t`. + ## Deletes `key` from hash table `t`. withLock t: delImpl(tabMakeEmpty, tabCellEmpty, tabCellHash) proc len*[A, B](t: var SharedTable[A, B]): int = - ## number of elements in `t` + ## Number of elements in `t`. withLock t: result = t.counter proc init*[A, B](t: var SharedTable[A, B], initialSize = 32) = - ## creates a new hash table that is empty. + ## Creates a new hash table that is empty. ## ## This proc must be called before any other usage of `t`. let initialSize = slotsNeeded(initialSize) diff --git a/lib/pure/collections/tableimpl.nim b/lib/pure/collections/tableimpl.nim index 27c339177..3542741fa 100644 --- a/lib/pure/collections/tableimpl.nim +++ b/lib/pure/collections/tableimpl.nim @@ -11,6 +11,9 @@ include hashcommon +const + defaultInitialSize* = 32 + template rawGetDeepImpl() {.dirty.} = # Search algo for unconditional add genHashImpl(key, hc) var h: Hash = hc and maxHash(t) @@ -23,7 +26,7 @@ template rawInsertImpl() {.dirty.} = data[h].val = val data[h].hcode = hc -proc rawGetDeep[X, A](t: X, key: A, hc: var Hash): int {.inline.} = +proc rawGetDeep[X, A](t: X, key: A, hc: var Hash): int {.inline, outParamsAt: [3].} = rawGetDeepImpl() proc rawInsert[X, A, B](t: var X, data: var KeyValuePairSeq[A, B], @@ -31,9 +34,8 @@ proc rawInsert[X, A, B](t: var X, data: var KeyValuePairSeq[A, B], rawInsertImpl() template checkIfInitialized() = - when compiles(defaultInitialSize): - if t.dataLen == 0: - initImpl(t, defaultInitialSize) + if t.dataLen == 0: + initImpl(t, defaultInitialSize) template addImpl(enlarge) {.dirty.} = checkIfInitialized() @@ -43,7 +45,7 @@ template addImpl(enlarge) {.dirty.} = rawInsert(t, t.data, key, val, hc, j) inc(t.counter) -template maybeRehashPutImpl(enlarge) {.dirty.} = +template maybeRehashPutImpl(enlarge, val) {.dirty.} = checkIfInitialized() if mustRehash(t): enlarge(t) @@ -54,28 +56,41 @@ template maybeRehashPutImpl(enlarge) {.dirty.} = template putImpl(enlarge) {.dirty.} = checkIfInitialized() - var hc: Hash + var hc: Hash = default(Hash) var index = rawGet(t, key, hc) if index >= 0: t.data[index].val = val - else: maybeRehashPutImpl(enlarge) + else: maybeRehashPutImpl(enlarge, val) template mgetOrPutImpl(enlarge) {.dirty.} = checkIfInitialized() - var hc: Hash + var hc: Hash = default(Hash) var index = rawGet(t, key, hc) if index < 0: # not present: insert (flipping index) - maybeRehashPutImpl(enlarge) + when declared(val): + maybeRehashPutImpl(enlarge, val) + else: + maybeRehashPutImpl(enlarge, default(B)) # either way return modifiable val result = t.data[index].val +# template mgetOrPutDefaultImpl(enlarge) {.dirty.} = +# checkIfInitialized() +# var hc: Hash = default(Hash) +# var index = rawGet(t, key, hc) +# if index < 0: +# # not present: insert (flipping index) +# maybeRehashPutImpl(enlarge, default(B)) +# # either way return modifiable val +# result = t.data[index].val + template hasKeyOrPutImpl(enlarge) {.dirty.} = checkIfInitialized() - var hc: Hash + var hc: Hash = default(Hash) var index = rawGet(t, key, hc) if index < 0: result = false - maybeRehashPutImpl(enlarge) + maybeRehashPutImpl(enlarge, val) else: result = true # delImplIdx is KnuthV3 Algo6.4R adapted to i=i+1 (from i=i-1) which has come to @@ -117,8 +132,10 @@ template delImplIdx(t, i, makeEmpty, cellEmpty, cellHash) = var j = i # The correctness of this depends on (h+1) in nextTry var r = j # though may be adaptable to other simple sequences. makeEmpty(i) # mark current EMPTY - t.data[i].key = default(typeof(t.data[i].key)) - t.data[i].val = default(typeof(t.data[i].val)) + {.push warning[UnsafeDefault]:off.} + reset(t.data[i].key) + reset(t.data[i].val) + {.pop.} while true: i = (i + 1) and msk # increment mod table size if cellEmpty(i): # end of collision cluster; So all done @@ -149,8 +166,10 @@ template clearImpl() {.dirty.} = for i in 0 ..< t.dataLen: when compiles(t.data[i].hcode): # CountTable records don't contain a hcode t.data[i].hcode = 0 - t.data[i].key = default(typeof(t.data[i].key)) - t.data[i].val = default(typeof(t.data[i].val)) + {.push warning[UnsafeDefault]:off.} + reset(t.data[i].key) + reset(t.data[i].val) + {.pop.} t.counter = 0 template ctAnd(a, b): bool = @@ -208,3 +227,5 @@ template equalsImpl(s, t: typed) = if not t.hasKey(key): return false if t.getOrDefault(key) != val: return false return true + else: + return false diff --git a/lib/pure/collections/tables.nim b/lib/pure/collections/tables.nim index f77642349..d414caeed 100644 --- a/lib/pure/collections/tables.nim +++ b/lib/pure/collections/tables.nim @@ -136,12 +136,11 @@ runnableExamples: ## a more complex object as a key you will be greeted by a strange compiler ## error: ## -## Error: type mismatch: got (Person) -## but expected one of: -## hashes.hash(x: openArray[A]): Hash -## hashes.hash(x: int): Hash -## hashes.hash(x: float): Hash -## … +## Error: type mismatch: got (Person) +## but expected one of: +## hashes.hash(x: openArray[A]): Hash +## hashes.hash(x: int): Hash +## hashes.hash(x: float): Hash ## ## What is happening here is that the types used for table keys require to have ## a `hash()` proc which will convert them to a `Hash <hashes.html#Hash>`_ @@ -189,7 +188,6 @@ runnableExamples: ## ## * `json module<json.html>`_ for table-like structure which allows ## heterogeneous members -## * `sharedtables module<sharedtables.html>`_ for shared hash table support ## * `strtabs module<strtabs.html>`_ for efficient hash tables ## mapping from strings to strings ## * `hashes module<hashes.html>`_ for helper functions for hashing @@ -198,6 +196,10 @@ runnableExamples: import std/private/since import std/[hashes, math, algorithm] + +when not defined(nimHasEffectsOf): + {.pragma: effectsOf.} + type KeyValuePair[A, B] = tuple[hcode: Hash, key: A, val: B] KeyValuePairSeq[A, B] = seq[KeyValuePair[A, B]] @@ -215,8 +217,6 @@ type ## For creating a new empty TableRef, use `newTable proc ## <#newTable>`_. -const - defaultInitialSize* = 32 # ------------------------------ helpers --------------------------------- @@ -227,6 +227,12 @@ template dataLen(t): untyped = len(t.data) include tableimpl +proc raiseKeyError[T](key: T) {.noinline, noreturn.} = + when compiles($key): + raise newException(KeyError, "key not found: " & $key) + else: + raise newException(KeyError, "key not found") + template get(t, key): untyped = ## retrieves the value at `t[key]`. The value can be modified. ## If `key` is not in `t`, the `KeyError` exception is raised. @@ -235,10 +241,7 @@ template get(t, key): untyped = var index = rawGet(t, key, hc) if index >= 0: result = t.data[index].val else: - when compiles($key): - raise newException(KeyError, "key not found: " & $key) - else: - raise newException(KeyError, "key not found") + raiseKeyError(key) proc enlarge[A, B](t: var Table[A, B]) = var n: KeyValuePairSeq[A, B] @@ -275,6 +278,7 @@ proc initTable*[A, B](initialSize = defaultInitialSize): Table[A, B] = let a = initTable[int, string]() b = initTable[char, seq[int]]() + result = default(Table[A, B]) initImpl(result, initialSize) proc `[]=`*[A, B](t: var Table[A, B], key: A, val: sink B) = @@ -309,7 +313,7 @@ proc toTable*[A, B](pairs: openArray[(A, B)]): Table[A, B] = result = initTable[A, B](pairs.len) for key, val in items(pairs): result[key] = val -proc `[]`*[A, B](t: Table[A, B], key: A): B = +proc `[]`*[A, B](t: Table[A, B], key: A): lent B = ## Retrieves the value at `t[key]`. ## ## If `key` is not in `t`, the `KeyError` exception is raised. @@ -321,7 +325,7 @@ proc `[]`*[A, B](t: Table[A, B], key: A): B = ## a default value (e.g. zero for int) if the key doesn't exist ## * `getOrDefault proc<#getOrDefault,Table[A,B],A,B>`_ to return ## a custom value if the key doesn't exist - ## * `[]= proc<#[]=,Table[A,B],A,B>`_ for inserting a new + ## * `[]= proc<#[]=,Table[A,B],A,sinkB>`_ for inserting a new ## (key, value) pair in the table ## * `hasKey proc<#hasKey,Table[A,B],A>`_ for checking if a key is in ## the table @@ -342,7 +346,7 @@ proc `[]`*[A, B](t: var Table[A, B], key: A): var B = ## a default value (e.g. zero for int) if the key doesn't exist ## * `getOrDefault proc<#getOrDefault,Table[A,B],A,B>`_ to return ## a custom value if the key doesn't exist - ## * `[]= proc<#[]=,Table[A,B],A,B>`_ for inserting a new + ## * `[]= proc<#[]=,Table[A,B],A,sinkB>`_ for inserting a new ## (key, value) pair in the table ## * `hasKey proc<#hasKey,Table[A,B],A>`_ for checking if a key is in ## the table @@ -412,7 +416,7 @@ proc getOrDefault*[A, B](t: Table[A, B], key: A): B = let a = {'a': 5, 'b': 9}.toTable doAssert a.getOrDefault('a') == 5 doAssert a.getOrDefault('z') == 0 - + result = default(B) getOrDefaultImpl(t, key) proc getOrDefault*[A, B](t: Table[A, B], key: A, default: B): B = @@ -430,7 +434,7 @@ proc getOrDefault*[A, B](t: Table[A, B], key: A, default: B): B = let a = {'a': 5, 'b': 9}.toTable doAssert a.getOrDefault('a', 99) == 5 doAssert a.getOrDefault('z', 99) == 99 - + result = default(B) getOrDefaultImpl(t, key, default) proc mgetOrPut*[A, B](t: var Table[A, B], key: A, val: B): var B = @@ -439,7 +443,7 @@ proc mgetOrPut*[A, B](t: var Table[A, B], key: A, val: B): var B = ## ## ## Note that while the value returned is of type `var B`, - ## it is easy to accidentally create an copy of the value at `t[key]`. + ## it is easy to accidentally create a copy of the value at `t[key]`. ## Remember that seqs and strings are value types, and therefore ## cannot be copied into a separate variable for modification. ## See the example below. @@ -471,6 +475,18 @@ proc mgetOrPut*[A, B](t: var Table[A, B], key: A, val: B): var B = mgetOrPutImpl(enlarge) +proc mgetOrPut*[A, B](t: var Table[A, B], key: A): var B = + ## Retrieves the value at `t[key]` or puts the + ## default initialization value for type `B` (e.g. 0 for any + ## integer type). + runnableExamples: + var a = {'a': 5}.newTable + doAssert a.mgetOrPut('a') == 5 + a.mgetOrPut('z').inc + doAssert a == {'a': 5, 'z': 1}.newTable + + mgetOrPutImpl(enlarge) + proc len*[A, B](t: Table[A, B]): int = ## Returns the number of keys in `t`. runnableExamples: @@ -485,7 +501,7 @@ proc add*[A, B](t: var Table[A, B], key: A, val: sink B) {.deprecated: ## ## **This can introduce duplicate keys into the table!** ## - ## Use `[]= proc<#[]=,Table[A,B],A,B>`_ for inserting a new + ## Use `[]= proc<#[]=,Table[A,B],A,sinkB>`_ for inserting a new ## (key, value) pair in the table without introducing duplicates. addImpl(enlarge) @@ -496,6 +512,9 @@ template tabCellHash(i) = t.data[i].hcode proc del*[A, B](t: var Table[A, B], key: A) = ## Deletes `key` from hash table `t`. Does nothing if the key does not exist. ## + ## .. warning:: If duplicate keys were added (via the now deprecated `add` proc), + ## this may need to be called multiple times. + ## ## See also: ## * `pop proc<#pop,Table[A,B],A,B>`_ ## * `clear proc<#clear,Table[A,B]>`_ to empty the whole table @@ -514,6 +533,9 @@ proc pop*[A, B](t: var Table[A, B], key: A, val: var B): bool = ## mapping of the key. Otherwise, returns `false`, and the `val` is ## unchanged. ## + ## .. warning:: If duplicate keys were added (via the now deprecated `add` proc), + ## this may need to be called multiple times. + ## ## See also: ## * `del proc<#del,Table[A,B],A>`_ ## * `clear proc<#clear,Table[A,B]>`_ to empty the whole table @@ -594,17 +616,17 @@ template withValue*[A, B](t: var Table[A, B], key: A, value, body: untyped) = let u = User(name: "Hello", uid: 99) t[1] = u - t.withValue(1, value) do: + t.withValue(1, value): # block is executed only if `key` in `t` value.name = "Nim" value.uid = 1314 - t.withValue(2, value) do: + t.withValue(2, value): value.name = "No" value.uid = 521 - doAssert t[1].name == "Nim" - doAssert t[1].uid == 1314 + assert t[1].name == "Nim" + assert t[1].uid == 1314 mixin rawGet var hc: Hash @@ -629,16 +651,22 @@ template withValue*[A, B](t: var Table[A, B], key: A, let u = User(name: "Hello", uid: 99) t[1] = u - t.withValue(1, value) do: + t.withValue(1, value): # block is executed only if `key` in `t` value.name = "Nim" value.uid = 1314 - # do: - # # block is executed when `key` not in `t` - # raise newException(KeyError, "Key not found") - doAssert t[1].name == "Nim" - doAssert t[1].uid == 1314 + t.withValue(521, value): + doAssert false + do: + # block is executed when `key` not in `t` + t[1314] = User(name: "exist", uid: 521) + + assert t[1].name == "Nim" + assert t[1].uid == 1314 + assert t[1314].name == "exist" + assert t[1314].uid == 521 + mixin rawGet var hc: Hash var index = rawGet(t, key, hc) @@ -660,7 +688,7 @@ iterator pairs*[A, B](t: Table[A, B]): (A, B) = ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## let a = { ## 'o': [1, 5, 7, 9], ## 'e': [2, 4, 6, 8] @@ -674,6 +702,7 @@ iterator pairs*[A, B](t: Table[A, B]): (A, B) = ## # value: [2, 4, 6, 8] ## # key: o ## # value: [1, 5, 7, 9] + ## ``` let L = len(t) for h in 0 .. high(t.data): if isFilled(t.data[h].hcode): @@ -702,7 +731,7 @@ iterator mpairs*[A, B](t: var Table[A, B]): (A, var B) = yield (t.data[h].key, t.data[h].val) assert(len(t) == L, "the length of the table changed while iterating over it") -iterator keys*[A, B](t: Table[A, B]): A = +iterator keys*[A, B](t: Table[A, B]): lent A = ## Iterates over any key in the table `t`. ## ## See also: @@ -723,7 +752,7 @@ iterator keys*[A, B](t: Table[A, B]): A = yield t.data[h].key assert(len(t) == L, "the length of the table changed while iterating over it") -iterator values*[A, B](t: Table[A, B]): B = +iterator values*[A, B](t: Table[A, B]): lent B = ## Iterates over any value in the table `t`. ## ## See also: @@ -795,7 +824,7 @@ iterator allValues*[A, B](t: Table[A, B]; key: A): B {.deprecated: # ------------------------------------------------------------------- -proc newTable*[A, B](initialSize = defaultInitialSize): <//>TableRef[A, B] = +proc newTable*[A, B](initialSize = defaultInitialSize): TableRef[A, B] = ## Creates a new ref hash table that is empty. ## ## See also: @@ -808,9 +837,10 @@ proc newTable*[A, B](initialSize = defaultInitialSize): <//>TableRef[A, B] = b = newTable[char, seq[int]]() new(result) - result[] = initTable[A, B](initialSize) + {.noSideEffect.}: + result[] = initTable[A, B](initialSize) -proc newTable*[A, B](pairs: openArray[(A, B)]): <//>TableRef[A, B] = +proc newTable*[A, B](pairs: openArray[(A, B)]): TableRef[A, B] = ## Creates a new ref hash table that contains the given `pairs`. ## ## `pairs` is a container consisting of `(key, value)` tuples. @@ -824,14 +854,16 @@ proc newTable*[A, B](pairs: openArray[(A, B)]): <//>TableRef[A, B] = assert b == {'a': 5, 'b': 9}.newTable new(result) - result[] = toTable[A, B](pairs) + {.noSideEffect.}: + result[] = toTable[A, B](pairs) -proc newTableFrom*[A, B, C](collection: A, index: proc(x: B): C): <//>TableRef[C, B] = +proc newTableFrom*[A, B, C](collection: A, index: proc(x: B): C): TableRef[C, B] = ## Index the collection with the proc provided. # TODO: As soon as supported, change collection: A to collection: A[B] result = newTable[C, B]() - for item in collection: - result[index(item)] = item + {.noSideEffect.}: + for item in collection: + result[index(item)] = item proc `[]`*[A, B](t: TableRef[A, B], key: A): var B = ## Retrieves the value at `t[key]`. @@ -901,7 +933,7 @@ proc contains*[A, B](t: TableRef[A, B], key: A): bool = return hasKey[A, B](t, key) -proc hasKeyOrPut*[A, B](t: var TableRef[A, B], key: A, val: B): bool = +proc hasKeyOrPut*[A, B](t: TableRef[A, B], key: A, val: B): bool = ## Returns true if `key` is in the table, otherwise inserts `value`. ## ## See also: @@ -994,6 +1026,18 @@ proc mgetOrPut*[A, B](t: TableRef[A, B], key: A, val: B): var B = doAssert t[25] == @[25, 35] t[].mgetOrPut(key, val) +proc mgetOrPut*[A, B](t: TableRef[A, B], key: A): var B = + ## Retrieves the value at `t[key]` or puts the + ## default initialization value for type `B` (e.g. 0 for any + ## integer type). + runnableExamples: + var a = {'a': 5}.newTable + doAssert a.mgetOrPut('a') == 5 + a.mgetOrPut('z').inc + doAssert a == {'a': 5, 'z': 1}.newTable + + t[].mgetOrPut(key) + proc len*[A, B](t: TableRef[A, B]): int = ## Returns the number of keys in `t`. runnableExamples: @@ -1015,7 +1059,8 @@ proc add*[A, B](t: TableRef[A, B], key: A, val: sink B) {.deprecated: proc del*[A, B](t: TableRef[A, B], key: A) = ## Deletes `key` from hash table `t`. Does nothing if the key does not exist. ## - ## **If duplicate keys were added, this may need to be called multiple times.** + ## .. warning:: If duplicate keys were added (via the now deprecated `add` proc), + ## this may need to be called multiple times. ## ## See also: ## * `pop proc<#pop,TableRef[A,B],A,B>`_ @@ -1035,7 +1080,8 @@ proc pop*[A, B](t: TableRef[A, B], key: A, val: var B): bool = ## mapping of the key. Otherwise, returns `false`, and the `val` is ## unchanged. ## - ## **If duplicate keys were added, this may need to be called multiple times.** + ## .. warning:: If duplicate keys were added (via the now deprecated `add` proc), + ## this may need to be called multiple times. ## ## See also: ## * `del proc<#del,TableRef[A,B],A>`_ @@ -1104,7 +1150,7 @@ iterator pairs*[A, B](t: TableRef[A, B]): (A, B) = ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## let a = { ## 'o': [1, 5, 7, 9], ## 'e': [2, 4, 6, 8] @@ -1118,6 +1164,7 @@ iterator pairs*[A, B](t: TableRef[A, B]): (A, B) = ## # value: [2, 4, 6, 8] ## # key: o ## # value: [1, 5, 7, 9] + ## ``` let L = len(t) for h in 0 .. high(t.data): if isFilled(t.data[h].hcode): @@ -1146,7 +1193,7 @@ iterator mpairs*[A, B](t: TableRef[A, B]): (A, var B) = yield (t.data[h].key, t.data[h].val) assert(len(t) == L, "the length of the table changed while iterating over it") -iterator keys*[A, B](t: TableRef[A, B]): A = +iterator keys*[A, B](t: TableRef[A, B]): lent A = ## Iterates over any key in the table `t`. ## ## See also: @@ -1167,7 +1214,7 @@ iterator keys*[A, B](t: TableRef[A, B]): A = yield t.data[h].key assert(len(t) == L, "the length of the table changed while iterating over it") -iterator values*[A, B](t: TableRef[A, B]): B = +iterator values*[A, B](t: TableRef[A, B]): lent B = ## Iterates over any value in the table `t`. ## ## See also: @@ -1300,6 +1347,7 @@ proc initOrderedTable*[A, B](initialSize = defaultInitialSize): OrderedTable[A, let a = initOrderedTable[int, string]() b = initOrderedTable[char, seq[int]]() + result = default(OrderedTable[A, B]) initImpl(result, initialSize) proc `[]=`*[A, B](t: var OrderedTable[A, B], key: A, val: sink B) = @@ -1335,7 +1383,7 @@ proc toOrderedTable*[A, B](pairs: openArray[(A, B)]): OrderedTable[A, B] = result = initOrderedTable[A, B](pairs.len) for key, val in items(pairs): result[key] = val -proc `[]`*[A, B](t: OrderedTable[A, B], key: A): B = +proc `[]`*[A, B](t: OrderedTable[A, B], key: A): lent B = ## Retrieves the value at `t[key]`. ## ## If `key` is not in `t`, the `KeyError` exception is raised. @@ -1391,7 +1439,7 @@ proc hasKey*[A, B](t: OrderedTable[A, B], key: A): bool = doAssert a.hasKey('a') == true doAssert a.hasKey('z') == false - var hc: Hash + var hc: Hash = default(Hash) result = rawGet(t, key, hc) >= 0 proc contains*[A, B](t: OrderedTable[A, B], key: A): bool = @@ -1440,7 +1488,7 @@ proc getOrDefault*[A, B](t: OrderedTable[A, B], key: A): B = let a = {'a': 5, 'b': 9}.toOrderedTable doAssert a.getOrDefault('a') == 5 doAssert a.getOrDefault('z') == 0 - + result = default(B) getOrDefaultImpl(t, key) proc getOrDefault*[A, B](t: OrderedTable[A, B], key: A, default: B): B = @@ -1458,7 +1506,7 @@ proc getOrDefault*[A, B](t: OrderedTable[A, B], key: A, default: B): B = let a = {'a': 5, 'b': 9}.toOrderedTable doAssert a.getOrDefault('a', 99) == 5 doAssert a.getOrDefault('z', 99) == 99 - + result = default(B) getOrDefaultImpl(t, key, default) proc mgetOrPut*[A, B](t: var OrderedTable[A, B], key: A, val: B): var B = @@ -1481,6 +1529,18 @@ proc mgetOrPut*[A, B](t: var OrderedTable[A, B], key: A, val: B): var B = mgetOrPutImpl(enlarge) +proc mgetOrPut*[A, B](t: var OrderedTable[A, B], key: A): var B = + ## Retrieves the value at `t[key]` or puts the + ## default initialization value for type `B` (e.g. 0 for any + ## integer type). + runnableExamples: + var a = {'a': 5}.toOrderedTable + doAssert a.mgetOrPut('a') == 5 + a.mgetOrPut('z').inc + doAssert a == {'a': 5, 'z': 1}.toOrderedTable + + mgetOrPutImpl(enlarge) + proc len*[A, B](t: OrderedTable[A, B]): int {.inline.} = ## Returns the number of keys in `t`. runnableExamples: @@ -1579,7 +1639,7 @@ proc clear*[A, B](t: var OrderedTable[A, B]) = t.last = -1 proc sort*[A, B](t: var OrderedTable[A, B], cmp: proc (x, y: (A, B)): int, - order = SortOrder.Ascending) = + order = SortOrder.Ascending) {.effectsOf: cmp.} = ## Sorts `t` according to the function `cmp`. ## ## This modifies the internal list @@ -1680,7 +1740,7 @@ iterator pairs*[A, B](t: OrderedTable[A, B]): (A, B) = ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## let a = { ## 'o': [1, 5, 7, 9], ## 'e': [2, 4, 6, 8] @@ -1694,6 +1754,7 @@ iterator pairs*[A, B](t: OrderedTable[A, B]): (A, B) = ## # value: [1, 5, 7, 9] ## # key: e ## # value: [2, 4, 6, 8] + ## ``` let L = len(t) forAllOrderedPairs: @@ -1722,7 +1783,7 @@ iterator mpairs*[A, B](t: var OrderedTable[A, B]): (A, var B) = yield (t.data[h].key, t.data[h].val) assert(len(t) == L, "the length of the table changed while iterating over it") -iterator keys*[A, B](t: OrderedTable[A, B]): A = +iterator keys*[A, B](t: OrderedTable[A, B]): lent A = ## Iterates over any key in the table `t` in insertion order. ## ## See also: @@ -1743,7 +1804,7 @@ iterator keys*[A, B](t: OrderedTable[A, B]): A = yield t.data[h].key assert(len(t) == L, "the length of the table changed while iterating over it") -iterator values*[A, B](t: OrderedTable[A, B]): B = +iterator values*[A, B](t: OrderedTable[A, B]): lent B = ## Iterates over any value in the table `t` in insertion order. ## ## See also: @@ -1790,7 +1851,7 @@ iterator mvalues*[A, B](t: var OrderedTable[A, B]): var B = # --------------------------- OrderedTableRef ------------------------------- # --------------------------------------------------------------------------- -proc newOrderedTable*[A, B](initialSize = defaultInitialSize): <//>OrderedTableRef[A, B] = +proc newOrderedTable*[A, B](initialSize = defaultInitialSize): OrderedTableRef[A, B] = ## Creates a new ordered ref hash table that is empty. ## ## See also: @@ -1803,9 +1864,10 @@ proc newOrderedTable*[A, B](initialSize = defaultInitialSize): <//>OrderedTableR a = newOrderedTable[int, string]() b = newOrderedTable[char, seq[int]]() new(result) - result[] = initOrderedTable[A, B](initialSize) + {.noSideEffect.}: + result[] = initOrderedTable[A, B](initialSize) -proc newOrderedTable*[A, B](pairs: openArray[(A, B)]): <//>OrderedTableRef[A, B] = +proc newOrderedTable*[A, B](pairs: openArray[(A, B)]): OrderedTableRef[A, B] = ## Creates a new ordered ref hash table that contains the given `pairs`. ## ## `pairs` is a container consisting of `(key, value)` tuples. @@ -1820,7 +1882,8 @@ proc newOrderedTable*[A, B](pairs: openArray[(A, B)]): <//>OrderedTableRef[A, B] assert b == {'a': 5, 'b': 9}.newOrderedTable result = newOrderedTable[A, B](pairs.len) - for key, val in items(pairs): result[key] = val + {.noSideEffect.}: + for key, val in items(pairs): result[key] = val proc `[]`*[A, B](t: OrderedTableRef[A, B], key: A): var B = @@ -1890,7 +1953,7 @@ proc contains*[A, B](t: OrderedTableRef[A, B], key: A): bool = return hasKey[A, B](t, key) -proc hasKeyOrPut*[A, B](t: var OrderedTableRef[A, B], key: A, val: B): bool = +proc hasKeyOrPut*[A, B](t: OrderedTableRef[A, B], key: A, val: B): bool = ## Returns true if `key` is in the table, otherwise inserts `value`. ## ## See also: @@ -1967,6 +2030,18 @@ proc mgetOrPut*[A, B](t: OrderedTableRef[A, B], key: A, val: B): var B = result = t[].mgetOrPut(key, val) +proc mgetOrPut*[A, B](t: OrderedTableRef[A, B], key: A): var B = + ## Retrieves the value at `t[key]` or puts the + ## default initialization value for type `B` (e.g. 0 for any + ## integer type). + runnableExamples: + var a = {'a': 5}.toOrderedTable + doAssert a.mgetOrPut('a') == 5 + a.mgetOrPut('z').inc + doAssert a == {'a': 5, 'z': 1}.toOrderedTable + + t[].mgetOrPut(key) + proc len*[A, B](t: OrderedTableRef[A, B]): int {.inline.} = ## Returns the number of keys in `t`. runnableExamples: @@ -2036,7 +2111,7 @@ proc clear*[A, B](t: OrderedTableRef[A, B]) = clear(t[]) proc sort*[A, B](t: OrderedTableRef[A, B], cmp: proc (x, y: (A, B)): int, - order = SortOrder.Ascending) = + order = SortOrder.Ascending) {.effectsOf: cmp.} = ## Sorts `t` according to the function `cmp`. ## ## This modifies the internal list @@ -2088,7 +2163,7 @@ iterator pairs*[A, B](t: OrderedTableRef[A, B]): (A, B) = ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## let a = { ## 'o': [1, 5, 7, 9], ## 'e': [2, 4, 6, 8] @@ -2102,6 +2177,7 @@ iterator pairs*[A, B](t: OrderedTableRef[A, B]): (A, B) = ## # value: [1, 5, 7, 9] ## # key: e ## # value: [2, 4, 6, 8] + ## ``` let L = len(t) forAllOrderedPairs: @@ -2130,7 +2206,7 @@ iterator mpairs*[A, B](t: OrderedTableRef[A, B]): (A, var B) = yield (t.data[h].key, t.data[h].val) assert(len(t) == L, "the length of the table changed while iterating over it") -iterator keys*[A, B](t: OrderedTableRef[A, B]): A = +iterator keys*[A, B](t: OrderedTableRef[A, B]): lent A = ## Iterates over any key in the table `t` in insertion order. ## ## See also: @@ -2151,7 +2227,7 @@ iterator keys*[A, B](t: OrderedTableRef[A, B]): A = yield t.data[h].key assert(len(t) == L, "the length of the table changed while iterating over it") -iterator values*[A, B](t: OrderedTableRef[A, B]): B = +iterator values*[A, B](t: OrderedTableRef[A, B]): lent B = ## Iterates over any value in the table `t` in insertion order. ## ## See also: @@ -2262,6 +2338,7 @@ proc initCountTable*[A](initialSize = defaultInitialSize): CountTable[A] = ## * `toCountTable proc<#toCountTable,openArray[A]>`_ ## * `newCountTable proc<#newCountTable>`_ for creating a ## `CountTableRef` + result = default(CountTable[A]) initImpl(result, initialSize) proc toCountTable*[A](keys: openArray[A]): CountTable[A] = @@ -2371,7 +2448,7 @@ proc contains*[A](t: CountTable[A], key: A): bool = return hasKey[A](t, key) proc getOrDefault*[A](t: CountTable[A], key: A; default: int = 0): int = - ## Retrieves the value at `t[key]` if`key` is in `t`. Otherwise, the + ## Retrieves the value at `t[key]` if `key` is in `t`. Otherwise, the ## integer value of `default` is returned. ## ## See also: @@ -2501,7 +2578,7 @@ iterator pairs*[A](t: CountTable[A]): (A, int) = ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## let a = toCountTable("abracadabra") ## ## for k, v in pairs(a): @@ -2518,6 +2595,7 @@ iterator pairs*[A](t: CountTable[A]): (A, int) = ## # value: 1 ## # key: r ## # value: 2 + ## ``` let L = len(t) for h in 0 .. high(t.data): if t.data[h].val != 0: @@ -2543,7 +2621,7 @@ iterator mpairs*[A](t: var CountTable[A]): (A, var int) = yield (t.data[h].key, t.data[h].val) assert(len(t) == L, "the length of the table changed while iterating over it") -iterator keys*[A](t: CountTable[A]): A = +iterator keys*[A](t: CountTable[A]): lent A = ## Iterates over any key in the table `t`. ## ## See also: @@ -2610,7 +2688,7 @@ iterator mvalues*[A](t: var CountTable[A]): var int = proc inc*[A](t: CountTableRef[A], key: A, val = 1) -proc newCountTable*[A](initialSize = defaultInitialSize): <//>CountTableRef[A] = +proc newCountTable*[A](initialSize = defaultInitialSize): CountTableRef[A] = ## Creates a new ref count table that is empty. ## ## See also: @@ -2619,13 +2697,15 @@ proc newCountTable*[A](initialSize = defaultInitialSize): <//>CountTableRef[A] = ## * `initCountTable proc<#initCountTable>`_ for creating a ## `CountTable` new(result) - result[] = initCountTable[A](initialSize) + {.noSideEffect.}: + result[] = initCountTable[A](initialSize) -proc newCountTable*[A](keys: openArray[A]): <//>CountTableRef[A] = +proc newCountTable*[A](keys: openArray[A]): CountTableRef[A] = ## Creates a new ref count table with every member of a container `keys` ## having a count of how many times it occurs in that container. result = newCountTable[A](keys.len) - for key in items(keys): result.inc(key) + {.noSideEffect.}: + for key in items(keys): result.inc(key) proc `[]`*[A](t: CountTableRef[A], key: A): int = ## Retrieves the value at `t[key]` if `key` is in `t`. @@ -2649,7 +2729,8 @@ proc `[]=`*[A](t: CountTableRef[A], key: A, val: int) = ## * `inc proc<#inc,CountTableRef[A],A,int>`_ for incrementing a ## value of a key assert val > 0 - t[][key] = val + {.noSideEffect.}: + t[][key] = val proc inc*[A](t: CountTableRef[A], key: A, val = 1) = ## Increments `t[key]` by `val` (default: 1). @@ -2658,7 +2739,8 @@ proc inc*[A](t: CountTableRef[A], key: A, val = 1) = a.inc('a') a.inc('b', 10) doAssert a == newCountTable("aaabbbbbbbbbbb") - t[].inc(key, val) + {.noSideEffect.}: + t[].inc(key, val) proc smallest*[A](t: CountTableRef[A]): tuple[key: A, val: int] = ## Returns the `(key, value)` pair with the smallest `val`. Efficiency: O(n) @@ -2691,7 +2773,7 @@ proc contains*[A](t: CountTableRef[A], key: A): bool = return hasKey[A](t, key) proc getOrDefault*[A](t: CountTableRef[A], key: A, default: int): int = - ## Retrieves the value at `t[key]` if`key` is in `t`. Otherwise, the + ## Retrieves the value at `t[key]` if `key` is in `t`. Otherwise, the ## integer value of `default` is returned. ## ## See also: @@ -2777,7 +2859,7 @@ iterator pairs*[A](t: CountTableRef[A]): (A, int) = ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## let a = newCountTable("abracadabra") ## ## for k, v in pairs(a): @@ -2794,6 +2876,7 @@ iterator pairs*[A](t: CountTableRef[A]): (A, int) = ## # value: 1 ## # key: r ## # value: 2 + ## ``` let L = len(t) for h in 0 .. high(t.data): if t.data[h].val != 0: @@ -2872,3 +2955,18 @@ iterator mvalues*[A](t: CountTableRef[A]): var int = if t.data[h].val != 0: yield t.data[h].val assert(len(t) == L, "the length of the table changed while iterating over it") + +proc hash*[K,V](s: Table[K,V]): Hash = + for p in pairs(s): + result = result xor hash(p) + result = !$result + +proc hash*[K,V](s: OrderedTable[K,V]): Hash = + for p in pairs(s): + result = result !& hash(p) + result = !$result + +proc hash*[V](s: CountTable[V]): Hash = + for p in pairs(s): + result = result xor hash(p) + result = !$result diff --git a/lib/pure/colors.nim b/lib/pure/colors.nim index 4a3f47e92..d3e6dc063 100644 --- a/lib/pure/colors.nim +++ b/lib/pure/colors.nim @@ -6,24 +6,26 @@ # distribution, for details about the copyright. # -## This module implements color handling for Nim. +## This module implements color handling for Nim, +## namely color mixing and parsing the CSS color names. -import strutils -from algorithm import binarySearch +import std/strutils +from std/algorithm import binarySearch type Color* = distinct int ## A color stored as RGB, e.g. `0xff00cc`. -proc `==` *(a, b: Color): bool {.borrow.} +proc `==`*(a, b: Color): bool {.borrow.} ## Compares two colors. ## - ## .. code-block:: + ## ```Nim ## var ## a = Color(0xff_00_ff) ## b = colFuchsia ## c = Color(0x00_ff_cc) ## assert a == b - ## assert not a == c + ## assert not (a == c) + ## ``` template extract(a: Color, r, g, b: untyped) = var r = a.int shr 16 and 0xff @@ -170,6 +172,7 @@ const colDarkGoldenRod* = Color(0xB8860B) colDarkGray* = Color(0xA9A9A9) colDarkGreen* = Color(0x006400) + colDarkGrey* = Color(0xA9A9A9) colDarkKhaki* = Color(0xBDB76B) colDarkMagenta* = Color(0x8B008B) colDarkOliveGreen* = Color(0x556B2F) @@ -180,11 +183,13 @@ const colDarkSeaGreen* = Color(0x8FBC8F) colDarkSlateBlue* = Color(0x483D8B) colDarkSlateGray* = Color(0x2F4F4F) + colDarkSlateGrey* = Color(0x2F4F4F) colDarkTurquoise* = Color(0x00CED1) colDarkViolet* = Color(0x9400D3) colDeepPink* = Color(0xFF1493) colDeepSkyBlue* = Color(0x00BFFF) colDimGray* = Color(0x696969) + colDimGrey* = Color(0x696969) colDodgerBlue* = Color(0x1E90FF) colFireBrick* = Color(0xB22222) colFloralWhite* = Color(0xFFFAF0) @@ -197,6 +202,7 @@ const colGray* = Color(0x808080) colGreen* = Color(0x008000) colGreenYellow* = Color(0xADFF2F) + colGrey* = Color(0x808080) colHoneyDew* = Color(0xF0FFF0) colHotPink* = Color(0xFF69B4) colIndianRed* = Color(0xCD5C5C) @@ -211,13 +217,15 @@ const colLightCoral* = Color(0xF08080) colLightCyan* = Color(0xE0FFFF) colLightGoldenRodYellow* = Color(0xFAFAD2) - colLightGrey* = Color(0xD3D3D3) + colLightGray* = Color(0xD3D3D3) colLightGreen* = Color(0x90EE90) + colLightGrey* = Color(0xD3D3D3) colLightPink* = Color(0xFFB6C1) colLightSalmon* = Color(0xFFA07A) colLightSeaGreen* = Color(0x20B2AA) colLightSkyBlue* = Color(0x87CEFA) colLightSlateGray* = Color(0x778899) + colLightSlateGrey* = Color(0x778899) colLightSteelBlue* = Color(0xB0C4DE) colLightYellow* = Color(0xFFFFE0) colLime* = Color(0x00FF00) @@ -228,7 +236,7 @@ const colMediumAquaMarine* = Color(0x66CDAA) colMediumBlue* = Color(0x0000CD) colMediumOrchid* = Color(0xBA55D3) - colMediumPurple* = Color(0x9370D8) + colMediumPurple* = Color(0x9370DB) colMediumSeaGreen* = Color(0x3CB371) colMediumSlateBlue* = Color(0x7B68EE) colMediumSpringGreen* = Color(0x00FA9A) @@ -249,7 +257,7 @@ const colPaleGoldenRod* = Color(0xEEE8AA) colPaleGreen* = Color(0x98FB98) colPaleTurquoise* = Color(0xAFEEEE) - colPaleVioletRed* = Color(0xD87093) + colPaleVioletRed* = Color(0xDB7093) colPapayaWhip* = Color(0xFFEFD5) colPeachPuff* = Color(0xFFDAB9) colPeru* = Color(0xCD853F) @@ -257,6 +265,7 @@ const colPlum* = Color(0xDDA0DD) colPowderBlue* = Color(0xB0E0E6) colPurple* = Color(0x800080) + colRebeccaPurple* = Color(0x663399) colRed* = Color(0xFF0000) colRosyBrown* = Color(0xBC8F8F) colRoyalBlue* = Color(0x4169E1) @@ -270,6 +279,7 @@ const colSkyBlue* = Color(0x87CEEB) colSlateBlue* = Color(0x6A5ACD) colSlateGray* = Color(0x708090) + colSlateGrey* = Color(0x708090) colSnow* = Color(0xFFFAFA) colSpringGreen* = Color(0x00FF7F) colSteelBlue* = Color(0x4682B4) @@ -285,147 +295,155 @@ const colYellow* = Color(0xFFFF00) colYellowGreen* = Color(0x9ACD32) - colorNames = [ - ("aliceblue", colAliceBlue), - ("antiquewhite", colAntiqueWhite), - ("aqua", colAqua), - ("aquamarine", colAquamarine), - ("azure", colAzure), - ("beige", colBeige), - ("bisque", colBisque), - ("black", colBlack), - ("blanchedalmond", colBlanchedAlmond), - ("blue", colBlue), - ("blueviolet", colBlueViolet), - ("brown", colBrown), - ("burlywood", colBurlyWood), - ("cadetblue", colCadetBlue), - ("chartreuse", colChartreuse), - ("chocolate", colChocolate), - ("coral", colCoral), - ("cornflowerblue", colCornflowerBlue), - ("cornsilk", colCornsilk), - ("crimson", colCrimson), - ("cyan", colCyan), - ("darkblue", colDarkBlue), - ("darkcyan", colDarkCyan), - ("darkgoldenrod", colDarkGoldenRod), - ("darkgray", colDarkGray), - ("darkgreen", colDarkGreen), - ("darkkhaki", colDarkKhaki), - ("darkmagenta", colDarkMagenta), - ("darkolivegreen", colDarkOliveGreen), - ("darkorange", colDarkorange), - ("darkorchid", colDarkOrchid), - ("darkred", colDarkRed), - ("darksalmon", colDarkSalmon), - ("darkseagreen", colDarkSeaGreen), - ("darkslateblue", colDarkSlateBlue), - ("darkslategray", colDarkSlateGray), - ("darkturquoise", colDarkTurquoise), - ("darkviolet", colDarkViolet), - ("deeppink", colDeepPink), - ("deepskyblue", colDeepSkyBlue), - ("dimgray", colDimGray), - ("dodgerblue", colDodgerBlue), - ("firebrick", colFireBrick), - ("floralwhite", colFloralWhite), - ("forestgreen", colForestGreen), - ("fuchsia", colFuchsia), - ("gainsboro", colGainsboro), - ("ghostwhite", colGhostWhite), - ("gold", colGold), - ("goldenrod", colGoldenRod), - ("gray", colGray), - ("green", colGreen), - ("greenyellow", colGreenYellow), - ("honeydew", colHoneyDew), - ("hotpink", colHotPink), - ("indianred", colIndianRed), - ("indigo", colIndigo), - ("ivory", colIvory), - ("khaki", colKhaki), - ("lavender", colLavender), - ("lavenderblush", colLavenderBlush), - ("lawngreen", colLawnGreen), - ("lemonchiffon", colLemonChiffon), - ("lightblue", colLightBlue), - ("lightcoral", colLightCoral), - ("lightcyan", colLightCyan), - ("lightgoldenrodyellow", colLightGoldenRodYellow), - ("lightgrey", colLightGrey), - ("lightgreen", colLightGreen), - ("lightpink", colLightPink), - ("lightsalmon", colLightSalmon), - ("lightseagreen", colLightSeaGreen), - ("lightskyblue", colLightSkyBlue), - ("lightslategray", colLightSlateGray), - ("lightsteelblue", colLightSteelBlue), - ("lightyellow", colLightYellow), - ("lime", colLime), - ("limegreen", colLimeGreen), - ("linen", colLinen), - ("magenta", colMagenta), - ("maroon", colMaroon), - ("mediumaquamarine", colMediumAquaMarine), - ("mediumblue", colMediumBlue), - ("mediumorchid", colMediumOrchid), - ("mediumpurple", colMediumPurple), - ("mediumseagreen", colMediumSeaGreen), - ("mediumslateblue", colMediumSlateBlue), - ("mediumspringgreen", colMediumSpringGreen), - ("mediumturquoise", colMediumTurquoise), - ("mediumvioletred", colMediumVioletRed), - ("midnightblue", colMidnightBlue), - ("mintcream", colMintCream), - ("mistyrose", colMistyRose), - ("moccasin", colMoccasin), - ("navajowhite", colNavajoWhite), - ("navy", colNavy), - ("oldlace", colOldLace), - ("olive", colOlive), - ("olivedrab", colOliveDrab), - ("orange", colOrange), - ("orangered", colOrangeRed), - ("orchid", colOrchid), - ("palegoldenrod", colPaleGoldenRod), - ("palegreen", colPaleGreen), - ("paleturquoise", colPaleTurquoise), - ("palevioletred", colPaleVioletRed), - ("papayawhip", colPapayaWhip), - ("peachpuff", colPeachPuff), - ("peru", colPeru), - ("pink", colPink), - ("plum", colPlum), - ("powderblue", colPowderBlue), - ("purple", colPurple), - ("red", colRed), - ("rosybrown", colRosyBrown), - ("royalblue", colRoyalBlue), - ("saddlebrown", colSaddleBrown), - ("salmon", colSalmon), - ("sandybrown", colSandyBrown), - ("seagreen", colSeaGreen), - ("seashell", colSeaShell), - ("sienna", colSienna), - ("silver", colSilver), - ("skyblue", colSkyBlue), - ("slateblue", colSlateBlue), - ("slategray", colSlateGray), - ("snow", colSnow), - ("springgreen", colSpringGreen), - ("steelblue", colSteelBlue), - ("tan", colTan), - ("teal", colTeal), - ("thistle", colThistle), - ("tomato", colTomato), - ("turquoise", colTurquoise), - ("violet", colViolet), - ("wheat", colWheat), - ("white", colWhite), - ("whitesmoke", colWhiteSmoke), - ("yellow", colYellow), - ("yellowgreen", colYellowGreen)] + colorNames = { + "aliceblue": colAliceBlue, + "antiquewhite": colAntiqueWhite, + "aqua": colAqua, + "aquamarine": colAquamarine, + "azure": colAzure, + "beige": colBeige, + "bisque": colBisque, + "black": colBlack, + "blanchedalmond": colBlanchedAlmond, + "blue": colBlue, + "blueviolet": colBlueViolet, + "brown": colBrown, + "burlywood": colBurlyWood, + "cadetblue": colCadetBlue, + "chartreuse": colChartreuse, + "chocolate": colChocolate, + "coral": colCoral, + "cornflowerblue": colCornflowerBlue, + "cornsilk": colCornsilk, + "crimson": colCrimson, + "cyan": colCyan, + "darkblue": colDarkBlue, + "darkcyan": colDarkCyan, + "darkgoldenrod": colDarkGoldenRod, + "darkgray": colDarkGray, + "darkgreen": colDarkGreen, + "darkgrey": colDarkGrey, + "darkkhaki": colDarkKhaki, + "darkmagenta": colDarkMagenta, + "darkolivegreen": colDarkOliveGreen, + "darkorange": colDarkorange, + "darkorchid": colDarkOrchid, + "darkred": colDarkRed, + "darksalmon": colDarkSalmon, + "darkseagreen": colDarkSeaGreen, + "darkslateblue": colDarkSlateBlue, + "darkslategray": colDarkSlateGray, + "darkslategrey": colDarkSlateGrey, + "darkturquoise": colDarkTurquoise, + "darkviolet": colDarkViolet, + "deeppink": colDeepPink, + "deepskyblue": colDeepSkyBlue, + "dimgray": colDimGray, + "dimgrey": colDimGrey, + "dodgerblue": colDodgerBlue, + "firebrick": colFireBrick, + "floralwhite": colFloralWhite, + "forestgreen": colForestGreen, + "fuchsia": colFuchsia, + "gainsboro": colGainsboro, + "ghostwhite": colGhostWhite, + "gold": colGold, + "goldenrod": colGoldenRod, + "gray": colGray, + "green": colGreen, + "greenyellow": colGreenYellow, + "grey": colGrey, + "honeydew": colHoneyDew, + "hotpink": colHotPink, + "indianred": colIndianRed, + "indigo": colIndigo, + "ivory": colIvory, + "khaki": colKhaki, + "lavender": colLavender, + "lavenderblush": colLavenderBlush, + "lawngreen": colLawnGreen, + "lemonchiffon": colLemonChiffon, + "lightblue": colLightBlue, + "lightcoral": colLightCoral, + "lightcyan": colLightCyan, + "lightgoldenrodyellow": colLightGoldenRodYellow, + "lightgray": colLightGray, + "lightgreen": colLightGreen, + "lightgrey": colLightGrey, + "lightpink": colLightPink, + "lightsalmon": colLightSalmon, + "lightseagreen": colLightSeaGreen, + "lightskyblue": colLightSkyBlue, + "lightslategray": colLightSlateGray, + "lightslategrey": colLightSlateGrey, + "lightsteelblue": colLightSteelBlue, + "lightyellow": colLightYellow, + "lime": colLime, + "limegreen": colLimeGreen, + "linen": colLinen, + "magenta": colMagenta, + "maroon": colMaroon, + "mediumaquamarine": colMediumAquaMarine, + "mediumblue": colMediumBlue, + "mediumorchid": colMediumOrchid, + "mediumpurple": colMediumPurple, + "mediumseagreen": colMediumSeaGreen, + "mediumslateblue": colMediumSlateBlue, + "mediumspringgreen": colMediumSpringGreen, + "mediumturquoise": colMediumTurquoise, + "mediumvioletred": colMediumVioletRed, + "midnightblue": colMidnightBlue, + "mintcream": colMintCream, + "mistyrose": colMistyRose, + "moccasin": colMoccasin, + "navajowhite": colNavajoWhite, + "navy": colNavy, + "oldlace": colOldLace, + "olive": colOlive, + "olivedrab": colOliveDrab, + "orange": colOrange, + "orangered": colOrangeRed, + "orchid": colOrchid, + "palegoldenrod": colPaleGoldenRod, + "palegreen": colPaleGreen, + "paleturquoise": colPaleTurquoise, + "palevioletred": colPaleVioletRed, + "papayawhip": colPapayaWhip, + "peachpuff": colPeachPuff, + "peru": colPeru, + "pink": colPink, + "plum": colPlum, + "powderblue": colPowderBlue, + "purple": colPurple, + "rebeccapurple": colRebeccaPurple, + "red": colRed, + "rosybrown": colRosyBrown, + "royalblue": colRoyalBlue, + "saddlebrown": colSaddleBrown, + "salmon": colSalmon, + "sandybrown": colSandyBrown, + "seagreen": colSeaGreen, + "seashell": colSeaShell, + "sienna": colSienna, + "silver": colSilver, + "skyblue": colSkyBlue, + "slateblue": colSlateBlue, + "slategray": colSlateGray, + "slategrey": colSlateGrey, + "snow": colSnow, + "springgreen": colSpringGreen, + "steelblue": colSteelBlue, + "tan": colTan, + "teal": colTeal, + "thistle": colThistle, + "tomato": colTomato, + "turquoise": colTurquoise, + "violet": colViolet, + "wheat": colWheat, + "white": colWhite, + "whitesmoke": colWhiteSmoke, + "yellow": colYellow, + "yellowgreen": colYellowGreen} proc `$`*(c: Color): string = ## Converts a color into its textual representation. diff --git a/lib/pure/complex.nim b/lib/pure/complex.nim index b9371c1e1..b48811eae 100644 --- a/lib/pure/complex.nim +++ b/lib/pure/complex.nim @@ -15,9 +15,6 @@ runnableExamples: from std/math import almostEqual, sqrt - func almostEqual(a, b: Complex): bool = - almostEqual(a.re, b.re) and almostEqual(a.im, b.im) - let z1 = complex(1.0, 2.0) z2 = complex(3.0, -4.0) @@ -36,7 +33,7 @@ runnableExamples: {.push checks: off, line_dir: off, stack_trace: off, debugger: off.} # the user does not want to trace a part of the standard library! -import std/math +import std/[math, strformat] type Complex*[T: SomeFloat] = object @@ -82,6 +79,13 @@ func abs2*[T](z: Complex[T]): T = ## This is more efficient than `abs(z) ^ 2`. result = z.re * z.re + z.im * z.im +func sgn*[T](z: Complex[T]): Complex[T] = + ## Returns the phase of `z` as a unit complex number, + ## or 0 if `z` is 0. + let a = abs(z) + if a != 0: + result = z / a + func conjugate*[T](z: Complex[T]): Complex[T] = ## Returns the complex conjugate of `z` (`complex(z.re, -z.im)`). result.re = z.re @@ -156,18 +160,7 @@ func `/`*[T](x: T; y: Complex[T]): Complex[T] = func `/`*[T](x, y: Complex[T]): Complex[T] = ## Divides two complex numbers. - var r, den: T - if abs(y.re) < abs(y.im): - r = y.re / y.im - den = y.im + r * y.re - result.re = (x.re * r + x.im) / den - result.im = (x.im * r - x.re) / den - else: - r = y.im / y.re - den = y.re + r * y.im - result.re = (x.re + r * x.im) / den - result.im = (x.im - r * x.re) / den - + x * conjugate(y) / abs2(y) func `+=`*[T](x: var Complex[T]; y: Complex[T]) = ## Adds `y` to `x`. @@ -253,10 +246,31 @@ func pow*[T](x, y: Complex[T]): Complex[T] = else: result.re = 0.0 result.im = 0.0 - elif y.re == 1.0 and y.im == 0.0: - result = x - elif y.re == -1.0 and y.im == 0.0: - result = T(1.0) / x + elif y.im == 0.0: + if y.re == 1.0: + result = x + elif y.re == -1.0: + result = T(1.0) / x + elif y.re == 2.0: + result = x * x + elif y.re == 0.5: + result = sqrt(x) + elif x.im == 0.0: + # Revert to real pow when both base and exponent are real + result.re = pow(x.re, y.re) + result.im = 0.0 + else: + # Special case when the exponent is real + let + rho = abs(x) + theta = arctan2(x.im, x.re) + s = pow(rho, y.re) + r = y.re * theta + result.re = s * cos(r) + result.im = s * sin(r) + elif x.im == 0.0 and x.re == E: + # Special case Euler's formula + result = exp(y) else: let rho = abs(x) @@ -395,6 +409,24 @@ func rect*[T](r, phi: T): Complex[T] = ## * `polar func<#polar,Complex[T]>`_ for the inverse operation complex(r * cos(phi), r * sin(phi)) +func almostEqual*[T: SomeFloat](x, y: Complex[T]; unitsInLastPlace: Natural = 4): bool = + ## Checks if two complex values are almost equal, using the + ## [machine epsilon](https://en.wikipedia.org/wiki/Machine_epsilon). + ## + ## Two complex values are considered almost equal if their real and imaginary + ## components are almost equal. + ## + ## `unitsInLastPlace` is the max number of + ## [units in the last place](https://en.wikipedia.org/wiki/Unit_in_the_last_place) + ## difference tolerated when comparing two numbers. The larger the value, the + ## more error is allowed. A `0` value means that two numbers must be exactly the + ## same to be considered equal. + ## + ## The machine epsilon has to be scaled to the magnitude of the values used + ## and multiplied by the desired precision in ULPs unless the difference is + ## subnormal. + almostEqual(x.re, y.re, unitsInLastPlace = unitsInLastPlace) and + almostEqual(x.im, y.im, unitsInLastPlace = unitsInLastPlace) func `$`*(z: Complex): string = ## Returns `z`'s string representation as `"(re, im)"`. @@ -403,4 +435,39 @@ func `$`*(z: Complex): string = result = "(" & $z.re & ", " & $z.im & ")" +proc formatValueAsTuple(result: var string; value: Complex; specifier: string) = + ## Format implementation for `Complex` representing the value as a (real, imaginary) tuple. + result.add "(" + formatValue(result, value.re, specifier) + result.add ", " + formatValue(result, value.im, specifier) + result.add ")" + +proc formatValueAsComplexNumber(result: var string; value: Complex; specifier: string) = + ## Format implementation for `Complex` representing the value as a (RE+IMj) number + ## By default, the real and imaginary parts are formatted using the general ('g') format + let specifier = if specifier.contains({'e', 'E', 'f', 'F', 'g', 'G'}): + specifier.replace("j") + else: + specifier.replace('j', 'g') + result.add "(" + formatValue(result, value.re, specifier) + if value.im >= 0 and not specifier.contains({'+', '-'}): + result.add "+" + formatValue(result, value.im, specifier) + result.add "j)" + +proc formatValue*(result: var string; value: Complex; specifier: string) = + ## Standard format implementation for `Complex`. It makes little + ## sense to call this directly, but it is required to exist + ## by the `&` macro. + ## For complex numbers, we add a specific 'j' specifier, which formats + ## the value as (A+Bj) like in mathematics. + if specifier.len == 0: + result.add $value + elif 'j' in specifier: + formatValueAsComplexNumber(result, value, specifier) + else: + formatValueAsTuple(result, value, specifier) + {.pop.} diff --git a/lib/pure/concurrency/atomics.nim b/lib/pure/concurrency/atomics.nim index bdf1e8cc2..818f1b37a 100644 --- a/lib/pure/concurrency/atomics.nim +++ b/lib/pure/concurrency/atomics.nim @@ -10,6 +10,9 @@ ## Types and operations for atomic operations and lockless algorithms. ## ## Unstable API. +## +## By default, C++ uses C11 atomic primitives. To use C++ `std::atomic`, +## `-d:nimUseCppAtomics` can be defined. runnableExamples: # Atomic @@ -50,8 +53,7 @@ runnableExamples: flag.clear(moRelaxed) assert not flag.testAndSet - -when defined(cpp) or defined(nimdoc): +when (defined(cpp) and defined(nimUseCppAtomics)) or defined(nimdoc): # For the C++ backend, types and operations map directly to C++11 atomics. {.push, header: "<atomic>".} @@ -211,8 +213,8 @@ else: # MSVC intrinsics proc interlockedExchange(location: pointer; desired: int8): int8 {.importc: "_InterlockedExchange8".} - proc interlockedExchange(location: pointer; desired: int16): int16 {.importc: "_InterlockedExchange".} - proc interlockedExchange(location: pointer; desired: int32): int32 {.importc: "_InterlockedExchange16".} + proc interlockedExchange(location: pointer; desired: int16): int16 {.importc: "_InterlockedExchange16".} + proc interlockedExchange(location: pointer; desired: int32): int32 {.importc: "_InterlockedExchange".} proc interlockedExchange(location: pointer; desired: int64): int64 {.importc: "_InterlockedExchange64".} proc interlockedCompareExchange(location: pointer; desired, expected: int8): int8 {.importc: "_InterlockedCompareExchange8".} @@ -274,10 +276,17 @@ else: cast[T](interlockedXor(addr(location.value), cast[nonAtomicType(T)](value))) else: - {.push, header: "<stdatomic.h>".} + when defined(cpp): + {.push, header: "<atomic>".} + template maybeWrapStd(x: string): string = + "std::" & x + else: + {.push, header: "<stdatomic.h>".} + template maybeWrapStd(x: string): string = + x type - MemoryOrder* {.importc: "memory_order".} = enum + MemoryOrder* {.importc: "memory_order".maybeWrapStd.} = enum moRelaxed moConsume moAcquire @@ -285,53 +294,59 @@ else: moAcquireRelease moSequentiallyConsistent - type - # Atomic* {.importcpp: "_Atomic('0)".} [T] = object + when defined(cpp): + type + # Atomic*[T] {.importcpp: "_Atomic('0)".} = object - AtomicInt8 {.importc: "_Atomic NI8", size: 1.} = object - AtomicInt16 {.importc: "_Atomic NI16", size: 2.} = object - AtomicInt32 {.importc: "_Atomic NI32", size: 4.} = object - AtomicInt64 {.importc: "_Atomic NI64", size: 8.} = object + AtomicInt8 {.importc: "std::atomic<NI8>".} = int8 + AtomicInt16 {.importc: "std::atomic<NI16>".} = int16 + AtomicInt32 {.importc: "std::atomic<NI32>".} = int32 + AtomicInt64 {.importc: "std::atomic<NI64>".} = int64 + else: + type + # Atomic*[T] {.importcpp: "_Atomic('0)".} = object - template atomicType*(T: typedesc[Trivial]): untyped = - # Maps the size of a trivial type to it's internal atomic type - when sizeof(T) == 1: AtomicInt8 - elif sizeof(T) == 2: AtomicInt16 - elif sizeof(T) == 4: AtomicInt32 - elif sizeof(T) == 8: AtomicInt64 + AtomicInt8 {.importc: "_Atomic NI8".} = int8 + AtomicInt16 {.importc: "_Atomic NI16".} = int16 + AtomicInt32 {.importc: "_Atomic NI32".} = int32 + AtomicInt64 {.importc: "_Atomic NI64".} = int64 type - AtomicFlag* {.importc: "atomic_flag", size: 1.} = object + AtomicFlag* {.importc: "atomic_flag".maybeWrapStd, size: 1.} = object Atomic*[T] = object when T is Trivial: - value: T.atomicType + # Maps the size of a trivial type to it's internal atomic type + when sizeof(T) == 1: value: AtomicInt8 + elif sizeof(T) == 2: value: AtomicInt16 + elif sizeof(T) == 4: value: AtomicInt32 + elif sizeof(T) == 8: value: AtomicInt64 else: nonAtomicValue: T guard: AtomicFlag #proc init*[T](location: var Atomic[T]; value: T): T {.importcpp: "atomic_init(@)".} - proc atomic_load_explicit[T, A](location: ptr A; order: MemoryOrder): T {.importc.} - proc atomic_store_explicit[T, A](location: ptr A; desired: T; order: MemoryOrder = moSequentiallyConsistent) {.importc.} - proc atomic_exchange_explicit[T, A](location: ptr A; desired: T; order: MemoryOrder = moSequentiallyConsistent): T {.importc.} - proc atomic_compare_exchange_strong_explicit[T, A](location: ptr A; expected: ptr T; desired: T; success, failure: MemoryOrder): bool {.importc.} - proc atomic_compare_exchange_weak_explicit[T, A](location: ptr A; expected: ptr T; desired: T; success, failure: MemoryOrder): bool {.importc.} + proc atomic_load_explicit[T, A](location: ptr A; order: MemoryOrder): T {.importc: "atomic_load_explicit".maybeWrapStd.} + proc atomic_store_explicit[T, A](location: ptr A; desired: T; order: MemoryOrder = moSequentiallyConsistent) {.importc: "atomic_store_explicit".maybeWrapStd.} + proc atomic_exchange_explicit[T, A](location: ptr A; desired: T; order: MemoryOrder = moSequentiallyConsistent): T {.importc: "atomic_exchange_explicit".maybeWrapStd.} + proc atomic_compare_exchange_strong_explicit[T, A](location: ptr A; expected: ptr T; desired: T; success, failure: MemoryOrder): bool {.importc: "atomic_compare_exchange_strong_explicit".maybeWrapStd.} + proc atomic_compare_exchange_weak_explicit[T, A](location: ptr A; expected: ptr T; desired: T; success, failure: MemoryOrder): bool {.importc: "atomic_compare_exchange_weak_explicit".maybeWrapStd.} # Numerical operations - proc atomic_fetch_add_explicit[T, A](location: ptr A; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.importc.} - proc atomic_fetch_sub_explicit[T, A](location: ptr A; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.importc.} - proc atomic_fetch_and_explicit[T, A](location: ptr A; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.importc.} - proc atomic_fetch_or_explicit[T, A](location: ptr A; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.importc.} - proc atomic_fetch_xor_explicit[T, A](location: ptr A; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.importc.} + proc atomic_fetch_add_explicit[T, A](location: ptr A; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.importc: "atomic_fetch_add_explicit".maybeWrapStd.} + proc atomic_fetch_sub_explicit[T, A](location: ptr A; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.importc: "atomic_fetch_sub_explicit".maybeWrapStd.} + proc atomic_fetch_and_explicit[T, A](location: ptr A; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.importc: "atomic_fetch_and_explicit".maybeWrapStd.} + proc atomic_fetch_or_explicit[T, A](location: ptr A; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.importc: "atomic_fetch_or_explicit".maybeWrapStd.} + proc atomic_fetch_xor_explicit[T, A](location: ptr A; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.importc: "atomic_fetch_xor_explicit".maybeWrapStd.} # Flag operations # var ATOMIC_FLAG_INIT {.importc, nodecl.}: AtomicFlag # proc init*(location: var AtomicFlag) {.inline.} = location = ATOMIC_FLAG_INIT - proc testAndSet*(location: var AtomicFlag; order: MemoryOrder = moSequentiallyConsistent): bool {.importc: "atomic_flag_test_and_set_explicit".} - proc clear*(location: var AtomicFlag; order: MemoryOrder = moSequentiallyConsistent) {.importc: "atomic_flag_clear_explicit".} + proc testAndSet*(location: var AtomicFlag; order: MemoryOrder = moSequentiallyConsistent): bool {.importc: "atomic_flag_test_and_set_explicit".maybeWrapStd.} + proc clear*(location: var AtomicFlag; order: MemoryOrder = moSequentiallyConsistent) {.importc: "atomic_flag_clear_explicit".maybeWrapStd.} - proc fence*(order: MemoryOrder) {.importc: "atomic_thread_fence".} - proc signalFence*(order: MemoryOrder) {.importc: "atomic_signal_fence".} + proc fence*(order: MemoryOrder) {.importc: "atomic_thread_fence".maybeWrapStd.} + proc signalFence*(order: MemoryOrder) {.importc: "atomic_signal_fence".maybeWrapStd.} {.pop.} @@ -364,11 +379,11 @@ else: cast[T](atomic_fetch_xor_explicit(addr(location.value), cast[nonAtomicType(T)](value), order)) template withLock[T: not Trivial](location: var Atomic[T]; order: MemoryOrder; body: untyped): untyped = - while location.guard.testAndSet(moAcquire): discard + while testAndSet(location.guard, moAcquire): discard try: body finally: - location.guard.clear(moRelease) + clear(location.guard, moRelease) proc load*[T: not Trivial](location: var Atomic[T]; order: MemoryOrder = moSequentiallyConsistent): T {.inline.} = withLock(location, order): diff --git a/lib/pure/concurrency/cpuinfo.nim b/lib/pure/concurrency/cpuinfo.nim index ee43b8e11..9bc3fd579 100644 --- a/lib/pure/concurrency/cpuinfo.nim +++ b/lib/pure/concurrency/cpuinfo.nim @@ -15,87 +15,96 @@ runnableExamples: include "system/inclrtl" -when defined(posix) and not (defined(macosx) or defined(bsd)): - import posix +when defined(js): + import std/jsffi + proc countProcessorsImpl(): int = + when defined(nodejs): + let jsOs = require("os") + let jsObj = jsOs.cpus().length + else: + # `navigator.hardwareConcurrency` + # works on browser as well as deno. + let navigator{.importcpp.}: JsObject + let jsObj = navigator.hardwareConcurrency + result = jsObj.to int +else: + when defined(posix) and not (defined(macosx) or defined(bsd)): + import std/posix -when defined(freebsd) or defined(macosx): - {.emit: "#include <sys/types.h>".} + when defined(windows): + import std/private/win_getsysteminfo + + when defined(freebsd) or defined(macosx): + {.emit: "#include <sys/types.h>".} + + when defined(openbsd) or defined(netbsd): + {.emit: "#include <sys/param.h>".} + + when defined(macosx) or defined(bsd): + # we HAVE to emit param.h before sysctl.h so we cannot use .header here + # either. The amount of archaic bullshit in Poonix based OSes is just insane. + {.emit: "#include <sys/sysctl.h>".} + {.push nodecl.} + when defined(macosx): + proc sysctlbyname(name: cstring, + oldp: pointer, oldlenp: var csize_t, + newp: pointer, newlen: csize_t): cint {.importc.} + let + CTL_HW{.importc.}: cint + HW_NCPU{.importc.}: cint + proc sysctl[I: static[int]](name: var array[I, cint], namelen: cuint, + oldp: pointer, oldlenp: var csize_t, + newp: pointer, newlen: csize_t): cint {.importc.} + {.pop.} -when defined(openbsd) or defined(netbsd): - {.emit: "#include <sys/param.h>".} + when defined(genode): + import genode/env -when defined(macosx) or defined(bsd): - # we HAVE to emit param.h before sysctl.h so we cannot use .header here - # either. The amount of archaic bullshit in Poonix based OSes is just insane. - {.emit: "#include <sys/sysctl.h>".} - const - CTL_HW = 6 - HW_AVAILCPU = 25 - HW_NCPU = 3 - proc sysctl(x: ptr array[0..3, cint], y: cint, z: pointer, - a: var csize_t, b: pointer, c: csize_t): cint {. - importc: "sysctl", nodecl.} + proc affinitySpaceTotal(env: GenodeEnvPtr): cuint {. + importcpp: "@->cpu().affinity_space().total()".} -when defined(genode): - include genode/env + when defined(haiku): + type + SystemInfo {.importc: "system_info", header: "<OS.h>".} = object + cpuCount {.importc: "cpu_count".}: uint32 + + proc getSystemInfo(info: ptr SystemInfo): int32 {.importc: "get_system_info", + header: "<OS.h>".} - proc affinitySpaceTotal(env: GenodeEnvPtr): cuint {. - importcpp: "@->cpu().affinity_space().total()".} + proc countProcessorsImpl(): int {.inline.} = + when defined(windows): + var + si: SystemInfo + getSystemInfo(addr si) + result = int(si.dwNumberOfProcessors) + elif defined(macosx) or defined(bsd): + let dest = addr result + var len = sizeof(result).csize_t + when defined(macosx): + # alias of "hw.activecpu" + if sysctlbyname("hw.logicalcpu", dest, len, nil, 0) == 0: + return + var mib = [CTL_HW, HW_NCPU] + if sysctl(mib, 2, dest, len, nil, 0) == 0: + return + elif defined(hpux): + result = mpctl(MPC_GETNUMSPUS, nil, nil) + elif defined(irix): + var SC_NPROC_ONLN {.importc: "_SC_NPROC_ONLN", header: "<unistd.h>".}: cint + result = sysconf(SC_NPROC_ONLN) + elif defined(genode): + result = runtimeEnv.affinitySpaceTotal().int + elif defined(haiku): + var sysinfo: SystemInfo + if getSystemInfo(addr sysinfo) == 0: + result = sysinfo.cpuCount.int + else: + result = sysconf(SC_NPROCESSORS_ONLN) + if result < 0: result = 0 -when defined(haiku): - type - SystemInfo {.importc: "system_info", header: "<OS.h>".} = object - cpuCount {.importc: "cpu_count".}: uint32 - proc getSystemInfo(info: ptr SystemInfo): int32 {.importc: "get_system_info", - header: "<OS.h>".} proc countProcessors*(): int {.rtl, extern: "ncpi$1".} = ## Returns the number of the processors/cores the machine has. ## Returns 0 if it cannot be detected. - when defined(windows): - type - SYSTEM_INFO {.final, pure.} = object - u1: int32 - dwPageSize: int32 - lpMinimumApplicationAddress: pointer - lpMaximumApplicationAddress: pointer - dwActiveProcessorMask: ptr int32 - dwNumberOfProcessors: int32 - dwProcessorType: int32 - dwAllocationGranularity: int32 - wProcessorLevel: int16 - wProcessorRevision: int16 - - proc GetSystemInfo(lpSystemInfo: var SYSTEM_INFO) {.stdcall, dynlib: "kernel32", importc: "GetSystemInfo".} - - var - si: SYSTEM_INFO - GetSystemInfo(si) - result = si.dwNumberOfProcessors - elif defined(macosx) or defined(bsd): - var - mib: array[0..3, cint] - numCPU: int - mib[0] = CTL_HW - mib[1] = HW_AVAILCPU - var len = sizeof(numCPU).csize_t - discard sysctl(addr(mib), 2, addr(numCPU), len, nil, 0) - if numCPU < 1: - mib[1] = HW_NCPU - discard sysctl(addr(mib), 2, addr(numCPU), len, nil, 0) - result = numCPU - elif defined(hpux): - result = mpctl(MPC_GETNUMSPUS, nil, nil) - elif defined(irix): - var SC_NPROC_ONLN {.importc: "_SC_NPROC_ONLN", header: "<unistd.h>".}: cint - result = sysconf(SC_NPROC_ONLN) - elif defined(genode): - result = runtimeEnv.affinitySpaceTotal().int - elif defined(haiku): - var sysinfo: SystemInfo - if getSystemInfo(addr sysinfo) == 0: - result = sysinfo.cpuCount.int - else: - result = sysconf(SC_NPROCESSORS_ONLN) - if result <= 0: result = 0 + countProcessorsImpl() diff --git a/lib/pure/concurrency/cpuload.nim b/lib/pure/concurrency/cpuload.nim index 841d58d86..bfbf16721 100644 --- a/lib/pure/concurrency/cpuload.nim +++ b/lib/pure/concurrency/cpuload.nim @@ -13,11 +13,14 @@ ## Unstable API. when defined(windows): - import winlean, os, strutils, math + import std/[winlean, os, strutils, math] proc `-`(a, b: FILETIME): int64 = a.rdFileTime - b.rdFileTime elif defined(linux): - from cpuinfo import countProcessors + from std/cpuinfo import countProcessors + +when defined(nimPreviewSlimSystem): + import std/syncio type ThreadPoolAdvice* = enum @@ -84,7 +87,7 @@ proc advice*(s: var ThreadPoolState): ThreadPoolAdvice = inc s.calls when not defined(testing) and isMainModule and not defined(nimdoc): - import random + import std/random proc busyLoop() = while true: diff --git a/lib/pure/concurrency/threadpool.nim b/lib/pure/concurrency/threadpool.nim index 9beb39522..06ed2fe54 100644 --- a/lib/pure/concurrency/threadpool.nim +++ b/lib/pure/concurrency/threadpool.nim @@ -7,21 +7,25 @@ # distribution, for details about the copyright. # +{.deprecated: "use the nimble packages `malebolgia`, `taskpools` or `weave` instead".} + ## Implements Nim's `parallel & spawn statements <manual_experimental.html#parallel-amp-spawn>`_. ## ## Unstable API. ## ## See also ## ======== -## * `threads module <threads.html>`_ for basic thread support -## * `channels module <channels_builtin.html>`_ for message passing support +## * `threads module <typedthreads.html>`_ for basic thread support ## * `locks module <locks.html>`_ for locks and condition variables ## * `asyncdispatch module <asyncdispatch.html>`_ for asynchronous IO when not compileOption("threads"): {.error: "Threadpool requires --threads:on option.".} -import cpuinfo, cpuload, locks, os +import std/[cpuinfo, cpuload, locks, os] + +when defined(nimPreviewSlimSystem): + import std/[assertions, typedthreads, sysatomics] {.push stackTrace:off.} @@ -52,17 +56,14 @@ proc signal(cv: var Semaphore) = release(cv.L) signal(cv.c) -const CacheLineSize = 32 # true for most archs +const CacheLineSize = 64 # true for most archs type Barrier {.compilerproc.} = object entered: int cv: Semaphore # Semaphore takes 3 words at least - when sizeof(int) < 8: - cacheAlign: array[CacheLineSize-4*sizeof(int), byte] - left: int - cacheAlign2: array[CacheLineSize-sizeof(int), byte] - interest: bool # whether the master is interested in the "all done" event + left {.align(CacheLineSize).}: int + interest {.align(CacheLineSize).} : bool # whether the master is interested in the "all done" event proc barrierEnter(b: ptr Barrier) {.compilerproc, inline.} = # due to the signaling between threads, it is ensured we are the only @@ -103,7 +104,7 @@ type idx: int FlowVarBase* = ref FlowVarBaseObj ## Untyped base class for `FlowVar[T] <#FlowVar>`_. - FlowVarBaseObj = object of RootObj + FlowVarBaseObj {.acyclic.} = object of RootObj ready, usesSemaphore, awaited: bool cv: Semaphore # for 'blockUntilAny' support ai: ptr AwaitInfo @@ -112,7 +113,7 @@ type # be RootRef here otherwise the wrong GC keeps track of it! owner: pointer # ptr Worker - FlowVarObj[T] = object of FlowVarBaseObj + FlowVarObj[T] {.acyclic.} = object of FlowVarBaseObj blob: T FlowVar*[T] {.compilerproc.} = ref FlowVarObj[T] ## A data flow variable. @@ -451,19 +452,21 @@ proc preferSpawn*(): bool = ## <#spawnX.t>`_ instead. result = gSomeReady.counter > 0 -proc spawn*(call: sink typed) {.magic: "Spawn".} +proc spawn*(call: sink typed) {.magic: "Spawn".} = ## Always spawns a new task, so that the `call` is never executed on ## the calling thread. ## ## `call` has to be a proc call `p(...)` where `p` is gcsafe and has a ## return type that is either `void` or compatible with `FlowVar[T]`. + discard "It uses `nimSpawn3` internally" -proc pinnedSpawn*(id: ThreadId; call: sink typed) {.magic: "Spawn".} +proc pinnedSpawn*(id: ThreadId; call: sink typed) {.magic: "Spawn".} = ## Always spawns a new task on the worker thread with `id`, so that ## the `call` is **always** executed on the thread. ## ## `call` has to be a proc call `p(...)` where `p` is gcsafe and has a ## return type that is either `void` or compatible with `FlowVar[T]`. + discard "It uses `nimSpawn4` internally" template spawnX*(call) = ## Spawns a new task if a CPU core is ready, otherwise executes the diff --git a/lib/pure/cookies.nim b/lib/pure/cookies.nim index 8d9cc0c95..f628aaf6b 100644 --- a/lib/pure/cookies.nim +++ b/lib/pure/cookies.nim @@ -11,6 +11,9 @@ import std/[strtabs, times, options] +when defined(nimPreviewSlimSystem): + import std/assertions + type SameSite* {.pure.} = enum ## The SameSite cookie attribute. @@ -25,7 +28,7 @@ proc parseCookies*(s: string): StringTableRef = ## "Set-Cookie" header set by servers. runnableExamples: import std/strtabs - let cookieJar = parseCookies("a=1; foo=bar") + let cookieJar = parseCookies("a=1; foo=bar") assert cookieJar["a"] == "1" assert cookieJar["foo"] == "bar" @@ -46,10 +49,12 @@ proc parseCookies*(s: string): StringTableRef = proc setCookie*(key, value: string, domain = "", path = "", expires = "", noName = false, - secure = false, httpOnly = false, + secure = false, httpOnly = false, maxAge = none(int), sameSite = SameSite.Default): string = ## Creates a command in the format of ## `Set-Cookie: key=value; Domain=...; ...` + ## + ## .. tip:: Cookies can be vulnerable. Consider setting `secure=true`, `httpOnly=true` and `sameSite=Strict`. result = "" if not noName: result.add("Set-Cookie: ") result.add key & "=" & value @@ -73,4 +78,4 @@ proc setCookie*(key, value: string, expires: DateTime|Time, ## `Set-Cookie: key=value; Domain=...; ...` result = setCookie(key, value, domain, path, format(expires.utc, "ddd',' dd MMM yyyy HH:mm:ss 'GMT'"), - noname, secure, httpOnly, maxAge, sameSite) + noName, secure, httpOnly, maxAge, sameSite) diff --git a/lib/pure/coro.nim b/lib/pure/coro.nim index af8acf268..24836e316 100644 --- a/lib/pure/coro.nim +++ b/lib/pure/coro.nim @@ -8,11 +8,11 @@ # ## Nim coroutines implementation, supports several context switching methods: -## -------- ------------ +## ======== ============ ## ucontext available on unix and alike (default) ## setjmp available on unix and alike (x86/64 only) ## fibers available and required on windows. -## -------- ------------ +## ======== ============ ## ## -d:nimCoroutines Required to build this module. ## -d:nimCoroutinesUcontext Use ucontext backend. @@ -21,18 +21,22 @@ ## ## Unstable API. +import system/coro_detection + when not nimCoroutines and not defined(nimdoc): when defined(noNimCoroutines): {.error: "Coroutines can not be used with -d:noNimCoroutines".} else: {.error: "Coroutines require -d:nimCoroutines".} -import os -import lists +import std/[os, lists] include system/timers +when defined(nimPreviewSlimSystem): + import std/assertions + const defaultStackSize = 512 * 1024 -const useOrcArc = defined(gcArc) or defined(gcOrc) +const useOrcArc = defined(gcArc) or defined(gcOrc) or defined(gcAtomicArc) when useOrcArc: proc nimGC_setStackBottom*(theStackBottom: pointer) = discard @@ -63,7 +67,7 @@ else: const coroBackend = CORO_BACKEND_UCONTEXT when coroBackend == CORO_BACKEND_FIBERS: - import windows/winlean + import std/winlean type Context = pointer @@ -219,7 +223,7 @@ proc switchTo(current, to: CoroutinePtr) = elif to.state == CORO_CREATED: # Coroutine is started. coroExecWithStack(runCurrentTask, to.stack.bottom) - #doAssert false + #raiseAssert "unreachable" else: {.error: "Invalid coroutine backend set.".} # Execution was just resumed. Restore frame information and set active stack. @@ -261,7 +265,7 @@ proc runCurrentTask() = current.state = CORO_FINISHED nimGC_setStackBottom(ctx.ncbottom) suspend(0) # Exit coroutine without returning from coroExecWithStack() - doAssert false + raiseAssert "unreachable" proc start*(c: proc(), stacksize: int = defaultStackSize): CoroutineRef {.discardable.} = ## Schedule coroutine for execution. It does not run immediately. @@ -274,11 +278,10 @@ proc start*(c: proc(), stacksize: int = defaultStackSize): CoroutineRef {.discar coro.execContext = CreateFiberEx(stacksize, stacksize, FIBER_FLAG_FLOAT_SWITCH, (proc(p: pointer) {.stdcall.} = runCurrentTask()), nil) - coro.stack.size = stacksize else: coro = cast[CoroutinePtr](alloc0(sizeof(Coroutine) + stacksize)) - coro.stack.top = cast[pointer](cast[ByteAddress](coro) + sizeof(Coroutine)) - coro.stack.bottom = cast[pointer](cast[ByteAddress](coro.stack.top) + stacksize) + coro.stack.top = cast[pointer](cast[int](coro) + sizeof(Coroutine)) + coro.stack.bottom = cast[pointer](cast[int](coro.stack.top) + stacksize) when coroBackend == CORO_BACKEND_UCONTEXT: discard getcontext(coro.execContext) coro.execContext.uc_stack.ss_sp = coro.stack.top @@ -293,9 +296,9 @@ proc start*(c: proc(), stacksize: int = defaultStackSize): CoroutineRef {.discar return coro.reference proc run*() = - initialize() ## Starts main coroutine scheduler loop which exits when all coroutines exit. ## Calling this proc starts execution of first coroutine. + initialize() ctx.current = ctx.coroutines.head var minDelay: float = 0 while ctx.current != nil: diff --git a/lib/pure/cstrutils.nim b/lib/pure/cstrutils.nim index 0eae00fba..c907e54d8 100644 --- a/lib/pure/cstrutils.nim +++ b/lib/pure/cstrutils.nim @@ -75,7 +75,7 @@ func cmpIgnoreStyle*(a, b: cstring): int {.rtl, extern: "csuCmpIgnoreStyle".} = ## for that. Returns: ## * 0 if `a == b` ## * < 0 if `a < b` - ## * > 0 if `a > b` + ## * \> 0 if `a > b` runnableExamples: assert cmpIgnoreStyle(cstring"hello", cstring"H_e_L_Lo") == 0 @@ -101,7 +101,7 @@ func cmpIgnoreCase*(a, b: cstring): int {.rtl, extern: "csuCmpIgnoreCase".} = ## Compares two strings in a case insensitive manner. Returns: ## * 0 if `a == b` ## * < 0 if `a < b` - ## * > 0 if `a > b` + ## * \> 0 if `a > b` runnableExamples: assert cmpIgnoreCase(cstring"hello", cstring"HeLLo") == 0 assert cmpIgnoreCase(cstring"echo", cstring"hello") < 0 diff --git a/lib/pure/db_common.nim b/lib/pure/db_common.nim deleted file mode 100644 index 852c8e1c3..000000000 --- a/lib/pure/db_common.nim +++ /dev/null @@ -1,100 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## Common datatypes and definitions for all `db_*.nim` ( -## `db_mysql <db_mysql.html>`_, `db_postgres <db_postgres.html>`_, -## and `db_sqlite <db_sqlite.html>`_) modules. - -type - DbError* = object of IOError ## exception that is raised if a database error occurs - - SqlQuery* = distinct string ## an SQL query string - - - DbEffect* = object of IOEffect ## effect that denotes a database operation - ReadDbEffect* = object of DbEffect ## effect that denotes a read operation - WriteDbEffect* = object of DbEffect ## effect that denotes a write operation - - DbTypeKind* = enum ## a superset of datatypes that might be supported. - dbUnknown, ## unknown datatype - dbSerial, ## datatype used for primary auto-increment keys - dbNull, ## datatype used for the NULL value - dbBit, ## bit datatype - dbBool, ## boolean datatype - dbBlob, ## blob datatype - dbFixedChar, ## string of fixed length - dbVarchar, ## string datatype - dbJson, ## JSON datatype - dbXml, ## XML datatype - dbInt, ## some integer type - dbUInt, ## some unsigned integer type - dbDecimal, ## decimal numbers (fixed-point number) - dbFloat, ## some floating point type - dbDate, ## a year-month-day description - dbTime, ## HH:MM:SS information - dbDatetime, ## year-month-day and HH:MM:SS information, - ## plus optional time or timezone information - dbTimestamp, ## Timestamp values are stored as the number of seconds - ## since the epoch ('1970-01-01 00:00:00' UTC). - dbTimeInterval, ## an interval [a,b] of times - dbEnum, ## some enum - dbSet, ## set of enum values - dbArray, ## an array of values - dbComposite, ## composite type (record, struct, etc) - dbUrl, ## a URL - dbUuid, ## a UUID - dbInet, ## an IP address - dbMacAddress, ## a MAC address - dbGeometry, ## some geometric type - dbPoint, ## Point on a plane (x,y) - dbLine, ## Infinite line ((x1,y1),(x2,y2)) - dbLseg, ## Finite line segment ((x1,y1),(x2,y2)) - dbBox, ## Rectangular box ((x1,y1),(x2,y2)) - dbPath, ## Closed or open path (similar to polygon) ((x1,y1),...) - dbPolygon, ## Polygon (similar to closed path) ((x1,y1),...) - dbCircle, ## Circle <(x,y),r> (center point and radius) - dbUser1, ## user definable datatype 1 (for unknown extensions) - dbUser2, ## user definable datatype 2 (for unknown extensions) - dbUser3, ## user definable datatype 3 (for unknown extensions) - dbUser4, ## user definable datatype 4 (for unknown extensions) - dbUser5 ## user definable datatype 5 (for unknown extensions) - - DbType* = object ## describes a database type - kind*: DbTypeKind ## the kind of the described type - notNull*: bool ## does the type contain NULL? - name*: string ## the name of the type - size*: Natural ## the size of the datatype; 0 if of variable size - maxReprLen*: Natural ## maximal length required for the representation - precision*, scale*: Natural ## precision and scale of the number - min*, max*: BiggestInt ## the minimum and maximum of allowed values - validValues*: seq[string] ## valid values of an enum or a set - - DbColumn* = object ## information about a database column - name*: string ## name of the column - tableName*: string ## name of the table the column belongs to (optional) - typ*: DbType ## type of the column - primaryKey*: bool ## is this a primary key? - foreignKey*: bool ## is this a foreign key? - DbColumns* = seq[DbColumn] - -template sql*(query: string): SqlQuery = - ## constructs a SqlQuery from the string `query`. This is supposed to be - ## used as a raw-string-literal modifier: - ## `sql"update user set counter = counter + 1"` - ## - ## If assertions are turned off, it does nothing. If assertions are turned - ## on, later versions will check the string for valid syntax. - SqlQuery(query) - -proc dbError*(msg: string) {.noreturn, noinline.} = - ## raises an DbError exception with message `msg`. - var e: ref DbError - new(e) - e.msg = msg - raise e diff --git a/lib/pure/distros.nim b/lib/pure/distros.nim index 2b119b92c..9e71d4ce0 100644 --- a/lib/pure/distros.nim +++ b/lib/pure/distros.nim @@ -9,21 +9,20 @@ ## This module implements the basics for Linux distribution ("distro") ## detection and the OS's native package manager. Its primary purpose is to -## produce output for Nimble packages, like:: +## produce output for Nimble packages, like: ## -## To complete the installation, run: +## To complete the installation, run: ## -## sudo apt-get install libblas-dev -## sudo apt-get install libvoodoo +## sudo apt-get install libblas-dev +## sudo apt-get install libvoodoo ## ## The above output could be the result of a code snippet like: ## -## .. code-block:: nim -## +## ```nim ## if detectOs(Ubuntu): ## foreignDep "lbiblas-dev" ## foreignDep "libvoodoo" -## +## ``` ## ## See `packaging <packaging.html>`_ for hints on distributing Nim using OS packages. @@ -31,7 +30,7 @@ from std/strutils import contains, toLowerAscii when not defined(nimscript): from std/osproc import execProcess - from std/os import existsEnv + from std/envvars import existsEnv type Distribution* {.pure.} = enum ## the list of known distributions @@ -52,6 +51,7 @@ type CentOS Deepin ArchLinux + Artix Antergos PCLinuxOS Mageia @@ -108,7 +108,7 @@ type Clonezilla SteamOS Absolute - NixOS ## NixOS or a Nix build environment + NixOS ## NixOS or a Nix build environment AUSTRUMI Arya Porteus @@ -123,9 +123,11 @@ type ExTiX Rockstor GoboLinux + Void BSD FreeBSD + NetBSD OpenBSD DragonFlyBSD @@ -133,7 +135,9 @@ type const - LacksDevPackages* = {Distribution.Gentoo, Distribution.Slackware, Distribution.ArchLinux} + LacksDevPackages* = {Distribution.Gentoo, Distribution.Slackware, + Distribution.ArchLinux, Distribution.Artix, Distribution.Antergos, + Distribution.BlackArch, Distribution.ArchBang} # we cache the result of the 'cmdRelease' # execution for faster platform detections. @@ -152,7 +156,8 @@ template hostnamectl(): untyped = cmdRelease("hostnamectl", hostnamectlRes) proc detectOsWithAllCmd(d: Distribution): bool = let dd = toLowerAscii($d) result = dd in toLowerAscii(osReleaseID()) or dd in toLowerAscii(release()) or - dd in toLowerAscii(uname()) or ("operating system: " & dd) in toLowerAscii(hostnamectl()) + dd in toLowerAscii(uname()) or ("operating system: " & dd) in + toLowerAscii(hostnamectl()) proc detectOsImpl(d: Distribution): bool = case d @@ -164,7 +169,7 @@ proc detectOsImpl(d: Distribution): bool = else: when defined(bsd): case d - of Distribution.FreeBSD, Distribution.OpenBSD: + of Distribution.FreeBSD, Distribution.NetBSD, Distribution.OpenBSD: result = $d in uname() else: result = false @@ -172,14 +177,16 @@ proc detectOsImpl(d: Distribution): bool = case d of Distribution.Gentoo: result = ("-" & $d & " ") in uname() - of Distribution.Elementary, Distribution.Ubuntu, Distribution.Debian, Distribution.Fedora, - Distribution.OpenMandriva, Distribution.CentOS, Distribution.Alpine, - Distribution.Mageia, Distribution.Zorin: + of Distribution.Elementary, Distribution.Ubuntu, Distribution.Debian, + Distribution.Fedora, Distribution.OpenMandriva, Distribution.CentOS, + Distribution.Alpine, Distribution.Mageia, Distribution.Zorin, Distribution.Void: result = toLowerAscii($d) in osReleaseID() of Distribution.RedHat: result = "rhel" in osReleaseID() of Distribution.ArchLinux: result = "arch" in osReleaseID() + of Distribution.Artix: + result = "artix" in osReleaseID() of Distribution.NixOS: # Check if this is a Nix build or NixOS environment result = existsEnv("NIX_BUILD_TOP") or existsEnv("__NIXOS_SET_ENVIRONMENT_DONE") @@ -204,7 +211,7 @@ template detectOs*(d: untyped): bool = detectOsImpl(Distribution.d) when not defined(nimble): - var foreignDeps: seq[string] = @[] + var foreignDeps*: seq[string] = @[] ## Registered foreign deps. proc foreignCmd*(cmd: string; requiresSudo = false) = ## Registers a foreign command to the internal list of commands @@ -245,12 +252,14 @@ proc foreignDepInstallCmd*(foreignPackageName: string): (string, bool) = result = ("nix-env -i " & p, false) elif detectOs(Solaris) or detectOs(FreeBSD): result = ("pkg install " & p, true) - elif detectOs(OpenBSD): + elif detectOs(NetBSD) or detectOs(OpenBSD): result = ("pkg_add " & p, true) elif detectOs(PCLinuxOS): result = ("rpm -ivh " & p, true) - elif detectOs(ArchLinux) or detectOs(Manjaro): + elif detectOs(ArchLinux) or detectOs(Manjaro) or detectOs(Artix): result = ("pacman -S " & p, true) + elif detectOs(Void): + result = ("xbps-install " & p, true) else: result = ("<your package manager here> install " & p, true) elif defined(haiku): diff --git a/lib/pure/dynlib.nim b/lib/pure/dynlib.nim index 3e1d84729..a162fe37f 100644 --- a/lib/pure/dynlib.nim +++ b/lib/pure/dynlib.nim @@ -17,75 +17,64 @@ ## Loading a simple C function ## --------------------------- ## -## The following example demonstrates loading a function called 'greet' +## The following example demonstrates loading a function called `greet` ## from a library that is determined at runtime based upon a language choice. -## If the library fails to load or the function 'greet' is not found, +## If the library fails to load or the function `greet` is not found, ## it quits with a failure error code. ## -## .. code-block::nim -## -## import std/dynlib -## -## type -## greetFunction = proc(): cstring {.gcsafe, stdcall.} -## -## let lang = stdin.readLine() -## -## let lib = case lang -## of "french": -## loadLib("french.dll") -## else: -## loadLib("english.dll") -## -## if lib == nil: -## echo "Error loading library" -## quit(QuitFailure) -## -## let greet = cast[greetFunction](lib.symAddr("greet")) -## -## if greet == nil: -## echo "Error loading 'greet' function from library" -## quit(QuitFailure) -## -## let greeting = greet() -## -## echo greeting -## -## unloadLib(lib) -## +runnableExamples: + type + GreetFunction = proc (): cstring {.gcsafe, stdcall.} + + proc loadGreet(lang: string) = + let lib = + case lang + of "french": + loadLib("french.dll") + else: + loadLib("english.dll") + assert lib != nil, "Error loading library" + + let greet = cast[GreetFunction](lib.symAddr("greet")) + assert greet != nil, "Error loading 'greet' function from library" + + echo greet() + + unloadLib(lib) -import strutils + +import std/strutils type - LibHandle* = pointer ## a handle to a dynamically loaded library + LibHandle* = pointer ## A handle to a dynamically loaded library. proc loadLib*(path: string, globalSymbols = false): LibHandle {.gcsafe.} - ## loads a library from `path`. Returns nil if the library could not + ## Loads a library from `path`. Returns nil if the library could not ## be loaded. proc loadLib*(): LibHandle {.gcsafe.} - ## gets the handle from the current executable. Returns nil if the + ## Gets the handle from the current executable. Returns nil if the ## library could not be loaded. proc unloadLib*(lib: LibHandle) {.gcsafe.} - ## unloads the library `lib` + ## Unloads the library `lib`. proc raiseInvalidLibrary*(name: cstring) {.noinline, noreturn.} = - ## raises an `EInvalidLibrary` exception. + ## Raises a `LibraryError` exception. raise newException(LibraryError, "could not find symbol: " & $name) proc symAddr*(lib: LibHandle, name: cstring): pointer {.gcsafe.} - ## retrieves the address of a procedure/variable from `lib`. Returns nil + ## Retrieves the address of a procedure/variable from `lib`. Returns nil ## if the symbol could not be found. proc checkedSymAddr*(lib: LibHandle, name: cstring): pointer = - ## retrieves the address of a procedure/variable from `lib`. Raises - ## `EInvalidLibrary` if the symbol could not be found. + ## Retrieves the address of a procedure/variable from `lib`. Raises + ## `LibraryError` if the symbol could not be found. result = symAddr(lib, name) if result == nil: raiseInvalidLibrary(name) proc libCandidates*(s: string, dest: var seq[string]) = - ## given a library name pattern `s` write possible library names to `dest`. + ## Given a library name pattern `s`, write possible library names to `dest`. var le = strutils.find(s, '(') var ri = strutils.find(s, ')', le+1) if le >= 0 and ri > le: @@ -97,8 +86,9 @@ proc libCandidates*(s: string, dest: var seq[string]) = add(dest, s) proc loadLibPattern*(pattern: string, globalSymbols = false): LibHandle = - ## loads a library with name matching `pattern`, similar to what `dynlib` + ## Loads a library with name matching `pattern`, similar to what the `dynlib` ## pragma does. Returns nil if the library could not be loaded. + ## ## .. warning:: this proc uses the GC and so cannot be used to load the GC. var candidates = newSeq[string]() libCandidates(pattern, candidates) @@ -115,7 +105,7 @@ when defined(posix) and not defined(nintendoswitch): # as an emulation layer on top of native functions. # ========================================================================= # - import posix + import std/posix proc loadLib(path: string, globalSymbols = false): LibHandle = let flags = @@ -150,6 +140,32 @@ elif defined(nintendoswitch): proc symAddr(lib: LibHandle, name: cstring): pointer = raise newException(OSError, "symAddr not implemented on Nintendo Switch!") +elif defined(genode): + # + # ========================================================================= + # Not implemented for Genode without POSIX. Raise an error if called. + # ========================================================================= + # + + template raiseErr(prc: string) = + raise newException(OSError, prc & " not implemented, compile with POSIX support") + + proc dlclose(lib: LibHandle) = + raiseErr(OSError, "dlclose") + proc dlopen(path: cstring, mode: int): LibHandle = + raiseErr(OSError, "dlopen") + proc dlsym(lib: LibHandle, name: cstring): pointer = + raiseErr(OSError, "dlsym") + proc loadLib(path: string, global_symbols = false): LibHandle = + raiseErr(OSError, "loadLib") + proc loadLib(): LibHandle = + raiseErr(OSError, "loadLib") + proc unloadLib(lib: LibHandle) = + raiseErr(OSError, "unloadLib") + proc symAddr(lib: LibHandle, name: cstring): pointer = + raiseErr(OSError, "symAddr") + + elif defined(windows) or defined(dos): # # ======================================================================= diff --git a/lib/pure/encodings.nim b/lib/pure/encodings.nim index 1d8512018..bbadca655 100644 --- a/lib/pure/encodings.nim +++ b/lib/pure/encodings.nim @@ -7,15 +7,46 @@ # distribution, for details about the copyright. # -## Converts between different character encodings. On UNIX, this uses +## Routines for converting between different character encodings. On UNIX, this uses ## the `iconv`:idx: library, on Windows the Windows API. +## +## The following example shows how to change character encodings. +runnableExamples: + when defined(windows): + let + orig = "öäüß" + # convert `orig` from "UTF-8" to "CP1252" + cp1252 = convert(orig, "CP1252", "UTF-8") + # convert `cp1252` from "CP1252" to "ibm850" + ibm850 = convert(cp1252, "ibm850", "CP1252") + current = getCurrentEncoding() + assert orig == "\195\182\195\164\195\188\195\159" + assert ibm850 == "\148\132\129\225" + assert convert(ibm850, current, "ibm850") == orig + +## The example below uses a reuseable `EncodingConverter` object which is +## created by `open` with `destEncoding` and `srcEncoding` specified. You can use +## `convert` on this object multiple times. +runnableExamples: + when defined(windows): + var fromGB2312 = open("utf-8", "gb2312") + let first = "\203\173\197\194\163\191\210\187" & + "\203\242\209\204\211\234\200\206\198\189\201\250" + assert fromGB2312.convert(first) == "谁怕？一蓑烟雨任平生" + + let second = "\211\208\176\215\205\183\200\231" & + "\208\194\163\172\199\227\184\199\200\231\185\202" + assert fromGB2312.convert(second) == "有白头如新，倾盖如故" -import os + +import std/os +when defined(nimPreviewSlimSystem): + import std/assertions when not defined(windows): type ConverterObj = object - EncodingConverter* = ptr ConverterObj ## can convert between two character sets + EncodingConverter* = ptr ConverterObj ## Can convert between two character sets. else: type @@ -24,11 +55,11 @@ else: dest, src: CodePage type - EncodingError* = object of ValueError ## exception that is raised - ## for encoding errors + EncodingError* = object of ValueError ## Exception that is raised + ## for encoding errors. when defined(windows): - import parseutils, strutils + import std/[parseutils, strutils] proc eqEncodingNames(a, b: string): bool = var i = 0 var j = 0 @@ -72,6 +103,7 @@ when defined(windows): (875, "cp875"), # IBM EBCDIC Greek Modern (932, "shift_jis"), # ANSI/OEM Japanese; Japanese (Shift-JIS) (936, "gb2312"), # ANSI/OEM Simplified Chinese (PRC, Singapore); Chinese Simplified (GB2312) + (936, "gbk"), # Alias for GB2312 encoding (949, "ks_c_5601-1987"), # ANSI/OEM Korean (Unified Hangul Code) (950, "big5"), # ANSI/OEM Traditional Chinese (Taiwan; Hong Kong SAR, PRC); Chinese Traditional (Big5) (1026, "IBM1026"), # IBM EBCDIC Turkish (Latin 5) @@ -281,8 +313,10 @@ else: var errno {.importc, header: "<errno.h>".}: cint - when defined(freebsd) or defined(netbsd): + when defined(bsd): {.pragma: importIconv, cdecl, header: "<iconv.h>".} + when defined(openbsd): + {.passL: "-liconv".} else: {.pragma: importIconv, cdecl, dynlib: iconvDll.} @@ -295,7 +329,7 @@ else: importc: "iconv", importIconv.} proc getCurrentEncoding*(uiApp = false): string = - ## retrieves the current encoding. On Unix, always "UTF-8" is returned. + ## Retrieves the current encoding. On Unix, "UTF-8" is always returned. ## The `uiApp` parameter is Windows specific. If true, the UI's code-page ## is returned, if false, the Console's code-page is returned. when defined(windows): @@ -304,11 +338,11 @@ proc getCurrentEncoding*(uiApp = false): string = result = "UTF-8" proc open*(destEncoding = "UTF-8", srcEncoding = "CP1252"): EncodingConverter = - ## opens a converter that can convert from `srcEncoding` to `destEncoding`. - ## Raises `IOError` if it cannot fulfill the request. + ## Opens a converter that can convert from `srcEncoding` to `destEncoding`. + ## Raises `EncodingError` if it cannot fulfill the request. when not defined(windows): result = iconvOpen(destEncoding, srcEncoding) - if result == nil: + if result == cast[EncodingConverter](-1): raise newException(EncodingError, "cannot create encoding converter from " & srcEncoding & " to " & destEncoding) @@ -323,7 +357,7 @@ proc open*(destEncoding = "UTF-8", srcEncoding = "CP1252"): EncodingConverter = "cannot find encoding " & srcEncoding) proc close*(c: EncodingConverter) = - ## frees the resources the converter `c` holds. + ## Frees the resources the converter `c` holds. when not defined(windows): iconvClose(c) @@ -418,12 +452,13 @@ when defined(windows): else: convertFromWideString(codePageTo, wideString) proc convert*(c: EncodingConverter, s: string): string = - ## converts `s` to `destEncoding` that was given to the converter `c`. It - ## assumed that `s` is in `srcEncoding`. - ## utf-16BE, utf-32 conversions not supported on windows result = convertWin(c.src, c.dest, s) else: proc convert*(c: EncodingConverter, s: string): string = + ## Converts `s` to `destEncoding` that was given to the converter `c`. It + ## assumes that `s` is in `srcEncoding`. + ## + ## .. warning:: UTF-16BE and UTF-32 conversions are not supported on Windows. result = newString(s.len) var inLen = csize_t len(s) var outLen = csize_t len(result) @@ -464,84 +499,13 @@ else: proc convert*(s: string, destEncoding = "UTF-8", srcEncoding = "CP1252"): string = - ## converts `s` to `destEncoding`. It assumed that `s` is in `srcEncoding`. + ## Converts `s` to `destEncoding`. It assumed that `s` is in `srcEncoding`. ## This opens a converter, uses it and closes it again and is thus more ## convenient but also likely less efficient than re-using a converter. - ## utf-16BE, utf-32 conversions not supported on windows + ## + ## .. warning:: UTF-16BE and UTF-32 conversions are not supported on Windows. var c = open(destEncoding, srcEncoding) try: result = convert(c, s) finally: close(c) - -when not defined(testing) and isMainModule: - let - orig = "öäüß" - cp1252 = convert(orig, "CP1252", "UTF-8") - ibm850 = convert(cp1252, "ibm850", "CP1252") - current = getCurrentEncoding() - echo "Original string from source code: ", orig - echo "Forced ibm850 encoding: ", ibm850 - echo "Current encoding: ", current - echo "From ibm850 to current: ", convert(ibm850, current, "ibm850") - -when not defined(testing) and isMainModule and defined(windows): - block should_throw_on_unsupported_conversions: - let original = "some string" - - doAssertRaises(EncodingError): - discard convert(original, "utf-8", "utf-32") - - doAssertRaises(EncodingError): - discard convert(original, "utf-8", "unicodeFFFE") - - doAssertRaises(EncodingError): - discard convert(original, "utf-8", "utf-32BE") - - doAssertRaises(EncodingError): - discard convert(original, "unicodeFFFE", "utf-8") - - doAssertRaises(EncodingError): - discard convert(original, "utf-32", "utf-8") - - doAssertRaises(EncodingError): - discard convert(original, "utf-32BE", "utf-8") - - block should_convert_from_utf16_to_utf8: - let original = "\x42\x04\x35\x04\x41\x04\x42\x04" # utf-16 little endian test string "тест" - let result = convert(original, "utf-8", "utf-16") - doAssert(result == "\xd1\x82\xd0\xb5\xd1\x81\xd1\x82") - - block should_convert_from_utf16_to_win1251: - let original = "\x42\x04\x35\x04\x41\x04\x42\x04" # utf-16 little endian test string "тест" - let result = convert(original, "windows-1251", "utf-16") - doAssert(result == "\xf2\xe5\xf1\xf2") - - block should_convert_from_win1251_to_koi8r: - let original = "\xf2\xe5\xf1\xf2" # win1251 test string "тест" - let result = convert(original, "koi8-r", "windows-1251") - doAssert(result == "\xd4\xc5\xd3\xd4") - - block should_convert_from_koi8r_to_win1251: - let original = "\xd4\xc5\xd3\xd4" # koi8r test string "тест" - let result = convert(original, "windows-1251", "koi8-r") - doAssert(result == "\xf2\xe5\xf1\xf2") - - block should_convert_from_utf8_to_win1251: - let original = "\xd1\x82\xd0\xb5\xd1\x81\xd1\x82" # utf-8 test string "тест" - let result = convert(original, "windows-1251", "utf-8") - doAssert(result == "\xf2\xe5\xf1\xf2") - - block should_convert_from_utf8_to_utf16: - let original = "\xd1\x82\xd0\xb5\xd1\x81\xd1\x82" # utf-8 test string "тест" - let result = convert(original, "utf-16", "utf-8") - doAssert(result == "\x42\x04\x35\x04\x41\x04\x42\x04") - - block should_handle_empty_string_for_any_conversion: - let original = "" - var result = convert(original, "utf-16", "utf-8") - doAssert(result == "") - result = convert(original, "utf-8", "utf-16") - doAssert(result == "") - result = convert(original, "windows-1251", "koi8-r") - doAssert(result == "") diff --git a/lib/pure/endians.nim b/lib/pure/endians.nim index 29fde1c80..4c1d45ae5 100644 --- a/lib/pure/endians.nim +++ b/lib/pure/endians.nim @@ -10,6 +10,11 @@ ## This module contains helpers that deal with different byte orders ## (`endian`:idx:). ## +## Endianness is the order of bytes of a value in memory. Big-endian means that +## the most significant byte is stored at the smallest memory address, +## while little endian means that the least-significant byte is stored +## at the smallest address. See also https://en.wikipedia.org/wiki/Endianness. +## ## Unstable API. when defined(gcc) or defined(llvm_gcc) or defined(clang): @@ -47,7 +52,7 @@ else: when useBuiltinSwap: template swapOpImpl(T: typedesc, op: untyped) = - ## We have to use `copyMem` here instead of a simple deference because they + ## We have to use `copyMem` here instead of a simple dereference because they ## may point to a unaligned address. A sufficiently smart compiler _should_ ## be able to elide them when they're not necessary. var tmp: T @@ -56,18 +61,40 @@ when useBuiltinSwap: copyMem(outp, addr tmp, sizeof(T)) proc swapEndian64*(outp, inp: pointer) {.inline, noSideEffect.} = + ## Copies `inp` to `outp`, reversing the byte order. + ## Both buffers are supposed to contain at least 8 bytes. + runnableExamples: + var a = [1'u8, 2, 3, 4, 5, 6, 7, 8] + var b: array[8, uint8] + swapEndian64(addr b, addr a) + assert b == [8'u8, 7, 6, 5, 4, 3, 2, 1] + swapOpImpl(uint64, builtin_bswap64) proc swapEndian32*(outp, inp: pointer) {.inline, noSideEffect.} = + ## Copies `inp` to `outp`, reversing the byte order. + ## Both buffers are supposed to contain at least 4 bytes. + runnableExamples: + var a = [1'u8, 2, 3, 4] + var b: array[4, uint8] + swapEndian32(addr b, addr a) + assert b == [4'u8, 3, 2, 1] + swapOpImpl(uint32, builtin_bswap32) proc swapEndian16*(outp, inp: pointer) {.inline, noSideEffect.} = + ## Copies `inp` to `outp`, reversing the byte order. + ## Both buffers are supposed to contain at least 2 bytes. + runnableExamples: + var a = [1'u8, 2] + var b: array[2, uint8] + swapEndian16(addr b, addr a) + assert b == [2'u8, 1] + swapOpImpl(uint16, builtin_bswap16) else: proc swapEndian64*(outp, inp: pointer) = - ## copies `inp` to `outp` swapping bytes. Both buffers are supposed to - ## contain at least 8 bytes. var i = cast[cstring](inp) var o = cast[cstring](outp) o[0] = i[7] @@ -80,8 +107,6 @@ else: o[7] = i[0] proc swapEndian32*(outp, inp: pointer) = - ## copies `inp` to `outp` swapping bytes. Both buffers are supposed to - ## contain at least 4 bytes. var i = cast[cstring](inp) var o = cast[cstring](outp) o[0] = i[3] @@ -90,8 +115,6 @@ else: o[3] = i[0] proc swapEndian16*(outp, inp: pointer) = - ## copies `inp` to `outp` swapping bytes. Both buffers are supposed to - ## contain at least 2 bytes. var i = cast[cstring](inp) var o = cast[cstring](outp) o[0] = i[1] @@ -106,8 +129,20 @@ when system.cpuEndian == bigEndian: proc bigEndian16*(outp, inp: pointer) {.inline.} = copyMem(outp, inp, 2) else: proc littleEndian64*(outp, inp: pointer) {.inline.} = copyMem(outp, inp, 8) + ## Copies `inp` to `outp`, storing it in 64-bit little-endian order. + ## Both buffers are supposed to contain at least 8 bytes. proc littleEndian32*(outp, inp: pointer) {.inline.} = copyMem(outp, inp, 4) + ## Copies `inp` to `outp`, storing it in 32-bit little-endian order. + ## Both buffers are supposed to contain at least 4 bytes. proc littleEndian16*(outp, inp: pointer){.inline.} = copyMem(outp, inp, 2) + ## Copies `inp` to `outp`, storing it in 16-bit little-endian order. + ## Both buffers are supposed to contain at least 2 bytes. proc bigEndian64*(outp, inp: pointer) {.inline.} = swapEndian64(outp, inp) + ## Copies `inp` to `outp`, storing it in 64-bit big-endian order. + ## Both buffers are supposed to contain at least 8 bytes. proc bigEndian32*(outp, inp: pointer) {.inline.} = swapEndian32(outp, inp) + ## Copies `inp` to `outp`, storing it in 32-bit big-endian order. + ## Both buffers are supposed to contain at least 4 bytes. proc bigEndian16*(outp, inp: pointer) {.inline.} = swapEndian16(outp, inp) + ## Copies `inp` to `outp`, storing it in 16-bit big-endian order. + ## Both buffers are supposed to contain at least 2 bytes. diff --git a/lib/pure/fenv.nim b/lib/pure/fenv.nim index ddb6a1a0c..1d96fd6be 100644 --- a/lib/pure/fenv.nim +++ b/lib/pure/fenv.nim @@ -12,7 +12,7 @@ ## The types, vars and procs are bindings for the C standard library ## [<fenv.h>](https://en.cppreference.com/w/c/numeric/fenv) header. -when defined(posix) and not defined(genode): +when defined(posix) and not defined(genode) and not defined(macosx): {.passl: "-lm".} var diff --git a/lib/pure/future.nim b/lib/pure/future.nim deleted file mode 100644 index 40dba7846..000000000 --- a/lib/pure/future.nim +++ /dev/null @@ -1,4 +0,0 @@ - -{.deprecated: "Use the new 'sugar' module instead".} - -include sugar diff --git a/lib/pure/hashes.nim b/lib/pure/hashes.nim index c526c976f..1038d55a1 100644 --- a/lib/pure/hashes.nim +++ b/lib/pure/hashes.nim @@ -21,7 +21,7 @@ runnableExamples: foo: int bar: string - iterator items(x: Something): int = + iterator items(x: Something): Hash = yield hash(x.foo) yield hash(x.bar) @@ -51,18 +51,26 @@ runnableExamples: h = h !& hash(x.bar) result = !$h -## **Note:** If the type has a `==` operator, the following must hold: -## If two values compare equal, their hashes must also be equal. +## .. important:: Use `-d:nimPreviewHashRef` to +## enable hashing `ref`s. It is expected that this behavior +## becomes the new default in upcoming versions. +## +## .. note:: If the type has a `==` operator, the following must hold: +## If two values compare equal, their hashes must also be equal. ## ## See also ## ======== ## * `md5 module <md5.html>`_ for the MD5 checksum algorithm ## * `base64 module <base64.html>`_ for a Base64 encoder and decoder -## * `std/sha1 module <sha1.html>`_ for a SHA-1 encoder and decoder +## * `sha1 module <sha1.html>`_ for the SHA-1 checksum algorithm ## * `tables module <tables.html>`_ for hash tables import std/private/since +when defined(nimPreviewSlimSystem): + import std/assertions + + type Hash* = int ## A hash value. Hash tables using these values should ## always have a size of a power of two so they can use the `and` @@ -182,7 +190,7 @@ proc hashData*(data: pointer, size: int): Hash = var h: Hash = 0 when defined(js): var p: cstring - asm """`p` = `Data`;""" + {.emit: """`p` = `Data`;""".} else: var p = cast[cstring](data) var i = 0 @@ -193,13 +201,23 @@ proc hashData*(data: pointer, size: int): Hash = dec(s) result = !$h +proc hashIdentity*[T: Ordinal|enum](x: T): Hash {.inline, since: (1, 3).} = + ## The identity hash, i.e. `hashIdentity(x) = x`. + cast[Hash](ord(x)) + +when defined(nimIntHash1): + proc hash*[T: Ordinal|enum](x: T): Hash {.inline.} = + ## Efficient hashing of integers. + cast[Hash](ord(x)) +else: + proc hash*[T: Ordinal|enum](x: T): Hash {.inline.} = + ## Efficient hashing of integers. + hashWangYi1(uint64(ord(x))) + when defined(js): var objectID = 0 - -proc hash*(x: pointer): Hash {.inline.} = - ## Efficient hashing of pointers. - when defined(js): - asm """ + proc getObjectId(x: pointer): int = + {.emit: """ if (typeof `x` == "object") { if ("_NimID" in `x`) `result` = `x`["_NimID"]; @@ -208,29 +226,48 @@ proc hash*(x: pointer): Hash {.inline.} = `x`["_NimID"] = `result`; } } - """ - else: - result = cast[Hash](cast[uint](x) shr 3) # skip the alignment + """.} -proc hash*[T: proc](x: T): Hash {.inline.} = - ## Efficient hashing of proc vars. Closures are supported too. - when T is "closure": - result = hash(rawProc(x)) !& hash(rawEnv(x)) +proc hash*(x: pointer): Hash {.inline.} = + ## Efficient `hash` overload. + when defined(js): + let y = getObjectId(x) else: - result = hash(pointer(x)) - -proc hashIdentity*[T: Ordinal|enum](x: T): Hash {.inline, since: (1, 3).} = - ## The identity hash, i.e. `hashIdentity(x) = x`. - cast[Hash](ord(x)) + let y = cast[int](x) + hash(y) # consistent with code expecting scrambled hashes depending on `nimIntHash1`. -when defined(nimIntHash1): - proc hash*[T: Ordinal|enum](x: T): Hash {.inline.} = - ## Efficient hashing of integers. - cast[Hash](ord(x)) -else: - proc hash*[T: Ordinal|enum](x: T): Hash {.inline.} = - ## Efficient hashing of integers. - hashWangYi1(uint64(ord(x))) +proc hash*[T](x: ptr[T]): Hash {.inline.} = + ## Efficient `hash` overload. + runnableExamples: + var a: array[10, uint8] + assert a[0].addr.hash != a[1].addr.hash + assert cast[pointer](a[0].addr).hash == a[0].addr.hash + hash(cast[pointer](x)) + +when defined(nimPreviewHashRef) or defined(nimdoc): + proc hash*[T](x: ref[T]): Hash {.inline.} = + ## Efficient `hash` overload. + ## + ## .. important:: Use `-d:nimPreviewHashRef` to + ## enable hashing `ref`s. It is expected that this behavior + ## becomes the new default in upcoming versions. + runnableExamples("-d:nimPreviewHashRef"): + type A = ref object + x: int + let a = A(x: 3) + let ha = a.hash + assert ha != A(x: 3).hash # A(x: 3) is a different ref object from `a`. + a.x = 4 + assert ha == a.hash # the hash only depends on the address + runnableExamples("-d:nimPreviewHashRef"): + # you can overload `hash` if you want to customize semantics + type A[T] = ref object + x, y: T + proc hash(a: A): Hash = hash(a.x) + assert A[int](x: 3, y: 4).hash == A[int](x: 3, y: 5).hash + # xxx pending bug #17733, merge as `proc hash*(pointer | ref | ptr): Hash` + # or `proc hash*[T: ref | ptr](x: T): Hash` + hash(cast[pointer](x)) proc hash*(x: float): Hash {.inline.} = ## Efficient hashing of floats. @@ -282,16 +319,24 @@ proc murmurHash(x: openArray[byte]): Hash = h1: uint32 i = 0 + + template impl = + var j = stepSize + while j > 0: + dec j + k1 = (k1 shl 8) or (ord(x[i+j])).uint32 + # body while i < n * stepSize: var k1: uint32 - when defined(js) or defined(sparc) or defined(sparc64): - var j = stepSize - while j > 0: - dec j - k1 = (k1 shl 8) or (ord(x[i+j])).uint32 + + when nimvm: + impl() else: - k1 = cast[ptr uint32](unsafeAddr x[i])[] + when declared(copyMem): + copyMem(addr k1, addr x[i], 4) + else: + impl() inc i, stepSize k1 = imul(k1, c1) @@ -323,16 +368,168 @@ proc murmurHash(x: openArray[byte]): Hash = return cast[Hash](h1) proc hashVmImpl(x: cstring, sPos, ePos: int): Hash = - doAssert false, "implementation override in compiler/vmops.nim" + raiseAssert "implementation override in compiler/vmops.nim" proc hashVmImpl(x: string, sPos, ePos: int): Hash = - doAssert false, "implementation override in compiler/vmops.nim" + raiseAssert "implementation override in compiler/vmops.nim" proc hashVmImplChar(x: openArray[char], sPos, ePos: int): Hash = - doAssert false, "implementation override in compiler/vmops.nim" + raiseAssert "implementation override in compiler/vmops.nim" proc hashVmImplByte(x: openArray[byte], sPos, ePos: int): Hash = - doAssert false, "implementation override in compiler/vmops.nim" + raiseAssert "implementation override in compiler/vmops.nim" + +const k0 = 0xc3a5c85c97cb3127u64 # Primes on (2^63, 2^64) for various uses +const k1 = 0xb492b66fbe98f273u64 +const k2 = 0x9ae16a3b2f90404fu64 + +proc load4e(s: openArray[byte], o=0): uint32 {.inline.} = + uint32(s[o + 3]) shl 24 or uint32(s[o + 2]) shl 16 or + uint32(s[o + 1]) shl 8 or uint32(s[o + 0]) + +proc load8e(s: openArray[byte], o=0): uint64 {.inline.} = + uint64(s[o + 7]) shl 56 or uint64(s[o + 6]) shl 48 or + uint64(s[o + 5]) shl 40 or uint64(s[o + 4]) shl 32 or + uint64(s[o + 3]) shl 24 or uint64(s[o + 2]) shl 16 or + uint64(s[o + 1]) shl 8 or uint64(s[o + 0]) + +proc load4(s: openArray[byte], o=0): uint32 {.inline.} = + when nimvm: result = load4e(s, o) + else: + when declared copyMem: copyMem result.addr, s[o].addr, result.sizeof + else: result = load4e(s, o) + +proc load8(s: openArray[byte], o=0): uint64 {.inline.} = + when nimvm: result = load8e(s, o) + else: + when declared copyMem: copyMem result.addr, s[o].addr, result.sizeof + else: result = load8e(s, o) + +proc lenU(s: openArray[byte]): uint64 {.inline.} = s.len.uint64 + +proc shiftMix(v: uint64): uint64 {.inline.} = v xor (v shr 47) + +proc rotR(v: uint64; bits: cint): uint64 {.inline.} = + (v shr bits) or (v shl (64 - bits)) + +proc len16(u: uint64; v: uint64; mul: uint64): uint64 {.inline.} = + var a = (u xor v)*mul + a = a xor (a shr 47) + var b = (v xor a)*mul + b = b xor (b shr 47) + b*mul + +proc len0_16(s: openArray[byte]): uint64 {.inline.} = + if s.len >= 8: + let mul = k2 + 2*s.lenU + let a = load8(s) + k2 + let b = load8(s, s.len - 8) + let c = rotR(b, 37)*mul + a + let d = (rotR(a, 25) + b)*mul + len16 c, d, mul + elif s.len >= 4: + let mul = k2 + 2*s.lenU + let a = load4(s).uint64 + len16 s.lenU + (a shl 3), load4(s, s.len - 4), mul + elif s.len > 0: + let a = uint32(s[0]) + let b = uint32(s[s.len shr 1]) + let c = uint32(s[s.len - 1]) + let y = a + (b shl 8) + let z = s.lenU + (c shl 2) + shiftMix(y*k2 xor z*k0)*k2 + else: k2 # s.len == 0 + +proc len17_32(s: openArray[byte]): uint64 {.inline.} = + let mul = k2 + 2*s.lenU + let a = load8(s)*k1 + let b = load8(s, 8) + let c = load8(s, s.len - 8)*mul + let d = load8(s, s.len - 16)*k2 + len16 rotR(a + b, 43) + rotR(c, 30) + d, a + rotR(b + k2, 18) + c, mul + +proc len33_64(s: openArray[byte]): uint64 {.inline.} = + let mul = k2 + 2*s.lenU + let a = load8(s)*k2 + let b = load8(s, 8) + let c = load8(s, s.len - 8)*mul + let d = load8(s, s.len - 16)*k2 + let y = rotR(a + b, 43) + rotR(c, 30) + d + let z = len16(y, a + rotR(b + k2, 18) + c, mul) + let e = load8(s, 16)*mul + let f = load8(s, 24) + let g = (y + load8(s, s.len - 32))*mul + let h = (z + load8(s, s.len - 24))*mul + len16 rotR(e + f, 43) + rotR(g, 30) + h, e + rotR(f + a, 18) + g, mul + +type Pair = tuple[first, second: uint64] + +proc weakLen32withSeeds2(w, x, y, z, a, b: uint64): Pair {.inline.} = + var a = a + w + var b = rotR(b + a + z, 21) + let c = a + a += x + a += y + b += rotR(a, 44) + result[0] = a + z + result[1] = b + c + +proc weakLen32withSeeds(s: openArray[byte]; o: int; a,b: uint64): Pair {.inline.} = + weakLen32withSeeds2 load8(s, o ), load8(s, o + 8), + load8(s, o + 16), load8(s, o + 24), a, b + +proc hashFarm(s: openArray[byte]): uint64 {.inline.} = + if s.len <= 16: return len0_16(s) + if s.len <= 32: return len17_32(s) + if s.len <= 64: return len33_64(s) + const seed = 81u64 # not const to use input `h` + var + o = 0 # s[] ptr arith -> variable origin variable `o` + x = seed + y = seed*k1 + 113 + z = shiftMix(y*k2 + 113)*k2 + v, w: Pair + x = x*k2 + load8(s) + let eos = ((s.len - 1) div 64)*64 + let last64 = eos + ((s.len - 1) and 63) - 63 + while true: + x = rotR(x + y + v[0] + load8(s, o+8), 37)*k1 + y = rotR(y + v[1] + load8(s, o+48), 42)*k1 + x = x xor w[1] + y += v[0] + load8(s, o+40) + z = rotR(z + w[0], 33)*k1 + v = weakLen32withSeeds(s, o+0 , v[1]*k1, x + w[0]) + w = weakLen32withSeeds(s, o+32, z + w[1], y + load8(s, o+16)) + swap z, x + inc o, 64 + if o == eos: break + let mul = k1 + ((z and 0xff) shl 1) + o = last64 + w[0] += (s.lenU - 1) and 63 + v[0] += w[0] + w[0] += v[0] + x = rotR(x + y + v[0] + load8(s, o+8), 37)*mul + y = rotR(y + v[1] + load8(s, o+48), 42)*mul + x = x xor w[1]*9 + y += v[0]*9 + load8(s, o+40) + z = rotR(z + w[0], 33)*mul + v = weakLen32withSeeds(s, o+0 , v[1]*mul, x + w[0]) + w = weakLen32withSeeds(s, o+32, z + w[1], y + load8(s, o+16)) + swap z, x + len16 len16(v[0],w[0],mul) + shiftMix(y)*k0 + z, len16(v[1],w[1],mul) + x, mul + +template jsNoInt64: untyped = + when defined js: + when compiles(compileOption("jsbigint64")): + when not compileOption("jsbigint64"): true + else: false + else: false + else: false +const sHash2 = (when defined(nimStringHash2) or jsNoInt64(): true else: false) + +template maybeFailJS_Number = + when jsNoInt64() and not defined(nimStringHash2): + {.error: "Must use `-d:nimStringHash2` when using `--jsbigint64:off`".} proc hash*(x: string): Hash = ## Efficient hashing of strings. @@ -342,16 +539,13 @@ proc hash*(x: string): Hash = ## * `hashIgnoreCase <#hashIgnoreCase,string>`_ runnableExamples: doAssert hash("abracadabra") != hash("AbracadabrA") - - when not defined(nimToOpenArrayCString): - result = 0 - for c in x: - result = result !& ord(c) - result = !$result + maybeFailJS_Number() + when not sHash2: + result = cast[Hash](hashFarm(toOpenArrayByte(x, 0, x.high))) else: - when nimvm: - result = hashVmImpl(x, 0, high(x)) - else: + #when nimvm: + # result = hashVmImpl(x, 0, high(x)) + when true: result = murmurHash(toOpenArrayByte(x, 0, high(x))) proc hash*(x: cstring): Hash = @@ -361,22 +555,22 @@ proc hash*(x: cstring): Hash = doAssert hash(cstring"AbracadabrA") == hash("AbracadabrA") doAssert hash(cstring"abracadabra") != hash(cstring"AbracadabrA") - when not defined(nimToOpenArrayCString): - result = 0 - var i = 0 - while x[i] != '\0': - result = result !& ord(x[i]) - inc i - result = !$result - else: - when nimvm: - hashVmImpl(x, 0, high(x)) + maybeFailJS_Number() + when not sHash2: + when defined js: + let xx = $x + result = cast[Hash](hashFarm(toOpenArrayByte(xx, 0, xx.high))) else: - when not defined(js) and defined(nimToOpenArrayCString): - murmurHash(toOpenArrayByte(x, 0, x.high)) + result = cast[Hash](hashFarm(toOpenArrayByte(x, 0, x.high))) + else: + #when nimvm: + # result = hashVmImpl(x, 0, high(x)) + when true: + when not defined(js): + result = murmurHash(toOpenArrayByte(x, 0, x.high)) else: let xx = $x - murmurHash(toOpenArrayByte(xx, 0, high(xx))) + result = murmurHash(toOpenArrayByte(xx, 0, high(xx))) proc hash*(sBuf: string, sPos, ePos: int): Hash = ## Efficient hashing of a string buffer, from starting @@ -387,11 +581,9 @@ proc hash*(sBuf: string, sPos, ePos: int): Hash = var a = "abracadabra" doAssert hash(a, 0, 3) == hash(a, 7, 10) - when not defined(nimToOpenArrayCString): - result = 0 - for i in sPos..ePos: - result = result !& ord(sBuf[i]) - result = !$result + maybeFailJS_Number() + when not sHash2: + result = cast[Hash](hashFarm(toOpenArrayByte(sBuf, sPos, ePos))) else: murmurHash(toOpenArrayByte(sBuf, sPos, ePos)) @@ -484,11 +676,10 @@ proc hashIgnoreCase*(sBuf: string, sPos, ePos: int): Hash = h = h !& ord(c) result = !$h - -proc hash*[T: tuple | object](x: T): Hash = - ## Efficient hashing of tuples and objects. - ## There must be a `hash` proc defined for each of the field types. +proc hash*[T: tuple | object | proc | iterator {.closure.}](x: T): Hash = + ## Efficient `hash` overload. runnableExamples: + # for `tuple|object`, `hash` must be defined for each component of `x`. type Obj = object x: int y: string @@ -499,21 +690,50 @@ proc hash*[T: tuple | object](x: T): Hash = # you can define custom hashes for objects (even if they're generic): proc hash(a: Obj2): Hash = hash((a.x)) assert hash(Obj2[float](x: 520, y: "Nim")) == hash(Obj2[float](x: 520, y: "Nim2")) - for f in fields(x): - result = result !& hash(f) - result = !$result + runnableExamples: + # proc + proc fn1() = discard + const fn1b = fn1 + assert hash(fn1b) == hash(fn1) + + # closure + proc outer = + var a = 0 + proc fn2() = a.inc + assert fn2 is "closure" + let fn2b = fn2 + assert hash(fn2b) == hash(fn2) + assert hash(fn2) != hash(fn1) + outer() + + when T is "closure": + result = hash((rawProc(x), rawEnv(x))) + elif T is (proc): + result = hash(cast[pointer](x)) + else: + result = 0 + for f in fields(x): + result = result !& hash(f) + result = !$result proc hash*[A](x: openArray[A]): Hash = ## Efficient hashing of arrays and sequences. ## There must be a `hash` proc defined for the element type `A`. when A is byte: - result = murmurHash(x) + when not sHash2: + result = cast[Hash](hashFarm(x)) + else: + result = murmurHash(x) elif A is char: - when nimvm: - result = hashVmImplChar(x, 0, x.high) + when not sHash2: + result = cast[Hash](hashFarm(toOpenArrayByte(x, 0, x.high))) else: - result = murmurHash(toOpenArrayByte(x, 0, x.high)) + #when nimvm: + # result = hashVmImplChar(x, 0, x.high) + when true: + result = murmurHash(toOpenArrayByte(x, 0, x.high)) else: + result = 0 for a in x: result = result !& hash(a) result = !$result @@ -527,17 +747,24 @@ proc hash*[A](aBuf: openArray[A], sPos, ePos: int): Hash = runnableExamples: let a = [1, 2, 5, 1, 2, 6] doAssert hash(a, 0, 1) == hash(a, 3, 4) - when A is byte: - when nimvm: - result = hashVmImplByte(aBuf, sPos, ePos) + maybeFailJS_Number() + when not sHash2: + result = cast[Hash](hashFarm(toOpenArray(aBuf, sPos, ePos))) else: - result = murmurHash(toOpenArray(aBuf, sPos, ePos)) + #when nimvm: + # result = hashVmImplByte(aBuf, sPos, ePos) + when true: + result = murmurHash(toOpenArray(aBuf, sPos, ePos)) elif A is char: - when nimvm: - result = hashVmImplChar(aBuf, sPos, ePos) + maybeFailJS_Number() + when not sHash2: + result = cast[Hash](hashFarm(toOpenArrayByte(aBuf, sPos, ePos))) else: - result = murmurHash(toOpenArrayByte(aBuf, sPos, ePos)) + #when nimvm: + # result = hashVmImplChar(aBuf, sPos, ePos) + when true: + result = murmurHash(toOpenArrayByte(aBuf, sPos, ePos)) else: for i in sPos .. ePos: result = result !& hash(aBuf[i]) @@ -546,6 +773,7 @@ proc hash*[A](aBuf: openArray[A], sPos, ePos: int): Hash = proc hash*[A](x: set[A]): Hash = ## Efficient hashing of sets. ## There must be a `hash` proc defined for the element type `A`. + result = 0 for it in items(x): result = result !& hash(it) result = !$result diff --git a/lib/pure/htmlgen.nim b/lib/pure/htmlgen.nim index 163f9303b..fafa72463 100644 --- a/lib/pure/htmlgen.nim +++ b/lib/pure/htmlgen.nim @@ -8,7 +8,7 @@ # ## Do yourself a favor and import the module -## as `from htmlgen import nil` and then fully qualify the macros. +## as `from std/htmlgen import nil` and then fully qualify the macros. ## ## *Note*: The Karax project (`nimble install karax`) has a better ## way to achieve the same, see https://github.com/pragmagic/karax/blob/master/tests/nativehtmlgen.nim @@ -30,17 +30,18 @@ ## Examples ## ======== ## -## .. code-block:: Nim +## ```Nim ## var nim = "Nim" ## echo h1(a(href="https://nim-lang.org", nim)) +## ``` ## -## Writes the string:: +## Writes the string: ## -## <h1><a href="https://nim-lang.org">Nim</a></h1> +## <h1><a href="https://nim-lang.org">Nim</a></h1> ## import - macros, strutils + std/[macros, strutils] const coreAttr* = " accesskey class contenteditable dir hidden id lang " & @@ -322,7 +323,7 @@ macro html*(e: varargs[untyped]): untyped = macro hr*(): untyped = ## Generates the HTML `hr` element. - result = xmlCheckedTag(newNimNode(nnkArglist), "hr", commonAttr, "", true) + result = xmlCheckedTag(newNimNode(nnkArgList), "hr", commonAttr, "", true) macro i*(e: varargs[untyped]): untyped = ## Generates the HTML `i` element. diff --git a/lib/pure/htmlparser.nim b/lib/pure/htmlparser.nim index c572146a7..62919546f 100644 --- a/lib/pure/htmlparser.nim +++ b/lib/pure/htmlparser.nim @@ -12,10 +12,9 @@ ## ## It can be used to parse a wild HTML document and output it as valid XHTML ## document (well, if you are lucky): -## -## .. code-block:: Nim -## +## ```Nim ## echo loadHtml("mydirty.html") +## ``` ## ## Every tag in the resulting tree is in lower case. ## @@ -29,9 +28,7 @@ ## and write back the modified version. In this case we look for hyperlinks ## ending with the extension `.rst` and convert them to `.html`. ## -## .. code-block:: Nim -## :test: -## +## ```Nim test ## import std/htmlparser ## import std/xmltree # To use '$' for XmlNode ## import std/strtabs # To access XmlAttributes @@ -48,8 +45,14 @@ ## a.attrs["href"] = dir / filename & ".html" ## ## writeFile("output.html", $html) +## ``` + +{.deprecated: "use `nimble install htmlparser` and import `pkg/htmlparser` instead".} + +import std/[strutils, streams, parsexml, xmltree, unicode, strtabs] -import strutils, streams, parsexml, xmltree, unicode, strtabs +when defined(nimPreviewSlimSystem): + import std/syncio type HtmlTag* = enum ## list of all supported HTML tags; order will always be @@ -215,7 +218,7 @@ const tagMenu, tagNoframes} SingleTags* = {tagArea, tagBase, tagBasefont, tagBr, tagCol, tagFrame, tagHr, tagImg, tagIsindex, - tagLink, tagMeta, tagParam, tagWbr} + tagLink, tagMeta, tagParam, tagWbr, tagSource} proc allLower(s: string): bool = for c in s: @@ -391,10 +394,10 @@ proc entityToRune*(entity: string): Rune = case entity[1] of '0'..'9': try: runeValue = parseInt(entity[1..^1]) - except: discard + except ValueError: discard of 'x', 'X': # not case sensitive here try: runeValue = parseHexInt(entity[2..^1]) - except: discard + except ValueError: discard else: discard # other entities are not defined with prefix `#` if runeValue notin 0..0x10FFFF: runeValue = 0 # only return legal values return Rune(runeValue) @@ -2061,7 +2064,7 @@ proc loadHtml*(path: string): XmlNode = result = loadHtml(path, errors) when not defined(testing) and isMainModule: - import os + import std/os var errors: seq[string] = @[] var x = loadHtml(paramStr(1), errors) diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index 14bcfd2fb..08ea99627 100644 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -10,28 +10,38 @@ ## This module implements a simple HTTP client that can be used to retrieve ## webpages and other data. ## +## .. warning:: Validate untrusted inputs: URI parsers and getters are not detecting malicious URIs. +## ## Retrieving a website ## ==================== ## ## This example uses HTTP GET to retrieve ## `http://google.com`: ## -## .. code-block:: Nim +## ```Nim ## import std/httpclient ## var client = newHttpClient() -## echo client.getContent("http://google.com") +## try: +## echo client.getContent("http://google.com") +## finally: +## client.close() +## ``` ## ## The same action can also be performed asynchronously, simply use the ## `AsyncHttpClient`: ## -## .. code-block:: Nim +## ```Nim ## import std/[asyncdispatch, httpclient] ## ## proc asyncProc(): Future[string] {.async.} = ## var client = newAsyncHttpClient() -## return await client.getContent("http://example.com") +## try: +## return await client.getContent("http://google.com") +## finally: +## client.close() ## ## echo waitFor asyncProc() +## ``` ## ## The functionality implemented by `HttpClient` and `AsyncHttpClient` ## is the same, so you can use whichever one suits you best in the examples @@ -40,6 +50,10 @@ ## **Note:** You need to run asynchronous examples in an async proc ## otherwise you will get an `Undeclared identifier: 'await'` error. ## +## **Note:** An asynchronous client instance can only deal with one +## request at a time. To send multiple requests in parallel, use +## multiple client instances. +## ## Using HTTP POST ## =============== ## @@ -47,33 +61,39 @@ ## uses `multipart/form-data` as the `Content-Type` to send the HTML to be ## validated to the server. ## -## .. code-block:: Nim +## ```Nim ## var client = newHttpClient() ## var data = newMultipartData() ## data["output"] = "soap12" ## data["uploaded_file"] = ("test.html", "text/html", ## "<html><head></head><body><p>test</p></body></html>") -## -## echo client.postContent("http://validator.w3.org/check", multipart=data) +## try: +## echo client.postContent("http://validator.w3.org/check", multipart=data) +## finally: +## client.close() +## ``` ## ## To stream files from disk when performing the request, use `addFiles`. ## ## **Note:** This will allocate a new `Mimetypes` database every time you call ## it, you can pass your own via the `mimeDb` parameter to avoid this. ## -## .. code-block:: Nim +## ```Nim ## let mimes = newMimetypes() ## var client = newHttpClient() ## var data = newMultipartData() ## data.addFiles({"uploaded_file": "test.html"}, mimeDb = mimes) -## -## echo client.postContent("http://validator.w3.org/check", multipart=data) +## try: +## echo client.postContent("http://validator.w3.org/check", multipart=data) +## finally: +## client.close() +## ``` ## ## You can also make post requests with custom headers. ## This example sets `Content-Type` to `application/json` ## and uses a json object for the body ## -## .. code-block:: Nim +## ```Nim ## import std/[httpclient, json] ## ## let client = newHttpClient() @@ -81,8 +101,12 @@ ## let body = %*{ ## "data": "some text" ## } -## let response = client.request("http://some.api", httpMethod = HttpPost, body = $body) -## echo response.status +## try: +## let response = client.request("http://some.api", httpMethod = HttpPost, body = $body) +## echo response.status +## finally: +## client.close() +## ``` ## ## Progress reporting ## ================== @@ -91,31 +115,36 @@ ## This callback will be executed every second with information about the ## progress of the HTTP request. ## -## .. code-block:: Nim -## import std/[asyncdispatch, httpclient] +## ```Nim +## import std/[asyncdispatch, httpclient] ## -## proc onProgressChanged(total, progress, speed: BiggestInt) {.async.} = -## echo("Downloaded ", progress, " of ", total) -## echo("Current rate: ", speed div 1000, "kb/s") +## proc onProgressChanged(total, progress, speed: BiggestInt) {.async.} = +## echo("Downloaded ", progress, " of ", total) +## echo("Current rate: ", speed div 1000, "kb/s") ## -## proc asyncProc() {.async.} = -## var client = newAsyncHttpClient() -## client.onProgressChanged = onProgressChanged -## discard await client.getContent("http://speedtest-ams2.digitalocean.com/100mb.test") +## proc asyncProc() {.async.} = +## var client = newAsyncHttpClient() +## client.onProgressChanged = onProgressChanged +## try: +## discard await client.getContent("http://speedtest-ams2.digitalocean.com/100mb.test") +## finally: +## client.close() ## -## waitFor asyncProc() +## waitFor asyncProc() +## ``` ## ## If you would like to remove the callback simply set it to `nil`. ## -## .. code-block:: Nim +## ```Nim ## client.onProgressChanged = nil +## ``` ## ## .. warning:: The `total` reported by httpclient may be 0 in some cases. ## ## ## SSL/TLS support ## =============== -## This requires the OpenSSL library, fortunately it's widely used and installed +## This requires the OpenSSL library. Fortunately it's widely used and installed ## on many operating systems. httpclient will use SSL automatically if you give ## any of the functions a url with the `https` schema, for example: ## `https://github.com/`. @@ -123,12 +152,26 @@ ## You will also have to compile with `ssl` defined like so: ## `nim c -d:ssl ...`. ## -## Certificate validation is NOT performed by default. -## This will change in the future. +## Certificate validation is performed by default. ## ## A set of directories and files from the `ssl_certs <ssl_certs.html>`_ ## module are scanned to locate CA certificates. ## +## Example of setting SSL verification parameters in a new client: +## +## ```Nim +## import httpclient +## var client = newHttpClient(sslContext=newContext(verifyMode=CVerifyPeer)) +## ``` +## +## There are three options for verify mode: +## +## * ``CVerifyNone``: certificates are not verified; +## * ``CVerifyPeer``: certificates are verified; +## * ``CVerifyPeerUseEnvVars``: certificates are verified and the optional +## environment variables SSL_CERT_FILE and SSL_CERT_DIR are also used to +## locate certificates +## ## See `newContext <net.html#newContext.string,string,string,string>`_ to tweak or disable certificate validation. ## ## Timeouts @@ -148,10 +191,11 @@ ## ## Here is how to set a timeout when creating an `HttpClient` instance: ## -## .. code-block:: Nim -## import std/httpclient +## ```Nim +## import std/httpclient ## -## let client = newHttpClient(timeout = 42) +## let client = newHttpClient(timeout = 42) +## ``` ## ## Proxy ## ===== @@ -162,28 +206,39 @@ ## ## Some examples on how to configure a Proxy for `HttpClient`: ## -## .. code-block:: Nim -## import std/httpclient +## ```Nim +## import std/httpclient +## +## let myProxy = newProxy("http://myproxy.network") +## let client = newHttpClient(proxy = myProxy) +## ``` +## +## Use proxies with basic authentication: +## +## ```Nim +## import std/httpclient ## -## let myProxy = newProxy("http://myproxy.network") -## let client = newHttpClient(proxy = myProxy) +## let myProxy = newProxy("http://myproxy.network", auth="user:password") +## let client = newHttpClient(proxy = myProxy) +## ``` ## ## Get Proxy URL from environment variables: ## -## .. code-block:: Nim -## import std/httpclient +## ```Nim +## import std/httpclient ## -## var url = "" -## try: -## if existsEnv("http_proxy"): -## url = getEnv("http_proxy") -## elif existsEnv("https_proxy"): -## url = getEnv("https_proxy") -## except ValueError: -## echo "Unable to parse proxy from environment variables." +## var url = "" +## try: +## if existsEnv("http_proxy"): +## url = getEnv("http_proxy") +## elif existsEnv("https_proxy"): +## url = getEnv("https_proxy") +## except ValueError: +## echo "Unable to parse proxy from environment variables." ## -## let myProxy = newProxy(url = url) -## let client = newHttpClient(proxy = myProxy) +## let myProxy = newProxy(url = url) +## let client = newHttpClient(proxy = myProxy) +## ``` ## ## Redirects ## ========= @@ -194,10 +249,11 @@ ## ## Here you can see an example about how to set the `maxRedirects` of `HttpClient`: ## -## .. code-block:: Nim -## import std/httpclient +## ```Nim +## import std/httpclient ## -## let client = newHttpClient(maxRedirects = 0) +## let client = newHttpClient(maxRedirects = 0) +## ``` ## import std/private/since @@ -208,6 +264,9 @@ import std/[ asyncnet, asyncdispatch, asyncfile, nativesockets, ] +when defined(nimPreviewSlimSystem): + import std/[assertions, syncio] + export httpcore except parseHeader # TODO: The `except` doesn't work type @@ -245,9 +304,9 @@ proc contentLength*(response: Response | AsyncResponse): int = ## This is effectively the value of the "Content-Length" header. ## ## A `ValueError` exception will be raised if the value is not an integer. - var contentLengthHeader = response.headers.getOrDefault("Content-Length") + ## If the Content-Length header is not set in the response, ContentLength is set to the value -1. + var contentLengthHeader = response.headers.getOrDefault("Content-Length", HttpHeaderValues(@["-1"])) result = contentLengthHeader.parseInt() - doAssert(result >= 0 and result <= high(int32)) proc lastModified*(response: Response | AsyncResponse): DateTime = ## Retrieves the specified response's last modified time. @@ -300,7 +359,7 @@ type ## and `postContent` proc, ## when the server returns an error -const defUserAgent* = "Nim httpclient/" & NimVersion +const defUserAgent* = "Nim-httpclient/" & NimVersion proc httpError(msg: string) = var e: ref ProtocolError @@ -383,8 +442,9 @@ proc add*(p: MultipartData, xs: MultipartEntries): MultipartData ## Add a list of multipart entries to the multipart data `p`. All values are ## added without a filename and without a content type. ## - ## .. code-block:: Nim + ## ```Nim ## data.add({"action": "login", "format": "json"}) + ## ``` for name, content in xs.items: p.add(name, content) result = p @@ -393,8 +453,9 @@ proc newMultipartData*(xs: MultipartEntries): MultipartData = ## Create a new multipart data object and fill it with the entries `xs` ## directly. ## - ## .. code-block:: Nim + ## ```Nim ## var data = newMultipartData({"action": "login", "format": "json"}) + ## ``` result = MultipartData() for entry in xs: result.add(entry.name, entry.content) @@ -409,8 +470,9 @@ proc addFiles*(p: MultipartData, xs: openArray[tuple[name, file: string]], ## Raises an `IOError` if the file cannot be opened or reading fails. To ## manually specify file content, filename and MIME type, use `[]=` instead. ## - ## .. code-block:: Nim + ## ```Nim ## data.addFiles({"uploaded_file": "public/test.html"}) + ## ``` for name, file in xs.items: var contentType: string let (_, fName, ext) = splitFile(file) @@ -424,8 +486,9 @@ proc `[]=`*(p: MultipartData, name, content: string) {.inline.} = ## Add a multipart entry to the multipart data `p`. The value is added ## without a filename and without a content type. ## - ## .. code-block:: Nim + ## ```Nim ## data["username"] = "NimUser" + ## ``` p.add(name, content) proc `[]=`*(p: MultipartData, name: string, @@ -433,9 +496,10 @@ proc `[]=`*(p: MultipartData, name: string, ## Add a file to the multipart data `p`, specifying filename, contentType ## and content manually. ## - ## .. code-block:: Nim + ## ```Nim ## data["uploaded_file"] = ("test.html", "text/html", ## "<html><head></head><body><p>test</p></body></html>") + ## ``` p.add(name, file.content, file.name, file.contentType, useStream = false) proc getBoundary(p: MultipartData): string = @@ -510,7 +574,7 @@ proc generateHeaders(requestUrl: Uri, httpMethod: HttpMethod, headers: HttpHeade # Proxy auth header. if not proxy.isNil and proxy.auth != "": let auth = base64.encode(proxy.auth) - add(result, "Proxy-Authorization: basic " & auth & httpNewLine) + add(result, "Proxy-Authorization: Basic " & auth & httpNewLine) for key, val in headers: add(result, key & ": " & val & httpNewLine) @@ -574,15 +638,11 @@ proc newHttpClient*(userAgent = defUserAgent, maxRedirects = 5, ## ## `headers` specifies the HTTP Headers. runnableExamples: - import std/[asyncdispatch, httpclient, strutils] - - proc asyncProc(): Future[string] {.async.} = - var client = newAsyncHttpClient() - return await client.getContent("http://example.com") + import std/strutils - let exampleHtml = waitFor asyncProc() + let exampleHtml = newHttpClient().getContent("http://example.com") assert "Example Domain" in exampleHtml - assert not ("Pizza" in exampleHtml) + assert "Pizza" notin exampleHtml new result result.headers = headers @@ -616,6 +676,17 @@ proc newAsyncHttpClient*(userAgent = defUserAgent, maxRedirects = 5, ## connections. ## ## `headers` specifies the HTTP Headers. + runnableExamples: + import std/[asyncdispatch, strutils] + + proc asyncProc(): Future[string] {.async.} = + let client = newAsyncHttpClient() + result = await client.getContent("http://example.com") + + let exampleHtml = waitFor asyncProc() + assert "Example Domain" in exampleHtml + assert "Pizza" notin exampleHtml + new result result.headers = headers result.userAgent = userAgent @@ -635,15 +706,15 @@ proc close*(client: HttpClient | AsyncHttpClient) = client.connected = false proc getSocket*(client: HttpClient): Socket {.inline.} = - ## Get network socket, useful if you want to find out more details about the connection + ## Get network socket, useful if you want to find out more details about the connection. ## - ## this example shows info about local and remote endpoints + ## This example shows info about local and remote endpoints: ## - ## .. code-block:: Nim + ## ```Nim ## if client.connected: ## echo client.getSocket.getLocalAddr ## echo client.getSocket.getPeerAddr - ## + ## ``` return client.socket proc getSocket*(client: AsyncHttpClient): AsyncSocket {.inline.} = @@ -653,7 +724,7 @@ proc reportProgress(client: HttpClient | AsyncHttpClient, progress: BiggestInt) {.multisync.} = client.contentProgress += progress client.oneSecondProgress += progress - if (getMonoTime() - client.lastProgressReport).inSeconds > 1: + if (getMonoTime() - client.lastProgressReport).inSeconds >= 1: if not client.onProgressChanged.isNil: await client.onProgressChanged(client.contentTotal, client.contentProgress, @@ -751,7 +822,7 @@ proc parseBody(client: HttpClient | AsyncHttpClient, headers: HttpHeaders, httpError("Got disconnected while trying to read body.") if recvLen != length: httpError("Received length doesn't match expected length. Wanted " & - $length & " got " & $recvLen) + $length & " got: " & $recvLen) else: # (http://tools.ietf.org/html/rfc2616#section-4.4) NR.4 TODO @@ -785,6 +856,7 @@ proc parseResponse(client: HttpClient | AsyncHttpClient, var parsedStatus = false var linei = 0 var fullyRead = false + var lastHeaderName = "" var line = "" result.headers = newHttpHeaders() while true: @@ -819,16 +891,29 @@ proc parseResponse(client: HttpClient | AsyncHttpClient, parsedStatus = true else: # Parse headers - var name = "" - var le = parseUntil(line, name, ':', linei) - if le <= 0: httpError("invalid headers") - inc(linei, le) - if line[linei] != ':': httpError("invalid headers") - inc(linei) # Skip : - - result.headers.add(name, line[linei .. ^1].strip()) - if result.headers.len > headerLimit: - httpError("too many headers") + # There's at least one char because empty lines are handled above (with client.close) + if line[0] in {' ', '\t'}: + # Check if it's a multiline header value, if so, append to the header we're currently parsing + # This works because a line with a header must start with the header name without any leading space + # See https://datatracker.ietf.org/doc/html/rfc7230, section 3.2 and 3.2.4 + # Multiline headers are deprecated in the spec, but it's better to parse them than crash + if lastHeaderName == "": + # Some extra unparsable lines in the HTTP output - we ignore them + discard + else: + result.headers.table[result.headers.toCaseInsensitive(lastHeaderName)][^1].add "\n" & line + else: + var name = "" + var le = parseUntil(line, name, ':', linei) + if le <= 0: httpError("Invalid headers - received empty header name") + if line.len == le: httpError("Invalid headers - no colon after header name") + inc(linei, le) # Skip the parsed header name + inc(linei) # Skip : + # If we want to be HTTP spec compliant later, error on linei == line.len (for empty header value) + lastHeaderName = name # Remember the header name for the possible multi-line header + result.headers.add(name, line[linei .. ^1].strip()) + if result.headers.len > headerLimit: + httpError("too many headers") if not fullyRead: httpError("Connection was closed before full request has been made") @@ -849,6 +934,9 @@ proc parseResponse(client: HttpClient | AsyncHttpClient, client.parseBodyFut.addCallback do(): if client.parseBodyFut.failed: client.bodyStream.fail(client.parseBodyFut.error) + else: + when client is AsyncHttpClient: + result.bodyStream.complete() proc newConnection(client: HttpClient | AsyncHttpClient, url: Uri) {.multisync.} = @@ -955,19 +1043,22 @@ proc format(client: HttpClient | AsyncHttpClient, if entry.isFile: length += entry.fileSize + httpNewLine.len - result.add "--" & bound & "--" + result.add "--" & bound & "--" & httpNewLine for s in result: length += s.len client.headers["Content-Length"] = $length proc override(fallback, override: HttpHeaders): HttpHeaders = # Right-biased map union for `HttpHeaders` - if override.isNil: - return fallback result = newHttpHeaders() # Copy by value result.table[] = fallback.table[] + + if override.isNil: + # Return the copy of fallback so it does not get modified + return result + for k, vs in override.table: result[k] = vs @@ -987,12 +1078,16 @@ proc requestAux(client: HttpClient | AsyncHttpClient, url: Uri, await newConnection(client, url) - let newHeaders = client.headers.override(headers) + var newHeaders: HttpHeaders var data: seq[string] if multipart != nil and multipart.content.len > 0: + # `format` modifies `client.headers`, see + # https://github.com/nim-lang/Nim/pull/18208#discussion_r647036979 data = await client.format(multipart) + newHeaders = client.headers.override(headers) else: + newHeaders = client.headers.override(headers) # Only change headers if they have not been specified already if not newHeaders.hasKey("Content-Length"): if body.len != 0: @@ -1137,7 +1232,7 @@ proc responseContent(resp: Response | AsyncResponse): Future[string] {.multisync ## A `HttpRequestError` will be raised if the server responds with a ## client error (status code 4xx) or a server error (status code 5xx). if resp.code.is4xx or resp.code.is5xx: - raise newException(HttpRequestError, resp.status) + raise newException(HttpRequestError, resp.status.move) else: return await resp.bodyStream.readAll() diff --git a/lib/pure/httpcore.nim b/lib/pure/httpcore.nim index fdd5926f7..5ccab379c 100644 --- a/lib/pure/httpcore.nim +++ b/lib/pure/httpcore.nim @@ -12,7 +12,7 @@ ## ## Unstable API. import std/private/since -import tables, strutils, parseutils +import std/[tables, strutils, parseutils] type HttpHeaders* = ref object @@ -126,23 +126,20 @@ func toTitleCase(s: string): string = result[i] = if upper: toUpperAscii(s[i]) else: toLowerAscii(s[i]) upper = s[i] == '-' -func toCaseInsensitive(headers: HttpHeaders, s: string): string {.inline.} = +func toCaseInsensitive*(headers: HttpHeaders, s: string): string {.inline.} = + ## For internal usage only. Do not use. return if headers.isTitleCase: toTitleCase(s) else: toLowerAscii(s) func newHttpHeaders*(titleCase=false): HttpHeaders = ## Returns a new `HttpHeaders` object. if `titleCase` is set to true, ## headers are passed to the server in title case (e.g. "Content-Length") - new result - result.table = newTable[string, seq[string]]() - result.isTitleCase = titleCase + result = HttpHeaders(table: newTable[string, seq[string]](), isTitleCase: titleCase) func newHttpHeaders*(keyValuePairs: openArray[tuple[key: string, val: string]], titleCase=false): HttpHeaders = ## Returns a new `HttpHeaders` object from an array. if `titleCase` is set to true, ## headers are passed to the server in title case (e.g. "Content-Length") - new result - result.table = newTable[string, seq[string]]() - result.isTitleCase = titleCase + result = HttpHeaders(table: newTable[string, seq[string]](), isTitleCase: titleCase) for pair in keyValuePairs: let key = result.toCaseInsensitive(pair.key) @@ -167,7 +164,8 @@ func `[]`*(headers: HttpHeaders, key: string): HttpHeaderValues = ## To access multiple values of a key, use the overloaded `[]` below or ## to get all of them access the `table` field directly. {.cast(noSideEffect).}: - return headers.table[headers.toCaseInsensitive(key)].HttpHeaderValues + let tmp = headers.table[headers.toCaseInsensitive(key)] + return HttpHeaderValues(tmp) converter toString*(values: HttpHeaderValues): string = return seq[string](values)[0] @@ -237,10 +235,9 @@ func parseList(line: string, list: var seq[string], start: int): int = while start+i < line.len and line[start + i] notin {'\c', '\l'}: i += line.skipWhitespace(start + i) i += line.parseUntil(current, {'\c', '\l', ','}, start + i) - list.add(current) + list.add(move current) # implicit current.setLen(0) if start+i < line.len and line[start + i] == ',': i.inc # Skip , - current.setLen(0) func parseHeader*(line: string): tuple[key: string, value: seq[string]] = ## Parses a single raw header HTTP line into key value pairs. @@ -347,35 +344,25 @@ func `$`*(code: HttpCode): string = func `==`*(a, b: HttpCode): bool {.borrow.} -proc `==`*(rawCode: string, code: HttpCode): bool - {.deprecated: "Deprecated since v1.2; use rawCode == $code instead".} = - ## Compare the string form of the status code with a HttpCode - ## - ## **Note**: According to HTTP/1.1 specification, the reason phrase is - ## optional and should be ignored by the client, making this - ## proc only suitable for comparing the `HttpCode` against the - ## string form of itself. - return cmpIgnoreCase(rawCode, $code) == 0 - func is1xx*(code: HttpCode): bool {.inline, since: (1, 5).} = ## Determines whether `code` is a 1xx HTTP status code. runnableExamples: doAssert is1xx(HttpCode(103)) - code.int in {100 .. 199} + code.int in 100 .. 199 func is2xx*(code: HttpCode): bool {.inline.} = ## Determines whether `code` is a 2xx HTTP status code. - code.int in {200 .. 299} + code.int in 200 .. 299 func is3xx*(code: HttpCode): bool {.inline.} = ## Determines whether `code` is a 3xx HTTP status code. - code.int in {300 .. 399} + code.int in 300 .. 399 func is4xx*(code: HttpCode): bool {.inline.} = ## Determines whether `code` is a 4xx HTTP status code. - code.int in {400 .. 499} + code.int in 400 .. 499 func is5xx*(code: HttpCode): bool {.inline.} = ## Determines whether `code` is a 5xx HTTP status code. - code.int in {500 .. 599} + code.int in 500 .. 599 diff --git a/lib/pure/includes/osenv.nim b/lib/pure/includes/osenv.nim deleted file mode 100644 index d0c92d566..000000000 --- a/lib/pure/includes/osenv.nim +++ /dev/null @@ -1,267 +0,0 @@ -# Include file that implements 'getEnv' and friends. Do not import it! - -when not declared(os) and not declared(ospaths): - {.error: "This is an include file for os.nim!".} - -when defined(nodejs): - proc getEnv*(key: string, default = ""): string {.tags: [ReadEnvEffect].} = - var ret = default.cstring - let key2 = key.cstring - {.emit: "const value = process.env[`key2`];".} - {.emit: "if (value !== undefined) { `ret` = value };".} - result = $ret - - proc existsEnv*(key: string): bool {.tags: [ReadEnvEffect].} = - var key2 = key.cstring - var ret: bool - {.emit: "`ret` = `key2` in process.env;".} - result = ret - - proc putEnv*(key, val: string) {.tags: [WriteEnvEffect].} = - var key2 = key.cstring - var val2 = val.cstring - {.emit: "process.env[`key2`] = `val2`;".} - - proc delEnv*(key: string) {.tags: [WriteEnvEffect].} = - var key2 = key.cstring - {.emit: "delete process.env[`key2`];".} - - iterator envPairs*(): tuple[key, value: string] {.tags: [ReadEnvEffect].} = - var num: int - var keys: RootObj - {.emit: "`keys` = Object.keys(process.env); `num` = `keys`.length;".} - for i in 0..<num: - var key, value: cstring - {.emit: "`key` = `keys`[`i`]; `value` = process.env[`key`];".} - yield ($key, $value) - -# commented because it must keep working with js+VM -# elif defined(js): -# {.error: "requires -d:nodejs".} - -else: - when defined(windows): - from parseutils import skipIgnoreCase - - proc c_getenv(env: cstring): cstring {. - importc: "getenv", header: "<stdlib.h>".} - proc c_putenv(env: cstring): cint {. - importc: "putenv", header: "<stdlib.h>".} - proc c_unsetenv(env: cstring): cint {. - importc: "unsetenv", header: "<stdlib.h>".} - - # Environment handling cannot be put into RTL, because the `envPairs` - # iterator depends on `environment`. - - var - envComputed {.threadvar.}: bool - environment {.threadvar.}: seq[string] - - when defined(nimV2): - proc unpairedEnvAllocs*(): int = - result = environment.len - if result > 0: inc result - - when defined(windows) and not defined(nimscript): - # because we support Windows GUI applications, things get really - # messy here... - when useWinUnicode: - when defined(cpp): - proc strEnd(cstr: WideCString, c = 0'i32): WideCString {. - importcpp: "(NI16*)wcschr((const wchar_t *)#, #)", header: "<string.h>".} - else: - proc strEnd(cstr: WideCString, c = 0'i32): WideCString {. - importc: "wcschr", header: "<string.h>".} - else: - proc strEnd(cstr: cstring, c = 0'i32): cstring {. - importc: "strchr", header: "<string.h>".} - - proc getEnvVarsC() = - if not envComputed: - environment = @[] - when useWinUnicode: - var - env = getEnvironmentStringsW() - e = env - if e == nil: return # an error occurred - while true: - var eend = strEnd(e) - add(environment, $e) - e = cast[WideCString](cast[ByteAddress](eend)+2) - if eend[1].int == 0: break - discard freeEnvironmentStringsW(env) - else: - var - env = getEnvironmentStringsA() - e = env - if e == nil: return # an error occurred - while true: - var eend = strEnd(e) - add(environment, $e) - e = cast[cstring](cast[ByteAddress](eend)+1) - if eend[1] == '\0': break - discard freeEnvironmentStringsA(env) - envComputed = true - - else: - const - useNSGetEnviron = (defined(macosx) and not defined(ios) and not defined(emscripten)) or defined(nimscript) - - when useNSGetEnviron: - # From the manual: - # Shared libraries and bundles don't have direct access to environ, - # which is only available to the loader ld(1) when a complete program - # is being linked. - # The environment routines can still be used, but if direct access to - # environ is needed, the _NSGetEnviron() routine, defined in - # <crt_externs.h>, can be used to retrieve the address of environ - # at runtime. - proc NSGetEnviron(): ptr cstringArray {. - importc: "_NSGetEnviron", header: "<crt_externs.h>".} - elif defined(haiku): - var gEnv {.importc: "environ", header: "<stdlib.h>".}: cstringArray - else: - var gEnv {.importc: "environ".}: cstringArray - - proc getEnvVarsC() = - # retrieves the variables of char** env of C's main proc - if not envComputed: - environment = @[] - when useNSGetEnviron: - var gEnv = NSGetEnviron()[] - var i = 0 - while gEnv[i] != nil: - add environment, $gEnv[i] - inc(i) - envComputed = true - - proc findEnvVar(key: string): int = - getEnvVarsC() - var temp = key & '=' - for i in 0..high(environment): - when defined(windows): - if skipIgnoreCase(environment[i], temp) == len(temp): return i - else: - if startsWith(environment[i], temp): return i - return -1 - - proc getEnv*(key: string, default = ""): string {.tags: [ReadEnvEffect].} = - ## Returns the value of the `environment variable`:idx: named `key`. - ## - ## If the variable does not exist, `""` is returned. To distinguish - ## whether a variable exists or it's value is just `""`, call - ## `existsEnv(key) proc <#existsEnv,string>`_. - ## - ## See also: - ## * `existsEnv proc <#existsEnv,string>`_ - ## * `putEnv proc <#putEnv,string,string>`_ - ## * `delEnv proc <#delEnv,string>`_ - ## * `envPairs iterator <#envPairs.i>`_ - runnableExamples: - assert getEnv("unknownEnv") == "" - assert getEnv("unknownEnv", "doesn't exist") == "doesn't exist" - - when nimvm: - discard "built into the compiler" - else: - var i = findEnvVar(key) - if i >= 0: - return substr(environment[i], find(environment[i], '=')+1) - else: - var env = c_getenv(key) - if env == nil: return default - result = $env - - proc existsEnv*(key: string): bool {.tags: [ReadEnvEffect].} = - ## Checks whether the environment variable named `key` exists. - ## Returns true if it exists, false otherwise. - ## - ## See also: - ## * `getEnv proc <#getEnv,string,string>`_ - ## * `putEnv proc <#putEnv,string,string>`_ - ## * `delEnv proc <#delEnv,string>`_ - ## * `envPairs iterator <#envPairs.i>`_ - runnableExamples: - assert not existsEnv("unknownEnv") - - when nimvm: - discard "built into the compiler" - else: - if c_getenv(key) != nil: return true - else: return findEnvVar(key) >= 0 - - proc putEnv*(key, val: string) {.tags: [WriteEnvEffect].} = - ## Sets the value of the `environment variable`:idx: named `key` to `val`. - ## If an error occurs, `OSError` is raised. - ## - ## See also: - ## * `getEnv proc <#getEnv,string,string>`_ - ## * `existsEnv proc <#existsEnv,string>`_ - ## * `delEnv proc <#delEnv,string>`_ - ## * `envPairs iterator <#envPairs.i>`_ - - # Note: by storing the string in the environment sequence, - # we guarantee that we don't free the memory before the program - # ends (this is needed for POSIX compliance). It is also needed so that - # the process itself may access its modified environment variables! - when nimvm: - discard "built into the compiler" - else: - var indx = findEnvVar(key) - if indx >= 0: - environment[indx] = key & '=' & val - else: - add environment, (key & '=' & val) - indx = high(environment) - when defined(windows) and not defined(nimscript): - when useWinUnicode: - var k = newWideCString(key) - var v = newWideCString(val) - if setEnvironmentVariableW(k, v) == 0'i32: raiseOSError(osLastError()) - else: - if setEnvironmentVariableA(key, val) == 0'i32: raiseOSError(osLastError()) - else: - if c_putenv(environment[indx]) != 0'i32: - raiseOSError(osLastError()) - - proc delEnv*(key: string) {.tags: [WriteEnvEffect].} = - ## Deletes the `environment variable`:idx: named `key`. - ## If an error occurs, `OSError` is raised. - ## - ## See also:ven - ## * `getEnv proc <#getEnv,string,string>`_ - ## * `existsEnv proc <#existsEnv,string>`_ - ## * `putEnv proc <#putEnv,string,string>`_ - ## * `envPairs iterator <#envPairs.i>`_ - when nimvm: - discard "built into the compiler" - else: - var indx = findEnvVar(key) - if indx < 0: return # Do nothing if the env var is not already set - when defined(windows) and not defined(nimscript): - when useWinUnicode: - var k = newWideCString(key) - if setEnvironmentVariableW(k, nil) == 0'i32: raiseOSError(osLastError()) - else: - if setEnvironmentVariableA(key, nil) == 0'i32: raiseOSError(osLastError()) - else: - if c_unsetenv(key) != 0'i32: - raiseOSError(osLastError()) - environment.delete(indx) - - iterator envPairs*(): tuple[key, value: string] {.tags: [ReadEnvEffect].} = - ## Iterate over all `environments variables`:idx:. - ## - ## In the first component of the tuple is the name of the current variable stored, - ## in the second its value. - ## - ## See also: - ## * `getEnv proc <#getEnv,string,string>`_ - ## * `existsEnv proc <#existsEnv,string>`_ - ## * `putEnv proc <#putEnv,string,string>`_ - ## * `delEnv proc <#delEnv,string>`_ - getEnvVarsC() - for i in 0..high(environment): - var p = find(environment[i], '=') - yield (substr(environment[i], 0, p-1), - substr(environment[i], p+1)) diff --git a/lib/pure/includes/oserr.nim b/lib/pure/includes/oserr.nim deleted file mode 100644 index b0740cbe6..000000000 --- a/lib/pure/includes/oserr.nim +++ /dev/null @@ -1,119 +0,0 @@ -# Include file that implements 'osErrorMsg' and friends. Do not import it! - -when not declared(os) and not declared(ospaths): - {.error: "This is an include file for os.nim!".} - -when not defined(nimscript): - var errno {.importc, header: "<errno.h>".}: cint - - proc c_strerror(errnum: cint): cstring {. - importc: "strerror", header: "<string.h>".} - - when defined(windows): - import winlean - -proc `==`*(err1, err2: OSErrorCode): bool {.borrow.} -proc `$`*(err: OSErrorCode): string {.borrow.} - -proc osErrorMsg*(errorCode: OSErrorCode): string = - ## Converts an OS error code into a human readable string. - ## - ## The error code can be retrieved using the `osLastError proc <#osLastError>`_. - ## - ## If conversion fails, or `errorCode` is `0` then `""` will be - ## returned. - ## - ## On Windows, the `-d:useWinAnsi` compilation flag can be used to - ## make this procedure use the non-unicode Win API calls to retrieve the - ## message. - ## - ## See also: - ## * `raiseOSError proc <#raiseOSError,OSErrorCode,string>`_ - ## * `osLastError proc <#osLastError>`_ - runnableExamples: - when defined(linux): - assert osErrorMsg(OSErrorCode(0)) == "" - assert osErrorMsg(OSErrorCode(1)) == "Operation not permitted" - assert osErrorMsg(OSErrorCode(2)) == "No such file or directory" - - result = "" - when defined(nimscript): - discard - elif defined(windows): - if errorCode != OSErrorCode(0'i32): - when useWinUnicode: - var msgbuf: WideCString - if formatMessageW(0x00000100 or 0x00001000 or 0x00000200, - nil, errorCode.int32, 0, addr(msgbuf), 0, nil) != 0'i32: - result = $msgbuf - if msgbuf != nil: localFree(cast[pointer](msgbuf)) - else: - var msgbuf: cstring - if formatMessageA(0x00000100 or 0x00001000 or 0x00000200, - nil, errorCode.int32, 0, addr(msgbuf), 0, nil) != 0'i32: - result = $msgbuf - if msgbuf != nil: localFree(msgbuf) - else: - if errorCode != OSErrorCode(0'i32): - result = $c_strerror(errorCode.int32) - -proc newOSError*( - errorCode: OSErrorCode, additionalInfo = "" -): owned(ref OSError) {.noinline.} = - ## Creates a new `OSError exception <system.html#OSError>`_. - ## - ## The `errorCode` will determine the - ## message, `osErrorMsg proc <#osErrorMsg,OSErrorCode>`_ will be used - ## to get this message. - ## - ## The error code can be retrieved using the `osLastError proc - ## <#osLastError>`_. - ## - ## If the error code is `0` or an error message could not be retrieved, - ## the message `unknown OS error` will be used. - ## - ## See also: - ## * `osErrorMsg proc <#osErrorMsg,OSErrorCode>`_ - ## * `osLastError proc <#osLastError>`_ - var e: owned(ref OSError); new(e) - e.errorCode = errorCode.int32 - e.msg = osErrorMsg(errorCode) - if additionalInfo.len > 0: - if e.msg.len > 0 and e.msg[^1] != '\n': e.msg.add '\n' - e.msg.add "Additional info: " - e.msg.addQuoted additionalInfo - if e.msg == "": - e.msg = "unknown OS error" - return e - -proc raiseOSError*(errorCode: OSErrorCode, additionalInfo = "") {.noinline.} = - ## Raises an `OSError exception <system.html#OSError>`_. - ## - ## Read the description of the `newOSError proc <#newOSError,OSErrorCode,string>`_ to learn - ## how the exception object is created. - raise newOSError(errorCode, additionalInfo) - -{.push stackTrace:off.} -proc osLastError*(): OSErrorCode {.sideEffect.} = - ## Retrieves the last operating system error code. - ## - ## This procedure is useful in the event when an OS call fails. In that case - ## this procedure will return the error code describing the reason why the - ## OS call failed. The `OSErrorMsg` procedure can then be used to convert - ## this code into a string. - ## - ## .. warning:: The behaviour of this procedure varies between Windows and POSIX systems. - ## On Windows some OS calls can reset the error code to `0` causing this - ## procedure to return `0`. It is therefore advised to call this procedure - ## immediately after an OS call fails. On POSIX systems this is not a problem. - ## - ## See also: - ## * `osErrorMsg proc <#osErrorMsg,OSErrorCode>`_ - ## * `raiseOSError proc <#raiseOSError,OSErrorCode,string>`_ - when defined(nimscript): - discard - elif defined(windows): - result = cast[OSErrorCode](getLastError()) - else: - result = OSErrorCode(errno) -{.pop.} diff --git a/lib/pure/includes/osseps.nim b/lib/pure/includes/osseps.nim deleted file mode 100644 index 10c85047b..000000000 --- a/lib/pure/includes/osseps.nim +++ /dev/null @@ -1,111 +0,0 @@ -# Include file that implements 'DirSep' and friends. Do not import this when -# you also import `os.nim`! - -# Improved based on info in 'compiler/platform.nim' - -const - doslikeFileSystem* = defined(windows) or defined(OS2) or defined(DOS) - -const - CurDir* = - when defined(macos): ':' - elif defined(genode): '/' - else: '.' - ## The constant character used by the operating system to refer to the - ## current directory. - ## - ## For example: `'.'` for POSIX or `':'` for the classic Macintosh. - - ParDir* = - when defined(macos): "::" - else: ".." - ## The constant string used by the operating system to refer to the - ## parent directory. - ## - ## For example: `".."` for POSIX or `"::"` for the classic Macintosh. - - DirSep* = - when defined(macos): ':' - elif doslikeFileSystem or defined(vxworks): '\\' - elif defined(RISCOS): '.' - else: '/' - ## The character used by the operating system to separate pathname - ## components, for example: `'/'` for POSIX, `':'` for the classic - ## Macintosh, and `'\\'` on Windows. - - AltSep* = - when doslikeFileSystem: '/' - else: DirSep - ## An alternative character used by the operating system to separate - ## pathname components, or the same as `DirSep <#DirSep>`_ if only one separator - ## character exists. This is set to `'/'` on Windows systems - ## where `DirSep <#DirSep>`_ is a backslash (`'\\'`). - - PathSep* = - when defined(macos) or defined(RISCOS): ',' - elif doslikeFileSystem or defined(vxworks): ';' - elif defined(PalmOS) or defined(MorphOS): ':' # platform has ':' but osseps has ';' - else: ':' - ## The character conventionally used by the operating system to separate - ## search path components (as in PATH), such as `':'` for POSIX - ## or `';'` for Windows. - - FileSystemCaseSensitive* = - when defined(macos) or defined(macosx) or doslikeFileSystem or defined(vxworks) or - defined(PalmOS) or defined(MorphOS): false - else: true - ## True if the file system is case sensitive, false otherwise. Used by - ## `cmpPaths proc <#cmpPaths,string,string>`_ to compare filenames properly. - - ExeExt* = - when doslikeFileSystem: "exe" - elif defined(atari): "tpp" - elif defined(netware): "nlm" - elif defined(vxworks): "vxe" - elif defined(nintendoswitch): "elf" - else: "" - ## The file extension of native executables. For example: - ## `""` for POSIX, `"exe"` on Windows (without a dot). - - ScriptExt* = - when doslikeFileSystem: "bat" - else: "" - ## The file extension of a script file. For example: `""` for POSIX, - ## `"bat"` on Windows. - - DynlibFormat* = - when defined(macos): "$1.dylib" # platform has $1Lib - elif defined(macosx): "lib$1.dylib" - elif doslikeFileSystem or defined(atari): "$1.dll" - elif defined(MorphOS): "$1.prc" - elif defined(PalmOS): "$1.prc" # platform has lib$1.so - elif defined(genode): "$1.lib.so" - elif defined(netware): "$1.nlm" - elif defined(amiga): "$1.Library" - else: "lib$1.so" - ## The format string to turn a filename into a `DLL`:idx: file (also - ## called `shared object`:idx: on some operating systems). - - ExtSep* = '.' - ## The character which separates the base filename from the extension; - ## for example, the `'.'` in `os.nim`. - - # MacOS paths - # =========== - # MacOS directory separator is a colon ":" which is the only character not - # allowed in filenames. - # - # A path containing no colon or which begins with a colon is a partial - # path. - # E.g. ":kalle:petter" ":kalle" "kalle" - # - # All other paths are full (absolute) paths. E.g. "HD:kalle:" "HD:" - # When generating paths, one is safe if one ensures that all partial paths - # begin with a colon, and all full paths end with a colon. - # In full paths the first name (e g HD above) is the name of a mounted - # volume. - # These names are not unique, because, for instance, two diskettes with the - # same names could be inserted. This means that paths on MacOS are not - # waterproof. In case of equal names the first volume found will do. - # Two colons "::" are the relative path to the parent. Three is to the - # grandparent etc. diff --git a/lib/pure/includes/unicode_ranges.nim b/lib/pure/includes/unicode_ranges.nim index 74c8c6399..04ccfb747 100644 --- a/lib/pure/includes/unicode_ranges.nim +++ b/lib/pure/includes/unicode_ranges.nim @@ -2,1988 +2,1979 @@ const toLowerRanges = [ - 0x00041, 0x0005A, 532, - 0x000C0, 0x000D6, 532, - 0x000D8, 0x000DE, 532, - 0x00189, 0x0018A, 705, - 0x001B1, 0x001B2, 717, - 0x00388, 0x0038A, 537, - 0x0038E, 0x0038F, 563, - 0x00391, 0x003A1, 532, - 0x003A3, 0x003AB, 532, - 0x003FD, 0x003FF, 370, - 0x00400, 0x0040F, 580, - 0x00410, 0x0042F, 532, - 0x00531, 0x00556, 548, - 0x010A0, 0x010C5, 7764, - 0x013A0, 0x013EF, 39364, - 0x013F0, 0x013F5, 508, - 0x01C90, 0x01CBA, -2508, - 0x01CBD, 0x01CBF, -2508, - 0x01F08, 0x01F0F, 492, - 0x01F18, 0x01F1D, 492, - 0x01F28, 0x01F2F, 492, - 0x01F38, 0x01F3F, 492, - 0x01F48, 0x01F4D, 492, - 0x01F68, 0x01F6F, 492, - 0x01F88, 0x01F8F, 492, - 0x01F98, 0x01F9F, 492, - 0x01FA8, 0x01FAF, 492, - 0x01FB8, 0x01FB9, 492, - 0x01FBA, 0x01FBB, 426, - 0x01FC8, 0x01FCB, 414, - 0x01FD8, 0x01FD9, 492, - 0x01FDA, 0x01FDB, 400, - 0x01FE8, 0x01FE9, 492, - 0x01FEA, 0x01FEB, 388, - 0x01FF8, 0x01FF9, 372, - 0x01FFA, 0x01FFB, 374, - 0x02C00, 0x02C2E, 548, - 0x02C7E, 0x02C7F, -10315, - 0x0FF21, 0x0FF3A, 532, - 0x10400, 0x10427, 540, - 0x104B0, 0x104D3, 540, - 0x10C80, 0x10CB2, 564, - 0x118A0, 0x118BF, 532, - 0x16E40, 0x16E5F, 532, - 0x1E900, 0x1E921, 534, + 0x00041'i32, 0x0005A'i32, 532, + 0x000C0'i32, 0x000D6'i32, 532, + 0x000D8'i32, 0x000DE'i32, 532, + 0x00189'i32, 0x0018A'i32, 705, + 0x001B1'i32, 0x001B2'i32, 717, + 0x00388'i32, 0x0038A'i32, 537, + 0x0038E'i32, 0x0038F'i32, 563, + 0x00391'i32, 0x003A1'i32, 532, + 0x003A3'i32, 0x003AB'i32, 532, + 0x003FD'i32, 0x003FF'i32, 370, + 0x00400'i32, 0x0040F'i32, 580, + 0x00410'i32, 0x0042F'i32, 532, + 0x00531'i32, 0x00556'i32, 548, + 0x010A0'i32, 0x010C5'i32, 7764, + 0x013A0'i32, 0x013EF'i32, 39364, + 0x013F0'i32, 0x013F5'i32, 508, + 0x01C90'i32, 0x01CBA'i32, -2508, + 0x01CBD'i32, 0x01CBF'i32, -2508, + 0x01F08'i32, 0x01F0F'i32, 492, + 0x01F18'i32, 0x01F1D'i32, 492, + 0x01F28'i32, 0x01F2F'i32, 492, + 0x01F38'i32, 0x01F3F'i32, 492, + 0x01F48'i32, 0x01F4D'i32, 492, + 0x01F68'i32, 0x01F6F'i32, 492, + 0x01F88'i32, 0x01F8F'i32, 492, + 0x01F98'i32, 0x01F9F'i32, 492, + 0x01FA8'i32, 0x01FAF'i32, 492, + 0x01FB8'i32, 0x01FB9'i32, 492, + 0x01FBA'i32, 0x01FBB'i32, 426, + 0x01FC8'i32, 0x01FCB'i32, 414, + 0x01FD8'i32, 0x01FD9'i32, 492, + 0x01FDA'i32, 0x01FDB'i32, 400, + 0x01FE8'i32, 0x01FE9'i32, 492, + 0x01FEA'i32, 0x01FEB'i32, 388, + 0x01FF8'i32, 0x01FF9'i32, 372, + 0x01FFA'i32, 0x01FFB'i32, 374, + 0x02C00'i32, 0x02C2E'i32, 548, + 0x02C7E'i32, 0x02C7F'i32, -10315, + 0x0FF21'i32, 0x0FF3A'i32, 532, + 0x10400'i32, 0x10427'i32, 540, + 0x104B0'i32, 0x104D3'i32, 540, + 0x10C80'i32, 0x10CB2'i32, 564, + 0x118A0'i32, 0x118BF'i32, 532, + 0x16E40'i32, 0x16E5F'i32, 532, + 0x1E900'i32, 0x1E921'i32, 534, ] toLowerSinglets = [ - 0x00100, 501, - 0x00102, 501, - 0x00104, 501, - 0x00106, 501, - 0x00108, 501, - 0x0010A, 501, - 0x0010C, 501, - 0x0010E, 501, - 0x00110, 501, - 0x00112, 501, - 0x00114, 501, - 0x00116, 501, - 0x00118, 501, - 0x0011A, 501, - 0x0011C, 501, - 0x0011E, 501, - 0x00120, 501, - 0x00122, 501, - 0x00124, 501, - 0x00126, 501, - 0x00128, 501, - 0x0012A, 501, - 0x0012C, 501, - 0x0012E, 501, - 0x00130, 301, - 0x00132, 501, - 0x00134, 501, - 0x00136, 501, - 0x00139, 501, - 0x0013B, 501, - 0x0013D, 501, - 0x0013F, 501, - 0x00141, 501, - 0x00143, 501, - 0x00145, 501, - 0x00147, 501, - 0x0014A, 501, - 0x0014C, 501, - 0x0014E, 501, - 0x00150, 501, - 0x00152, 501, - 0x00154, 501, - 0x00156, 501, - 0x00158, 501, - 0x0015A, 501, - 0x0015C, 501, - 0x0015E, 501, - 0x00160, 501, - 0x00162, 501, - 0x00164, 501, - 0x00166, 501, - 0x00168, 501, - 0x0016A, 501, - 0x0016C, 501, - 0x0016E, 501, - 0x00170, 501, - 0x00172, 501, - 0x00174, 501, - 0x00176, 501, - 0x00178, 379, - 0x00179, 501, - 0x0017B, 501, - 0x0017D, 501, - 0x00181, 710, - 0x00182, 501, - 0x00184, 501, - 0x00186, 706, - 0x00187, 501, - 0x0018B, 501, - 0x0018E, 579, - 0x0018F, 702, - 0x00190, 703, - 0x00191, 501, - 0x00193, 705, - 0x00194, 707, - 0x00196, 711, - 0x00197, 709, - 0x00198, 501, - 0x0019C, 711, - 0x0019D, 713, - 0x0019F, 714, - 0x001A0, 501, - 0x001A2, 501, - 0x001A4, 501, - 0x001A6, 718, - 0x001A7, 501, - 0x001A9, 718, - 0x001AC, 501, - 0x001AE, 718, - 0x001AF, 501, - 0x001B3, 501, - 0x001B5, 501, - 0x001B7, 719, - 0x001B8, 501, - 0x001BC, 501, - 0x001C4, 502, - 0x001C5, 501, - 0x001C7, 502, - 0x001C8, 501, - 0x001CA, 502, - 0x001CB, 501, - 0x001CD, 501, - 0x001CF, 501, - 0x001D1, 501, - 0x001D3, 501, - 0x001D5, 501, - 0x001D7, 501, - 0x001D9, 501, - 0x001DB, 501, - 0x001DE, 501, - 0x001E0, 501, - 0x001E2, 501, - 0x001E4, 501, - 0x001E6, 501, - 0x001E8, 501, - 0x001EA, 501, - 0x001EC, 501, - 0x001EE, 501, - 0x001F1, 502, - 0x001F2, 501, - 0x001F4, 501, - 0x001F6, 403, - 0x001F7, 444, - 0x001F8, 501, - 0x001FA, 501, - 0x001FC, 501, - 0x001FE, 501, - 0x00200, 501, - 0x00202, 501, - 0x00204, 501, - 0x00206, 501, - 0x00208, 501, - 0x0020A, 501, - 0x0020C, 501, - 0x0020E, 501, - 0x00210, 501, - 0x00212, 501, - 0x00214, 501, - 0x00216, 501, - 0x00218, 501, - 0x0021A, 501, - 0x0021C, 501, - 0x0021E, 501, - 0x00220, 370, - 0x00222, 501, - 0x00224, 501, - 0x00226, 501, - 0x00228, 501, - 0x0022A, 501, - 0x0022C, 501, - 0x0022E, 501, - 0x00230, 501, - 0x00232, 501, - 0x0023A, 11295, - 0x0023B, 501, - 0x0023D, 337, - 0x0023E, 11292, - 0x00241, 501, - 0x00243, 305, - 0x00244, 569, - 0x00245, 571, - 0x00246, 501, - 0x00248, 501, - 0x0024A, 501, - 0x0024C, 501, - 0x0024E, 501, - 0x00370, 501, - 0x00372, 501, - 0x00376, 501, - 0x0037F, 616, - 0x00386, 538, - 0x0038C, 564, - 0x003CF, 508, - 0x003D8, 501, - 0x003DA, 501, - 0x003DC, 501, - 0x003DE, 501, - 0x003E0, 501, - 0x003E2, 501, - 0x003E4, 501, - 0x003E6, 501, - 0x003E8, 501, - 0x003EA, 501, - 0x003EC, 501, - 0x003EE, 501, - 0x003F4, 440, - 0x003F7, 501, - 0x003F9, 493, - 0x003FA, 501, - 0x00460, 501, - 0x00462, 501, - 0x00464, 501, - 0x00466, 501, - 0x00468, 501, - 0x0046A, 501, - 0x0046C, 501, - 0x0046E, 501, - 0x00470, 501, - 0x00472, 501, - 0x00474, 501, - 0x00476, 501, - 0x00478, 501, - 0x0047A, 501, - 0x0047C, 501, - 0x0047E, 501, - 0x00480, 501, - 0x0048A, 501, - 0x0048C, 501, - 0x0048E, 501, - 0x00490, 501, - 0x00492, 501, - 0x00494, 501, - 0x00496, 501, - 0x00498, 501, - 0x0049A, 501, - 0x0049C, 501, - 0x0049E, 501, - 0x004A0, 501, - 0x004A2, 501, - 0x004A4, 501, - 0x004A6, 501, - 0x004A8, 501, - 0x004AA, 501, - 0x004AC, 501, - 0x004AE, 501, - 0x004B0, 501, - 0x004B2, 501, - 0x004B4, 501, - 0x004B6, 501, - 0x004B8, 501, - 0x004BA, 501, - 0x004BC, 501, - 0x004BE, 501, - 0x004C0, 515, - 0x004C1, 501, - 0x004C3, 501, - 0x004C5, 501, - 0x004C7, 501, - 0x004C9, 501, - 0x004CB, 501, - 0x004CD, 501, - 0x004D0, 501, - 0x004D2, 501, - 0x004D4, 501, - 0x004D6, 501, - 0x004D8, 501, - 0x004DA, 501, - 0x004DC, 501, - 0x004DE, 501, - 0x004E0, 501, - 0x004E2, 501, - 0x004E4, 501, - 0x004E6, 501, - 0x004E8, 501, - 0x004EA, 501, - 0x004EC, 501, - 0x004EE, 501, - 0x004F0, 501, - 0x004F2, 501, - 0x004F4, 501, - 0x004F6, 501, - 0x004F8, 501, - 0x004FA, 501, - 0x004FC, 501, - 0x004FE, 501, - 0x00500, 501, - 0x00502, 501, - 0x00504, 501, - 0x00506, 501, - 0x00508, 501, - 0x0050A, 501, - 0x0050C, 501, - 0x0050E, 501, - 0x00510, 501, - 0x00512, 501, - 0x00514, 501, - 0x00516, 501, - 0x00518, 501, - 0x0051A, 501, - 0x0051C, 501, - 0x0051E, 501, - 0x00520, 501, - 0x00522, 501, - 0x00524, 501, - 0x00526, 501, - 0x00528, 501, - 0x0052A, 501, - 0x0052C, 501, - 0x0052E, 501, - 0x010C7, 7764, - 0x010CD, 7764, - 0x01E00, 501, - 0x01E02, 501, - 0x01E04, 501, - 0x01E06, 501, - 0x01E08, 501, - 0x01E0A, 501, - 0x01E0C, 501, - 0x01E0E, 501, - 0x01E10, 501, - 0x01E12, 501, - 0x01E14, 501, - 0x01E16, 501, - 0x01E18, 501, - 0x01E1A, 501, - 0x01E1C, 501, - 0x01E1E, 501, - 0x01E20, 501, - 0x01E22, 501, - 0x01E24, 501, - 0x01E26, 501, - 0x01E28, 501, - 0x01E2A, 501, - 0x01E2C, 501, - 0x01E2E, 501, - 0x01E30, 501, - 0x01E32, 501, - 0x01E34, 501, - 0x01E36, 501, - 0x01E38, 501, - 0x01E3A, 501, - 0x01E3C, 501, - 0x01E3E, 501, - 0x01E40, 501, - 0x01E42, 501, - 0x01E44, 501, - 0x01E46, 501, - 0x01E48, 501, - 0x01E4A, 501, - 0x01E4C, 501, - 0x01E4E, 501, - 0x01E50, 501, - 0x01E52, 501, - 0x01E54, 501, - 0x01E56, 501, - 0x01E58, 501, - 0x01E5A, 501, - 0x01E5C, 501, - 0x01E5E, 501, - 0x01E60, 501, - 0x01E62, 501, - 0x01E64, 501, - 0x01E66, 501, - 0x01E68, 501, - 0x01E6A, 501, - 0x01E6C, 501, - 0x01E6E, 501, - 0x01E70, 501, - 0x01E72, 501, - 0x01E74, 501, - 0x01E76, 501, - 0x01E78, 501, - 0x01E7A, 501, - 0x01E7C, 501, - 0x01E7E, 501, - 0x01E80, 501, - 0x01E82, 501, - 0x01E84, 501, - 0x01E86, 501, - 0x01E88, 501, - 0x01E8A, 501, - 0x01E8C, 501, - 0x01E8E, 501, - 0x01E90, 501, - 0x01E92, 501, - 0x01E94, 501, - 0x01E9E, -7115, - 0x01EA0, 501, - 0x01EA2, 501, - 0x01EA4, 501, - 0x01EA6, 501, - 0x01EA8, 501, - 0x01EAA, 501, - 0x01EAC, 501, - 0x01EAE, 501, - 0x01EB0, 501, - 0x01EB2, 501, - 0x01EB4, 501, - 0x01EB6, 501, - 0x01EB8, 501, - 0x01EBA, 501, - 0x01EBC, 501, - 0x01EBE, 501, - 0x01EC0, 501, - 0x01EC2, 501, - 0x01EC4, 501, - 0x01EC6, 501, - 0x01EC8, 501, - 0x01ECA, 501, - 0x01ECC, 501, - 0x01ECE, 501, - 0x01ED0, 501, - 0x01ED2, 501, - 0x01ED4, 501, - 0x01ED6, 501, - 0x01ED8, 501, - 0x01EDA, 501, - 0x01EDC, 501, - 0x01EDE, 501, - 0x01EE0, 501, - 0x01EE2, 501, - 0x01EE4, 501, - 0x01EE6, 501, - 0x01EE8, 501, - 0x01EEA, 501, - 0x01EEC, 501, - 0x01EEE, 501, - 0x01EF0, 501, - 0x01EF2, 501, - 0x01EF4, 501, - 0x01EF6, 501, - 0x01EF8, 501, - 0x01EFA, 501, - 0x01EFC, 501, - 0x01EFE, 501, - 0x01F59, 492, - 0x01F5B, 492, - 0x01F5D, 492, - 0x01F5F, 492, - 0x01FBC, 491, - 0x01FCC, 491, - 0x01FEC, 493, - 0x01FFC, 491, - 0x02126, -7017, - 0x0212A, -7883, - 0x0212B, -7762, - 0x02132, 528, - 0x02183, 501, - 0x02C60, 501, - 0x02C62, -10243, - 0x02C63, -3314, - 0x02C64, -10227, - 0x02C67, 501, - 0x02C69, 501, - 0x02C6B, 501, - 0x02C6D, -10280, - 0x02C6E, -10249, - 0x02C6F, -10283, - 0x02C70, -10282, - 0x02C72, 501, - 0x02C75, 501, - 0x02C80, 501, - 0x02C82, 501, - 0x02C84, 501, - 0x02C86, 501, - 0x02C88, 501, - 0x02C8A, 501, - 0x02C8C, 501, - 0x02C8E, 501, - 0x02C90, 501, - 0x02C92, 501, - 0x02C94, 501, - 0x02C96, 501, - 0x02C98, 501, - 0x02C9A, 501, - 0x02C9C, 501, - 0x02C9E, 501, - 0x02CA0, 501, - 0x02CA2, 501, - 0x02CA4, 501, - 0x02CA6, 501, - 0x02CA8, 501, - 0x02CAA, 501, - 0x02CAC, 501, - 0x02CAE, 501, - 0x02CB0, 501, - 0x02CB2, 501, - 0x02CB4, 501, - 0x02CB6, 501, - 0x02CB8, 501, - 0x02CBA, 501, - 0x02CBC, 501, - 0x02CBE, 501, - 0x02CC0, 501, - 0x02CC2, 501, - 0x02CC4, 501, - 0x02CC6, 501, - 0x02CC8, 501, - 0x02CCA, 501, - 0x02CCC, 501, - 0x02CCE, 501, - 0x02CD0, 501, - 0x02CD2, 501, - 0x02CD4, 501, - 0x02CD6, 501, - 0x02CD8, 501, - 0x02CDA, 501, - 0x02CDC, 501, - 0x02CDE, 501, - 0x02CE0, 501, - 0x02CE2, 501, - 0x02CEB, 501, - 0x02CED, 501, - 0x02CF2, 501, - 0x0A640, 501, - 0x0A642, 501, - 0x0A644, 501, - 0x0A646, 501, - 0x0A648, 501, - 0x0A64A, 501, - 0x0A64C, 501, - 0x0A64E, 501, - 0x0A650, 501, - 0x0A652, 501, - 0x0A654, 501, - 0x0A656, 501, - 0x0A658, 501, - 0x0A65A, 501, - 0x0A65C, 501, - 0x0A65E, 501, - 0x0A660, 501, - 0x0A662, 501, - 0x0A664, 501, - 0x0A666, 501, - 0x0A668, 501, - 0x0A66A, 501, - 0x0A66C, 501, - 0x0A680, 501, - 0x0A682, 501, - 0x0A684, 501, - 0x0A686, 501, - 0x0A688, 501, - 0x0A68A, 501, - 0x0A68C, 501, - 0x0A68E, 501, - 0x0A690, 501, - 0x0A692, 501, - 0x0A694, 501, - 0x0A696, 501, - 0x0A698, 501, - 0x0A69A, 501, - 0x0A722, 501, - 0x0A724, 501, - 0x0A726, 501, - 0x0A728, 501, - 0x0A72A, 501, - 0x0A72C, 501, - 0x0A72E, 501, - 0x0A732, 501, - 0x0A734, 501, - 0x0A736, 501, - 0x0A738, 501, - 0x0A73A, 501, - 0x0A73C, 501, - 0x0A73E, 501, - 0x0A740, 501, - 0x0A742, 501, - 0x0A744, 501, - 0x0A746, 501, - 0x0A748, 501, - 0x0A74A, 501, - 0x0A74C, 501, - 0x0A74E, 501, - 0x0A750, 501, - 0x0A752, 501, - 0x0A754, 501, - 0x0A756, 501, - 0x0A758, 501, - 0x0A75A, 501, - 0x0A75C, 501, - 0x0A75E, 501, - 0x0A760, 501, - 0x0A762, 501, - 0x0A764, 501, - 0x0A766, 501, - 0x0A768, 501, - 0x0A76A, 501, - 0x0A76C, 501, - 0x0A76E, 501, - 0x0A779, 501, - 0x0A77B, 501, - 0x0A77D, -34832, - 0x0A77E, 501, - 0x0A780, 501, - 0x0A782, 501, - 0x0A784, 501, - 0x0A786, 501, - 0x0A78B, 501, - 0x0A78D, -41780, - 0x0A790, 501, - 0x0A792, 501, - 0x0A796, 501, - 0x0A798, 501, - 0x0A79A, 501, - 0x0A79C, 501, - 0x0A79E, 501, - 0x0A7A0, 501, - 0x0A7A2, 501, - 0x0A7A4, 501, - 0x0A7A6, 501, - 0x0A7A8, 501, - 0x0A7AA, -41808, - 0x0A7AB, -41819, - 0x0A7AC, -41815, - 0x0A7AD, -41805, - 0x0A7AE, -41808, - 0x0A7B0, -41758, - 0x0A7B1, -41782, - 0x0A7B2, -41761, - 0x0A7B3, 1428, - 0x0A7B4, 501, - 0x0A7B6, 501, - 0x0A7B8, 501, - 0x0A7BA, 501, - 0x0A7BC, 501, - 0x0A7BE, 501, - 0x0A7C2, 501, - 0x0A7C4, 452, - 0x0A7C5, -41807, - 0x0A7C6, -34884, + 0x00100'i32, 501, + 0x00102'i32, 501, + 0x00104'i32, 501, + 0x00106'i32, 501, + 0x00108'i32, 501, + 0x0010A'i32, 501, + 0x0010C'i32, 501, + 0x0010E'i32, 501, + 0x00110'i32, 501, + 0x00112'i32, 501, + 0x00114'i32, 501, + 0x00116'i32, 501, + 0x00118'i32, 501, + 0x0011A'i32, 501, + 0x0011C'i32, 501, + 0x0011E'i32, 501, + 0x00120'i32, 501, + 0x00122'i32, 501, + 0x00124'i32, 501, + 0x00126'i32, 501, + 0x00128'i32, 501, + 0x0012A'i32, 501, + 0x0012C'i32, 501, + 0x0012E'i32, 501, + 0x00130'i32, 301, + 0x00132'i32, 501, + 0x00134'i32, 501, + 0x00136'i32, 501, + 0x00139'i32, 501, + 0x0013B'i32, 501, + 0x0013D'i32, 501, + 0x0013F'i32, 501, + 0x00141'i32, 501, + 0x00143'i32, 501, + 0x00145'i32, 501, + 0x00147'i32, 501, + 0x0014A'i32, 501, + 0x0014C'i32, 501, + 0x0014E'i32, 501, + 0x00150'i32, 501, + 0x00152'i32, 501, + 0x00154'i32, 501, + 0x00156'i32, 501, + 0x00158'i32, 501, + 0x0015A'i32, 501, + 0x0015C'i32, 501, + 0x0015E'i32, 501, + 0x00160'i32, 501, + 0x00162'i32, 501, + 0x00164'i32, 501, + 0x00166'i32, 501, + 0x00168'i32, 501, + 0x0016A'i32, 501, + 0x0016C'i32, 501, + 0x0016E'i32, 501, + 0x00170'i32, 501, + 0x00172'i32, 501, + 0x00174'i32, 501, + 0x00176'i32, 501, + 0x00178'i32, 379, + 0x00179'i32, 501, + 0x0017B'i32, 501, + 0x0017D'i32, 501, + 0x00181'i32, 710, + 0x00182'i32, 501, + 0x00184'i32, 501, + 0x00186'i32, 706, + 0x00187'i32, 501, + 0x0018B'i32, 501, + 0x0018E'i32, 579, + 0x0018F'i32, 702, + 0x00190'i32, 703, + 0x00191'i32, 501, + 0x00193'i32, 705, + 0x00194'i32, 707, + 0x00196'i32, 711, + 0x00197'i32, 709, + 0x00198'i32, 501, + 0x0019C'i32, 711, + 0x0019D'i32, 713, + 0x0019F'i32, 714, + 0x001A0'i32, 501, + 0x001A2'i32, 501, + 0x001A4'i32, 501, + 0x001A6'i32, 718, + 0x001A7'i32, 501, + 0x001A9'i32, 718, + 0x001AC'i32, 501, + 0x001AE'i32, 718, + 0x001AF'i32, 501, + 0x001B3'i32, 501, + 0x001B5'i32, 501, + 0x001B7'i32, 719, + 0x001B8'i32, 501, + 0x001BC'i32, 501, + 0x001C4'i32, 502, + 0x001C5'i32, 501, + 0x001C7'i32, 502, + 0x001C8'i32, 501, + 0x001CA'i32, 502, + 0x001CB'i32, 501, + 0x001CD'i32, 501, + 0x001CF'i32, 501, + 0x001D1'i32, 501, + 0x001D3'i32, 501, + 0x001D5'i32, 501, + 0x001D7'i32, 501, + 0x001D9'i32, 501, + 0x001DB'i32, 501, + 0x001DE'i32, 501, + 0x001E0'i32, 501, + 0x001E2'i32, 501, + 0x001E4'i32, 501, + 0x001E6'i32, 501, + 0x001E8'i32, 501, + 0x001EA'i32, 501, + 0x001EC'i32, 501, + 0x001EE'i32, 501, + 0x001F1'i32, 502, + 0x001F2'i32, 501, + 0x001F4'i32, 501, + 0x001F6'i32, 403, + 0x001F7'i32, 444, + 0x001F8'i32, 501, + 0x001FA'i32, 501, + 0x001FC'i32, 501, + 0x001FE'i32, 501, + 0x00200'i32, 501, + 0x00202'i32, 501, + 0x00204'i32, 501, + 0x00206'i32, 501, + 0x00208'i32, 501, + 0x0020A'i32, 501, + 0x0020C'i32, 501, + 0x0020E'i32, 501, + 0x00210'i32, 501, + 0x00212'i32, 501, + 0x00214'i32, 501, + 0x00216'i32, 501, + 0x00218'i32, 501, + 0x0021A'i32, 501, + 0x0021C'i32, 501, + 0x0021E'i32, 501, + 0x00220'i32, 370, + 0x00222'i32, 501, + 0x00224'i32, 501, + 0x00226'i32, 501, + 0x00228'i32, 501, + 0x0022A'i32, 501, + 0x0022C'i32, 501, + 0x0022E'i32, 501, + 0x00230'i32, 501, + 0x00232'i32, 501, + 0x0023A'i32, 11295, + 0x0023B'i32, 501, + 0x0023D'i32, 337, + 0x0023E'i32, 11292, + 0x00241'i32, 501, + 0x00243'i32, 305, + 0x00244'i32, 569, + 0x00245'i32, 571, + 0x00246'i32, 501, + 0x00248'i32, 501, + 0x0024A'i32, 501, + 0x0024C'i32, 501, + 0x0024E'i32, 501, + 0x00370'i32, 501, + 0x00372'i32, 501, + 0x00376'i32, 501, + 0x0037F'i32, 616, + 0x00386'i32, 538, + 0x0038C'i32, 564, + 0x003CF'i32, 508, + 0x003D8'i32, 501, + 0x003DA'i32, 501, + 0x003DC'i32, 501, + 0x003DE'i32, 501, + 0x003E0'i32, 501, + 0x003E2'i32, 501, + 0x003E4'i32, 501, + 0x003E6'i32, 501, + 0x003E8'i32, 501, + 0x003EA'i32, 501, + 0x003EC'i32, 501, + 0x003EE'i32, 501, + 0x003F4'i32, 440, + 0x003F7'i32, 501, + 0x003F9'i32, 493, + 0x003FA'i32, 501, + 0x00460'i32, 501, + 0x00462'i32, 501, + 0x00464'i32, 501, + 0x00466'i32, 501, + 0x00468'i32, 501, + 0x0046A'i32, 501, + 0x0046C'i32, 501, + 0x0046E'i32, 501, + 0x00470'i32, 501, + 0x00472'i32, 501, + 0x00474'i32, 501, + 0x00476'i32, 501, + 0x00478'i32, 501, + 0x0047A'i32, 501, + 0x0047C'i32, 501, + 0x0047E'i32, 501, + 0x00480'i32, 501, + 0x0048A'i32, 501, + 0x0048C'i32, 501, + 0x0048E'i32, 501, + 0x00490'i32, 501, + 0x00492'i32, 501, + 0x00494'i32, 501, + 0x00496'i32, 501, + 0x00498'i32, 501, + 0x0049A'i32, 501, + 0x0049C'i32, 501, + 0x0049E'i32, 501, + 0x004A0'i32, 501, + 0x004A2'i32, 501, + 0x004A4'i32, 501, + 0x004A6'i32, 501, + 0x004A8'i32, 501, + 0x004AA'i32, 501, + 0x004AC'i32, 501, + 0x004AE'i32, 501, + 0x004B0'i32, 501, + 0x004B2'i32, 501, + 0x004B4'i32, 501, + 0x004B6'i32, 501, + 0x004B8'i32, 501, + 0x004BA'i32, 501, + 0x004BC'i32, 501, + 0x004BE'i32, 501, + 0x004C0'i32, 515, + 0x004C1'i32, 501, + 0x004C3'i32, 501, + 0x004C5'i32, 501, + 0x004C7'i32, 501, + 0x004C9'i32, 501, + 0x004CB'i32, 501, + 0x004CD'i32, 501, + 0x004D0'i32, 501, + 0x004D2'i32, 501, + 0x004D4'i32, 501, + 0x004D6'i32, 501, + 0x004D8'i32, 501, + 0x004DA'i32, 501, + 0x004DC'i32, 501, + 0x004DE'i32, 501, + 0x004E0'i32, 501, + 0x004E2'i32, 501, + 0x004E4'i32, 501, + 0x004E6'i32, 501, + 0x004E8'i32, 501, + 0x004EA'i32, 501, + 0x004EC'i32, 501, + 0x004EE'i32, 501, + 0x004F0'i32, 501, + 0x004F2'i32, 501, + 0x004F4'i32, 501, + 0x004F6'i32, 501, + 0x004F8'i32, 501, + 0x004FA'i32, 501, + 0x004FC'i32, 501, + 0x004FE'i32, 501, + 0x00500'i32, 501, + 0x00502'i32, 501, + 0x00504'i32, 501, + 0x00506'i32, 501, + 0x00508'i32, 501, + 0x0050A'i32, 501, + 0x0050C'i32, 501, + 0x0050E'i32, 501, + 0x00510'i32, 501, + 0x00512'i32, 501, + 0x00514'i32, 501, + 0x00516'i32, 501, + 0x00518'i32, 501, + 0x0051A'i32, 501, + 0x0051C'i32, 501, + 0x0051E'i32, 501, + 0x00520'i32, 501, + 0x00522'i32, 501, + 0x00524'i32, 501, + 0x00526'i32, 501, + 0x00528'i32, 501, + 0x0052A'i32, 501, + 0x0052C'i32, 501, + 0x0052E'i32, 501, + 0x010C7'i32, 7764, + 0x010CD'i32, 7764, + 0x01E00'i32, 501, + 0x01E02'i32, 501, + 0x01E04'i32, 501, + 0x01E06'i32, 501, + 0x01E08'i32, 501, + 0x01E0A'i32, 501, + 0x01E0C'i32, 501, + 0x01E0E'i32, 501, + 0x01E10'i32, 501, + 0x01E12'i32, 501, + 0x01E14'i32, 501, + 0x01E16'i32, 501, + 0x01E18'i32, 501, + 0x01E1A'i32, 501, + 0x01E1C'i32, 501, + 0x01E1E'i32, 501, + 0x01E20'i32, 501, + 0x01E22'i32, 501, + 0x01E24'i32, 501, + 0x01E26'i32, 501, + 0x01E28'i32, 501, + 0x01E2A'i32, 501, + 0x01E2C'i32, 501, + 0x01E2E'i32, 501, + 0x01E30'i32, 501, + 0x01E32'i32, 501, + 0x01E34'i32, 501, + 0x01E36'i32, 501, + 0x01E38'i32, 501, + 0x01E3A'i32, 501, + 0x01E3C'i32, 501, + 0x01E3E'i32, 501, + 0x01E40'i32, 501, + 0x01E42'i32, 501, + 0x01E44'i32, 501, + 0x01E46'i32, 501, + 0x01E48'i32, 501, + 0x01E4A'i32, 501, + 0x01E4C'i32, 501, + 0x01E4E'i32, 501, + 0x01E50'i32, 501, + 0x01E52'i32, 501, + 0x01E54'i32, 501, + 0x01E56'i32, 501, + 0x01E58'i32, 501, + 0x01E5A'i32, 501, + 0x01E5C'i32, 501, + 0x01E5E'i32, 501, + 0x01E60'i32, 501, + 0x01E62'i32, 501, + 0x01E64'i32, 501, + 0x01E66'i32, 501, + 0x01E68'i32, 501, + 0x01E6A'i32, 501, + 0x01E6C'i32, 501, + 0x01E6E'i32, 501, + 0x01E70'i32, 501, + 0x01E72'i32, 501, + 0x01E74'i32, 501, + 0x01E76'i32, 501, + 0x01E78'i32, 501, + 0x01E7A'i32, 501, + 0x01E7C'i32, 501, + 0x01E7E'i32, 501, + 0x01E80'i32, 501, + 0x01E82'i32, 501, + 0x01E84'i32, 501, + 0x01E86'i32, 501, + 0x01E88'i32, 501, + 0x01E8A'i32, 501, + 0x01E8C'i32, 501, + 0x01E8E'i32, 501, + 0x01E90'i32, 501, + 0x01E92'i32, 501, + 0x01E94'i32, 501, + 0x01E9E'i32, -7115, + 0x01EA0'i32, 501, + 0x01EA2'i32, 501, + 0x01EA4'i32, 501, + 0x01EA6'i32, 501, + 0x01EA8'i32, 501, + 0x01EAA'i32, 501, + 0x01EAC'i32, 501, + 0x01EAE'i32, 501, + 0x01EB0'i32, 501, + 0x01EB2'i32, 501, + 0x01EB4'i32, 501, + 0x01EB6'i32, 501, + 0x01EB8'i32, 501, + 0x01EBA'i32, 501, + 0x01EBC'i32, 501, + 0x01EBE'i32, 501, + 0x01EC0'i32, 501, + 0x01EC2'i32, 501, + 0x01EC4'i32, 501, + 0x01EC6'i32, 501, + 0x01EC8'i32, 501, + 0x01ECA'i32, 501, + 0x01ECC'i32, 501, + 0x01ECE'i32, 501, + 0x01ED0'i32, 501, + 0x01ED2'i32, 501, + 0x01ED4'i32, 501, + 0x01ED6'i32, 501, + 0x01ED8'i32, 501, + 0x01EDA'i32, 501, + 0x01EDC'i32, 501, + 0x01EDE'i32, 501, + 0x01EE0'i32, 501, + 0x01EE2'i32, 501, + 0x01EE4'i32, 501, + 0x01EE6'i32, 501, + 0x01EE8'i32, 501, + 0x01EEA'i32, 501, + 0x01EEC'i32, 501, + 0x01EEE'i32, 501, + 0x01EF0'i32, 501, + 0x01EF2'i32, 501, + 0x01EF4'i32, 501, + 0x01EF6'i32, 501, + 0x01EF8'i32, 501, + 0x01EFA'i32, 501, + 0x01EFC'i32, 501, + 0x01EFE'i32, 501, + 0x01F59'i32, 492, + 0x01F5B'i32, 492, + 0x01F5D'i32, 492, + 0x01F5F'i32, 492, + 0x01FBC'i32, 491, + 0x01FCC'i32, 491, + 0x01FEC'i32, 493, + 0x01FFC'i32, 491, + 0x02126'i32, -7017, + 0x0212A'i32, -7883, + 0x0212B'i32, -7762, + 0x02132'i32, 528, + 0x02183'i32, 501, + 0x02C60'i32, 501, + 0x02C62'i32, -10243, + 0x02C63'i32, -3314, + 0x02C64'i32, -10227, + 0x02C67'i32, 501, + 0x02C69'i32, 501, + 0x02C6B'i32, 501, + 0x02C6D'i32, -10280, + 0x02C6E'i32, -10249, + 0x02C6F'i32, -10283, + 0x02C70'i32, -10282, + 0x02C72'i32, 501, + 0x02C75'i32, 501, + 0x02C80'i32, 501, + 0x02C82'i32, 501, + 0x02C84'i32, 501, + 0x02C86'i32, 501, + 0x02C88'i32, 501, + 0x02C8A'i32, 501, + 0x02C8C'i32, 501, + 0x02C8E'i32, 501, + 0x02C90'i32, 501, + 0x02C92'i32, 501, + 0x02C94'i32, 501, + 0x02C96'i32, 501, + 0x02C98'i32, 501, + 0x02C9A'i32, 501, + 0x02C9C'i32, 501, + 0x02C9E'i32, 501, + 0x02CA0'i32, 501, + 0x02CA2'i32, 501, + 0x02CA4'i32, 501, + 0x02CA6'i32, 501, + 0x02CA8'i32, 501, + 0x02CAA'i32, 501, + 0x02CAC'i32, 501, + 0x02CAE'i32, 501, + 0x02CB0'i32, 501, + 0x02CB2'i32, 501, + 0x02CB4'i32, 501, + 0x02CB6'i32, 501, + 0x02CB8'i32, 501, + 0x02CBA'i32, 501, + 0x02CBC'i32, 501, + 0x02CBE'i32, 501, + 0x02CC0'i32, 501, + 0x02CC2'i32, 501, + 0x02CC4'i32, 501, + 0x02CC6'i32, 501, + 0x02CC8'i32, 501, + 0x02CCA'i32, 501, + 0x02CCC'i32, 501, + 0x02CCE'i32, 501, + 0x02CD0'i32, 501, + 0x02CD2'i32, 501, + 0x02CD4'i32, 501, + 0x02CD6'i32, 501, + 0x02CD8'i32, 501, + 0x02CDA'i32, 501, + 0x02CDC'i32, 501, + 0x02CDE'i32, 501, + 0x02CE0'i32, 501, + 0x02CE2'i32, 501, + 0x02CEB'i32, 501, + 0x02CED'i32, 501, + 0x02CF2'i32, 501, + 0x0A640'i32, 501, + 0x0A642'i32, 501, + 0x0A644'i32, 501, + 0x0A646'i32, 501, + 0x0A648'i32, 501, + 0x0A64A'i32, 501, + 0x0A64C'i32, 501, + 0x0A64E'i32, 501, + 0x0A650'i32, 501, + 0x0A652'i32, 501, + 0x0A654'i32, 501, + 0x0A656'i32, 501, + 0x0A658'i32, 501, + 0x0A65A'i32, 501, + 0x0A65C'i32, 501, + 0x0A65E'i32, 501, + 0x0A660'i32, 501, + 0x0A662'i32, 501, + 0x0A664'i32, 501, + 0x0A666'i32, 501, + 0x0A668'i32, 501, + 0x0A66A'i32, 501, + 0x0A66C'i32, 501, + 0x0A680'i32, 501, + 0x0A682'i32, 501, + 0x0A684'i32, 501, + 0x0A686'i32, 501, + 0x0A688'i32, 501, + 0x0A68A'i32, 501, + 0x0A68C'i32, 501, + 0x0A68E'i32, 501, + 0x0A690'i32, 501, + 0x0A692'i32, 501, + 0x0A694'i32, 501, + 0x0A696'i32, 501, + 0x0A698'i32, 501, + 0x0A69A'i32, 501, + 0x0A722'i32, 501, + 0x0A724'i32, 501, + 0x0A726'i32, 501, + 0x0A728'i32, 501, + 0x0A72A'i32, 501, + 0x0A72C'i32, 501, + 0x0A72E'i32, 501, + 0x0A732'i32, 501, + 0x0A734'i32, 501, + 0x0A736'i32, 501, + 0x0A738'i32, 501, + 0x0A73A'i32, 501, + 0x0A73C'i32, 501, + 0x0A73E'i32, 501, + 0x0A740'i32, 501, + 0x0A742'i32, 501, + 0x0A744'i32, 501, + 0x0A746'i32, 501, + 0x0A748'i32, 501, + 0x0A74A'i32, 501, + 0x0A74C'i32, 501, + 0x0A74E'i32, 501, + 0x0A750'i32, 501, + 0x0A752'i32, 501, + 0x0A754'i32, 501, + 0x0A756'i32, 501, + 0x0A758'i32, 501, + 0x0A75A'i32, 501, + 0x0A75C'i32, 501, + 0x0A75E'i32, 501, + 0x0A760'i32, 501, + 0x0A762'i32, 501, + 0x0A764'i32, 501, + 0x0A766'i32, 501, + 0x0A768'i32, 501, + 0x0A76A'i32, 501, + 0x0A76C'i32, 501, + 0x0A76E'i32, 501, + 0x0A779'i32, 501, + 0x0A77B'i32, 501, + 0x0A77D'i32, -34832, + 0x0A77E'i32, 501, + 0x0A780'i32, 501, + 0x0A782'i32, 501, + 0x0A784'i32, 501, + 0x0A786'i32, 501, + 0x0A78B'i32, 501, + 0x0A78D'i32, -41780, + 0x0A790'i32, 501, + 0x0A792'i32, 501, + 0x0A796'i32, 501, + 0x0A798'i32, 501, + 0x0A79A'i32, 501, + 0x0A79C'i32, 501, + 0x0A79E'i32, 501, + 0x0A7A0'i32, 501, + 0x0A7A2'i32, 501, + 0x0A7A4'i32, 501, + 0x0A7A6'i32, 501, + 0x0A7A8'i32, 501, + 0x0A7AA'i32, -41808, + 0x0A7AB'i32, -41819, + 0x0A7AC'i32, -41815, + 0x0A7AD'i32, -41805, + 0x0A7AE'i32, -41808, + 0x0A7B0'i32, -41758, + 0x0A7B1'i32, -41782, + 0x0A7B2'i32, -41761, + 0x0A7B3'i32, 1428, + 0x0A7B4'i32, 501, + 0x0A7B6'i32, 501, + 0x0A7B8'i32, 501, + 0x0A7BA'i32, 501, + 0x0A7BC'i32, 501, + 0x0A7BE'i32, 501, + 0x0A7C2'i32, 501, + 0x0A7C4'i32, 452, + 0x0A7C5'i32, -41807, + 0x0A7C6'i32, -34884, ] toUpperRanges = [ - 0x00061, 0x0007A, 468, - 0x000E0, 0x000F6, 468, - 0x000F8, 0x000FE, 468, - 0x0023F, 0x00240, 11315, - 0x00256, 0x00257, 295, - 0x0028A, 0x0028B, 283, - 0x0037B, 0x0037D, 630, - 0x003AD, 0x003AF, 463, - 0x003B1, 0x003C1, 468, - 0x003C3, 0x003CB, 468, - 0x003CD, 0x003CE, 437, - 0x00430, 0x0044F, 468, - 0x00450, 0x0045F, 420, - 0x00561, 0x00586, 452, - 0x010D0, 0x010FA, 3508, - 0x010FD, 0x010FF, 3508, - 0x013F8, 0x013FD, 492, - 0x01C83, 0x01C84, -5742, - 0x01F00, 0x01F07, 508, - 0x01F10, 0x01F15, 508, - 0x01F20, 0x01F27, 508, - 0x01F30, 0x01F37, 508, - 0x01F40, 0x01F45, 508, - 0x01F60, 0x01F67, 508, - 0x01F70, 0x01F71, 574, - 0x01F72, 0x01F75, 586, - 0x01F76, 0x01F77, 600, - 0x01F78, 0x01F79, 628, - 0x01F7A, 0x01F7B, 612, - 0x01F7C, 0x01F7D, 626, - 0x01F80, 0x01F87, 508, - 0x01F90, 0x01F97, 508, - 0x01FA0, 0x01FA7, 508, - 0x01FB0, 0x01FB1, 508, - 0x01FD0, 0x01FD1, 508, - 0x01FE0, 0x01FE1, 508, - 0x02C30, 0x02C5E, 452, - 0x02D00, 0x02D25, -6764, - 0x0AB70, 0x0ABBF, -38364, - 0x0FF41, 0x0FF5A, 468, - 0x10428, 0x1044F, 460, - 0x104D8, 0x104FB, 460, - 0x10CC0, 0x10CF2, 436, - 0x118C0, 0x118DF, 468, - 0x16E60, 0x16E7F, 468, - 0x1E922, 0x1E943, 466, + 0x00061'i32, 0x0007A'i32, 468, + 0x000E0'i32, 0x000F6'i32, 468, + 0x000F8'i32, 0x000FE'i32, 468, + 0x0023F'i32, 0x00240'i32, 11315, + 0x00256'i32, 0x00257'i32, 295, + 0x0028A'i32, 0x0028B'i32, 283, + 0x0037B'i32, 0x0037D'i32, 630, + 0x003AD'i32, 0x003AF'i32, 463, + 0x003B1'i32, 0x003C1'i32, 468, + 0x003C3'i32, 0x003CB'i32, 468, + 0x003CD'i32, 0x003CE'i32, 437, + 0x00430'i32, 0x0044F'i32, 468, + 0x00450'i32, 0x0045F'i32, 420, + 0x00561'i32, 0x00586'i32, 452, + 0x010D0'i32, 0x010FA'i32, 3508, + 0x010FD'i32, 0x010FF'i32, 3508, + 0x013F8'i32, 0x013FD'i32, 492, + 0x01C83'i32, 0x01C84'i32, -5742, + 0x01F00'i32, 0x01F07'i32, 508, + 0x01F10'i32, 0x01F15'i32, 508, + 0x01F20'i32, 0x01F27'i32, 508, + 0x01F30'i32, 0x01F37'i32, 508, + 0x01F40'i32, 0x01F45'i32, 508, + 0x01F60'i32, 0x01F67'i32, 508, + 0x01F70'i32, 0x01F71'i32, 574, + 0x01F72'i32, 0x01F75'i32, 586, + 0x01F76'i32, 0x01F77'i32, 600, + 0x01F78'i32, 0x01F79'i32, 628, + 0x01F7A'i32, 0x01F7B'i32, 612, + 0x01F7C'i32, 0x01F7D'i32, 626, + 0x01F80'i32, 0x01F87'i32, 508, + 0x01F90'i32, 0x01F97'i32, 508, + 0x01FA0'i32, 0x01FA7'i32, 508, + 0x01FB0'i32, 0x01FB1'i32, 508, + 0x01FD0'i32, 0x01FD1'i32, 508, + 0x01FE0'i32, 0x01FE1'i32, 508, + 0x02C30'i32, 0x02C5E'i32, 452, + 0x02D00'i32, 0x02D25'i32, -6764, + 0x0AB70'i32, 0x0ABBF'i32, -38364, + 0x0FF41'i32, 0x0FF5A'i32, 468, + 0x10428'i32, 0x1044F'i32, 460, + 0x104D8'i32, 0x104FB'i32, 460, + 0x10CC0'i32, 0x10CF2'i32, 436, + 0x118C0'i32, 0x118DF'i32, 468, + 0x16E60'i32, 0x16E7F'i32, 468, + 0x1E922'i32, 0x1E943'i32, 466, ] toUpperSinglets = [ - 0x000B5, 1243, - 0x000FF, 621, - 0x00101, 499, - 0x00103, 499, - 0x00105, 499, - 0x00107, 499, - 0x00109, 499, - 0x0010B, 499, - 0x0010D, 499, - 0x0010F, 499, - 0x00111, 499, - 0x00113, 499, - 0x00115, 499, - 0x00117, 499, - 0x00119, 499, - 0x0011B, 499, - 0x0011D, 499, - 0x0011F, 499, - 0x00121, 499, - 0x00123, 499, - 0x00125, 499, - 0x00127, 499, - 0x00129, 499, - 0x0012B, 499, - 0x0012D, 499, - 0x0012F, 499, - 0x00131, 268, - 0x00133, 499, - 0x00135, 499, - 0x00137, 499, - 0x0013A, 499, - 0x0013C, 499, - 0x0013E, 499, - 0x00140, 499, - 0x00142, 499, - 0x00144, 499, - 0x00146, 499, - 0x00148, 499, - 0x0014B, 499, - 0x0014D, 499, - 0x0014F, 499, - 0x00151, 499, - 0x00153, 499, - 0x00155, 499, - 0x00157, 499, - 0x00159, 499, - 0x0015B, 499, - 0x0015D, 499, - 0x0015F, 499, - 0x00161, 499, - 0x00163, 499, - 0x00165, 499, - 0x00167, 499, - 0x00169, 499, - 0x0016B, 499, - 0x0016D, 499, - 0x0016F, 499, - 0x00171, 499, - 0x00173, 499, - 0x00175, 499, - 0x00177, 499, - 0x0017A, 499, - 0x0017C, 499, - 0x0017E, 499, - 0x0017F, 200, - 0x00180, 695, - 0x00183, 499, - 0x00185, 499, - 0x00188, 499, - 0x0018C, 499, - 0x00192, 499, - 0x00195, 597, - 0x00199, 499, - 0x0019A, 663, - 0x0019E, 630, - 0x001A1, 499, - 0x001A3, 499, - 0x001A5, 499, - 0x001A8, 499, - 0x001AD, 499, - 0x001B0, 499, - 0x001B4, 499, - 0x001B6, 499, - 0x001B9, 499, - 0x001BD, 499, - 0x001BF, 556, - 0x001C5, 499, - 0x001C6, 498, - 0x001C8, 499, - 0x001C9, 498, - 0x001CB, 499, - 0x001CC, 498, - 0x001CE, 499, - 0x001D0, 499, - 0x001D2, 499, - 0x001D4, 499, - 0x001D6, 499, - 0x001D8, 499, - 0x001DA, 499, - 0x001DC, 499, - 0x001DD, 421, - 0x001DF, 499, - 0x001E1, 499, - 0x001E3, 499, - 0x001E5, 499, - 0x001E7, 499, - 0x001E9, 499, - 0x001EB, 499, - 0x001ED, 499, - 0x001EF, 499, - 0x001F2, 499, - 0x001F3, 498, - 0x001F5, 499, - 0x001F9, 499, - 0x001FB, 499, - 0x001FD, 499, - 0x001FF, 499, - 0x00201, 499, - 0x00203, 499, - 0x00205, 499, - 0x00207, 499, - 0x00209, 499, - 0x0020B, 499, - 0x0020D, 499, - 0x0020F, 499, - 0x00211, 499, - 0x00213, 499, - 0x00215, 499, - 0x00217, 499, - 0x00219, 499, - 0x0021B, 499, - 0x0021D, 499, - 0x0021F, 499, - 0x00223, 499, - 0x00225, 499, - 0x00227, 499, - 0x00229, 499, - 0x0022B, 499, - 0x0022D, 499, - 0x0022F, 499, - 0x00231, 499, - 0x00233, 499, - 0x0023C, 499, - 0x00242, 499, - 0x00247, 499, - 0x00249, 499, - 0x0024B, 499, - 0x0024D, 499, - 0x0024F, 499, - 0x00250, 11283, - 0x00251, 11280, - 0x00252, 11282, - 0x00253, 290, - 0x00254, 294, - 0x00259, 298, - 0x0025B, 297, - 0x0025C, 42819, - 0x00260, 295, - 0x00261, 42815, - 0x00263, 293, - 0x00265, 42780, - 0x00266, 42808, - 0x00268, 291, - 0x00269, 289, - 0x0026A, 42808, - 0x0026B, 11243, - 0x0026C, 42805, - 0x0026F, 289, - 0x00271, 11249, - 0x00272, 287, - 0x00275, 286, - 0x0027D, 11227, - 0x00280, 282, - 0x00282, 42807, - 0x00283, 282, - 0x00287, 42782, - 0x00288, 282, - 0x00289, 431, - 0x0028C, 429, - 0x00292, 281, - 0x0029D, 42761, - 0x0029E, 42758, - 0x00371, 499, - 0x00373, 499, - 0x00377, 499, - 0x003AC, 462, - 0x003C2, 469, - 0x003CC, 436, - 0x003D0, 438, - 0x003D1, 443, - 0x003D5, 453, - 0x003D6, 446, - 0x003D7, 492, - 0x003D9, 499, - 0x003DB, 499, - 0x003DD, 499, - 0x003DF, 499, - 0x003E1, 499, - 0x003E3, 499, - 0x003E5, 499, - 0x003E7, 499, - 0x003E9, 499, - 0x003EB, 499, - 0x003ED, 499, - 0x003EF, 499, - 0x003F0, 414, - 0x003F1, 420, - 0x003F2, 507, - 0x003F3, 384, - 0x003F5, 404, - 0x003F8, 499, - 0x003FB, 499, - 0x00461, 499, - 0x00463, 499, - 0x00465, 499, - 0x00467, 499, - 0x00469, 499, - 0x0046B, 499, - 0x0046D, 499, - 0x0046F, 499, - 0x00471, 499, - 0x00473, 499, - 0x00475, 499, - 0x00477, 499, - 0x00479, 499, - 0x0047B, 499, - 0x0047D, 499, - 0x0047F, 499, - 0x00481, 499, - 0x0048B, 499, - 0x0048D, 499, - 0x0048F, 499, - 0x00491, 499, - 0x00493, 499, - 0x00495, 499, - 0x00497, 499, - 0x00499, 499, - 0x0049B, 499, - 0x0049D, 499, - 0x0049F, 499, - 0x004A1, 499, - 0x004A3, 499, - 0x004A5, 499, - 0x004A7, 499, - 0x004A9, 499, - 0x004AB, 499, - 0x004AD, 499, - 0x004AF, 499, - 0x004B1, 499, - 0x004B3, 499, - 0x004B5, 499, - 0x004B7, 499, - 0x004B9, 499, - 0x004BB, 499, - 0x004BD, 499, - 0x004BF, 499, - 0x004C2, 499, - 0x004C4, 499, - 0x004C6, 499, - 0x004C8, 499, - 0x004CA, 499, - 0x004CC, 499, - 0x004CE, 499, - 0x004CF, 485, - 0x004D1, 499, - 0x004D3, 499, - 0x004D5, 499, - 0x004D7, 499, - 0x004D9, 499, - 0x004DB, 499, - 0x004DD, 499, - 0x004DF, 499, - 0x004E1, 499, - 0x004E3, 499, - 0x004E5, 499, - 0x004E7, 499, - 0x004E9, 499, - 0x004EB, 499, - 0x004ED, 499, - 0x004EF, 499, - 0x004F1, 499, - 0x004F3, 499, - 0x004F5, 499, - 0x004F7, 499, - 0x004F9, 499, - 0x004FB, 499, - 0x004FD, 499, - 0x004FF, 499, - 0x00501, 499, - 0x00503, 499, - 0x00505, 499, - 0x00507, 499, - 0x00509, 499, - 0x0050B, 499, - 0x0050D, 499, - 0x0050F, 499, - 0x00511, 499, - 0x00513, 499, - 0x00515, 499, - 0x00517, 499, - 0x00519, 499, - 0x0051B, 499, - 0x0051D, 499, - 0x0051F, 499, - 0x00521, 499, - 0x00523, 499, - 0x00525, 499, - 0x00527, 499, - 0x00529, 499, - 0x0052B, 499, - 0x0052D, 499, - 0x0052F, 499, - 0x01C80, -5754, - 0x01C81, -5753, - 0x01C82, -5744, - 0x01C85, -5743, - 0x01C86, -5736, - 0x01C87, -5681, - 0x01C88, 35766, - 0x01D79, 35832, - 0x01D7D, 4314, - 0x01D8E, 35884, - 0x01E01, 499, - 0x01E03, 499, - 0x01E05, 499, - 0x01E07, 499, - 0x01E09, 499, - 0x01E0B, 499, - 0x01E0D, 499, - 0x01E0F, 499, - 0x01E11, 499, - 0x01E13, 499, - 0x01E15, 499, - 0x01E17, 499, - 0x01E19, 499, - 0x01E1B, 499, - 0x01E1D, 499, - 0x01E1F, 499, - 0x01E21, 499, - 0x01E23, 499, - 0x01E25, 499, - 0x01E27, 499, - 0x01E29, 499, - 0x01E2B, 499, - 0x01E2D, 499, - 0x01E2F, 499, - 0x01E31, 499, - 0x01E33, 499, - 0x01E35, 499, - 0x01E37, 499, - 0x01E39, 499, - 0x01E3B, 499, - 0x01E3D, 499, - 0x01E3F, 499, - 0x01E41, 499, - 0x01E43, 499, - 0x01E45, 499, - 0x01E47, 499, - 0x01E49, 499, - 0x01E4B, 499, - 0x01E4D, 499, - 0x01E4F, 499, - 0x01E51, 499, - 0x01E53, 499, - 0x01E55, 499, - 0x01E57, 499, - 0x01E59, 499, - 0x01E5B, 499, - 0x01E5D, 499, - 0x01E5F, 499, - 0x01E61, 499, - 0x01E63, 499, - 0x01E65, 499, - 0x01E67, 499, - 0x01E69, 499, - 0x01E6B, 499, - 0x01E6D, 499, - 0x01E6F, 499, - 0x01E71, 499, - 0x01E73, 499, - 0x01E75, 499, - 0x01E77, 499, - 0x01E79, 499, - 0x01E7B, 499, - 0x01E7D, 499, - 0x01E7F, 499, - 0x01E81, 499, - 0x01E83, 499, - 0x01E85, 499, - 0x01E87, 499, - 0x01E89, 499, - 0x01E8B, 499, - 0x01E8D, 499, - 0x01E8F, 499, - 0x01E91, 499, - 0x01E93, 499, - 0x01E95, 499, - 0x01E9B, 441, - 0x01EA1, 499, - 0x01EA3, 499, - 0x01EA5, 499, - 0x01EA7, 499, - 0x01EA9, 499, - 0x01EAB, 499, - 0x01EAD, 499, - 0x01EAF, 499, - 0x01EB1, 499, - 0x01EB3, 499, - 0x01EB5, 499, - 0x01EB7, 499, - 0x01EB9, 499, - 0x01EBB, 499, - 0x01EBD, 499, - 0x01EBF, 499, - 0x01EC1, 499, - 0x01EC3, 499, - 0x01EC5, 499, - 0x01EC7, 499, - 0x01EC9, 499, - 0x01ECB, 499, - 0x01ECD, 499, - 0x01ECF, 499, - 0x01ED1, 499, - 0x01ED3, 499, - 0x01ED5, 499, - 0x01ED7, 499, - 0x01ED9, 499, - 0x01EDB, 499, - 0x01EDD, 499, - 0x01EDF, 499, - 0x01EE1, 499, - 0x01EE3, 499, - 0x01EE5, 499, - 0x01EE7, 499, - 0x01EE9, 499, - 0x01EEB, 499, - 0x01EED, 499, - 0x01EEF, 499, - 0x01EF1, 499, - 0x01EF3, 499, - 0x01EF5, 499, - 0x01EF7, 499, - 0x01EF9, 499, - 0x01EFB, 499, - 0x01EFD, 499, - 0x01EFF, 499, - 0x01F51, 508, - 0x01F53, 508, - 0x01F55, 508, - 0x01F57, 508, - 0x01FB3, 509, - 0x01FBE, -6705, - 0x01FC3, 509, - 0x01FE5, 507, - 0x01FF3, 509, - 0x0214E, 472, - 0x02184, 499, - 0x02C61, 499, - 0x02C65, -10295, - 0x02C66, -10292, - 0x02C68, 499, - 0x02C6A, 499, - 0x02C6C, 499, - 0x02C73, 499, - 0x02C76, 499, - 0x02C81, 499, - 0x02C83, 499, - 0x02C85, 499, - 0x02C87, 499, - 0x02C89, 499, - 0x02C8B, 499, - 0x02C8D, 499, - 0x02C8F, 499, - 0x02C91, 499, - 0x02C93, 499, - 0x02C95, 499, - 0x02C97, 499, - 0x02C99, 499, - 0x02C9B, 499, - 0x02C9D, 499, - 0x02C9F, 499, - 0x02CA1, 499, - 0x02CA3, 499, - 0x02CA5, 499, - 0x02CA7, 499, - 0x02CA9, 499, - 0x02CAB, 499, - 0x02CAD, 499, - 0x02CAF, 499, - 0x02CB1, 499, - 0x02CB3, 499, - 0x02CB5, 499, - 0x02CB7, 499, - 0x02CB9, 499, - 0x02CBB, 499, - 0x02CBD, 499, - 0x02CBF, 499, - 0x02CC1, 499, - 0x02CC3, 499, - 0x02CC5, 499, - 0x02CC7, 499, - 0x02CC9, 499, - 0x02CCB, 499, - 0x02CCD, 499, - 0x02CCF, 499, - 0x02CD1, 499, - 0x02CD3, 499, - 0x02CD5, 499, - 0x02CD7, 499, - 0x02CD9, 499, - 0x02CDB, 499, - 0x02CDD, 499, - 0x02CDF, 499, - 0x02CE1, 499, - 0x02CE3, 499, - 0x02CEC, 499, - 0x02CEE, 499, - 0x02CF3, 499, - 0x02D27, -6764, - 0x02D2D, -6764, - 0x0A641, 499, - 0x0A643, 499, - 0x0A645, 499, - 0x0A647, 499, - 0x0A649, 499, - 0x0A64B, 499, - 0x0A64D, 499, - 0x0A64F, 499, - 0x0A651, 499, - 0x0A653, 499, - 0x0A655, 499, - 0x0A657, 499, - 0x0A659, 499, - 0x0A65B, 499, - 0x0A65D, 499, - 0x0A65F, 499, - 0x0A661, 499, - 0x0A663, 499, - 0x0A665, 499, - 0x0A667, 499, - 0x0A669, 499, - 0x0A66B, 499, - 0x0A66D, 499, - 0x0A681, 499, - 0x0A683, 499, - 0x0A685, 499, - 0x0A687, 499, - 0x0A689, 499, - 0x0A68B, 499, - 0x0A68D, 499, - 0x0A68F, 499, - 0x0A691, 499, - 0x0A693, 499, - 0x0A695, 499, - 0x0A697, 499, - 0x0A699, 499, - 0x0A69B, 499, - 0x0A723, 499, - 0x0A725, 499, - 0x0A727, 499, - 0x0A729, 499, - 0x0A72B, 499, - 0x0A72D, 499, - 0x0A72F, 499, - 0x0A733, 499, - 0x0A735, 499, - 0x0A737, 499, - 0x0A739, 499, - 0x0A73B, 499, - 0x0A73D, 499, - 0x0A73F, 499, - 0x0A741, 499, - 0x0A743, 499, - 0x0A745, 499, - 0x0A747, 499, - 0x0A749, 499, - 0x0A74B, 499, - 0x0A74D, 499, - 0x0A74F, 499, - 0x0A751, 499, - 0x0A753, 499, - 0x0A755, 499, - 0x0A757, 499, - 0x0A759, 499, - 0x0A75B, 499, - 0x0A75D, 499, - 0x0A75F, 499, - 0x0A761, 499, - 0x0A763, 499, - 0x0A765, 499, - 0x0A767, 499, - 0x0A769, 499, - 0x0A76B, 499, - 0x0A76D, 499, - 0x0A76F, 499, - 0x0A77A, 499, - 0x0A77C, 499, - 0x0A77F, 499, - 0x0A781, 499, - 0x0A783, 499, - 0x0A785, 499, - 0x0A787, 499, - 0x0A78C, 499, - 0x0A791, 499, - 0x0A793, 499, - 0x0A794, 548, - 0x0A797, 499, - 0x0A799, 499, - 0x0A79B, 499, - 0x0A79D, 499, - 0x0A79F, 499, - 0x0A7A1, 499, - 0x0A7A3, 499, - 0x0A7A5, 499, - 0x0A7A7, 499, - 0x0A7A9, 499, - 0x0A7B5, 499, - 0x0A7B7, 499, - 0x0A7B9, 499, - 0x0A7BB, 499, - 0x0A7BD, 499, - 0x0A7BF, 499, - 0x0A7C3, 499, - 0x0AB53, -428, + 0x000B5'i32, 1243, + 0x000FF'i32, 621, + 0x00101'i32, 499, + 0x00103'i32, 499, + 0x00105'i32, 499, + 0x00107'i32, 499, + 0x00109'i32, 499, + 0x0010B'i32, 499, + 0x0010D'i32, 499, + 0x0010F'i32, 499, + 0x00111'i32, 499, + 0x00113'i32, 499, + 0x00115'i32, 499, + 0x00117'i32, 499, + 0x00119'i32, 499, + 0x0011B'i32, 499, + 0x0011D'i32, 499, + 0x0011F'i32, 499, + 0x00121'i32, 499, + 0x00123'i32, 499, + 0x00125'i32, 499, + 0x00127'i32, 499, + 0x00129'i32, 499, + 0x0012B'i32, 499, + 0x0012D'i32, 499, + 0x0012F'i32, 499, + 0x00131'i32, 268, + 0x00133'i32, 499, + 0x00135'i32, 499, + 0x00137'i32, 499, + 0x0013A'i32, 499, + 0x0013C'i32, 499, + 0x0013E'i32, 499, + 0x00140'i32, 499, + 0x00142'i32, 499, + 0x00144'i32, 499, + 0x00146'i32, 499, + 0x00148'i32, 499, + 0x0014B'i32, 499, + 0x0014D'i32, 499, + 0x0014F'i32, 499, + 0x00151'i32, 499, + 0x00153'i32, 499, + 0x00155'i32, 499, + 0x00157'i32, 499, + 0x00159'i32, 499, + 0x0015B'i32, 499, + 0x0015D'i32, 499, + 0x0015F'i32, 499, + 0x00161'i32, 499, + 0x00163'i32, 499, + 0x00165'i32, 499, + 0x00167'i32, 499, + 0x00169'i32, 499, + 0x0016B'i32, 499, + 0x0016D'i32, 499, + 0x0016F'i32, 499, + 0x00171'i32, 499, + 0x00173'i32, 499, + 0x00175'i32, 499, + 0x00177'i32, 499, + 0x0017A'i32, 499, + 0x0017C'i32, 499, + 0x0017E'i32, 499, + 0x0017F'i32, 200, + 0x00180'i32, 695, + 0x00183'i32, 499, + 0x00185'i32, 499, + 0x00188'i32, 499, + 0x0018C'i32, 499, + 0x00192'i32, 499, + 0x00195'i32, 597, + 0x00199'i32, 499, + 0x0019A'i32, 663, + 0x0019E'i32, 630, + 0x001A1'i32, 499, + 0x001A3'i32, 499, + 0x001A5'i32, 499, + 0x001A8'i32, 499, + 0x001AD'i32, 499, + 0x001B0'i32, 499, + 0x001B4'i32, 499, + 0x001B6'i32, 499, + 0x001B9'i32, 499, + 0x001BD'i32, 499, + 0x001BF'i32, 556, + 0x001C5'i32, 499, + 0x001C6'i32, 498, + 0x001C8'i32, 499, + 0x001C9'i32, 498, + 0x001CB'i32, 499, + 0x001CC'i32, 498, + 0x001CE'i32, 499, + 0x001D0'i32, 499, + 0x001D2'i32, 499, + 0x001D4'i32, 499, + 0x001D6'i32, 499, + 0x001D8'i32, 499, + 0x001DA'i32, 499, + 0x001DC'i32, 499, + 0x001DD'i32, 421, + 0x001DF'i32, 499, + 0x001E1'i32, 499, + 0x001E3'i32, 499, + 0x001E5'i32, 499, + 0x001E7'i32, 499, + 0x001E9'i32, 499, + 0x001EB'i32, 499, + 0x001ED'i32, 499, + 0x001EF'i32, 499, + 0x001F2'i32, 499, + 0x001F3'i32, 498, + 0x001F5'i32, 499, + 0x001F9'i32, 499, + 0x001FB'i32, 499, + 0x001FD'i32, 499, + 0x001FF'i32, 499, + 0x00201'i32, 499, + 0x00203'i32, 499, + 0x00205'i32, 499, + 0x00207'i32, 499, + 0x00209'i32, 499, + 0x0020B'i32, 499, + 0x0020D'i32, 499, + 0x0020F'i32, 499, + 0x00211'i32, 499, + 0x00213'i32, 499, + 0x00215'i32, 499, + 0x00217'i32, 499, + 0x00219'i32, 499, + 0x0021B'i32, 499, + 0x0021D'i32, 499, + 0x0021F'i32, 499, + 0x00223'i32, 499, + 0x00225'i32, 499, + 0x00227'i32, 499, + 0x00229'i32, 499, + 0x0022B'i32, 499, + 0x0022D'i32, 499, + 0x0022F'i32, 499, + 0x00231'i32, 499, + 0x00233'i32, 499, + 0x0023C'i32, 499, + 0x00242'i32, 499, + 0x00247'i32, 499, + 0x00249'i32, 499, + 0x0024B'i32, 499, + 0x0024D'i32, 499, + 0x0024F'i32, 499, + 0x00250'i32, 11283, + 0x00251'i32, 11280, + 0x00252'i32, 11282, + 0x00253'i32, 290, + 0x00254'i32, 294, + 0x00259'i32, 298, + 0x0025B'i32, 297, + 0x0025C'i32, 42819, + 0x00260'i32, 295, + 0x00261'i32, 42815, + 0x00263'i32, 293, + 0x00265'i32, 42780, + 0x00266'i32, 42808, + 0x00268'i32, 291, + 0x00269'i32, 289, + 0x0026A'i32, 42808, + 0x0026B'i32, 11243, + 0x0026C'i32, 42805, + 0x0026F'i32, 289, + 0x00271'i32, 11249, + 0x00272'i32, 287, + 0x00275'i32, 286, + 0x0027D'i32, 11227, + 0x00280'i32, 282, + 0x00282'i32, 42807, + 0x00283'i32, 282, + 0x00287'i32, 42782, + 0x00288'i32, 282, + 0x00289'i32, 431, + 0x0028C'i32, 429, + 0x00292'i32, 281, + 0x0029D'i32, 42761, + 0x0029E'i32, 42758, + 0x00371'i32, 499, + 0x00373'i32, 499, + 0x00377'i32, 499, + 0x003AC'i32, 462, + 0x003C2'i32, 469, + 0x003CC'i32, 436, + 0x003D0'i32, 438, + 0x003D1'i32, 443, + 0x003D5'i32, 453, + 0x003D6'i32, 446, + 0x003D7'i32, 492, + 0x003D9'i32, 499, + 0x003DB'i32, 499, + 0x003DD'i32, 499, + 0x003DF'i32, 499, + 0x003E1'i32, 499, + 0x003E3'i32, 499, + 0x003E5'i32, 499, + 0x003E7'i32, 499, + 0x003E9'i32, 499, + 0x003EB'i32, 499, + 0x003ED'i32, 499, + 0x003EF'i32, 499, + 0x003F0'i32, 414, + 0x003F1'i32, 420, + 0x003F2'i32, 507, + 0x003F3'i32, 384, + 0x003F5'i32, 404, + 0x003F8'i32, 499, + 0x003FB'i32, 499, + 0x00461'i32, 499, + 0x00463'i32, 499, + 0x00465'i32, 499, + 0x00467'i32, 499, + 0x00469'i32, 499, + 0x0046B'i32, 499, + 0x0046D'i32, 499, + 0x0046F'i32, 499, + 0x00471'i32, 499, + 0x00473'i32, 499, + 0x00475'i32, 499, + 0x00477'i32, 499, + 0x00479'i32, 499, + 0x0047B'i32, 499, + 0x0047D'i32, 499, + 0x0047F'i32, 499, + 0x00481'i32, 499, + 0x0048B'i32, 499, + 0x0048D'i32, 499, + 0x0048F'i32, 499, + 0x00491'i32, 499, + 0x00493'i32, 499, + 0x00495'i32, 499, + 0x00497'i32, 499, + 0x00499'i32, 499, + 0x0049B'i32, 499, + 0x0049D'i32, 499, + 0x0049F'i32, 499, + 0x004A1'i32, 499, + 0x004A3'i32, 499, + 0x004A5'i32, 499, + 0x004A7'i32, 499, + 0x004A9'i32, 499, + 0x004AB'i32, 499, + 0x004AD'i32, 499, + 0x004AF'i32, 499, + 0x004B1'i32, 499, + 0x004B3'i32, 499, + 0x004B5'i32, 499, + 0x004B7'i32, 499, + 0x004B9'i32, 499, + 0x004BB'i32, 499, + 0x004BD'i32, 499, + 0x004BF'i32, 499, + 0x004C2'i32, 499, + 0x004C4'i32, 499, + 0x004C6'i32, 499, + 0x004C8'i32, 499, + 0x004CA'i32, 499, + 0x004CC'i32, 499, + 0x004CE'i32, 499, + 0x004CF'i32, 485, + 0x004D1'i32, 499, + 0x004D3'i32, 499, + 0x004D5'i32, 499, + 0x004D7'i32, 499, + 0x004D9'i32, 499, + 0x004DB'i32, 499, + 0x004DD'i32, 499, + 0x004DF'i32, 499, + 0x004E1'i32, 499, + 0x004E3'i32, 499, + 0x004E5'i32, 499, + 0x004E7'i32, 499, + 0x004E9'i32, 499, + 0x004EB'i32, 499, + 0x004ED'i32, 499, + 0x004EF'i32, 499, + 0x004F1'i32, 499, + 0x004F3'i32, 499, + 0x004F5'i32, 499, + 0x004F7'i32, 499, + 0x004F9'i32, 499, + 0x004FB'i32, 499, + 0x004FD'i32, 499, + 0x004FF'i32, 499, + 0x00501'i32, 499, + 0x00503'i32, 499, + 0x00505'i32, 499, + 0x00507'i32, 499, + 0x00509'i32, 499, + 0x0050B'i32, 499, + 0x0050D'i32, 499, + 0x0050F'i32, 499, + 0x00511'i32, 499, + 0x00513'i32, 499, + 0x00515'i32, 499, + 0x00517'i32, 499, + 0x00519'i32, 499, + 0x0051B'i32, 499, + 0x0051D'i32, 499, + 0x0051F'i32, 499, + 0x00521'i32, 499, + 0x00523'i32, 499, + 0x00525'i32, 499, + 0x00527'i32, 499, + 0x00529'i32, 499, + 0x0052B'i32, 499, + 0x0052D'i32, 499, + 0x0052F'i32, 499, + 0x01C80'i32, -5754, + 0x01C81'i32, -5753, + 0x01C82'i32, -5744, + 0x01C85'i32, -5743, + 0x01C86'i32, -5736, + 0x01C87'i32, -5681, + 0x01C88'i32, 35766, + 0x01D79'i32, 35832, + 0x01D7D'i32, 4314, + 0x01D8E'i32, 35884, + 0x01E01'i32, 499, + 0x01E03'i32, 499, + 0x01E05'i32, 499, + 0x01E07'i32, 499, + 0x01E09'i32, 499, + 0x01E0B'i32, 499, + 0x01E0D'i32, 499, + 0x01E0F'i32, 499, + 0x01E11'i32, 499, + 0x01E13'i32, 499, + 0x01E15'i32, 499, + 0x01E17'i32, 499, + 0x01E19'i32, 499, + 0x01E1B'i32, 499, + 0x01E1D'i32, 499, + 0x01E1F'i32, 499, + 0x01E21'i32, 499, + 0x01E23'i32, 499, + 0x01E25'i32, 499, + 0x01E27'i32, 499, + 0x01E29'i32, 499, + 0x01E2B'i32, 499, + 0x01E2D'i32, 499, + 0x01E2F'i32, 499, + 0x01E31'i32, 499, + 0x01E33'i32, 499, + 0x01E35'i32, 499, + 0x01E37'i32, 499, + 0x01E39'i32, 499, + 0x01E3B'i32, 499, + 0x01E3D'i32, 499, + 0x01E3F'i32, 499, + 0x01E41'i32, 499, + 0x01E43'i32, 499, + 0x01E45'i32, 499, + 0x01E47'i32, 499, + 0x01E49'i32, 499, + 0x01E4B'i32, 499, + 0x01E4D'i32, 499, + 0x01E4F'i32, 499, + 0x01E51'i32, 499, + 0x01E53'i32, 499, + 0x01E55'i32, 499, + 0x01E57'i32, 499, + 0x01E59'i32, 499, + 0x01E5B'i32, 499, + 0x01E5D'i32, 499, + 0x01E5F'i32, 499, + 0x01E61'i32, 499, + 0x01E63'i32, 499, + 0x01E65'i32, 499, + 0x01E67'i32, 499, + 0x01E69'i32, 499, + 0x01E6B'i32, 499, + 0x01E6D'i32, 499, + 0x01E6F'i32, 499, + 0x01E71'i32, 499, + 0x01E73'i32, 499, + 0x01E75'i32, 499, + 0x01E77'i32, 499, + 0x01E79'i32, 499, + 0x01E7B'i32, 499, + 0x01E7D'i32, 499, + 0x01E7F'i32, 499, + 0x01E81'i32, 499, + 0x01E83'i32, 499, + 0x01E85'i32, 499, + 0x01E87'i32, 499, + 0x01E89'i32, 499, + 0x01E8B'i32, 499, + 0x01E8D'i32, 499, + 0x01E8F'i32, 499, + 0x01E91'i32, 499, + 0x01E93'i32, 499, + 0x01E95'i32, 499, + 0x01E9B'i32, 441, + 0x01EA1'i32, 499, + 0x01EA3'i32, 499, + 0x01EA5'i32, 499, + 0x01EA7'i32, 499, + 0x01EA9'i32, 499, + 0x01EAB'i32, 499, + 0x01EAD'i32, 499, + 0x01EAF'i32, 499, + 0x01EB1'i32, 499, + 0x01EB3'i32, 499, + 0x01EB5'i32, 499, + 0x01EB7'i32, 499, + 0x01EB9'i32, 499, + 0x01EBB'i32, 499, + 0x01EBD'i32, 499, + 0x01EBF'i32, 499, + 0x01EC1'i32, 499, + 0x01EC3'i32, 499, + 0x01EC5'i32, 499, + 0x01EC7'i32, 499, + 0x01EC9'i32, 499, + 0x01ECB'i32, 499, + 0x01ECD'i32, 499, + 0x01ECF'i32, 499, + 0x01ED1'i32, 499, + 0x01ED3'i32, 499, + 0x01ED5'i32, 499, + 0x01ED7'i32, 499, + 0x01ED9'i32, 499, + 0x01EDB'i32, 499, + 0x01EDD'i32, 499, + 0x01EDF'i32, 499, + 0x01EE1'i32, 499, + 0x01EE3'i32, 499, + 0x01EE5'i32, 499, + 0x01EE7'i32, 499, + 0x01EE9'i32, 499, + 0x01EEB'i32, 499, + 0x01EED'i32, 499, + 0x01EEF'i32, 499, + 0x01EF1'i32, 499, + 0x01EF3'i32, 499, + 0x01EF5'i32, 499, + 0x01EF7'i32, 499, + 0x01EF9'i32, 499, + 0x01EFB'i32, 499, + 0x01EFD'i32, 499, + 0x01EFF'i32, 499, + 0x01F51'i32, 508, + 0x01F53'i32, 508, + 0x01F55'i32, 508, + 0x01F57'i32, 508, + 0x01FB3'i32, 509, + 0x01FBE'i32, -6705, + 0x01FC3'i32, 509, + 0x01FE5'i32, 507, + 0x01FF3'i32, 509, + 0x0214E'i32, 472, + 0x02184'i32, 499, + 0x02C61'i32, 499, + 0x02C65'i32, -10295, + 0x02C66'i32, -10292, + 0x02C68'i32, 499, + 0x02C6A'i32, 499, + 0x02C6C'i32, 499, + 0x02C73'i32, 499, + 0x02C76'i32, 499, + 0x02C81'i32, 499, + 0x02C83'i32, 499, + 0x02C85'i32, 499, + 0x02C87'i32, 499, + 0x02C89'i32, 499, + 0x02C8B'i32, 499, + 0x02C8D'i32, 499, + 0x02C8F'i32, 499, + 0x02C91'i32, 499, + 0x02C93'i32, 499, + 0x02C95'i32, 499, + 0x02C97'i32, 499, + 0x02C99'i32, 499, + 0x02C9B'i32, 499, + 0x02C9D'i32, 499, + 0x02C9F'i32, 499, + 0x02CA1'i32, 499, + 0x02CA3'i32, 499, + 0x02CA5'i32, 499, + 0x02CA7'i32, 499, + 0x02CA9'i32, 499, + 0x02CAB'i32, 499, + 0x02CAD'i32, 499, + 0x02CAF'i32, 499, + 0x02CB1'i32, 499, + 0x02CB3'i32, 499, + 0x02CB5'i32, 499, + 0x02CB7'i32, 499, + 0x02CB9'i32, 499, + 0x02CBB'i32, 499, + 0x02CBD'i32, 499, + 0x02CBF'i32, 499, + 0x02CC1'i32, 499, + 0x02CC3'i32, 499, + 0x02CC5'i32, 499, + 0x02CC7'i32, 499, + 0x02CC9'i32, 499, + 0x02CCB'i32, 499, + 0x02CCD'i32, 499, + 0x02CCF'i32, 499, + 0x02CD1'i32, 499, + 0x02CD3'i32, 499, + 0x02CD5'i32, 499, + 0x02CD7'i32, 499, + 0x02CD9'i32, 499, + 0x02CDB'i32, 499, + 0x02CDD'i32, 499, + 0x02CDF'i32, 499, + 0x02CE1'i32, 499, + 0x02CE3'i32, 499, + 0x02CEC'i32, 499, + 0x02CEE'i32, 499, + 0x02CF3'i32, 499, + 0x02D27'i32, -6764, + 0x02D2D'i32, -6764, + 0x0A641'i32, 499, + 0x0A643'i32, 499, + 0x0A645'i32, 499, + 0x0A647'i32, 499, + 0x0A649'i32, 499, + 0x0A64B'i32, 499, + 0x0A64D'i32, 499, + 0x0A64F'i32, 499, + 0x0A651'i32, 499, + 0x0A653'i32, 499, + 0x0A655'i32, 499, + 0x0A657'i32, 499, + 0x0A659'i32, 499, + 0x0A65B'i32, 499, + 0x0A65D'i32, 499, + 0x0A65F'i32, 499, + 0x0A661'i32, 499, + 0x0A663'i32, 499, + 0x0A665'i32, 499, + 0x0A667'i32, 499, + 0x0A669'i32, 499, + 0x0A66B'i32, 499, + 0x0A66D'i32, 499, + 0x0A681'i32, 499, + 0x0A683'i32, 499, + 0x0A685'i32, 499, + 0x0A687'i32, 499, + 0x0A689'i32, 499, + 0x0A68B'i32, 499, + 0x0A68D'i32, 499, + 0x0A68F'i32, 499, + 0x0A691'i32, 499, + 0x0A693'i32, 499, + 0x0A695'i32, 499, + 0x0A697'i32, 499, + 0x0A699'i32, 499, + 0x0A69B'i32, 499, + 0x0A723'i32, 499, + 0x0A725'i32, 499, + 0x0A727'i32, 499, + 0x0A729'i32, 499, + 0x0A72B'i32, 499, + 0x0A72D'i32, 499, + 0x0A72F'i32, 499, + 0x0A733'i32, 499, + 0x0A735'i32, 499, + 0x0A737'i32, 499, + 0x0A739'i32, 499, + 0x0A73B'i32, 499, + 0x0A73D'i32, 499, + 0x0A73F'i32, 499, + 0x0A741'i32, 499, + 0x0A743'i32, 499, + 0x0A745'i32, 499, + 0x0A747'i32, 499, + 0x0A749'i32, 499, + 0x0A74B'i32, 499, + 0x0A74D'i32, 499, + 0x0A74F'i32, 499, + 0x0A751'i32, 499, + 0x0A753'i32, 499, + 0x0A755'i32, 499, + 0x0A757'i32, 499, + 0x0A759'i32, 499, + 0x0A75B'i32, 499, + 0x0A75D'i32, 499, + 0x0A75F'i32, 499, + 0x0A761'i32, 499, + 0x0A763'i32, 499, + 0x0A765'i32, 499, + 0x0A767'i32, 499, + 0x0A769'i32, 499, + 0x0A76B'i32, 499, + 0x0A76D'i32, 499, + 0x0A76F'i32, 499, + 0x0A77A'i32, 499, + 0x0A77C'i32, 499, + 0x0A77F'i32, 499, + 0x0A781'i32, 499, + 0x0A783'i32, 499, + 0x0A785'i32, 499, + 0x0A787'i32, 499, + 0x0A78C'i32, 499, + 0x0A791'i32, 499, + 0x0A793'i32, 499, + 0x0A794'i32, 548, + 0x0A797'i32, 499, + 0x0A799'i32, 499, + 0x0A79B'i32, 499, + 0x0A79D'i32, 499, + 0x0A79F'i32, 499, + 0x0A7A1'i32, 499, + 0x0A7A3'i32, 499, + 0x0A7A5'i32, 499, + 0x0A7A7'i32, 499, + 0x0A7A9'i32, 499, + 0x0A7B5'i32, 499, + 0x0A7B7'i32, 499, + 0x0A7B9'i32, 499, + 0x0A7BB'i32, 499, + 0x0A7BD'i32, 499, + 0x0A7BF'i32, 499, + 0x0A7C3'i32, 499, + 0x0AB53'i32, -428, ] toTitleSinglets = [ - 0x001C4, 501, - 0x001C6, 499, - 0x001C7, 501, - 0x001C9, 499, - 0x001CA, 501, - 0x001CC, 499, - 0x001F1, 501, - 0x001F3, 499, + 0x001C4'i32, 501, + 0x001C6'i32, 499, + 0x001C7'i32, 501, + 0x001C9'i32, 499, + 0x001CA'i32, 501, + 0x001CC'i32, 499, + 0x001F1'i32, 501, + 0x001F3'i32, 499, ] alphaRanges = [ - 0x00041, 0x0005A, - 0x00061, 0x0007A, - 0x000C0, 0x000D6, - 0x000D8, 0x000F6, - 0x000F8, 0x002C1, - 0x002C6, 0x002D1, - 0x002E0, 0x002E4, - 0x00370, 0x00374, - 0x00376, 0x00377, - 0x0037A, 0x0037D, - 0x00388, 0x0038A, - 0x0038E, 0x003A1, - 0x003A3, 0x003F5, - 0x003F7, 0x00481, - 0x0048A, 0x0052F, - 0x00531, 0x00556, - 0x00560, 0x00588, - 0x005D0, 0x005EA, - 0x005EF, 0x005F2, - 0x00620, 0x0064A, - 0x0066E, 0x0066F, - 0x00671, 0x006D3, - 0x006E5, 0x006E6, - 0x006EE, 0x006EF, - 0x006FA, 0x006FC, - 0x00712, 0x0072F, - 0x0074D, 0x007A5, - 0x007CA, 0x007EA, - 0x007F4, 0x007F5, - 0x00800, 0x00815, - 0x00840, 0x00858, - 0x00860, 0x0086A, - 0x008A0, 0x008B4, - 0x008B6, 0x008BD, - 0x00904, 0x00939, - 0x00958, 0x00961, - 0x00971, 0x00980, - 0x00985, 0x0098C, - 0x0098F, 0x00990, - 0x00993, 0x009A8, - 0x009AA, 0x009B0, - 0x009B6, 0x009B9, - 0x009DC, 0x009DD, - 0x009DF, 0x009E1, - 0x009F0, 0x009F1, - 0x00A05, 0x00A0A, - 0x00A0F, 0x00A10, - 0x00A13, 0x00A28, - 0x00A2A, 0x00A30, - 0x00A32, 0x00A33, - 0x00A35, 0x00A36, - 0x00A38, 0x00A39, - 0x00A59, 0x00A5C, - 0x00A72, 0x00A74, - 0x00A85, 0x00A8D, - 0x00A8F, 0x00A91, - 0x00A93, 0x00AA8, - 0x00AAA, 0x00AB0, - 0x00AB2, 0x00AB3, - 0x00AB5, 0x00AB9, - 0x00AE0, 0x00AE1, - 0x00B05, 0x00B0C, - 0x00B0F, 0x00B10, - 0x00B13, 0x00B28, - 0x00B2A, 0x00B30, - 0x00B32, 0x00B33, - 0x00B35, 0x00B39, - 0x00B5C, 0x00B5D, - 0x00B5F, 0x00B61, - 0x00B85, 0x00B8A, - 0x00B8E, 0x00B90, - 0x00B92, 0x00B95, - 0x00B99, 0x00B9A, - 0x00B9E, 0x00B9F, - 0x00BA3, 0x00BA4, - 0x00BA8, 0x00BAA, - 0x00BAE, 0x00BB9, - 0x00C05, 0x00C0C, - 0x00C0E, 0x00C10, - 0x00C12, 0x00C28, - 0x00C2A, 0x00C39, - 0x00C58, 0x00C5A, - 0x00C60, 0x00C61, - 0x00C85, 0x00C8C, - 0x00C8E, 0x00C90, - 0x00C92, 0x00CA8, - 0x00CAA, 0x00CB3, - 0x00CB5, 0x00CB9, - 0x00CE0, 0x00CE1, - 0x00CF1, 0x00CF2, - 0x00D05, 0x00D0C, - 0x00D0E, 0x00D10, - 0x00D12, 0x00D3A, - 0x00D54, 0x00D56, - 0x00D5F, 0x00D61, - 0x00D7A, 0x00D7F, - 0x00D85, 0x00D96, - 0x00D9A, 0x00DB1, - 0x00DB3, 0x00DBB, - 0x00DC0, 0x00DC6, - 0x00E01, 0x00E30, - 0x00E32, 0x00E33, - 0x00E40, 0x00E46, - 0x00E81, 0x00E82, - 0x00E86, 0x00E8A, - 0x00E8C, 0x00EA3, - 0x00EA7, 0x00EB0, - 0x00EB2, 0x00EB3, - 0x00EC0, 0x00EC4, - 0x00EDC, 0x00EDF, - 0x00F40, 0x00F47, - 0x00F49, 0x00F6C, - 0x00F88, 0x00F8C, - 0x01000, 0x0102A, - 0x01050, 0x01055, - 0x0105A, 0x0105D, - 0x01065, 0x01066, - 0x0106E, 0x01070, - 0x01075, 0x01081, - 0x010A0, 0x010C5, - 0x010D0, 0x010FA, - 0x010FC, 0x01248, - 0x0124A, 0x0124D, - 0x01250, 0x01256, - 0x0125A, 0x0125D, - 0x01260, 0x01288, - 0x0128A, 0x0128D, - 0x01290, 0x012B0, - 0x012B2, 0x012B5, - 0x012B8, 0x012BE, - 0x012C2, 0x012C5, - 0x012C8, 0x012D6, - 0x012D8, 0x01310, - 0x01312, 0x01315, - 0x01318, 0x0135A, - 0x01380, 0x0138F, - 0x013A0, 0x013F5, - 0x013F8, 0x013FD, - 0x01401, 0x0166C, - 0x0166F, 0x0167F, - 0x01681, 0x0169A, - 0x016A0, 0x016EA, - 0x016F1, 0x016F8, - 0x01700, 0x0170C, - 0x0170E, 0x01711, - 0x01720, 0x01731, - 0x01740, 0x01751, - 0x01760, 0x0176C, - 0x0176E, 0x01770, - 0x01780, 0x017B3, - 0x01820, 0x01878, - 0x01880, 0x01884, - 0x01887, 0x018A8, - 0x018B0, 0x018F5, - 0x01900, 0x0191E, - 0x01950, 0x0196D, - 0x01970, 0x01974, - 0x01980, 0x019AB, - 0x019B0, 0x019C9, - 0x01A00, 0x01A16, - 0x01A20, 0x01A54, - 0x01B05, 0x01B33, - 0x01B45, 0x01B4B, - 0x01B83, 0x01BA0, - 0x01BAE, 0x01BAF, - 0x01BBA, 0x01BE5, - 0x01C00, 0x01C23, - 0x01C4D, 0x01C4F, - 0x01C5A, 0x01C7D, - 0x01C80, 0x01C88, - 0x01C90, 0x01CBA, - 0x01CBD, 0x01CBF, - 0x01CE9, 0x01CEC, - 0x01CEE, 0x01CF3, - 0x01CF5, 0x01CF6, - 0x01D00, 0x01DBF, - 0x01E00, 0x01F15, - 0x01F18, 0x01F1D, - 0x01F20, 0x01F45, - 0x01F48, 0x01F4D, - 0x01F50, 0x01F57, - 0x01F5F, 0x01F7D, - 0x01F80, 0x01FB4, - 0x01FB6, 0x01FBC, - 0x01FC2, 0x01FC4, - 0x01FC6, 0x01FCC, - 0x01FD0, 0x01FD3, - 0x01FD6, 0x01FDB, - 0x01FE0, 0x01FEC, - 0x01FF2, 0x01FF4, - 0x01FF6, 0x01FFC, - 0x02090, 0x0209C, - 0x0210A, 0x02113, - 0x02119, 0x0211D, - 0x0212A, 0x0212D, - 0x0212F, 0x02139, - 0x0213C, 0x0213F, - 0x02145, 0x02149, - 0x02183, 0x02184, - 0x02C00, 0x02C2E, - 0x02C30, 0x02C5E, - 0x02C60, 0x02CE4, - 0x02CEB, 0x02CEE, - 0x02CF2, 0x02CF3, - 0x02D00, 0x02D25, - 0x02D30, 0x02D67, - 0x02D80, 0x02D96, - 0x02DA0, 0x02DA6, - 0x02DA8, 0x02DAE, - 0x02DB0, 0x02DB6, - 0x02DB8, 0x02DBE, - 0x02DC0, 0x02DC6, - 0x02DC8, 0x02DCE, - 0x02DD0, 0x02DD6, - 0x02DD8, 0x02DDE, - 0x03005, 0x03006, - 0x03031, 0x03035, - 0x0303B, 0x0303C, - 0x03041, 0x03096, - 0x0309D, 0x0309F, - 0x030A1, 0x030FA, - 0x030FC, 0x030FF, - 0x03105, 0x0312F, - 0x03131, 0x0318E, - 0x031A0, 0x031BA, - 0x031F0, 0x031FF, - 0x0A000, 0x0A48C, - 0x0A4D0, 0x0A4FD, - 0x0A500, 0x0A60C, - 0x0A610, 0x0A61F, - 0x0A62A, 0x0A62B, - 0x0A640, 0x0A66E, - 0x0A67F, 0x0A69D, - 0x0A6A0, 0x0A6E5, - 0x0A717, 0x0A71F, - 0x0A722, 0x0A788, - 0x0A78B, 0x0A7BF, - 0x0A7C2, 0x0A7C6, - 0x0A7F7, 0x0A801, - 0x0A803, 0x0A805, - 0x0A807, 0x0A80A, - 0x0A80C, 0x0A822, - 0x0A840, 0x0A873, - 0x0A882, 0x0A8B3, - 0x0A8F2, 0x0A8F7, - 0x0A8FD, 0x0A8FE, - 0x0A90A, 0x0A925, - 0x0A930, 0x0A946, - 0x0A960, 0x0A97C, - 0x0A984, 0x0A9B2, - 0x0A9E0, 0x0A9E4, - 0x0A9E6, 0x0A9EF, - 0x0A9FA, 0x0A9FE, - 0x0AA00, 0x0AA28, - 0x0AA40, 0x0AA42, - 0x0AA44, 0x0AA4B, - 0x0AA60, 0x0AA76, - 0x0AA7E, 0x0AAAF, - 0x0AAB5, 0x0AAB6, - 0x0AAB9, 0x0AABD, - 0x0AADB, 0x0AADD, - 0x0AAE0, 0x0AAEA, - 0x0AAF2, 0x0AAF4, - 0x0AB01, 0x0AB06, - 0x0AB09, 0x0AB0E, - 0x0AB11, 0x0AB16, - 0x0AB20, 0x0AB26, - 0x0AB28, 0x0AB2E, - 0x0AB30, 0x0AB5A, - 0x0AB5C, 0x0AB67, - 0x0AB70, 0x0ABE2, - 0x0D7B0, 0x0D7C6, - 0x0D7CB, 0x0D7FB, - 0x0F900, 0x0FA6D, - 0x0FA70, 0x0FAD9, - 0x0FB00, 0x0FB06, - 0x0FB13, 0x0FB17, - 0x0FB1F, 0x0FB28, - 0x0FB2A, 0x0FB36, - 0x0FB38, 0x0FB3C, - 0x0FB40, 0x0FB41, - 0x0FB43, 0x0FB44, - 0x0FB46, 0x0FBB1, - 0x0FBD3, 0x0FD3D, - 0x0FD50, 0x0FD8F, - 0x0FD92, 0x0FDC7, - 0x0FDF0, 0x0FDFB, - 0x0FE70, 0x0FE74, - 0x0FE76, 0x0FEFC, - 0x0FF21, 0x0FF3A, - 0x0FF41, 0x0FF5A, - 0x0FF66, 0x0FFBE, - 0x0FFC2, 0x0FFC7, - 0x0FFCA, 0x0FFCF, - 0x0FFD2, 0x0FFD7, - 0x0FFDA, 0x0FFDC, - 0x10000, 0x1000B, - 0x1000D, 0x10026, - 0x10028, 0x1003A, - 0x1003C, 0x1003D, - 0x1003F, 0x1004D, - 0x10050, 0x1005D, - 0x10080, 0x100FA, - 0x10280, 0x1029C, - 0x102A0, 0x102D0, - 0x10300, 0x1031F, - 0x1032D, 0x10340, - 0x10342, 0x10349, - 0x10350, 0x10375, - 0x10380, 0x1039D, - 0x103A0, 0x103C3, - 0x103C8, 0x103CF, - 0x10400, 0x1049D, - 0x104B0, 0x104D3, - 0x104D8, 0x104FB, - 0x10500, 0x10527, - 0x10530, 0x10563, - 0x10600, 0x10736, - 0x10740, 0x10755, - 0x10760, 0x10767, - 0x10800, 0x10805, - 0x1080A, 0x10835, - 0x10837, 0x10838, - 0x1083F, 0x10855, - 0x10860, 0x10876, - 0x10880, 0x1089E, - 0x108E0, 0x108F2, - 0x108F4, 0x108F5, - 0x10900, 0x10915, - 0x10920, 0x10939, - 0x10980, 0x109B7, - 0x109BE, 0x109BF, - 0x10A10, 0x10A13, - 0x10A15, 0x10A17, - 0x10A19, 0x10A35, - 0x10A60, 0x10A7C, - 0x10A80, 0x10A9C, - 0x10AC0, 0x10AC7, - 0x10AC9, 0x10AE4, - 0x10B00, 0x10B35, - 0x10B40, 0x10B55, - 0x10B60, 0x10B72, - 0x10B80, 0x10B91, - 0x10C00, 0x10C48, - 0x10C80, 0x10CB2, - 0x10CC0, 0x10CF2, - 0x10D00, 0x10D23, - 0x10F00, 0x10F1C, - 0x10F30, 0x10F45, - 0x10FE0, 0x10FF6, - 0x11003, 0x11037, - 0x11083, 0x110AF, - 0x110D0, 0x110E8, - 0x11103, 0x11126, - 0x11150, 0x11172, - 0x11183, 0x111B2, - 0x111C1, 0x111C4, - 0x11200, 0x11211, - 0x11213, 0x1122B, - 0x11280, 0x11286, - 0x1128A, 0x1128D, - 0x1128F, 0x1129D, - 0x1129F, 0x112A8, - 0x112B0, 0x112DE, - 0x11305, 0x1130C, - 0x1130F, 0x11310, - 0x11313, 0x11328, - 0x1132A, 0x11330, - 0x11332, 0x11333, - 0x11335, 0x11339, - 0x1135D, 0x11361, - 0x11400, 0x11434, - 0x11447, 0x1144A, - 0x11480, 0x114AF, - 0x114C4, 0x114C5, - 0x11580, 0x115AE, - 0x115D8, 0x115DB, - 0x11600, 0x1162F, - 0x11680, 0x116AA, - 0x11700, 0x1171A, - 0x11800, 0x1182B, - 0x118A0, 0x118DF, - 0x119A0, 0x119A7, - 0x119AA, 0x119D0, - 0x11A0B, 0x11A32, - 0x11A5C, 0x11A89, - 0x11AC0, 0x11AF8, - 0x11C00, 0x11C08, - 0x11C0A, 0x11C2E, - 0x11C72, 0x11C8F, - 0x11D00, 0x11D06, - 0x11D08, 0x11D09, - 0x11D0B, 0x11D30, - 0x11D60, 0x11D65, - 0x11D67, 0x11D68, - 0x11D6A, 0x11D89, - 0x11EE0, 0x11EF2, - 0x12000, 0x12399, - 0x12480, 0x12543, - 0x13000, 0x1342E, - 0x14400, 0x14646, - 0x16800, 0x16A38, - 0x16A40, 0x16A5E, - 0x16AD0, 0x16AED, - 0x16B00, 0x16B2F, - 0x16B40, 0x16B43, - 0x16B63, 0x16B77, - 0x16B7D, 0x16B8F, - 0x16E40, 0x16E7F, - 0x16F00, 0x16F4A, - 0x16F93, 0x16F9F, - 0x16FE0, 0x16FE1, - 0x18800, 0x18AF2, - 0x1B000, 0x1B11E, - 0x1B150, 0x1B152, - 0x1B164, 0x1B167, - 0x1B170, 0x1B2FB, - 0x1BC00, 0x1BC6A, - 0x1BC70, 0x1BC7C, - 0x1BC80, 0x1BC88, - 0x1BC90, 0x1BC99, - 0x1D400, 0x1D454, - 0x1D456, 0x1D49C, - 0x1D49E, 0x1D49F, - 0x1D4A5, 0x1D4A6, - 0x1D4A9, 0x1D4AC, - 0x1D4AE, 0x1D4B9, - 0x1D4BD, 0x1D4C3, - 0x1D4C5, 0x1D505, - 0x1D507, 0x1D50A, - 0x1D50D, 0x1D514, - 0x1D516, 0x1D51C, - 0x1D51E, 0x1D539, - 0x1D53B, 0x1D53E, - 0x1D540, 0x1D544, - 0x1D54A, 0x1D550, - 0x1D552, 0x1D6A5, - 0x1D6A8, 0x1D6C0, - 0x1D6C2, 0x1D6DA, - 0x1D6DC, 0x1D6FA, - 0x1D6FC, 0x1D714, - 0x1D716, 0x1D734, - 0x1D736, 0x1D74E, - 0x1D750, 0x1D76E, - 0x1D770, 0x1D788, - 0x1D78A, 0x1D7A8, - 0x1D7AA, 0x1D7C2, - 0x1D7C4, 0x1D7CB, - 0x1E100, 0x1E12C, - 0x1E137, 0x1E13D, - 0x1E2C0, 0x1E2EB, - 0x1E800, 0x1E8C4, - 0x1E900, 0x1E943, - 0x1EE00, 0x1EE03, - 0x1EE05, 0x1EE1F, - 0x1EE21, 0x1EE22, - 0x1EE29, 0x1EE32, - 0x1EE34, 0x1EE37, - 0x1EE4D, 0x1EE4F, - 0x1EE51, 0x1EE52, - 0x1EE61, 0x1EE62, - 0x1EE67, 0x1EE6A, - 0x1EE6C, 0x1EE72, - 0x1EE74, 0x1EE77, - 0x1EE79, 0x1EE7C, - 0x1EE80, 0x1EE89, - 0x1EE8B, 0x1EE9B, - 0x1EEA1, 0x1EEA3, - 0x1EEA5, 0x1EEA9, - 0x1EEAB, 0x1EEBB, - 0x2F800, 0x2FA1D, + 0x00041'i32, 0x0005A'i32, + 0x00061'i32, 0x0007A'i32, + 0x000C0'i32, 0x000D6'i32, + 0x000D8'i32, 0x000F6'i32, + 0x000F8'i32, 0x002C1'i32, + 0x002C6'i32, 0x002D1'i32, + 0x002E0'i32, 0x002E4'i32, + 0x00370'i32, 0x00374'i32, + 0x00376'i32, 0x00377'i32, + 0x0037A'i32, 0x0037D'i32, + 0x00388'i32, 0x0038A'i32, + 0x0038E'i32, 0x003A1'i32, + 0x003A3'i32, 0x003F5'i32, + 0x003F7'i32, 0x00481'i32, + 0x0048A'i32, 0x0052F'i32, + 0x00531'i32, 0x00556'i32, + 0x00560'i32, 0x00588'i32, + 0x005D0'i32, 0x005EA'i32, + 0x005EF'i32, 0x005F2'i32, + 0x00620'i32, 0x0064A'i32, + 0x0066E'i32, 0x0066F'i32, + 0x00671'i32, 0x006D3'i32, + 0x006E5'i32, 0x006E6'i32, + 0x006EE'i32, 0x006EF'i32, + 0x006FA'i32, 0x006FC'i32, + 0x00712'i32, 0x0072F'i32, + 0x0074D'i32, 0x007A5'i32, + 0x007CA'i32, 0x007EA'i32, + 0x007F4'i32, 0x007F5'i32, + 0x00800'i32, 0x00815'i32, + 0x00840'i32, 0x00858'i32, + 0x00860'i32, 0x0086A'i32, + 0x008A0'i32, 0x008B4'i32, + 0x008B6'i32, 0x008BD'i32, + 0x00904'i32, 0x00939'i32, + 0x00958'i32, 0x00961'i32, + 0x00971'i32, 0x00980'i32, + 0x00985'i32, 0x0098C'i32, + 0x0098F'i32, 0x00990'i32, + 0x00993'i32, 0x009A8'i32, + 0x009AA'i32, 0x009B0'i32, + 0x009B6'i32, 0x009B9'i32, + 0x009DC'i32, 0x009DD'i32, + 0x009DF'i32, 0x009E1'i32, + 0x009F0'i32, 0x009F1'i32, + 0x00A05'i32, 0x00A0A'i32, + 0x00A0F'i32, 0x00A10'i32, + 0x00A13'i32, 0x00A28'i32, + 0x00A2A'i32, 0x00A30'i32, + 0x00A32'i32, 0x00A33'i32, + 0x00A35'i32, 0x00A36'i32, + 0x00A38'i32, 0x00A39'i32, + 0x00A59'i32, 0x00A5C'i32, + 0x00A72'i32, 0x00A74'i32, + 0x00A85'i32, 0x00A8D'i32, + 0x00A8F'i32, 0x00A91'i32, + 0x00A93'i32, 0x00AA8'i32, + 0x00AAA'i32, 0x00AB0'i32, + 0x00AB2'i32, 0x00AB3'i32, + 0x00AB5'i32, 0x00AB9'i32, + 0x00AE0'i32, 0x00AE1'i32, + 0x00B05'i32, 0x00B0C'i32, + 0x00B0F'i32, 0x00B10'i32, + 0x00B13'i32, 0x00B28'i32, + 0x00B2A'i32, 0x00B30'i32, + 0x00B32'i32, 0x00B33'i32, + 0x00B35'i32, 0x00B39'i32, + 0x00B5C'i32, 0x00B5D'i32, + 0x00B5F'i32, 0x00B61'i32, + 0x00B85'i32, 0x00B8A'i32, + 0x00B8E'i32, 0x00B90'i32, + 0x00B92'i32, 0x00B95'i32, + 0x00B99'i32, 0x00B9A'i32, + 0x00B9E'i32, 0x00B9F'i32, + 0x00BA3'i32, 0x00BA4'i32, + 0x00BA8'i32, 0x00BAA'i32, + 0x00BAE'i32, 0x00BB9'i32, + 0x00C05'i32, 0x00C0C'i32, + 0x00C0E'i32, 0x00C10'i32, + 0x00C12'i32, 0x00C28'i32, + 0x00C2A'i32, 0x00C39'i32, + 0x00C58'i32, 0x00C5A'i32, + 0x00C60'i32, 0x00C61'i32, + 0x00C85'i32, 0x00C8C'i32, + 0x00C8E'i32, 0x00C90'i32, + 0x00C92'i32, 0x00CA8'i32, + 0x00CAA'i32, 0x00CB3'i32, + 0x00CB5'i32, 0x00CB9'i32, + 0x00CE0'i32, 0x00CE1'i32, + 0x00CF1'i32, 0x00CF2'i32, + 0x00D05'i32, 0x00D0C'i32, + 0x00D0E'i32, 0x00D10'i32, + 0x00D12'i32, 0x00D3A'i32, + 0x00D54'i32, 0x00D56'i32, + 0x00D5F'i32, 0x00D61'i32, + 0x00D7A'i32, 0x00D7F'i32, + 0x00D85'i32, 0x00D96'i32, + 0x00D9A'i32, 0x00DB1'i32, + 0x00DB3'i32, 0x00DBB'i32, + 0x00DC0'i32, 0x00DC6'i32, + 0x00E01'i32, 0x00E30'i32, + 0x00E32'i32, 0x00E33'i32, + 0x00E40'i32, 0x00E46'i32, + 0x00E81'i32, 0x00E82'i32, + 0x00E86'i32, 0x00E8A'i32, + 0x00E8C'i32, 0x00EA3'i32, + 0x00EA7'i32, 0x00EB0'i32, + 0x00EB2'i32, 0x00EB3'i32, + 0x00EC0'i32, 0x00EC4'i32, + 0x00EDC'i32, 0x00EDF'i32, + 0x00F40'i32, 0x00F47'i32, + 0x00F49'i32, 0x00F6C'i32, + 0x00F88'i32, 0x00F8C'i32, + 0x01000'i32, 0x0102A'i32, + 0x01050'i32, 0x01055'i32, + 0x0105A'i32, 0x0105D'i32, + 0x01065'i32, 0x01066'i32, + 0x0106E'i32, 0x01070'i32, + 0x01075'i32, 0x01081'i32, + 0x010A0'i32, 0x010C5'i32, + 0x010D0'i32, 0x010FA'i32, + 0x010FC'i32, 0x01248'i32, + 0x0124A'i32, 0x0124D'i32, + 0x01250'i32, 0x01256'i32, + 0x0125A'i32, 0x0125D'i32, + 0x01260'i32, 0x01288'i32, + 0x0128A'i32, 0x0128D'i32, + 0x01290'i32, 0x012B0'i32, + 0x012B2'i32, 0x012B5'i32, + 0x012B8'i32, 0x012BE'i32, + 0x012C2'i32, 0x012C5'i32, + 0x012C8'i32, 0x012D6'i32, + 0x012D8'i32, 0x01310'i32, + 0x01312'i32, 0x01315'i32, + 0x01318'i32, 0x0135A'i32, + 0x01380'i32, 0x0138F'i32, + 0x013A0'i32, 0x013F5'i32, + 0x013F8'i32, 0x013FD'i32, + 0x01401'i32, 0x0166C'i32, + 0x0166F'i32, 0x0167F'i32, + 0x01681'i32, 0x0169A'i32, + 0x016A0'i32, 0x016EA'i32, + 0x016F1'i32, 0x016F8'i32, + 0x01700'i32, 0x0170C'i32, + 0x0170E'i32, 0x01711'i32, + 0x01720'i32, 0x01731'i32, + 0x01740'i32, 0x01751'i32, + 0x01760'i32, 0x0176C'i32, + 0x0176E'i32, 0x01770'i32, + 0x01780'i32, 0x017B3'i32, + 0x01820'i32, 0x01878'i32, + 0x01880'i32, 0x01884'i32, + 0x01887'i32, 0x018A8'i32, + 0x018B0'i32, 0x018F5'i32, + 0x01900'i32, 0x0191E'i32, + 0x01950'i32, 0x0196D'i32, + 0x01970'i32, 0x01974'i32, + 0x01980'i32, 0x019AB'i32, + 0x019B0'i32, 0x019C9'i32, + 0x01A00'i32, 0x01A16'i32, + 0x01A20'i32, 0x01A54'i32, + 0x01B05'i32, 0x01B33'i32, + 0x01B45'i32, 0x01B4B'i32, + 0x01B83'i32, 0x01BA0'i32, + 0x01BAE'i32, 0x01BAF'i32, + 0x01BBA'i32, 0x01BE5'i32, + 0x01C00'i32, 0x01C23'i32, + 0x01C4D'i32, 0x01C4F'i32, + 0x01C5A'i32, 0x01C7D'i32, + 0x01C80'i32, 0x01C88'i32, + 0x01C90'i32, 0x01CBA'i32, + 0x01CBD'i32, 0x01CBF'i32, + 0x01CE9'i32, 0x01CEC'i32, + 0x01CEE'i32, 0x01CF3'i32, + 0x01CF5'i32, 0x01CF6'i32, + 0x01D00'i32, 0x01DBF'i32, + 0x01E00'i32, 0x01F15'i32, + 0x01F18'i32, 0x01F1D'i32, + 0x01F20'i32, 0x01F45'i32, + 0x01F48'i32, 0x01F4D'i32, + 0x01F50'i32, 0x01F57'i32, + 0x01F5F'i32, 0x01F7D'i32, + 0x01F80'i32, 0x01FB4'i32, + 0x01FB6'i32, 0x01FBC'i32, + 0x01FC2'i32, 0x01FC4'i32, + 0x01FC6'i32, 0x01FCC'i32, + 0x01FD0'i32, 0x01FD3'i32, + 0x01FD6'i32, 0x01FDB'i32, + 0x01FE0'i32, 0x01FEC'i32, + 0x01FF2'i32, 0x01FF4'i32, + 0x01FF6'i32, 0x01FFC'i32, + 0x02090'i32, 0x0209C'i32, + 0x0210A'i32, 0x02113'i32, + 0x02119'i32, 0x0211D'i32, + 0x0212A'i32, 0x0212D'i32, + 0x0212F'i32, 0x02139'i32, + 0x0213C'i32, 0x0213F'i32, + 0x02145'i32, 0x02149'i32, + 0x02183'i32, 0x02184'i32, + 0x02C00'i32, 0x02C2E'i32, + 0x02C30'i32, 0x02C5E'i32, + 0x02C60'i32, 0x02CE4'i32, + 0x02CEB'i32, 0x02CEE'i32, + 0x02CF2'i32, 0x02CF3'i32, + 0x02D00'i32, 0x02D25'i32, + 0x02D30'i32, 0x02D67'i32, + 0x02D80'i32, 0x02D96'i32, + 0x02DA0'i32, 0x02DA6'i32, + 0x02DA8'i32, 0x02DAE'i32, + 0x02DB0'i32, 0x02DB6'i32, + 0x02DB8'i32, 0x02DBE'i32, + 0x02DC0'i32, 0x02DC6'i32, + 0x02DC8'i32, 0x02DCE'i32, + 0x02DD0'i32, 0x02DD6'i32, + 0x02DD8'i32, 0x02DDE'i32, + 0x03005'i32, 0x03006'i32, + 0x03031'i32, 0x03035'i32, + 0x0303B'i32, 0x0303C'i32, + 0x03041'i32, 0x03096'i32, + 0x0309D'i32, 0x0309F'i32, + 0x030A1'i32, 0x030FA'i32, + 0x030FC'i32, 0x030FF'i32, + 0x03105'i32, 0x0312F'i32, + 0x03131'i32, 0x0318E'i32, + 0x031A0'i32, 0x031BA'i32, + 0x031F0'i32, 0x031FF'i32, + 0x03400'i32, 0x04DB5'i32, + 0x04E00'i32, 0x09FEF'i32, + 0x0A000'i32, 0x0A48C'i32, + 0x0A4D0'i32, 0x0A4FD'i32, + 0x0A500'i32, 0x0A60C'i32, + 0x0A610'i32, 0x0A61F'i32, + 0x0A62A'i32, 0x0A62B'i32, + 0x0A640'i32, 0x0A66E'i32, + 0x0A67F'i32, 0x0A69D'i32, + 0x0A6A0'i32, 0x0A6E5'i32, + 0x0A717'i32, 0x0A71F'i32, + 0x0A722'i32, 0x0A788'i32, + 0x0A78B'i32, 0x0A7BF'i32, + 0x0A7C2'i32, 0x0A7C6'i32, + 0x0A7F7'i32, 0x0A801'i32, + 0x0A803'i32, 0x0A805'i32, + 0x0A807'i32, 0x0A80A'i32, + 0x0A80C'i32, 0x0A822'i32, + 0x0A840'i32, 0x0A873'i32, + 0x0A882'i32, 0x0A8B3'i32, + 0x0A8F2'i32, 0x0A8F7'i32, + 0x0A8FD'i32, 0x0A8FE'i32, + 0x0A90A'i32, 0x0A925'i32, + 0x0A930'i32, 0x0A946'i32, + 0x0A960'i32, 0x0A97C'i32, + 0x0A984'i32, 0x0A9B2'i32, + 0x0A9E0'i32, 0x0A9E4'i32, + 0x0A9E6'i32, 0x0A9EF'i32, + 0x0A9FA'i32, 0x0A9FE'i32, + 0x0AA00'i32, 0x0AA28'i32, + 0x0AA40'i32, 0x0AA42'i32, + 0x0AA44'i32, 0x0AA4B'i32, + 0x0AA60'i32, 0x0AA76'i32, + 0x0AA7E'i32, 0x0AAAF'i32, + 0x0AAB5'i32, 0x0AAB6'i32, + 0x0AAB9'i32, 0x0AABD'i32, + 0x0AADB'i32, 0x0AADD'i32, + 0x0AAE0'i32, 0x0AAEA'i32, + 0x0AAF2'i32, 0x0AAF4'i32, + 0x0AB01'i32, 0x0AB06'i32, + 0x0AB09'i32, 0x0AB0E'i32, + 0x0AB11'i32, 0x0AB16'i32, + 0x0AB20'i32, 0x0AB26'i32, + 0x0AB28'i32, 0x0AB2E'i32, + 0x0AB30'i32, 0x0AB5A'i32, + 0x0AB5C'i32, 0x0AB67'i32, + 0x0AB70'i32, 0x0ABE2'i32, + 0x0AC00'i32, 0x0D7A3'i32, + 0x0D7B0'i32, 0x0D7C6'i32, + 0x0D7CB'i32, 0x0D7FB'i32, + 0x0F900'i32, 0x0FA6D'i32, + 0x0FA70'i32, 0x0FAD9'i32, + 0x0FB00'i32, 0x0FB06'i32, + 0x0FB13'i32, 0x0FB17'i32, + 0x0FB1F'i32, 0x0FB28'i32, + 0x0FB2A'i32, 0x0FB36'i32, + 0x0FB38'i32, 0x0FB3C'i32, + 0x0FB40'i32, 0x0FB41'i32, + 0x0FB43'i32, 0x0FB44'i32, + 0x0FB46'i32, 0x0FBB1'i32, + 0x0FBD3'i32, 0x0FD3D'i32, + 0x0FD50'i32, 0x0FD8F'i32, + 0x0FD92'i32, 0x0FDC7'i32, + 0x0FDF0'i32, 0x0FDFB'i32, + 0x0FE70'i32, 0x0FE74'i32, + 0x0FE76'i32, 0x0FEFC'i32, + 0x0FF21'i32, 0x0FF3A'i32, + 0x0FF41'i32, 0x0FF5A'i32, + 0x0FF66'i32, 0x0FFBE'i32, + 0x0FFC2'i32, 0x0FFC7'i32, + 0x0FFCA'i32, 0x0FFCF'i32, + 0x0FFD2'i32, 0x0FFD7'i32, + 0x0FFDA'i32, 0x0FFDC'i32, + 0x10000'i32, 0x1000B'i32, + 0x1000D'i32, 0x10026'i32, + 0x10028'i32, 0x1003A'i32, + 0x1003C'i32, 0x1003D'i32, + 0x1003F'i32, 0x1004D'i32, + 0x10050'i32, 0x1005D'i32, + 0x10080'i32, 0x100FA'i32, + 0x10280'i32, 0x1029C'i32, + 0x102A0'i32, 0x102D0'i32, + 0x10300'i32, 0x1031F'i32, + 0x1032D'i32, 0x10340'i32, + 0x10342'i32, 0x10349'i32, + 0x10350'i32, 0x10375'i32, + 0x10380'i32, 0x1039D'i32, + 0x103A0'i32, 0x103C3'i32, + 0x103C8'i32, 0x103CF'i32, + 0x10400'i32, 0x1049D'i32, + 0x104B0'i32, 0x104D3'i32, + 0x104D8'i32, 0x104FB'i32, + 0x10500'i32, 0x10527'i32, + 0x10530'i32, 0x10563'i32, + 0x10600'i32, 0x10736'i32, + 0x10740'i32, 0x10755'i32, + 0x10760'i32, 0x10767'i32, + 0x10800'i32, 0x10805'i32, + 0x1080A'i32, 0x10835'i32, + 0x10837'i32, 0x10838'i32, + 0x1083F'i32, 0x10855'i32, + 0x10860'i32, 0x10876'i32, + 0x10880'i32, 0x1089E'i32, + 0x108E0'i32, 0x108F2'i32, + 0x108F4'i32, 0x108F5'i32, + 0x10900'i32, 0x10915'i32, + 0x10920'i32, 0x10939'i32, + 0x10980'i32, 0x109B7'i32, + 0x109BE'i32, 0x109BF'i32, + 0x10A10'i32, 0x10A13'i32, + 0x10A15'i32, 0x10A17'i32, + 0x10A19'i32, 0x10A35'i32, + 0x10A60'i32, 0x10A7C'i32, + 0x10A80'i32, 0x10A9C'i32, + 0x10AC0'i32, 0x10AC7'i32, + 0x10AC9'i32, 0x10AE4'i32, + 0x10B00'i32, 0x10B35'i32, + 0x10B40'i32, 0x10B55'i32, + 0x10B60'i32, 0x10B72'i32, + 0x10B80'i32, 0x10B91'i32, + 0x10C00'i32, 0x10C48'i32, + 0x10C80'i32, 0x10CB2'i32, + 0x10CC0'i32, 0x10CF2'i32, + 0x10D00'i32, 0x10D23'i32, + 0x10F00'i32, 0x10F1C'i32, + 0x10F30'i32, 0x10F45'i32, + 0x10FE0'i32, 0x10FF6'i32, + 0x11003'i32, 0x11037'i32, + 0x11083'i32, 0x110AF'i32, + 0x110D0'i32, 0x110E8'i32, + 0x11103'i32, 0x11126'i32, + 0x11150'i32, 0x11172'i32, + 0x11183'i32, 0x111B2'i32, + 0x111C1'i32, 0x111C4'i32, + 0x11200'i32, 0x11211'i32, + 0x11213'i32, 0x1122B'i32, + 0x11280'i32, 0x11286'i32, + 0x1128A'i32, 0x1128D'i32, + 0x1128F'i32, 0x1129D'i32, + 0x1129F'i32, 0x112A8'i32, + 0x112B0'i32, 0x112DE'i32, + 0x11305'i32, 0x1130C'i32, + 0x1130F'i32, 0x11310'i32, + 0x11313'i32, 0x11328'i32, + 0x1132A'i32, 0x11330'i32, + 0x11332'i32, 0x11333'i32, + 0x11335'i32, 0x11339'i32, + 0x1135D'i32, 0x11361'i32, + 0x11400'i32, 0x11434'i32, + 0x11447'i32, 0x1144A'i32, + 0x11480'i32, 0x114AF'i32, + 0x114C4'i32, 0x114C5'i32, + 0x11580'i32, 0x115AE'i32, + 0x115D8'i32, 0x115DB'i32, + 0x11600'i32, 0x1162F'i32, + 0x11680'i32, 0x116AA'i32, + 0x11700'i32, 0x1171A'i32, + 0x11800'i32, 0x1182B'i32, + 0x118A0'i32, 0x118DF'i32, + 0x119A0'i32, 0x119A7'i32, + 0x119AA'i32, 0x119D0'i32, + 0x11A0B'i32, 0x11A32'i32, + 0x11A5C'i32, 0x11A89'i32, + 0x11AC0'i32, 0x11AF8'i32, + 0x11C00'i32, 0x11C08'i32, + 0x11C0A'i32, 0x11C2E'i32, + 0x11C72'i32, 0x11C8F'i32, + 0x11D00'i32, 0x11D06'i32, + 0x11D08'i32, 0x11D09'i32, + 0x11D0B'i32, 0x11D30'i32, + 0x11D60'i32, 0x11D65'i32, + 0x11D67'i32, 0x11D68'i32, + 0x11D6A'i32, 0x11D89'i32, + 0x11EE0'i32, 0x11EF2'i32, + 0x12000'i32, 0x12399'i32, + 0x12480'i32, 0x12543'i32, + 0x13000'i32, 0x1342E'i32, + 0x14400'i32, 0x14646'i32, + 0x16800'i32, 0x16A38'i32, + 0x16A40'i32, 0x16A5E'i32, + 0x16AD0'i32, 0x16AED'i32, + 0x16B00'i32, 0x16B2F'i32, + 0x16B40'i32, 0x16B43'i32, + 0x16B63'i32, 0x16B77'i32, + 0x16B7D'i32, 0x16B8F'i32, + 0x16E40'i32, 0x16E7F'i32, + 0x16F00'i32, 0x16F4A'i32, + 0x16F93'i32, 0x16F9F'i32, + 0x16FE0'i32, 0x16FE1'i32, + 0x17000'i32, 0x187F7'i32, + 0x18800'i32, 0x18AF2'i32, + 0x1B000'i32, 0x1B11E'i32, + 0x1B150'i32, 0x1B152'i32, + 0x1B164'i32, 0x1B167'i32, + 0x1B170'i32, 0x1B2FB'i32, + 0x1BC00'i32, 0x1BC6A'i32, + 0x1BC70'i32, 0x1BC7C'i32, + 0x1BC80'i32, 0x1BC88'i32, + 0x1BC90'i32, 0x1BC99'i32, + 0x1D400'i32, 0x1D454'i32, + 0x1D456'i32, 0x1D49C'i32, + 0x1D49E'i32, 0x1D49F'i32, + 0x1D4A5'i32, 0x1D4A6'i32, + 0x1D4A9'i32, 0x1D4AC'i32, + 0x1D4AE'i32, 0x1D4B9'i32, + 0x1D4BD'i32, 0x1D4C3'i32, + 0x1D4C5'i32, 0x1D505'i32, + 0x1D507'i32, 0x1D50A'i32, + 0x1D50D'i32, 0x1D514'i32, + 0x1D516'i32, 0x1D51C'i32, + 0x1D51E'i32, 0x1D539'i32, + 0x1D53B'i32, 0x1D53E'i32, + 0x1D540'i32, 0x1D544'i32, + 0x1D54A'i32, 0x1D550'i32, + 0x1D552'i32, 0x1D6A5'i32, + 0x1D6A8'i32, 0x1D6C0'i32, + 0x1D6C2'i32, 0x1D6DA'i32, + 0x1D6DC'i32, 0x1D6FA'i32, + 0x1D6FC'i32, 0x1D714'i32, + 0x1D716'i32, 0x1D734'i32, + 0x1D736'i32, 0x1D74E'i32, + 0x1D750'i32, 0x1D76E'i32, + 0x1D770'i32, 0x1D788'i32, + 0x1D78A'i32, 0x1D7A8'i32, + 0x1D7AA'i32, 0x1D7C2'i32, + 0x1D7C4'i32, 0x1D7CB'i32, + 0x1E100'i32, 0x1E12C'i32, + 0x1E137'i32, 0x1E13D'i32, + 0x1E2C0'i32, 0x1E2EB'i32, + 0x1E800'i32, 0x1E8C4'i32, + 0x1E900'i32, 0x1E943'i32, + 0x1EE00'i32, 0x1EE03'i32, + 0x1EE05'i32, 0x1EE1F'i32, + 0x1EE21'i32, 0x1EE22'i32, + 0x1EE29'i32, 0x1EE32'i32, + 0x1EE34'i32, 0x1EE37'i32, + 0x1EE4D'i32, 0x1EE4F'i32, + 0x1EE51'i32, 0x1EE52'i32, + 0x1EE61'i32, 0x1EE62'i32, + 0x1EE67'i32, 0x1EE6A'i32, + 0x1EE6C'i32, 0x1EE72'i32, + 0x1EE74'i32, 0x1EE77'i32, + 0x1EE79'i32, 0x1EE7C'i32, + 0x1EE80'i32, 0x1EE89'i32, + 0x1EE8B'i32, 0x1EE9B'i32, + 0x1EEA1'i32, 0x1EEA3'i32, + 0x1EEA5'i32, 0x1EEA9'i32, + 0x1EEAB'i32, 0x1EEBB'i32, + 0x20000'i32, 0x2A6D6'i32, + 0x2A700'i32, 0x2B734'i32, + 0x2B740'i32, 0x2B81D'i32, + 0x2B820'i32, 0x2CEA1'i32, + 0x2CEB0'i32, 0x2EBE0'i32, + 0x2F800'i32, 0x2FA1D'i32, ] alphaSinglets = [ - 0x000AA, - 0x000B5, - 0x000BA, - 0x002EC, - 0x002EE, - 0x0037F, - 0x00386, - 0x0038C, - 0x00559, - 0x006D5, - 0x006FF, - 0x00710, - 0x007B1, - 0x007FA, - 0x0081A, - 0x00824, - 0x00828, - 0x0093D, - 0x00950, - 0x009B2, - 0x009BD, - 0x009CE, - 0x009FC, - 0x00A5E, - 0x00ABD, - 0x00AD0, - 0x00AF9, - 0x00B3D, - 0x00B71, - 0x00B83, - 0x00B9C, - 0x00BD0, - 0x00C3D, - 0x00C80, - 0x00CBD, - 0x00CDE, - 0x00D3D, - 0x00D4E, - 0x00DBD, - 0x00E84, - 0x00EA5, - 0x00EBD, - 0x00EC6, - 0x00F00, - 0x0103F, - 0x01061, - 0x0108E, - 0x010C7, - 0x010CD, - 0x01258, - 0x012C0, - 0x017D7, - 0x017DC, - 0x018AA, - 0x01AA7, - 0x01CFA, - 0x01F59, - 0x01F5B, - 0x01F5D, - 0x01FBE, - 0x02071, - 0x0207F, - 0x02102, - 0x02107, - 0x02115, - 0x02124, - 0x02126, - 0x02128, - 0x0214E, - 0x02D27, - 0x02D2D, - 0x02D6F, - 0x02E2F, - 0x03400, - 0x04DB5, - 0x04E00, - 0x09FEF, - 0x0A8FB, - 0x0A9CF, - 0x0AA7A, - 0x0AAB1, - 0x0AAC0, - 0x0AAC2, - 0x0AC00, - 0x0D7A3, - 0x0FB1D, - 0x0FB3E, - 0x10808, - 0x1083C, - 0x10A00, - 0x10F27, - 0x11144, - 0x11176, - 0x111DA, - 0x111DC, - 0x11288, - 0x1133D, - 0x11350, - 0x1145F, - 0x114C7, - 0x11644, - 0x116B8, - 0x118FF, - 0x119E1, - 0x119E3, - 0x11A00, - 0x11A3A, - 0x11A50, - 0x11A9D, - 0x11C40, - 0x11D46, - 0x11D98, - 0x16F50, - 0x16FE3, - 0x17000, - 0x187F7, - 0x1D4A2, - 0x1D4BB, - 0x1D546, - 0x1E14E, - 0x1E94B, - 0x1EE24, - 0x1EE27, - 0x1EE39, - 0x1EE3B, - 0x1EE42, - 0x1EE47, - 0x1EE49, - 0x1EE4B, - 0x1EE54, - 0x1EE57, - 0x1EE59, - 0x1EE5B, - 0x1EE5D, - 0x1EE5F, - 0x1EE64, - 0x1EE7E, - 0x20000, - 0x2A6D6, - 0x2A700, - 0x2B734, - 0x2B740, - 0x2B81D, - 0x2B820, - 0x2CEA1, - 0x2CEB0, - 0x2EBE0, + 0x000AA'i32, + 0x000B5'i32, + 0x000BA'i32, + 0x002EC'i32, + 0x002EE'i32, + 0x0037F'i32, + 0x00386'i32, + 0x0038C'i32, + 0x00559'i32, + 0x006D5'i32, + 0x006FF'i32, + 0x00710'i32, + 0x007B1'i32, + 0x007FA'i32, + 0x0081A'i32, + 0x00824'i32, + 0x00828'i32, + 0x0093D'i32, + 0x00950'i32, + 0x009B2'i32, + 0x009BD'i32, + 0x009CE'i32, + 0x009FC'i32, + 0x00A5E'i32, + 0x00ABD'i32, + 0x00AD0'i32, + 0x00AF9'i32, + 0x00B3D'i32, + 0x00B71'i32, + 0x00B83'i32, + 0x00B9C'i32, + 0x00BD0'i32, + 0x00C3D'i32, + 0x00C80'i32, + 0x00CBD'i32, + 0x00CDE'i32, + 0x00D3D'i32, + 0x00D4E'i32, + 0x00DBD'i32, + 0x00E84'i32, + 0x00EA5'i32, + 0x00EBD'i32, + 0x00EC6'i32, + 0x00F00'i32, + 0x0103F'i32, + 0x01061'i32, + 0x0108E'i32, + 0x010C7'i32, + 0x010CD'i32, + 0x01258'i32, + 0x012C0'i32, + 0x017D7'i32, + 0x017DC'i32, + 0x018AA'i32, + 0x01AA7'i32, + 0x01CFA'i32, + 0x01F59'i32, + 0x01F5B'i32, + 0x01F5D'i32, + 0x01FBE'i32, + 0x02071'i32, + 0x0207F'i32, + 0x02102'i32, + 0x02107'i32, + 0x02115'i32, + 0x02124'i32, + 0x02126'i32, + 0x02128'i32, + 0x0214E'i32, + 0x02D27'i32, + 0x02D2D'i32, + 0x02D6F'i32, + 0x02E2F'i32, + 0x0A8FB'i32, + 0x0A9CF'i32, + 0x0AA7A'i32, + 0x0AAB1'i32, + 0x0AAC0'i32, + 0x0AAC2'i32, + 0x0FB1D'i32, + 0x0FB3E'i32, + 0x10808'i32, + 0x1083C'i32, + 0x10A00'i32, + 0x10F27'i32, + 0x11144'i32, + 0x11176'i32, + 0x111DA'i32, + 0x111DC'i32, + 0x11288'i32, + 0x1133D'i32, + 0x11350'i32, + 0x1145F'i32, + 0x114C7'i32, + 0x11644'i32, + 0x116B8'i32, + 0x118FF'i32, + 0x119E1'i32, + 0x119E3'i32, + 0x11A00'i32, + 0x11A3A'i32, + 0x11A50'i32, + 0x11A9D'i32, + 0x11C40'i32, + 0x11D46'i32, + 0x11D98'i32, + 0x16F50'i32, + 0x16FE3'i32, + 0x1D4A2'i32, + 0x1D4BB'i32, + 0x1D546'i32, + 0x1E14E'i32, + 0x1E94B'i32, + 0x1EE24'i32, + 0x1EE27'i32, + 0x1EE39'i32, + 0x1EE3B'i32, + 0x1EE42'i32, + 0x1EE47'i32, + 0x1EE49'i32, + 0x1EE4B'i32, + 0x1EE54'i32, + 0x1EE57'i32, + 0x1EE59'i32, + 0x1EE5B'i32, + 0x1EE5D'i32, + 0x1EE5F'i32, + 0x1EE64'i32, + 0x1EE7E'i32, ] spaceRanges = [ - 0x00009, 0x0000D, - 0x00020, 0x00020, - 0x00085, 0x00085, - 0x000A0, 0x000A0, - 0x01680, 0x01680, - 0x02000, 0x0200A, - 0x02028, 0x02029, - 0x0202F, 0x0202F, - 0x0205F, 0x0205F, - 0x03000, 0x03000, + 0x00009'i32, 0x0000D'i32, + 0x00020'i32, 0x00020'i32, + 0x00085'i32, 0x00085'i32, + 0x000A0'i32, 0x000A0'i32, + 0x01680'i32, 0x01680'i32, + 0x02000'i32, 0x0200A'i32, + 0x02028'i32, 0x02029'i32, + 0x0202F'i32, 0x0202F'i32, + 0x0205F'i32, 0x0205F'i32, + 0x03000'i32, 0x03000'i32, ] unicodeSpaces = [ diff --git a/lib/pure/ioselects/ioselectors_epoll.nim b/lib/pure/ioselects/ioselectors_epoll.nim index b62b4c2db..10658b78e 100644 --- a/lib/pure/ioselects/ioselectors_epoll.nim +++ b/lib/pure/ioselects/ioselectors_epoll.nim @@ -9,7 +9,7 @@ # This module implements Linux epoll(). -import posix, times, epoll +import std/[posix, times, epoll] # Maximum number of events that can be returned const MAX_EPOLL_EVENTS = 64 @@ -72,14 +72,16 @@ type SelectEvent* = ptr SelectEventImpl proc newSelector*[T](): Selector[T] = + proc initialNumFD(): int {.inline.} = + when defined(nuttx): + result = NEPOLL_MAX + else: + result = 1024 # Retrieve the maximum fd count (for current OS) via getrlimit() - var a = RLimit() - if getrlimit(posix.RLIMIT_NOFILE, a) != 0: - raiseOSError(osLastError()) - var maxFD = int(a.rlim_max) + var maxFD = maxDescriptors() doAssert(maxFD > 0) # Start with a reasonable size, checkFd() will grow this on demand - const numFD = 1024 + let numFD = initialNumFD() var epollFD = epoll_create1(O_CLOEXEC) if epollFD < 0: @@ -136,7 +138,7 @@ template checkFd(s, f) = var numFD = s.numFD while numFD <= f: numFD *= 2 when hasThreadSupport: - s.fds = reallocSharedArray(s.fds, numFD) + s.fds = reallocSharedArray(s.fds, s.numFD, numFD) else: s.fds.setLen(numFD) for i in s.numFD ..< numFD: @@ -529,4 +531,4 @@ template withData*[T](s: Selector[T], fd: SocketHandle|int, value, body1, body2 proc getFd*[T](s: Selector[T]): int = - return s.epollFd.int + return s.epollFD.int diff --git a/lib/pure/ioselects/ioselectors_kqueue.nim b/lib/pure/ioselects/ioselectors_kqueue.nim index 68be969c7..513578eda 100644 --- a/lib/pure/ioselects/ioselectors_kqueue.nim +++ b/lib/pure/ioselects/ioselectors_kqueue.nim @@ -9,7 +9,7 @@ # This module implements BSD kqueue(). -import posix, times, kqueue, nativesockets +import std/[posix, times, kqueue, nativesockets] const # Maximum number of events that can be returned. @@ -194,7 +194,9 @@ when hasThreadSupport: if s.changesLength > 0: if kevent(s.kqFD, addr(s.changes[0]), cint(s.changesLength), nil, 0, nil) == -1: - raiseIOSelectorsError(osLastError()) + let res = osLastError() + if cint(res) != ENOENT: # ignore pipes whose read end is closed + raiseIOSelectorsError(res) s.changesLength = 0 else: template modifyKQueue[T](s: Selector[T], nident: uint, nfilter: cshort, @@ -211,7 +213,9 @@ else: if length > 0: if kevent(s.kqFD, addr(s.changes[0]), length, nil, 0, nil) == -1: - raiseIOSelectorsError(osLastError()) + let res = osLastError() + if cint(res) != ENOENT: # ignore pipes whose read end is closed + raiseIOSelectorsError(res) s.changes.setLen(0) proc registerHandle*[T](s: Selector[T], fd: int | SocketHandle, diff --git a/lib/pure/ioselects/ioselectors_poll.nim b/lib/pure/ioselects/ioselectors_poll.nim index 6b10e19ae..7c5347156 100644 --- a/lib/pure/ioselects/ioselectors_poll.nim +++ b/lib/pure/ioselects/ioselectors_poll.nim @@ -9,11 +9,17 @@ # This module implements Posix poll(). -import posix, times +import std/[posix, times] # Maximum number of events that can be returned const MAX_POLL_EVENTS = 64 +const hasEventFds = defined(zephyr) or defined(nimPollHasEventFds) + +when hasEventFds: + proc eventfd(count: cuint, flags: cint): cint + {.cdecl, importc: "eventfd", header: "<sys/eventfd.h>".} + when hasThreadSupport: type SelectorImpl[T] = object @@ -53,10 +59,7 @@ else: body proc newSelector*[T](): Selector[T] = - var a = RLimit() - if getrlimit(posix.RLIMIT_NOFILE, a) != 0: - raiseIOSelectorsError(osLastError()) - var maxFD = int(a.rlim_max) + var maxFD = maxDescriptors() when hasThreadSupport: result = cast[Selector[T]](allocShared0(sizeof(SelectorImpl[T]))) @@ -187,14 +190,22 @@ proc unregister*[T](s: Selector[T], ev: SelectEvent) = s.pollRemove(fdi.cint) proc newSelectEvent*(): SelectEvent = - var fds: array[2, cint] - if posix.pipe(fds) != 0: - raiseIOSelectorsError(osLastError()) - setNonBlocking(fds[0]) - setNonBlocking(fds[1]) - result = cast[SelectEvent](allocShared0(sizeof(SelectEventImpl))) - result.rfd = fds[0] - result.wfd = fds[1] + when not hasEventFds: + var fds: array[2, cint] + if posix.pipe(fds) != 0: + raiseIOSelectorsError(osLastError()) + setNonBlocking(fds[0]) + setNonBlocking(fds[1]) + result = cast[SelectEvent](allocShared0(sizeof(SelectEventImpl))) + result.rfd = fds[0] + result.wfd = fds[1] + else: + let fdci = eventfd(0, posix.O_NONBLOCK) + if fdci == -1: + raiseIOSelectorsError(osLastError()) + result = cast[SelectEvent](allocShared0(sizeof(SelectEventImpl))) + result.rfd = fdci + result.wfd = fdci proc trigger*(ev: SelectEvent) = var data: uint64 = 1 @@ -203,7 +214,10 @@ proc trigger*(ev: SelectEvent) = proc close*(ev: SelectEvent) = let res1 = posix.close(ev.rfd) - let res2 = posix.close(ev.wfd) + let res2 = + when hasEventFds: 0 + else: posix.close(ev.wfd) + deallocShared(cast[pointer](ev)) if res1 != 0 or res2 != 0: raiseIOSelectorsError(osLastError()) diff --git a/lib/pure/ioselects/ioselectors_select.nim b/lib/pure/ioselects/ioselectors_select.nim index 6f216ac85..6c516395b 100644 --- a/lib/pure/ioselects/ioselectors_select.nim +++ b/lib/pure/ioselects/ioselectors_select.nim @@ -9,10 +9,10 @@ # This module implements Posix and Windows select(). -import times, nativesockets +import std/[times, nativesockets] when defined(windows): - import winlean + import std/winlean when defined(gcc): {.passl: "-lws2_32".} elif defined(vcc): @@ -26,27 +26,27 @@ else: #include <sys/types.h> #include <unistd.h>""" type - Fdset {.importc: "fd_set", header: platformHeaders, pure, final.} = object + FdSet {.importc: "fd_set", header: platformHeaders, pure, final.} = object var FD_SETSIZE {.importc: "FD_SETSIZE", header: platformHeaders.}: cint -proc IOFD_SET(fd: SocketHandle, fdset: ptr Fdset) +proc IOFD_SET(fd: SocketHandle, fdset: ptr FdSet) {.cdecl, importc: "FD_SET", header: platformHeaders, inline.} -proc IOFD_CLR(fd: SocketHandle, fdset: ptr Fdset) +proc IOFD_CLR(fd: SocketHandle, fdset: ptr FdSet) {.cdecl, importc: "FD_CLR", header: platformHeaders, inline.} -proc IOFD_ZERO(fdset: ptr Fdset) +proc IOFD_ZERO(fdset: ptr FdSet) {.cdecl, importc: "FD_ZERO", header: platformHeaders, inline.} when defined(windows): - proc IOFD_ISSET(fd: SocketHandle, fdset: ptr Fdset): cint + proc IOFD_ISSET(fd: SocketHandle, fdset: ptr FdSet): cint {.stdcall, importc: "FD_ISSET", header: platformHeaders, inline.} - proc ioselect(nfds: cint, readFds, writeFds, exceptFds: ptr Fdset, + proc ioselect(nfds: cint, readFds, writeFds, exceptFds: ptr FdSet, timeout: ptr Timeval): cint {.stdcall, importc: "select", header: platformHeaders.} else: - proc IOFD_ISSET(fd: SocketHandle, fdset: ptr Fdset): cint + proc IOFD_ISSET(fd: SocketHandle, fdset: ptr FdSet): cint {.cdecl, importc: "FD_ISSET", header: platformHeaders, inline.} - proc ioselect(nfds: cint, readFds, writeFds, exceptFds: ptr Fdset, + proc ioselect(nfds: cint, readFds, writeFds, exceptFds: ptr FdSet, timeout: ptr Timeval): cint {.cdecl, importc: "select", header: platformHeaders.} @@ -313,8 +313,8 @@ proc selectInto*[T](s: Selector[T], timeout: int, verifySelectParams(timeout) if timeout != -1: - when defined(genode) or defined(freertos): - tv.tv_sec = Time(timeout div 1_000) + when defined(genode) or defined(freertos) or defined(zephyr) or defined(nuttx): + tv.tv_sec = posix.Time(timeout div 1_000) else: tv.tv_sec = timeout.int32 div 1_000 tv.tv_usec = (timeout.int32 %% 1_000) * 1_000 @@ -398,18 +398,6 @@ proc contains*[T](s: Selector[T], fd: SocketHandle|int): bool {.inline.} = if s.fds[i].ident == fdi: return true -when hasThreadSupport: - template withSelectLock[T](s: Selector[T], body: untyped) = - acquire(s.lock) - {.locks: [s.lock].}: - try: - body - finally: - release(s.lock) -else: - template withSelectLock[T](s: Selector[T], body: untyped) = - body - proc getData*[T](s: Selector[T], fd: SocketHandle|int): var T = s.withSelectLock(): let fdi = int(fd) diff --git a/lib/pure/json.nim b/lib/pure/json.nim index 048ed2797..53fa7553a 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -41,13 +41,14 @@ ## For a `JsonNode` who's kind is `JObject`, you can access its fields using ## the `[]` operator. The following example shows how to do this: ## -## .. code-block:: Nim +## ```Nim ## import std/json ## ## let jsonNode = parseJson("""{"key": 3.14}""") ## ## doAssert jsonNode.kind == JObject ## doAssert jsonNode["key"].kind == JFloat +## ``` ## ## Reading values ## -------------- @@ -62,12 +63,13 @@ ## ## To retrieve the value of `"key"` you can do the following: ## -## .. code-block:: Nim +## ```Nim ## import std/json ## ## let jsonNode = parseJson("""{"key": 3.14}""") ## ## doAssert jsonNode["key"].getFloat() == 3.14 +## ``` ## ## **Important:** The `[]` operator will raise an exception when the ## specified field does not exist. @@ -79,7 +81,7 @@ ## when the field is not found. The `get`-family of procedures will return a ## type's default value when called on `nil`. ## -## .. code-block:: Nim +## ```Nim ## import std/json ## ## let jsonNode = parseJson("{}") @@ -88,6 +90,7 @@ ## doAssert jsonNode{"nope"}.getFloat() == 0 ## doAssert jsonNode{"nope"}.getStr() == "" ## doAssert jsonNode{"nope"}.getBool() == false +## ``` ## ## Using default values ## -------------------- @@ -95,7 +98,7 @@ ## The `get`-family helpers also accept an additional parameter which allow ## you to fallback to a default value should the key's values be `null`: ## -## .. code-block:: Nim +## ```Nim ## import std/json ## ## let jsonNode = parseJson("""{"key": 3.14, "key2": null}""") @@ -103,6 +106,7 @@ ## doAssert jsonNode["key"].getFloat(6.28) == 3.14 ## doAssert jsonNode["key2"].getFloat(3.14) == 3.14 ## doAssert jsonNode{"nope"}.getFloat(3.14) == 3.14 # note the {} +## ``` ## ## Unmarshalling ## ------------- @@ -113,7 +117,7 @@ ## Note: Use `Option <options.html#Option>`_ for keys sometimes missing in json ## responses, and backticks around keys with a reserved keyword as name. ## -## .. code-block:: Nim +## ```Nim ## import std/json ## import std/options ## @@ -127,6 +131,7 @@ ## let user = to(userJson, User) ## if user.`type`.isSome(): ## assert user.`type`.get() != "robot" +## ``` ## ## Creating JSON ## ============= @@ -134,7 +139,7 @@ ## This module can also be used to comfortably create JSON using the `%*` ## operator: ## -## .. code-block:: nim +## ```nim ## import std/json ## ## var hisName = "John" @@ -148,6 +153,7 @@ ## var j2 = %* {"name": "Isaac", "books": ["Robot Dreams"]} ## j2["details"] = %* {"age":35, "pi":3.1415} ## echo j2 +## ``` ## ## See also: std/jsonutils for hookable json serialization/deserialization ## of arbitrary types. @@ -159,12 +165,14 @@ runnableExamples: a1, a2, a0, a3, a4: int doAssert $(%* Foo()) == """{"a1":0,"a2":0,"a0":0,"a3":0,"a4":0}""" -import - std/[hashes, tables, strutils, lexbase, streams, macros, parsejson] +import std/[hashes, tables, strutils, lexbase, streams, macros, parsejson] import std/options # xxx remove this dependency using same approach as https://github.com/nim-lang/Nim/pull/14563 import std/private/since +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions, formatfloat] + export tables.`$` @@ -203,6 +211,8 @@ type of JArray: elems*: seq[JsonNode] +const DepthLimit = 1000 + proc newJString*(s: string): JsonNode = ## Creates a new `JString JsonNode`. result = JsonNode(kind: JString, str: s) @@ -214,10 +224,6 @@ proc newJRawNumber(s: string): JsonNode = ## to the string representation without the quotes. result = JsonNode(kind: JString, str: s, isUnquoted: true) -proc newJStringMove(s: string): JsonNode = - result = JsonNode(kind: JString) - shallowCopy(result.str, s) - proc newJInt*(n: BiggestInt): JsonNode = ## Creates a new `JInt JsonNode`. result = JsonNode(kind: JInt, num: n) @@ -334,7 +340,16 @@ proc `%`*(n: BiggestInt): JsonNode = proc `%`*(n: float): JsonNode = ## Generic constructor for JSON data. Creates a new `JFloat JsonNode`. - result = JsonNode(kind: JFloat, fnum: n) + runnableExamples: + assert $(%[NaN, Inf, -Inf, 0.0, -0.0, 1.0, 1e-2]) == """["nan","inf","-inf",0.0,-0.0,1.0,0.01]""" + assert (%NaN).kind == JString + assert (%0.0).kind == JFloat + # for those special cases, we could also have used `newJRawNumber` but then + # it would've been inconsisten with the case of `parseJson` vs `%` for representing them. + if n != n: newJString("nan") + elif n == Inf: newJString("inf") + elif n == -Inf: newJString("-inf") + else: JsonNode(kind: JFloat, fnum: n) proc `%`*(b: bool): JsonNode = ## Generic constructor for JSON data. Creates a new `JBool JsonNode`. @@ -429,7 +444,7 @@ macro `%*`*(x: untyped): untyped = ## `%` for every element. result = toJsonImpl(x) -proc `==`*(a, b: JsonNode): bool = +proc `==`*(a, b: JsonNode): bool {.noSideEffect, raises: [].} = ## Check two nodes for equality if a.isNil: if b.isNil: return true @@ -449,19 +464,25 @@ proc `==`*(a, b: JsonNode): bool = of JNull: result = true of JArray: - result = a.elems == b.elems + {.cast(raises: []).}: # bug #19303 + result = a.elems == b.elems of JObject: # we cannot use OrderedTable's equality here as # the order does not matter for equality here. if a.fields.len != b.fields.len: return false for key, val in a.fields: if not b.fields.hasKey(key): return false - if b.fields[key] != val: return false + {.cast(raises: []).}: + when defined(nimHasEffectsOf): + {.noSideEffect.}: + if b.fields[key] != val: return false + else: + if b.fields[key] != val: return false result = true proc hash*(n: OrderedTable[string, JsonNode]): Hash {.noSideEffect.} -proc hash*(n: JsonNode): Hash = +proc hash*(n: JsonNode): Hash {.noSideEffect.} = ## Compute the hash for a JSON node case n.kind of JArray: @@ -523,6 +544,24 @@ proc `[]`*(node: JsonNode, index: BackwardsIndex): JsonNode {.inline, since: (1, `[]`(node, node.len - int(index)) +proc `[]`*[U, V](a: JsonNode, x: HSlice[U, V]): JsonNode = + ## Slice operation for JArray. + ## + ## Returns the inclusive range `[a[x.a], a[x.b]]`: + runnableExamples: + import std/json + let arr = %[0,1,2,3,4,5] + doAssert arr[2..4] == %[2,3,4] + doAssert arr[2..^2] == %[2,3,4] + doAssert arr[^4..^2] == %[2,3,4] + + assert(a.kind == JArray) + result = newJArray() + let xa = (when x.a is BackwardsIndex: a.len - int(x.a) else: int(x.a)) + let L = (when x.b is BackwardsIndex: a.len - int(x.b) else: int(x.b)) - xa + 1 + for i in 0..<L: + result.add(a[i + xa]) + proc hasKey*(node: JsonNode, key: string): bool = ## Checks if `key` exists in `node`. assert(node.kind == JObject) @@ -595,7 +634,7 @@ proc delete*(obj: JsonNode, key: string) = obj.fields.del(key) proc copy*(p: JsonNode): JsonNode = - ## Performs a deep copy of `a`. + ## Performs a deep copy of `p`. case p.kind of JString: result = newJString(p.str) @@ -663,6 +702,47 @@ proc escapeJson*(s: string): string = result = newStringOfCap(s.len + s.len shr 3) escapeJson(s, result) +proc toUgly*(result: var string, node: JsonNode) = + ## Converts `node` to its JSON Representation, without + ## regard for human readability. Meant to improve `$` string + ## conversion performance. + ## + ## JSON representation is stored in the passed `result` + ## + ## This provides higher efficiency than the `pretty` procedure as it + ## does **not** attempt to format the resulting JSON to make it human readable. + var comma = false + case node.kind: + of JArray: + result.add "[" + for child in node.elems: + if comma: result.add "," + else: comma = true + result.toUgly child + result.add "]" + of JObject: + result.add "{" + for key, value in pairs(node.fields): + if comma: result.add "," + else: comma = true + key.escapeJson(result) + result.add ":" + result.toUgly value + result.add "}" + of JString: + if node.isUnquoted: + result.add node.str + else: + escapeJson(node.str, result) + of JInt: + result.addInt(node.num) + of JFloat: + result.addFloat(node.fnum) + of JBool: + result.add(if node.bval: "true" else: "false") + of JNull: + result.add "null" + proc toPretty(result: var string, node: JsonNode, indent = 2, ml = true, lstArr = false, currIndent = 0) = case node.kind @@ -690,10 +770,7 @@ proc toPretty(result: var string, node: JsonNode, indent = 2, ml = true, result.add("{}") of JString: if lstArr: result.indent(currIndent) - if node.isUnquoted: - result.add node.str - else: - escapeJson(node.str, result) + toUgly(result, node) of JInt: if lstArr: result.indent(currIndent) result.addInt(node.num) @@ -744,47 +821,6 @@ proc pretty*(node: JsonNode, indent = 2): string = result = "" toPretty(result, node, indent) -proc toUgly*(result: var string, node: JsonNode) = - ## Converts `node` to its JSON Representation, without - ## regard for human readability. Meant to improve `$` string - ## conversion performance. - ## - ## JSON representation is stored in the passed `result` - ## - ## This provides higher efficiency than the `pretty` procedure as it - ## does **not** attempt to format the resulting JSON to make it human readable. - var comma = false - case node.kind: - of JArray: - result.add "[" - for child in node.elems: - if comma: result.add "," - else: comma = true - result.toUgly child - result.add "]" - of JObject: - result.add "{" - for key, value in pairs(node.fields): - if comma: result.add "," - else: comma = true - key.escapeJson(result) - result.add ":" - result.toUgly value - result.add "}" - of JString: - if node.isUnquoted: - result.add node.str - else: - node.str.escapeJson(result) - of JInt: - result.addInt(node.num) - of JFloat: - result.addFloat(node.fnum) - of JBool: - result.add(if node.bval: "true" else: "false") - of JNull: - result.add "null" - proc `$`*(node: JsonNode): string = ## Converts `node` to its JSON Representation on one line. result = newStringOfCap(node.len shl 1) @@ -822,13 +858,17 @@ iterator mpairs*(node: var JsonNode): tuple[key: string, val: var JsonNode] = for key, val in mpairs(node.fields): yield (key, val) -proc parseJson(p: var JsonParser; rawIntegers, rawFloats: bool): JsonNode = +proc parseJson(p: var JsonParser; rawIntegers, rawFloats: bool, depth = 0): JsonNode = ## Parses JSON from a JSON Parser `p`. case p.tok of tkString: # we capture 'p.a' here, so we need to give it a fresh buffer afterwards: - result = newJStringMove(p.a) - p.a = "" + when defined(gcArc) or defined(gcOrc) or defined(gcAtomicArc): + result = JsonNode(kind: JString, str: move p.a) + else: + result = JsonNode(kind: JString) + shallowCopy(result.str, p.a) + p.a = "" discard getTok(p) of tkInt: if rawIntegers: @@ -858,6 +898,8 @@ proc parseJson(p: var JsonParser; rawIntegers, rawFloats: bool): JsonNode = result = newJNull() discard getTok(p) of tkCurlyLe: + if depth > DepthLimit: + raiseParseErr(p, "}") result = newJObject() discard getTok(p) while p.tok != tkCurlyRi: @@ -866,16 +908,18 @@ proc parseJson(p: var JsonParser; rawIntegers, rawFloats: bool): JsonNode = var key = p.a discard getTok(p) eat(p, tkColon) - var val = parseJson(p, rawIntegers, rawFloats) + var val = parseJson(p, rawIntegers, rawFloats, depth+1) result[key] = val if p.tok != tkComma: break discard getTok(p) eat(p, tkCurlyRi) of tkBracketLe: + if depth > DepthLimit: + raiseParseErr(p, "]") result = newJArray() discard getTok(p) while p.tok != tkBracketRi: - result.add(parseJson(p, rawIntegers, rawFloats)) + result.add(parseJson(p, rawIntegers, rawFloats, depth+1)) if p.tok != tkComma: break discard getTok(p) eat(p, tkBracketRi) @@ -922,21 +966,22 @@ proc parseJson*(s: Stream, filename: string = ""; rawIntegers = false, rawFloats when defined(js): from std/math import `mod` - from std/jsffi import JSObject, `[]`, to + from std/jsffi import JsObject, `[]`, to from std/private/jsutils import getProtoName, isInteger, isSafeInteger - proc parseNativeJson(x: cstring): JSObject {.importjs: "JSON.parse(#)".} + proc parseNativeJson(x: cstring): JsObject {.importjs: "JSON.parse(#)".} - proc getVarType(x: JSObject): JsonNodeKind = + proc getVarType(x: JsObject, isRawNumber: var bool): JsonNodeKind = result = JNull case $getProtoName(x) # TODO: Implicit returns fail here. of "[object Array]": return JArray of "[object Object]": return JObject of "[object Number]": - if isInteger(x): + if isInteger(x) and 1.0 / cast[float](x) != -Inf: # preserve -0.0 as float if isSafeInteger(x): return JInt else: + isRawNumber = true return JString else: return JFloat @@ -945,35 +990,41 @@ when defined(js): of "[object String]": return JString else: assert false - proc len(x: JSObject): int = - assert x.getVarType == JArray - asm """ + proc len(x: JsObject): int = + {.emit: """ `result` = `x`.length; - """ + """.} - proc convertObject(x: JSObject): JsonNode = - case getVarType(x) + proc convertObject(x: JsObject): JsonNode = + var isRawNumber = false + case getVarType(x, isRawNumber) of JArray: result = newJArray() for i in 0 ..< x.len: result.add(x[i].convertObject()) of JObject: result = newJObject() - asm """for (var property in `x`) { + {.emit: """for (var property in `x`) { if (`x`.hasOwnProperty(property)) { - """ + """.} + var nimProperty: cstring - var nimValue: JSObject - asm "`nimProperty` = property; `nimValue` = `x`[property];" + var nimValue: JsObject + {.emit: "`nimProperty` = property; `nimValue` = `x`[property];".} result[$nimProperty] = nimValue.convertObject() - asm "}}" + {.emit: "}}".} of JInt: result = newJInt(x.to(int)) of JFloat: result = newJFloat(x.to(float)) of JString: # Dunno what to do with isUnquoted here - result = newJString($x.to(cstring)) + if isRawNumber: + var value: cstring + {.emit: "`value` = `x`.toString();".} + result = newJRawNumber($value) + else: + result = newJString($x.to(cstring)) of JBool: result = newJBool(x.to(bool)) of JNull: @@ -1017,297 +1068,306 @@ template verifyJsonKind(node: JsonNode, kinds: set[JsonNodeKind], ] raise newException(JsonKindError, msg) -when defined(nimFixedForwardGeneric): - - macro isRefSkipDistinct*(arg: typed): untyped = - ## internal only, do not use - var impl = getTypeImpl(arg) - if impl.kind == nnkBracketExpr and impl[0].eqIdent("typeDesc"): - impl = getTypeImpl(impl[1]) - while impl.kind == nnkDistinctTy: - impl = getTypeImpl(impl[0]) - result = newLit(impl.kind == nnkRefTy) - - # The following forward declarations don't work in older versions of Nim - - # forward declare all initFromJson - - proc initFromJson(dst: var string; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson(dst: var bool; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson(dst: var JsonNode; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[T: SomeInteger](dst: var T; jsonNode: JsonNode, jsonPath: var string) - proc initFromJson[T: SomeFloat](dst: var T; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[T: enum](dst: var T; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[T](dst: var seq[T]; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[S, T](dst: var array[S, T]; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[T](dst: var Table[string, T]; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[T](dst: var OrderedTable[string, T]; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[T](dst: var ref T; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[T](dst: var Option[T]; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[T: distinct](dst: var T; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[T: object|tuple](dst: var T; jsonNode: JsonNode; jsonPath: var string) - - # initFromJson definitions - - proc initFromJson(dst: var string; jsonNode: JsonNode; jsonPath: var string) = - verifyJsonKind(jsonNode, {JString, JNull}, jsonPath) - # since strings don't have a nil state anymore, this mapping of - # JNull to the default string is questionable. `none(string)` and - # `some("")` have the same potentional json value `JNull`. - if jsonNode.kind == JNull: - dst = "" - else: - dst = jsonNode.str - - proc initFromJson(dst: var bool; jsonNode: JsonNode; jsonPath: var string) = - verifyJsonKind(jsonNode, {JBool}, jsonPath) - dst = jsonNode.bval - - proc initFromJson(dst: var JsonNode; jsonNode: JsonNode; jsonPath: var string) = - if jsonNode == nil: - raise newException(KeyError, "key not found: " & jsonPath) - dst = jsonNode.copy - - proc initFromJson[T: SomeInteger](dst: var T; jsonNode: JsonNode, jsonPath: var string) = - when T is uint|uint64: - case jsonNode.kind - of JString: - dst = T(parseBiggestUInt(jsonNode.str)) - else: - verifyJsonKind(jsonNode, {JInt}, jsonPath) - dst = T(jsonNode.num) - else: - verifyJsonKind(jsonNode, {JInt}, jsonPath) - dst = cast[T](jsonNode.num) +macro isRefSkipDistinct*(arg: typed): untyped = + ## internal only, do not use + var impl = getTypeImpl(arg) + if impl.kind == nnkBracketExpr and impl[0].eqIdent("typeDesc"): + impl = getTypeImpl(impl[1]) + while impl.kind == nnkDistinctTy: + impl = getTypeImpl(impl[0]) + result = newLit(impl.kind == nnkRefTy) + +# The following forward declarations don't work in older versions of Nim + +# forward declare all initFromJson + +proc initFromJson(dst: var string; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson(dst: var bool; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson(dst: var JsonNode; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T: SomeInteger](dst: var T; jsonNode: JsonNode, jsonPath: var string) +proc initFromJson[T: SomeFloat](dst: var T; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T: enum](dst: var T; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T](dst: var seq[T]; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[S, T](dst: var array[S, T]; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T](dst: var Table[string, T]; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T](dst: var OrderedTable[string, T]; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T](dst: var ref T; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T](dst: var Option[T]; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T: distinct](dst: var T; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T: object|tuple](dst: var T; jsonNode: JsonNode; jsonPath: var string) + +# initFromJson definitions + +proc initFromJson(dst: var string; jsonNode: JsonNode; jsonPath: var string) = + verifyJsonKind(jsonNode, {JString, JNull}, jsonPath) + # since strings don't have a nil state anymore, this mapping of + # JNull to the default string is questionable. `none(string)` and + # `some("")` have the same potentional json value `JNull`. + if jsonNode.kind == JNull: + dst = "" + else: + dst = jsonNode.str + +proc initFromJson(dst: var bool; jsonNode: JsonNode; jsonPath: var string) = + verifyJsonKind(jsonNode, {JBool}, jsonPath) + dst = jsonNode.bval + +proc initFromJson(dst: var JsonNode; jsonNode: JsonNode; jsonPath: var string) = + if jsonNode == nil: + raise newException(KeyError, "key not found: " & jsonPath) + dst = jsonNode.copy - proc initFromJson[T: SomeFloat](dst: var T; jsonNode: JsonNode; jsonPath: var string) = - verifyJsonKind(jsonNode, {JInt, JFloat}, jsonPath) +proc initFromJson[T: SomeInteger](dst: var T; jsonNode: JsonNode, jsonPath: var string) = + when T is uint|uint64 or int.sizeof == 4: + verifyJsonKind(jsonNode, {JInt, JString}, jsonPath) + case jsonNode.kind + of JString: + let x = parseBiggestUInt(jsonNode.str) + dst = cast[T](x) + else: + dst = T(jsonNode.num) + else: + verifyJsonKind(jsonNode, {JInt}, jsonPath) + dst = cast[T](jsonNode.num) + +proc initFromJson[T: SomeFloat](dst: var T; jsonNode: JsonNode; jsonPath: var string) = + verifyJsonKind(jsonNode, {JInt, JFloat, JString}, jsonPath) + if jsonNode.kind == JString: + case jsonNode.str + of "nan": + let b = NaN + dst = T(b) + # dst = NaN # would fail some tests because range conversions would cause CT error + # in some cases; but this is not a hot-spot inside this branch and backend can optimize this. + of "inf": + let b = Inf + dst = T(b) + of "-inf": + let b = -Inf + dst = T(b) + else: raise newException(JsonKindError, "expected 'nan|inf|-inf', got " & jsonNode.str) + else: if jsonNode.kind == JFloat: dst = T(jsonNode.fnum) else: dst = T(jsonNode.num) - proc initFromJson[T: enum](dst: var T; jsonNode: JsonNode; jsonPath: var string) = - verifyJsonKind(jsonNode, {JString}, jsonPath) - dst = parseEnum[T](jsonNode.getStr) - - proc initFromJson[T](dst: var seq[T]; jsonNode: JsonNode; jsonPath: var string) = - verifyJsonKind(jsonNode, {JArray}, jsonPath) - dst.setLen jsonNode.len - let orignalJsonPathLen = jsonPath.len - for i in 0 ..< jsonNode.len: - jsonPath.add '[' - jsonPath.addInt i - jsonPath.add ']' - initFromJson(dst[i], jsonNode[i], jsonPath) - jsonPath.setLen orignalJsonPathLen - - proc initFromJson[S,T](dst: var array[S,T]; jsonNode: JsonNode; jsonPath: var string) = - verifyJsonKind(jsonNode, {JArray}, jsonPath) - let originalJsonPathLen = jsonPath.len - for i in 0 ..< jsonNode.len: - jsonPath.add '[' - jsonPath.addInt i - jsonPath.add ']' - initFromJson(dst[i.S], jsonNode[i], jsonPath) # `.S` for enum indexed arrays - jsonPath.setLen originalJsonPathLen - - proc initFromJson[T](dst: var Table[string,T]; jsonNode: JsonNode; jsonPath: var string) = - dst = initTable[string, T]() - verifyJsonKind(jsonNode, {JObject}, jsonPath) - let originalJsonPathLen = jsonPath.len - for key in keys(jsonNode.fields): - jsonPath.add '.' - jsonPath.add key - initFromJson(mgetOrPut(dst, key, default(T)), jsonNode[key], jsonPath) - jsonPath.setLen originalJsonPathLen - - proc initFromJson[T](dst: var OrderedTable[string,T]; jsonNode: JsonNode; jsonPath: var string) = - dst = initOrderedTable[string,T]() - verifyJsonKind(jsonNode, {JObject}, jsonPath) - let originalJsonPathLen = jsonPath.len - for key in keys(jsonNode.fields): - jsonPath.add '.' - jsonPath.add key - initFromJson(mgetOrPut(dst, key, default(T)), jsonNode[key], jsonPath) - jsonPath.setLen originalJsonPathLen - - proc initFromJson[T](dst: var ref T; jsonNode: JsonNode; jsonPath: var string) = - verifyJsonKind(jsonNode, {JObject, JNull}, jsonPath) - if jsonNode.kind == JNull: - dst = nil - else: - dst = new(T) - initFromJson(dst[], jsonNode, jsonPath) - - proc initFromJson[T](dst: var Option[T]; jsonNode: JsonNode; jsonPath: var string) = - if jsonNode != nil and jsonNode.kind != JNull: - when T is ref: - dst = some(new(T)) - else: - dst = some(default(T)) - initFromJson(dst.get, jsonNode, jsonPath) - - macro assignDistinctImpl[T: distinct](dst: var T;jsonNode: JsonNode; jsonPath: var string) = - let typInst = getTypeInst(dst) - let typImpl = getTypeImpl(dst) - let baseTyp = typImpl[0] +proc initFromJson[T: enum](dst: var T; jsonNode: JsonNode; jsonPath: var string) = + verifyJsonKind(jsonNode, {JString}, jsonPath) + dst = parseEnum[T](jsonNode.getStr) + +proc initFromJson[T](dst: var seq[T]; jsonNode: JsonNode; jsonPath: var string) = + verifyJsonKind(jsonNode, {JArray}, jsonPath) + dst.setLen jsonNode.len + let orignalJsonPathLen = jsonPath.len + for i in 0 ..< jsonNode.len: + jsonPath.add '[' + jsonPath.addInt i + jsonPath.add ']' + initFromJson(dst[i], jsonNode[i], jsonPath) + jsonPath.setLen orignalJsonPathLen + +proc initFromJson[S,T](dst: var array[S,T]; jsonNode: JsonNode; jsonPath: var string) = + verifyJsonKind(jsonNode, {JArray}, jsonPath) + let originalJsonPathLen = jsonPath.len + for i in 0 ..< jsonNode.len: + jsonPath.add '[' + jsonPath.addInt i + jsonPath.add ']' + initFromJson(dst[i.S], jsonNode[i], jsonPath) # `.S` for enum indexed arrays + jsonPath.setLen originalJsonPathLen + +proc initFromJson[T](dst: var Table[string,T]; jsonNode: JsonNode; jsonPath: var string) = + dst = initTable[string, T]() + verifyJsonKind(jsonNode, {JObject}, jsonPath) + let originalJsonPathLen = jsonPath.len + for key in keys(jsonNode.fields): + jsonPath.add '.' + jsonPath.add key + initFromJson(mgetOrPut(dst, key, default(T)), jsonNode[key], jsonPath) + jsonPath.setLen originalJsonPathLen + +proc initFromJson[T](dst: var OrderedTable[string,T]; jsonNode: JsonNode; jsonPath: var string) = + dst = initOrderedTable[string,T]() + verifyJsonKind(jsonNode, {JObject}, jsonPath) + let originalJsonPathLen = jsonPath.len + for key in keys(jsonNode.fields): + jsonPath.add '.' + jsonPath.add key + initFromJson(mgetOrPut(dst, key, default(T)), jsonNode[key], jsonPath) + jsonPath.setLen originalJsonPathLen + +proc initFromJson[T](dst: var ref T; jsonNode: JsonNode; jsonPath: var string) = + verifyJsonKind(jsonNode, {JObject, JNull}, jsonPath) + if jsonNode.kind == JNull: + dst = nil + else: + dst = new(T) + initFromJson(dst[], jsonNode, jsonPath) - result = quote do: +proc initFromJson[T](dst: var Option[T]; jsonNode: JsonNode; jsonPath: var string) = + if jsonNode != nil and jsonNode.kind != JNull: + when T is ref: + dst = some(new(T)) + else: + dst = some(default(T)) + initFromJson(dst.get, jsonNode, jsonPath) + +macro assignDistinctImpl[T: distinct](dst: var T;jsonNode: JsonNode; jsonPath: var string) = + let typInst = getTypeInst(dst) + let typImpl = getTypeImpl(dst) + let baseTyp = typImpl[0] + + result = quote do: + initFromJson(`baseTyp`(`dst`), `jsonNode`, `jsonPath`) + +proc initFromJson[T: distinct](dst: var T; jsonNode: JsonNode; jsonPath: var string) = + assignDistinctImpl(dst, jsonNode, jsonPath) + +proc detectIncompatibleType(typeExpr, lineinfoNode: NimNode) = + if typeExpr.kind == nnkTupleConstr: + error("Use a named tuple instead of: " & typeExpr.repr, lineinfoNode) + +proc foldObjectBody(dst, typeNode, tmpSym, jsonNode, jsonPath, originalJsonPathLen: NimNode) = + case typeNode.kind + of nnkEmpty: + discard + of nnkRecList, nnkTupleTy: + for it in typeNode: + foldObjectBody(dst, it, tmpSym, jsonNode, jsonPath, originalJsonPathLen) + + of nnkIdentDefs: + typeNode.expectLen 3 + let fieldSym = typeNode[0] + let fieldNameLit = newLit(fieldSym.strVal) + let fieldPathLit = newLit("." & fieldSym.strVal) + let fieldType = typeNode[1] + + # Detecting incompatiple tuple types in `assignObjectImpl` only + # would be much cleaner, but the ast for tuple types does not + # contain usable type information. + detectIncompatibleType(fieldType, fieldSym) + + dst.add quote do: + jsonPath.add `fieldPathLit` when nimvm: - # workaround #12282 - var tmp: `baseTyp` - initFromJson( tmp, `jsonNode`, `jsonPath`) - `dst` = `typInst`(tmp) - else: - initFromJson( `baseTyp`(`dst`), `jsonNode`, `jsonPath`) - - proc initFromJson[T: distinct](dst: var T; jsonNode: JsonNode; jsonPath: var string) = - assignDistinctImpl(dst, jsonNode, jsonPath) - - proc detectIncompatibleType(typeExpr, lineinfoNode: NimNode) = - if typeExpr.kind == nnkTupleConstr: - error("Use a named tuple instead of: " & typeExpr.repr, lineinfoNode) - - proc foldObjectBody(dst, typeNode, tmpSym, jsonNode, jsonPath, originalJsonPathLen: NimNode) = - case typeNode.kind - of nnkEmpty: - discard - of nnkRecList, nnkTupleTy: - for it in typeNode: - foldObjectBody(dst, it, tmpSym, jsonNode, jsonPath, originalJsonPathLen) - - of nnkIdentDefs: - typeNode.expectLen 3 - let fieldSym = typeNode[0] - let fieldNameLit = newLit(fieldSym.strVal) - let fieldPathLit = newLit("." & fieldSym.strVal) - let fieldType = typeNode[1] - - # Detecting incompatiple tuple types in `assignObjectImpl` only - # would be much cleaner, but the ast for tuple types does not - # contain usable type information. - detectIncompatibleType(fieldType, fieldSym) - - dst.add quote do: - jsonPath.add `fieldPathLit` - when nimvm: - when isRefSkipDistinct(`tmpSym`.`fieldSym`): - # workaround #12489 - var tmp: `fieldType` - initFromJson(tmp, getOrDefault(`jsonNode`,`fieldNameLit`), `jsonPath`) - `tmpSym`.`fieldSym` = tmp - else: - initFromJson(`tmpSym`.`fieldSym`, getOrDefault(`jsonNode`,`fieldNameLit`), `jsonPath`) + when isRefSkipDistinct(`tmpSym`.`fieldSym`): + # workaround #12489 + var tmp: `fieldType` + initFromJson(tmp, getOrDefault(`jsonNode`,`fieldNameLit`), `jsonPath`) + `tmpSym`.`fieldSym` = tmp else: initFromJson(`tmpSym`.`fieldSym`, getOrDefault(`jsonNode`,`fieldNameLit`), `jsonPath`) - jsonPath.setLen `originalJsonPathLen` - - of nnkRecCase: - let kindSym = typeNode[0][0] - let kindNameLit = newLit(kindSym.strVal) - let kindPathLit = newLit("." & kindSym.strVal) - let kindType = typeNode[0][1] - let kindOffsetLit = newLit(uint(getOffset(kindSym))) - dst.add quote do: - var kindTmp: `kindType` - jsonPath.add `kindPathLit` - initFromJson(kindTmp, `jsonNode`[`kindNameLit`], `jsonPath`) - jsonPath.setLen `originalJsonPathLen` - when defined js: + else: + initFromJson(`tmpSym`.`fieldSym`, getOrDefault(`jsonNode`,`fieldNameLit`), `jsonPath`) + jsonPath.setLen `originalJsonPathLen` + + of nnkRecCase: + let kindSym = typeNode[0][0] + let kindNameLit = newLit(kindSym.strVal) + let kindPathLit = newLit("." & kindSym.strVal) + let kindType = typeNode[0][1] + let kindOffsetLit = newLit(uint(getOffset(kindSym))) + dst.add quote do: + var kindTmp: `kindType` + jsonPath.add `kindPathLit` + initFromJson(kindTmp, `jsonNode`[`kindNameLit`], `jsonPath`) + jsonPath.setLen `originalJsonPathLen` + when defined js: + `tmpSym`.`kindSym` = kindTmp + else: + when nimvm: `tmpSym`.`kindSym` = kindTmp else: - when nimvm: - `tmpSym`.`kindSym` = kindTmp - else: - # fuck it, assign kind field anyway - ((cast[ptr `kindType`](cast[uint](`tmpSym`.addr) + `kindOffsetLit`))[]) = kindTmp - dst.add nnkCaseStmt.newTree(nnkDotExpr.newTree(tmpSym, kindSym)) - for i in 1 ..< typeNode.len: - foldObjectBody(dst, typeNode[i], tmpSym, jsonNode, jsonPath, originalJsonPathLen) - - of nnkOfBranch, nnkElse: - let ofBranch = newNimNode(typeNode.kind) - for i in 0 ..< typeNode.len-1: - ofBranch.add copyNimTree(typeNode[i]) - let dstInner = newNimNode(nnkStmtListExpr) - foldObjectBody(dstInner, typeNode[^1], tmpSym, jsonNode, jsonPath, originalJsonPathLen) - # resOuter now contains the inner stmtList - ofBranch.add dstInner - dst[^1].expectKind nnkCaseStmt - dst[^1].add ofBranch - - of nnkObjectTy: - typeNode[0].expectKind nnkEmpty - typeNode[1].expectKind {nnkEmpty, nnkOfInherit} - if typeNode[1].kind == nnkOfInherit: - let base = typeNode[1][0] - var impl = getTypeImpl(base) - while impl.kind in {nnkRefTy, nnkPtrTy}: - impl = getTypeImpl(impl[0]) - foldObjectBody(dst, impl, tmpSym, jsonNode, jsonPath, originalJsonPathLen) - let body = typeNode[2] - foldObjectBody(dst, body, tmpSym, jsonNode, jsonPath, originalJsonPathLen) + # fuck it, assign kind field anyway + ((cast[ptr `kindType`](cast[uint](`tmpSym`.addr) + `kindOffsetLit`))[]) = kindTmp + dst.add nnkCaseStmt.newTree(nnkDotExpr.newTree(tmpSym, kindSym)) + for i in 1 ..< typeNode.len: + foldObjectBody(dst, typeNode[i], tmpSym, jsonNode, jsonPath, originalJsonPathLen) + + of nnkOfBranch, nnkElse: + let ofBranch = newNimNode(typeNode.kind) + for i in 0 ..< typeNode.len-1: + ofBranch.add copyNimTree(typeNode[i]) + let dstInner = newNimNode(nnkStmtListExpr) + foldObjectBody(dstInner, typeNode[^1], tmpSym, jsonNode, jsonPath, originalJsonPathLen) + # resOuter now contains the inner stmtList + ofBranch.add dstInner + dst[^1].expectKind nnkCaseStmt + dst[^1].add ofBranch + + of nnkObjectTy: + typeNode[0].expectKind nnkEmpty + typeNode[1].expectKind {nnkEmpty, nnkOfInherit} + if typeNode[1].kind == nnkOfInherit: + let base = typeNode[1][0] + var impl = getTypeImpl(base) + while impl.kind in {nnkRefTy, nnkPtrTy}: + impl = getTypeImpl(impl[0]) + foldObjectBody(dst, impl, tmpSym, jsonNode, jsonPath, originalJsonPathLen) + let body = typeNode[2] + foldObjectBody(dst, body, tmpSym, jsonNode, jsonPath, originalJsonPathLen) - else: - error("unhandled kind: " & $typeNode.kind, typeNode) - - macro assignObjectImpl[T](dst: var T; jsonNode: JsonNode; jsonPath: var string) = - let typeSym = getTypeInst(dst) - let originalJsonPathLen = genSym(nskLet, "originalJsonPathLen") - result = newStmtList() - result.add quote do: - let `originalJsonPathLen` = len(`jsonPath`) - if typeSym.kind in {nnkTupleTy, nnkTupleConstr}: - # both, `dst` and `typeSym` don't have good lineinfo. But nothing - # else is available here. - detectIncompatibleType(typeSym, dst) - foldObjectBody(result, typeSym, dst, jsonNode, jsonPath, originalJsonPathLen) - else: - foldObjectBody(result, typeSym.getTypeImpl, dst, jsonNode, jsonPath, originalJsonPathLen) - - proc initFromJson[T: object|tuple](dst: var T; jsonNode: JsonNode; jsonPath: var string) = - assignObjectImpl(dst, jsonNode, jsonPath) - - proc to*[T](node: JsonNode, t: typedesc[T]): T = - ## `Unmarshals`:idx: the specified node into the object type specified. - ## - ## Known limitations: - ## - ## * Heterogeneous arrays are not supported. - ## * Sets in object variants are not supported. - ## * Not nil annotations are not supported. - ## - runnableExamples: - let jsonNode = parseJson(""" - { - "person": { - "name": "Nimmer", - "age": 21 - }, - "list": [1, 2, 3, 4] - } - """) - - type - Person = object - name: string - age: int - - Data = object - person: Person - list: seq[int] - - var data = to(jsonNode, Data) - doAssert data.person.name == "Nimmer" - doAssert data.person.age == 21 - doAssert data.list == @[1, 2, 3, 4] - - var jsonPath = "" - initFromJson(result, node, jsonPath) + else: + error("unhandled kind: " & $typeNode.kind, typeNode) + +macro assignObjectImpl[T](dst: var T; jsonNode: JsonNode; jsonPath: var string) = + let typeSym = getTypeInst(dst) + let originalJsonPathLen = genSym(nskLet, "originalJsonPathLen") + result = newStmtList() + result.add quote do: + let `originalJsonPathLen` = len(`jsonPath`) + if typeSym.kind in {nnkTupleTy, nnkTupleConstr}: + # both, `dst` and `typeSym` don't have good lineinfo. But nothing + # else is available here. + detectIncompatibleType(typeSym, dst) + foldObjectBody(result, typeSym, dst, jsonNode, jsonPath, originalJsonPathLen) + else: + foldObjectBody(result, typeSym.getTypeImpl, dst, jsonNode, jsonPath, originalJsonPathLen) + +proc initFromJson[T: object|tuple](dst: var T; jsonNode: JsonNode; jsonPath: var string) = + assignObjectImpl(dst, jsonNode, jsonPath) + +proc to*[T](node: JsonNode, t: typedesc[T]): T = + ## `Unmarshals`:idx: the specified node into the object type specified. + ## + ## Known limitations: + ## + ## * Heterogeneous arrays are not supported. + ## * Sets in object variants are not supported. + ## * Not nil annotations are not supported. + ## + runnableExamples: + let jsonNode = parseJson(""" + { + "person": { + "name": "Nimmer", + "age": 21 + }, + "list": [1, 2, 3, 4] + } + """) + + type + Person = object + name: string + age: int + + Data = object + person: Person + list: seq[int] + + var data = to(jsonNode, Data) + doAssert data.person.name == "Nimmer" + doAssert data.person.age == 21 + doAssert data.list == @[1, 2, 3, 4] + + var jsonPath = "" + result = default(T) + initFromJson(result, node, jsonPath) when false: - import os + import std/os var s = newFileStream(paramStr(1), fmRead) if s == nil: quit("cannot open the file" & paramStr(1)) var x: JsonParser diff --git a/lib/pure/lexbase.nim b/lib/pure/lexbase.nim index 166d8286b..1efd97b24 100644 --- a/lib/pure/lexbase.nim +++ b/lib/pure/lexbase.nim @@ -1,6 +1,6 @@ # # -# The Nim Compiler +# Nim's Runtime Library # (c) Copyright 2009 Andreas Rumpf # # See the file "copying.txt", included in this @@ -12,7 +12,10 @@ ## needs refilling. import - strutils, streams + std/[strutils, streams] + +when defined(nimPreviewSlimSystem): + import std/assertions const EndOfFile* = '\0' ## end of file marker @@ -101,9 +104,9 @@ proc fillBaseLexer(L: var BaseLexer, pos: int): int = result = 0 proc handleCR*(L: var BaseLexer, pos: int): int = - ## Call this if you scanned over '\c' in the buffer; it returns the + ## Call this if you scanned over `'\c'` in the buffer; it returns the ## position to continue the scanning from. `pos` must be the position - ## of the '\c'. + ## of the `'\c'`. assert(L.buf[pos] == '\c') inc(L.lineNumber) result = fillBaseLexer(L, pos) @@ -112,9 +115,9 @@ proc handleCR*(L: var BaseLexer, pos: int): int = L.lineStart = result proc handleLF*(L: var BaseLexer, pos: int): int = - ## Call this if you scanned over '\L' in the buffer; it returns the + ## Call this if you scanned over `'\L'` in the buffer; it returns the ## position to continue the scanning from. `pos` must be the position - ## of the '\L'. + ## of the `'\L'`. assert(L.buf[pos] == '\L') inc(L.lineNumber) result = fillBaseLexer(L, pos) #L.lastNL := result-1; // BUGFIX: was: result; diff --git a/lib/pure/logging.nim b/lib/pure/logging.nim index b2ace79ab..c30f68af8 100644 --- a/lib/pure/logging.nim +++ b/lib/pure/logging.nim @@ -17,10 +17,11 @@ ## ## To get started, first create a logger: ## -## .. code-block:: +## ```Nim ## import std/logging ## ## var logger = newConsoleLogger() +## ``` ## ## The logger that was created above logs to the console, but this module ## also provides loggers that log to files, such as the @@ -30,9 +31,10 @@ ## Once a logger has been created, call its `log proc ## <#log.e,ConsoleLogger,Level,varargs[string,]>`_ to log a message: ## -## .. code-block:: +## ```Nim ## logger.log(lvlInfo, "a log message") ## # Output: INFO a log message +## ``` ## ## The ``INFO`` within the output is the result of a format string being ## prepended to the message, and it will differ depending on the message's @@ -47,9 +49,8 @@ ## ## .. warning:: ## For loggers that log to a console or to files, only error and fatal -## messages will cause their output buffers to be flushed immediately. -## Use the `flushFile proc <io.html#flushFile,File>`_ to flush the buffer -## manually if needed. +## messages will cause their output buffers to be flushed immediately by default. +## set ``flushThreshold`` when creating the logger to change this. ## ## Handlers ## -------- @@ -59,7 +60,7 @@ ## used with the `addHandler proc<#addHandler,Logger>`_, which is demonstrated ## in the following example: ## -## .. code-block:: +## ```Nim ## import std/logging ## ## var consoleLog = newConsoleLogger() @@ -69,17 +70,19 @@ ## addHandler(consoleLog) ## addHandler(fileLog) ## addHandler(rollingLog) +## ``` ## ## After doing this, use either the `log template ## <#log.t,Level,varargs[string,]>`_ or one of the level-specific templates, ## such as the `error template<#error.t,varargs[string,]>`_, to log messages ## to all registered handlers at once. ## -## .. code-block:: +## ```Nim ## # This example uses the loggers created above ## log(lvlError, "an error occurred") ## error("an error occurred") # Equivalent to the above line ## info("something normal happened") # Will not be written to errors.log +## ``` ## ## Note that a message's level is still checked against each handler's ## ``levelThreshold`` and the global log filter. @@ -117,12 +120,13 @@ ## ## The following example illustrates how to use format strings: ## -## .. code-block:: +## ```Nim ## import std/logging ## ## var logger = newConsoleLogger(fmtStr="[$time] - $levelname: ") ## logger.log(lvlInfo, "this is a message") ## # Output: [19:50:13] - INFO: this is a message +## ``` ## ## Notes when using multiple threads ## --------------------------------- @@ -142,9 +146,12 @@ ## * `strscans module<strscans.html>`_ for ``scanf`` and ``scanp`` macros, which ## offer easier substring extraction than regular expressions -import strutils, times +import std/[strutils, times] when not defined(js): - import os + import std/os + +when defined(nimPreviewSlimSystem): + import std/syncio type Level* = enum ## \ @@ -197,6 +204,15 @@ const ## If a different format string is preferred, refer to the ## `documentation about format strings<#basic-usage-format-strings>`_ ## for more information, including a list of available variables. + defaultFlushThreshold = when NimMajor >= 2: + when defined(nimV1LogFlushBehavior): lvlError else: lvlAll + else: + when defined(nimFlushAllLogs): lvlAll else: lvlError + ## The threshold above which log messages to file-like loggers + ## are automatically flushed. + ## + ## By default, only error and fatal messages are logged, + ## but defining ``-d:nimFlushAllLogs`` will make all levels be flushed type Logger* = ref object of RootObj @@ -225,6 +241,8 @@ type ## * `FileLogger<#FileLogger>`_ ## * `RollingFileLogger<#RollingFileLogger>`_ useStderr*: bool ## If true, writes to stderr; otherwise, writes to stdout + flushThreshold*: Level ## Only messages that are at or above this + ## threshold will be flushed immediately when not defined(js): type @@ -240,6 +258,8 @@ when not defined(js): ## * `ConsoleLogger<#ConsoleLogger>`_ ## * `RollingFileLogger<#RollingFileLogger>`_ file*: File ## The wrapped file + flushThreshold*: Level ## Only messages that are at or above this + ## threshold will be flushed immediately RollingFileLogger* = ref object of FileLogger ## A logger that writes log messages to a file while performing log @@ -345,8 +365,8 @@ method log*(logger: ConsoleLogger, level: Level, args: varargs[string, `$`]) = ## `setLogFilter proc<#setLogFilter,Level>`_. ## ## **Note:** Only error and fatal messages will cause the output buffer - ## to be flushed immediately. Use the `flushFile proc - ## <io.html#flushFile,File>`_ to flush the buffer manually if needed. + ## to be flushed immediately by default. Set ``flushThreshold`` when creating + ## the logger to change this. ## ## See also: ## * `log method<#log.e,FileLogger,Level,varargs[string,]>`_ @@ -357,14 +377,15 @@ method log*(logger: ConsoleLogger, level: Level, args: varargs[string, `$`]) = ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## var consoleLog = newConsoleLogger() ## consoleLog.log(lvlInfo, "this is a message") ## consoleLog.log(lvlError, "error code is: ", 404) + ## ``` if level >= logging.level and level >= logger.levelThreshold: let ln = substituteLog(logger.fmtStr, level, args) when defined(js): - let cln: cstring = ln + let cln = ln.cstring case level of lvlDebug: {.emit: "console.debug(`cln`);".} of lvlInfo: {.emit: "console.info(`cln`);".} @@ -377,12 +398,12 @@ method log*(logger: ConsoleLogger, level: Level, args: varargs[string, `$`]) = if logger.useStderr: handle = stderr writeLine(handle, ln) - if level in {lvlError, lvlFatal}: flushFile(handle) + if level >= logger.flushThreshold: flushFile(handle) except IOError: discard proc newConsoleLogger*(levelThreshold = lvlAll, fmtStr = defaultFmtStr, - useStderr = false): ConsoleLogger = + useStderr = false, flushThreshold = defaultFlushThreshold): ConsoleLogger = ## Creates a new `ConsoleLogger<#ConsoleLogger>`_. ## ## By default, log messages are written to ``stdout``. If ``useStderr`` is @@ -399,13 +420,15 @@ proc newConsoleLogger*(levelThreshold = lvlAll, fmtStr = defaultFmtStr, ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## var normalLog = newConsoleLogger() ## var formatLog = newConsoleLogger(fmtStr=verboseFmtStr) ## var errorLog = newConsoleLogger(levelThreshold=lvlError, useStderr=true) + ## ``` new result result.fmtStr = fmtStr result.levelThreshold = levelThreshold + result.flushThreshold = flushThreshold result.useStderr = useStderr when not defined(js): @@ -421,8 +444,8 @@ when not defined(js): ## ## **Notes:** ## * Only error and fatal messages will cause the output buffer - ## to be flushed immediately. Use the `flushFile proc - ## <io.html#flushFile,File>`_ to flush the buffer manually if needed. + ## to be flushed immediately by default. Set ``flushThreshold`` when creating + ## the logger to change this. ## * This method is not available for the JavaScript backend. ## ## See also: @@ -434,13 +457,14 @@ when not defined(js): ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## var fileLog = newFileLogger("messages.log") ## fileLog.log(lvlInfo, "this is a message") ## fileLog.log(lvlError, "error code is: ", 404) + ## ``` if level >= logging.level and level >= logger.levelThreshold: writeLine(logger.file, substituteLog(logger.fmtStr, level, args)) - if level in {lvlError, lvlFatal}: flushFile(logger.file) + if level >= logger.flushThreshold: flushFile(logger.file) proc defaultFilename*(): string = ## Returns the filename that is used by default when naming log files. @@ -451,7 +475,8 @@ when not defined(js): proc newFileLogger*(file: File, levelThreshold = lvlAll, - fmtStr = defaultFmtStr): FileLogger = + fmtStr = defaultFmtStr, + flushThreshold = defaultFlushThreshold): FileLogger = ## Creates a new `FileLogger<#FileLogger>`_ that uses the given file handle. ## ## **Note:** This proc is not available for the JavaScript backend. @@ -464,7 +489,7 @@ when not defined(js): ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## var messages = open("messages.log", fmWrite) ## var formatted = open("formatted.log", fmWrite) ## var errors = open("errors.log", fmWrite) @@ -472,16 +497,19 @@ when not defined(js): ## var normalLog = newFileLogger(messages) ## var formatLog = newFileLogger(formatted, fmtStr=verboseFmtStr) ## var errorLog = newFileLogger(errors, levelThreshold=lvlError) + ## ``` new(result) result.file = file result.levelThreshold = levelThreshold + result.flushThreshold = flushThreshold result.fmtStr = fmtStr proc newFileLogger*(filename = defaultFilename(), mode: FileMode = fmAppend, levelThreshold = lvlAll, fmtStr = defaultFmtStr, - bufSize: int = -1): FileLogger = + bufSize: int = -1, + flushThreshold = defaultFlushThreshold): FileLogger = ## Creates a new `FileLogger<#FileLogger>`_ that logs to a file with the ## given filename. ## @@ -500,12 +528,13 @@ when not defined(js): ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## var normalLog = newFileLogger("messages.log") ## var formatLog = newFileLogger("formatted.log", fmtStr=verboseFmtStr) ## var errorLog = newFileLogger("errors.log", levelThreshold=lvlError) + ## ``` let file = open(filename, mode, bufSize = bufSize) - newFileLogger(file, levelThreshold, fmtStr) + newFileLogger(file, levelThreshold, fmtStr, flushThreshold) # ------ @@ -537,7 +566,8 @@ when not defined(js): levelThreshold = lvlAll, fmtStr = defaultFmtStr, maxLines: Positive = 1000, - bufSize: int = -1): RollingFileLogger = + bufSize: int = -1, + flushThreshold = defaultFlushThreshold): RollingFileLogger = ## Creates a new `RollingFileLogger<#RollingFileLogger>`_. ## ## Once the current log file being written to contains ``maxLines`` lines, @@ -559,11 +589,12 @@ when not defined(js): ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## var normalLog = newRollingFileLogger("messages.log") ## var formatLog = newRollingFileLogger("formatted.log", fmtStr=verboseFmtStr) ## var shortLog = newRollingFileLogger("short.log", maxLines=200) ## var errorLog = newRollingFileLogger("errors.log", levelThreshold=lvlError) + ## ``` new(result) result.levelThreshold = levelThreshold result.fmtStr = fmtStr @@ -573,6 +604,7 @@ when not defined(js): result.curLine = 0 result.baseName = filename result.baseMode = mode + result.flushThreshold = flushThreshold result.logFiles = countFiles(filename) @@ -599,8 +631,8 @@ when not defined(js): ## ## **Notes:** ## * Only error and fatal messages will cause the output buffer - ## to be flushed immediately. Use the `flushFile proc - ## <io.html#flushFile,File>`_ to flush the buffer manually if needed. + ## to be flushed immediately by default. Set ``flushThreshold`` when creating + ## the logger to change this. ## * This method is not available for the JavaScript backend. ## ## See also: @@ -612,10 +644,11 @@ when not defined(js): ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## var rollingLog = newRollingFileLogger("messages.log") ## rollingLog.log(lvlInfo, "this is a message") ## rollingLog.log(lvlError, "error code is: ", 404) + ## ``` if level >= logging.level and level >= logger.levelThreshold: if logger.curLine >= logger.maxLines: logger.file.close() @@ -626,7 +659,7 @@ when not defined(js): bufSize = logger.bufSize) writeLine(logger.file, substituteLog(logger.fmtStr, level, args)) - if level in {lvlError, lvlFatal}: flushFile(logger.file) + if level >= logger.flushThreshold: flushFile(logger.file) logger.curLine.inc # -------- @@ -645,11 +678,12 @@ template log*(level: Level, args: varargs[string, `$`]) = ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## var logger = newConsoleLogger() ## addHandler(logger) ## ## log(lvlInfo, "This is an example.") + ## ``` ## ## See also: ## * `debug template<#debug.t,varargs[string,]>`_ @@ -674,11 +708,12 @@ template debug*(args: varargs[string, `$`]) = ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## var logger = newConsoleLogger() ## addHandler(logger) ## ## debug("myProc called with arguments: foo, 5") + ## ``` ## ## See also: ## * `log template<#log.t,Level,varargs[string,]>`_ @@ -695,11 +730,12 @@ template info*(args: varargs[string, `$`]) = ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## var logger = newConsoleLogger() ## addHandler(logger) ## ## info("Application started successfully.") + ## ``` ## ## See also: ## * `log template<#log.t,Level,varargs[string,]>`_ @@ -716,11 +752,12 @@ template notice*(args: varargs[string, `$`]) = ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## var logger = newConsoleLogger() ## addHandler(logger) ## ## notice("An important operation has completed.") + ## ``` ## ## See also: ## * `log template<#log.t,Level,varargs[string,]>`_ @@ -736,11 +773,12 @@ template warn*(args: varargs[string, `$`]) = ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## var logger = newConsoleLogger() ## addHandler(logger) ## ## warn("The previous operation took too long to process.") + ## ``` ## ## See also: ## * `log template<#log.t,Level,varargs[string,]>`_ @@ -758,11 +796,12 @@ template error*(args: varargs[string, `$`]) = ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## var logger = newConsoleLogger() ## addHandler(logger) ## ## error("An exception occurred while processing the form.") + ## ``` ## ## See also: ## * `log template<#log.t,Level,varargs[string,]>`_ @@ -779,11 +818,12 @@ template fatal*(args: varargs[string, `$`]) = ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## var logger = newConsoleLogger() ## addHandler(logger) ## ## fatal("Can't open database -- exiting.") + ## ``` ## ## See also: ## * `log template<#log.t,Level,varargs[string,]>`_ @@ -799,6 +839,7 @@ proc addHandler*(handler: Logger) = ## each of those threads. ## ## See also: + ## * `removeHandler proc`_ ## * `getHandlers proc<#getHandlers>`_ runnableExamples: var logger = newConsoleLogger() @@ -806,6 +847,16 @@ proc addHandler*(handler: Logger) = doAssert logger in getHandlers() handlers.add(handler) +proc removeHandler*(handler: Logger) = + ## Removes a logger from the list of registered handlers. + ## + ## Note that for n times a logger is registered, n calls to this proc + ## are required to remove that logger. + for i, hnd in handlers: + if hnd == handler: + handlers.delete(i) + return + proc getHandlers*(): seq[Logger] = ## Returns a list of all the registered handlers. ## diff --git a/lib/pure/marshal.nim b/lib/pure/marshal.nim index 936b8fe94..f9b3d3e4c 100644 --- a/lib/pure/marshal.nim +++ b/lib/pure/marshal.nim @@ -56,6 +56,9 @@ Please contribute a new implementation.""".} import std/[streams, typeinfo, json, intsets, tables, unicode] +when defined(nimPreviewSlimSystem): + import std/[assertions, formatfloat] + proc ptrToInt(x: pointer): int {.inline.} = result = cast[int](x) # don't skip alignment @@ -163,7 +166,10 @@ proc loadAny(p: var JsonParser, a: Any, t: var Table[BiggestInt, pointer]) = of akSequence: case p.kind of jsonNull: - setPointer(a, nil) + when defined(nimSeqsV2): + invokeNewSeq(a, 0) + else: + setPointer(a, nil) next(p) of jsonArrayStart: next(p) @@ -204,7 +210,8 @@ proc loadAny(p: var JsonParser, a: Any, t: var Table[BiggestInt, pointer]) = setPointer(a, nil) next(p) of jsonInt: - setPointer(a, t.getOrDefault(p.getInt)) + var raw = t.getOrDefault(p.getInt) + setPointer(a, addr raw) next(p) of jsonArrayStart: next(p) @@ -230,7 +237,10 @@ proc loadAny(p: var JsonParser, a: Any, t: var Table[BiggestInt, pointer]) = of akString: case p.kind of jsonNull: - setPointer(a, nil) + when defined(nimSeqsV2): + setString(a, "") + else: + setPointer(a, nil) next(p) of jsonString: setString(a, p.str) @@ -282,7 +292,7 @@ proc load*[T](s: Stream, data: var T) = var tab = initTable[BiggestInt, pointer]() loadAny(s, toAny(data), tab) -proc store*[T](s: Stream, data: T) = +proc store*[T](s: Stream, data: sink T) = ## Stores `data` into the stream `s`. Raises `IOError` in case of an error. runnableExamples: import std/streams @@ -295,10 +305,16 @@ proc store*[T](s: Stream, data: T) = var stored = initIntSet() var d: T - shallowCopy(d, data) + when defined(gcArc) or defined(gcOrc)or defined(gcAtomicArc): + d = data + else: + shallowCopy(d, data) storeAny(s, toAny(d), stored) -proc `$$`*[T](x: T): string = +proc loadVM[T](typ: typedesc[T], x: T): string = + discard "the implementation is in the compiler/vmops" + +proc `$$`*[T](x: sink T): string = ## Returns a string representation of `x` (serialization, marshalling). ## ## **Note:** to serialize `x` to JSON use `%x` from the `json` module @@ -313,12 +329,21 @@ proc `$$`*[T](x: T): string = let y = $$x assert y == """{"id": 1, "bar": "baz"}""" - var stored = initIntSet() - var d: T - shallowCopy(d, x) - var s = newStringStream() - storeAny(s, toAny(d), stored) - result = s.data + when nimvm: + result = loadVM(T, x) + else: + var stored = initIntSet() + var d: T + when defined(gcArc) or defined(gcOrc) or defined(gcAtomicArc): + d = x + else: + shallowCopy(d, x) + var s = newStringStream() + storeAny(s, toAny(d), stored) + result = s.data + +proc toVM[T](typ: typedesc[T], data: string): T = + discard "the implementation is in the compiler/vmops" proc to*[T](data: string): T = ## Reads data and transforms it to a type `T` (deserialization, unmarshalling). @@ -335,5 +360,8 @@ proc to*[T](data: string): T = assert z.id == 1 assert z.bar == "baz" - var tab = initTable[BiggestInt, pointer]() - loadAny(newStringStream(data), toAny(result), tab) + when nimvm: + result = toVM(T, data) + else: + var tab = initTable[BiggestInt, pointer]() + loadAny(newStringStream(data), toAny(result), tab) diff --git a/lib/pure/math.nim b/lib/pure/math.nim index 74a98e655..ed7d2382f 100644 --- a/lib/pure/math.nim +++ b/lib/pure/math.nim @@ -47,7 +47,6 @@ runnableExamples: ## * `fenv module <fenv.html>`_ for handling of floating-point rounding ## and exceptions (overflow, zero-divide, etc.) ## * `random module <random.html>`_ for a fast and tiny random number generator -## * `mersenne module <mersenne.html>`_ for the Mersenne Twister random number generator ## * `stats module <stats.html>`_ for statistical analysis ## * `strformat module <strformat.html>`_ for formatting floats for printing ## * `system module <system.html>`_ for some very basic and trivial math operators @@ -59,8 +58,13 @@ import std/private/since # of the standard library! import std/[bitops, fenv] +import system/countbits_impl -when defined(c) or defined(cpp): +when defined(nimPreviewSlimSystem): + import std/assertions + + +when not defined(js) and not defined(nimscript): # C proc c_isnan(x: float): bool {.importc: "isnan", header: "<math.h>".} # a generic like `x: SomeFloat` might work too if this is implemented via a C macro. @@ -69,16 +73,46 @@ when defined(c) or defined(cpp): proc c_signbit(x: SomeFloat): cint {.importc: "signbit", header: "<math.h>".} - func c_frexp*(x: cfloat, exponent: var cint): cfloat {. - importc: "frexpf", header: "<math.h>", deprecated: "Use `frexp` instead".} - func c_frexp*(x: cdouble, exponent: var cint): cdouble {. - importc: "frexp", header: "<math.h>", deprecated: "Use `frexp` instead".} - # don't export `c_frexp` in the future and remove `c_frexp2`. func c_frexp2(x: cfloat, exponent: var cint): cfloat {. importc: "frexpf", header: "<math.h>".} func c_frexp2(x: cdouble, exponent: var cint): cdouble {. importc: "frexp", header: "<math.h>".} + + type + div_t {.importc, header: "<stdlib.h>".} = object + quot: cint + rem: cint + ldiv_t {.importc, header: "<stdlib.h>".} = object + quot: clong + rem: clong + lldiv_t {.importc, header: "<stdlib.h>".} = object + quot: clonglong + rem: clonglong + + when cint isnot clong: + func divmod_c(x, y: cint): div_t {.importc: "div", header: "<stdlib.h>".} + when clong isnot clonglong: + func divmod_c(x, y: clonglong): lldiv_t {.importc: "lldiv", header: "<stdlib.h>".} + func divmod_c(x, y: clong): ldiv_t {.importc: "ldiv", header: "<stdlib.h>".} + func divmod*[T: SomeInteger](x, y: T): (T, T) {.inline.} = + ## Specialized instructions for computing both division and modulus. + ## Return structure is: (quotient, remainder) + runnableExamples: + doAssert divmod(5, 2) == (2, 1) + doAssert divmod(5, -3) == (-1, 2) + when T is cint | clong | clonglong: + when compileOption("overflowChecks"): + if y == 0: + raise new(DivByZeroDefect) + elif (x == T.low and y == -1.T): + raise new(OverflowDefect) + let res = divmod_c(x, y) + result[0] = res.quot + result[1] = res.rem + else: + result[0] = x div y + result[1] = x mod y func binom*(n, k: int): int = ## Computes the [binomial coefficient](https://en.wikipedia.org/wiki/Binomial_coefficient). @@ -122,7 +156,7 @@ func fac*(n: int): int = {.push checks: off, line_dir: off, stack_trace: off.} -when defined(posix) and not defined(genode): +when defined(posix) and not defined(genode) and not defined(macosx): {.passl: "-lm".} const @@ -167,7 +201,7 @@ func isNaN*(x: SomeFloat): bool {.inline, since: (1,5,1).} = template fn: untyped = result = x != x when nimvm: fn() else: - when defined(js): fn() + when defined(js) or defined(nimscript): fn() else: result = c_isnan(x) when defined(js): @@ -186,13 +220,13 @@ when defined(js): let a = newFloat64Array(buffer) let b = newUint32Array(buffer) a[0] = x - asm """ + {.emit: """ function updateBit(num, bitPos, bitVal) { return (num & ~(1 << bitPos)) | (bitVal << bitPos); } `b`[1] = updateBit(`b`[1], 31, `sgn`); - `result` = `a`[0] - """ + `result` = `a`[0]; + """.} proc signbit*(x: SomeFloat): bool {.inline, since: (1, 5, 1).} = ## Returns true if `x` is negative, false otherwise. @@ -237,7 +271,6 @@ func classify*(x: float): FloatClass = ## Classifies a floating point value. ## ## Returns `x`'s class as specified by the `FloatClass enum<#FloatClass>`_. - ## Doesn't work with `--passc:-ffast-math`. runnableExamples: doAssert classify(0.3) == fcNormal doAssert classify(0.0) == fcZero @@ -246,6 +279,7 @@ func classify*(x: float): FloatClass = doAssert classify(5.0e-324) == fcSubnormal # JavaScript and most C compilers have no classify: + if isNan(x): return fcNan if x == 0.0: if 1.0 / x == Inf: return fcZero @@ -254,7 +288,6 @@ func classify*(x: float): FloatClass = if x * 0.5 == x: if x > 0.0: return fcInf else: return fcNegInf - if x != x: return fcNan if abs(x) < MinFloatNormal: return fcSubnormal return fcNormal @@ -328,68 +361,8 @@ func nextPowerOfTwo*(x: int): int = result = result or (result shr 1) result += 1 + ord(x <= 0) -func sum*[T](x: openArray[T]): T = - ## Computes the sum of the elements in `x`. - ## - ## If `x` is empty, 0 is returned. - ## - ## **See also:** - ## * `prod func <#prod,openArray[T]>`_ - ## * `cumsum func <#cumsum,openArray[T]>`_ - ## * `cumsummed func <#cumsummed,openArray[T]>`_ - runnableExamples: - doAssert sum([1, 2, 3, 4]) == 10 - doAssert sum([-4, 3, 5]) == 4 - - for i in items(x): result = result + i - -func prod*[T](x: openArray[T]): T = - ## Computes the product of the elements in `x`. - ## - ## If `x` is empty, 1 is returned. - ## - ## **See also:** - ## * `sum func <#sum,openArray[T]>`_ - ## * `fac func <#fac,int>`_ - runnableExamples: - doAssert prod([1, 2, 3, 4]) == 24 - doAssert prod([-4, 3, 5]) == -60 - - result = T(1) - for i in items(x): result = result * i - -func cumsummed*[T](x: openArray[T]): seq[T] = - ## Returns the cumulative (aka prefix) summation of `x`. - ## - ## If `x` is empty, `@[]` is returned. - ## - ## **See also:** - ## * `sum func <#sum,openArray[T]>`_ - ## * `cumsum func <#cumsum,openArray[T]>`_ for the in-place version - runnableExamples: - doAssert cumsummed([1, 2, 3, 4]) == @[1, 3, 6, 10] - - let xLen = x.len - if xLen == 0: - return @[] - result.setLen(xLen) - result[0] = x[0] - for i in 1 ..< xLen: result[i] = result[i - 1] + x[i] -func cumsum*[T](x: var openArray[T]) = - ## Transforms `x` in-place (must be declared as `var`) into its - ## cumulative (aka prefix) summation. - ## - ## **See also:** - ## * `sum func <#sum,openArray[T]>`_ - ## * `cumsummed func <#cumsummed,openArray[T]>`_ for a version which - ## returns a cumsummed sequence - runnableExamples: - var a = [1, 2, 3, 4] - cumsum(a) - doAssert a == @[1, 3, 6, 10] - for i in 1 ..< x.len: x[i] = x[i - 1] + x[i] when not defined(js): # C func sqrt*(x: float32): float32 {.importc: "sqrtf", header: "<math.h>".} @@ -855,6 +828,14 @@ else: # JS doAssert -6.5 mod 2.5 == -1.5 doAssert 6.5 mod -2.5 == 1.5 doAssert -6.5 mod -2.5 == -1.5 + + func divmod*[T:SomeInteger](num, denom: T): (T, T) = + runnableExamples: + doAssert divmod(5, 2) == (2, 1) + doAssert divmod(5, -3) == (-1, 2) + result[0] = num div denom + result[1] = num mod denom + func round*[T: float32|float64](x: T, places: int): T = ## Decimal rounding on a binary floating point number. @@ -941,6 +922,58 @@ func euclMod*[T: SomeNumber](x, y: T): T {.since: (1, 5, 1).} = if result < 0: result += abs(y) +func ceilDiv*[T: SomeInteger](x, y: T): T {.inline, since: (1, 5, 1).} = + ## Ceil division is conceptually defined as `ceil(x / y)`. + ## + ## Assumes `x >= 0` and `y > 0` (and `x + y - 1 <= high(T)` if T is SomeUnsignedInt). + ## + ## This is different from the `system.div <system.html#div,int,int>`_ + ## operator, which works like `trunc(x / y)`. + ## That is, `div` rounds towards `0` and `ceilDiv` rounds up. + ## + ## This function has the above input limitation, because that allows the + ## compiler to generate faster code and it is rarely used with + ## negative values or unsigned integers close to `high(T)/2`. + ## If you need a `ceilDiv` that works with any input, see: + ## https://github.com/demotomohiro/divmath. + ## + ## **See also:** + ## * `system.div proc <system.html#div,int,int>`_ for integer division + ## * `floorDiv func <#floorDiv,T,T>`_ for integer division which rounds down. + runnableExamples: + assert ceilDiv(12, 3) == 4 + assert ceilDiv(13, 3) == 5 + + when sizeof(T) == 8: + type UT = uint64 + elif sizeof(T) == 4: + type UT = uint32 + elif sizeof(T) == 2: + type UT = uint16 + elif sizeof(T) == 1: + type UT = uint8 + else: + {.fatal: "Unsupported int type".} + + assert x >= 0 and y > 0 + when T is SomeUnsignedInt: + assert x + y - 1 >= x + + # If the divisor is const, the backend C/C++ compiler generates code without a `div` + # instruction, as it is slow on most CPUs. + # If the divisor is a power of 2 and a const unsigned integer type, the + # compiler generates faster code. + # If the divisor is const and a signed integer, generated code becomes slower + # than the code with unsigned integers, because division with signed integers + # need to works for both positive and negative value without `idiv`/`sdiv`. + # That is why this code convert parameters to unsigned. + # This post contains a comparison of the performance of signed/unsigned integers: + # https://github.com/nim-lang/Nim/pull/18596#issuecomment-894420984. + # If signed integer arguments were not converted to unsigned integers, + # `ceilDiv` wouldn't work for any positive signed integer value, because + # `x + (y - 1)` can overflow. + ((x.UT + (y.UT - 1.UT)) div y.UT).T + func frexp*[T: float32|float64](x: T): tuple[frac: T, exp: int] {.inline.} = ## Splits `x` into a normalized fraction `frac` and an integral power of 2 `exp`, ## such that `abs(frac) in 0.5..<1` and `x == frac * 2 ^ exp`, except for special @@ -949,18 +982,26 @@ func frexp*[T: float32|float64](x: T): tuple[frac: T, exp: int] {.inline.} = doAssert frexp(8.0) == (0.5, 4) doAssert frexp(-8.0) == (-0.5, 4) doAssert frexp(0.0) == (0.0, 0) + # special cases: - when not defined(windows): - doAssert frexp(-0.0) == (-0.0, 0) # signbit preserved for +-0 + when sizeof(int) == 8: + doAssert frexp(-0.0).frac.signbit # signbit preserved for +-0 doAssert frexp(Inf).frac == Inf # +- Inf preserved doAssert frexp(NaN).frac.isNaN + when not defined(js): var exp: cint result.frac = c_frexp2(x, exp) result.exp = exp else: if x == 0.0: - result = (0.0, 0) + # reuse signbit implementation + let uintBuffer = toBitsImpl(x) + if (uintBuffer[1] shr 31) != 0: + # x is -0.0 + result = (-0.0, 0) + else: + result = (0.0, 0) elif x < 0.0: result = frexp(-x) result.frac = -result.frac @@ -980,6 +1021,7 @@ func frexp*[T: float32|float64](x: T, exponent: var int): T {.inline.} = var x: int doAssert frexp(5.0, x) == 0.625 doAssert x == 3 + (result, exponent) = frexp(x) @@ -988,7 +1030,7 @@ when not defined(js): # taken from Go-lang Math.Log2 const ln2 = 0.693147180559945309417232121458176568075500134360255254120680009 template log2Impl[T](x: T): T = - var exp: int32 + var exp: int var frac = frexp(x, exp) # Make sure exact powers of two give an exact answer. # Don't depend on Log(0.5)*(1/Ln2)+exp being exactly exp-1. @@ -1074,6 +1116,69 @@ func sgn*[T: SomeNumber](x: T): int {.inline.} = {.pop.} {.pop.} +func sum*[T](x: openArray[T]): T = + ## Computes the sum of the elements in `x`. + ## + ## If `x` is empty, 0 is returned. + ## + ## **See also:** + ## * `prod func <#prod,openArray[T]>`_ + ## * `cumsum func <#cumsum,openArray[T]>`_ + ## * `cumsummed func <#cumsummed,openArray[T]>`_ + runnableExamples: + doAssert sum([1, 2, 3, 4]) == 10 + doAssert sum([-4, 3, 5]) == 4 + + for i in items(x): result = result + i + +func prod*[T](x: openArray[T]): T = + ## Computes the product of the elements in `x`. + ## + ## If `x` is empty, 1 is returned. + ## + ## **See also:** + ## * `sum func <#sum,openArray[T]>`_ + ## * `fac func <#fac,int>`_ + runnableExamples: + doAssert prod([1, 2, 3, 4]) == 24 + doAssert prod([-4, 3, 5]) == -60 + + result = T(1) + for i in items(x): result = result * i + +func cumsummed*[T](x: openArray[T]): seq[T] = + ## Returns the cumulative (aka prefix) summation of `x`. + ## + ## If `x` is empty, `@[]` is returned. + ## + ## **See also:** + ## * `sum func <#sum,openArray[T]>`_ + ## * `cumsum func <#cumsum,openArray[T]>`_ for the in-place version + runnableExamples: + doAssert cumsummed([1, 2, 3, 4]) == @[1, 3, 6, 10] + + let xLen = x.len + if xLen == 0: + return @[] + result.setLen(xLen) + result[0] = x[0] + for i in 1 ..< xLen: result[i] = result[i - 1] + x[i] + +func cumsum*[T](x: var openArray[T]) = + ## Transforms `x` in-place (must be declared as `var`) into its + ## cumulative (aka prefix) summation. + ## + ## **See also:** + ## * `sum func <#sum,openArray[T]>`_ + ## * `cumsummed func <#cumsummed,openArray[T]>`_ for a version which + ## returns a cumsummed sequence + runnableExamples: + var a = [1, 2, 3, 4] + cumsum(a) + doAssert a == @[1, 3, 6, 10] + + for i in 1 ..< x.len: x[i] = x[i - 1] + x[i] + func `^`*[T: SomeNumber](x: T, y: Natural): T = ## Computes `x` to the power of `y`. ## @@ -1125,40 +1230,42 @@ func gcd*[T](x, y: T): T = swap x, y abs x -func gcd*(x, y: SomeInteger): SomeInteger = - ## Computes the greatest common (positive) divisor of `x` and `y`, - ## using the binary GCD (aka Stein's) algorithm. - ## - ## **See also:** - ## * `gcd func <#gcd,T,T>`_ for a float version - ## * `lcm func <#lcm,T,T>`_ - runnableExamples: - doAssert gcd(12, 8) == 4 - doAssert gcd(17, 63) == 1 - - when x is SomeSignedInt: - var x = abs(x) - else: - var x = x - when y is SomeSignedInt: - var y = abs(y) - else: - var y = y - - if x == 0: - return y - if y == 0: - return x - - let shift = countTrailingZeroBits(x or y) - y = y shr countTrailingZeroBits(y) - while x != 0: - x = x shr countTrailingZeroBits(x) - if y > x: - swap y, x - x -= y - y shl shift - +when useBuiltins: + ## this func uses bitwise comparisons from C compilers, which are not always available. + func gcd*(x, y: SomeInteger): SomeInteger = + ## Computes the greatest common (positive) divisor of `x` and `y`, + ## using the binary GCD (aka Stein's) algorithm. + ## + ## **See also:** + ## * `gcd func <#gcd,T,T>`_ for a float version + ## * `lcm func <#lcm,T,T>`_ + runnableExamples: + doAssert gcd(12, 8) == 4 + doAssert gcd(17, 63) == 1 + + when x is SomeSignedInt: + var x = abs(x) + else: + var x = x + when y is SomeSignedInt: + var y = abs(y) + else: + var y = y + + if x == 0: + return y + if y == 0: + return x + + let shift = countTrailingZeroBits(x or y) + y = y shr countTrailingZeroBits(y) + while x != 0: + x = x shr countTrailingZeroBits(x) + if y > x: + swap y, x + x -= y + y shl shift + func gcd*[T](x: openArray[T]): T {.since: (1, 1).} = ## Computes the greatest common (positive) divisor of the elements of `x`. ## diff --git a/lib/pure/md5.nim b/lib/pure/md5.nim index 11c324548..9c3f6d51b 100644 --- a/lib/pure/md5.nim +++ b/lib/pure/md5.nim @@ -9,15 +9,17 @@ ## Module for computing [MD5 checksums](https://en.wikipedia.org/wiki/MD5). ## -## **Note:** The procs in this module can be used at compile time. +## This module also works at compile time and in JavaScript. ## ## See also ## ======== -## * `base64 module<base64.html>`_ implements a Base64 encoder and decoder -## * `std/sha1 module <sha1.html>`_ for a SHA-1 encoder and decoder +## * `base64 module<base64.html>`_ for a Base64 encoder and decoder +## * `sha1 module <sha1.html>`_ for the SHA-1 checksum algorithm ## * `hashes module<hashes.html>`_ for efficient computations of hash values ## for diverse Nim types +{.deprecated: "use command `nimble install checksums` and import `checksums/md5` instead".} + when defined(nimHasStyleChecks): {.push styleChecks: off.} @@ -34,15 +36,16 @@ type buffer: MD5Buffer const - padding: cstring = "\x80\0\0\0" & - "\0\0\0\0\0\0\0\0" & - "\0\0\0\0\0\0\0\0" & - "\0\0\0\0\0\0\0\0" & - "\0\0\0\0\0\0\0\0" & - "\0\0\0\0\0\0\0\0" & - "\0\0\0\0\0\0\0\0" & - "\0\0\0\0\0\0\0\0" & - "\0\0\0\0" + padding: array[0..63, uint8] = [ + 0x80'u8, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0 + ] proc F(x, y, z: uint32): uint32 {.inline.} = result = (x and y) or ((not x) and z) @@ -79,7 +82,7 @@ proc II(a: var uint32, b, c, d, x: uint32, s: uint8, ac: uint32) = rot(a, s) a = a + b -proc encode(dest: var MD5Block, src: cstring) = +proc encode(dest: var MD5Block, src: openArray[uint8]) = var j = 0 for i in 0..high(dest): dest[i] = uint32(ord(src[j])) or @@ -97,10 +100,35 @@ proc decode(dest: var openArray[uint8], src: openArray[uint32]) = dest[i+3] = uint8(src[j] shr 24 and 0xff'u32) inc(i, 4) -proc transform(buffer: pointer, state: var MD5State) = +template slice(s: cstring, a, b): openArray[uint8] = + when nimvm: + # toOpenArray is not implemented in VM + toOpenArrayByte($s, a, b) + else: + when defined(js): + # toOpenArrayByte for cstring is not implemented in JS + toOpenArrayByte($s, a, b) + else: + s.toOpenArrayByte(a, b) + +template slice(s: openArray[uint8], a, b): openArray[uint8] = + s.toOpenArray(a, b) + +const useMem = declared(copyMem) + +template memOrNot(withMem, withoutMem): untyped = + when nimvm: + withoutMem + else: + when useMem: + withMem + else: + withoutMem + +proc transform(buffer: openArray[uint8], state: var MD5State) = var myBlock: MD5Block - encode(myBlock, cast[cstring](buffer)) + encode(myBlock, buffer) var a = state[0] var b = state[1] var c = state[2] @@ -175,10 +203,18 @@ proc transform(buffer: pointer, state: var MD5State) = state[3] = state[3] + d proc md5Init*(c: var MD5Context) {.raises: [], tags: [], gcsafe.} -proc md5Update*(c: var MD5Context, input: cstring, len: int) {.raises: [], +proc md5Update*(c: var MD5Context, input: openArray[uint8]) {.raises: [], tags: [], gcsafe.} proc md5Final*(c: var MD5Context, digest: var MD5Digest) {.raises: [], tags: [], gcsafe.} +proc md5Update*(c: var MD5Context, input: cstring, len: int) {.raises: [], + tags: [], gcsafe.} = + ## Updates the `MD5Context` with the `input` data of length `len`. + ## + ## If you use the `toMD5 proc <#toMD5,string>`_, there's no need to call this + ## function explicitly. + md5Update(c, input.slice(0, len - 1)) + proc toMD5*(s: string): MD5Digest = ## Computes the `MD5Digest` value for a string `s`. @@ -192,7 +228,7 @@ proc toMD5*(s: string): MD5Digest = var c: MD5Context md5Init(c) - md5Update(c, cstring(s), len(s)) + md5Update(c, s.slice(0, s.len - 1)) md5Final(c, result) proc `$`*(d: MD5Digest): string = @@ -215,7 +251,7 @@ proc getMD5*(s: string): string = c: MD5Context d: MD5Digest md5Init(c) - md5Update(c, cstring(s), len(s)) + md5Update(c, s.slice(0, s.len - 1)) md5Final(c, d) result = $d @@ -226,6 +262,12 @@ proc `==`*(D1, D2: MD5Digest): bool = return true +proc clearBuffer(c: var MD5Context) {.inline.} = + memOrNot: + zeroMem(addr(c.buffer), sizeof(MD5Buffer)) + do: + reset(c.buffer) + proc md5Init*(c: var MD5Context) = ## Initializes an `MD5Context`. ## @@ -237,29 +279,39 @@ proc md5Init*(c: var MD5Context) = c.state[3] = 0x10325476'u32 c.count[0] = 0'u32 c.count[1] = 0'u32 - zeroMem(addr(c.buffer), sizeof(MD5Buffer)) + clearBuffer(c) -proc md5Update*(c: var MD5Context, input: cstring, len: int) = - ## Updates the `MD5Context` with the `input` data of length `len`. +proc writeBuffer(c: var MD5Context, index: int, + input: openArray[uint8], inputIndex, len: int) {.inline.} = + memOrNot: + copyMem(addr(c.buffer[index]), unsafeAddr(input[inputIndex]), len) + do: + # cannot use system.`[]=` for arrays and openarrays as + # it can raise RangeDefect which gets tracked + for i in 0..<len: + c.buffer[index + i] = input[inputIndex + i] + +proc md5Update*(c: var MD5Context, input: openArray[uint8]) = + ## Updates the `MD5Context` with the `input` data. ## ## If you use the `toMD5 proc <#toMD5,string>`_, there's no need to call this ## function explicitly. - var input = input var Index = int((c.count[0] shr 3) and 0x3F) - c.count[0] = c.count[0] + (uint32(len) shl 3) - if c.count[0] < (uint32(len) shl 3): c.count[1] = c.count[1] + 1'u32 - c.count[1] = c.count[1] + (uint32(len) shr 29) + c.count[0] = c.count[0] + (uint32(input.len) shl 3) + if c.count[0] < (uint32(input.len) shl 3): c.count[1] = c.count[1] + 1'u32 + c.count[1] = c.count[1] + (uint32(input.len) shr 29) var PartLen = 64 - Index - if len >= PartLen: - copyMem(addr(c.buffer[Index]), input, PartLen) - transform(addr(c.buffer), c.state) + if input.len >= PartLen: + writeBuffer(c, Index, input, 0, PartLen) + transform(c.buffer, c.state) var i = PartLen - while i + 63 < len: - transform(addr(input[i]), c.state) + while i + 63 < input.len: + transform(input.slice(i, i + 63), c.state) inc(i, 64) - copyMem(addr(c.buffer[0]), addr(input[i]), len-i) - else: - copyMem(addr(c.buffer[Index]), addr(input[0]), len) + if i < input.len: + writeBuffer(c, 0, input, i, input.len - i) + elif input.len > 0: + writeBuffer(c, Index, input, 0, input.len) proc md5Final*(c: var MD5Context, digest: var MD5Digest) = ## Finishes the `MD5Context` and stores the result in `digest`. @@ -273,11 +325,11 @@ proc md5Final*(c: var MD5Context, digest: var MD5Digest) = var Index = int((c.count[0] shr 3) and 0x3F) if Index < 56: PadLen = 56 - Index else: PadLen = 120 - Index - md5Update(c, padding, PadLen) - md5Update(c, cast[cstring](addr(Bits)), 8) + md5Update(c, padding.slice(0, PadLen - 1)) + md5Update(c, Bits) decode(digest, c.state) - zeroMem(addr(c), sizeof(MD5Context)) + clearBuffer(c) when defined(nimHasStyleChecks): - {.pop.} #{.push styleChecks: off.} + {.pop.} #{.push styleChecks: off.} \ No newline at end of file diff --git a/lib/pure/memfiles.nim b/lib/pure/memfiles.nim index 407a358fa..8eec551c4 100644 --- a/lib/pure/memfiles.nim +++ b/lib/pure/memfiles.nim @@ -16,18 +16,54 @@ ## other "line-like", variable length, delimited records). when defined(windows): - import winlean + import std/winlean + when defined(nimPreviewSlimSystem): + import std/widestrs elif defined(posix): - import posix + import std/posix else: {.error: "the memfiles module is not supported on your operating system!".} -import os, streams +import std/streams +import std/oserrors + +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions] + proc newEIO(msg: string): ref IOError = new(result) result.msg = msg +proc setFileSize(fh: FileHandle, newFileSize = -1, oldSize = -1): OSErrorCode = + ## Set the size of open file pointed to by `fh` to `newFileSize` if != -1, + ## allocating | freeing space from the file system. This routine returns the + ## last OSErrorCode found rather than raising to support old rollback/clean-up + ## code style. [ Should maybe move to std/osfiles. ] + if newFileSize < 0 or newFileSize == oldSize: + return + when defined(windows): + var sizeHigh = int32(newFileSize shr 32) + let sizeLow = int32(newFileSize and 0xffffffff) + let status = setFilePointer(fh, sizeLow, addr(sizeHigh), FILE_BEGIN) + let lastErr = osLastError() + if (status == INVALID_SET_FILE_POINTER and lastErr.int32 != NO_ERROR) or + setEndOfFile(fh) == 0: + result = lastErr + else: + if newFileSize > oldSize: # grow the file + var e: cint # posix_fallocate truncates up when needed. + when declared(posix_fallocate): + while (e = posix_fallocate(fh, 0, newFileSize); e == EINTR): + discard + if e in [EINVAL, EOPNOTSUPP] and ftruncate(fh, newFileSize) == -1: + result = osLastError() # fallback arguable; Most portable BUT allows SEGV + elif e != 0: + result = osLastError() + else: # shrink the file + if ftruncate(fh.cint, newFileSize) == -1: + result = osLastError() + type MemFile* = object ## represents a memory mapped file mem*: pointer ## a pointer to the memory mapped file. The pointer @@ -118,7 +154,7 @@ proc open*(filename: string, mode: FileMode = fmRead, ## ## Example: ## - ## .. code-block:: nim + ## ```nim ## var ## mm, mm_full, mm_half: MemFile ## @@ -130,6 +166,7 @@ proc open*(filename: string, mode: FileMode = fmRead, ## ## # Read the first 512 bytes ## mm_half = memfiles.open("/tmp/test.mmap", mode = fmReadWrite, mappedSize = 512) + ## ``` # The file can be resized only when write mode is used: if mode == fmAppend: @@ -167,25 +204,13 @@ proc open*(filename: string, mode: FileMode = fmRead, else: FILE_ATTRIBUTE_NORMAL or flags, 0) - when useWinUnicode: - result.fHandle = callCreateFile(createFileW, newWideCString(filename)) - else: - result.fHandle = callCreateFile(createFileA, filename) + result.fHandle = callCreateFile(createFileW, newWideCString(filename)) if result.fHandle == INVALID_HANDLE_VALUE: fail(osLastError(), "error opening file") - if newFileSize != -1: - var - sizeHigh = int32(newFileSize shr 32) - sizeLow = int32(newFileSize and 0xffffffff) - - var status = setFilePointer(result.fHandle, sizeLow, addr(sizeHigh), - FILE_BEGIN) - let lastErr = osLastError() - if (status == INVALID_SET_FILE_POINTER and lastErr.int32 != NO_ERROR) or - (setEndOfFile(result.fHandle) == 0): - fail(lastErr, "error setting file size") + if (let e = setFileSize(result.fHandle.FileHandle, newFileSize); + e != 0.OSErrorCode): fail(e, "error setting file size") # since the strings are always 'nil', we simply always call # CreateFileMappingW which should be slightly faster anyway: @@ -219,7 +244,7 @@ proc open*(filename: string, mode: FileMode = fmRead, result.wasOpened = true if not allowRemap and result.fHandle != INVALID_HANDLE_VALUE: - if closeHandle(result.fHandle) == 0: + if closeHandle(result.fHandle) != 0: result.fHandle = INVALID_HANDLE_VALUE else: @@ -234,42 +259,31 @@ proc open*(filename: string, mode: FileMode = fmRead, flags = flags or O_CREAT or O_TRUNC var permissionsMode = S_IRUSR or S_IWUSR result.handle = open(filename, flags, permissionsMode) + if result.handle != -1: + if (let e = setFileSize(result.handle.FileHandle, newFileSize); + e != 0.OSErrorCode): fail(e, "error setting file size") else: result.handle = open(filename, flags) if result.handle == -1: - # XXX: errno is supposed to be set here - # Is there an exception that wraps it? fail(osLastError(), "error opening file") - if newFileSize != -1: - if ftruncate(result.handle, newFileSize) == -1: - fail(osLastError(), "error setting file size") - - if mappedSize != -1: - result.size = mappedSize - else: - var stat: Stat + if mappedSize != -1: #XXX Logic here differs from `when windows` branch .. + result.size = mappedSize #.. which always fstats&Uses min(mappedSize, st). + else: # if newFileSize!=-1: result.size=newFileSize # if trust setFileSize + var stat: Stat #^^.. BUT some FSes (eg. Linux HugeTLBfs) round to 2MiB. if fstat(result.handle, stat) != -1: - # XXX: Hmm, this could be unsafe - # Why is mmap taking int anyway? - result.size = int(stat.st_size) + result.size = stat.st_size.int # int may be 32-bit-unsafe for 2..<4 GiB else: fail(osLastError(), "error getting file size") result.flags = if mapFlags == cint(-1): MAP_SHARED else: mapFlags - #Ensure exactly one of MAP_PRIVATE cr MAP_SHARED is set + # Ensure exactly one of MAP_PRIVATE cr MAP_SHARED is set if int(result.flags and MAP_PRIVATE) == 0: result.flags = result.flags or MAP_SHARED - result.mem = mmap( - nil, - result.size, - if readonly: PROT_READ else: PROT_READ or PROT_WRITE, - result.flags, - result.handle, - offset) - + let pr = if readonly: PROT_READ else: PROT_READ or PROT_WRITE + result.mem = mmap(nil, result.size, pr, result.flags, result.handle, offset) if result.mem == cast[pointer](MAP_FAILED): fail(osLastError(), "file mapping failed") @@ -299,34 +313,58 @@ proc flush*(f: var MemFile; attempts: Natural = 3) = if lastErr != EBUSY.OSErrorCode: raiseOSError(lastErr, "error flushing mapping") -when defined(posix) or defined(nimdoc): - proc resize*(f: var MemFile, newFileSize: int) {.raises: [IOError, OSError].} = - ## resize and re-map the file underlying an `allowRemap MemFile`. - ## **Note**: this assumes the entire file is mapped read-write at offset zero. - ## Also, the value of `.mem` will probably change. - ## **Note**: This is not (yet) available on Windows. - when defined(posix): - if f.handle == -1: - raise newException(IOError, - "Cannot resize MemFile opened with allowRemap=false") - if ftruncate(f.handle, newFileSize) == -1: - raiseOSError(osLastError()) - when defined(linux): #Maybe NetBSD, too? - #On Linux this can be over 100 times faster than a munmap,mmap cycle. - proc mremap(old: pointer; oldSize, newSize: csize; flags: cint): - pointer {.importc: "mremap", header: "<sys/mman.h>".} - let newAddr = mremap(f.mem, csize(f.size), csize(newFileSize), cint(1)) - if newAddr == cast[pointer](MAP_FAILED): - raiseOSError(osLastError()) - else: - if munmap(f.mem, f.size) != 0: - raiseOSError(osLastError()) - let newAddr = mmap(nil, newFileSize, PROT_READ or PROT_WRITE, - f.flags, f.handle, 0) - if newAddr == cast[pointer](MAP_FAILED): - raiseOSError(osLastError()) - f.mem = newAddr +proc resize*(f: var MemFile, newFileSize: int) {.raises: [IOError, OSError].} = + ## Resize & re-map the file underlying an `allowRemap MemFile`. If the OS/FS + ## supports it, file space is reserved to ensure room for new virtual pages. + ## Caller should wait often enough for `flush` to finish to limit use of + ## system RAM for write buffering, perhaps just prior to this call. + ## **Note**: this assumes the entire file is mapped read-write at offset 0. + ## Also, the value of `.mem` will probably change. + if newFileSize < 1: # Q: include system/bitmasks & use PageSize ? + raise newException(IOError, "Cannot resize MemFile to < 1 byte") + when defined(windows): + if not f.wasOpened: + raise newException(IOError, "Cannot resize unopened MemFile") + if f.fHandle == INVALID_HANDLE_VALUE: + raise newException(IOError, + "Cannot resize MemFile opened with allowRemap=false") + if unmapViewOfFile(f.mem) == 0 or closeHandle(f.mapHandle) == 0: # Un-do map + raiseOSError(osLastError()) + if newFileSize != f.size: # Seek to size & `setEndOfFile` => allocated. + if (let e = setFileSize(f.fHandle.FileHandle, newFileSize); + e != 0.OSErrorCode): raiseOSError(e) + f.mapHandle = createFileMappingW(f.fHandle, nil, PAGE_READWRITE, 0,0,nil) + if f.mapHandle == 0: # Re-do map + raiseOSError(osLastError()) + if (let m = mapViewOfFileEx(f.mapHandle, FILE_MAP_READ or FILE_MAP_WRITE, + 0, 0, WinSizeT(newFileSize), nil); m != nil): + f.mem = m f.size = newFileSize + else: + raiseOSError(osLastError()) + elif defined(posix): + if f.handle == -1: + raise newException(IOError, + "Cannot resize MemFile opened with allowRemap=false") + if newFileSize != f.size: + if (let e = setFileSize(f.handle.FileHandle, newFileSize, f.size); + e != 0.OSErrorCode): raiseOSError(e) + when defined(linux): #Maybe NetBSD, too? + # On Linux this can be over 100 times faster than a munmap,mmap cycle. + proc mremap(old: pointer; oldSize, newSize: csize_t; flags: cint): + pointer {.importc: "mremap", header: "<sys/mman.h>".} + let newAddr = mremap(f.mem, csize_t(f.size), csize_t(newFileSize), 1.cint) + if newAddr == cast[pointer](MAP_FAILED): + raiseOSError(osLastError()) + else: + if munmap(f.mem, f.size) != 0: + raiseOSError(osLastError()) + let newAddr = mmap(nil, newFileSize, PROT_READ or PROT_WRITE, + f.flags, f.handle, 0) + if newAddr == cast[pointer](MAP_FAILED): + raiseOSError(osLastError()) + f.mem = newAddr + f.size = newFileSize proc close*(f: var MemFile) = ## closes the memory mapped file `f`. All changes are written back to the @@ -374,10 +412,10 @@ proc `==`*(x, y: MemSlice): bool = proc `$`*(ms: MemSlice): string {.inline.} = ## Return a Nim string built from a MemSlice. result.setLen(ms.size) - copyMem(addr(result[0]), ms.data, ms.size) + copyMem(result.cstring, ms.data, ms.size) iterator memSlices*(mfile: MemFile, delim = '\l', eat = '\r'): MemSlice {.inline.} = - ## Iterates over [optional `eat`] `delim`-delimited slices in MemFile `mfile`. + ## Iterates over \[optional `eat`] `delim`-delimited slices in MemFile `mfile`. ## ## Default parameters parse lines ending in either Unix(\\l) or Windows(\\r\\l) ## style on on a line-by-line basis. I.e., not every line needs the same ending. @@ -400,15 +438,15 @@ iterator memSlices*(mfile: MemFile, delim = '\l', eat = '\r'): MemSlice {.inline ## functions, not str* functions). ## ## Example: - ## - ## .. code-block:: nim + ## ```nim ## var count = 0 ## for slice in memSlices(memfiles.open("foo")): ## if slice.size > 0 and cast[cstring](slice.data)[0] != '#': ## inc(count) ## echo count + ## ``` - proc c_memchr(cstr: pointer, c: char, n: csize): pointer {. + proc c_memchr(cstr: pointer, c: char, n: csize_t): pointer {. importc: "memchr", header: "<string.h>".} proc `-!`(p, q: pointer): int {.inline.} = return cast[int](p) -% cast[int](q) var ms: MemSlice @@ -416,7 +454,7 @@ iterator memSlices*(mfile: MemFile, delim = '\l', eat = '\r'): MemSlice {.inline ms.data = mfile.mem var remaining = mfile.size while remaining > 0: - ending = c_memchr(ms.data, delim, remaining) + ending = c_memchr(ms.data, delim, csize_t(remaining)) if ending == nil: # unterminated final slice ms.size = remaining # Weird case..check eat? yield ms @@ -431,16 +469,16 @@ iterator memSlices*(mfile: MemFile, delim = '\l', eat = '\r'): MemSlice {.inline iterator lines*(mfile: MemFile, buf: var string, delim = '\l', eat = '\r'): string {.inline.} = ## Replace contents of passed buffer with each new line, like - ## `readLine(File) <io.html#readLine,File,string>`_. + ## `readLine(File) <syncio.html#readLine,File,string>`_. ## `delim`, `eat`, and delimiting logic is exactly as for `memSlices ## <#memSlices.i,MemFile,char,char>`_, but Nim strings are returned. ## ## Example: - ## - ## .. code-block:: nim + ## ```nim ## var buffer: string = "" ## for line in lines(memfiles.open("foo"), buffer): ## echo line + ## ``` for ms in memSlices(mfile, delim, eat): setLen(buf, ms.size) @@ -450,15 +488,15 @@ iterator lines*(mfile: MemFile, buf: var string, delim = '\l', iterator lines*(mfile: MemFile, delim = '\l', eat = '\r'): string {.inline.} = ## Return each line in a file as a Nim string, like - ## `lines(File) <io.html#lines.i,File>`_. + ## `lines(File) <syncio.html#lines.i,File>`_. ## `delim`, `eat`, and delimiting logic is exactly as for `memSlices ## <#memSlices.i,MemFile,char,char>`_, but Nim strings are returned. ## ## Example: - ## - ## .. code-block:: nim + ## ```nim ## for line in lines(memfiles.open("foo")): ## echo line + ## ``` var buf = newStringOfCap(80) for line in lines(mfile, buf, delim, eat): @@ -488,8 +526,8 @@ proc mmsSetPosition(s: Stream, pos: int) = proc mmsGetPosition(s: Stream): int = MemMapFileStream(s).pos proc mmsPeekData(s: Stream, buffer: pointer, bufLen: int): int = - let startAddress = cast[ByteAddress](MemMapFileStream(s).mf.mem) - let p = cast[ByteAddress](MemMapFileStream(s).pos) + let startAddress = cast[int](MemMapFileStream(s).mf.mem) + let p = cast[int](MemMapFileStream(s).pos) let l = min(bufLen, MemMapFileStream(s).mf.size - p) moveMem(buffer, cast[pointer](startAddress + p), l) result = l @@ -504,8 +542,8 @@ proc mmsWriteData(s: Stream, buffer: pointer, bufLen: int) = let size = MemMapFileStream(s).mf.size if MemMapFileStream(s).pos + bufLen > size: raise newEIO("cannot write to stream") - let p = cast[ByteAddress](MemMapFileStream(s).mf.mem) + - cast[ByteAddress](MemMapFileStream(s).pos) + let p = cast[int](MemMapFileStream(s).mf.mem) + + cast[int](MemMapFileStream(s).pos) moveMem(cast[pointer](p), buffer, bufLen) inc(MemMapFileStream(s).pos, bufLen) diff --git a/lib/pure/mersenne.nim b/lib/pure/mersenne.nim deleted file mode 100644 index 6778e2d62..000000000 --- a/lib/pure/mersenne.nim +++ /dev/null @@ -1,55 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Nim Contributors -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## The [Mersenne Twister](https://en.wikipedia.org/wiki/Mersenne_Twister) -## random number generator. -## -## **Note:** The procs in this module work at compile-time. - -runnableExamples: - var rand = newMersenneTwister(uint32.high) ## must be "var" - doAssert rand.getNum() != rand.getNum() ## pseudorandom number - -## See also -## ======== -## * `random module<random.html>`_ for Nim's standard random number generator - -type - MersenneTwister* = object - ## The Mersenne Twister. - mt: array[0..623, uint32] - index: int - -proc newMersenneTwister*(seed: uint32): MersenneTwister = - ## Creates a new `MersenneTwister` with seed `seed`. - result.index = 0 - result.mt[0] = seed - for i in 1'u32 .. 623'u32: - result.mt[i] = (0x6c078965'u32 * (result.mt[i-1] xor - (result.mt[i-1] shr 30'u32)) + i) - -proc generateNumbers(m: var MersenneTwister) = - for i in 0..623: - var y = (m.mt[i] and 0x80000000'u32) + - (m.mt[(i+1) mod 624] and 0x7fffffff'u32) - m.mt[i] = m.mt[(i+397) mod 624] xor uint32(y shr 1'u32) - if (y mod 2'u32) != 0: - m.mt[i] = m.mt[i] xor 0x9908b0df'u32 - -proc getNum*(m: var MersenneTwister): uint32 = - ## Returns the next pseudorandom `uint32`. - if m.index == 0: - generateNumbers(m) - result = m.mt[m.index] - m.index = (m.index + 1) mod m.mt.len - - result = result xor (result shr 11'u32) - result = result xor ((result shl 7'u32) and 0x9d2c5680'u32) - result = result xor ((result shl 15'u32) and 0xefc60000'u32) - result = result xor (result shr 18'u32) diff --git a/lib/pure/mimetypes.nim b/lib/pure/mimetypes.nim index c10739fec..ff639e8e5 100644 --- a/lib/pure/mimetypes.nim +++ b/lib/pure/mimetypes.nim @@ -26,1856 +26,716 @@ runnableExamples: doAssert m.getMimetype("fakext") == "text/fakelang" doAssert m.getMimetype("FaKeXT") == "text/fakelang" -import strtabs -from strutils import startsWith, toLowerAscii, strip +import std/tables +from std/strutils import startsWith, toLowerAscii, strip + +when defined(nimPreviewSlimSystem): + import std/assertions + type MimeDB* = object - mimes: StringTableRef + mimes: OrderedTable[string, string] const mimes* = { - "123": "application/vnd.lotus-1-2-3", - "1km": "application/vnd.1000minds.decision-model+xml", - "323": "text/h323", - "3dm": "text/vnd.in3d.3dml", - "3dmf": "x-world/x-3dmf", - "3dml": "text/vnd.in3d.3dml", - "3ds": "image/x-3ds", - "3g2": "video/3gpp2", - "3gp": "video/3gpp", - "3gpp": "audio/3gpp", - "3gpp2": "video/3gpp2", - "3mf": "application/vnd.ms-3mfdocument", - "669": "audio/x-mod", - "726": "audio/32kadpcm", - "7z": "application/x-7z-compressed", - "a": "text/plain", - "a2l": "application/a2l", - "aa3": "audio/atrac3", - "aab": "application/x-authorware-bin", - "aac": "audio/x-aac", - "aal": "audio/atrac-advanced-lossless", - "aam": "application/x-authorware-map", - "aas": "application/x-authorware-seg", - "abc": "text/vnd.abc", - "abw": "application/x-abiword", + "ez": "application/andrew-inset", + "aw": "application/applixware", + "atom": "application/atom+xml", + "atomcat": "application/atomcat+xml", + "atomsvc": "application/atomsvc+xml", + "ccxml": "application/ccxml+xml", + "cdmia": "application/cdmi-capability", + "cdmic": "application/cdmi-container", + "cdmid": "application/cdmi-domain", + "cdmio": "application/cdmi-object", + "cdmiq": "application/cdmi-queue", + "cu": "application/cu-seeme", + "davmount": "application/davmount+xml", + "dbk": "application/docbook+xml", + "dssc": "application/dssc+der", + "xdssc": "application/dssc+xml", + "ecma": "application/ecmascript", + "emma": "application/emma+xml", + "epub": "application/epub+zip", + "exi": "application/exi", + "pfr": "application/font-tdpfr", + "gml": "application/gml+xml", + "gpx": "application/gpx+xml", + "gxf": "application/gxf", + "stk": "application/hyperstudio", + "ink": "application/inkml+xml", + "inkml": "application/inkml+xml", + "ipfix": "application/ipfix", + "jar": "application/java-archive", + "ser": "application/java-serialized-object", + "class": "application/java-vm", + "json": "application/json", + "jsonml": "application/jsonml+json", + "lostxml": "application/lost+xml", + "hqx": "application/mac-binhex40", + "cpt": "application/mac-compactpro", + "mads": "application/mads+xml", + "mrc": "application/marc", + "mrcx": "application/marcxml+xml", + "ma": "application/mathematica", + "nb": "application/mathematica", + "mb": "application/mathematica", + "mathml": "application/mathml+xml", + "mbox": "application/mbox", + "mscml": "application/mediaservercontrol+xml", + "metalink": "application/metalink+xml", + "meta4": "application/metalink4+xml", + "mets": "application/mets+xml", + "mods": "application/mods+xml", + "m21": "application/mp21", + "mp21": "application/mp21", + "mp4s": "application/mp4", + "doc": "application/msword", + "dot": "application/msword", + "mxf": "application/mxf", + "bin": "application/octet-stream", + "dms": "application/octet-stream", + "lrf": "application/octet-stream", + "mar": "application/octet-stream", + "so": "application/octet-stream", + "dist": "application/octet-stream", + "distz": "application/octet-stream", + "pkg": "application/octet-stream", + "bpk": "application/octet-stream", + "dump": "application/octet-stream", + "elc": "application/octet-stream", + "deploy": "application/octet-stream", + "oda": "application/oda", + "opf": "application/oebps-package+xml", + "ogx": "application/ogg", + "omdoc": "application/omdoc+xml", + "onetoc": "application/onenote", + "onetoc2": "application/onenote", + "onetmp": "application/onenote", + "onepkg": "application/onenote", + "oxps": "application/oxps", + "xer": "application/patch-ops-error+xml", + "pdf": "application/pdf", + "pgp": "application/pgp-encrypted", + "asc": "application/pgp-signature", + "sig": "application/pgp-signature", + "prf": "application/pics-rules", + "p10": "application/pkcs10", + "p7m": "application/pkcs7-mime", + "p7c": "application/pkcs7-mime", + "p7s": "application/pkcs7-signature", + "p8": "application/pkcs8", "ac": "application/pkix-attr-cert", - "ac3": "audio/ac3", - "acc": "application/vnd.americandynamics.acc", - "ace": "application/x-ace-compressed", - "acn": "audio/asc", - "acu": "application/vnd.acucobol", - "acutc": "application/vnd.acucorp", - "acx": "application/internet-property-stream", - "adp": "audio/adpcm", - "aep": "application/vnd.audiograph", - "afl": "video/animaflex", - "afm": "application/x-font-type1", - "afp": "application/vnd.ibm.modcap", - "ahead": "application/vnd.ahead.space", + "cer": "application/pkix-cert", + "crl": "application/pkix-crl", + "pkipath": "application/pkix-pkipath", + "pki": "application/pkixcmp", + "pls": "application/pls+xml", "ai": "application/postscript", - "aif": "audio/x-aiff", - "aifc": "audio/x-aiff", - "aiff": "audio/x-aiff", - "aim": "application/x-aim", - "aip": "text/x-audiosoft-intra", - "air": "application/vnd.adobe.air-application-installer-package+zip", - "ait": "application/vnd.dvb.ait", - "alc": "chemical/x-alchemy", - "ami": "application/vnd.amiga.ami", - "aml": "application/aml", - "amr": "audio/amr", - "ani": "application/x-navi-animation", - "anx": "application/x-annodex", - "aos": "application/x-nokia-9000-communicator-add-on-software", - "apinotes": "text/apinotes", - "apk": "application/vnd.android.package-archive", - "apkg": "application/vnd.anki", - "apng": "image/apng", - "appcache": "text/cache-manifest", - "appimage": "application/appimage", - "application": "application/x-ms-application", - "apr": "application/vnd.lotus-approach", - "aps": "application/mime", - "apxml": "application/auth-policy+xml", - "arc": "application/x-freearc", - "arj": "application/x-arj", - "art": "message/rfc822", - "asar": "binary/asar", - "asc": "text/plain", - "ascii": "text/vnd.ascii-art", - "asf": "application/vnd.ms-asf", - "asice": "application/vnd.etsi.asic-e+zip", - "asics": "application/vnd.etsi.asic-s+zip", - "asm": "text/x-asm", - "asn": "chemical/x-ncbi-asn1-spec", + "eps": "application/postscript", + "ps": "application/postscript", + "cww": "application/prs.cww", + "pskcxml": "application/pskc+xml", + "rdf": "application/rdf+xml", + "rif": "application/reginfo+xml", + "rnc": "application/relax-ng-compact-syntax", + "rl": "application/resource-lists+xml", + "rld": "application/resource-lists-diff+xml", + "rs": "application/rls-services+xml", + "gbr": "application/rpki-ghostbusters", + "mft": "application/rpki-manifest", + "roa": "application/rpki-roa", + "rsd": "application/rsd+xml", + "rss": "application/rss+xml", + "rtf": "application/rtf", + "sbml": "application/sbml+xml", + "scq": "application/scvp-cv-request", + "scs": "application/scvp-cv-response", + "spq": "application/scvp-vp-request", + "spp": "application/scvp-vp-response", + "sdp": "application/sdp", + "setpay": "application/set-payment-initiation", + "setreg": "application/set-registration-initiation", + "shf": "application/shf+xml", + "smi": "application/smil+xml", + "smil": "application/smil+xml", + "rq": "application/sparql-query", + "srx": "application/sparql-results+xml", + "gram": "application/srgs", + "grxml": "application/srgs+xml", + "sru": "application/sru+xml", + "ssdl": "application/ssdl+xml", + "ssml": "application/ssml+xml", + "tei": "application/tei+xml", + "teicorpus": "application/tei+xml", + "tfi": "application/thraud+xml", + "tsd": "application/timestamped-data", + "plb": "application/vnd.3gpp.pic-bw-large", + "psb": "application/vnd.3gpp.pic-bw-small", + "pvb": "application/vnd.3gpp.pic-bw-var", + "tcap": "application/vnd.3gpp2.tcap", + "pwn": "application/vnd.3m.post-it-notes", "aso": "application/vnd.accpac.simply.aso", - "asp": "text/asp", - "asr": "video/x-ms-asf", - "asx": "video/x-ms-asf", - "at3": "audio/atrac3", + "imp": "application/vnd.accpac.simply.imp", + "acu": "application/vnd.acucobol", "atc": "application/vnd.acucorp", - "atf": "application/atf", - "atfx": "application/atfx", - "atom": "application/atom+xml", - "atomcat": "application/atomcat+xml", - "atomdeleted": "application/atomdeleted+xml", - "atomsrv": "application/atomserv+xml", - "atomsvc": "application/atomsvc+xml", - "atx": "application/vnd.antix.game-component", - "atxml": "application/atxml", - "au": "audio/basic", - "auc": "application/tamp-apex-update-confirm", - "avi": "video/x-msvideo", - "avs": "video/avs-video", - "aw": "application/applixware", - "awb": "audio/amr-wb", - "axa": "audio/x-annodex", - "axs": "application/olescript", - "axv": "video/x-annodex", + "acutc": "application/vnd.acucorp", + "air": "application/vnd.adobe.air-application-installer-package+zip", + "fcdt": "application/vnd.adobe.formscentral.fcdt", + "fxp": "application/vnd.adobe.fxp", + "fxpl": "application/vnd.adobe.fxp", + "xdp": "application/vnd.adobe.xdp+xml", + "xfdf": "application/vnd.adobe.xfdf", + "ahead": "application/vnd.ahead.space", "azf": "application/vnd.airzip.filesecure.azf", "azs": "application/vnd.airzip.filesecure.azs", - "azv": "image/vnd.airzip.accelerator.azv", "azw": "application/vnd.amazon.ebook", - "azw3": "application/vnd.amazon.mobi8-ebook", - "b": "chemical/x-molconn-Z", - "bak": "application/x-trash", - "bar": "application/vnd.qualcomm.brew-app-res", - "bas": "text/plain", - "bash": "text/shell", - "bat": "application/x-msdos-program", - "bcpio": "application/x-bcpio", - "bdf": "application/x-font-bdf", - "bdm": "application/vnd.syncml.dm+wbxml", - "bdoc": "application/bdoc", - "bed": "application/vnd.realvnc.bed", - "bh2": "application/vnd.fujitsu.oasysprs", - "bib": "text/x-bibtex", - "bik": "video/vnd.radgamettools.bink", - "bin": "application/octet-stream", - "bk2": "video/vnd.radgamettools.bink", - "bkm": "application/vnd.nervana", - "blb": "application/x-blorb", - "blend": "binary/blender", - "blorb": "application/x-blorb", - "bm": "image/bmp", - "bmed": "multipart/vnd.bint.med-plus", + "acc": "application/vnd.americandynamics.acc", + "ami": "application/vnd.amiga.ami", + "apk": "application/vnd.android.package-archive", + "cii": "application/vnd.anser-web-certificate-issue-initiation", + "fti": "application/vnd.anser-web-funds-transfer-initiation", + "atx": "application/vnd.antix.game-component", + "mpkg": "application/vnd.apple.installer+xml", + "m3u8": "application/vnd.apple.mpegurl", + "swi": "application/vnd.aristanetworks.swi", + "iota": "application/vnd.astraea-software.iota", + "aep": "application/vnd.audiograph", + "mpm": "application/vnd.blueice.multipass", "bmi": "application/vnd.bmi", - "bmml": "application/vnd.balsamiq.bmml+xml", - "bmp": "image/bmp", - "bmpr": "application/vnd.balsamiq.bmpr", - "boo": "application/book", - "book": "application/book", - "box": "application/vnd.previewsystems.box", - "boz": "application/x-bzip2", - "bpd": "application/vnd.hbci", - "bpk": "application/octet-stream", - "brf": "text/plain", - "bsd": "chemical/x-crossfire", - "bsh": "application/x-bsh", - "bsp": "model/vnd.valve.source.compiled-map", - "btf": "image/prs.btif", - "btif": "image/prs.btif", - "bz": "application/x-bzip", - "bz2": "application/x-bzip2", - "c": "text/x-csrc", - "c++": "text/x-c++src", - "c11amc": "application/vnd.cluetrust.cartomobile-config", - "c11amz": "application/vnd.cluetrust.cartomobile-config-pkg", - "c3d": "chemical/x-chem3d", - "c3ex": "application/cccex", + "rep": "application/vnd.businessobjects", + "cdxml": "application/vnd.chemdraw+xml", + "mmd": "application/vnd.chipnuts.karaoke-mmd", + "cdy": "application/vnd.cinderella", + "cla": "application/vnd.claymore", + "rp9": "application/vnd.cloanto.rp9", + "c4g": "application/vnd.clonk.c4group", "c4d": "application/vnd.clonk.c4group", "c4f": "application/vnd.clonk.c4group", - "c4g": "application/vnd.clonk.c4group", "c4p": "application/vnd.clonk.c4group", "c4u": "application/vnd.clonk.c4group", - "cab": "application/vnd.ms-cab-compressed", - "cac": "chemical/x-cache", - "cache": "application/x-cache", - "caf": "audio/x-caf", - "cap": "application/vnd.tcpdump.pcap", - "car": "application/vnd.curl.car", - "cascii": "chemical/x-cactvs-binary", - "cat": "application/vnd.ms-pki.seccat", - "cb7": "application/x-cbr", - "cba": "application/x-cbr", - "cbin": "chemical/x-cactvs-binary", - "cbor": "application/cbor", - "cbr": "application/x-cbr", - "cbt": "application/x-cbr", - "cbz": "application/vnd.comicbook+zip", - "cc": "text/plain", - "ccad": "application/clariscad", - "ccc": "text/vnd.net2phone.commcenter.command", - "ccmp": "application/ccmp+xml", - "cco": "application/x-cocoa", - "cct": "application/x-director", - "ccxml": "application/ccxml+xml", - "cda": "application/x-cdf", + "c11amc": "application/vnd.cluetrust.cartomobile-config", + "c11amz": "application/vnd.cluetrust.cartomobile-config-pkg", + "csp": "application/vnd.commonspace", "cdbcmsg": "application/vnd.contact.cmsg", - "cdf": "application/x-netcdf", - "cdfx": "application/cdfx+xml", - "cdkey": "application/vnd.mediastation.cdkey", - "cdmia": "application/cdmi-capability", - "cdmic": "application/cdmi-container", - "cdmid": "application/cdmi-domain", - "cdmio": "application/cdmi-object", - "cdmiq": "application/cdmi-queue", - "cdr": "image/x-coreldraw", - "cdt": "image/x-coreldrawtemplate", - "cdx": "chemical/x-cdx", - "cdxml": "application/vnd.chemdraw+xml", - "cdy": "application/vnd.cinderella", - "cea": "application/cea", - "cef": "chemical/x-cxf", - "cellml": "application/cellml+xml", - "cer": "application/pkix-cert", - "cfg": "text/cfg", - "cfs": "application/x-cfs-compressed", - "cgm": "image/cgm", - "cha": "application/x-chat", - "chat": "application/x-chat", - "chm": "application/vnd.ms-htmlhelp", - "chrt": "application/vnd.kde.kchart", - "cif": "chemical/x-cif", - "cii": "application/vnd.anser-web-certificate-issue-initiation", - "cil": "application/vnd.ms-artgalry", - "cl": "application/simple-filter+xml", - "cla": "application/vnd.claymore", - "class": "application/java-vm", + "cmc": "application/vnd.cosmocaller", + "clkx": "application/vnd.crick.clicker", "clkk": "application/vnd.crick.clicker.keyboard", "clkp": "application/vnd.crick.clicker.palette", "clkt": "application/vnd.crick.clicker.template", "clkw": "application/vnd.crick.clicker.wordbank", - "clkx": "application/vnd.crick.clicker", - "clp": "application/x-msclip", - "cls": "text/x-tex", - "clue": "application/clue_info+xml", - "cmake": "text/cmake", - "cmc": "application/vnd.cosmocaller", - "cmdf": "chemical/x-cmdf", - "cml": "chemical/x-cml", - "cmp": "application/vnd.yellowriver-custom-menu", - "cmsc": "application/cms", - "cmx": "image/x-cmx", - "cnd": "text/jcr-cnd", - "cnf": "text/cnf", - "cod": "application/vnd.rim.cod", - "coffee": "application/vnd.coffeescript", - "com": "application/x-msdos-program", - "conf": "text/plain", - "copyright": "text/vnd.debian.copyright", - "cpa": "chemical/x-compass", - "cpio": "application/x-cpio", - "cpkg": "application/vnd.xmpie.cpkg", - "cpl": "application/cpl+xml", - "cpp": "text/x-c++src", - "cpt": "application/mac-compactpro", - "cr2": "image/x-canon-cr2", - "crd": "application/x-mscardfile", - "crl": "application/pkix-crl", - "crt": "application/x-x509-ca-cert", - "crtr": "application/vnd.multiad.creator", - "crw": "image/x-canon-crw", - "crx": "application/x-chrome-extension", - "cryptonote": "application/vnd.rig.cryptonote", - "cs": "text/c#", - "csf": "chemical/x-cache-csf", - "csh": "application/x-csh", - "csl": "application/vnd.citationstyles.style+xml", - "csm": "chemical/x-csml", - "csml": "chemical/x-csml", - "cson": "text/cson", - "csp": "application/vnd.commonspace", - "csrattrs": "application/csrattrs", - "css": "text/css", - "cst": "application/vnd.commonspace", - "csv": "text/csv", - "csvs": "text/csv-schema", - "ctab": "chemical/x-cactvs-binary", - "ctx": "chemical/x-ctx", - "cu": "application/cu-seeme", - "cub": "chemical/x-gaussian-cube", - "cuc": "application/tamp-community-update-confirm", - "curl": "text/vnd.curl", - "cw": "application/prs.cww", - "cww": "application/prs.cww", - "cxf": "chemical/x-cxf", - "cxt": "application/x-director", - "cxx": "text/plain", - "d": "text/x-dsrc", - "dae": "model/vnd.collada+xml", - "daf": "application/vnd.mobius.daf", + "wbs": "application/vnd.criticaltools.wbs+xml", + "pml": "application/vnd.ctc-posml", + "ppd": "application/vnd.cups-ppd", + "car": "application/vnd.curl.car", + "pcurl": "application/vnd.curl.pcurl", "dart": "application/vnd.dart", - "dat": "application/x-ns-proxy-autoconfig", - "dataless": "application/vnd.fdsn.seed", - "davmount": "application/davmount+xml", - "dbk": "application/docbook+xml", - "dcd": "application/dcd", - "dcf": "application/vnd.oma.drm.content", - "dcm": "application/dicom", - "dcr": "application/x-director", - "dcurl": "text/vnd.curl.dcurl", - "dd": "application/vnd.oma.dd+xml", - "dd2": "application/vnd.oma.dd2+xml", - "ddd": "application/vnd.fujixerox.ddd", - "ddf": "application/vnd.syncml.dmddf+xml", - "deb": "application/vnd.debian.binary-package", - "deepv": "application/x-deepv", - "def": "text/plain", - "deploy": "application/octet-stream", - "der": "application/x-x509-ca-cert", - "dfac": "application/vnd.dreamfactory", - "dgc": "application/x-dgc-compressed", - "dib": "image/bmp", - "dic": "text/x-c", - "dif": "video/x-dv", - "diff": "text/x-diff", - "dii": "application/dii", - "dim": "application/vnd.fastcopy-disk-image", - "dir": "application/x-director", - "dis": "application/vnd.mobius.dis", - "disposition-notification": "message/disposition-notification", - "dist": "application/vnd.apple.installer+xml", - "distz": "application/vnd.apple.installer+xml", - "dit": "application/dit", - "djv": "image/vnd.djvu", - "djvu": "image/vnd.djvu", - "dl": "video/dl", - "dll": "application/x-msdos-program", - "dls": "audio/dls", - "dm": "application/vnd.oma.drm.message", - "dmg": "application/x-apple-diskimage", - "dmp": "application/vnd.tcpdump.pcap", - "dms": "text/vnd.dmclientscript", + "rdz": "application/vnd.data-vision.rdz", + "uvf": "application/vnd.dece.data", + "uvvf": "application/vnd.dece.data", + "uvd": "application/vnd.dece.data", + "uvvd": "application/vnd.dece.data", + "uvt": "application/vnd.dece.ttml+xml", + "uvvt": "application/vnd.dece.ttml+xml", + "uvx": "application/vnd.dece.unspecified", + "uvvx": "application/vnd.dece.unspecified", + "uvz": "application/vnd.dece.zip", + "uvvz": "application/vnd.dece.zip", + "fe_launch": "application/vnd.denovo.fcselayout-link", "dna": "application/vnd.dna", - "doc": "application/msword", - "docjson": "application/vnd.document+json", - "docm": "application/vnd.ms-word.document.macroenabled.12", - "docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", - "dor": "model/vnd.gdl", - "dot": "text/vnd.graphviz", - "dotm": "application/vnd.ms-word.template.macroenabled.12", - "dotx": "application/vnd.openxmlformats-officedocument.wordprocessingml.template", - "dp": "application/vnd.osgi.dp", + "mlp": "application/vnd.dolby.mlp", "dpg": "application/vnd.dpgraph", - "dpgraph": "application/vnd.dpgraph", - "dpkg": "application/vnd.xmpie.dpkg", - "dr": "application/vnd.oma.drm.rights+xml", - "dra": "audio/vnd.dra", - "drc": "application/vnd.oma.drm.rights+wbxml", - "drle": "image/dicom-rle", - "drw": "application/drafting", - "dsc": "text/prs.lines.tag", - "dsm": "application/vnd.desmume.movie", - "dssc": "application/dssc+der", - "dtb": "application/x-dtbook+xml", - "dtd": "application/xml-dtd", - "dts": "audio/vnd.dts", - "dtshd": "audio/vnd.dts.hd", - "dump": "application/octet-stream", - "dv": "video/x-dv", - "dvb": "video/vnd.dvb.file", - "dvc": "application/dvcs", - "dvi": "application/x-dvi", - "dwf": "model/vnd.dwf", - "dwg": "image/vnd.dwg", - "dx": "chemical/x-jcamp-dx", - "dxf": "image/vnd.dxf", - "dxp": "application/vnd.spotfire.dxp", - "dxr": "application/x-director", - "dzr": "application/vnd.dzr", - "ear": "binary/zip", - "ecelp4800": "audio/vnd.nuera.ecelp4800", - "ecelp7470": "audio/vnd.nuera.ecelp7470", - "ecelp9600": "audio/vnd.nuera.ecelp9600", - "ecig": "application/vnd.evolv.ecig.settings", - "ecigprofile": "application/vnd.evolv.ecig.profile", - "ecigtheme": "application/vnd.evolv.ecig.theme", - "ecma": "application/ecmascript", - "edm": "application/vnd.novadigm.edm", - "edx": "application/vnd.novadigm.edx", - "efi": "application/efi", - "efif": "application/vnd.picsel", - "ei6": "application/vnd.pg.osasli", - "ejs": "text/ejs", - "el": "text/plain", - "elc": "application/x-bytecode.elisp", - "emb": "chemical/x-embl-dl-nucleotide", - "embl": "chemical/x-embl-dl-nucleotide", - "emf": "image/emf", - "eml": "message/rfc822", - "emm": "application/vnd.ibm.electronic-media", - "emma": "application/emma+xml", - "emotionml": "application/emotionml+xml", - "emz": "application/x-msmetafile", - "ent": "text/xml-external-parsed-entity", - "entity": "application/vnd.nervana", - "env": "application/x-envoy", - "enw": "audio/evrcnw", - "eol": "audio/vnd.digital-winds", - "eot": "application/vnd.ms-fontobject", - "ep": "application/vnd.bluetooth.ep.oob", - "eps": "application/postscript", - "eps2": "application/postscript", - "eps3": "application/postscript", - "epsf": "application/postscript", - "epsi": "application/postscript", - "epub": "application/epub+zip", - "erb": "text/erb", - "erf": "image/x-epson-erf", - "es": "application/ecmascript", - "es3": "application/vnd.eszigno3+xml", - "esa": "application/vnd.osgi.subsystem", - "escn": "text/godot", + "dfac": "application/vnd.dreamfactory", + "kpxx": "application/vnd.ds-keypoint", + "ait": "application/vnd.dvb.ait", + "svc": "application/vnd.dvb.service", + "geo": "application/vnd.dynageo", + "mag": "application/vnd.ecowin.chart", + "nml": "application/vnd.enliven", "esf": "application/vnd.epson.esf", - "espass": "application/vnd.espass-espass+zip", + "msf": "application/vnd.epson.msf", + "qam": "application/vnd.epson.quickanime", + "slt": "application/vnd.epson.salt", + "ssf": "application/vnd.epson.ssf", + "es3": "application/vnd.eszigno3+xml", "et3": "application/vnd.eszigno3+xml", - "etx": "text/x-setext", - "eva": "application/x-eva", - "evb": "audio/evrcb", - "evc": "audio/evrc", - "evw": "audio/evrcwb", - "evy": "application/x-envoy", - "exe": "application/x-msdos-program", - "exi": "application/exi", - "exr": "image/aces", - "ext": "application/vnd.novadigm.ext", - "eyaml": "text/yaml", - "ez": "application/andrew-inset", "ez2": "application/vnd.ezpix-album", "ez3": "application/vnd.ezpix-package", - "f": "text/x-fortran", - "f4v": "video/x-f4v", - "f77": "text/x-fortran", - "f90": "text/plain", - "fb": "application/x-maker", - "fbdoc": "application/x-maker", - "fbs": "image/vnd.fastbidsheet", - "fbx": "model/filmbox", - "fcdt": "application/vnd.adobe.formscentral.fcdt", - "fch": "chemical/x-gaussian-checkpoint", - "fchk": "chemical/x-gaussian-checkpoint", - "fcs": "application/vnd.isac.fcs", "fdf": "application/vnd.fdf", - "fdt": "application/fdt+xml", - "fe_launch": "application/vnd.denovo.fcselayout-link", - "feature": "text/gherkin", - "fg5": "application/vnd.fujitsu.oasysgp", - "fgd": "application/x-director", - "fh": "image/x-freehand", - "fh4": "image/x-freehand", - "fh5": "image/x-freehand", - "fh7": "image/x-freehand", - "fhc": "image/x-freehand", - "fif": "image/fif", - "fig": "application/x-xfig", - "finf": "application/fastinfoset", - "fish": "text/fish", - "fit": "image/fits", - "fits": "image/fits", - "fla": "application/vnd.dtg.local.flash", - "flac": "audio/x-flac", - "fli": "video/x-fli", - "flo": "application/vnd.micrografx.flo", - "flr": "x-world/x-vrml", - "flv": "video/x-flv", - "flw": "application/vnd.kde.kivio", - "flx": "text/vnd.fmi.flexstor", - "fly": "text/vnd.fly", + "mseed": "application/vnd.fdsn.mseed", + "seed": "application/vnd.fdsn.seed", + "dataless": "application/vnd.fdsn.seed", + "gph": "application/vnd.flographit", + "ftc": "application/vnd.fluxtime.clip", "fm": "application/vnd.framemaker", - "fmf": "video/x-atomic3d-feature", - "fnc": "application/vnd.frogans.fnc", - "fo": "application/vnd.software602.filler.form+xml", - "for": "text/x-fortran", - "fpx": "image/vnd.fpx", "frame": "application/vnd.framemaker", - "frl": "application/freeloader", - "frm": "application/vnd.ufdl", + "maker": "application/vnd.framemaker", + "book": "application/vnd.framemaker", + "fnc": "application/vnd.frogans.fnc", + "ltf": "application/vnd.frogans.ltf", "fsc": "application/vnd.fsc.weblaunch", - "fst": "image/vnd.fst", - "ftc": "application/vnd.fluxtime.clip", - "fti": "application/vnd.anser-web-funds-transfer-initiation", - "fts": "image/fits", - "funk": "audio/make", - "fvt": "video/vnd.fvt", - "fxm": "video/x-javafx", - "fxp": "application/vnd.adobe.fxp", - "fxpl": "application/vnd.adobe.fxp", + "oas": "application/vnd.fujitsu.oasys", + "oa2": "application/vnd.fujitsu.oasys2", + "oa3": "application/vnd.fujitsu.oasys3", + "fg5": "application/vnd.fujitsu.oasysgp", + "bh2": "application/vnd.fujitsu.oasysprs", + "ddd": "application/vnd.fujixerox.ddd", + "xdw": "application/vnd.fujixerox.docuworks", + "xbd": "application/vnd.fujixerox.docuworks.binder", "fzs": "application/vnd.fuzzysheet", - "g": "text/plain", - "g2w": "application/vnd.geoplan", - "g3": "image/g3fax", - "g3w": "application/vnd.geospace", - "gac": "application/vnd.groove-account", - "gal": "chemical/x-gaussian-log", - "gam": "application/x-tads", - "gamin": "chemical/x-gamess-input", - "gau": "chemical/x-gaussian-input", - "gbr": "application/rpki-ghostbusters", - "gca": "application/x-gca-compressed", - "gcd": "text/x-pcs-gcd", - "gcf": "application/x-graphing-calculator", - "gcg": "chemical/x-gcg8-sequence", - "gdl": "model/vnd.gdl", - "gdoc": "application/vnd.google-apps.document", - "gemspec": "text/ruby", - "gen": "chemical/x-genbank", - "geo": "application/vnd.dynageo", - "geojson": "application/geo+json", - "gex": "application/vnd.geometry-explorer", - "gf": "application/x-tex-gf", + "txd": "application/vnd.genomatix.tuxedo", "ggb": "application/vnd.geogebra.file", + "ggs": "application/vnd.geogebra.slides", "ggt": "application/vnd.geogebra.tool", - "ghf": "application/vnd.groove-help", - "gif": "image/gif", - "gim": "application/vnd.groove-identity-message", - "gjc": "chemical/x-gaussian-input", - "gjf": "chemical/x-gaussian-input", - "gl": "video/gl", - "glb": "model/gltf-binary", - "gltf": "model/gltf+json", - "gml": "application/gml+xml", + "gex": "application/vnd.geometry-explorer", + "gre": "application/vnd.geometry-explorer", + "gxt": "application/vnd.geonext", + "g2w": "application/vnd.geoplan", + "g3w": "application/vnd.geospace", "gmx": "application/vnd.gmx", - "gnumeric": "application/x-gnumeric", - "go": "text/go", - "gotmpl": "text/gotmpl", - "gph": "application/vnd.flographit", - "gpt": "chemical/x-mopac-graph", - "gpx": "application/gpx+xml", + "kml": "application/vnd.google-earth.kml+xml", + "kmz": "application/vnd.google-earth.kmz", "gqf": "application/vnd.grafeq", "gqs": "application/vnd.grafeq", - "gradle": "text/groovy", - "gram": "application/srgs", - "gramps": "application/x-gramps-xml", - "gre": "application/vnd.geometry-explorer", - "groovy": "text/groovy", + "gac": "application/vnd.groove-account", + "ghf": "application/vnd.groove-help", + "gim": "application/vnd.groove-identity-message", "grv": "application/vnd.groove-injector", - "grxml": "application/srgs+xml", - "gsd": "audio/x-gsm", - "gsf": "application/x-font-ghostscript", - "gsheet": "application/vnd.google-apps.spreadsheet", - "gslides": "application/vnd.google-apps.presentation", - "gsm": "model/vnd.gdl", - "gsp": "application/x-gsp", - "gss": "application/x-gss", - "gtar": "application/x-gtar", "gtm": "application/vnd.groove-tool-message", - "gtw": "model/vnd.gtw", - "gv": "text/vnd.graphviz", - "gxf": "application/gxf", - "gxt": "application/vnd.geonext", - "gyb": "text/gyb", - "gyp": "text/gyp", - "gypi": "text/gyp", - "gz": "application/gzip", - "h": "text/x-chdr", - "h++": "text/x-c++hdr", - "h261": "video/h261", - "h263": "video/h263", - "h264": "video/h264", + "tpl": "application/vnd.groove-tool-template", + "vcg": "application/vnd.groove-vcard", "hal": "application/vnd.hal+xml", - "hbc": "application/vnd.hbci", + "zmm": "application/vnd.handheld-entertainment+xml", "hbci": "application/vnd.hbci", - "hbs": "text/x-handlebars-template", - "hdd": "application/x-virtualbox-hdd", - "hdf": "application/x-hdf", - "hdr": "image/vnd.radiance", - "hdt": "application/vnd.hdt", - "heic": "image/heic", - "heics": "image/heic-sequence", - "heif": "image/heif", - "heifs": "image/heif-sequence", - "help": "application/x-helpfile", - "hgl": "application/vnd.hp-hpgl", - "hh": "text/plain", - "hin": "chemical/x-hin", - "hjson": "application/hjson", - "hlb": "text/x-script", - "hlp": "application/winhlp", - "hpg": "application/vnd.hp-hpgl", + "les": "application/vnd.hhe.lesson-player", "hpgl": "application/vnd.hp-hpgl", - "hpi": "application/vnd.hp-hpid", "hpid": "application/vnd.hp-hpid", - "hpp": "text/x-c++hdr", "hps": "application/vnd.hp-hps", - "hpub": "application/prs.hpub+zip", - "hqx": "application/mac-binhex40", - "hs": "text/x-haskell", - "hta": "application/hta", - "htc": "text/x-component", - "htke": "application/vnd.kenameaapp", - "html": "text/html", - "htt": "text/webviewhtml", - "hvd": "application/vnd.yamaha.hv-dic", - "hvp": "application/vnd.yamaha.hv-voice", - "hvs": "application/vnd.yamaha.hv-script", - "hx": "text/haxe", - "hxml": "text/haxe", - "hxx": "text/plain", - "i2g": "application/vnd.intergeo", - "ic0": "application/vnd.commerce-battelle", - "ic1": "application/vnd.commerce-battelle", - "ic2": "application/vnd.commerce-battelle", - "ic3": "application/vnd.commerce-battelle", - "ic4": "application/vnd.commerce-battelle", - "ic5": "application/vnd.commerce-battelle", - "ic6": "application/vnd.commerce-battelle", - "ic7": "application/vnd.commerce-battelle", - "ic8": "application/vnd.commerce-battelle", - "ica": "application/vnd.commerce-battelle", + "jlt": "application/vnd.hp-jlyt", + "pcl": "application/vnd.hp-pcl", + "pclxl": "application/vnd.hp-pclxl", + "sfd-hdstx": "application/vnd.hydrostatix.sof-data", + "mpy": "application/vnd.ibm.minipay", + "afp": "application/vnd.ibm.modcap", + "listafp": "application/vnd.ibm.modcap", + "list3820": "application/vnd.ibm.modcap", + "irm": "application/vnd.ibm.rights-management", + "sc": "application/vnd.ibm.secure-container", "icc": "application/vnd.iccprofile", - "icd": "application/vnd.commerce-battelle", - "ice": "x-conference/x-cooltalk", - "icf": "application/vnd.commerce-battelle", "icm": "application/vnd.iccprofile", - "icns": "binary/icns", - "ico": "image/x-icon", - "ics": "text/calendar", - "icz": "text/calendar", - "idc": "text/plain", - "idl": "text/idl", - "ief": "image/ief", - "iefs": "image/ief", - "ifb": "text/calendar", - "ifm": "application/vnd.shana.informed.formdata", - "iges": "model/iges", "igl": "application/vnd.igloader", - "igm": "application/vnd.insors.igm", - "ign": "application/vnd.coreos.ignition+json", - "ignition": "application/vnd.coreos.ignition+json", - "igs": "model/iges", - "igx": "application/vnd.micrografx.igx", - "iif": "application/vnd.shana.informed.interchange", - "iii": "application/x-iphone", - "ima": "application/x-ima", - "imap": "application/x-httpd-imap", - "imf": "application/vnd.imagemeter.folder+zip", - "img": "application/octet-stream", - "imgcal": "application/vnd.3lightssoftware.imagescal", - "imi": "application/vnd.imagemeter.image+zip", - "imp": "application/vnd.accpac.simply.imp", - "ims": "application/vnd.ms-ims", - "imscc": "application/vnd.ims.imsccv1p1", - "in": "text/plain", - "inc": "text/inc", - "inf": "application/inf", - "info": "application/x-info", - "ini": "text/ini", - "ink": "application/inkml+xml", - "inkml": "application/inkml+xml", - "inp": "chemical/x-gamess-input", - "ins": "application/x-internet-signup", - "install": "application/x-install-instructions", - "iota": "application/vnd.astraea-software.iota", - "ip": "application/x-ip2", - "ipfix": "application/ipfix", - "ipk": "application/vnd.shana.informed.package", - "irm": "application/vnd.ibm.rights-management", - "irp": "application/vnd.irepository.package+xml", - "ism": "model/vnd.gdl", - "iso": "application/x-iso9660-image", - "isp": "application/x-internet-signup", - "ist": "chemical/x-isostar", - "istr": "chemical/x-isostar", - "isu": "video/x-isvideo", - "it": "audio/it", - "itp": "application/vnd.shana.informed.formtemplate", - "its": "application/its+xml", - "iv": "application/x-inventor", "ivp": "application/vnd.immervision-ivp", - "ivr": "i-world/i-vrml", "ivu": "application/vnd.immervision-ivu", - "ivy": "application/x-livescreen", - "j2": "text/jinja", - "jad": "text/vnd.sun.j2me.app-descriptor", - "jade": "text/jade", + "igm": "application/vnd.insors.igm", + "xpw": "application/vnd.intercon.formnet", + "xpx": "application/vnd.intercon.formnet", + "i2g": "application/vnd.intergeo", + "qbo": "application/vnd.intu.qbo", + "qfx": "application/vnd.intu.qfx", + "rcprofile": "application/vnd.ipunplugged.rcprofile", + "irp": "application/vnd.irepository.package+xml", + "xpr": "application/vnd.is-xpr", + "fcs": "application/vnd.isac.fcs", "jam": "application/vnd.jam", - "jar": "application/x-java-archive", - "jardiff": "application/x-java-archive-diff", - "java": "text/x-java-source", - "jcm": "application/x-java-commerce", - "jdx": "chemical/x-jcamp-dx", - "jenkinsfile": "text/groovy", - "jfif": "image/jpeg", - "jinja": "text/jinja", - "jinja2": "text/jinja", + "rms": "application/vnd.jcp.javame.midlet-rms", "jisp": "application/vnd.jisp", - "jls": "image/jls", - "jlt": "application/vnd.hp-jlyt", - "jl": "text/julia", - "jmz": "application/x-jmol", - "jng": "image/x-jng", - "jnlp": "application/x-java-jnlp-file", "joda": "application/vnd.joost.joda-archive", - "jp2": "image/jp2", - "jpe": "image/jpeg", - "jpeg": "image/jpeg", - "jpf": "image/jpx", - "jpg": "image/jpeg", - "jpg2": "image/jp2", - "jpgm": "image/jpm", - "jpgv": "video/jpeg", - "jpm": "image/jpm", - "jps": "image/x-jps", - "jpx": "image/jpx", - "jrd": "application/jrd+json", - "js": "application/javascript", - "json": "application/json", - "json-patch": "application/json-patch+json", - "json5": "application/json5", - "jsonld": "application/ld+json", - "jsonml": "application/jsonml+json", - "jsx": "text/jsx", - "jtd": "text/vnd.esmertec.theme-descriptor", - "jut": "image/jutvision", - "kar": "audio/midi", + "ktz": "application/vnd.kahootz", + "ktr": "application/vnd.kahootz", "karbon": "application/vnd.kde.karbon", - "kcm": "application/vnd.nervana", - "key": "application/pgp-keys", - "keynote": "application/vnd.apple.keynote", + "chrt": "application/vnd.kde.kchart", "kfo": "application/vnd.kde.kformula", - "kia": "application/vnd.kidspiration", - "kil": "application/x-killustrator", - "kin": "chemical/x-kinemage", - "kml": "application/vnd.google-earth.kml+xml", - "kmz": "application/vnd.google-earth.kmz", - "kne": "application/vnd.kinar", - "knp": "application/vnd.kinar", - "kom": "application/vnd.hbci", + "flw": "application/vnd.kde.kivio", "kon": "application/vnd.kde.kontour", - "koz": "audio/vnd.audikoz", "kpr": "application/vnd.kde.kpresenter", "kpt": "application/vnd.kde.kpresenter", - "kpxx": "application/vnd.ds-keypoint", - "ksh": "application/x-ksh", "ksp": "application/vnd.kde.kspread", - "kt": "text/kotlin", - "ktr": "application/vnd.kahootz", - "ktx": "image/ktx", - "ktz": "application/vnd.kahootz", "kwd": "application/vnd.kde.kword", "kwt": "application/vnd.kde.kword", - "l16": "audio/l16", - "la": "audio/nspaudio", - "lam": "audio/x-liveaudio", - "lasjson": "application/vnd.las.las+json", + "htke": "application/vnd.kenameaapp", + "kia": "application/vnd.kidspiration", + "kne": "application/vnd.kinar", + "knp": "application/vnd.kinar", + "skp": "application/vnd.koan", + "skd": "application/vnd.koan", + "skt": "application/vnd.koan", + "skm": "application/vnd.koan", + "sse": "application/vnd.kodak-descriptor", "lasxml": "application/vnd.las.las+xml", - "latex": "application/x-latex", - "lbc": "audio/ilbc", "lbd": "application/vnd.llamagraphics.life-balance.desktop", "lbe": "application/vnd.llamagraphics.life-balance.exchange+xml", - "le": "application/vnd.bluetooth.le.oob", - "les": "application/vnd.hhe.lesson-player", - "less": "text/less", - "lgr": "application/lgr+xml", - "lha": "application/octet-stream", - "lhs": "text/x-literate-haskell", - "lhx": "application/octet-stream", - "lin": "application/bbolin", - "link66": "application/vnd.route66.link66+xml", - "list": "text/plain", - "list3820": "application/vnd.ibm.modcap", - "listafp": "application/vnd.ibm.modcap", - "lmp": "model/vnd.gdl", - "lnk": "application/x-ms-shortcut", - "log": "text/plain", - "lostsyncxml": "application/lostsync+xml", - "lostxml": "application/lost+xml", - "lrf": "application/octet-stream", - "lrm": "application/vnd.ms-lrm", - "lsf": "video/x-la-asf", - "lsp": "text/x-script.lisp", - "lst": "text/plain", - "lsx": "video/x-la-asf", - "ltf": "application/vnd.frogans.ltf", - "ltx": "application/x-latex", - "lua": "text/x-lua", - "luac": "application/x-lua-bytecode", - "lvp": "audio/vnd.lucent.voice", + "123": "application/vnd.lotus-1-2-3", + "apr": "application/vnd.lotus-approach", + "pre": "application/vnd.lotus-freelance", + "nsf": "application/vnd.lotus-notes", + "org": "application/vnd.lotus-organizer", + "scm": "application/vnd.lotus-screencam", "lwp": "application/vnd.lotus-wordpro", - "lxf": "application/lxf", - "lyx": "application/x-lyx", - "lzh": "application/octet-stream", - "lzx": "application/x-lzx", - "m": "application/vnd.wolfram.mathematica.package", - "m13": "application/x-msmediaview", - "m14": "application/x-msmediaview", - "m15": "audio/x-mod", - "m1v": "video/mpeg", - "m21": "application/mp21", - "m2a": "audio/mpeg", - "m2v": "video/mpeg", - "m3a": "audio/mpeg", - "m3g": "application/m3g", - "m3u": "audio/x-mpegurl", - "m3u8": "application/vnd.apple.mpegurl", - "m4a": "audio/x-m4a", - "m4s": "video/iso.segment", - "m4u": "video/vnd.mpegurl", - "m4v": "video/x-m4v", - "ma": "application/mathematica", - "mads": "application/mads+xml", - "mag": "application/vnd.ecowin.chart", - "mail": "message/rfc822", - "maker": "application/vnd.framemaker", - "man": "application/x-troff-man", - "manifest": "text/cache-manifest", - "map": "application/x-navimap", - "mar": "text/plain", - "markdown": "text/markdown", - "mathml": "application/mathml+xml", - "mb": "application/mathematica", - "mbd": "application/mbedlet", - "mbk": "application/vnd.mobius.mbk", - "mbox": "application/mbox", - "mc$": "application/x-magic-cap-package-1.0", - "mc1": "application/vnd.medcalcdata", + "portpkg": "application/vnd.macports.portpkg", "mcd": "application/vnd.mcd", - "mcf": "image/vasa", - "mcif": "chemical/x-mmcif", - "mcm": "chemical/x-macmolecule", - "mcp": "application/netmc", - "mcurl": "text/vnd.curl.mcurl", - "md": "text/markdown", - "mdb": "application/x-msaccess", - "mdc": "application/vnd.marlin.drm.mdcf", - "mdi": "image/vnd.ms-modi", - "me": "application/x-troff-me", - "med": "audio/x-mod", - "mesh": "model/mesh", - "meta4": "application/metalink4+xml", - "metalink": "application/metalink+xml", - "mets": "application/mets+xml", - "mf4": "application/mf4", + "mc1": "application/vnd.medcalcdata", + "cdkey": "application/vnd.mediastation.cdkey", + "mwf": "application/vnd.mfer", "mfm": "application/vnd.mfmp", - "mft": "application/rpki-manifest", - "mgp": "application/vnd.osgeo.mapguide.package", - "mgz": "application/vnd.proteus.magazine", - "mht": "message/rfc822", - "mhtml": "message/rfc822", - "mib": "text/mib", - "mid": "audio/midi", - "midi": "audio/midi", - "mie": "application/x-mie", - "mif": "application/x-mif", - "mime": "message/rfc822", - "miz": "text/mizar", - "mj2": "video/mj2", - "mjf": "audio/x-vnd.audioexplosion.mjuicemediafile", - "mjp2": "video/mj2", - "mjpg": "video/x-motion-jpeg", - "mjs": "application/javascript", - "mk": "text/makefile", - "mk3d": "video/x-matroska-3d", - "mka": "audio/x-matroska", - "mkd": "text/x-markdown", - "mks": "video/x-matroska", - "mkv": "video/x-matroska", - "mlp": "application/vnd.dolby.mlp", - "mm": "application/x-freemind", - "mmd": "application/vnd.chipnuts.karaoke-mmd", - "mmdb": "application/vnd.maxmind.maxmind-db", - "mme": "application/base64", - "mmf": "application/vnd.smaf", - "mml": "text/mathml", - "mmod": "chemical/x-macromodel-input", - "mmr": "image/vnd.fujixerox.edmics-mmr", - "mms": "application/vnd.wap.mms-message", - "mng": "video/x-mng", - "mny": "application/x-msmoney", - "mobi": "application/x-mobipocket-ebook", - "moc": "text/x-moc", - "mod": "audio/x-mod", - "model-inter": "application/vnd.vd-study", - "mods": "application/mods+xml", - "modulemap": "text/modulemap", - "mol": "chemical/x-mdl-molfile", - "mol2": "chemical/x-mol2", - "moml": "model/vnd.moml+xml", - "moo": "chemical/x-mopac-out", - "moov": "video/quicktime", - "mop": "chemical/x-mopac-input", - "mopcrt": "chemical/x-mopac-input", - "mov": "video/quicktime", - "movie": "video/x-sgi-movie", - "mp1": "audio/mpeg", - "mp2": "audio/mpeg", - "mp21": "application/mp21", - "mp2a": "audio/mpeg", - "mp3": "audio/mp3", - "mp4": "video/mp4", - "mp4a": "audio/mp4", - "mp4s": "application/mp4", - "mp4v": "video/mp4", - "mpa": "video/mpeg", - "mpc": "application/vnd.mophun.certificate", - "mpd": "application/dash+xml", - "mpdd": "application/dashdelta", - "mpe": "video/mpeg", - "mpeg": "video/mpeg", - "mpega": "audio/mpeg", - "mpf": "text/vnd.ms-mediapackage", - "mpg": "video/mpeg", - "mpg4": "video/mp4", - "mpga": "audio/mpeg", - "mpkg": "application/vnd.apple.installer+xml", - "mpm": "application/vnd.blueice.multipass", + "flo": "application/vnd.micrografx.flo", + "igx": "application/vnd.micrografx.igx", + "mif": "application/vnd.mif", + "daf": "application/vnd.mobius.daf", + "dis": "application/vnd.mobius.dis", + "mbk": "application/vnd.mobius.mbk", + "mqy": "application/vnd.mobius.mqy", + "msl": "application/vnd.mobius.msl", + "plc": "application/vnd.mobius.plc", + "txf": "application/vnd.mobius.txf", "mpn": "application/vnd.mophun.application", + "mpc": "application/vnd.mophun.certificate", + "xul": "application/vnd.mozilla.xul+xml", + "cil": "application/vnd.ms-artgalry", + "cab": "application/vnd.ms-cab-compressed", + "xls": "application/vnd.ms-excel", + "xlm": "application/vnd.ms-excel", + "xla": "application/vnd.ms-excel", + "xlc": "application/vnd.ms-excel", + "xlt": "application/vnd.ms-excel", + "xlw": "application/vnd.ms-excel", + "xlam": "application/vnd.ms-excel.addin.macroenabled.12", + "xlsb": "application/vnd.ms-excel.sheet.binary.macroenabled.12", + "xlsm": "application/vnd.ms-excel.sheet.macroenabled.12", + "xltm": "application/vnd.ms-excel.template.macroenabled.12", + "eot": "application/vnd.ms-fontobject", + "chm": "application/vnd.ms-htmlhelp", + "ims": "application/vnd.ms-ims", + "lrm": "application/vnd.ms-lrm", + "thmx": "application/vnd.ms-officetheme", + "cat": "application/vnd.ms-pki.seccat", + "stl": "application/vnd.ms-pki.stl", + "ppt": "application/vnd.ms-powerpoint", + "pps": "application/vnd.ms-powerpoint", + "pot": "application/vnd.ms-powerpoint", + "ppam": "application/vnd.ms-powerpoint.addin.macroenabled.12", + "pptm": "application/vnd.ms-powerpoint.presentation.macroenabled.12", + "sldm": "application/vnd.ms-powerpoint.slide.macroenabled.12", + "ppsm": "application/vnd.ms-powerpoint.slideshow.macroenabled.12", + "potm": "application/vnd.ms-powerpoint.template.macroenabled.12", "mpp": "application/vnd.ms-project", "mpt": "application/vnd.ms-project", - "mpv": "application/x-project", - "mpv2": "video/mpeg", - "mpx": "application/x-project", - "mpy": "application/vnd.ibm.minipay", - "mqy": "application/vnd.mobius.mqy", - "mrc": "application/marc", - "mrcx": "application/marcxml+xml", - "ms": "application/x-troff-ms", - "msa": "application/vnd.msa-disk-image", - "mscml": "application/mediaservercontrol+xml", - "msd": "application/vnd.fdsn.mseed", - "mseed": "application/vnd.fdsn.mseed", + "docm": "application/vnd.ms-word.document.macroenabled.12", + "dotm": "application/vnd.ms-word.template.macroenabled.12", + "wps": "application/vnd.ms-works", + "wks": "application/vnd.ms-works", + "wcm": "application/vnd.ms-works", + "wdb": "application/vnd.ms-works", + "wpl": "application/vnd.ms-wpl", + "xps": "application/vnd.ms-xpsdocument", "mseq": "application/vnd.mseq", - "msf": "application/vnd.epson.msf", - "msg": "application/vnd.ms-outlook", - "msh": "model/mesh", - "msi": "application/x-msi", - "msl": "application/vnd.mobius.msl", - "msm": "model/vnd.gdl", - "msty": "application/vnd.muvee.style", - "mtm": "audio/x-mod", - "mts": "model/vnd.mts", - "multitrack": "audio/vnd.presonus.multitrack", "mus": "application/vnd.musician", - "musd": "application/mmt-usd+xml", - "musicxml": "application/vnd.recordare.musicxml+xml", - "mv": "video/x-sgi-movie", - "mvb": "application/x-msmediaview", - "mvt": "application/vnd.mapbox-vector-tile", - "mwc": "application/vnd.dpgraph", - "mwf": "application/vnd.mfer", - "mxf": "application/mxf", - "mxi": "application/vnd.vd-study", - "mxl": "application/vnd.recordare.musicxml", - "mxmf": "audio/mobile-xmf", - "mxml": "application/xv+xml", - "mxs": "application/vnd.triscape.mxs", - "mxu": "video/vnd.mpegurl", - "my": "audio/make", - "mzz": "application/x-vnd.audioexplosion.mzz", - "n-gage": "application/vnd.nokia.n-gage.symbian.install", - "n3": "text/n3", - "nap": "image/naplps", - "naplps": "image/naplps", - "nb": "application/mathematica", - "nbp": "application/vnd.wolfram.player", - "nc": "application/x-netcdf", - "ncm": "application/vnd.nokia.configuration-message", - "ncx": "application/x-dtbncx+xml", - "ndc": "application/vnd.osa.netdeploy", - "ndjson": "application/json", - "ndl": "application/vnd.lotus-notes", - "nds": "application/vnd.nintendo.nitro.rom", - "nef": "image/x-nikon-nef", - "nfo": "text/x-nfo", - "ngdat": "application/vnd.nokia.n-gage.data", - "ngdoc": "text/ngdoc", - "nif": "image/x-niff", - "niff": "image/x-niff", + "msty": "application/vnd.muvee.style", + "taglet": "application/vnd.mynfc", + "nlu": "application/vnd.neurolanguage.nlu", "nim": "text/nim", "nimble": "text/nimble", "nimf": "text/nim", "nims": "text/nim", + "ntf": "application/vnd.nitf", "nitf": "application/vnd.nitf", - "nix": "application/x-mix-transfer", - "nlu": "application/vnd.neurolanguage.nlu", - "nml": "application/vnd.enliven", "nnd": "application/vnd.noblenet-directory", "nns": "application/vnd.noblenet-sealer", "nnw": "application/vnd.noblenet-web", - "notebook": "application/vnd.smart.notebook", - "npx": "image/vnd.net-fpx", - "nq": "application/n-quads", - "ns2": "application/vnd.lotus-notes", - "ns3": "application/vnd.lotus-notes", - "ns4": "application/vnd.lotus-notes", - "nsc": "application/x-conference", - "nsf": "application/vnd.lotus-notes", - "nsg": "application/vnd.lotus-notes", - "nsh": "application/vnd.lotus-notes", - "nt": "application/n-triples", - "ntf": "application/vnd.lotus-notes", - "numbers": "application/vnd.apple.numbers", - "nvd": "application/x-navidoc", - "nwc": "application/x-nwc", - "nws": "message/rfc822", - "nzb": "application/x-nzb", - "o": "application/x-object", - "o4a": "application/vnd.oma.drm.dcf", - "o4v": "application/vnd.oma.drm.dcf", - "oa2": "application/vnd.fujitsu.oasys2", - "oa3": "application/vnd.fujitsu.oasys3", - "oas": "application/vnd.fujitsu.oasys", - "obd": "application/x-msbinder", - "obg": "application/vnd.openblox.game-binary", - "obgx": "application/vnd.openblox.game+xml", - "obj": "application/x-tgif", - "oda": "application/oda", - "odb": "application/vnd.oasis.opendocument.database", + "ngdat": "application/vnd.nokia.n-gage.data", + "n-gage": "application/vnd.nokia.n-gage.symbian.install", + "rpst": "application/vnd.nokia.radio-preset", + "rpss": "application/vnd.nokia.radio-presets", + "edm": "application/vnd.novadigm.edm", + "edx": "application/vnd.novadigm.edx", + "ext": "application/vnd.novadigm.ext", "odc": "application/vnd.oasis.opendocument.chart", - "odd": "application/tei+xml", + "otc": "application/vnd.oasis.opendocument.chart-template", + "odb": "application/vnd.oasis.opendocument.database", "odf": "application/vnd.oasis.opendocument.formula", "odft": "application/vnd.oasis.opendocument.formula-template", "odg": "application/vnd.oasis.opendocument.graphics", - "odi": "application/vnd.oasis.opendocument.image", - "odm": "application/vnd.oasis.opendocument.text-master", - "odp": "application/vnd.oasis.opendocument.presentation", - "ods": "application/vnd.oasis.opendocument.spreadsheet", - "odt": "application/vnd.oasis.opendocument.text", - "odx": "application/odx", - "oeb": "application/vnd.openeye.oeb", - "oga": "audio/ogg", - "ogex": "model/vnd.opengex", - "ogg": "audio/ogg", - "ogv": "video/ogg", - "ogx": "application/ogg", - "old": "application/x-trash", - "omc": "application/x-omc", - "omcd": "application/x-omcdatamaker", - "omcr": "application/x-omcregerator", - "omdoc": "application/omdoc+xml", - "omg": "audio/atrac3", - "onepkg": "application/onenote", - "onetmp": "application/onenote", - "onetoc": "application/onenote", - "onetoc2": "application/onenote", - "opf": "application/oebps-package+xml", - "opml": "text/x-opml", - "oprc": "application/vnd.palm", - "opus": "audio/ogg", - "or2": "application/vnd.lotus-organizer", - "or3": "application/vnd.lotus-organizer", - "orf": "image/x-olympus-orf", - "org": "text/x-org", - "orq": "application/ocsp-request", - "ors": "application/ocsp-response", - "osf": "application/vnd.yamaha.openscoreformat", - "osfpvg": "application/vnd.yamaha.openscoreformat.osfpvg+xml", - "osm": "application/vnd.openstreetmap.data+xml", - "otc": "application/vnd.oasis.opendocument.chart-template", - "otf": "font/otf", "otg": "application/vnd.oasis.opendocument.graphics-template", - "oth": "application/vnd.oasis.opendocument.text-web", + "odi": "application/vnd.oasis.opendocument.image", "oti": "application/vnd.oasis.opendocument.image-template", + "odp": "application/vnd.oasis.opendocument.presentation", "otp": "application/vnd.oasis.opendocument.presentation-template", + "ods": "application/vnd.oasis.opendocument.spreadsheet", "ots": "application/vnd.oasis.opendocument.spreadsheet-template", + "odt": "application/vnd.oasis.opendocument.text", + "odm": "application/vnd.oasis.opendocument.text-master", "ott": "application/vnd.oasis.opendocument.text-template", - "ova": "application/x-virtualbox-ova", - "ovf": "application/x-virtualbox-ovf", - "owx": "application/owl+xml", - "oxlicg": "application/vnd.oxli.countgraph", - "oxps": "application/oxps", + "oth": "application/vnd.oasis.opendocument.text-web", + "xo": "application/vnd.olpc-sugar", + "dd2": "application/vnd.oma.dd2+xml", "oxt": "application/vnd.openofficeorg.extension", - "oza": "application/x-oz-application", - "p": "text/x-pascal", - "p10": "application/pkcs10", - "p12": "application/pkcs12", - "p2p": "application/vnd.wfa.p2p", - "p7a": "application/x-pkcs7-signature", - "p7b": "application/x-pkcs7-certificates", - "p7c": "application/pkcs7-mime", - "p7m": "application/pkcs7-mime", - "p7r": "application/x-pkcs7-certreqresp", - "p7s": "application/pkcs7-signature", - "p8": "application/pkcs8", - "pac": "application/x-ns-proxy-autoconfig", - "pack": "application/x-java-pack200", - "package": "application/vnd.autopackage", - "pages": "application/vnd.apple.pages", - "par": "text/plain-bas", - "part": "application/pro_eng", - "pas": "text/pascal", - "pat": "image/x-coreldrawpattern", - "patch": "text/x-diff", - "paw": "application/vnd.pawaafile", - "pbd": "application/vnd.powerbuilder6", - "pbm": "image/x-portable-bitmap", - "pcap": "application/vnd.tcpdump.pcap", - "pcf": "application/x-font-pcf", - "pcl": "application/vnd.hp-pcl", - "pclxl": "application/vnd.hp-pclxl", - "pct": "image/x-pict", - "pcurl": "application/vnd.curl.pcurl", - "pcx": "image/x-pcx", - "pdb": "application/vnd.palm", - "pde": "text/x-processing", - "pdf": "application/pdf", - "pdx": "application/pdx", - "pem": "text/pem", - "pfa": "application/x-font-type1", - "pfb": "application/x-font-type1", - "pfm": "application/x-font-type1", - "pfr": "application/font-tdpfr", - "pfunk": "audio/make", - "pfx": "application/pkcs12", - "pgb": "image/vnd.globalgraphics.pgb", - "pgm": "image/x-portable-graymap", - "pgn": "application/x-chess-pgn", - "pgp": "application/pgp-encrypted", - "php": "application/x-httpd-php", - "php3": "application/x-httpd-php3", - "php3p": "application/x-httpd-php3-preprocessed", - "php4": "application/x-httpd-php4", - "php5": "application/x-httpd-php5", - "phps": "application/x-httpd-php-source", - "pht": "application/x-httpd-php", - "phtml": "application/x-httpd-php", - "pic": "image/pict", - "pict": "image/pict", - "pil": "application/vnd.piaccess.application-license", - "pk": "application/x-tex-pk", - "pkd": "application/vnd.hbci", - "pkg": "application/vnd.apple.installer+xml", - "pki": "application/pkixcmp", - "pkipath": "application/pkix-pkipath", - "pko": "application/ynd.ms-pkipko", - "pkpass": "application/vnd.apple.pkpass", - "pl": "application/x-perl", - "plantuml": "text/plantuml", - "plb": "application/vnd.3gpp.pic-bw-large", - "plc": "application/vnd.mobius.plc", - "plf": "application/vnd.pocketlearn", - "plj": "audio/vnd.everad.plj", - "plp": "application/vnd.panoply", - "pls": "application/pls+xml", - "plx": "application/x-pixclscript", - "ply": "model/stanford", - "pm": "text/plain", - "pm4": "application/x-pagemaker", - "pm5": "application/x-pagemaker", - "pma": "application/x-perfmon", - "pmc": "application/x-perfmon", - "pml": "application/vnd.ctc-posml", - "pmr": "application/x-perfmon", - "pmw": "application/x-perfmon", - "png": "image/png", - "pnm": "image/x-portable-anymap", - "po": "text/pofile", - "pod": "text/x-pod", - "portpkg": "application/vnd.macports.portpkg", - "pot": "application/vnd.ms-powerpoint", - "potm": "application/vnd.ms-powerpoint.template.macroenabled.12", - "potx": "application/vnd.openxmlformats-officedocument.presentationml.template", - "pov": "model/x-pov", - "pp": "text/puppet", - "ppa": "application/vnd.ms-powerpoint", - "ppam": "application/vnd.ms-powerpoint.addin.macroenabled.12", - "ppd": "application/vnd.cups-ppd", - "ppkg": "application/vnd.xmpie.ppkg", - "ppm": "image/x-portable-pixmap", - "pps": "application/vnd.ms-powerpoint", - "ppsm": "application/vnd.ms-powerpoint.slideshow.macroenabled.12", - "ppsx": "application/vnd.openxmlformats-officedocument.presentationml.slideshow", - "ppt": "application/vnd.ms-powerpoint", - "pptm": "application/vnd.ms-powerpoint.presentation.macroenabled.12", "pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation", - "ppz": "application/mspowerpoint", + "sldx": "application/vnd.openxmlformats-officedocument.presentationml.slide", + "ppsx": "application/vnd.openxmlformats-officedocument.presentationml.slideshow", + "potx": "application/vnd.openxmlformats-officedocument.presentationml.template", + "xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "xltx": "application/vnd.openxmlformats-officedocument.spreadsheetml.template", + "docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "dotx": "application/vnd.openxmlformats-officedocument.wordprocessingml.template", + "mgp": "application/vnd.osgeo.mapguide.package", + "dp": "application/vnd.osgi.dp", + "esa": "application/vnd.osgi.subsystem", + "pdb": "application/vnd.palm", "pqa": "application/vnd.palm", - "prc": "application/vnd.palm", - "pre": "application/vnd.lotus-freelance", - "preminet": "application/vnd.preminet", - "prf": "application/pics-rules", - "proto": "text/proto", - "provn": "text/provenance-notation", - "provx": "application/provenance+xml", - "prt": "application/pro_eng", - "prz": "application/vnd.lotus-freelance", - "ps": "application/postscript", - "psb": "application/vnd.3gpp.pic-bw-small", - "psd": "image/vnd.adobe.photoshop", - "pseg3820": "application/vnd.ibm.modcap", - "psf": "application/x-font-linux-psf", - "psid": "audio/prs.sid", - "pskcxml": "application/pskc+xml", - "pti": "image/prs.pti", - "ptid": "application/vnd.pvi.ptid1", - "pub": "application/x-mspublisher", - "purs": "text/purescript", - "pvb": "application/vnd.3gpp.pic-bw-var", - "pvu": "paleovu/x-pv", - "pwn": "application/vnd.3m.post-it-notes", - "pwz": "application/vnd.ms-powerpoint", - "pxd": "text/cython", - "pxi": "text/cython", - "py": "text/x-script.phyton", - "pya": "audio/vnd.ms-playready.media.pya", - "pyc": "application/x-python-code", - "pyi": "text/pyi", - "pyo": "application/x-python-code", - "pyv": "video/vnd.ms-playready.media.pyv", - "pyx": "text/cython", - "qam": "application/vnd.epson.quickanime", - "qbo": "application/vnd.intu.qbo", - "qca": "application/vnd.ericsson.quickcall", - "qcall": "application/vnd.ericsson.quickcall", - "qcp": "audio/qcelp", - "qd3": "x-world/x-3dmf", - "qd3d": "x-world/x-3dmf", - "qfx": "application/vnd.intu.qfx", - "qgs": "application/x-qgis", - "qif": "image/x-quicktime", + "oprc": "application/vnd.palm", + "paw": "application/vnd.pawaafile", + "str": "application/vnd.pg.format", + "ei6": "application/vnd.pg.osasli", + "efif": "application/vnd.picsel", + "wg": "application/vnd.pmi.widget", + "plf": "application/vnd.pocketlearn", + "pbd": "application/vnd.powerbuilder6", + "box": "application/vnd.previewsystems.box", + "mgz": "application/vnd.proteus.magazine", "qps": "application/vnd.publishare-delta-tree", - "qt": "video/quicktime", - "qtc": "video/x-qtc", - "qti": "image/x-quicktime", - "qtif": "image/x-quicktime", - "qtl": "application/x-quicktimeplayer", - "quiz": "application/vnd.quobject-quoxdocument", - "quox": "application/vnd.quobject-quoxdocument", - "qvd": "application/vnd.theqvd", + "ptid": "application/vnd.pvi.ptid1", + "qxd": "application/vnd.quark.quarkxpress", + "qxt": "application/vnd.quark.quarkxpress", "qwd": "application/vnd.quark.quarkxpress", "qwt": "application/vnd.quark.quarkxpress", - "qxb": "application/vnd.quark.quarkxpress", - "qxd": "application/vnd.quark.quarkxpress", "qxl": "application/vnd.quark.quarkxpress", - "qxt": "application/vnd.quark.quarkxpress", - "r": "text/r", - "ra": "audio/x-realaudio", - "ram": "audio/x-pn-realaudio", - "raml": "application/raml+yaml", - "rapd": "application/route-apd+xml", - "rar": "application/x-rar-compressed", - "ras": "image/x-cmu-raster", - "rast": "image/cmu-raster", - "rb": "application/x-ruby", - "rcprofile": "application/vnd.ipunplugged.rcprofile", - "rct": "application/prs.nprend", - "rd": "chemical/x-mdl-rdfile", - "rda": "text/r", - "rdata": "text/r", - "rds": "text/r", - "rdf": "application/rdf+xml", - "rdf-crypt": "application/prs.rdf-xml-crypt", - "rdz": "application/vnd.data-vision.rdz", - "relo": "application/p2p-overlay+xml", - "rep": "application/vnd.businessobjects", - "request": "application/vnd.nervana", - "res": "application/x-dtbresource+xml", - "rexx": "text/x-script.rexx", - "rf": "image/vnd.rn-realflash", - "rfcxml": "application/rfc+xml", - "rgb": "image/x-rgb", - "rgbe": "image/vnd.radiance", - "rhtml": "application/x-httpd-eruby", - "rif": "application/reginfo+xml", - "rip": "audio/vnd.rip", - "ris": "application/x-research-info-systems", - "rl": "application/resource-lists+xml", - "rlc": "image/vnd.fujixerox.edmics-rlc", - "rld": "application/resource-lists-diff+xml", - "rlib": "text/rust", + "qxb": "application/vnd.quark.quarkxpress", + "bed": "application/vnd.realvnc.bed", + "mxl": "application/vnd.recordare.musicxml", + "musicxml": "application/vnd.recordare.musicxml+xml", + "cryptonote": "application/vnd.rig.cryptonote", + "cod": "application/vnd.rim.cod", "rm": "application/vnd.rn-realmedia", - "rmi": "audio/mid", - "rmm": "audio/x-pn-realaudio", - "rmp": "audio/x-pn-realaudio-plugin", - "rms": "application/vnd.jcp.javame.midlet-rms", "rmvb": "application/vnd.rn-realmedia-vbr", - "rnc": "application/relax-ng-compact-syntax", - "rnd": "application/prs.nprend", - "rng": "text/xml", - "rnx": "application/vnd.rn-realplayer", - "roa": "application/rpki-roa", - "roff": "text/troff", - "ros": "chemical/x-rosdal", - "rp": "image/vnd.rn-realpix", - "rp9": "application/vnd.cloanto.rp9", - "rpm": "application/x-redhat-package-manager", - "rpss": "application/vnd.nokia.radio-presets", - "rpst": "application/vnd.nokia.radio-preset", - "rq": "application/sparql-query", - "rs": "application/rls-services+xml", - "rsd": "application/rsd+xml", - "rsheet": "application/urc-ressheet+xml", - "rsm": "model/vnd.gdl", - "rss": "application/rss+xml", - "rst": "text/prs.fallenstein.rst", - "rt": "text/richtext", - "rtf": "text/rtf", - "rtx": "text/richtext", - "run": "application/x-makeself", - "rusd": "application/route-usd+xml", - "rv": "video/vnd.rn-realvideo", - "rxn": "chemical/x-mdl-rxnfile", - "s": "text/x-asm", - "s11": "video/vnd.sealed.mpeg1", - "s14": "video/vnd.sealed.mpeg4", - "s1a": "application/vnd.sealedmedia.softseal.pdf", - "s1e": "application/vnd.sealed.xls", - "s1g": "image/vnd.sealedmedia.softseal.gif", - "s1h": "application/vnd.sealedmedia.softseal.html", - "s1j": "image/vnd.sealedmedia.softseal.jpg", - "s1m": "audio/vnd.sealedmedia.softseal.mpeg", - "s1n": "image/vnd.sealed.png", - "s1p": "application/vnd.sealed.ppt", - "s1q": "video/vnd.sealedmedia.softseal.mov", - "s1w": "application/vnd.sealed.doc", - "s3df": "application/vnd.sealed.3df", - "s3m": "audio/s3m", - "sac": "application/tamp-sequence-adjust-confirm", - "saf": "application/vnd.yamaha.smaf-audio", - "sam": "application/vnd.lotus-wordpro", - "sandboxed": "text/html-sandboxed", - "sass": "text/x-sass", - "saveme": "application/octet-stream", - "sbk": "application/x-tbook", - "sbml": "application/sbml+xml", - "sc": "application/vnd.ibm.secure-container", - "scala": "text/x-scala", - "scd": "application/x-msschedule", - "sce": "application/vnd.etsi.asic-e+zip", - "scim": "application/scim+json", - "scld": "application/vnd.doremir.scorecloud-binary-document", - "scm": "application/vnd.lotus-screencam", - "scq": "application/scvp-cv-request", - "scr": "application/x-silverlight", - "scs": "application/scvp-cv-response", - "scsf": "application/vnd.sealed.csf", - "scss": "text/x-scss", - "sct": "text/scriptlet", - "scurl": "text/vnd.curl.scurl", - "sd": "chemical/x-mdl-sdfile", - "sd2": "audio/x-sd2", - "sda": "application/vnd.stardivision.draw", - "sdc": "application/vnd.stardivision.calc", - "sdd": "application/vnd.stardivision.impress", - "sdf": "application/vnd.kinar", - "sdkd": "application/vnd.solent.sdkm+xml", - "sdkm": "application/vnd.solent.sdkm+xml", - "sdml": "text/plain", - "sdo": "application/vnd.sealed.doc", - "sdoc": "application/vnd.sealed.doc", - "sdp": "application/sdp", - "sdr": "application/sounder", - "sdw": "application/vnd.stardivision.writer", - "sea": "application/x-sea", + "link66": "application/vnd.route66.link66+xml", + "st": "application/vnd.sailingtracker.track", "see": "application/vnd.seemail", - "seed": "application/vnd.fdsn.seed", - "sem": "application/vnd.sealed.eml", "sema": "application/vnd.sema", "semd": "application/vnd.semd", "semf": "application/vnd.semf", - "seml": "application/vnd.sealed.eml", - "ser": "application/java-serialized-object", - "set": "application/set", - "setpay": "application/set-payment-initiation", - "setreg": "application/set-registration-initiation", - "sfc": "application/vnd.nintendo.snes.rom", - "sfd": "application/vnd.font-fontforge-sfd", - "sfd-hdstx": "application/vnd.hydrostatix.sof-data", + "ifm": "application/vnd.shana.informed.formdata", + "itp": "application/vnd.shana.informed.formtemplate", + "iif": "application/vnd.shana.informed.interchange", + "ipk": "application/vnd.shana.informed.package", + "twd": "application/vnd.simtech-mindmapper", + "twds": "application/vnd.simtech-mindmapper", + "mmf": "application/vnd.smaf", + "teacher": "application/vnd.smart.teacher", + "sdkm": "application/vnd.solent.sdkm+xml", + "sdkd": "application/vnd.solent.sdkm+xml", + "dxp": "application/vnd.spotfire.dxp", "sfs": "application/vnd.spotfire.sfs", - "sfv": "text/x-sfv", - "sgf": "application/x-go-sgf", - "sgi": "image/sgi", - "sgif": "image/vnd.sealedmedia.softseal.gif", - "sgl": "application/vnd.stardivision.writer-global", - "sgm": "text/sgml", - "sgml": "text/sgml", - "sh": "application/x-sh", - "shar": "application/x-shar", - "shex": "text/shex", - "shf": "application/shf+xml", - "shp": "application/x-qgis", - "shx": "application/x-qgis", - "si": "text/vnd.wap.si", - "sic": "application/vnd.wap.sic", - "sid": "image/x-mrsid-image", - "sieve": "application/sieve", - "sig": "application/pgp-signature", - "sik": "application/x-trash", - "sil": "audio/silk", - "silo": "model/mesh", - "sis": "application/vnd.symbian.install", - "sisx": "x-epoc/x-sisx-app", - "sit": "application/x-stuffit", - "sitx": "application/x-stuffitx", - "siv": "application/sieve", - "sjp": "image/vnd.sealedmedia.softseal.jpg", - "sjpg": "image/vnd.sealedmedia.softseal.jpg", - "skd": "application/vnd.koan", - "skm": "application/vnd.koan", - "skp": "application/vnd.koan", - "skt": "application/vnd.koan", - "sl": "text/vnd.wap.sl", - "sla": "application/vnd.scribus", - "slaz": "application/vnd.scribus", - "slc": "application/vnd.wap.slc", - "sldm": "application/vnd.ms-powerpoint.slide.macroenabled.12", - "sldx": "application/vnd.openxmlformats-officedocument.presentationml.slide", - "sls": "application/route-s-tsid+xml", - "slt": "application/vnd.epson.salt", - "sm": "application/vnd.stepmania.stepchart", - "smc": "application/vnd.nintendo.snes.rom", + "sdc": "application/vnd.stardivision.calc", + "sda": "application/vnd.stardivision.draw", + "sdd": "application/vnd.stardivision.impress", "smf": "application/vnd.stardivision.math", - "smh": "application/vnd.sealed.mht", - "smht": "application/vnd.sealed.mht", - "smi": "application/smil+xml", - "smil": "application/smil+xml", - "smk": "video/vnd.radgamettools.smacker", - "sml": "application/smil+xml", - "smo": "video/vnd.sealedmedia.softseal.mov", - "smov": "video/vnd.sealedmedia.softseal.mov", - "smp": "audio/vnd.sealedmedia.softseal.mpeg", - "smp3": "audio/vnd.sealedmedia.softseal.mpeg", - "smpg": "video/vnd.sealed.mpeg1", - "sms": "application/vnd.3gpp2.sms", - "smv": "video/x-smv", + "sdw": "application/vnd.stardivision.writer", + "vor": "application/vnd.stardivision.writer", + "sgl": "application/vnd.stardivision.writer-global", "smzip": "application/vnd.stepmania.package", - "snd": "audio/basic", - "snf": "application/x-font-snf", - "so": "application/octet-stream", - "soa": "text/dns", - "soc": "application/sgml-open-catalog", - "sol": "application/solids", - "spc": "text/x-speech", - "spd": "application/vnd.sealedmedia.softseal.pdf", - "spdf": "application/vnd.sealedmedia.softseal.pdf", - "spec": "text/spec", - "spf": "application/vnd.yamaha.smaf-phrase", - "spl": "application/x-futuresplash", - "spn": "image/vnd.sealed.png", - "spng": "image/vnd.sealed.png", - "spo": "text/vnd.in3d.spot", - "spot": "text/vnd.in3d.spot", - "spp": "application/scvp-vp-response", - "sppt": "application/vnd.sealed.ppt", - "spq": "application/scvp-vp-request", - "spr": "application/x-sprite", - "sprite": "application/x-sprite", - "spx": "audio/ogg", - "sql": "application/x-sql", - "sr": "application/vnd.sigrok.session", - "src": "application/x-wais-source", - "srt": "application/x-subrip", - "sru": "application/sru+xml", - "srx": "application/sparql-results+xml", - "ssdl": "application/ssdl+xml", - "sse": "application/vnd.kodak-descriptor", - "ssf": "application/vnd.epson.ssf", - "ssi": "text/x-server-parsed-html", - "ssm": "application/streamingmedia", - "ssml": "application/ssml+xml", - "sst": "application/vnd.ms-pki.certstore", - "ssw": "video/vnd.sealed.swf", - "sswf": "video/vnd.sealed.swf", - "st": "application/vnd.sailingtracker.track", + "sm": "application/vnd.stepmania.stepchart", + "sxc": "application/vnd.sun.xml.calc", "stc": "application/vnd.sun.xml.calc.template", + "sxd": "application/vnd.sun.xml.draw", "std": "application/vnd.sun.xml.draw.template", - "step": "application/step", - "stf": "application/vnd.wt.stf", + "sxi": "application/vnd.sun.xml.impress", "sti": "application/vnd.sun.xml.impress.template", - "stif": "application/vnd.sealed.tiff", - "stk": "application/hyperstudio", - "stl": "application/vnd.ms-pki.stl", - "stm": "audio/x-stm", - "stml": "application/vnd.sealedmedia.softseal.html", - "stp": "application/step", - "str": "application/vnd.pg.format", - "study-inter": "application/vnd.vd-study", + "sxm": "application/vnd.sun.xml.math", + "sxw": "application/vnd.sun.xml.writer", + "sxg": "application/vnd.sun.xml.writer.global", "stw": "application/vnd.sun.xml.writer.template", - "sty": "text/x-tex", - "styl": "text/stylus", - "sub": "text/vnd.dvb.subtitle", "sus": "application/vnd.sus-calendar", "susp": "application/vnd.sus-calendar", - "sv4cpio": "application/x-sv4cpio", - "sv4crc": "application/x-sv4crc", - "svc": "application/vnd.dvb.service", "svd": "application/vnd.svd", - "svf": "image/x-dwg", - "svg": "image/svg+xml", - "svgz": "image/svg+xml", - "sw": "chemical/x-swissprot", - "swa": "application/x-director", - "swf": "application/x-shockwave-flash", - "swfl": "application/x-shockwave-flash", - "swi": "application/vnd.aristanetworks.swi", - "swift": "text/swift", - "swiftdeps": "text/swiftdeps", - "sxc": "application/vnd.sun.xml.calc", - "sxd": "application/vnd.sun.xml.draw", - "sxg": "application/vnd.sun.xml.writer.global", - "sxi": "application/vnd.sun.xml.impress", - "sxl": "application/vnd.sealed.xls", - "sxls": "application/vnd.sealed.xls", - "sxm": "application/vnd.sun.xml.math", - "sxw": "application/vnd.sun.xml.writer", - "t": "text/troff", - "t3": "application/x-t3vm-image", - "t38": "image/t38", - "tac": "text/twisted", - "tag": "text/prs.lines.tag", - "taglet": "application/vnd.mynfc", - "talk": "text/x-speech", - "tam": "application/vnd.onepager", - "tamp": "application/vnd.onepagertamp", - "tamx": "application/vnd.onepagertamx", + "sis": "application/vnd.symbian.install", + "sisx": "application/vnd.symbian.install", + "xsm": "application/vnd.syncml+xml", + "bdm": "application/vnd.syncml.dm+wbxml", + "xdm": "application/vnd.syncml.dm+xml", "tao": "application/vnd.tao.intent-module-archive", - "tap": "image/vnd.tencent.tap", - "tar": "application/x-tar", - "tat": "application/vnd.onepagertat", - "tatp": "application/vnd.onepagertatp", - "tatx": "application/vnd.onepagertatx", - "tau": "application/tamp-apex-update", - "taz": "application/x-gtar", - "tbk": "application/toolbook", - "tcap": "application/vnd.3gpp2.tcap", - "tcl": "application/x-tcl", - "tcsh": "text/x-script.tcsh", - "tcu": "application/tamp-community-update", - "td": "application/urc-targetdesc+xml", - "teacher": "application/vnd.smart.teacher", - "tei": "application/tei+xml", - "teicorpus": "application/tei+xml", - "ter": "application/tamp-error", - "tex": "application/x-tex", - "texi": "application/x-texinfo", - "texinfo": "application/x-texinfo", - "text": "text/plain", - "tf": "text/terraform", - "tfi": "application/thraud+xml", - "tfm": "application/x-tex-tfm", - "tfx": "image/tiff-fx", - "tga": "image/x-tga", - "tgf": "chemical/x-mdl-tgf", - "tgz": "application/gzip", - "thmx": "application/vnd.ms-officetheme", - "thrift": "text/thrift", - "tif": "image/tiff", - "tiff": "image/tiff", - "tk": "text/x-tcl", - "tlclient": "application/vnd.cendio.thinlinc.clientconf", - "tm": "text/texmacs", + "pcap": "application/vnd.tcpdump.pcap", + "cap": "application/vnd.tcpdump.pcap", + "dmp": "application/vnd.tcpdump.pcap", "tmo": "application/vnd.tmobile-livetv", - "tnef": "application/vnd.ms-tnef", - "tnf": "application/vnd.ms-tnef", - "toml": "text/toml", - "torrent": "application/x-bittorrent", - "tpl": "application/vnd.groove-tool-template", "tpt": "application/vnd.trid.tpt", - "tr": "text/troff", + "mxs": "application/vnd.triscape.mxs", "tra": "application/vnd.trueapp", - "tree": "application/vnd.rainstor.data", - "trig": "application/trig", - "trm": "application/x-msterminal", - "ts": "video/mp2t", - "tsa": "application/tamp-sequence-adjust", - "tscn": "text/godot", - "tsd": "application/timestamped-data", - "tsi": "audio/tsp-audio", - "tsp": "audio/tsplayer", - "tsq": "application/timestamp-query", - "tsr": "application/timestamp-reply", - "tst": "application/vnd.etsi.timestamp-token", - "tsv": "text/tab-separated-values", - "tsx": "text/tsx", - "ttc": "font/collection", - "ttf": "font/ttf", - "ttl": "text/turtle", - "ttml": "application/ttml+xml", - "tuc": "application/tamp-update-confirm", - "tur": "application/tamp-update", - "turbot": "image/florian", - "twd": "application/vnd.simtech-mindmapper", - "twds": "application/vnd.simtech-mindmapper", - "txd": "application/vnd.genomatix.tuxedo", - "txf": "application/vnd.mobius.txf", - "txt": "text/plain", - "u32": "application/x-authorware-bin", - "u8dsn": "message/global-delivery-status", - "u8hdr": "message/global-headers", - "u8mdn": "message/global-disposition-notification", - "u8msg": "message/global", - "udeb": "application/vnd.debian.binary-package", "ufd": "application/vnd.ufdl", "ufdl": "application/vnd.ufdl", - "uil": "text/x-uil", - "uis": "application/urc-uisocketdesc+xml", - "uls": "text/iuls", - "ult": "audio/x-mod", - "ulx": "application/x-glulx", + "utz": "application/vnd.uiq.theme", "umj": "application/vnd.umajin", - "uni": "audio/x-mod", - "unis": "text/uri-list", "unityweb": "application/vnd.unity", - "unv": "application/i-deas", - "uo": "application/vnd.uoml+xml", "uoml": "application/vnd.uoml+xml", - "upa": "application/vnd.hbci", - "uri": "text/uri-list", - "uric": "text/vnd.si.uricatalogue", - "urim": "application/vnd.uri-map", - "urimap": "application/vnd.uri-map", - "uris": "text/uri-list", - "urls": "text/uri-list", - "ustar": "application/x-ustar", - "utz": "application/vnd.uiq.theme", - "uu": "text/x-uuencode", - "uue": "text/x-uuencode", - "uva": "audio/vnd.dece.audio", - "uvd": "application/vnd.dece.data", - "uvf": "application/vnd.dece.data", - "uvg": "image/vnd.dece.graphic", - "uvh": "video/vnd.dece.hd", - "uvi": "image/vnd.dece.graphic", - "uvm": "video/vnd.dece.mobile", - "uvp": "video/vnd.dece.pd", - "uvs": "video/vnd.dece.sd", - "uvt": "application/vnd.dece.ttml+xml", - "uvu": "video/vnd.dece.mp4", - "uvv": "video/vnd.dece.video", - "uvva": "audio/vnd.dece.audio", - "uvvd": "application/vnd.dece.data", - "uvvf": "application/vnd.dece.data", - "uvvg": "image/vnd.dece.graphic", - "uvvh": "video/vnd.dece.hd", - "uvvi": "image/vnd.dece.graphic", - "uvvm": "video/vnd.dece.mobile", - "uvvp": "video/vnd.dece.pd", - "uvvs": "video/vnd.dece.sd", - "uvvt": "application/vnd.dece.ttml+xml", - "uvvu": "video/vnd.dece.mp4", - "uvvv": "video/vnd.dece.video", - "uvvx": "application/vnd.dece.unspecified", - "uvvz": "application/vnd.dece.zip", - "uvx": "application/vnd.dece.unspecified", - "uvz": "application/vnd.dece.zip", - "val": "chemical/x-ncbi-asn1-binary", - "vbk": "audio/vnd.nortel.vbk", - "vbox": "application/x-virtualbox-vbox", - "vbox-extpack": "application/x-virtualbox-vbox-extpack", - "vcard": "text/vcard", - "vcd": "application/x-cdlink", - "vcf": "text/x-vcard", - "vcg": "application/vnd.groove-vcard", - "vcs": "text/x-vcalendar", "vcx": "application/vnd.vcx", - "vda": "application/vda", - "vdi": "application/x-virtualbox-vdi", - "vdo": "video/vdo", - "vdx": "text/vdx", - "vew": "application/vnd.lotus-approach", - "vfr": "application/vnd.tml", - "vhd": "application/x-virtualbox-vhd", - "viaframe": "application/vnd.tml", - "vim": "text/vim", - "vis": "application/vnd.visionary", - "viv": "video/vnd.vivo", - "vivo": "video/vivo", - "vmd": "application/vocaltec-media-desc", - "vmdk": "application/x-virtualbox-vmdk", - "vmf": "application/vocaltec-media-file", - "vms": "chemical/x-vamas-iso14976", - "vmt": "application/vnd.valve.source.material", - "vob": "video/x-ms-vob", - "voc": "audio/voc", - "vor": "application/vnd.stardivision.writer", - "vos": "video/vosaic", - "vox": "audio/voxware", - "vpm": "multipart/voice-message", - "vqe": "audio/x-twinvq-plugin", - "vqf": "audio/x-twinvq", - "vql": "audio/x-twinvq-plugin", - "vrm": "x-world/x-vrml", - "vrml": "model/vrml", - "vrt": "x-world/x-vrt", - "vsc": "application/vnd.vidsoft.vidconference", "vsd": "application/vnd.visio", - "vsf": "application/vnd.vsf", - "vss": "application/vnd.visio", "vst": "application/vnd.visio", + "vss": "application/vnd.visio", "vsw": "application/vnd.visio", - "vtf": "image/vnd.valve.source.texture", - "vtt": "text/vtt", - "vtu": "model/vnd.vtu", - "vue": "text/vue", - "vwx": "application/vnd.vectorworks", - "vxml": "application/voicexml+xml", - "w3d": "application/x-director", - "w60": "application/wordperfect6.0", - "w61": "application/wordperfect6.1", - "w6w": "application/msword", - "wad": "application/x-doom", - "wadl": "application/vnd.sun.wadl+xml", - "war": "binary/zip", - "wasm": "application/wasm", - "wav": "audio/wave", - "wax": "audio/x-ms-wax", - "wb1": "application/x-qpro", - "wbmp": "image/vnd.wap.wbmp", - "wbs": "application/vnd.criticaltools.wbs+xml", + "vis": "application/vnd.visionary", + "vsf": "application/vnd.vsf", "wbxml": "application/vnd.wap.wbxml", - "wcm": "application/vnd.ms-works", - "wdb": "application/vnd.ms-works", - "wdp": "image/vnd.ms-photo", - "web": "application/vnd.xara", - "weba": "audio/webm", - "webapp": "application/x-web-app-manifest+json", - "webm": "video/webm", - "webmanifest": "application/manifest+json", - "webp": "image/webp", - "wg": "application/vnd.pmi.widget", - "wgt": "application/widget", - "whl": "binary/wheel", - "wif": "application/watcherinfo+xml", - "win": "model/vnd.gdl", - "wiz": "application/msword", - "wk": "application/x-123", - "wk1": "application/vnd.lotus-1-2-3", - "wk3": "application/vnd.lotus-1-2-3", - "wk4": "application/vnd.lotus-1-2-3", - "wks": "application/vnd.ms-works", - "wkt": "text/wkt", - "wlnk": "application/link-format", - "wm": "video/x-ms-wm", - "wma": "audio/x-ms-wma", - "wmc": "application/vnd.wmc", - "wmd": "application/x-ms-wmd", - "wmf": "image/wmf", - "wml": "text/vnd.wap.wml", "wmlc": "application/vnd.wap.wmlc", - "wmls": "text/vnd.wap.wmlscript", "wmlsc": "application/vnd.wap.wmlscriptc", - "wmv": "video/x-ms-wmv", - "wmx": "video/x-ms-wmx", - "wmz": "application/x-ms-wmz", - "woff": "font/woff", - "woff2": "font/woff2", - "word": "application/msword", - "wp": "application/wordperfect", - "wp5": "application/wordperfect", - "wp6": "application/wordperfect", + "wtb": "application/vnd.webturbo", + "nbp": "application/vnd.wolfram.player", "wpd": "application/vnd.wordperfect", - "wpl": "application/vnd.ms-wpl", - "wps": "application/vnd.ms-works", - "wq1": "application/x-lotus", "wqd": "application/vnd.wqd", - "wri": "application/x-mswrite", - "wrl": "model/vrml", - "wrz": "model/vrml", - "wsc": "message/vnd.wfa.wsc", + "stf": "application/vnd.wt.stf", + "xar": "application/vnd.xara", + "xfdl": "application/vnd.xfdl", + "hvd": "application/vnd.yamaha.hv-dic", + "hvs": "application/vnd.yamaha.hv-script", + "hvp": "application/vnd.yamaha.hv-voice", + "osf": "application/vnd.yamaha.openscoreformat", + "osfpvg": "application/vnd.yamaha.openscoreformat.osfpvg+xml", + "saf": "application/vnd.yamaha.smaf-audio", + "spf": "application/vnd.yamaha.smaf-phrase", + "cmp": "application/vnd.yellowriver-custom-menu", + "zir": "application/vnd.zul", + "zirz": "application/vnd.zul", + "zaz": "application/vnd.zzazz.deck+xml", + "vxml": "application/voicexml+xml", + "wasm": "application/wasm", + "wgt": "application/widget", + "hlp": "application/winhlp", "wsdl": "application/wsdl+xml", - "wsgi": "text/wsgi", "wspolicy": "application/wspolicy+xml", - "wsrc": "application/x-wais-source", - "wtb": "application/vnd.webturbo", - "wtk": "application/x-wintalk", - "wv": "application/vnd.wv.csp+wbxml", - "wvx": "video/x-ms-wvx", - "wz": "application/x-wingz", - "x-png": "image/png", + "7z": "application/x-7z-compressed", + "abw": "application/x-abiword", + "ace": "application/x-ace-compressed", + "dmg": "application/x-apple-diskimage", + "aab": "application/x-authorware-bin", "x32": "application/x-authorware-bin", - "x3d": "application/vnd.hzn-3d-crossword", - "x3db": "model/x3d+xml", - "x3dbz": "model/x3d+binary", - "x3dv": "model/x3d-vrml", - "x3dvz": "model/x3d-vrml", - "x3dz": "model/x3d+xml", - "x_b": "model/vnd.parasolid.transmit.binary", - "x_t": "model/vnd.parasolid.transmit.text", - "xaf": "x-world/x-vrml", - "xaml": "application/xaml+xml", - "xap": "application/x-silverlight-app", - "xar": "application/vnd.xara", - "xav": "application/xcap-att+xml", + "u32": "application/x-authorware-bin", + "vox": "application/x-authorware-bin", + "aam": "application/x-authorware-map", + "aas": "application/x-authorware-seg", + "bcpio": "application/x-bcpio", + "torrent": "application/x-bittorrent", + "blb": "application/x-blorb", + "blorb": "application/x-blorb", + "bz": "application/x-bzip", + "bz2": "application/x-bzip2", + "boz": "application/x-bzip2", + "cbr": "application/x-cbr", + "cba": "application/x-cbr", + "cbt": "application/x-cbr", + "cbz": "application/x-cbr", + "cb7": "application/x-cbr", + "vcd": "application/x-cdlink", + "cfs": "application/x-cfs-compressed", + "chat": "application/x-chat", + "pgn": "application/x-chess-pgn", + "nsc": "application/x-conference", + "cpio": "application/x-cpio", + "csh": "application/x-csh", + "deb": "application/x-debian-package", + "udeb": "application/x-debian-package", + "dgc": "application/x-dgc-compressed", + "dir": "application/x-director", + "dcr": "application/x-director", + "dxr": "application/x-director", + "cst": "application/x-director", + "cct": "application/x-director", + "cxt": "application/x-director", + "w3d": "application/x-director", + "fgd": "application/x-director", + "swa": "application/x-director", + "wad": "application/x-doom", + "ncx": "application/x-dtbncx+xml", + "dtb": "application/x-dtbook+xml", + "res": "application/x-dtbresource+xml", + "dvi": "application/x-dvi", + "evy": "application/x-envoy", + "eva": "application/x-eva", + "bdf": "application/x-font-bdf", + "gsf": "application/x-font-ghostscript", + "psf": "application/x-font-linux-psf", + "pcf": "application/x-font-pcf", + "snf": "application/x-font-snf", + "pfa": "application/x-font-type1", + "pfb": "application/x-font-type1", + "pfm": "application/x-font-type1", + "afm": "application/x-font-type1", + "arc": "application/x-freearc", + "spl": "application/x-futuresplash", + "gca": "application/x-gca-compressed", + "ulx": "application/x-glulx", + "gnumeric": "application/x-gnumeric", + "gramps": "application/x-gramps-xml", + "gtar": "application/x-gtar", + "hdf": "application/x-hdf", + "install": "application/x-install-instructions", + "iso": "application/x-iso9660-image", + "jnlp": "application/x-java-jnlp-file", + "latex": "application/x-latex", + "lzh": "application/x-lzh-compressed", + "lha": "application/x-lzh-compressed", + "mie": "application/x-mie", + "prc": "application/x-mobipocket-ebook", + "mobi": "application/x-mobipocket-ebook", + "application": "application/x-ms-application", + "lnk": "application/x-ms-shortcut", + "wmd": "application/x-ms-wmd", + "wmz": "application/x-ms-wmz", "xbap": "application/x-ms-xbap", - "xbd": "application/vnd.fujixerox.docuworks.binder", - "xbm": "image/x-xbitmap", - "xca": "application/xcap-caps+xml", - "xcf": "application/x-xcf", - "xcs": "application/calendar+xml", - "xct": "application/vnd.fujixerox.docuworks.container", - "xdd": "application/bacnet-xdd+zip", - "xdf": "application/xcap-diff+xml", - "xdm": "application/vnd.syncml.dm+xml", - "xdp": "application/vnd.adobe.xdp+xml", - "xdr": "video/x-amt-demorun", - "xdssc": "application/dssc+xml", - "xdw": "application/vnd.fujixerox.docuworks", - "xel": "application/xcap-el+xml", - "xenc": "application/xenc+xml", - "xer": "application/patch-ops-error+xml", - "xfd": "application/vnd.xfdl", - "xfdf": "application/vnd.adobe.xfdf", - "xfdl": "application/vnd.xfdl", - "xgz": "xgl/drawing", - "xht": "application/xhtml+xml", - "xhtm": "application/xhtml+xml", - "xhtml": "application/xhtml+xml", - "xhvml": "application/xv+xml", - "xif": "image/vnd.xiff", - "xl": "application/excel", - "xla": "application/vnd.ms-excel", - "xlam": "application/vnd.ms-excel.addin.macroenabled.12", - "xlb": "application/vndms-excel", - "xlc": "application/vnd.ms-excel", + "mdb": "application/x-msaccess", + "obd": "application/x-msbinder", + "crd": "application/x-mscardfile", + "clp": "application/x-msclip", + "exe": "application/x-msdownload", + "dll": "application/x-msdownload", + "com": "application/x-msdownload", + "bat": "application/x-msdownload", + "msi": "application/x-msdownload", + "mvb": "application/x-msmediaview", + "m13": "application/x-msmediaview", + "m14": "application/x-msmediaview", + "wmf": "application/x-msmetafile", + "wmz": "application/x-msmetafile", + "emf": "application/x-msmetafile", + "emz": "application/x-msmetafile", + "mny": "application/x-msmoney", + "pub": "application/x-mspublisher", + "scd": "application/x-msschedule", + "trm": "application/x-msterminal", + "wri": "application/x-mswrite", + "nc": "application/x-netcdf", + "cdf": "application/x-netcdf", + "nzb": "application/x-nzb", + "p12": "application/x-pkcs12", + "pfx": "application/x-pkcs12", + "p7b": "application/x-pkcs7-certificates", + "spc": "application/x-pkcs7-certificates", + "p7r": "application/x-pkcs7-certreqresp", + "rar": "application/x-rar-compressed", + "ris": "application/x-research-info-systems", + "sh": "application/x-sh", + "shar": "application/x-shar", + "swf": "application/x-shockwave-flash", + "xap": "application/x-silverlight-app", + "sql": "application/x-sql", + "sit": "application/x-stuffit", + "sitx": "application/x-stuffitx", + "srt": "application/x-subrip", + "sv4cpio": "application/x-sv4cpio", + "sv4crc": "application/x-sv4crc", + "t3": "application/x-t3vm-image", + "gam": "application/x-tads", + "tar": "application/x-tar", + "tcl": "application/x-tcl", + "tex": "application/x-tex", + "tfm": "application/x-tex-tfm", + "texinfo": "application/x-texinfo", + "texi": "application/x-texinfo", + "obj": "application/x-tgif", + "ustar": "application/x-ustar", + "src": "application/x-wais-source", + "der": "application/x-x509-ca-cert", + "crt": "application/x-x509-ca-cert", + "fig": "application/x-xfig", "xlf": "application/x-xliff+xml", - "xlim": "application/vnd.xmpie.xlim", - "xlm": "application/vnd.ms-excel", - "xls": "application/vnd.ms-excel", - "xlsb": "application/vnd.ms-excel.sheet.binary.macroenabled.12", - "xlsm": "application/vnd.ms-excel.sheet.macroenabled.12", - "xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", - "xlt": "application/vnd.ms-excel", - "xltm": "application/vnd.ms-excel.template.macroenabled.12", - "xltx": "application/vnd.openxmlformats-officedocument.spreadsheetml.template", - "xlw": "application/vnd.ms-excel", - "xm": "audio/xm", - "xml": "text/xml", - "xmls": "application/dskpp+xml", - "xmt_bin": "model/vnd.parasolid.transmit.binary", - "xmt_txt": "model/vnd.parasolid.transmit.text", - "xmz": "xgl/movie", - "xns": "application/xcap-ns+xml", - "xo": "application/vnd.olpc-sugar", - "xof": "x-world/x-vrml", - "xop": "application/xop+xml", - "xpdl": "application/xml", "xpi": "application/x-xpinstall", - "xpix": "application/x-vnd.ls-xpix", - "xpl": "application/xproc+xml", - "xpm": "image/x-xpixmap", - "xpr": "application/vnd.is-xpr", - "xps": "application/vnd.ms-xpsdocument", - "xpw": "application/vnd.intercon.formnet", - "xpx": "application/vnd.intercon.formnet", - "xq": "text/xquery", - "xql": "text/xquery", - "xqm": "text/xquery", - "xqu": "text/xquery", - "xquery": "text/xquery", - "xqy": "text/xquery", - "xsd": "text/xml", - "xsf": "application/prs.xsf+xml", - "xsl": "application/xslt+xml", - "xslt": "application/xslt+xml", - "xsm": "application/vnd.syncml+xml", - "xspf": "application/xspf+xml", - "xsr": "video/x-amt-showrun", - "xtel": "chemical/x-xtel", - "xul": "application/vnd.mozilla.xul+xml", - "xvm": "application/xv+xml", - "xvml": "application/xv+xml", - "xwd": "image/x-xwindowdump", - "xyz": "chemical/x-xyz", - "xyze": "image/vnd.radiance", "xz": "application/x-xz", - "yaml": "text/yaml", - "yang": "application/yang", - "yin": "application/yin+xml", - "yme": "application/vnd.yaoweme", - "yml": "text/yaml", - "ymp": "text/x-suse-ymp", "z1": "application/x-zmachine", "z2": "application/x-zmachine", "z3": "application/x-zmachine", @@ -1884,26 +744,301 @@ const mimes* = { "z6": "application/x-zmachine", "z7": "application/x-zmachine", "z8": "application/x-zmachine", - "zaz": "application/vnd.zzazz.deck+xml", - "zfc": "application/vnd.filmit.zfc", - "zfo": "application/vnd.software602.filler.form-xml-zip", - "zig": "text/zig", + "xaml": "application/xaml+xml", + "xdf": "application/xcap-diff+xml", + "xenc": "application/xenc+xml", + "xhtml": "application/xhtml+xml", + "xht": "application/xhtml+xml", + "xml": "application/xml", + "xsl": "application/xml", + "dtd": "application/xml-dtd", + "xop": "application/xop+xml", + "xpl": "application/xproc+xml", + "xslt": "application/xslt+xml", + "xspf": "application/xspf+xml", + "mxml": "application/xv+xml", + "xhvml": "application/xv+xml", + "xvml": "application/xv+xml", + "xvm": "application/xv+xml", + "yang": "application/yang", + "yin": "application/yin+xml", "zip": "application/zip", - "zir": "application/vnd.zul", - "zirz": "application/vnd.zul", - "zmm": "application/vnd.handheld-entertainment+xml", - "zmt": "chemical/x-mopac-input", - "zone": "text/dns", - "zoo": "application/octet-stream", - "zsh": "text/x-script.zsh", - "~": "application/x-trash" + "adp": "audio/adpcm", + "au": "audio/basic", + "snd": "audio/basic", + "mid": "audio/midi", + "midi": "audio/midi", + "kar": "audio/midi", + "rmi": "audio/midi", + "m4a": "audio/mp4", + "mp4a": "audio/mp4", + "mpga": "audio/mpeg", + "mp2": "audio/mpeg", + "mp2a": "audio/mpeg", + "mp3": "audio/mpeg", + "m2a": "audio/mpeg", + "m3a": "audio/mpeg", + "oga": "audio/ogg", + "ogg": "audio/ogg", + "spx": "audio/ogg", + "opus": "audio/ogg", + "s3m": "audio/s3m", + "sil": "audio/silk", + "uva": "audio/vnd.dece.audio", + "uvva": "audio/vnd.dece.audio", + "eol": "audio/vnd.digital-winds", + "dra": "audio/vnd.dra", + "dts": "audio/vnd.dts", + "dtshd": "audio/vnd.dts.hd", + "lvp": "audio/vnd.lucent.voice", + "pya": "audio/vnd.ms-playready.media.pya", + "ecelp4800": "audio/vnd.nuera.ecelp4800", + "ecelp7470": "audio/vnd.nuera.ecelp7470", + "ecelp9600": "audio/vnd.nuera.ecelp9600", + "rip": "audio/vnd.rip", + "weba": "audio/webm", + "aac": "audio/x-aac", + "aif": "audio/x-aiff", + "aiff": "audio/x-aiff", + "aifc": "audio/x-aiff", + "caf": "audio/x-caf", + "flac": "audio/x-flac", + "mka": "audio/x-matroska", + "m3u": "audio/x-mpegurl", + "wax": "audio/x-ms-wax", + "wma": "audio/x-ms-wma", + "ram": "audio/x-pn-realaudio", + "ra": "audio/x-pn-realaudio", + "rmp": "audio/x-pn-realaudio-plugin", + "wav": "audio/x-wav", + "xm": "audio/xm", + "cdx": "chemical/x-cdx", + "cif": "chemical/x-cif", + "cmdf": "chemical/x-cmdf", + "cml": "chemical/x-cml", + "csml": "chemical/x-csml", + "xyz": "chemical/x-xyz", + "ttc": "font/collection", + "otf": "font/otf", + "ttf": "font/ttf", + "woff": "font/woff", + "woff2": "font/woff2", + "bmp": "image/bmp", + "cgm": "image/cgm", + "g3": "image/g3fax", + "gif": "image/gif", + "ief": "image/ief", + "jpeg": "image/jpeg", + "jpg": "image/jpeg", + "jpe": "image/jpeg", + "ktx": "image/ktx", + "png": "image/png", + "btif": "image/prs.btif", + "sgi": "image/sgi", + "svg": "image/svg+xml", + "svgz": "image/svg+xml", + "tiff": "image/tiff", + "tif": "image/tiff", + "psd": "image/vnd.adobe.photoshop", + "uvi": "image/vnd.dece.graphic", + "uvvi": "image/vnd.dece.graphic", + "uvg": "image/vnd.dece.graphic", + "uvvg": "image/vnd.dece.graphic", + "djvu": "image/vnd.djvu", + "djv": "image/vnd.djvu", + "sub": "image/vnd.dvb.subtitle", + "dwg": "image/vnd.dwg", + "dxf": "image/vnd.dxf", + "fbs": "image/vnd.fastbidsheet", + "fpx": "image/vnd.fpx", + "fst": "image/vnd.fst", + "mmr": "image/vnd.fujixerox.edmics-mmr", + "rlc": "image/vnd.fujixerox.edmics-rlc", + "mdi": "image/vnd.ms-modi", + "wdp": "image/vnd.ms-photo", + "npx": "image/vnd.net-fpx", + "wbmp": "image/vnd.wap.wbmp", + "xif": "image/vnd.xiff", + "webp": "image/webp", + "3ds": "image/x-3ds", + "ras": "image/x-cmu-raster", + "cmx": "image/x-cmx", + "fh": "image/x-freehand", + "fhc": "image/x-freehand", + "fh4": "image/x-freehand", + "fh5": "image/x-freehand", + "fh7": "image/x-freehand", + "ico": "image/x-icon", + "sid": "image/x-mrsid-image", + "pcx": "image/x-pcx", + "pic": "image/x-pict", + "pct": "image/x-pict", + "pnm": "image/x-portable-anymap", + "pbm": "image/x-portable-bitmap", + "pgm": "image/x-portable-graymap", + "ppm": "image/x-portable-pixmap", + "rgb": "image/x-rgb", + "tga": "image/x-tga", + "xbm": "image/x-xbitmap", + "xpm": "image/x-xpixmap", + "xwd": "image/x-xwindowdump", + "eml": "message/rfc822", + "mime": "message/rfc822", + "igs": "model/iges", + "iges": "model/iges", + "msh": "model/mesh", + "mesh": "model/mesh", + "silo": "model/mesh", + "dae": "model/vnd.collada+xml", + "dwf": "model/vnd.dwf", + "gdl": "model/vnd.gdl", + "gtw": "model/vnd.gtw", + "mts": "model/vnd.mts", + "vtu": "model/vnd.vtu", + "wrl": "model/vrml", + "vrml": "model/vrml", + "x3db": "model/x3d+binary", + "x3dbz": "model/x3d+binary", + "x3dv": "model/x3d+vrml", + "x3dvz": "model/x3d+vrml", + "x3d": "model/x3d+xml", + "x3dz": "model/x3d+xml", + "appcache": "text/cache-manifest", + "ics": "text/calendar", + "ifb": "text/calendar", + "css": "text/css", + "csv": "text/csv", + "html": "text/html", + "htm": "text/html", + "js": "text/javascript", + "mjs": "text/javascript", + "n3": "text/n3", + "txt": "text/plain", + "text": "text/plain", + "conf": "text/plain", + "def": "text/plain", + "list": "text/plain", + "log": "text/plain", + "in": "text/plain", + "dsc": "text/prs.lines.tag", + "rtx": "text/richtext", + "sgml": "text/sgml", + "sgm": "text/sgml", + "tsv": "text/tab-separated-values", + "t": "text/troff", + "tr": "text/troff", + "roff": "text/troff", + "man": "text/troff", + "me": "text/troff", + "ms": "text/troff", + "ttl": "text/turtle", + "uri": "text/uri-list", + "uris": "text/uri-list", + "urls": "text/uri-list", + "vcard": "text/vcard", + "curl": "text/vnd.curl", + "dcurl": "text/vnd.curl.dcurl", + "mcurl": "text/vnd.curl.mcurl", + "scurl": "text/vnd.curl.scurl", + "sub": "text/vnd.dvb.subtitle", + "fly": "text/vnd.fly", + "flx": "text/vnd.fmi.flexstor", + "gv": "text/vnd.graphviz", + "3dml": "text/vnd.in3d.3dml", + "spot": "text/vnd.in3d.spot", + "jad": "text/vnd.sun.j2me.app-descriptor", + "wml": "text/vnd.wap.wml", + "wmls": "text/vnd.wap.wmlscript", + "s": "text/x-asm", + "asm": "text/x-asm", + "c": "text/x-c", + "cc": "text/x-c", + "cxx": "text/x-c", + "cpp": "text/x-c", + "h": "text/x-c", + "hh": "text/x-c", + "dic": "text/x-c", + "f": "text/x-fortran", + "for": "text/x-fortran", + "f77": "text/x-fortran", + "f90": "text/x-fortran", + "java": "text/x-java-source", + "nfo": "text/x-nfo", + "opml": "text/x-opml", + "p": "text/x-pascal", + "pas": "text/x-pascal", + "etx": "text/x-setext", + "sfv": "text/x-sfv", + "uu": "text/x-uuencode", + "vcs": "text/x-vcalendar", + "vcf": "text/x-vcard", + "3gp": "video/3gpp", + "3g2": "video/3gpp2", + "h261": "video/h261", + "h263": "video/h263", + "h264": "video/h264", + "jpgv": "video/jpeg", + "jpm": "video/jpm", + "jpgm": "video/jpm", + "mj2": "video/mj2", + "mjp2": "video/mj2", + "mp4": "video/mp4", + "mp4v": "video/mp4", + "mpg4": "video/mp4", + "mpeg": "video/mpeg", + "mpg": "video/mpeg", + "mpe": "video/mpeg", + "m1v": "video/mpeg", + "m2v": "video/mpeg", + "ogv": "video/ogg", + "qt": "video/quicktime", + "mov": "video/quicktime", + "uvh": "video/vnd.dece.hd", + "uvvh": "video/vnd.dece.hd", + "uvm": "video/vnd.dece.mobile", + "uvvm": "video/vnd.dece.mobile", + "uvp": "video/vnd.dece.pd", + "uvvp": "video/vnd.dece.pd", + "uvs": "video/vnd.dece.sd", + "uvvs": "video/vnd.dece.sd", + "uvv": "video/vnd.dece.video", + "uvvv": "video/vnd.dece.video", + "dvb": "video/vnd.dvb.file", + "fvt": "video/vnd.fvt", + "mxu": "video/vnd.mpegurl", + "m4u": "video/vnd.mpegurl", + "pyv": "video/vnd.ms-playready.media.pyv", + "uvu": "video/vnd.uvvu.mp4", + "uvvu": "video/vnd.uvvu.mp4", + "viv": "video/vnd.vivo", + "webm": "video/webm", + "f4v": "video/x-f4v", + "fli": "video/x-fli", + "flv": "video/x-flv", + "m4v": "video/x-m4v", + "mkv": "video/x-matroska", + "mk3d": "video/x-matroska", + "mks": "video/x-matroska", + "mng": "video/x-mng", + "asf": "video/x-ms-asf", + "asx": "video/x-ms-asf", + "vob": "video/x-ms-vob", + "wm": "video/x-ms-wm", + "wmv": "video/x-ms-wmv", + "wmx": "video/x-ms-wmx", + "wvx": "video/x-ms-wvx", + "avi": "video/x-msvideo", + "movie": "video/x-sgi-movie", + "smv": "video/x-smv", + "ice": "x-conference/x-cooltalk", } func newMimetypes*(): MimeDB = ## Creates a new Mimetypes database. The database will contain the most ## common mimetypes. - result.mimes = mimes.newStringTable() + {.cast(noSideEffect).}: + result.mimes = mimes.toOrderedTable() func getMimetype*(mimedb: MimeDB, ext: string, default = "text/plain"): string = ## Gets mimetype which corresponds to `ext`. Returns `default` if `ext` diff --git a/lib/pure/nativesockets.nim b/lib/pure/nativesockets.nim index 605f62321..656c98a20 100644 --- a/lib/pure/nativesockets.nim +++ b/lib/pure/nativesockets.nim @@ -12,31 +12,39 @@ # TODO: Clean up the exports a bit and everything else in general. -import os, options +import std/[os, options] import std/private/since +import std/strbasics +when defined(nimPreviewSlimSystem): + import std/[assertions, syncio] when hostOS == "solaris": {.passl: "-lsocket -lnsl".} const useWinVersion = defined(windows) or defined(nimdoc) +const useNimNetLite = defined(nimNetLite) or defined(freertos) or defined(zephyr) or + defined(nuttx) when useWinVersion: - import winlean + import std/winlean export WSAEWOULDBLOCK, WSAECONNRESET, WSAECONNABORTED, WSAENETRESET, WSANOTINITIALISED, WSAENOTSOCK, WSAEINPROGRESS, WSAEINTR, WSAEDISCON, ERROR_NETNAME_DELETED else: - import posix + import std/posix export fcntl, F_GETFL, O_NONBLOCK, F_SETFL, EAGAIN, EWOULDBLOCK, MSG_NOSIGNAL, EINTR, EINPROGRESS, ECONNRESET, EPIPE, ENETRESET, EBADF export Sockaddr_storage, Sockaddr_un, Sockaddr_un_path_length export SocketHandle, Sockaddr_in, Addrinfo, INADDR_ANY, SockAddr, SockLen, Sockaddr_in6, Sockaddr_storage, - inet_ntoa, recv, `==`, connect, send, accept, recvfrom, sendto, + recv, `==`, connect, send, accept, recvfrom, sendto, freeAddrInfo +when not useNimNetLite: + export inet_ntoa + export SO_ERROR, SOL_SOCKET, @@ -59,7 +67,7 @@ type ## some procedures, such as getaddrinfo) AF_UNIX = 1, ## for local socket (using a file). Unsupported on Windows. AF_INET = 2, ## for network protocol IPv4 or - AF_INET6 = when defined(macosx): 30 else: 23 ## for network protocol IPv6. + AF_INET6 = when defined(macosx): 30 elif defined(windows): 23 else: 10 ## for network protocol IPv6. SockType* = enum ## second argument to `socket` proc SOCK_STREAM = 1, ## reliable stream-oriented service or Stream Sockets @@ -89,6 +97,8 @@ type length*: int addrList*: seq[string] +const IPPROTO_NONE* = IPPROTO_IP ## Use this if your socket type requires a protocol value of zero (e.g. Unix sockets). + when useWinVersion: let osInvalidSocket* = winlean.INVALID_SOCKET @@ -162,7 +172,7 @@ when not useWinVersion: else: proc toInt(domain: Domain): cint = - result = toU32(ord(domain)).cint + result = cast[cint](uint32(ord(domain))) proc toKnownDomain*(family: cint): Option[Domain] = ## Converts the platform-dependent `cint` to the Domain or none(), @@ -209,7 +219,7 @@ proc getProtoByName*(name: string): int {.since: (1, 3, 5).} = let protoent = posix.getprotobyname(name.cstring) if protoent == nil: - raise newException(OSError, "protocol not found") + raise newException(OSError, "protocol not found: " & name) result = protoent.p_proto.int @@ -295,9 +305,9 @@ proc getAddrInfo*(address: string, port: Port, domain: Domain = AF_INET, if domain == AF_INET6: hints.ai_flags = AI_V4MAPPED let socketPort = if sockType == SOCK_RAW: "" else: $port - var gaiResult = getaddrinfo(address, socketPort, addr(hints), result) + var gaiResult = getaddrinfo(address, socketPort.cstring, addr(hints), result) if gaiResult != 0'i32: - when useWinVersion or defined(freertos): + when useWinVersion or defined(freertos) or defined(nuttx): raiseOSError(osLastError()) else: raiseOSError(osLastError(), $gai_strerror(gaiResult)) @@ -331,125 +341,6 @@ template htons*(x: uint16): untyped = ## order, this is a no-op; otherwise, it performs a 2-byte swap operation. nativesockets.ntohs(x) -proc getServByName*(name, proto: string): Servent {.tags: [ReadIOEffect].} = - ## Searches the database from the beginning and finds the first entry for - ## which the service name specified by `name` matches the s_name member - ## and the protocol name specified by `proto` matches the s_proto member. - ## - ## On posix this will search through the `/etc/services` file. - when useWinVersion: - var s = winlean.getservbyname(name, proto) - else: - var s = posix.getservbyname(name, proto) - if s == nil: raiseOSError(osLastError(), "Service not found.") - result.name = $s.s_name - result.aliases = cstringArrayToSeq(s.s_aliases) - result.port = Port(s.s_port) - result.proto = $s.s_proto - -proc getServByPort*(port: Port, proto: string): Servent {.tags: [ReadIOEffect].} = - ## Searches the database from the beginning and finds the first entry for - ## which the port specified by `port` matches the s_port member and the - ## protocol name specified by `proto` matches the s_proto member. - ## - ## On posix this will search through the `/etc/services` file. - when useWinVersion: - var s = winlean.getservbyport(ze(int16(port)).cint, proto) - else: - var s = posix.getservbyport(ze(int16(port)).cint, proto) - if s == nil: raiseOSError(osLastError(), "Service not found.") - result.name = $s.s_name - result.aliases = cstringArrayToSeq(s.s_aliases) - result.port = Port(s.s_port) - result.proto = $s.s_proto - -proc getHostByAddr*(ip: string): Hostent {.tags: [ReadIOEffect].} = - ## This function will lookup the hostname of an IP Address. - var myaddr: InAddr - myaddr.s_addr = inet_addr(ip) - - when useWinVersion: - var s = winlean.gethostbyaddr(addr(myaddr), sizeof(myaddr).cuint, - cint(AF_INET)) - if s == nil: raiseOSError(osLastError()) - else: - var s = - when defined(android4): - posix.gethostbyaddr(cast[cstring](addr(myaddr)), sizeof(myaddr).cint, - cint(posix.AF_INET)) - else: - posix.gethostbyaddr(addr(myaddr), sizeof(myaddr).SockLen, - cint(posix.AF_INET)) - if s == nil: - raiseOSError(osLastError(), $hstrerror(h_errno)) - - result.name = $s.h_name - result.aliases = cstringArrayToSeq(s.h_aliases) - when useWinVersion: - result.addrtype = Domain(s.h_addrtype) - else: - if s.h_addrtype == posix.AF_INET: - result.addrtype = AF_INET - elif s.h_addrtype == posix.AF_INET6: - result.addrtype = AF_INET6 - else: - raiseOSError(osLastError(), "unknown h_addrtype") - if result.addrtype == AF_INET: - result.addrList = @[] - var i = 0 - while not isNil(s.h_addr_list[i]): - var inaddrPtr = cast[ptr InAddr](s.h_addr_list[i]) - result.addrList.add($inet_ntoa(inaddrPtr[])) - inc(i) - else: - result.addrList = cstringArrayToSeq(s.h_addr_list) - result.length = int(s.h_length) - -proc getHostByName*(name: string): Hostent {.tags: [ReadIOEffect].} = - ## This function will lookup the IP address of a hostname. - when useWinVersion: - var s = winlean.gethostbyname(name) - else: - var s = posix.gethostbyname(name) - if s == nil: raiseOSError(osLastError()) - result.name = $s.h_name - result.aliases = cstringArrayToSeq(s.h_aliases) - when useWinVersion: - result.addrtype = Domain(s.h_addrtype) - else: - if s.h_addrtype == posix.AF_INET: - result.addrtype = AF_INET - elif s.h_addrtype == posix.AF_INET6: - result.addrtype = AF_INET6 - else: - raiseOSError(osLastError(), "unknown h_addrtype") - if result.addrtype == AF_INET: - result.addrList = @[] - var i = 0 - while not isNil(s.h_addr_list[i]): - var inaddrPtr = cast[ptr InAddr](s.h_addr_list[i]) - result.addrList.add($inet_ntoa(inaddrPtr[])) - inc(i) - else: - result.addrList = cstringArrayToSeq(s.h_addr_list) - result.length = int(s.h_length) - -proc getHostname*(): string {.tags: [ReadIOEffect].} = - ## Returns the local hostname (not the FQDN) - # https://tools.ietf.org/html/rfc1035#section-2.3.1 - # https://tools.ietf.org/html/rfc2181#section-11 - const size = 64 - result = newString(size) - when useWinVersion: - let success = winlean.gethostname(result, size) - else: - # Posix - let success = posix.gethostname(result, size) - if success != 0.cint: - raiseOSError(osLastError()) - let x = len(cstring(result)) - result.setLen(x) - proc getSockDomain*(socket: SocketHandle): Domain = ## Returns the socket's domain (AF_INET or AF_INET6). var name: Sockaddr_in6 @@ -463,161 +354,383 @@ proc getSockDomain*(socket: SocketHandle): Domain = else: raise newException(IOError, "Unknown socket family in getSockDomain") -proc getAddrString*(sockAddr: ptr SockAddr): string = - ## Returns the string representation of address within sockAddr - if sockAddr.sa_family.cint == nativeAfInet: - result = $inet_ntoa(cast[ptr Sockaddr_in](sockAddr).sin_addr) - elif sockAddr.sa_family.cint == nativeAfInet6: - let addrLen = when not useWinVersion: posix.INET6_ADDRSTRLEN - else: 46 # it's actually 46 in both cases - result = newString(addrLen) - let addr6 = addr cast[ptr Sockaddr_in6](sockAddr).sin6_addr - when not useWinVersion: - if posix.inet_ntop(posix.AF_INET6, addr6, addr result[0], - result.len.int32) == nil: - raiseOSError(osLastError()) - if posix.IN6_IS_ADDR_V4MAPPED(addr6) != 0: - result = result.substr("::ffff:".len) +when not useNimNetLite: + proc getServByName*(name, proto: string): Servent {.tags: [ReadIOEffect].} = + ## Searches the database from the beginning and finds the first entry for + ## which the service name specified by `name` matches the s_name member + ## and the protocol name specified by `proto` matches the s_proto member. + ## + ## On posix this will search through the `/etc/services` file. + when useWinVersion: + var s = winlean.getservbyname(name, proto) else: - if winlean.inet_ntop(winlean.AF_INET6, addr6, addr result[0], - result.len.int32) == nil: - raiseOSError(osLastError()) - setLen(result, len(cstring(result))) - else: - when defined(posix) and not defined(nimdoc): - if sockAddr.sa_family.cint == nativeAfUnix: - return "unix" - raise newException(IOError, "Unknown socket family in getAddrString") - -proc getAddrString*(sockAddr: ptr SockAddr, strAddress: var string) = - ## Stores in `strAddress` the string representation of the address inside - ## `sockAddr` - ## - ## **Note** - ## * `strAddress` must be initialized to 46 in length. - assert(46 == len(strAddress), - "`strAddress` was not initialized correctly. 46 != `len(strAddress)`") - if sockAddr.sa_family.cint == nativeAfInet: - let addr4 = addr cast[ptr Sockaddr_in](sockAddr).sin_addr - when not useWinVersion: - if posix.inet_ntop(posix.AF_INET, addr4, addr strAddress[0], - strAddress.len.int32) == nil: - raiseOSError(osLastError()) + var s = posix.getservbyname(name, proto) + if s == nil: raiseOSError(osLastError(), "Service not found.") + result.name = $s.s_name + result.aliases = cstringArrayToSeq(s.s_aliases) + result.port = Port(s.s_port) + result.proto = $s.s_proto + + proc getServByPort*(port: Port, proto: string): Servent {.tags: [ReadIOEffect].} = + ## Searches the database from the beginning and finds the first entry for + ## which the port specified by `port` matches the s_port member and the + ## protocol name specified by `proto` matches the s_proto member. + ## + ## On posix this will search through the `/etc/services` file. + when useWinVersion: + var s = winlean.getservbyport(uint16(port).cint, proto) else: - if winlean.inet_ntop(winlean.AF_INET, addr4, addr strAddress[0], - strAddress.len.int32) == nil: - raiseOSError(osLastError()) - elif sockAddr.sa_family.cint == nativeAfInet6: - let addr6 = addr cast[ptr Sockaddr_in6](sockAddr).sin6_addr - when not useWinVersion: - if posix.inet_ntop(posix.AF_INET6, addr6, addr strAddress[0], - strAddress.len.int32) == nil: - raiseOSError(osLastError()) - if posix.IN6_IS_ADDR_V4MAPPED(addr6) != 0: - strAddress = strAddress.substr("::ffff:".len) + var s = posix.getservbyport(uint16(port).cint, proto) + if s == nil: raiseOSError(osLastError(), "Service not found.") + result.name = $s.s_name + result.aliases = cstringArrayToSeq(s.s_aliases) + result.port = Port(s.s_port) + result.proto = $s.s_proto + + proc getHostByAddr*(ip: string): Hostent {.tags: [ReadIOEffect].} = + ## This function will lookup the hostname of an IP Address. + var + addrInfo = getAddrInfo(ip, Port(0), AF_UNSPEC) + myAddr: pointer + addrLen = 0 + family = 0 + + defer: freeAddrInfo(addrInfo) + + if addrInfo.ai_addr.sa_family.cint == nativeAfInet: + family = nativeAfInet + myAddr = addr cast[ptr Sockaddr_in](addrInfo.ai_addr).sin_addr + addrLen = 4 + elif addrInfo.ai_addr.sa_family.cint == nativeAfInet6: + family = nativeAfInet6 + myAddr = addr cast[ptr Sockaddr_in6](addrInfo.ai_addr).sin6_addr + addrLen = 16 else: - if winlean.inet_ntop(winlean.AF_INET6, addr6, addr strAddress[0], - strAddress.len.int32) == nil: - raiseOSError(osLastError()) - else: - raise newException(IOError, "Unknown socket family in getAddrString") - setLen(strAddress, len(cstring(strAddress))) - -when defined(posix) and not defined(nimdoc): - proc makeUnixAddr*(path: string): Sockaddr_un = - result.sun_family = AF_UNIX.TSa_Family - if path.len >= Sockaddr_un_path_length: - raise newException(ValueError, "socket path too long") - copyMem(addr result.sun_path, path.cstring, path.len + 1) - -proc getSockName*(socket: SocketHandle): Port = - ## Returns the socket's associated port number. - var name: Sockaddr_in - when useWinVersion: - name.sin_family = uint16(ord(AF_INET)) - else: - name.sin_family = TSa_Family(posix.AF_INET) - #name.sin_port = htons(cint16(port)) - #name.sin_addr.s_addr = htonl(INADDR_ANY) - var namelen = sizeof(name).SockLen - if getsockname(socket, cast[ptr SockAddr](addr(name)), - addr(namelen)) == -1'i32: - raiseOSError(osLastError()) - result = Port(nativesockets.ntohs(name.sin_port)) + raise newException(IOError, "Unknown socket family in `getHostByAddr()`") -proc getLocalAddr*(socket: SocketHandle, domain: Domain): (string, Port) = - ## Returns the socket's local address and port number. - ## - ## Similar to POSIX's `getsockname`:idx:. - case domain - of AF_INET: - var name: Sockaddr_in when useWinVersion: - name.sin_family = uint16(ord(AF_INET)) + var s = winlean.gethostbyaddr(cast[ptr InAddr](myAddr), addrLen.cuint, + cint(family)) + if s == nil: raiseOSError(osLastError()) else: - name.sin_family = TSa_Family(posix.AF_INET) - var namelen = sizeof(name).SockLen - if getsockname(socket, cast[ptr SockAddr](addr(name)), - addr(namelen)) == -1'i32: - raiseOSError(osLastError()) - result = ($inet_ntoa(name.sin_addr), - Port(nativesockets.ntohs(name.sin_port))) - of AF_INET6: - var name: Sockaddr_in6 + var s = + when defined(android4): + posix.gethostbyaddr(cast[cstring](myAddr), addrLen.cint, + cint(family)) + else: + posix.gethostbyaddr(myAddr, addrLen.SockLen, + cint(family)) + if s == nil: + raiseOSError(osLastError(), $hstrerror(h_errno)) + + result.name = $s.h_name + result.aliases = cstringArrayToSeq(s.h_aliases) when useWinVersion: - name.sin6_family = uint16(ord(AF_INET6)) + result.addrtype = Domain(s.h_addrtype) else: - name.sin6_family = TSa_Family(posix.AF_INET6) - var namelen = sizeof(name).SockLen - if getsockname(socket, cast[ptr SockAddr](addr(name)), - addr(namelen)) == -1'i32: - raiseOSError(osLastError()) - # Cannot use INET6_ADDRSTRLEN here, because it's a C define. - result[0] = newString(64) - if inet_ntop(name.sin6_family.cint, - addr name.sin6_addr, addr result[0][0], (result[0].len+1).int32).isNil: + if s.h_addrtype == posix.AF_INET: + result.addrtype = AF_INET + elif s.h_addrtype == posix.AF_INET6: + result.addrtype = AF_INET6 + else: + raiseOSError(osLastError(), "unknown h_addrtype") + if result.addrtype == AF_INET: + result.addrList = @[] + var i = 0 + while not isNil(s.h_addr_list[i]): + var inaddrPtr = cast[ptr InAddr](s.h_addr_list[i]) + result.addrList.add($inet_ntoa(inaddrPtr[])) + inc(i) + else: + let strAddrLen = when not useWinVersion: posix.INET6_ADDRSTRLEN.int + else: 46 + var i = 0 + while not isNil(s.h_addr_list[i]): + var ipStr = newString(strAddrLen) + if inet_ntop(nativeAfInet6, cast[pointer](s.h_addr_list[i]), + cstring(ipStr), len(ipStr).int32) == nil: + raiseOSError(osLastError()) + when not useWinVersion: + if posix.IN6_IS_ADDR_V4MAPPED(cast[ptr In6Addr](s.h_addr_list[i])) != 0: + ipStr.setSlice("::ffff:".len..<strAddrLen) + setLen(ipStr, len(cstring(ipStr))) + result.addrList.add(ipStr) + inc(i) + result.length = int(s.h_length) + + proc getHostByName*(name: string): Hostent {.tags: [ReadIOEffect].} = + ## This function will lookup the IP address of a hostname. + when useWinVersion: + var s = winlean.gethostbyname(name) + else: + var s = posix.gethostbyname(name) + if s == nil: raiseOSError(osLastError()) + result.name = $s.h_name + result.aliases = cstringArrayToSeq(s.h_aliases) + when useWinVersion: + result.addrtype = Domain(s.h_addrtype) + else: + if s.h_addrtype == posix.AF_INET: + result.addrtype = AF_INET + elif s.h_addrtype == posix.AF_INET6: + result.addrtype = AF_INET6 + else: + raiseOSError(osLastError(), "unknown h_addrtype") + if result.addrtype == AF_INET: + result.addrList = @[] + var i = 0 + while not isNil(s.h_addr_list[i]): + var inaddrPtr = cast[ptr InAddr](s.h_addr_list[i]) + result.addrList.add($inet_ntoa(inaddrPtr[])) + inc(i) + else: + result.addrList = cstringArrayToSeq(s.h_addr_list) + result.length = int(s.h_length) + + proc getHostname*(): string {.tags: [ReadIOEffect].} = + ## Returns the local hostname (not the FQDN) + # https://tools.ietf.org/html/rfc1035#section-2.3.1 + # https://tools.ietf.org/html/rfc2181#section-11 + const size = 256 + result = newString(size) + when useWinVersion: + let success = winlean.gethostname(result.cstring, size) + else: + # Posix + let success = posix.gethostname(result.cstring, size) + if success != 0.cint: raiseOSError(osLastError()) - setLen(result[0], result[0].cstring.len) - result[1] = Port(nativesockets.ntohs(name.sin6_port)) - else: - raiseOSError(OSErrorCode(-1), "invalid socket family in getLocalAddr") - -proc getPeerAddr*(socket: SocketHandle, domain: Domain): (string, Port) = - ## Returns the socket's peer address and port number. - ## - ## Similar to POSIX's `getpeername`:idx: - case domain - of AF_INET: + let x = len(cstring(result)) + result.setLen(x) + + proc getAddrString*(sockAddr: ptr SockAddr): string = + ## Returns the string representation of address within sockAddr + if sockAddr.sa_family.cint == nativeAfInet: + result = $inet_ntoa(cast[ptr Sockaddr_in](sockAddr).sin_addr) + elif sockAddr.sa_family.cint == nativeAfInet6: + let addrLen = when not useWinVersion: posix.INET6_ADDRSTRLEN.int + else: 46 # it's actually 46 in both cases + result = newString(addrLen) + let addr6 = addr cast[ptr Sockaddr_in6](sockAddr).sin6_addr + when not useWinVersion: + if posix.inet_ntop(posix.AF_INET6, addr6, cast[cstring](addr result[0]), + result.len.int32) == nil: + raiseOSError(osLastError()) + if posix.IN6_IS_ADDR_V4MAPPED(addr6) != 0: + result.setSlice("::ffff:".len..<addrLen) + else: + if winlean.inet_ntop(winlean.AF_INET6, addr6, cast[cstring](addr result[0]), + result.len.int32) == nil: + raiseOSError(osLastError()) + setLen(result, len(cstring(result))) + else: + when defined(posix) and not defined(nimdoc): + if sockAddr.sa_family.cint == nativeAfUnix: + return "unix" + raise newException(IOError, "Unknown socket family in getAddrString") + + proc getAddrString*(sockAddr: ptr SockAddr, strAddress: var string) = + ## Stores in `strAddress` the string representation of the address inside + ## `sockAddr` + ## + ## **Note** + ## * `strAddress` must be initialized to 46 in length. + const length = 46 + assert(length == len(strAddress), + "`strAddress` was not initialized correctly. 46 != `len(strAddress)`") + if sockAddr.sa_family.cint == nativeAfInet: + let addr4 = addr cast[ptr Sockaddr_in](sockAddr).sin_addr + when not useWinVersion: + if posix.inet_ntop(posix.AF_INET, addr4, cast[cstring](addr strAddress[0]), + strAddress.len.int32) == nil: + raiseOSError(osLastError()) + else: + if winlean.inet_ntop(winlean.AF_INET, addr4, cast[cstring](addr strAddress[0]), + strAddress.len.int32) == nil: + raiseOSError(osLastError()) + elif sockAddr.sa_family.cint == nativeAfInet6: + let addr6 = addr cast[ptr Sockaddr_in6](sockAddr).sin6_addr + when not useWinVersion: + if posix.inet_ntop(posix.AF_INET6, addr6, cast[cstring](addr strAddress[0]), + strAddress.len.int32) == nil: + raiseOSError(osLastError()) + if posix.IN6_IS_ADDR_V4MAPPED(addr6) != 0: + strAddress.setSlice("::ffff:".len..<length) + else: + if winlean.inet_ntop(winlean.AF_INET6, addr6, cast[cstring](addr strAddress[0]), + strAddress.len.int32) == nil: + raiseOSError(osLastError()) + else: + raise newException(IOError, "Unknown socket family in getAddrString") + setLen(strAddress, len(cstring(strAddress))) + + when defined(posix) and not defined(nimdoc): + proc makeUnixAddr*(path: string): Sockaddr_un = + result.sun_family = AF_UNIX.TSa_Family + if path.len >= Sockaddr_un_path_length: + raise newException(ValueError, "socket path too long") + copyMem(addr result.sun_path, path.cstring, path.len + 1) + + proc getSockName*(socket: SocketHandle): Port = + ## Returns the socket's associated port number. var name: Sockaddr_in when useWinVersion: name.sin_family = uint16(ord(AF_INET)) else: name.sin_family = TSa_Family(posix.AF_INET) + #name.sin_port = htons(cint16(port)) + #name.sin_addr.s_addr = htonl(INADDR_ANY) var namelen = sizeof(name).SockLen - if getpeername(socket, cast[ptr SockAddr](addr(name)), - addr(namelen)) == -1'i32: + if getsockname(socket, cast[ptr SockAddr](addr(name)), + addr(namelen)) == -1'i32: raiseOSError(osLastError()) - result = ($inet_ntoa(name.sin_addr), - Port(nativesockets.ntohs(name.sin_port))) - of AF_INET6: - var name: Sockaddr_in6 - when useWinVersion: - name.sin6_family = uint16(ord(AF_INET6)) + result = Port(nativesockets.ntohs(name.sin_port)) + + proc getLocalAddr*(socket: SocketHandle, domain: Domain): (string, Port) = + ## Returns the socket's local address and port number. + ## + ## Similar to POSIX's `getsockname`:idx:. + case domain + of AF_INET: + var name: Sockaddr_in + when useWinVersion: + name.sin_family = uint16(ord(AF_INET)) + else: + name.sin_family = TSa_Family(posix.AF_INET) + var namelen = sizeof(name).SockLen + if getsockname(socket, cast[ptr SockAddr](addr(name)), + addr(namelen)) == -1'i32: + raiseOSError(osLastError()) + result = ($inet_ntoa(name.sin_addr), + Port(nativesockets.ntohs(name.sin_port))) + of AF_INET6: + var name: Sockaddr_in6 + when useWinVersion: + name.sin6_family = uint16(ord(AF_INET6)) + else: + name.sin6_family = TSa_Family(posix.AF_INET6) + var namelen = sizeof(name).SockLen + if getsockname(socket, cast[ptr SockAddr](addr(name)), + addr(namelen)) == -1'i32: + raiseOSError(osLastError()) + # Cannot use INET6_ADDRSTRLEN here, because it's a C define. + result[0] = newString(64) + if inet_ntop(name.sin6_family.cint, + addr name.sin6_addr, cast[cstring](addr result[0][0]), (result[0].len+1).int32).isNil: + raiseOSError(osLastError()) + setLen(result[0], result[0].cstring.len) + result[1] = Port(nativesockets.ntohs(name.sin6_port)) else: - name.sin6_family = TSa_Family(posix.AF_INET6) - var namelen = sizeof(name).SockLen - if getpeername(socket, cast[ptr SockAddr](addr(name)), - addr(namelen)) == -1'i32: - raiseOSError(osLastError()) - # Cannot use INET6_ADDRSTRLEN here, because it's a C define. - result[0] = newString(64) - if inet_ntop(name.sin6_family.cint, - addr name.sin6_addr, addr result[0][0], (result[0].len+1).int32).isNil: + raiseOSError(OSErrorCode(-1), "invalid socket family in getLocalAddr") + + proc getPeerAddr*(socket: SocketHandle, domain: Domain): (string, Port) = + ## Returns the socket's peer address and port number. + ## + ## Similar to POSIX's `getpeername`:idx: + case domain + of AF_INET: + var name: Sockaddr_in + when useWinVersion: + name.sin_family = uint16(ord(AF_INET)) + else: + name.sin_family = TSa_Family(posix.AF_INET) + var namelen = sizeof(name).SockLen + if getpeername(socket, cast[ptr SockAddr](addr(name)), + addr(namelen)) == -1'i32: + raiseOSError(osLastError()) + result = ($inet_ntoa(name.sin_addr), + Port(nativesockets.ntohs(name.sin_port))) + of AF_INET6: + var name: Sockaddr_in6 + when useWinVersion: + name.sin6_family = uint16(ord(AF_INET6)) + else: + name.sin6_family = TSa_Family(posix.AF_INET6) + var namelen = sizeof(name).SockLen + if getpeername(socket, cast[ptr SockAddr](addr(name)), + addr(namelen)) == -1'i32: + raiseOSError(osLastError()) + # Cannot use INET6_ADDRSTRLEN here, because it's a C define. + result[0] = newString(64) + if inet_ntop(name.sin6_family.cint, + addr name.sin6_addr, cast[cstring](addr result[0][0]), (result[0].len+1).int32).isNil: + raiseOSError(osLastError()) + setLen(result[0], result[0].cstring.len) + result[1] = Port(nativesockets.ntohs(name.sin6_port)) + else: + raiseOSError(OSErrorCode(-1), "invalid socket family in getLocalAddr") + +when useNimNetLite: + + when useWinVersion: + const + INET_ADDRSTRLEN = 16 + INET6_ADDRSTRLEN = 46 # it's actually 46 in both cases + + proc sockAddrToStr(sa: ptr SockAddr): string {.noinit.} = + let af_family = sa.sa_family + var nl, v4Slice: cint + var si_addr: ptr InAddr + + if af_family == AF_INET.TSa_Family: + nl = INET_ADDRSTRLEN + si_addr = cast[ptr Sockaddr_in](sa).sin_addr.addr() + elif af_family == AF_INET6.TSa_Family: + nl = INET6_ADDRSTRLEN + let si6_addr = cast[ptr Sockaddr_in6](sa).sin6_addr.addr() + si_addr = cast[ptr InAddr](si6_addr) # let's us reuse logic below + when defined(posix) and not defined(nimdoc) and not defined(zephyr): + if posix.IN6_IS_ADDR_V4MAPPED(si6_addr) != 0: + v4Slice = "::ffff:".len() + else: + when defined(posix) and not defined(nimdoc): + if af_family.cint == nativeAfUnix: + return "unix" + return "" + + result = newString(nl) + let namePtr = result.cstring() + if namePtr == inet_ntop(af_family.cint, si_addr, namePtr, nl): + result.setLen(len(namePtr)) + if v4Slice > 0: result.setSlice(v4Slice.int ..< nl.int) + else: + return "" + + proc sockAddrToStr(sa: var Sockaddr_in | var Sockaddr_in6): string = + result = sockAddrToStr(cast[ptr SockAddr](unsafeAddr(sa))) + + proc getAddrString*(sockAddr: ptr SockAddr): string = + result = sockAddrToStr(sockAddr) + if result.len() == 0: raiseOSError(osLastError()) - setLen(result[0], result[0].cstring.len) - result[1] = Port(nativesockets.ntohs(name.sin6_port)) - else: - raiseOSError(OSErrorCode(-1), "invalid socket family in getLocalAddr") + + proc getAddrString*(sockAddr: ptr SockAddr, strAddress: var string) {.noinit.} = + strAddress = getAddrString(sockAddr) + + proc getLocalAddr*(socket: SocketHandle, domain: Domain): (string, Port) = + ## Returns the socket's local address and port number. + ## + ## Similar to POSIX's `getsockname`:idx:. + template sockGetNameOrRaiseError(socket: untyped, name: untyped) = + var namelen = sizeof(socket).SockLen + if getsockname(socket, cast[ptr SockAddr](addr(name)), + addr(namelen)) == -1'i32: + raiseOSError(osLastError()) + + case domain + of AF_INET: + var name = Sockaddr_in(sin_family: TSa_Family(posix.AF_INET)) + sockGetNameOrRaiseError(socket, name) + result = (sockAddrToStr(name), + Port(nativesockets.ntohs(name.sin_port))) + of AF_INET6: + var name = Sockaddr_in6(sin6_family: TSa_Family(posix.AF_INET6)) + sockGetNameOrRaiseError(socket, name) + result = (sockAddrToStr(name), + Port(nativesockets.ntohs(name.sin6_port))) + else: + raiseOSError(OSErrorCode(-1), "invalid socket family in getLocalAddr") + proc getSockOptInt*(socket: SocketHandle, level, optname: int): int {. tags: [ReadIOEffect].} = @@ -731,14 +844,14 @@ proc accept*(fd: SocketHandle, inheritable = defined(nimInheritHandles)): (Socke ## child processes. ## ## Returns (osInvalidSocket, "") if an error occurred. - var sockAddress: Sockaddr_in + var sockAddress: SockAddr var addrLen = sizeof(sockAddress).SockLen var sock = when (defined(linux) or defined(bsd)) and not defined(nimdoc): - accept4(fd, cast[ptr SockAddr](addr(sockAddress)), addr(addrLen), + accept4(fd, addr(sockAddress), addr(addrLen), if inheritable: 0 else: SOCK_CLOEXEC) else: - accept(fd, cast[ptr SockAddr](addr(sockAddress)), addr(addrLen)) + accept(fd, addr(sockAddress), addr(addrLen)) when declared(setInheritable) and not (defined(linux) or defined(bsd)): if not setInheritable(sock, inheritable): close sock @@ -746,7 +859,11 @@ proc accept*(fd: SocketHandle, inheritable = defined(nimInheritHandles)): (Socke if sock == osInvalidSocket: return (osInvalidSocket, "") else: - return (sock, $inet_ntoa(sockAddress.sin_addr)) + when useNimNetLite: + var name = sockAddrToStr(addr sockAddress) + return (sock, name) + else: + return (sock, $inet_ntoa(cast[Sockaddr_in](sockAddress).sin_addr)) when defined(windows): var wsa: WSAData diff --git a/lib/pure/net.nim b/lib/pure/net.nim index b37782271..24c94b651 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -44,64 +44,73 @@ ## After you create a socket with the `newSocket` procedure, you can easily ## connect it to a server running at a known hostname (or IP address) and port. ## To do so over TCP, use the example below. -## -## .. code-block:: Nim -## var socket = newSocket() -## socket.connect("google.com", Port(80)) -## -## For SSL, use the following example (and make sure to compile with `-d:ssl`): -## -## .. code-block:: Nim -## var socket = newSocket() -## var ctx = newContext() -## wrapSocket(ctx, socket) -## socket.connect("google.com", Port(443)) -## + +runnableExamples("-r:off"): + let socket = newSocket() + socket.connect("google.com", Port(80)) + +## For SSL, use the following example: + +runnableExamples("-r:off -d:ssl"): + let socket = newSocket() + let ctx = newContext() + wrapSocket(ctx, socket) + socket.connect("google.com", Port(443)) + ## UDP is a connectionless protocol, so UDP sockets don't have to explicitly ## call the `connect <net.html#connect%2CSocket%2Cstring>`_ procedure. They can ## simply start sending data immediately. -## -## .. code-block:: Nim -## var socket = newSocket(AF_INET, SOCK_DGRAM, IPPROTO_UDP) -## socket.sendTo("192.168.0.1", Port(27960), "status\n") -## + +runnableExamples("-r:off"): + let socket = newSocket(AF_INET, SOCK_DGRAM, IPPROTO_UDP) + socket.sendTo("192.168.0.1", Port(27960), "status\n") + +runnableExamples("-r:off"): + let socket = newSocket(AF_INET, SOCK_DGRAM, IPPROTO_UDP) + let ip = parseIpAddress("192.168.0.1") + doAssert socket.sendTo(ip, Port(27960), "status\c\l") == 8 + ## Creating a server ## ----------------- ## ## After you create a socket with the `newSocket` procedure, you can create a ## TCP server by calling the `bindAddr` and `listen` procedures. -## -## .. code-block:: Nim -## var socket = newSocket() -## socket.bindAddr(Port(1234)) -## socket.listen() -## -## You can then begin accepting connections using the `accept` procedure. -## -## .. code-block:: Nim -## var client: Socket -## var address = "" -## while true: -## socket.acceptAddr(client, address) -## echo("Client connected from: ", address) + +runnableExamples("-r:off"): + let socket = newSocket() + socket.bindAddr(Port(1234)) + socket.listen() + + # You can then begin accepting connections using the `accept` procedure. + var client: Socket + var address = "" + while true: + socket.acceptAddr(client, address) + echo "Client connected from: ", address import std/private/since -import nativesockets, os, strutils, times, sets, options, std/monotimes -import ssl_config +when defined(nimPreviewSlimSystem): + import std/assertions + +import std/nativesockets +import std/[os, strutils, times, sets, options, monotimes] +import std/ssl_config export nativesockets.Port, nativesockets.`$`, nativesockets.`==` -export Domain, SockType, Protocol +export Domain, SockType, Protocol, IPPROTO_NONE const useWinVersion = defined(windows) or defined(nimdoc) +const useNimNetLite = defined(nimNetLite) or defined(freertos) or defined(zephyr) or + defined(nuttx) const defineSsl = defined(ssl) or defined(nimdoc) when useWinVersion: - from winlean import WSAESHUTDOWN + from std/winlean import WSAESHUTDOWN when defineSsl: - import openssl + import std/openssl when not defined(nimDisableCertificateValidation): - from ssl_certs import scanSSLCertificates + from std/ssl_certs import scanSSLCertificates # Note: The enumerations are mapped to Window's constants. @@ -198,6 +207,30 @@ type when defined(nimHasStyleChecks): {.pop.} + +when defined(posix) and not defined(lwip): + from std/posix import TPollfd, POLLIN, POLLPRI, POLLOUT, POLLWRBAND, Tnfds + + template monitorPollEvent(x: var SocketHandle, y: cint, timeout: int): int = + var tpollfd: TPollfd + tpollfd.fd = cast[cint](x) + tpollfd.events = y + posix.poll(addr(tpollfd), Tnfds(1), timeout) + +proc timeoutRead(fd: var SocketHandle, timeout = 500): int = + when defined(windows) or defined(lwip): + var fds = @[fd] + selectRead(fds, timeout) + else: + monitorPollEvent(fd, POLLIN or POLLPRI, timeout) + +proc timeoutWrite(fd: var SocketHandle, timeout = 500): int = + when defined(windows) or defined(lwip): + var fds = @[fd] + selectWrite(fds, timeout) + else: + monitorPollEvent(fd, POLLOUT or POLLWRBAND, timeout) + proc socketError*(socket: Socket, err: int = -1, async = false, lastError = (-1).OSErrorCode, flags: set[SocketFlag] = {}) {.gcsafe.} @@ -283,14 +316,20 @@ proc parseIPv4Address(addressStr: string): IpAddress = byteCount = 0 currentByte: uint16 = 0 separatorValid = false + leadingZero = false result = IpAddress(family: IpAddressFamily.IPv4) for i in 0 .. high(addressStr): if addressStr[i] in strutils.Digits: # Character is a number + if leadingZero: + raise newException(ValueError, + "Invalid IP address. Octal numbers are not allowed") currentByte = currentByte * 10 + cast[uint16](ord(addressStr[i]) - ord('0')) - if currentByte > 255'u16: + if currentByte == 0'u16: + leadingZero = true + elif currentByte > 255'u16: raise newException(ValueError, "Invalid IP Address. Value is out of range") separatorValid = true @@ -302,6 +341,7 @@ proc parseIPv4Address(addressStr: string): IpAddress = currentByte = 0 byteCount.inc separatorValid = false + leadingZero = false else: raise newException(ValueError, "Invalid IP Address. Address contains an invalid character") @@ -390,10 +430,16 @@ proc parseIPv6Address(addressStr: string): IpAddress = result.address_v6[groupCount*2+1] = cast[uint8](currentShort and 0xFF) groupCount.inc() else: # Must parse IPv4 address + var leadingZero = false for i, c in addressStr[v4StartPos..high(addressStr)]: if c in strutils.Digits: # Character is a number + if leadingZero: + raise newException(ValueError, + "Invalid IP address. Octal numbers not allowed") currentShort = currentShort * 10 + cast[uint32](ord(c) - ord('0')) - if currentShort > 255'u32: + if currentShort == 0'u32: + leadingZero = true + elif currentShort > 255'u32: raise newException(ValueError, "Invalid IP Address. Value is out of range") separatorValid = true @@ -404,6 +450,7 @@ proc parseIPv6Address(addressStr: string): IpAddress = currentShort = 0 byteCount.inc() separatorValid = false + leadingZero = false else: # Invalid character raise newException(ValueError, "Invalid IP Address. Address contains an invalid character") @@ -433,7 +480,12 @@ proc parseIPv6Address(addressStr: string): IpAddress = proc parseIpAddress*(addressStr: string): IpAddress = ## Parses an IP address - ## Raises ValueError on error + ## + ## Raises ValueError on error. + ## + ## For IPv4 addresses, only the strict form as + ## defined in RFC 6943 is considered valid, see + ## https://datatracker.ietf.org/doc/html/rfc6943#section-3.1.1. if addressStr.len == 0: raise newException(ValueError, "IP Address string is empty") if addressStr.contains(':'): @@ -495,13 +547,20 @@ proc fromSockAddr*(sa: Sockaddr_storage | SockAddr | Sockaddr_in | Sockaddr_in6, fromSockAddrAux(cast[ptr Sockaddr_storage](unsafeAddr sa), sl, address, port) when defineSsl: - CRYPTO_malloc_init() - doAssert SslLibraryInit() == 1 - SSL_load_error_strings() - ERR_load_BIO_strings() - OpenSSL_add_all_algorithms() - - proc raiseSSLError*(s = "") = + # OpenSSL >= 1.1.0 does not need explicit init. + when not useOpenssl3: + CRYPTO_malloc_init() + doAssert SslLibraryInit() == 1 + SSL_load_error_strings() + ERR_load_BIO_strings() + OpenSSL_add_all_algorithms() + + proc sslHandle*(self: Socket): SslPtr = + ## Retrieve the ssl pointer of `socket`. + ## Useful for interfacing with `openssl`. + self.sslHandle + + proc raiseSSLError*(s = "") {.raises: [SslError].}= ## Raises a new SSL error. if s != "": raise newException(SslError, s) @@ -563,12 +622,11 @@ when defineSsl: proc newContext*(protVersion = protSSLv23, verifyMode = CVerifyPeer, certFile = "", keyFile = "", cipherList = CiphersIntermediate, - caDir = "", caFile = ""): SslContext = + caDir = "", caFile = "", ciphersuites = CiphersModern): SslContext = ## Creates an SSL context. ## - ## Protocol version specifies the protocol to use. SSLv2, SSLv3, TLSv1 - ## are available with the addition of `protSSLv23` which allows for - ## compatibility with all of them. + ## Protocol version is currently ignored by default and TLS is used. + ## With `-d:openssl10`, only SSLv23 and TLSv1 may be used. ## ## There are three options for verify mode: ## `CVerifyNone`: certificates are not verified; @@ -582,10 +640,10 @@ when defineSsl: ## ## CA certificates will be loaded, in the following order, from: ## - ## - caFile, caDir, parameters, if set - ## - if `verifyMode` is set to `CVerifyPeerUseEnvVars`, - ## the SSL_CERT_FILE and SSL_CERT_DIR environment variables are used - ## - a set of files and directories from the `ssl_certs <ssl_certs.html>`_ file. + ## - caFile, caDir, parameters, if set + ## - if `verifyMode` is set to `CVerifyPeerUseEnvVars`, + ## the SSL_CERT_FILE and SSL_CERT_DIR environment variables are used + ## - a set of files and directories from the `ssl_certs <ssl_certs.html>`_ file. ## ## The last two parameters specify the certificate file path and the key file ## path, a server socket will most likely not work without these. @@ -595,31 +653,39 @@ when defineSsl: ## or using ECDSA: ## - `openssl ecparam -out mykey.pem -name secp256k1 -genkey` ## - `openssl req -new -key mykey.pem -x509 -nodes -days 365 -out mycert.pem` - var newCTX: SslCtx - case protVersion - of protSSLv23: - newCTX = SSL_CTX_new(SSLv23_method()) # SSlv2,3 and TLS1 support. - of protSSLv2: - raiseSSLError("SSLv2 is no longer secure and has been deprecated, use protSSLv23") - of protSSLv3: - raiseSSLError("SSLv3 is no longer secure and has been deprecated, use protSSLv23") - of protTLSv1: - newCTX = SSL_CTX_new(TLSv1_method()) + var mtd: PSSL_METHOD + when defined(openssl10): + case protVersion + of protSSLv23: + mtd = SSLv23_method() + of protSSLv2: + raiseSSLError("SSLv2 is no longer secure and has been deprecated, use protSSLv23") + of protSSLv3: + raiseSSLError("SSLv3 is no longer secure and has been deprecated, use protSSLv23") + of protTLSv1: + mtd = TLSv1_method() + else: + mtd = TLS_method() + if mtd == nil: + raiseSSLError("Failed to create TLS context") + var newCTX = SSL_CTX_new(mtd) + if newCTX == nil: + raiseSSLError("Failed to create TLS context") if newCTX.SSL_CTX_set_cipher_list(cipherList) != 1: raiseSSLError() when not defined(openssl10) and not defined(libressl): let sslVersion = getOpenSSLVersion() - if sslVersion >= 0x010101000 and not sslVersion == 0x020000000: + if sslVersion >= 0x010101000 and sslVersion != 0x020000000: # In OpenSSL >= 1.1.1, TLSv1.3 cipher suites can only be configured via # this API. - if newCTX.SSL_CTX_set_ciphersuites(cipherList) != 1: + if newCTX.SSL_CTX_set_ciphersuites(ciphersuites) != 1: raiseSSLError() # Automatically the best ECDH curve for client exchange. Without this, ECDH # ciphers will be ignored by the server. # # From OpenSSL >= 1.1.0, this setting is set by default and can't be - # overriden. + # overridden. if newCTX.SSL_CTX_set_ecdh_auto(1) != 1: raiseSSLError() @@ -644,15 +710,20 @@ when defineSsl: if verifyMode != CVerifyNone: # Use the caDir and caFile parameters if set if caDir != "" or caFile != "": - if newCTX.SSL_CTX_load_verify_locations(caFile, caDir) != VerifySuccess: + if newCTX.SSL_CTX_load_verify_locations(if caFile == "": nil else: caFile.cstring, if caDir == "": nil else: caDir.cstring) != VerifySuccess: raise newException(IOError, "Failed to load SSL/TLS CA certificate(s).") else: # Scan for certs in known locations. For CVerifyPeerUseEnvVars also scan # the SSL_CERT_FILE and SSL_CERT_DIR env vars var found = false - for fn in scanSSLCertificates(): - if newCTX.SSL_CTX_load_verify_locations(fn, nil) == VerifySuccess: + let useEnvVars = (if verifyMode == CVerifyPeerUseEnvVars: true else: false) + for fn in scanSSLCertificates(useEnvVars = useEnvVars): + if fn.extractFilename == "": + if newCTX.SSL_CTX_load_verify_locations(nil, cstring(fn.normalizePathEnd(false))) == VerifySuccess: + found = true + break + elif newCTX.SSL_CTX_load_verify_locations(cstring(fn), nil) == VerifySuccess: found = true break if not found: @@ -685,17 +756,16 @@ when defineSsl: return ctx.getExtraInternal().clientGetPskFunc proc pskClientCallback(ssl: SslPtr; hint: cstring; identity: cstring; - max_identity_len: cuint; psk: ptr cuchar; + max_identity_len: cuint; psk: ptr uint8; max_psk_len: cuint): cuint {.cdecl.} = let ctx = SslContext(context: ssl.SSL_get_SSL_CTX) let hintString = if hint == nil: "" else: $hint let (identityString, pskString) = (ctx.clientGetPskFunc)(hintString) - if psk.len.cuint > max_psk_len: + if pskString.len.cuint > max_psk_len: return 0 if identityString.len.cuint >= max_identity_len: return 0 - - copyMem(identity, identityString.cstring, pskString.len + 1) # with the last zero byte + copyMem(identity, identityString.cstring, identityString.len + 1) # with the last zero byte copyMem(psk, pskString.cstring, pskString.len) return pskString.len.cuint @@ -712,11 +782,11 @@ when defineSsl: proc serverGetPskFunc*(ctx: SslContext): SslServerGetPskFunc = return ctx.getExtraInternal().serverGetPskFunc - proc pskServerCallback(ssl: SslCtx; identity: cstring; psk: ptr cuchar; + proc pskServerCallback(ssl: SslCtx; identity: cstring; psk: ptr uint8; max_psk_len: cint): cuint {.cdecl.} = let ctx = SslContext(context: ssl.SSL_get_SSL_CTX) let pskString = (ctx.serverGetPskFunc)($identity) - if psk.len.cint > max_psk_len: + if pskString.len.cint > max_psk_len: return 0 copyMem(psk, pskString.cstring, pskString.len) @@ -759,23 +829,28 @@ when defineSsl: if SSL_set_fd(socket.sslHandle, socket.fd) != 1: raiseSSLError() - proc checkCertName(socket: Socket, hostname: string) = + proc checkCertName(socket: Socket, hostname: string) {.raises: [SslError], tags:[RootEffect].} = ## Check if the certificate Subject Alternative Name (SAN) or Subject CommonName (CN) matches hostname. ## Wildcards match only in the left-most label. ## When name starts with a dot it will be matched by a certificate valid for any subdomain when not defined(nimDisableCertificateValidation) and not defined(windows): assert socket.isSsl - let certificate = socket.sslHandle.SSL_get_peer_certificate() - if certificate.isNil: - raiseSSLError("No SSL certificate found.") - - const X509_CHECK_FLAG_ALWAYS_CHECK_SUBJECT = 0x1.cuint - const size = 1024 - var peername: string = newString(size) - let match = certificate.X509_check_host(hostname.cstring, hostname.len.cint, - X509_CHECK_FLAG_ALWAYS_CHECK_SUBJECT, peername) - if match != 1: - raiseSSLError("SSL Certificate check failed.") + try: + let certificate = socket.sslHandle.SSL_get_peer_certificate() + if certificate.isNil: + raiseSSLError("No SSL certificate found.") + + const X509_CHECK_FLAG_ALWAYS_CHECK_SUBJECT = 0x1.cuint + # https://www.openssl.org/docs/man1.1.1/man3/X509_check_host.html + let match = certificate.X509_check_host(hostname.cstring, hostname.len.cint, + X509_CHECK_FLAG_ALWAYS_CHECK_SUBJECT, nil) + # https://www.openssl.org/docs/man1.1.1/man3/SSL_get_peer_certificate.html + X509_free(certificate) + if match != 1: + raiseSSLError("SSL Certificate check failed.") + + except LibraryError: + raiseSSLError("SSL import failed") proc wrapConnectedSocket*(ctx: SslContext, socket: Socket, handshake: SslHandshakeType, @@ -802,6 +877,7 @@ when defineSsl: let ret = SSL_connect(socket.sslHandle) socketError(socket, ret) when not defined(nimDisableCertificateValidation) and not defined(windows): + # FIXME: this should be skipped on CVerifyNone if hostname.len > 0 and not isIpAddress(hostname): socket.checkCertName(hostname) of handshakeAsServer: @@ -954,14 +1030,16 @@ proc bindAddr*(socket: Socket, port = Port(0), address = "") {. var aiList = getAddrInfo(realaddr, port, socket.domain) if bindAddr(socket.fd, aiList.ai_addr, aiList.ai_addrlen.SockLen) < 0'i32: - freeaddrinfo(aiList) - raiseOSError(osLastError()) - freeaddrinfo(aiList) + freeAddrInfo(aiList) + var address2: string + address2.addQuoted address + raiseOSError(osLastError(), "address: $# port: $#" % [address2, $port]) + freeAddrInfo(aiList) proc acceptAddr*(server: Socket, client: var owned(Socket), address: var string, flags = {SocketFlag.SafeDisconn}, inheritable = defined(nimInheritHandles)) {. - tags: [ReadIOEffect], gcsafe, locks: 0.} = + tags: [ReadIOEffect], gcsafe.} = ## Blocks until a connection is being made from a client. When a connection ## is made sets `client` to the client socket and `address` to the address ## of the connecting client. @@ -1076,7 +1154,7 @@ proc accept*(server: Socket, client: var owned(Socket), acceptAddr(server, client, addrDummy, flags) when defined(posix) and not defined(lwip): - from posix import Sigset, sigwait, sigismember, sigemptyset, sigaddset, + from std/posix import Sigset, sigwait, sigismember, sigemptyset, sigaddset, sigprocmask, pthread_sigmask, SIGPIPE, SIG_BLOCK, SIG_UNBLOCK template blockSigpipe(body: untyped): untyped = @@ -1190,9 +1268,9 @@ proc close*(socket: Socket, flags = {SocketFlag.SafeDisconn}) = socket.fd = osInvalidSocket when defined(posix): - from posix import TCP_NODELAY + from std/posix import TCP_NODELAY else: - from winlean import TCP_NODELAY + from std/winlean import TCP_NODELAY proc toCInt*(opt: SOBool): cint = ## Converts a `SOBool` into its Socket Option cint representation. @@ -1219,32 +1297,31 @@ proc getLocalAddr*(socket: Socket): (string, Port) = ## This is high-level interface for `getsockname`:idx:. getLocalAddr(socket.fd, socket.domain) -proc getPeerAddr*(socket: Socket): (string, Port) = - ## Get the socket's peer address and port number. - ## - ## This is high-level interface for `getpeername`:idx:. - getPeerAddr(socket.fd, socket.domain) +when not useNimNetLite: + proc getPeerAddr*(socket: Socket): (string, Port) = + ## Get the socket's peer address and port number. + ## + ## This is high-level interface for `getpeername`:idx:. + getPeerAddr(socket.fd, socket.domain) proc setSockOpt*(socket: Socket, opt: SOBool, value: bool, level = SOL_SOCKET) {.tags: [WriteIOEffect].} = ## Sets option `opt` to a boolean value specified by `value`. - ## - ## .. code-block:: Nim - ## var socket = newSocket() - ## socket.setSockOpt(OptReusePort, true) - ## socket.setSockOpt(OptNoDelay, true, level=IPPROTO_TCP.toInt) - ## + runnableExamples("-r:off"): + let socket = newSocket() + socket.setSockOpt(OptReusePort, true) + socket.setSockOpt(OptNoDelay, true, level = IPPROTO_TCP.cint) var valuei = cint(if value: 1 else: 0) setSockOptInt(socket.fd, cint(level), toCInt(opt), valuei) -when defined(posix) or defined(nimdoc): +when defined(nimdoc) or (defined(posix) and not useNimNetLite): proc connectUnix*(socket: Socket, path: string) = ## Connects to Unix socket on `path`. ## This only works on Unix-style systems: Mac OS X, BSD and Linux when not defined(nimdoc): var socketAddr = makeUnixAddr(path) if socket.fd.connect(cast[ptr SockAddr](addr socketAddr), - (sizeof(socketAddr.sun_family) + path.len).SockLen) != 0'i32: + (offsetOf(socketAddr, sun_path) + path.len + 1).SockLen) != 0'i32: raiseOSError(osLastError()) proc bindUnix*(socket: Socket, path: string) = @@ -1253,10 +1330,10 @@ when defined(posix) or defined(nimdoc): when not defined(nimdoc): var socketAddr = makeUnixAddr(path) if socket.fd.bindAddr(cast[ptr SockAddr](addr socketAddr), - (sizeof(socketAddr.sun_family) + path.len).SockLen) != 0'i32: + (offsetOf(socketAddr, sun_path) + path.len + 1).SockLen) != 0'i32: raiseOSError(osLastError()) -when defined(ssl): +when defineSsl: proc gotHandshake*(socket: Socket): bool = ## Determines whether a handshake has occurred between a client (`socket`) ## and the server that `socket` is connected to. @@ -1277,14 +1354,6 @@ proc hasDataBuffered*(s: Socket): bool = if s.isSsl and not result: result = s.sslHasPeekChar -proc select(readfd: Socket, timeout = 500): int = - ## Used for socket operation timeouts. - if readfd.hasDataBuffered: - return 1 - - var fds = @[readfd.fd] - result = selectRead(fds, timeout) - proc isClosed(socket: Socket): bool = socket.fd == osInvalidSocket @@ -1398,7 +1467,9 @@ proc waitFor(socket: Socket, waited: var Duration, timeout, size: int, return min(sslPending, size) var startTime = getMonoTime() - let selRet = select(socket, (timeout - waited.inMilliseconds).int) + let selRet = if socket.hasDataBuffered: 1 + else: + timeoutRead(socket.fd, (timeout - waited.inMilliseconds).int) if selRet < 0: raiseOSError(osLastError()) if selRet != 1: raise newException(TimeoutError, "Call to '" & funcName & "' timed out.") @@ -1426,7 +1497,7 @@ proc recv*(socket: Socket, data: var string, size: int, timeout = -1, flags = {SocketFlag.SafeDisconn}): int = ## Higher-level version of `recv`. ## - ## Reads **up to** `size` bytes from `socket` into `buf`. + ## Reads **up to** `size` bytes from `socket` into `data`. ## ## For buffered sockets this function will attempt to read all the requested ## data. It will read this data in `BufferSize` chunks. @@ -1443,8 +1514,6 @@ proc recv*(socket: Socket, data: var string, size: int, timeout = -1, ## A timeout may be specified in milliseconds, if enough data is not received ## within the time specified a TimeoutError exception will be raised. ## - ## **Note**: `data` must be initialised. - ## ## .. warning:: Only the `SafeDisconn` flag is currently supported. data.setLen(size) result = @@ -1463,7 +1532,7 @@ proc recv*(socket: Socket, size: int, timeout = -1, flags = {SocketFlag.SafeDisconn}): string {.inline.} = ## Higher-level version of `recv` which returns a string. ## - ## Reads **up to** `size` bytes from `socket` into `buf`. + ## Reads **up to** `size` bytes from `socket` into the result. ## ## For buffered sockets this function will attempt to read all the requested ## data. It will read this data in `BufferSize` chunks. @@ -1534,6 +1603,7 @@ proc readLine*(socket: Socket, line: var string, timeout = -1, if flags.isDisconnectionError(lastError): setLen(line, 0) socket.socketError(n, lastError = lastError, flags = flags) + return var waited: Duration @@ -1583,11 +1653,12 @@ proc recvLine*(socket: Socket, timeout = -1, result = "" readLine(socket, result, timeout, flags, maxLength) -proc recvFrom*(socket: Socket, data: var string, length: int, - address: var string, port: var Port, flags = 0'i32): int {. +proc recvFrom*[T: string | IpAddress](socket: Socket, data: var string, length: int, + address: var T, port: var Port, flags = 0'i32): int {. tags: [ReadIOEffect].} = ## Receives data from `socket`. This function should normally be used with - ## connection-less sockets (UDP sockets). + ## connection-less sockets (UDP sockets). The source address of the data + ## packet is stored in the `address` argument as either a string or an IpAddress. ## ## If an error occurs an OSError exception will be raised. Otherwise the return ## value will be the length of data received. @@ -1596,31 +1667,37 @@ proc recvFrom*(socket: Socket, data: var string, length: int, ## so when `socket` is buffered the non-buffered implementation will be ## used. Therefore if `socket` contains something in its buffer this ## function will make no effort to return it. - template adaptRecvFromToDomain(domain: Domain) = - var addrLen = sizeof(sockAddress).SockLen + template adaptRecvFromToDomain(sockAddress: untyped, domain: Domain) = + var addrLen = SockLen(sizeof(sockAddress)) result = recvfrom(socket.fd, cstring(data), length.cint, flags.cint, cast[ptr SockAddr](addr(sockAddress)), addr(addrLen)) if result != -1: data.setLen(result) - address = getAddrString(cast[ptr SockAddr](addr(sockAddress))) - when domain == AF_INET6: - port = ntohs(sockAddress.sin6_port).Port + + when typeof(address) is string: + address = getAddrString(cast[ptr SockAddr](addr(sockAddress))) + when domain == AF_INET6: + port = ntohs(sockAddress.sin6_port).Port + else: + port = ntohs(sockAddress.sin_port).Port else: - port = ntohs(sockAddress.sin_port).Port + data.setLen(result) + sockAddress.fromSockAddr(addrLen, address, port) else: raiseOSError(osLastError()) assert(socket.protocol != IPPROTO_TCP, "Cannot `recvFrom` on a TCP socket") # TODO: Buffered sockets data.setLen(length) + case socket.domain of AF_INET6: var sockAddress: Sockaddr_in6 - adaptRecvFromToDomain(AF_INET6) + adaptRecvFromToDomain(sockAddress, AF_INET6) of AF_INET: var sockAddress: Sockaddr_in - adaptRecvFromToDomain(AF_INET) + adaptRecvFromToDomain(sockAddress, AF_INET) else: raise newException(ValueError, "Unknown socket address family") @@ -1659,15 +1736,36 @@ proc send*(socket: Socket, data: pointer, size: int): int {. result = send(socket.fd, data, size, int32(MSG_NOSIGNAL)) proc send*(socket: Socket, data: string, - flags = {SocketFlag.SafeDisconn}) {.tags: [WriteIOEffect].} = - ## sends data to a socket. - let sent = send(socket, cstring(data), data.len) - if sent < 0: - let lastError = osLastError() - socketError(socket, lastError = lastError, flags = flags) + flags = {SocketFlag.SafeDisconn}, maxRetries = 100) {.tags: [WriteIOEffect].} = + ## Sends data to a socket. Will try to send all the data by handling interrupts + ## and incomplete writes up to `maxRetries`. + var written = 0 + var attempts = 0 + while data.len - written > 0: + let sent = send(socket, cstring(data), data.len) + + if sent < 0: + let lastError = osLastError() + let isBlockingErr = + when defined(nimdoc): + false + elif useWinVersion: + lastError.int32 == WSAEINTR or + lastError.int32 == WSAEWOULDBLOCK + else: + lastError.int32 == EINTR or + lastError.int32 == EWOULDBLOCK or + lastError.int32 == EAGAIN - if sent != data.len: - raiseOSError(osLastError(), "Could not send all data.") + if not isBlockingErr: + let lastError = osLastError() + socketError(socket, lastError = lastError, flags = flags) + else: + attempts.inc() + if attempts > maxRetries: + raiseOSError(osLastError(), "Could not send all data.") + else: + written.inc(sent) template `&=`*(socket: Socket; data: typed) = ## an alias for 'send'. @@ -1683,7 +1781,8 @@ proc sendTo*(socket: Socket, address: string, port: Port, data: pointer, tags: [WriteIOEffect].} = ## This proc sends `data` to the specified `address`, ## which may be an IP address or a hostname, if a hostname is specified - ## this function will try each IP of that hostname. + ## this function will try each IP of that hostname. This function + ## should normally be used with connection-less sockets (UDP sockets). ## ## If an error occurs an OSError exception will be raised. ## @@ -1707,7 +1806,7 @@ proc sendTo*(socket: Socket, address: string, port: Port, data: pointer, it = it.ai_next let osError = osLastError() - freeaddrinfo(aiList) + freeAddrInfo(aiList) if not success: raiseOSError(osError) @@ -1718,11 +1817,37 @@ proc sendTo*(socket: Socket, address: string, port: Port, ## which may be an IP address or a hostname, if a hostname is specified ## this function will try each IP of that hostname. ## + ## Generally for use with connection-less (UDP) sockets. + ## ## If an error occurs an OSError exception will be raised. ## ## This is the high-level version of the above `sendTo` function. socket.sendTo(address, port, cstring(data), data.len, socket.domain) +proc sendTo*(socket: Socket, address: IpAddress, port: Port, + data: string, flags = 0'i32): int {. + discardable, tags: [WriteIOEffect].} = + ## This proc sends `data` to the specified `IpAddress` and returns + ## the number of bytes written. + ## + ## Generally for use with connection-less (UDP) sockets. + ## + ## If an error occurs an OSError exception will be raised. + ## + ## This is the high-level version of the above `sendTo` function. + assert(socket.protocol != IPPROTO_TCP, "Cannot `sendTo` on a TCP socket") + assert(not socket.isClosed, "Cannot `sendTo` on a closed socket") + + var sa: Sockaddr_storage + var sl: SockLen + toSockAddr(address, port, sa, sl) + result = sendto(socket.fd, cstring(data), data.len().cint, flags.cint, + cast[ptr SockAddr](addr sa), sl) + + if result == -1'i32: + let osError = osLastError() + raiseOSError(osError) + proc isSsl*(socket: Socket): bool = ## Determines whether `socket` is a SSL socket. @@ -1734,6 +1859,16 @@ proc isSsl*(socket: Socket): bool = proc getFd*(socket: Socket): SocketHandle = return socket.fd ## Returns the socket's file descriptor +when defined(zephyr) or defined(nimNetSocketExtras): # Remove in future + proc getDomain*(socket: Socket): Domain = return socket.domain + ## Returns the socket's domain + + proc getType*(socket: Socket): SockType = return socket.sockType + ## Returns the socket's type + + proc getProtocol*(socket: Socket): Protocol = return socket.protocol + ## Returns the socket's protocol + when defined(nimHasStyleChecks): {.push styleChecks: off.} @@ -1785,14 +1920,18 @@ proc `==`*(lhs, rhs: IpAddress): bool = proc `$`*(address: IpAddress): string = ## Converts an IpAddress into the textual representation - result = "" case address.family of IpAddressFamily.IPv4: - for i in 0 .. 3: - if i != 0: - result.add('.') - result.add($address.address_v4[i]) + result = newStringOfCap(15) + result.addInt address.address_v4[0] + result.add '.' + result.addInt address.address_v4[1] + result.add '.' + result.addInt address.address_v4[2] + result.add '.' + result.addInt address.address_v4[3] of IpAddressFamily.IPv6: + result = newStringOfCap(39) var currentZeroStart = -1 currentZeroCount = 0 @@ -1886,7 +2025,7 @@ proc dial*(address: string, port: Port, # network system problem (e.g. not enough FDs), and not an unreachable # address. let err = osLastError() - freeaddrinfo(aiList) + freeAddrInfo(aiList) closeUnusedFds() raiseOSError(err) fdPerDomain[ord(domain)] = lastFd @@ -1895,18 +2034,20 @@ proc dial*(address: string, port: Port, break lastError = osLastError() it = it.ai_next - freeaddrinfo(aiList) + freeAddrInfo(aiList) closeUnusedFds(ord(domain)) if success: - result = newSocket(lastFd, domain, sockType, protocol) + result = newSocket(lastFd, domain, sockType, protocol, buffered) elif lastError != 0.OSErrorCode: + lastFd.close() raiseOSError(lastError) else: + lastFd.close() raise newException(IOError, "Couldn't resolve address: " & address) proc connect*(socket: Socket, address: string, - port = Port(0)) {.tags: [ReadIOEffect].} = + port = Port(0)) {.tags: [ReadIOEffect, RootEffect].} = ## Connects socket to `address`:`port`. `Address` can be an IP address or a ## host name. If `address` is a host name, this function will try each IP ## of that host name. `htons` is already performed on `port` so you must @@ -1925,7 +2066,7 @@ proc connect*(socket: Socket, address: string, else: lastError = osLastError() it = it.ai_next - freeaddrinfo(aiList) + freeAddrInfo(aiList) if not success: raiseOSError(lastError) when defineSsl: @@ -1977,11 +2118,11 @@ proc connectAsync(socket: Socket, name: string, port = Port(0), it = it.ai_next - freeaddrinfo(aiList) + freeAddrInfo(aiList) if not success: raiseOSError(lastError) proc connect*(socket: Socket, address: string, port = Port(0), - timeout: int) {.tags: [ReadIOEffect, WriteIOEffect].} = + timeout: int) {.tags: [ReadIOEffect, WriteIOEffect, RootEffect].} = ## Connects to server as specified by `address` on port specified by `port`. ## ## The `timeout` parameter specifies the time in milliseconds to allow for @@ -1989,8 +2130,7 @@ proc connect*(socket: Socket, address: string, port = Port(0), socket.fd.setBlocking(false) socket.connectAsync(address, port, socket.domain) - var s = @[socket.fd] - if selectWrite(s, timeout) != 1: + if timeoutWrite(socket.fd, timeout) != 1: raise newException(TimeoutError, "Call to 'connect' timed out.") else: let res = getSockOptInt(socket.fd, SOL_SOCKET, SO_ERROR) @@ -2021,10 +2161,8 @@ proc getPrimaryIPAddr*(dest = parseIpAddress("8.8.8.8")): IpAddress = ## ## Supports IPv4 and v6. ## Raises OSError if external networking is not set up. - ## - ## .. code-block:: Nim - ## echo $getPrimaryIPAddr() # "192.168.1.2" - + runnableExamples("-r:off"): + echo getPrimaryIPAddr() # "192.168.1.2" let socket = if dest.family == IpAddressFamily.IPv4: newSocket(AF_INET, SOCK_DGRAM, IPPROTO_UDP) diff --git a/lib/pure/nimprof.nim b/lib/pure/nimprof.nim index 721ae35c3..bf8367d1d 100644 --- a/lib/pure/nimprof.nim +++ b/lib/pure/nimprof.nim @@ -9,18 +9,21 @@ ## Profiling support for Nim. This is an embedded profiler that requires ## `--profiler:on`. You only need to import this module to get a profiling -## report at program exit. +## report at program exit. See `Embedded Stack Trace Profiler <estp.html>`_ +## for usage. when not defined(profiler) and not defined(memProfiler): {.error: "Profiling support is turned off! Enable profiling by passing `--profiler:on --stackTrace:on` to the compiler (see the Nim Compiler User Guide for more options).".} -when defined(nimHasUsed): - {.used.} +{.used.} # We don't want to profile the profiling code ... {.push profiler: off.} -import hashes, algorithm, strutils, tables, sets +import std/[hashes, algorithm, strutils, tables, sets] + +when defined(nimPreviewSlimSystem): + import std/[syncio, sysatomics] when not defined(memProfiler): include "system/timers" @@ -66,7 +69,7 @@ when not defined(memProfiler): else: interval = intervalInUs * 1000 - tickCountCorrection when withThreads: - import locks + import std/locks var profilingLock: Lock @@ -121,13 +124,13 @@ when defined(memProfiler): var gTicker {.threadvar.}: int - proc requestedHook(): bool {.nimcall, locks: 0.} = + proc requestedHook(): bool {.nimcall.} = if gTicker == 0: gTicker = SamplingInterval result = true dec gTicker - proc hook(st: StackTrace, size: int) {.nimcall, locks: 0.} = + proc hook(st: StackTrace, size: int) {.nimcall.} = when defined(ignoreAllocationSize): hookAux(st, 1) else: @@ -139,7 +142,7 @@ else: gTicker: int # we use an additional counter to # avoid calling 'getTicks' too frequently - proc requestedHook(): bool {.nimcall, locks: 0.} = + proc requestedHook(): bool {.nimcall.} = if interval == 0: result = true elif gTicker == 0: gTicker = 500 @@ -148,7 +151,7 @@ else: else: dec gTicker - proc hook(st: StackTrace) {.nimcall, locks: 0.} = + proc hook(st: StackTrace) {.nimcall.} = #echo "profiling! ", interval if interval == 0: hookAux(st, 1) diff --git a/lib/pure/nimtracker.nim b/lib/pure/nimtracker.nim deleted file mode 100644 index a66dfc2ea..000000000 --- a/lib/pure/nimtracker.nim +++ /dev/null @@ -1,88 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2016 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## Memory tracking support for Nim. - -when not defined(memTracker) and not isMainModule: - {.error: "Memory tracking support is turned off!".} - -{.push memtracker: off.} -# we import the low level wrapper and are careful not to use Nim's -# memory manager for anything here. -import sqlite3 - -var - dbHandle: PSqlite3 - insertStmt {.threadvar.}: Pstmt - -const insertQuery = "INSERT INTO tracking(op, address, size, file, line) values (?, ?, ?, ?, ?)" - -template sbind(x: int; value) = - when value is cstring: - let ret = insertStmt.bindText(x, value, value.len.int32, SQLITE_TRANSIENT) - if ret != SQLITE_OK: - quit "could not bind value" - else: - let ret = insertStmt.bindInt64(x, value) - if ret != SQLITE_OK: - quit "could not bind value" - -when defined(memTracker): - proc logEntries(log: TrackLog) {.nimcall, locks: 0, tags: [], gcsafe.} = - if insertStmt.isNil: - if prepare_v2(dbHandle, insertQuery, - insertQuery.len, insertStmt, nil) != SQLITE_OK: - quit "could not bind query to insertStmt " & $sqlite3.errmsg(dbHandle) - for i in 0..log.count-1: - var success = false - let e = log.data[i] - discard sqlite3.reset(insertStmt) - discard clearBindings(insertStmt) - sbind 1, e.op - sbind(2, cast[int](e.address)) - sbind 3, e.size - sbind 4, e.file - sbind 5, e.line - if step(insertStmt) == SQLITE_DONE: - success = true - if not success: - quit "could not write to database! " & $sqlite3.errmsg(dbHandle) - -proc execQuery(q: string) = - var s: Pstmt - if prepare_v2(dbHandle, q, q.len.int32, s, nil) == SQLITE_OK: - discard step(s) - if finalize(s) != SQLITE_OK: - quit "could not finalize " & $sqlite3.errmsg(dbHandle) - else: - quit "could not prepare statement " & $sqlite3.errmsg(dbHandle) - -proc setupDb() = - execQuery """create table if not exists tracking( - id integer primary key, - op varchar not null, - address integer not null, - size integer not null, - file varchar not null, - line integer not null - )""" - execQuery "delete from tracking" - -if sqlite3.open("memtrack.db", dbHandle) == SQLITE_OK: - setupDb() - const query = "INSERT INTO tracking(op, address, size, file, line) values (?, ?, ?, ?, ?)" - if prepare_v2(dbHandle, insertQuery, - insertQuery.len, insertStmt, nil) == SQLITE_OK: - when defined(memTracker): - setTrackLogger logEntries - else: - quit "could not prepare statement B " & $sqlite3.errmsg(dbHandle) -else: - quit "could not setup sqlite " & $sqlite3.errmsg(dbHandle) -{.pop.} diff --git a/lib/pure/oids.nim b/lib/pure/oids.nim index fb70047b6..4d6ceefd7 100644 --- a/lib/pure/oids.nim +++ b/lib/pure/oids.nim @@ -9,8 +9,7 @@ ## Nim OID support. An OID is a global ID that consists of a timestamp, ## a unique counter and a random value. This combination should suffice to -## produce a globally distributed unique ID. This implementation was extracted -## from the MongoDB interface and is thus binary compatible with a MongoDB OID. +## produce a globally distributed unique ID. ## ## This implementation calls `initRand()` for the first call of ## `genOid`. @@ -18,9 +17,12 @@ import std/[hashes, times, endians, random] from std/private/decode_helpers import handleHexChar +when defined(nimPreviewSlimSystem): + import std/sysatomics + type Oid* = object ## An OID. - time: int32 + time: int64 fuzz: int32 count: int32 @@ -42,44 +44,29 @@ proc hexbyte*(hex: char): int {.inline.} = proc parseOid*(str: cstring): Oid = ## Parses an OID. - var bytes = cast[cstring](addr(result.time)) + var bytes = cast[cstring](cast[pointer](cast[int](addr(result.time)) + 4)) var i = 0 while i < 12: bytes[i] = chr((hexbyte(str[2 * i]) shl 4) or hexbyte(str[2 * i + 1])) inc(i) -template toStringImpl[T: string | cstring](result: var T, oid: Oid) = - ## Stringifies `oid`. +proc `$`*(oid: Oid): string = + ## Converts an OID to a string. const hex = "0123456789abcdef" - const N = 24 - when T is string: - result.setLen N + result.setLen 24 var o = oid - var bytes = cast[cstring](addr(o)) + var bytes = cast[cstring](cast[pointer](cast[int](addr(o)) + 4)) var i = 0 while i < 12: let b = bytes[i].ord result[2 * i] = hex[(b and 0xF0) shr 4] result[2 * i + 1] = hex[b and 0xF] inc(i) - when T is cstring: - result[N] = '\0' - -proc oidToString*(oid: Oid, str: cstring) {.deprecated: "unsafe; use `$`".} = - ## Converts an oid to a string which must have space allocated for 25 elements. - # work around a compiler bug: - var str = str - toStringImpl(str, oid) - -proc `$`*(oid: Oid): string = - ## Converts an OID to a string. - toStringImpl(result, oid) - let - t = getTime().toUnix.int32 + t = getTime().toUnix var seed = initRand(t) @@ -89,10 +76,10 @@ let fuzz = cast[int32](seed.rand(high(int))) template genOid(result: var Oid, incr: var int, fuzz: int32) = - var time = getTime().toUnix.int32 + var time = getTime().toUnix var i = cast[int32](atomicInc(incr)) - bigEndian32(addr result.time, addr(time)) + bigEndian64(addr result.time, addr(time)) result.fuzz = fuzz bigEndian32(addr result.count, addr(i)) @@ -106,7 +93,7 @@ proc genOid*(): Oid = proc generatedTime*(oid: Oid): Time = ## Returns the generated timestamp of the OID. - var tmp: int32 + var tmp: int64 var dummy = oid.time - bigEndian32(addr(tmp), addr(dummy)) + bigEndian64(addr(tmp), addr(dummy)) result = fromUnix(tmp) diff --git a/lib/pure/options.nim b/lib/pure/options.nim index 63e3598f0..b34ff72c0 100644 --- a/lib/pure/options.nim +++ b/lib/pure/options.nim @@ -53,7 +53,7 @@ Pattern matching supports pattern matching on `Option`s, with the `Some(<pattern>)` and `None()` patterns. -.. code-block:: nim + ```nim {.experimental: "caseStmtMacros".} import fusion/matching @@ -65,15 +65,24 @@ supports pattern matching on `Option`s, with the `Some(<pattern>)` and assert false assertMatch(some(some(none(int))), Some(Some(None()))) + ``` ]## # xxx pending https://github.com/timotheecour/Nim/issues/376 use `runnableExamples` and `whichModule` +when defined(nimHasEffectsOf): + {.experimental: "strictEffects".} +else: + {.pragma: effectsOf.} import std/typetraits +when defined(nimPreviewSlimSystem): + import std/assertions + + when (NimMajor, NimMinor) >= (1, 1): type - SomePointer = ref | ptr | pointer | proc + SomePointer = ref | ptr | pointer | proc | iterator {.closure.} else: type SomePointer = ref | ptr | pointer @@ -81,7 +90,7 @@ else: type Option*[T] = object ## An optional type that may or may not contain a value of type `T`. - ## When `T` is a a pointer type (`ptr`, `pointer`, `ref` or `proc`), + ## When `T` is a a pointer type (`ptr`, `pointer`, `ref`, `proc` or `iterator {.closure.}`), ## `none(T)` is represented as `nil`. when T is SomePointer: val: T @@ -92,7 +101,7 @@ type UnpackDefect* = object of Defect UnpackError* {.deprecated: "See corresponding Defect".} = UnpackDefect -proc option*[T](val: T): Option[T] {.inline.} = +proc option*[T](val: sink T): Option[T] {.inline.} = ## Can be used to convert a pointer type (`ptr`, `pointer`, `ref` or `proc`) to an option type. ## It converts `nil` to `none(T)`. When `T` is no pointer type, this is equivalent to `some(val)`. ## @@ -108,11 +117,12 @@ proc option*[T](val: T): Option[T] {.inline.} = assert option[Foo](nil).isNone assert option(42).isSome - result.val = val - when T isnot SomePointer: - result.has = true + when T is SomePointer: + result = Option[T](val: val) + else: + result = Option[T](has: true, val: val) -proc some*[T](val: T): Option[T] {.inline.} = +proc some*[T](val: sink T): Option[T] {.inline.} = ## Returns an `Option` that has the value `val`. ## ## **See also:** @@ -127,10 +137,9 @@ proc some*[T](val: T): Option[T] {.inline.} = when T is SomePointer: assert not val.isNil - result.val = val + result = Option[T](val: val) else: - result.has = true - result.val = val + result = Option[T](has: true, val: val) proc none*(T: typedesc): Option[T] {.inline.} = ## Returns an `Option` for this type that has no value. @@ -143,7 +152,7 @@ proc none*(T: typedesc): Option[T] {.inline.} = assert none(int).isNone # the default is the none type - discard + result = Option[T]() proc none*[T]: Option[T] {.inline.} = ## Alias for `none(T) <#none,typedesc>`_. @@ -222,7 +231,7 @@ proc get*[T](self: var Option[T]): var T {.inline.} = raise newException(UnpackDefect, "Can't obtain a value from a `none`") return self.val -proc map*[T](self: Option[T], callback: proc (input: T)) {.inline.} = +proc map*[T](self: Option[T], callback: proc (input: T)) {.inline, effectsOf: callback.} = ## Applies a `callback` function to the value of the `Option`, if it has one. ## ## **See also:** @@ -241,7 +250,7 @@ proc map*[T](self: Option[T], callback: proc (input: T)) {.inline.} = if self.isSome: callback(self.val) -proc map*[T, R](self: Option[T], callback: proc (input: T): R): Option[R] {.inline.} = +proc map*[T, R](self: Option[T], callback: proc (input: T): R): Option[R] {.inline, effectsOf: callback.} = ## Applies a `callback` function to the value of the `Option` and returns an ## `Option` containing the new value. ## @@ -278,7 +287,7 @@ proc flatten*[T](self: Option[Option[T]]): Option[T] {.inline.} = none(T) proc flatMap*[T, R](self: Option[T], - callback: proc (input: T): Option[R]): Option[R] {.inline.} = + callback: proc (input: T): Option[R]): Option[R] {.inline, effectsOf: callback.} = ## Applies a `callback` function to the value of the `Option` and returns the new value. ## ## If the `Option` has no value, `none(R)` will be returned. @@ -303,7 +312,7 @@ proc flatMap*[T, R](self: Option[T], map(self, callback).flatten() -proc filter*[T](self: Option[T], callback: proc (input: T): bool): Option[T] {.inline.} = +proc filter*[T](self: Option[T], callback: proc (input: T): bool): Option[T] {.inline, effectsOf: callback.} = ## Applies a `callback` to the value of the `Option`. ## ## If the `callback` returns `true`, the option is returned as `some`. diff --git a/lib/pure/os.nim b/lib/pure/os.nim index d0b3aef1a..78ebb1c88 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -8,51 +8,62 @@ # ## This module contains basic operating system facilities like -## retrieving environment variables, reading command line arguments, -## working with directories, running shell commands, etc. -## -## .. code-block:: -## import std/os -## -## let myFile = "/path/to/my/file.nim" -## -## let pathSplit = splitPath(myFile) -## assert pathSplit.head == "/path/to/my" -## assert pathSplit.tail == "file.nim" -## -## assert parentDir(myFile) == "/path/to/my" -## -## let fileSplit = splitFile(myFile) -## assert fileSplit.dir == "/path/to/my" -## assert fileSplit.name == "file" -## assert fileSplit.ext == ".nim" -## -## assert myFile.changeFileExt("c") == "/path/to/my/file.c" - -## -## +## retrieving environment variables, working with directories, +## running shell commands, etc. + +## .. importdoc:: symlinks.nim, appdirs.nim, dirs.nim, ospaths2.nim + +runnableExamples: + let myFile = "/path/to/my/file.nim" + assert splitPath(myFile) == (head: "/path/to/my", tail: "file.nim") + when defined(posix): + assert parentDir(myFile) == "/path/to/my" + assert splitFile(myFile) == (dir: "/path/to/my", name: "file", ext: ".nim") + assert myFile.changeFileExt("c") == "/path/to/my/file.c" + ## **See also:** +## * `paths <paths.html>`_ and `files <files.html>`_ modules for high-level file manipulation ## * `osproc module <osproc.html>`_ for process communication beyond -## `execShellCmd proc <#execShellCmd,string>`_ -## * `parseopt module <parseopt.html>`_ for command-line parser beyond -## `parseCmdLine proc <#parseCmdLine,string>`_ +## `execShellCmd proc`_ ## * `uri module <uri.html>`_ ## * `distros module <distros.html>`_ ## * `dynlib module <dynlib.html>`_ ## * `streams module <streams.html>`_ +import std/private/ospaths2 +export ospaths2 + +import std/private/osfiles +export osfiles + +import std/private/osdirs +export osdirs + +import std/private/ossymlinks +export ossymlinks + +import std/private/osappdirs +export osappdirs + +import std/private/oscommon include system/inclrtl import std/private/since +import std/cmdline +export cmdline + import std/[strutils, pathnorm] +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions, widestrs] + const weirdTarget = defined(nimscript) or defined(js) since (1, 1): const invalidFilenameChars* = {'/', '\\', ':', '*', '?', '"', '<', '>', '|', '^', '\0'} ## \ - ## Characters that may produce invalid filenames across Linux, Windows, Mac, etc. - ## You can check if your filename contains these char and strip them for safety. + ## Characters that may produce invalid filenames across Linux, Windows and Mac. + ## You can check if your filename contains any of these chars and strip them for safety. ## Mac bans ``':'``, Linux bans ``'/'``, Windows bans all others. invalidFilenames* = [ "CON", "PRN", "AUX", "NUL", @@ -87,943 +98,35 @@ elif defined(js): else: {.pragma: noNimJs.} -proc normalizePathAux(path: var string){.inline, raises: [], noSideEffect.} - -type - ReadEnvEffect* = object of ReadIOEffect ## Effect that denotes a read - ## from an environment variable. - WriteEnvEffect* = object of WriteIOEffect ## Effect that denotes a write - ## to an environment variable. - - ReadDirEffect* = object of ReadIOEffect ## Effect that denotes a read - ## operation from the directory - ## structure. - WriteDirEffect* = object of WriteIOEffect ## Effect that denotes a write - ## operation to - ## the directory structure. - - OSErrorCode* = distinct int32 ## Specifies an OS Error Code. - -include "includes/osseps" - -proc absolutePathInternal(path: string): string {.gcsafe.} - -proc normalizePathEnd(path: var string, trailingSep = false) = - ## Ensures ``path`` has exactly 0 or 1 trailing `DirSep`, depending on - ## ``trailingSep``, and taking care of edge cases: it preservers whether - ## a path is absolute or relative, and makes sure trailing sep is `DirSep`, - ## not `AltSep`. Trailing `/.` are compressed, see examples. - if path.len == 0: return - var i = path.len - while i >= 1: - if path[i-1] in {DirSep, AltSep}: dec(i) - elif path[i-1] == '.' and i >= 2 and path[i-2] in {DirSep, AltSep}: dec(i) - else: break - if trailingSep: - # foo// => foo - path.setLen(i) - # foo => foo/ - path.add DirSep - elif i > 0: - # foo// => foo - path.setLen(i) - else: - # // => / (empty case was already taken care of) - path = $DirSep - -proc normalizePathEnd(path: string, trailingSep = false): string = - ## outplace overload - runnableExamples: - when defined(posix): - assert normalizePathEnd("/lib//.//", trailingSep = true) == "/lib/" - assert normalizePathEnd("lib/./.", trailingSep = false) == "lib" - assert normalizePathEnd(".//./.", trailingSep = false) == "." - assert normalizePathEnd("", trailingSep = true) == "" # not / ! - assert normalizePathEnd("/", trailingSep = false) == "/" # not "" ! - result = path - result.normalizePathEnd(trailingSep) - -since((1, 1)): - export normalizePathEnd - -template endsWith(a: string, b: set[char]): bool = - a.len > 0 and a[^1] in b - -proc joinPathImpl(result: var string, state: var int, tail: string) = - let trailingSep = tail.endsWith({DirSep, AltSep}) or tail.len == 0 and result.endsWith({DirSep, AltSep}) - normalizePathEnd(result, trailingSep=false) - addNormalizePath(tail, result, state, DirSep) - normalizePathEnd(result, trailingSep=trailingSep) - -proc joinPath*(head, tail: string): string {. - noSideEffect, rtl, extern: "nos$1".} = - ## Joins two directory names to one. - ## - ## returns normalized path concatenation of `head` and `tail`, preserving - ## whether or not `tail` has a trailing slash (or, if tail if empty, whether - ## head has one). - ## - ## See also: - ## * `joinPath(varargs) proc <#joinPath,varargs[string]>`_ - ## * `/ proc <#/,string,string>`_ - ## * `splitPath proc <#splitPath,string>`_ - ## * `uri.combine proc <uri.html#combine,Uri,Uri>`_ - ## * `uri./ proc <uri.html#/,Uri,string>`_ - runnableExamples: - when defined(posix): - assert joinPath("usr", "lib") == "usr/lib" - assert joinPath("usr", "lib/") == "usr/lib/" - assert joinPath("usr", "") == "usr" - assert joinPath("usr/", "") == "usr/" - assert joinPath("", "") == "" - assert joinPath("", "lib") == "lib" - assert joinPath("", "/lib") == "/lib" - assert joinPath("usr/", "/lib") == "usr/lib" - assert joinPath("usr/lib", "../bin") == "usr/bin" - - result = newStringOfCap(head.len + tail.len) - var state = 0 - joinPathImpl(result, state, head) - joinPathImpl(result, state, tail) - when false: - if len(head) == 0: - result = tail - elif head[len(head)-1] in {DirSep, AltSep}: - if tail.len > 0 and tail[0] in {DirSep, AltSep}: - result = head & substr(tail, 1) - else: - result = head & tail - else: - if tail.len > 0 and tail[0] in {DirSep, AltSep}: - result = head & tail - else: - result = head & DirSep & tail - -proc joinPath*(parts: varargs[string]): string {.noSideEffect, - rtl, extern: "nos$1OpenArray".} = - ## The same as `joinPath(head, tail) proc <#joinPath,string,string>`_, - ## but works with any number of directory parts. - ## - ## You need to pass at least one element or the proc - ## will assert in debug builds and crash on release builds. - ## - ## See also: - ## * `joinPath(head, tail) proc <#joinPath,string,string>`_ - ## * `/ proc <#/,string,string>`_ - ## * `/../ proc <#/../,string,string>`_ - ## * `splitPath proc <#splitPath,string>`_ - runnableExamples: - when defined(posix): - assert joinPath("a") == "a" - assert joinPath("a", "b", "c") == "a/b/c" - assert joinPath("usr/lib", "../../var", "log") == "var/log" - - var estimatedLen = 0 - for p in parts: estimatedLen += p.len - result = newStringOfCap(estimatedLen) - var state = 0 - for i in 0..high(parts): - joinPathImpl(result, state, parts[i]) - -proc `/`*(head, tail: string): string {.noSideEffect, inline.} = - ## The same as `joinPath(head, tail) proc <#joinPath,string,string>`_. - ## - ## See also: - ## * `/../ proc <#/../,string,string>`_ - ## * `joinPath(head, tail) proc <#joinPath,string,string>`_ - ## * `joinPath(varargs) proc <#joinPath,varargs[string]>`_ - ## * `splitPath proc <#splitPath,string>`_ - ## * `uri.combine proc <uri.html#combine,Uri,Uri>`_ - ## * `uri./ proc <uri.html#/,Uri,string>`_ - runnableExamples: - when defined(posix): - assert "usr" / "" == "usr" - assert "" / "lib" == "lib" - assert "" / "/lib" == "/lib" - assert "usr/" / "/lib/" == "usr/lib/" - assert "usr" / "lib" / "../bin" == "usr/bin" - - result = joinPath(head, tail) - -proc splitPath*(path: string): tuple[head, tail: string] {. - noSideEffect, rtl, extern: "nos$1".} = - ## Splits a directory into `(head, tail)` tuple, so that - ## ``head / tail == path`` (except for edge cases like "/usr"). - ## - ## See also: - ## * `joinPath(head, tail) proc <#joinPath,string,string>`_ - ## * `joinPath(varargs) proc <#joinPath,varargs[string]>`_ - ## * `/ proc <#/,string,string>`_ - ## * `/../ proc <#/../,string,string>`_ - ## * `relativePath proc <#relativePath,string,string>`_ - runnableExamples: - assert splitPath("usr/local/bin") == ("usr/local", "bin") - assert splitPath("usr/local/bin/") == ("usr/local/bin", "") - assert splitPath("/bin/") == ("/bin", "") - when (NimMajor, NimMinor) <= (1, 0): - assert splitPath("/bin") == ("", "bin") - else: - assert splitPath("/bin") == ("/", "bin") - assert splitPath("bin") == ("", "bin") - assert splitPath("") == ("", "") - - var sepPos = -1 - for i in countdown(len(path)-1, 0): - if path[i] in {DirSep, AltSep}: - sepPos = i - break - if sepPos >= 0: - result.head = substr(path, 0, - when (NimMajor, NimMinor) <= (1, 0): - sepPos-1 - else: - if likely(sepPos >= 1): sepPos-1 else: 0 - ) - result.tail = substr(path, sepPos+1) - else: - result.head = "" - result.tail = path - -proc isAbsolute*(path: string): bool {.rtl, noSideEffect, extern: "nos$1", raises: [].} = - ## Checks whether a given `path` is absolute. - ## - ## On Windows, network paths are considered absolute too. - runnableExamples: - assert not "".isAbsolute - assert not ".".isAbsolute - when defined(posix): - assert "/".isAbsolute - assert not "a/".isAbsolute - assert "/a/".isAbsolute - - if len(path) == 0: return false - - when doslikeFileSystem: - var len = len(path) - result = (path[0] in {'/', '\\'}) or - (len > 1 and path[0] in {'a'..'z', 'A'..'Z'} and path[1] == ':') - elif defined(macos): - # according to https://perldoc.perl.org/File/Spec/Mac.html `:a` is a relative path - result = path[0] != ':' - elif defined(RISCOS): - result = path[0] == '$' - elif defined(posix) or defined(js): - # `or defined(js)` wouldn't be needed pending https://github.com/nim-lang/Nim/issues/13469 - # This works around the problem for posix, but windows is still broken with nim js -d:nodejs - result = path[0] == '/' - else: - doAssert false # if ever hits here, adapt as needed - -when FileSystemCaseSensitive: - template `!=?`(a, b: char): bool = a != b -else: - template `!=?`(a, b: char): bool = toLowerAscii(a) != toLowerAscii(b) - -when doslikeFileSystem: - proc isAbsFromCurrentDrive(path: string): bool {.noSideEffect, raises: []} = - ## An absolute path from the root of the current drive (e.g. "\foo") - path.len > 0 and - (path[0] == AltSep or - (path[0] == DirSep and - (path.len == 1 or path[1] notin {DirSep, AltSep, ':'}))) - - proc isUNCPrefix(path: string): bool {.noSideEffect, raises: []} = - path[0] == DirSep and path[1] == DirSep - - proc sameRoot(path1, path2: string): bool {.noSideEffect, raises: []} = - ## Return true if path1 and path2 have a same root. - ## - ## Detail of windows path formats: - ## https://docs.microsoft.com/en-us/dotnet/standard/io/file-path-formats - - assert(isAbsolute(path1)) - assert(isAbsolute(path2)) - - let - len1 = path1.len - len2 = path2.len - assert(len1 != 0 and len2 != 0) - - if isAbsFromCurrentDrive(path1) and isAbsFromCurrentDrive(path2): - return true - elif len1 == 1 or len2 == 1: - return false - else: - if path1[1] == ':' and path2[1] == ':': - return path1[0].toLowerAscii() == path2[0].toLowerAscii() - else: - var - p1, p2: PathIter - pp1 = next(p1, path1) - pp2 = next(p2, path2) - if pp1[1] - pp1[0] == 1 and pp2[1] - pp2[0] == 1 and - isUNCPrefix(path1) and isUNCPrefix(path2): - #UNC - var h = 0 - while p1.hasNext(path1) and p2.hasNext(path2) and h < 2: - pp1 = next(p1, path1) - pp2 = next(p2, path2) - let diff = pp1[1] - pp1[0] - if diff != pp2[1] - pp2[0]: - return false - for i in 0..diff: - if path1[i + pp1[0]] !=? path2[i + pp2[0]]: - return false - inc h - return h == 2 - else: - return false - -proc relativePath*(path, base: string, sep = DirSep): string {. - rtl, extern: "nos$1".} = - ## Converts `path` to a path relative to `base`. - ## - ## The `sep` (default: `DirSep <#DirSep>`_) is used for the path normalizations, - ## this can be useful to ensure the relative path only contains `'/'` - ## so that it can be used for URL constructions. - ## - ## On windows, if a root of `path` and a root of `base` are different, - ## returns `path` as is because it is impossible to make a relative path. - ## That means an absolute path can be returned. - ## - ## See also: - ## * `splitPath proc <#splitPath,string>`_ - ## * `parentDir proc <#parentDir,string>`_ - ## * `tailDir proc <#tailDir,string>`_ - runnableExamples: - assert relativePath("/Users/me/bar/z.nim", "/Users/other/bad", '/') == "../../me/bar/z.nim" - assert relativePath("/Users/me/bar/z.nim", "/Users/other", '/') == "../me/bar/z.nim" - assert relativePath("/Users///me/bar//z.nim", "//Users/", '/') == "me/bar/z.nim" - assert relativePath("/Users/me/bar/z.nim", "/Users/me", '/') == "bar/z.nim" - assert relativePath("", "/users/moo", '/') == "" - assert relativePath("foo", ".", '/') == "foo" - assert relativePath("foo", "foo", '/') == "." - - if path.len == 0: return "" - var base = if base == ".": "" else: base - var path = path - path.normalizePathAux - base.normalizePathAux - let a1 = isAbsolute(path) - let a2 = isAbsolute(base) - if a1 and not a2: - base = absolutePathInternal(base) - elif a2 and not a1: - path = absolutePathInternal(path) - - when doslikeFileSystem: - if isAbsolute(path) and isAbsolute(base): - if not sameRoot(path, base): - return path - - var f = default PathIter - var b = default PathIter - var ff = (0, -1) - var bb = (0, -1) # (int, int) - result = newStringOfCap(path.len) - # skip the common prefix: - while f.hasNext(path) and b.hasNext(base): - ff = next(f, path) - bb = next(b, base) - let diff = ff[1] - ff[0] - if diff != bb[1] - bb[0]: break - var same = true - for i in 0..diff: - if path[i + ff[0]] !=? base[i + bb[0]]: - same = false - break - if not same: break - ff = (0, -1) - bb = (0, -1) - # for i in 0..diff: - # result.add base[i + bb[0]] - - # /foo/bar/xxx/ -- base - # /foo/bar/baz -- path path - # ../baz - # every directory that is in 'base', needs to add '..' - while true: - if bb[1] >= bb[0]: - if result.len > 0 and result[^1] != sep: - result.add sep - result.add ".." - if not b.hasNext(base): break - bb = b.next(base) - - # add the rest of 'path': - while true: - if ff[1] >= ff[0]: - if result.len > 0 and result[^1] != sep: - result.add sep - for i in 0..ff[1] - ff[0]: - result.add path[i + ff[0]] - if not f.hasNext(path): break - ff = f.next(path) - - when not defined(nimOldRelativePathBehavior): - if result.len == 0: result.add "." - -proc isRelativeTo*(path: string, base: string): bool {.since: (1, 1).} = - ## Returns true if `path` is relative to `base`. - runnableExamples: - doAssert isRelativeTo("./foo//bar", "foo") - doAssert isRelativeTo("foo/bar", ".") - doAssert isRelativeTo("/foo/bar.nim", "/foo/bar.nim") - doAssert not isRelativeTo("foo/bar.nims", "foo/bar.nim") - let path = path.normalizePath - let base = base.normalizePath - let ret = relativePath(path, base) - result = path.len > 0 and not ret.startsWith ".." - -proc parentDirPos(path: string): int = - var q = 1 - if len(path) >= 1 and path[len(path)-1] in {DirSep, AltSep}: q = 2 - for i in countdown(len(path)-q, 0): - if path[i] in {DirSep, AltSep}: return i - result = -1 - -proc parentDir*(path: string): string {. - noSideEffect, rtl, extern: "nos$1".} = - ## Returns the parent directory of `path`. - ## - ## This is similar to ``splitPath(path).head`` when ``path`` doesn't end - ## in a dir separator, but also takes care of path normalizations. - ## The remainder can be obtained with `lastPathPart(path) proc - ## <#lastPathPart,string>`_. - ## - ## See also: - ## * `relativePath proc <#relativePath,string,string>`_ - ## * `splitPath proc <#splitPath,string>`_ - ## * `tailDir proc <#tailDir,string>`_ - ## * `parentDirs iterator <#parentDirs.i,string>`_ - runnableExamples: - assert parentDir("") == "" - when defined(posix): - assert parentDir("/usr/local/bin") == "/usr/local" - assert parentDir("foo/bar//") == "foo" - assert parentDir("//foo//bar//.") == "/foo" - assert parentDir("./foo") == "." - assert parentDir("/./foo//./") == "/" - assert parentDir("a//./") == "." - assert parentDir("a/b/c/..") == "a" - result = pathnorm.normalizePath(path) - var sepPos = parentDirPos(result) - if sepPos >= 0: - result = substr(result, 0, sepPos) - normalizePathEnd(result) - elif result == ".." or result == "." or result.len == 0 or result[^1] in {DirSep, AltSep}: - # `.` => `..` and .. => `../..`(etc) would be a sensible alternative - # `/` => `/` (as done with splitFile) would be a sensible alternative - result = "" - else: - result = "." - -proc tailDir*(path: string): string {. - noSideEffect, rtl, extern: "nos$1".} = - ## Returns the tail part of `path`. - ## - ## See also: - ## * `relativePath proc <#relativePath,string,string>`_ - ## * `splitPath proc <#splitPath,string>`_ - ## * `parentDir proc <#parentDir,string>`_ - runnableExamples: - assert tailDir("/bin") == "bin" - assert tailDir("bin") == "" - assert tailDir("bin/") == "" - assert tailDir("/usr/local/bin") == "usr/local/bin" - assert tailDir("//usr//local//bin//") == "usr//local//bin//" - assert tailDir("./usr/local/bin") == "usr/local/bin" - assert tailDir("usr/local/bin") == "local/bin" - - var i = 0 - while i < len(path): - if path[i] in {DirSep, AltSep}: - while i < len(path) and path[i] in {DirSep, AltSep}: inc i - return substr(path, i) - inc i - result = "" - -proc isRootDir*(path: string): bool {. - noSideEffect, rtl, extern: "nos$1".} = - ## Checks whether a given `path` is a root directory. - runnableExamples: - assert isRootDir("") - assert isRootDir(".") - assert isRootDir("/") - assert isRootDir("a") - assert not isRootDir("/a") - assert not isRootDir("a/b/c") - - result = parentDirPos(path) < 0 - -iterator parentDirs*(path: string, fromRoot=false, inclusive=true): string = - ## Walks over all parent directories of a given `path`. - ## - ## If `fromRoot` is true (default: false), the traversal will start from - ## the file system root directory. - ## If `inclusive` is true (default), the original argument will be included - ## in the traversal. - ## - ## Relative paths won't be expanded by this iterator. Instead, it will traverse - ## only the directories appearing in the relative path. - ## - ## See also: - ## * `parentDir proc <#parentDir,string>`_ - ## - ## **Examples:** - ## - ## .. code-block:: - ## let g = "a/b/c" - ## - ## for p in g.parentDirs: - ## echo p - ## # a/b/c - ## # a/b - ## # a - ## - ## for p in g.parentDirs(fromRoot=true): - ## echo p - ## # a/ - ## # a/b/ - ## # a/b/c - ## - ## for p in g.parentDirs(inclusive=false): - ## echo p - ## # a/b - ## # a - - if not fromRoot: - var current = path - if inclusive: yield path - while true: - if current.isRootDir: break - current = current.parentDir - yield current - else: - for i in countup(0, path.len - 2): # ignore the last / - # deal with non-normalized paths such as /foo//bar//baz - if path[i] in {DirSep, AltSep} and - (i == 0 or path[i-1] notin {DirSep, AltSep}): - yield path.substr(0, i) - - if inclusive: yield path - -proc `/../`*(head, tail: string): string {.noSideEffect.} = - ## The same as ``parentDir(head) / tail``, unless there is no parent - ## directory. Then ``head / tail`` is performed instead. - ## - ## See also: - ## * `/ proc <#/,string,string>`_ - ## * `parentDir proc <#parentDir,string>`_ - runnableExamples: - when defined(posix): - assert "a/b/c" /../ "d/e" == "a/b/d/e" - assert "a" /../ "d/e" == "a/d/e" - - let sepPos = parentDirPos(head) - if sepPos >= 0: - result = substr(head, 0, sepPos-1) / tail - else: - result = head / tail - -proc normExt(ext: string): string = - if ext == "" or ext[0] == ExtSep: result = ext # no copy needed here - else: result = ExtSep & ext - -proc searchExtPos*(path: string): int = - ## Returns index of the `'.'` char in `path` if it signifies the beginning - ## of extension. Returns -1 otherwise. - ## - ## See also: - ## * `splitFile proc <#splitFile,string>`_ - ## * `extractFilename proc <#extractFilename,string>`_ - ## * `lastPathPart proc <#lastPathPart,string>`_ - ## * `changeFileExt proc <#changeFileExt,string,string>`_ - ## * `addFileExt proc <#addFileExt,string,string>`_ - runnableExamples: - assert searchExtPos("a/b/c") == -1 - assert searchExtPos("c.nim") == 1 - assert searchExtPos("a/b/c.nim") == 5 - assert searchExtPos("a.b.c.nim") == 5 - - # BUGFIX: do not search until 0! .DS_Store is no file extension! - result = -1 - for i in countdown(len(path)-1, 1): - if path[i] == ExtSep: - result = i - break - elif path[i] in {DirSep, AltSep}: - break # do not skip over path - -proc splitFile*(path: string): tuple[dir, name, ext: string] {. - noSideEffect, rtl, extern: "nos$1".} = - ## Splits a filename into `(dir, name, extension)` tuple. - ## - ## `dir` does not end in `DirSep <#DirSep>`_ unless it's `/`. - ## `extension` includes the leading dot. - ## - ## If `path` has no extension, `ext` is the empty string. - ## If `path` has no directory component, `dir` is the empty string. - ## If `path` has no filename component, `name` and `ext` are empty strings. - ## - ## See also: - ## * `searchExtPos proc <#searchExtPos,string>`_ - ## * `extractFilename proc <#extractFilename,string>`_ - ## * `lastPathPart proc <#lastPathPart,string>`_ - ## * `changeFileExt proc <#changeFileExt,string,string>`_ - ## * `addFileExt proc <#addFileExt,string,string>`_ - runnableExamples: - var (dir, name, ext) = splitFile("usr/local/nimc.html") - assert dir == "usr/local" - assert name == "nimc" - assert ext == ".html" - (dir, name, ext) = splitFile("/usr/local/os") - assert dir == "/usr/local" - assert name == "os" - assert ext == "" - (dir, name, ext) = splitFile("/usr/local/") - assert dir == "/usr/local" - assert name == "" - assert ext == "" - (dir, name, ext) = splitFile("/tmp.txt") - assert dir == "/" - assert name == "tmp" - assert ext == ".txt" - - var namePos = 0 - var dotPos = 0 - for i in countdown(len(path) - 1, 0): - if path[i] in {DirSep, AltSep} or i == 0: - if path[i] in {DirSep, AltSep}: - result.dir = substr(path, 0, if likely(i >= 1): i - 1 else: 0) - namePos = i + 1 - if dotPos > i: - result.name = substr(path, namePos, dotPos - 1) - result.ext = substr(path, dotPos) - else: - result.name = substr(path, namePos) - break - elif path[i] == ExtSep and i > 0 and i < len(path) - 1 and - path[i - 1] notin {DirSep, AltSep} and - path[i + 1] != ExtSep and dotPos == 0: - dotPos = i - -proc extractFilename*(path: string): string {. - noSideEffect, rtl, extern: "nos$1".} = - ## Extracts the filename of a given `path`. - ## - ## This is the same as ``name & ext`` from `splitFile(path) proc - ## <#splitFile,string>`_. - ## - ## See also: - ## * `searchExtPos proc <#searchExtPos,string>`_ - ## * `splitFile proc <#splitFile,string>`_ - ## * `lastPathPart proc <#lastPathPart,string>`_ - ## * `changeFileExt proc <#changeFileExt,string,string>`_ - ## * `addFileExt proc <#addFileExt,string,string>`_ - runnableExamples: - assert extractFilename("foo/bar/") == "" - assert extractFilename("foo/bar") == "bar" - assert extractFilename("foo/bar.baz") == "bar.baz" - - if path.len == 0 or path[path.len-1] in {DirSep, AltSep}: - result = "" - else: - result = splitPath(path).tail - -proc lastPathPart*(path: string): string {. - noSideEffect, rtl, extern: "nos$1".} = - ## Like `extractFilename proc <#extractFilename,string>`_, but ignores - ## trailing dir separator; aka: `baseName`:idx: in some other languages. - ## - ## See also: - ## * `searchExtPos proc <#searchExtPos,string>`_ - ## * `splitFile proc <#splitFile,string>`_ - ## * `extractFilename proc <#extractFilename,string>`_ - ## * `changeFileExt proc <#changeFileExt,string,string>`_ - ## * `addFileExt proc <#addFileExt,string,string>`_ - runnableExamples: - assert lastPathPart("foo/bar/") == "bar" - assert lastPathPart("foo/bar") == "bar" - - let path = path.normalizePathEnd(trailingSep = false) - result = extractFilename(path) - -proc changeFileExt*(filename, ext: string): string {. - noSideEffect, rtl, extern: "nos$1".} = - ## Changes the file extension to `ext`. - ## - ## If the `filename` has no extension, `ext` will be added. - ## If `ext` == "" then any extension is removed. - ## - ## `Ext` should be given without the leading `'.'`, because some - ## filesystems may use a different character. (Although I know - ## of none such beast.) - ## - ## See also: - ## * `searchExtPos proc <#searchExtPos,string>`_ - ## * `splitFile proc <#splitFile,string>`_ - ## * `extractFilename proc <#extractFilename,string>`_ - ## * `lastPathPart proc <#lastPathPart,string>`_ - ## * `addFileExt proc <#addFileExt,string,string>`_ - runnableExamples: - assert changeFileExt("foo.bar", "baz") == "foo.baz" - assert changeFileExt("foo.bar", "") == "foo" - assert changeFileExt("foo", "baz") == "foo.baz" - - var extPos = searchExtPos(filename) - if extPos < 0: result = filename & normExt(ext) - else: result = substr(filename, 0, extPos-1) & normExt(ext) - -proc addFileExt*(filename, ext: string): string {. - noSideEffect, rtl, extern: "nos$1".} = - ## Adds the file extension `ext` to `filename`, unless - ## `filename` already has an extension. - ## - ## `Ext` should be given without the leading `'.'`, because some - ## filesystems may use a different character. - ## (Although I know of none such beast.) - ## - ## See also: - ## * `searchExtPos proc <#searchExtPos,string>`_ - ## * `splitFile proc <#splitFile,string>`_ - ## * `extractFilename proc <#extractFilename,string>`_ - ## * `lastPathPart proc <#lastPathPart,string>`_ - ## * `changeFileExt proc <#changeFileExt,string,string>`_ - runnableExamples: - assert addFileExt("foo.bar", "baz") == "foo.bar" - assert addFileExt("foo.bar", "") == "foo.bar" - assert addFileExt("foo", "baz") == "foo.baz" - - var extPos = searchExtPos(filename) - if extPos < 0: result = filename & normExt(ext) - else: result = filename -proc cmpPaths*(pathA, pathB: string): int {. - noSideEffect, rtl, extern: "nos$1".} = - ## Compares two paths. - ## - ## On a case-sensitive filesystem this is done - ## case-sensitively otherwise case-insensitively. Returns: - ## - ## | 0 if pathA == pathB - ## | < 0 if pathA < pathB - ## | > 0 if pathA > pathB - runnableExamples: - when defined(macosx): - assert cmpPaths("foo", "Foo") == 0 - elif defined(posix): - assert cmpPaths("foo", "Foo") > 0 - - let a = normalizePath(pathA) - let b = normalizePath(pathB) - if FileSystemCaseSensitive: - result = cmp(a, b) - else: - when defined(nimscript): - result = cmpic(a, b) - elif defined(nimdoc): discard - else: - result = cmpIgnoreCase(a, b) +import std/oserrors +export oserrors +import std/envvars +export envvars -proc unixToNativePath*(path: string, drive=""): string {. - noSideEffect, rtl, extern: "nos$1".} = - ## Converts an UNIX-like path to a native one. - ## - ## On an UNIX system this does nothing. Else it converts - ## `'/'`, `'.'`, `'..'` to the appropriate things. - ## - ## On systems with a concept of "drives", `drive` is used to determine - ## which drive label to use during absolute path conversion. - ## `drive` defaults to the drive of the current working directory, and is - ## ignored on systems that do not have a concept of "drives". - when defined(unix): - result = path - else: - if path.len == 0: return "" - - var start: int - if path[0] == '/': - # an absolute path - when doslikeFileSystem: - if drive != "": - result = drive & ":" & DirSep - else: - result = $DirSep - elif defined(macos): - result = "" # must not start with ':' - else: - result = $DirSep - start = 1 - elif path[0] == '.' and (path.len == 1 or path[1] == '/'): - # current directory - result = $CurDir - start = when doslikeFileSystem: 1 else: 2 - else: - result = "" - start = 0 - - var i = start - while i < len(path): # ../../../ --> :::: - if i+2 < path.len and path[i] == '.' and path[i+1] == '.' and path[i+2] == '/': - # parent directory - when defined(macos): - if result[high(result)] == ':': - add result, ':' - else: - add result, ParDir - else: - add result, ParDir & DirSep - inc(i, 3) - elif path[i] == '/': - add result, DirSep - inc(i) - else: - add result, path[i] - inc(i) +import std/private/osseps +export osseps -include "includes/oserr" -when not defined(nimscript): - include "includes/osenv" -proc getHomeDir*(): string {.rtl, extern: "nos$1", - tags: [ReadEnvEffect, ReadIOEffect].} = - ## Returns the home directory of the current user. - ## - ## This proc is wrapped by the `expandTilde proc <#expandTilde,string>`_ - ## for the convenience of processing paths coming from user configuration files. - ## - ## See also: - ## * `getConfigDir proc <#getConfigDir>`_ - ## * `getTempDir proc <#getTempDir>`_ - ## * `expandTilde proc <#expandTilde,string>`_ - ## * `getCurrentDir proc <#getCurrentDir>`_ - ## * `setCurrentDir proc <#setCurrentDir,string>`_ - runnableExamples: - assert getHomeDir() == expandTilde("~") - # `getHomeDir()` doesn't end in `DirSep` even if `$HOME` (on posix) or - # `$USERPROFILE` (on windows) does, unless `-d:nimLegacyHomeDir` is specified. - from std/strutils import endsWith - assert not getHomeDir().endsWith DirSep - - when defined(windows): result = getEnv("USERPROFILE") - else: result = getEnv("HOME") - result.normalizePathEnd(trailingSep = defined(nimLegacyHomeDir)) - -proc getConfigDir*(): string {.rtl, extern: "nos$1", - tags: [ReadEnvEffect, ReadIOEffect].} = - ## Returns the config directory of the current user for applications. - ## - ## On non-Windows OSs, this proc conforms to the XDG Base Directory - ## spec. Thus, this proc returns the value of the `XDG_CONFIG_HOME` environment - ## variable if it is set, otherwise it returns the default configuration - ## directory ("~/.config"). - ## - ## See also: - ## * `getHomeDir proc <#getHomeDir>`_ - ## * `getTempDir proc <#getTempDir>`_ - ## * `expandTilde proc <#expandTilde,string>`_ - ## * `getCurrentDir proc <#getCurrentDir>`_ - ## * `setCurrentDir proc <#setCurrentDir,string>`_ - runnableExamples: - from std/strutils import endsWith - # See `getHomeDir` for behavior regarding trailing DirSep. - assert not getConfigDir().endsWith DirSep - when defined(windows): - result = getEnv("APPDATA") - else: - result = getEnv("XDG_CONFIG_HOME", getEnv("HOME") / ".config") - result.normalizePathEnd(trailingSep = defined(nimLegacyHomeDir)) - -when defined(windows): - type DWORD = uint32 - - proc getTempPath( - nBufferLength: DWORD, lpBuffer: WideCString - ): DWORD {.stdcall, dynlib: "kernel32.dll", importc: "GetTempPathW".} = - ## Retrieves the path of the directory designated for temporary files. - -template getEnvImpl(result: var string, tempDirList: openArray[string]) = - for dir in tempDirList: - if existsEnv(dir): - result = getEnv(dir) - break - -template getTempDirImpl(result: var string) = - when defined(windows): - getEnvImpl(result, ["TMP", "TEMP", "USERPROFILE"]) - else: - getEnvImpl(result, ["TMPDIR", "TEMP", "TMP", "TEMPDIR"]) - -proc getTempDir*(): string {.rtl, extern: "nos$1", - tags: [ReadEnvEffect, ReadIOEffect].} = - ## Returns the temporary directory of the current user for applications to - ## save temporary files in. - ## - ## On Windows, it calls [GetTempPath](https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-gettemppathw). - ## On Posix based platforms, it will check `TMPDIR`, `TEMP`, `TMP` and `TEMPDIR` environment variables in order. - ## On all platforms, `/tmp` will be returned if the procs fails. - ## - ## You can override this implementation - ## by adding `-d:tempDir=mytempname` to your compiler invocation. - ## - ## **Note:** This proc does not check whether the returned path exists. - ## - ## See also: - ## * `getHomeDir proc <#getHomeDir>`_ - ## * `getConfigDir proc <#getConfigDir>`_ - ## * `expandTilde proc <#expandTilde,string>`_ - ## * `getCurrentDir proc <#getCurrentDir>`_ - ## * `setCurrentDir proc <#setCurrentDir,string>`_ - runnableExamples: - from std/strutils import endsWith - # See `getHomeDir` for behavior regarding trailing DirSep. - assert not getTempDir().endsWith(DirSep) - const tempDirDefault = "/tmp" - when defined(tempDir): - const tempDir {.strdefine.}: string = tempDirDefault - result = tempDir - else: - when nimvm: - getTempDirImpl(result) - else: - when defined(windows): - let size = getTempPath(0, nil) - # If the function fails, the return value is zero. - if size > 0: - let buffer = newWideCString(size.int) - if getTempPath(size, buffer) > 0: - result = $buffer - elif defined(android): result = "/data/local/tmp" - else: - getTempDirImpl(result) - if result.len == 0: - result = tempDirDefault - result.normalizePathEnd(trailingSep = defined(nimLegacyHomeDir)) proc expandTilde*(path: string): string {. tags: [ReadEnvEffect, ReadIOEffect].} = ## Expands ``~`` or a path starting with ``~/`` to a full path, replacing - ## ``~`` with `getHomeDir() <#getHomeDir>`_ (otherwise returns ``path`` unmodified). + ## ``~`` with `getHomeDir()`_ (otherwise returns ``path`` unmodified). ## - ## Windows: this is still supported despite Windows platform not having this + ## Windows: this is still supported despite the Windows platform not having this ## convention; also, both ``~/`` and ``~\`` are handled. - ## - ## .. warning:: `~bob` and `~bob/` are not yet handled correctly. ## ## See also: - ## * `getHomeDir proc <#getHomeDir>`_ - ## * `getConfigDir proc <#getConfigDir>`_ - ## * `getTempDir proc <#getTempDir>`_ - ## * `getCurrentDir proc <#getCurrentDir>`_ - ## * `setCurrentDir proc <#setCurrentDir,string>`_ + ## * `getHomeDir proc`_ + ## * `getConfigDir proc`_ + ## * `getTempDir proc`_ + ## * `getCurrentDir proc`_ + ## * `setCurrentDir proc`_ runnableExamples: assert expandTilde("~" / "appname.cfg") == getHomeDir() / "appname.cfg" assert expandTilde("~/foo/bar") == getHomeDir() / "foo/bar" assert expandTilde("/foo/bar") == "/foo/bar" - assert expandTilde("~") == getHomeDir() - from std/strutils import endsWith - assert not expandTilde("~").endsWith(DirSep) if len(path) == 0 or path[0] != '~': result = path @@ -1035,15 +138,12 @@ proc expandTilde*(path: string): string {. # TODO: handle `~bob` and `~bob/` which means home of bob result = path -# TODO: consider whether quoteShellPosix, quoteShellWindows, quoteShell, quoteShellCommand -# belong in `strutils` instead; they are not specific to paths proc quoteShellWindows*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".} = ## Quote `s`, so it can be safely passed to Windows API. ## ## Based on Python's `subprocess.list2cmdline`. ## See `this link <http://msdn.microsoft.com/en-us/library/17w5ykft.aspx>`_ ## for more details. - let needQuote = {' ', '\t'} in s or s.len == 0 result = "" var backslashBuff = "" @@ -1054,8 +154,8 @@ proc quoteShellWindows*(s: string): string {.noSideEffect, rtl, extern: "nosp$1" if c == '\\': backslashBuff.add(c) elif c == '\"': - result.add(backslashBuff) - result.add(backslashBuff) + for i in 0..<backslashBuff.len*2: + result.add('\\') backslashBuff.setLen(0) result.add("\\\"") else: @@ -1064,35 +164,34 @@ proc quoteShellWindows*(s: string): string {.noSideEffect, rtl, extern: "nosp$1" backslashBuff.setLen(0) result.add(c) + if backslashBuff.len > 0: + result.add(backslashBuff) if needQuote: + result.add(backslashBuff) result.add("\"") + proc quoteShellPosix*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".} = ## Quote ``s``, so it can be safely passed to POSIX shell. - ## Based on Python's `pipes.quote`. const safeUnixChars = {'%', '+', '-', '.', '/', '_', ':', '=', '@', '0'..'9', 'A'..'Z', 'a'..'z'} if s.len == 0: - return "''" - - let safe = s.allCharsInSet(safeUnixChars) - - if safe: - return s + result = "''" + elif s.allCharsInSet(safeUnixChars): + result = s else: - return "'" & s.replace("'", "'\"'\"'") & "'" + result = "'" & s.replace("'", "'\"'\"'") & "'" when defined(windows) or defined(posix) or defined(nintendoswitch): proc quoteShell*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".} = ## Quote ``s``, so it can be safely passed to shell. ## - ## When on Windows, it calls `quoteShellWindows proc - ## <#quoteShellWindows,string>`_. Otherwise, calls `quoteShellPosix proc - ## <#quoteShellPosix,string>`_. + ## When on Windows, it calls `quoteShellWindows proc`_. + ## Otherwise, calls `quoteShellPosix proc`_. when defined(windows): - return quoteShellWindows(s) + result = quoteShellWindows(s) else: - return quoteShellPosix(s) + result = quoteShellPosix(s) proc quoteShellCommand*(args: openArray[string]): string = ## Concatenates and quotes shell arguments `args`. @@ -1113,106 +212,10 @@ when not weirdTarget: importc: "system", header: "<stdlib.h>".} when not defined(windows): - proc c_rename(oldname, newname: cstring): cint {. - importc: "rename", header: "<stdio.h>".} - proc c_strlen(a: cstring): cint {. - importc: "strlen", header: "<string.h>", noSideEffect.} proc c_free(p: pointer) {. importc: "free", header: "<stdlib.h>".} -when defined(windows) and not weirdTarget: - when useWinUnicode: - template wrapUnary(varname, winApiProc, arg: untyped) = - var varname = winApiProc(newWideCString(arg)) - - template wrapBinary(varname, winApiProc, arg, arg2: untyped) = - var varname = winApiProc(newWideCString(arg), arg2) - proc findFirstFile(a: string, b: var WIN32_FIND_DATA): Handle = - result = findFirstFileW(newWideCString(a), b) - template findNextFile(a, b: untyped): untyped = findNextFileW(a, b) - template getCommandLine(): untyped = getCommandLineW() - - template getFilename(f: untyped): untyped = - $cast[WideCString](addr(f.cFileName[0])) - else: - template findFirstFile(a, b: untyped): untyped = findFirstFileA(a, b) - template findNextFile(a, b: untyped): untyped = findNextFileA(a, b) - template getCommandLine(): untyped = getCommandLineA() - - template getFilename(f: untyped): untyped = $cstring(addr f.cFileName) - - proc skipFindData(f: WIN32_FIND_DATA): bool {.inline.} = - # Note - takes advantage of null delimiter in the cstring - const dot = ord('.') - result = f.cFileName[0].int == dot and (f.cFileName[1].int == 0 or - f.cFileName[1].int == dot and f.cFileName[2].int == 0) - -proc fileExists*(filename: string): bool {.rtl, extern: "nos$1", - tags: [ReadDirEffect], noNimJs.} = - ## Returns true if `filename` exists and is a regular file or symlink. - ## - ## Directories, device files, named pipes and sockets return false. - ## - ## See also: - ## * `dirExists proc <#dirExists,string>`_ - ## * `symlinkExists proc <#symlinkExists,string>`_ - when defined(windows): - when useWinUnicode: - wrapUnary(a, getFileAttributesW, filename) - else: - var a = getFileAttributesA(filename) - if a != -1'i32: - result = (a and FILE_ATTRIBUTE_DIRECTORY) == 0'i32 - else: - var res: Stat - return stat(filename, res) >= 0'i32 and S_ISREG(res.st_mode) - -proc dirExists*(dir: string): bool {.rtl, extern: "nos$1", tags: [ReadDirEffect], - noNimJs.} = - ## Returns true if the directory `dir` exists. If `dir` is a file, false - ## is returned. Follows symlinks. - ## - ## See also: - ## * `fileExists proc <#fileExists,string>`_ - ## * `symlinkExists proc <#symlinkExists,string>`_ - when defined(windows): - when useWinUnicode: - wrapUnary(a, getFileAttributesW, dir) - else: - var a = getFileAttributesA(dir) - if a != -1'i32: - result = (a and FILE_ATTRIBUTE_DIRECTORY) != 0'i32 - else: - var res: Stat - return stat(dir, res) >= 0'i32 and S_ISDIR(res.st_mode) - -proc symlinkExists*(link: string): bool {.rtl, extern: "nos$1", - tags: [ReadDirEffect], - noWeirdTarget.} = - ## Returns true if the symlink `link` exists. Will return true - ## regardless of whether the link points to a directory or file. - ## - ## See also: - ## * `fileExists proc <#fileExists,string>`_ - ## * `dirExists proc <#dirExists,string>`_ - when defined(windows): - when useWinUnicode: - wrapUnary(a, getFileAttributesW, link) - else: - var a = getFileAttributesA(link) - if a != -1'i32: - # xxx see: bug #16784 (bug9); checking `IO_REPARSE_TAG_SYMLINK` - # may also be needed. - result = (a and FILE_ATTRIBUTE_REPARSE_POINT) != 0'i32 - else: - var res: Stat - return lstat(link, res) >= 0'i32 and S_ISLNK(res.st_mode) - - -when not defined(windows): - const maxSymlinkLen = 1024 - const ExeExts* = ## Platform specific file extension for executables. ## On Windows ``["exe", "cmd", "bat"]``, on Posix ``[""]``. @@ -1225,7 +228,7 @@ proc findExe*(exe: string, followSymlinks: bool = true; ## in directories listed in the ``PATH`` environment variable. ## ## Returns `""` if the `exe` cannot be found. `exe` - ## is added the `ExeExts <#ExeExts>`_ file extensions if it has none. + ## is added the `ExeExts`_ file extensions if it has none. ## ## If the system supports symlinks it also resolves them until it ## meets the actual file. This behavior can be disabled if desired @@ -1252,16 +255,16 @@ proc findExe*(exe: string, followSymlinks: bool = true; for ext in extensions: var x = addFileExt(x, ext) if fileExists(x): - when not defined(windows): + when not (defined(windows) or defined(nintendoswitch)): while followSymlinks: # doubles as if here if x.symlinkExists: var r = newString(maxSymlinkLen) - var len = readlink(x, r, maxSymlinkLen) + var len = readlink(x.cstring, r.cstring, maxSymlinkLen) if len < 0: raiseOSError(osLastError(), exe) if len > maxSymlinkLen: r = newString(len+1) - len = readlink(x, r, len) + len = readlink(x.cstring, r.cstring, len) setLen(r, len) if isAbsolute(r): x = r @@ -1280,9 +283,9 @@ proc getLastModificationTime*(file: string): times.Time {.rtl, extern: "nos$1", ## Returns the `file`'s last modification time. ## ## See also: - ## * `getLastAccessTime proc <#getLastAccessTime,string>`_ - ## * `getCreationTime proc <#getCreationTime,string>`_ - ## * `fileNewer proc <#fileNewer,string,string>`_ + ## * `getLastAccessTime proc`_ + ## * `getCreationTime proc`_ + ## * `fileNewer proc`_ when defined(posix): var res: Stat if stat(file, res) < 0'i32: raiseOSError(osLastError(), file) @@ -1298,9 +301,9 @@ proc getLastAccessTime*(file: string): times.Time {.rtl, extern: "nos$1", noWeir ## Returns the `file`'s last read or write access time. ## ## See also: - ## * `getLastModificationTime proc <#getLastModificationTime,string>`_ - ## * `getCreationTime proc <#getCreationTime,string>`_ - ## * `fileNewer proc <#fileNewer,string,string>`_ + ## * `getLastModificationTime proc`_ + ## * `getCreationTime proc`_ + ## * `fileNewer proc`_ when defined(posix): var res: Stat if stat(file, res) < 0'i32: raiseOSError(osLastError(), file) @@ -1320,9 +323,9 @@ proc getCreationTime*(file: string): times.Time {.rtl, extern: "nos$1", noWeirdT ## `here <https://github.com/nim-lang/Nim/issues/1058>`_ for details. ## ## See also: - ## * `getLastModificationTime proc <#getLastModificationTime,string>`_ - ## * `getLastAccessTime proc <#getLastAccessTime,string>`_ - ## * `fileNewer proc <#fileNewer,string,string>`_ + ## * `getLastModificationTime proc`_ + ## * `getLastAccessTime proc`_ + ## * `fileNewer proc`_ when defined(posix): var res: Stat if stat(file, res) < 0'i32: raiseOSError(osLastError(), file) @@ -1339,9 +342,9 @@ proc fileNewer*(a, b: string): bool {.rtl, extern: "nos$1", noWeirdTarget.} = ## modification time is later than `b`'s. ## ## See also: - ## * `getLastModificationTime proc <#getLastModificationTime,string>`_ - ## * `getLastAccessTime proc <#getLastAccessTime,string>`_ - ## * `getCreationTime proc <#getCreationTime,string>`_ + ## * `getLastModificationTime proc`_ + ## * `getLastAccessTime proc`_ + ## * `getCreationTime proc`_ when defined(posix): # If we don't have access to nanosecond resolution, use '>=' when not StatHasNanoseconds: @@ -1351,356 +354,6 @@ proc fileNewer*(a, b: string): bool {.rtl, extern: "nos$1", noWeirdTarget.} = else: result = getLastModificationTime(a) > getLastModificationTime(b) -when not defined(nimscript): - proc getCurrentDir*(): string {.rtl, extern: "nos$1", tags: [].} = - ## Returns the `current working directory`:idx: i.e. where the built - ## binary is run. - ## - ## So the path returned by this proc is determined at run time. - ## - ## See also: - ## * `getHomeDir proc <#getHomeDir>`_ - ## * `getConfigDir proc <#getConfigDir>`_ - ## * `getTempDir proc <#getTempDir>`_ - ## * `setCurrentDir proc <#setCurrentDir,string>`_ - ## * `currentSourcePath template <system.html#currentSourcePath.t>`_ - ## * `getProjectPath proc <macros.html#getProjectPath>`_ - when defined(nodejs): - var ret: cstring - {.emit: "`ret` = process.cwd();".} - return $ret - elif defined(js): - doAssert false, "use -d:nodejs to have `getCurrentDir` defined" - elif defined(windows): - var bufsize = MAX_PATH.int32 - when useWinUnicode: - var res = newWideCString("", bufsize) - while true: - var L = getCurrentDirectoryW(bufsize, res) - if L == 0'i32: - raiseOSError(osLastError()) - elif L > bufsize: - res = newWideCString("", L) - bufsize = L - else: - result = res$L - break - else: - result = newString(bufsize) - while true: - var L = getCurrentDirectoryA(bufsize, result) - if L == 0'i32: - raiseOSError(osLastError()) - elif L > bufsize: - result = newString(L) - bufsize = L - else: - setLen(result, L) - break - else: - var bufsize = 1024 # should be enough - result = newString(bufsize) - while true: - if getcwd(result, bufsize) != nil: - setLen(result, c_strlen(result)) - break - else: - let err = osLastError() - if err.int32 == ERANGE: - bufsize = bufsize shl 1 - doAssert(bufsize >= 0) - result = newString(bufsize) - else: - raiseOSError(osLastError()) - -proc setCurrentDir*(newDir: string) {.inline, tags: [], noWeirdTarget.} = - ## Sets the `current working directory`:idx:; `OSError` - ## is raised if `newDir` cannot been set. - ## - ## See also: - ## * `getHomeDir proc <#getHomeDir>`_ - ## * `getConfigDir proc <#getConfigDir>`_ - ## * `getTempDir proc <#getTempDir>`_ - ## * `getCurrentDir proc <#getCurrentDir>`_ - when defined(windows): - when useWinUnicode: - if setCurrentDirectoryW(newWideCString(newDir)) == 0'i32: - raiseOSError(osLastError(), newDir) - else: - if setCurrentDirectoryA(newDir) == 0'i32: raiseOSError(osLastError(), newDir) - else: - if chdir(newDir) != 0'i32: raiseOSError(osLastError(), newDir) - - -proc absolutePath*(path: string, root = getCurrentDir()): string = - ## Returns the absolute path of `path`, rooted at `root` (which must be absolute; - ## default: current directory). - ## If `path` is absolute, return it, ignoring `root`. - ## - ## See also: - ## * `normalizedPath proc <#normalizedPath,string>`_ - ## * `normalizePath proc <#normalizePath,string>`_ - runnableExamples: - assert absolutePath("a") == getCurrentDir() / "a" - - if isAbsolute(path): path - else: - if not root.isAbsolute: - raise newException(ValueError, "The specified root is not absolute: " & root) - joinPath(root, path) - -proc absolutePathInternal(path: string): string = - absolutePath(path, getCurrentDir()) - -proc normalizeExe*(file: var string) {.since: (1, 3, 5).} = - ## on posix, prepends `./` if `file` doesn't contain `/` and is not `"", ".", ".."`. - runnableExamples: - import std/sugar - when defined(posix): - doAssert "foo".dup(normalizeExe) == "./foo" - doAssert "foo/../bar".dup(normalizeExe) == "foo/../bar" - doAssert "".dup(normalizeExe) == "" - when defined(posix): - if file.len > 0 and DirSep notin file and file != "." and file != "..": - file = "./" & file - -proc normalizePath*(path: var string) {.rtl, extern: "nos$1", tags: [].} = - ## Normalize a path. - ## - ## Consecutive directory separators are collapsed, including an initial double slash. - ## - ## On relative paths, double dot (`..`) sequences are collapsed if possible. - ## On absolute paths they are always collapsed. - ## - ## .. warning:: URL-encoded and Unicode attempts at directory traversal are not detected. - ## Triple dot is not handled. - ## - ## See also: - ## * `absolutePath proc <#absolutePath,string>`_ - ## * `normalizedPath proc <#normalizedPath,string>`_ for outplace version - ## * `normalizeExe proc <#normalizeExe,string>`_ - runnableExamples: - when defined(posix): - var a = "a///b//..//c///d" - a.normalizePath() - assert a == "a/c/d" - - path = pathnorm.normalizePath(path) - when false: - let isAbs = isAbsolute(path) - var stack: seq[string] = @[] - for p in split(path, {DirSep}): - case p - of "", ".": - continue - of "..": - if stack.len == 0: - if isAbs: - discard # collapse all double dots on absoluta paths - else: - stack.add(p) - elif stack[^1] == "..": - stack.add(p) - else: - discard stack.pop() - else: - stack.add(p) - - if isAbs: - path = DirSep & join(stack, $DirSep) - elif stack.len > 0: - path = join(stack, $DirSep) - else: - path = "." - -proc normalizePathAux(path: var string) = normalizePath(path) - -proc normalizedPath*(path: string): string {.rtl, extern: "nos$1", tags: [].} = - ## Returns a normalized path for the current OS. - ## - ## See also: - ## * `absolutePath proc <#absolutePath,string>`_ - ## * `normalizePath proc <#normalizePath,string>`_ for the in-place version - runnableExamples: - when defined(posix): - assert normalizedPath("a///b//..//c///d") == "a/c/d" - result = pathnorm.normalizePath(path) - -when defined(windows) and not weirdTarget: - proc openHandle(path: string, followSymlink=true, writeAccess=false): Handle = - var flags = FILE_FLAG_BACKUP_SEMANTICS or FILE_ATTRIBUTE_NORMAL - if not followSymlink: - flags = flags or FILE_FLAG_OPEN_REPARSE_POINT - let access = if writeAccess: GENERIC_WRITE else: 0'i32 - - when useWinUnicode: - result = createFileW( - newWideCString(path), access, - FILE_SHARE_DELETE or FILE_SHARE_READ or FILE_SHARE_WRITE, - nil, OPEN_EXISTING, flags, 0 - ) - else: - result = createFileA( - path, access, - FILE_SHARE_DELETE or FILE_SHARE_READ or FILE_SHARE_WRITE, - nil, OPEN_EXISTING, flags, 0 - ) - -proc sameFile*(path1, path2: string): bool {.rtl, extern: "nos$1", - tags: [ReadDirEffect], noWeirdTarget.} = - ## Returns true if both pathname arguments refer to the same physical - ## file or directory. - ## - ## Raises `OSError` if any of the files does not - ## exist or information about it can not be obtained. - ## - ## This proc will return true if given two alternative hard-linked or - ## sym-linked paths to the same file or directory. - ## - ## See also: - ## * `sameFileContent proc <#sameFileContent,string,string>`_ - when defined(windows): - var success = true - var f1 = openHandle(path1) - var f2 = openHandle(path2) - - var lastErr: OSErrorCode - if f1 != INVALID_HANDLE_VALUE and f2 != INVALID_HANDLE_VALUE: - var fi1, fi2: BY_HANDLE_FILE_INFORMATION - - if getFileInformationByHandle(f1, addr(fi1)) != 0 and - getFileInformationByHandle(f2, addr(fi2)) != 0: - result = fi1.dwVolumeSerialNumber == fi2.dwVolumeSerialNumber and - fi1.nFileIndexHigh == fi2.nFileIndexHigh and - fi1.nFileIndexLow == fi2.nFileIndexLow - else: - lastErr = osLastError() - success = false - else: - lastErr = osLastError() - success = false - - discard closeHandle(f1) - discard closeHandle(f2) - - if not success: raiseOSError(lastErr, $(path1, path2)) - else: - var a, b: Stat - if stat(path1, a) < 0'i32 or stat(path2, b) < 0'i32: - raiseOSError(osLastError(), $(path1, path2)) - else: - result = a.st_dev == b.st_dev and a.st_ino == b.st_ino - -type - FilePermission* = enum ## File access permission, modelled after UNIX. - ## - ## See also: - ## * `getFilePermissions <#getFilePermissions,string>`_ - ## * `setFilePermissions <#setFilePermissions,string,set[FilePermission]>`_ - ## * `FileInfo object <#FileInfo>`_ - fpUserExec, ## execute access for the file owner - fpUserWrite, ## write access for the file owner - fpUserRead, ## read access for the file owner - fpGroupExec, ## execute access for the group - fpGroupWrite, ## write access for the group - fpGroupRead, ## read access for the group - fpOthersExec, ## execute access for others - fpOthersWrite, ## write access for others - fpOthersRead ## read access for others - -proc getFilePermissions*(filename: string): set[FilePermission] {. - rtl, extern: "nos$1", tags: [ReadDirEffect], noWeirdTarget.} = - ## Retrieves file permissions for `filename`. - ## - ## `OSError` is raised in case of an error. - ## On Windows, only the ``readonly`` flag is checked, every other - ## permission is available in any case. - ## - ## See also: - ## * `setFilePermissions proc <#setFilePermissions,string,set[FilePermission]>`_ - ## * `FilePermission enum <#FilePermission>`_ - when defined(posix): - var a: Stat - if stat(filename, a) < 0'i32: raiseOSError(osLastError(), filename) - result = {} - if (a.st_mode and S_IRUSR.Mode) != 0.Mode: result.incl(fpUserRead) - if (a.st_mode and S_IWUSR.Mode) != 0.Mode: result.incl(fpUserWrite) - if (a.st_mode and S_IXUSR.Mode) != 0.Mode: result.incl(fpUserExec) - - if (a.st_mode and S_IRGRP.Mode) != 0.Mode: result.incl(fpGroupRead) - if (a.st_mode and S_IWGRP.Mode) != 0.Mode: result.incl(fpGroupWrite) - if (a.st_mode and S_IXGRP.Mode) != 0.Mode: result.incl(fpGroupExec) - - if (a.st_mode and S_IROTH.Mode) != 0.Mode: result.incl(fpOthersRead) - if (a.st_mode and S_IWOTH.Mode) != 0.Mode: result.incl(fpOthersWrite) - if (a.st_mode and S_IXOTH.Mode) != 0.Mode: result.incl(fpOthersExec) - else: - when useWinUnicode: - wrapUnary(res, getFileAttributesW, filename) - else: - var res = getFileAttributesA(filename) - if res == -1'i32: raiseOSError(osLastError(), filename) - if (res and FILE_ATTRIBUTE_READONLY) != 0'i32: - result = {fpUserExec, fpUserRead, fpGroupExec, fpGroupRead, - fpOthersExec, fpOthersRead} - else: - result = {fpUserExec..fpOthersRead} - -proc setFilePermissions*(filename: string, permissions: set[FilePermission], - followSymlinks = true) - {.rtl, extern: "nos$1", tags: [ReadDirEffect, WriteDirEffect], - noWeirdTarget.} = - ## Sets the file permissions for `filename`. - ## - ## If `followSymlinks` set to true (default) and ``filename`` points to a - ## symlink, permissions are set to the file symlink points to. - ## `followSymlinks` set to false is a noop on Windows and some POSIX - ## systems (including Linux) on which `lchmod` is either unavailable or always - ## fails, given that symlinks permissions there are not observed. - ## - ## `OSError` is raised in case of an error. - ## On Windows, only the ``readonly`` flag is changed, depending on - ## ``fpUserWrite`` permission. - ## - ## See also: - ## * `getFilePermissions <#getFilePermissions,string>`_ - ## * `FilePermission enum <#FilePermission>`_ - when defined(posix): - var p = 0.Mode - if fpUserRead in permissions: p = p or S_IRUSR.Mode - if fpUserWrite in permissions: p = p or S_IWUSR.Mode - if fpUserExec in permissions: p = p or S_IXUSR.Mode - - if fpGroupRead in permissions: p = p or S_IRGRP.Mode - if fpGroupWrite in permissions: p = p or S_IWGRP.Mode - if fpGroupExec in permissions: p = p or S_IXGRP.Mode - - if fpOthersRead in permissions: p = p or S_IROTH.Mode - if fpOthersWrite in permissions: p = p or S_IWOTH.Mode - if fpOthersExec in permissions: p = p or S_IXOTH.Mode - - if not followSymlinks and filename.symlinkExists: - when declared(lchmod): - if lchmod(filename, cast[Mode](p)) != 0: - raiseOSError(osLastError(), $(filename, permissions)) - else: - if chmod(filename, cast[Mode](p)) != 0: - raiseOSError(osLastError(), $(filename, permissions)) - else: - when useWinUnicode: - wrapUnary(res, getFileAttributesW, filename) - else: - var res = getFileAttributesA(filename) - if res == -1'i32: raiseOSError(osLastError(), filename) - if fpUserWrite in permissions: - res = res and not FILE_ATTRIBUTE_READONLY - else: - res = res or FILE_ATTRIBUTE_READONLY - when useWinUnicode: - wrapBinary(res2, setFileAttributesW, filename, res) - else: - var res2 = setFileAttributesA(filename, res) - if res2 == - 1'i32: raiseOSError(osLastError(), $(filename, permissions)) proc isAdmin*: bool {.noWeirdTarget.} = ## Returns whether the caller's process is a member of the Administrators local @@ -1720,314 +373,19 @@ proc isAdmin*: bool {.noWeirdTarget.} = addr administratorsGroup)): raiseOSError(osLastError(), "could not get SID for Administrators group") - defer: + try: + var b: WINBOOL + if not isSuccess(checkTokenMembership(0, administratorsGroup, addr b)): + raiseOSError(osLastError(), "could not check access token membership") + + result = isSuccess(b) + finally: if freeSid(administratorsGroup) != nil: raiseOSError(osLastError(), "failed to free SID for Administrators group") - var b: WINBOOL - if not isSuccess(checkTokenMembership(0, administratorsGroup, addr b)): - raiseOSError(osLastError(), "could not check access token membership") - - return isSuccess(b) - else: - return geteuid() == 0 - -proc createSymlink*(src, dest: string) {.noWeirdTarget.} = - ## Create a symbolic link at `dest` which points to the item specified - ## by `src`. On most operating systems, will fail if a link already exists. - ## - ## .. warning:: Some OS's (such as Microsoft Windows) restrict the creation - ## of symlinks to root users (administrators) or users with developper mode enabled. - ## - ## See also: - ## * `createHardlink proc <#createHardlink,string,string>`_ - ## * `expandSymlink proc <#expandSymlink,string>`_ - - when defined(windows): - const SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE = 2 - # allows anyone with developer mode on to create a link - let flag = dirExists(src).int32 or SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE - when useWinUnicode: - var wSrc = newWideCString(src) - var wDst = newWideCString(dest) - if createSymbolicLinkW(wDst, wSrc, flag) == 0 or getLastError() != 0: - raiseOSError(osLastError(), $(src, dest)) - else: - if createSymbolicLinkA(dest, src, flag) == 0 or getLastError() != 0: - raiseOSError(osLastError(), $(src, dest)) - else: - if symlink(src, dest) != 0: - raiseOSError(osLastError(), $(src, dest)) - -proc expandSymlink*(symlinkPath: string): string {.noWeirdTarget.} = - ## Returns a string representing the path to which the symbolic link points. - ## - ## On Windows this is a noop, `symlinkPath` is simply returned. - ## - ## See also: - ## * `createSymlink proc <#createSymlink,string,string>`_ - when defined(windows): - result = symlinkPath - else: - result = newString(maxSymlinkLen) - var len = readlink(symlinkPath, result, maxSymlinkLen) - if len < 0: - raiseOSError(osLastError(), symlinkPath) - if len > maxSymlinkLen: - result = newString(len+1) - len = readlink(symlinkPath, result, len) - setLen(result, len) - -const hasCCopyfile = defined(osx) and not defined(nimLegacyCopyFile) - # xxx instead of `nimLegacyCopyFile`, support something like: `when osxVersion >= (10, 5)` - -when hasCCopyfile: - # `copyfile` API available since osx 10.5. - {.push nodecl, header: "<copyfile.h>".} - type - copyfile_state_t {.nodecl.} = pointer - copyfile_flags_t = cint - proc copyfile_state_alloc(): copyfile_state_t - proc copyfile_state_free(state: copyfile_state_t): cint - proc c_copyfile(src, dst: cstring, state: copyfile_state_t, flags: copyfile_flags_t): cint {.importc: "copyfile".} - # replace with `let` pending bootstrap >= 1.4.0 - var - COPYFILE_DATA {.nodecl.}: copyfile_flags_t - COPYFILE_XATTR {.nodecl.}: copyfile_flags_t - {.pop.} - -type CopyFlag* = enum ## Copy options. - cfSymlinkAsIs, ## Copy symlinks as symlinks - cfSymlinkFollow, ## Copy the files symlinks point to - cfSymlinkIgnore ## Ignore symlinks - -const copyFlagSymlink = {cfSymlinkAsIs, cfSymlinkFollow, cfSymlinkIgnore} - -proc copyFile*(source, dest: string, options = {cfSymlinkFollow}) {.rtl, - extern: "nos$1", tags: [ReadDirEffect, ReadIOEffect, WriteIOEffect], - noWeirdTarget.} = - ## Copies a file from `source` to `dest`, where `dest.parentDir` must exist. - ## - ## On non-Windows OSes, `options` specify the way file is copied; by default, - ## if `source` is a symlink, copies the file symlink points to. `options` is - ## ignored on Windows: symlinks are skipped. - ## - ## If this fails, `OSError` is raised. - ## - ## On the Windows platform this proc will - ## copy the source file's attributes into dest. - ## - ## On other platforms you need - ## to use `getFilePermissions <#getFilePermissions,string>`_ and - ## `setFilePermissions <#setFilePermissions,string,set[FilePermission]>`_ - ## procs - ## to copy them by hand (or use the convenience `copyFileWithPermissions - ## proc <#copyFileWithPermissions,string,string>`_), - ## otherwise `dest` will inherit the default permissions of a newly - ## created file for the user. - ## - ## If `dest` already exists, the file attributes - ## will be preserved and the content overwritten. - ## - ## On OSX, `copyfile` C api will be used (available since OSX 10.5) unless - ## `-d:nimLegacyCopyFile` is used. - ## - ## See also: - ## * `CopyFlag enum <#CopyFlag>`_ - ## * `copyDir proc <#copyDir,string,string>`_ - ## * `copyFileWithPermissions proc <#copyFileWithPermissions,string,string>`_ - ## * `tryRemoveFile proc <#tryRemoveFile,string>`_ - ## * `removeFile proc <#removeFile,string>`_ - ## * `moveFile proc <#moveFile,string,string>`_ - - doAssert card(copyFlagSymlink * options) == 1, "There should be exactly " & - "one cfSymlink* in options" - let isSymlink = source.symlinkExists - if isSymlink and (cfSymlinkIgnore in options or defined(windows)): - return - when defined(windows): - when useWinUnicode: - let s = newWideCString(source) - let d = newWideCString(dest) - if copyFileW(s, d, 0'i32) == 0'i32: - raiseOSError(osLastError(), $(source, dest)) - else: - if copyFileA(source, dest, 0'i32) == 0'i32: - raiseOSError(osLastError(), $(source, dest)) - else: - if isSymlink and cfSymlinkAsIs in options: - createSymlink(expandSymlink(source), dest) - else: - when hasCCopyfile: - let state = copyfile_state_alloc() - # xxx `COPYFILE_STAT` could be used for one-shot - # `copyFileWithPermissions`. - let status = c_copyfile(source.cstring, dest.cstring, state, - COPYFILE_DATA) - if status != 0: - let err = osLastError() - discard copyfile_state_free(state) - raiseOSError(err, $(source, dest)) - let status2 = copyfile_state_free(state) - if status2 != 0: raiseOSError(osLastError(), $(source, dest)) - else: - # generic version of copyFile which works for any platform: - const bufSize = 8000 # better for memory manager - var d, s: File - if not open(s, source):raiseOSError(osLastError(), source) - if not open(d, dest, fmWrite): - close(s) - raiseOSError(osLastError(), dest) - var buf = alloc(bufSize) - while true: - var bytesread = readBuffer(s, buf, bufSize) - if bytesread > 0: - var byteswritten = writeBuffer(d, buf, bytesread) - if bytesread != byteswritten: - dealloc(buf) - close(s) - close(d) - raiseOSError(osLastError(), dest) - if bytesread != bufSize: break - dealloc(buf) - close(s) - flushFile(d) - close(d) - -proc copyFileToDir*(source, dir: string, options = {cfSymlinkFollow}) - {.noWeirdTarget, since: (1,3,7).} = - ## Copies a file `source` into directory `dir`, which must exist. - ## - ## On non-Windows OSes, `options` specify the way file is copied; by default, - ## if `source` is a symlink, copies the file symlink points to. `options` is - ## ignored on Windows: symlinks are skipped. - ## - ## See also: - ## * `CopyFlag enum <#CopyFlag>`_ - ## * `copyFile proc <#copyDir,string,string>`_ - if dir.len == 0: # treating "" as "." is error prone - raise newException(ValueError, "dest is empty") - copyFile(source, dir / source.lastPathPart, options) - -when not declared(ENOENT) and not defined(windows): - when NoFakeVars: - when not defined(haiku): - const ENOENT = cint(2) # 2 on most systems including Solaris - else: - const ENOENT = cint(-2147459069) - else: - var ENOENT {.importc, header: "<errno.h>".}: cint - -when defined(windows) and not weirdTarget: - when useWinUnicode: - template deleteFile(file: untyped): untyped = deleteFileW(file) - template setFileAttributes(file, attrs: untyped): untyped = - setFileAttributesW(file, attrs) else: - template deleteFile(file: untyped): untyped = deleteFileA(file) - template setFileAttributes(file, attrs: untyped): untyped = - setFileAttributesA(file, attrs) + result = geteuid() == 0 -proc tryRemoveFile*(file: string): bool {.rtl, extern: "nos$1", tags: [WriteDirEffect], noWeirdTarget.} = - ## Removes the `file`. - ## - ## If this fails, returns `false`. This does not fail - ## if the file never existed in the first place. - ## - ## On Windows, ignores the read-only attribute. - ## - ## See also: - ## * `copyFile proc <#copyFile,string,string>`_ - ## * `copyFileWithPermissions proc <#copyFileWithPermissions,string,string>`_ - ## * `removeFile proc <#removeFile,string>`_ - ## * `moveFile proc <#moveFile,string,string>`_ - result = true - when defined(windows): - when useWinUnicode: - let f = newWideCString(file) - else: - let f = file - if deleteFile(f) == 0: - result = false - let err = getLastError() - if err == ERROR_FILE_NOT_FOUND or err == ERROR_PATH_NOT_FOUND: - result = true - elif err == ERROR_ACCESS_DENIED and - setFileAttributes(f, FILE_ATTRIBUTE_NORMAL) != 0 and - deleteFile(f) != 0: - result = true - else: - if unlink(file) != 0'i32 and errno != ENOENT: - result = false - -proc removeFile*(file: string) {.rtl, extern: "nos$1", tags: [WriteDirEffect], noWeirdTarget.} = - ## Removes the `file`. - ## - ## If this fails, `OSError` is raised. This does not fail - ## if the file never existed in the first place. - ## - ## On Windows, ignores the read-only attribute. - ## - ## See also: - ## * `removeDir proc <#removeDir,string>`_ - ## * `copyFile proc <#copyFile,string,string>`_ - ## * `copyFileWithPermissions proc <#copyFileWithPermissions,string,string>`_ - ## * `tryRemoveFile proc <#tryRemoveFile,string>`_ - ## * `moveFile proc <#moveFile,string,string>`_ - if not tryRemoveFile(file): - raiseOSError(osLastError(), file) - -proc tryMoveFSObject(source, dest: string): bool {.noWeirdTarget.} = - ## Moves a file or directory from `source` to `dest`. - ## - ## Returns false in case of `EXDEV` error. - ## In case of other errors `OSError` is raised. - ## Returns true in case of success. - when defined(windows): - when useWinUnicode: - let s = newWideCString(source) - let d = newWideCString(dest) - if moveFileExW(s, d, MOVEFILE_COPY_ALLOWED or MOVEFILE_REPLACE_EXISTING) == 0'i32: raiseOSError(osLastError(), $(source, dest)) - else: - if moveFileExA(source, dest, MOVEFILE_COPY_ALLOWED or MOVEFILE_REPLACE_EXISTING) == 0'i32: raiseOSError(osLastError(), $(source, dest)) - else: - if c_rename(source, dest) != 0'i32: - let err = osLastError() - if err == EXDEV.OSErrorCode: - return false - else: - # see whether `strerror(errno)` is redundant with what raiseOSError already shows - raiseOSError(err, $(source, dest, strerror(errno))) - return true - -proc moveFile*(source, dest: string) {.rtl, extern: "nos$1", - tags: [ReadDirEffect, ReadIOEffect, WriteIOEffect], noWeirdTarget.} = - ## Moves a file from `source` to `dest`. - ## - ## Symlinks are not followed: if `source` is a symlink, it is itself moved, - ## not its target. - ## - ## If this fails, `OSError` is raised. - ## If `dest` already exists, it will be overwritten. - ## - ## Can be used to `rename files`:idx:. - ## - ## See also: - ## * `moveDir proc <#moveDir,string,string>`_ - ## * `copyFile proc <#copyFile,string,string>`_ - ## * `copyFileWithPermissions proc <#copyFileWithPermissions,string,string>`_ - ## * `removeFile proc <#removeFile,string>`_ - ## * `tryRemoveFile proc <#tryRemoveFile,string>`_ - - if not tryMoveFSObject(source, dest): - when not defined(windows): - # Fallback to copy & del - copyFile(source, dest, {cfSymlinkAsIs}) - try: - removeFile(source) - except: - discard tryRemoveFile(dest) - raise proc exitStatusLikeShell*(status: cint): cint = ## Converts exit code from `c_system` into a shell exit code. @@ -2053,122 +411,11 @@ proc execShellCmd*(command: string): int {.rtl, extern: "nos$1", ## <osproc.html#execProcess,string,string,openArray[string],StringTableRef,set[ProcessOption]>`_. ## ## **Examples:** - ## - ## .. code-block:: + ## ```Nim ## discard execShellCmd("ls -la") + ## ``` result = exitStatusLikeShell(c_system(command)) -# Templates for filtering directories and files -when defined(windows) and not weirdTarget: - template isDir(f: WIN32_FIND_DATA): bool = - (f.dwFileAttributes and FILE_ATTRIBUTE_DIRECTORY) != 0'i32 - template isFile(f: WIN32_FIND_DATA): bool = - not isDir(f) -else: - template isDir(f: string): bool {.dirty.} = - dirExists(f) - template isFile(f: string): bool {.dirty.} = - fileExists(f) - -template defaultWalkFilter(item): bool = - ## Walk filter used to return true on both - ## files and directories - true - -template walkCommon(pattern: string, filter) = - ## Common code for getting the files and directories with the - ## specified `pattern` - when defined(windows): - var - f: WIN32_FIND_DATA - res: int - res = findFirstFile(pattern, f) - if res != -1: - defer: findClose(res) - let dotPos = searchExtPos(pattern) - while true: - if not skipFindData(f) and filter(f): - # Windows bug/gotcha: 't*.nim' matches 'tfoo.nims' -.- so we check - # that the file extensions have the same length ... - let ff = getFilename(f) - let idx = ff.len - pattern.len + dotPos - if dotPos < 0 or idx >= ff.len or (idx >= 0 and ff[idx] == '.') or - (dotPos >= 0 and dotPos+1 < pattern.len and pattern[dotPos+1] == '*'): - yield splitFile(pattern).dir / extractFilename(ff) - if findNextFile(res, f) == 0'i32: - let errCode = getLastError() - if errCode == ERROR_NO_MORE_FILES: break - else: raiseOSError(errCode.OSErrorCode) - else: # here we use glob - var - f: Glob - res: int - f.gl_offs = 0 - f.gl_pathc = 0 - f.gl_pathv = nil - res = glob(pattern, 0, nil, addr(f)) - defer: globfree(addr(f)) - if res == 0: - for i in 0.. f.gl_pathc - 1: - assert(f.gl_pathv[i] != nil) - let path = $f.gl_pathv[i] - if filter(path): - yield path - -iterator walkPattern*(pattern: string): string {.tags: [ReadDirEffect], noWeirdTarget.} = - ## Iterate over all the files and directories that match the `pattern`. - ## - ## On POSIX this uses the `glob`:idx: call. - ## `pattern` is OS dependent, but at least the `"\*.ext"` - ## notation is supported. - ## - ## See also: - ## * `walkFiles iterator <#walkFiles.i,string>`_ - ## * `walkDirs iterator <#walkDirs.i,string>`_ - ## * `walkDir iterator <#walkDir.i,string>`_ - ## * `walkDirRec iterator <#walkDirRec.i,string>`_ - runnableExamples: - import std/sequtils - let paths = toSeq(walkPattern("lib/pure/*")) # works on windows too - assert "lib/pure/concurrency".unixToNativePath in paths - assert "lib/pure/os.nim".unixToNativePath in paths - walkCommon(pattern, defaultWalkFilter) - -iterator walkFiles*(pattern: string): string {.tags: [ReadDirEffect], noWeirdTarget.} = - ## Iterate over all the files that match the `pattern`. - ## - ## On POSIX this uses the `glob`:idx: call. - ## `pattern` is OS dependent, but at least the `"\*.ext"` - ## notation is supported. - ## - ## See also: - ## * `walkPattern iterator <#walkPattern.i,string>`_ - ## * `walkDirs iterator <#walkDirs.i,string>`_ - ## * `walkDir iterator <#walkDir.i,string>`_ - ## * `walkDirRec iterator <#walkDirRec.i,string>`_ - runnableExamples: - import std/sequtils - assert "lib/pure/os.nim".unixToNativePath in toSeq(walkFiles("lib/pure/*.nim")) # works on windows too - walkCommon(pattern, isFile) - -iterator walkDirs*(pattern: string): string {.tags: [ReadDirEffect], noWeirdTarget.} = - ## Iterate over all the directories that match the `pattern`. - ## - ## On POSIX this uses the `glob`:idx: call. - ## `pattern` is OS dependent, but at least the `"\*.ext"` - ## notation is supported. - ## - ## See also: - ## * `walkPattern iterator <#walkPattern.i,string>`_ - ## * `walkFiles iterator <#walkFiles.i,string>`_ - ## * `walkDir iterator <#walkDir.i,string>`_ - ## * `walkDirRec iterator <#walkDirRec.i,string>`_ - runnableExamples: - import std/sequtils - let paths = toSeq(walkDirs("lib/pure/*")) # works on windows too - assert "lib/pure/concurrency".unixToNativePath in paths - walkCommon(pattern, isDir) - proc expandFilename*(filename: string): string {.rtl, extern: "nos$1", tags: [ReadDirEffect], noWeirdTarget.} = ## Returns the full (`absolute`:idx:) path of an existing file `filename`. @@ -2176,32 +423,18 @@ proc expandFilename*(filename: string): string {.rtl, extern: "nos$1", ## Raises `OSError` in case of an error. Follows symlinks. when defined(windows): var bufsize = MAX_PATH.int32 - when useWinUnicode: - var unused: WideCString = nil - var res = newWideCString("", bufsize) - while true: - var L = getFullPathNameW(newWideCString(filename), bufsize, res, unused) - if L == 0'i32: - raiseOSError(osLastError(), filename) - elif L > bufsize: - res = newWideCString("", L) - bufsize = L - else: - result = res$L - break - else: - var unused: cstring = nil - result = newString(bufsize) - while true: - var L = getFullPathNameA(filename, bufsize, result, unused) - if L == 0'i32: - raiseOSError(osLastError(), filename) - elif L > bufsize: - result = newString(L) - bufsize = L - else: - setLen(result, L) - break + var unused: WideCString = nil + var res = newWideCString(bufsize) + while true: + var L = getFullPathNameW(newWideCString(filename), bufsize, res, unused) + if L == 0'i32: + raiseOSError(osLastError(), filename) + elif L > bufsize: + res = newWideCString(L) + bufsize = L + else: + result = res$L + break # getFullPathName doesn't do case corrections, so we have to use this convoluted # way of retrieving the true filename for x in walkFiles(result): @@ -2219,379 +452,14 @@ proc expandFilename*(filename: string): string {.rtl, extern: "nos$1", result = $r c_free(cast[pointer](r)) -type - PathComponent* = enum ## Enumeration specifying a path component. - ## - ## See also: - ## * `walkDirRec iterator <#walkDirRec.i,string>`_ - ## * `FileInfo object <#FileInfo>`_ - pcFile, ## path refers to a file - pcLinkToFile, ## path refers to a symbolic link to a file - pcDir, ## path refers to a directory - pcLinkToDir ## path refers to a symbolic link to a directory - proc getCurrentCompilerExe*(): string {.compileTime.} = discard - ## This is `getAppFilename() <#getAppFilename>`_ at compile time. + ## Returns the path of the currently running Nim compiler or nimble executable. ## ## Can be used to retrieve the currently executing ## Nim compiler from a Nim or nimscript program, or the nimble binary ## inside a nimble program (likewise with other binaries built from ## compiler API). -when defined(posix) and not weirdTarget: - proc getSymlinkFileKind(path: string): PathComponent = - # Helper function. - var s: Stat - assert(path != "") - if stat(path, s) == 0'i32 and S_ISDIR(s.st_mode): - result = pcLinkToDir - else: - result = pcLinkToFile - -proc staticWalkDir(dir: string; relative: bool): seq[ - tuple[kind: PathComponent, path: string]] = - discard - -iterator walkDir*(dir: string; relative = false, checkDir = false): - tuple[kind: PathComponent, path: string] {.tags: [ReadDirEffect].} = - ## Walks over the directory `dir` and yields for each directory or file in - ## `dir`. The component type and full path for each item are returned. - ## - ## Walking is not recursive. If ``relative`` is true (default: false) - ## the resulting path is shortened to be relative to ``dir``. - ## Example: This directory structure:: - ## dirA / dirB / fileB1.txt - ## / dirC - ## / fileA1.txt - ## / fileA2.txt - ## - ## and this code: - ## - ## .. code-block:: Nim - ## for kind, path in walkDir("dirA"): - ## echo(path) - ## - ## produce this output (but not necessarily in this order!):: - ## dirA/dirB - ## dirA/dirC - ## dirA/fileA1.txt - ## dirA/fileA2.txt - ## - ## See also: - ## * `walkPattern iterator <#walkPattern.i,string>`_ - ## * `walkFiles iterator <#walkFiles.i,string>`_ - ## * `walkDirs iterator <#walkDirs.i,string>`_ - ## * `walkDirRec iterator <#walkDirRec.i,string>`_ - - when nimvm: - for k, v in items(staticWalkDir(dir, relative)): - yield (k, v) - else: - when weirdTarget: - for k, v in items(staticWalkDir(dir, relative)): - yield (k, v) - elif defined(windows): - var f: WIN32_FIND_DATA - var h = findFirstFile(dir / "*", f) - if h == -1: - if checkDir: - raiseOSError(osLastError(), dir) - else: - defer: findClose(h) - while true: - var k = pcFile - if not skipFindData(f): - if (f.dwFileAttributes and FILE_ATTRIBUTE_DIRECTORY) != 0'i32: - k = pcDir - if (f.dwFileAttributes and FILE_ATTRIBUTE_REPARSE_POINT) != 0'i32: - k = succ(k) - let xx = if relative: extractFilename(getFilename(f)) - else: dir / extractFilename(getFilename(f)) - yield (k, xx) - if findNextFile(h, f) == 0'i32: - let errCode = getLastError() - if errCode == ERROR_NO_MORE_FILES: break - else: raiseOSError(errCode.OSErrorCode) - else: - var d = opendir(dir) - if d == nil: - if checkDir: - raiseOSError(osLastError(), dir) - else: - defer: discard closedir(d) - while true: - var x = readdir(d) - if x == nil: break - var y = $cstring(addr x.d_name) - if y != "." and y != "..": - var s: Stat - let path = dir / y - if not relative: - y = path - var k = pcFile - - template kSetGeneric() = # pure Posix component `k` resolution - if lstat(path, s) < 0'i32: continue # don't yield - elif S_ISDIR(s.st_mode): - k = pcDir - elif S_ISLNK(s.st_mode): - k = getSymlinkFileKind(path) - - when defined(linux) or defined(macosx) or - defined(bsd) or defined(genode) or defined(nintendoswitch): - case x.d_type - of DT_DIR: k = pcDir - of DT_LNK: - if dirExists(path): k = pcLinkToDir - else: k = pcLinkToFile - of DT_UNKNOWN: - kSetGeneric() - else: # e.g. DT_REG etc - discard # leave it as pcFile - else: # assuming that field `d_type` is not present - kSetGeneric() - - yield (k, y) - -iterator walkDirRec*(dir: string, - yieldFilter = {pcFile}, followFilter = {pcDir}, - relative = false, checkDir = false): string {.tags: [ReadDirEffect].} = - ## Recursively walks over the directory `dir` and yields for each file - ## or directory in `dir`. - ## - ## If ``relative`` is true (default: false) the resulting path is - ## shortened to be relative to ``dir``, otherwise the full path is returned. - ## - ## .. warning:: Modifying the directory structure while the iterator - ## is traversing may result in undefined behavior! - ## - ## Walking is recursive. `followFilter` controls the behaviour of the iterator: - ## - ## --------------------- --------------------------------------------- - ## yieldFilter meaning - ## --------------------- --------------------------------------------- - ## ``pcFile`` yield real files (default) - ## ``pcLinkToFile`` yield symbolic links to files - ## ``pcDir`` yield real directories - ## ``pcLinkToDir`` yield symbolic links to directories - ## --------------------- --------------------------------------------- - ## - ## --------------------- --------------------------------------------- - ## followFilter meaning - ## --------------------- --------------------------------------------- - ## ``pcDir`` follow real directories (default) - ## ``pcLinkToDir`` follow symbolic links to directories - ## --------------------- --------------------------------------------- - ## - ## - ## See also: - ## * `walkPattern iterator <#walkPattern.i,string>`_ - ## * `walkFiles iterator <#walkFiles.i,string>`_ - ## * `walkDirs iterator <#walkDirs.i,string>`_ - ## * `walkDir iterator <#walkDir.i,string>`_ - - var stack = @[""] - var checkDir = checkDir - while stack.len > 0: - let d = stack.pop() - for k, p in walkDir(dir / d, relative = true, checkDir = checkDir): - let rel = d / p - if k in {pcDir, pcLinkToDir} and k in followFilter: - stack.add rel - if k in yieldFilter: - yield if relative: rel else: dir / rel - checkDir = false - # We only check top-level dir, otherwise if a subdir is invalid (eg. wrong - # permissions), it'll abort iteration and there would be no way to - # continue iteration. - # Future work can provide a way to customize this and do error reporting. - -proc rawRemoveDir(dir: string) {.noWeirdTarget.} = - when defined(windows): - when useWinUnicode: - wrapUnary(res, removeDirectoryW, dir) - else: - var res = removeDirectoryA(dir) - let lastError = osLastError() - if res == 0'i32 and lastError.int32 != 3'i32 and - lastError.int32 != 18'i32 and lastError.int32 != 2'i32: - raiseOSError(lastError, dir) - else: - if rmdir(dir) != 0'i32 and errno != ENOENT: raiseOSError(osLastError(), dir) - -proc removeDir*(dir: string, checkDir = false) {.rtl, extern: "nos$1", tags: [ - WriteDirEffect, ReadDirEffect], benign, noWeirdTarget.} = - ## Removes the directory `dir` including all subdirectories and files - ## in `dir` (recursively). - ## - ## If this fails, `OSError` is raised. This does not fail if the directory never - ## existed in the first place, unless `checkDir` = true - ## - ## See also: - ## * `tryRemoveFile proc <#tryRemoveFile,string>`_ - ## * `removeFile proc <#removeFile,string>`_ - ## * `existsOrCreateDir proc <#existsOrCreateDir,string>`_ - ## * `createDir proc <#createDir,string>`_ - ## * `copyDir proc <#copyDir,string,string>`_ - ## * `copyDirWithPermissions proc <#copyDirWithPermissions,string,string>`_ - ## * `moveDir proc <#moveDir,string,string>`_ - for kind, path in walkDir(dir, checkDir = checkDir): - case kind - of pcFile, pcLinkToFile, pcLinkToDir: removeFile(path) - of pcDir: removeDir(path, true) - # for subdirectories there is no benefit in `checkDir = false` - # (unless perhaps for edge case of concurrent processes also deleting - # the same files) - rawRemoveDir(dir) - -proc rawCreateDir(dir: string): bool {.noWeirdTarget.} = - # Try to create one directory (not the whole path). - # returns `true` for success, `false` if the path has previously existed - # - # This is a thin wrapper over mkDir (or alternatives on other systems), - # so in case of a pre-existing path we don't check that it is a directory. - when defined(solaris): - let res = mkdir(dir, 0o777) - if res == 0'i32: - result = true - elif errno in {EEXIST, ENOSYS}: - result = false - else: - raiseOSError(osLastError(), dir) - elif defined(haiku): - let res = mkdir(dir, 0o777) - if res == 0'i32: - result = true - elif errno == EEXIST or errno == EROFS: - result = false - else: - raiseOSError(osLastError(), dir) - elif defined(posix): - let res = mkdir(dir, 0o777) - if res == 0'i32: - result = true - elif errno == EEXIST: - result = false - else: - #echo res - raiseOSError(osLastError(), dir) - else: - when useWinUnicode: - wrapUnary(res, createDirectoryW, dir) - else: - let res = createDirectoryA(dir) - - if res != 0'i32: - result = true - elif getLastError() == 183'i32: - result = false - else: - raiseOSError(osLastError(), dir) - -proc existsOrCreateDir*(dir: string): bool {.rtl, extern: "nos$1", - tags: [WriteDirEffect, ReadDirEffect], noWeirdTarget.} = - ## Checks if a `directory`:idx: `dir` exists, and creates it otherwise. - ## - ## Does not create parent directories (raises `OSError` if parent directories do not exist). - ## Returns `true` if the directory already exists, and `false` otherwise. - ## - ## See also: - ## * `removeDir proc <#removeDir,string>`_ - ## * `createDir proc <#createDir,string>`_ - ## * `copyDir proc <#copyDir,string,string>`_ - ## * `copyDirWithPermissions proc <#copyDirWithPermissions,string,string>`_ - ## * `moveDir proc <#moveDir,string,string>`_ - result = not rawCreateDir(dir) - if result: - # path already exists - need to check that it is indeed a directory - if not dirExists(dir): - raise newException(IOError, "Failed to create '" & dir & "'") - -proc createDir*(dir: string) {.rtl, extern: "nos$1", - tags: [WriteDirEffect, ReadDirEffect], noWeirdTarget.} = - ## Creates the `directory`:idx: `dir`. - ## - ## The directory may contain several subdirectories that do not exist yet. - ## The full path is created. If this fails, `OSError` is raised. - ## - ## It does **not** fail if the directory already exists because for - ## most usages this does not indicate an error. - ## - ## See also: - ## * `removeDir proc <#removeDir,string>`_ - ## * `existsOrCreateDir proc <#existsOrCreateDir,string>`_ - ## * `copyDir proc <#copyDir,string,string>`_ - ## * `copyDirWithPermissions proc <#copyDirWithPermissions,string,string>`_ - ## * `moveDir proc <#moveDir,string,string>`_ - var omitNext = false - when doslikeFileSystem: - omitNext = isAbsolute(dir) - for i in 1.. dir.len-1: - if dir[i] in {DirSep, AltSep}: - if omitNext: - omitNext = false - else: - discard existsOrCreateDir(substr(dir, 0, i-1)) - - # The loop does not create the dir itself if it doesn't end in separator - if dir.len > 0 and not omitNext and - dir[^1] notin {DirSep, AltSep}: - discard existsOrCreateDir(dir) - -proc copyDir*(source, dest: string) {.rtl, extern: "nos$1", - tags: [ReadDirEffect, WriteIOEffect, ReadIOEffect], benign, noWeirdTarget.} = - ## Copies a directory from `source` to `dest`. - ## - ## On non-Windows OSes, symlinks are copied as symlinks. On Windows, symlinks - ## are skipped. - ## - ## If this fails, `OSError` is raised. - ## - ## On the Windows platform this proc will copy the attributes from - ## `source` into `dest`. - ## - ## On other platforms created files and directories will inherit the - ## default permissions of a newly created file/directory for the user. - ## Use `copyDirWithPermissions proc <#copyDirWithPermissions,string,string>`_ - ## to preserve attributes recursively on these platforms. - ## - ## See also: - ## * `copyDirWithPermissions proc <#copyDirWithPermissions,string,string>`_ - ## * `copyFile proc <#copyFile,string,string>`_ - ## * `copyFileWithPermissions proc <#copyFileWithPermissions,string,string>`_ - ## * `removeDir proc <#removeDir,string>`_ - ## * `existsOrCreateDir proc <#existsOrCreateDir,string>`_ - ## * `createDir proc <#createDir,string>`_ - ## * `moveDir proc <#moveDir,string,string>`_ - createDir(dest) - for kind, path in walkDir(source): - var noSource = splitPath(path).tail - if kind == pcDir: - copyDir(path, dest / noSource) - else: - copyFile(path, dest / noSource, {cfSymlinkAsIs}) - -proc moveDir*(source, dest: string) {.tags: [ReadIOEffect, WriteIOEffect], noWeirdTarget.} = - ## Moves a directory from `source` to `dest`. - ## - ## Symlinks are not followed: if `source` contains symlinks, they themself are - ## moved, not their target. - ## - ## If this fails, `OSError` is raised. - ## - ## See also: - ## * `moveFile proc <#moveFile,string,string>`_ - ## * `copyDir proc <#copyDir,string,string>`_ - ## * `copyDirWithPermissions proc <#copyDirWithPermissions,string,string>`_ - ## * `removeDir proc <#removeDir,string>`_ - ## * `existsOrCreateDir proc <#existsOrCreateDir,string>`_ - ## * `createDir proc <#createDir,string>`_ - if not tryMoveFSObject(source, dest): - when not defined(windows): - # Fallback to copy & del - copyDir(source, dest) - removeDir(source) - proc createHardlink*(src, dest: string) {.noWeirdTarget.} = ## Create a hard link at `dest` which points to the item specified ## by `src`. @@ -2600,403 +468,39 @@ proc createHardlink*(src, dest: string) {.noWeirdTarget.} = ## root users (administrators). ## ## See also: - ## * `createSymlink proc <#createSymlink,string,string>`_ + ## * `createSymlink proc`_ when defined(windows): - when useWinUnicode: - var wSrc = newWideCString(src) - var wDst = newWideCString(dest) - if createHardLinkW(wDst, wSrc, nil) == 0: - raiseOSError(osLastError(), $(src, dest)) - else: - if createHardLinkA(dest, src, nil) == 0: - raiseOSError(osLastError(), $(src, dest)) + var wSrc = newWideCString(src) + var wDst = newWideCString(dest) + if createHardLinkW(wDst, wSrc, nil) == 0: + raiseOSError(osLastError(), $(src, dest)) else: if link(src, dest) != 0: raiseOSError(osLastError(), $(src, dest)) -proc copyFileWithPermissions*(source, dest: string, - ignorePermissionErrors = true, - options = {cfSymlinkFollow}) {.noWeirdTarget.} = - ## Copies a file from `source` to `dest` preserving file permissions. - ## - ## On non-Windows OSes, `options` specify the way file is copied; by default, - ## if `source` is a symlink, copies the file symlink points to. `options` is - ## ignored on Windows: symlinks are skipped. - ## - ## This is a wrapper proc around `copyFile <#copyFile,string,string>`_, - ## `getFilePermissions <#getFilePermissions,string>`_ and - ## `setFilePermissions<#setFilePermissions,string,set[FilePermission]>`_ - ## procs on non-Windows platforms. - ## - ## On Windows this proc is just a wrapper for `copyFile proc - ## <#copyFile,string,string>`_ since that proc already copies attributes. - ## - ## On non-Windows systems permissions are copied after the file itself has - ## been copied, which won't happen atomically and could lead to a race - ## condition. If `ignorePermissionErrors` is true (default), errors while - ## reading/setting file attributes will be ignored, otherwise will raise - ## `OSError`. - ## - ## See also: - ## * `CopyFlag enum <#CopyFlag>`_ - ## * `copyFile proc <#copyFile,string,string>`_ - ## * `copyDir proc <#copyDir,string,string>`_ - ## * `tryRemoveFile proc <#tryRemoveFile,string>`_ - ## * `removeFile proc <#removeFile,string>`_ - ## * `moveFile proc <#moveFile,string,string>`_ - ## * `copyDirWithPermissions proc <#copyDirWithPermissions,string,string>`_ - copyFile(source, dest, options) - when not defined(windows): - try: - setFilePermissions(dest, getFilePermissions(source), followSymlinks = - (cfSymlinkFollow in options)) - except: - if not ignorePermissionErrors: - raise - -proc copyDirWithPermissions*(source, dest: string, - ignorePermissionErrors = true) - {.rtl, extern: "nos$1", tags: [ReadDirEffect, WriteIOEffect, ReadIOEffect], - benign, noWeirdTarget.} = - ## Copies a directory from `source` to `dest` preserving file permissions. - ## - ## On non-Windows OSes, symlinks are copied as symlinks. On Windows, symlinks - ## are skipped. - ## - ## If this fails, `OSError` is raised. This is a wrapper proc around `copyDir - ## <#copyDir,string,string>`_ and `copyFileWithPermissions - ## <#copyFileWithPermissions,string,string>`_ procs - ## on non-Windows platforms. - ## - ## On Windows this proc is just a wrapper for `copyDir proc - ## <#copyDir,string,string>`_ since that proc already copies attributes. - ## - ## On non-Windows systems permissions are copied after the file or directory - ## itself has been copied, which won't happen atomically and could lead to a - ## race condition. If `ignorePermissionErrors` is true (default), errors while - ## reading/setting file attributes will be ignored, otherwise will raise - ## `OSError`. - ## - ## See also: - ## * `copyDir proc <#copyDir,string,string>`_ - ## * `copyFile proc <#copyFile,string,string>`_ - ## * `copyFileWithPermissions proc <#copyFileWithPermissions,string,string>`_ - ## * `removeDir proc <#removeDir,string>`_ - ## * `moveDir proc <#moveDir,string,string>`_ - ## * `existsOrCreateDir proc <#existsOrCreateDir,string>`_ - ## * `createDir proc <#createDir,string>`_ - createDir(dest) - when not defined(windows): - try: - setFilePermissions(dest, getFilePermissions(source), followSymlinks = - false) - except: - if not ignorePermissionErrors: - raise - for kind, path in walkDir(source): - var noSource = splitPath(path).tail - if kind == pcDir: - copyDirWithPermissions(path, dest / noSource, ignorePermissionErrors) - else: - copyFileWithPermissions(path, dest / noSource, ignorePermissionErrors, {cfSymlinkAsIs}) - proc inclFilePermissions*(filename: string, permissions: set[FilePermission]) {. rtl, extern: "nos$1", tags: [ReadDirEffect, WriteDirEffect], noWeirdTarget.} = ## A convenience proc for: - ## - ## .. code-block:: nim + ## ```nim ## setFilePermissions(filename, getFilePermissions(filename)+permissions) + ## ``` setFilePermissions(filename, getFilePermissions(filename)+permissions) proc exclFilePermissions*(filename: string, permissions: set[FilePermission]) {. rtl, extern: "nos$1", tags: [ReadDirEffect, WriteDirEffect], noWeirdTarget.} = ## A convenience proc for: - ## - ## .. code-block:: nim + ## ```nim ## setFilePermissions(filename, getFilePermissions(filename)-permissions) + ## ``` setFilePermissions(filename, getFilePermissions(filename)-permissions) -proc parseCmdLine*(c: string): seq[string] {. - noSideEffect, rtl, extern: "nos$1".} = - ## Splits a `command line`:idx: into several components. - ## - ## **Note**: This proc is only occasionally useful, better use the - ## `parseopt module <parseopt.html>`_. - ## - ## On Windows, it uses the `following parsing rules - ## <http://msdn.microsoft.com/en-us/library/17w5ykft.aspx>`_: - ## - ## * Arguments are delimited by white space, which is either a space or a tab. - ## * The caret character (^) is not recognized as an escape character or - ## delimiter. The character is handled completely by the command-line parser - ## in the operating system before being passed to the argv array in the - ## program. - ## * A string surrounded by double quotation marks ("string") is interpreted - ## as a single argument, regardless of white space contained within. A - ## quoted string can be embedded in an argument. - ## * A double quotation mark preceded by a backslash (\") is interpreted as a - ## literal double quotation mark character ("). - ## * Backslashes are interpreted literally, unless they immediately precede - ## a double quotation mark. - ## * If an even number of backslashes is followed by a double quotation mark, - ## one backslash is placed in the argv array for every pair of backslashes, - ## and the double quotation mark is interpreted as a string delimiter. - ## * If an odd number of backslashes is followed by a double quotation mark, - ## one backslash is placed in the argv array for every pair of backslashes, - ## and the double quotation mark is "escaped" by the remaining backslash, - ## causing a literal double quotation mark (") to be placed in argv. - ## - ## On Posix systems, it uses the following parsing rules: - ## Components are separated by whitespace unless the whitespace - ## occurs within ``"`` or ``'`` quotes. - ## - ## See also: - ## * `parseopt module <parseopt.html>`_ - ## * `paramCount proc <#paramCount>`_ - ## * `paramStr proc <#paramStr,int>`_ - ## * `commandLineParams proc <#commandLineParams>`_ - - result = @[] - var i = 0 - var a = "" - while true: - setLen(a, 0) - # eat all delimiting whitespace - while i < c.len and c[i] in {' ', '\t', '\l', '\r'}: inc(i) - if i >= c.len: break - when defined(windows): - # parse a single argument according to the above rules: - var inQuote = false - while i < c.len: - case c[i] - of '\\': - var j = i - while j < c.len and c[j] == '\\': inc(j) - if j < c.len and c[j] == '"': - for k in 1..(j-i) div 2: a.add('\\') - if (j-i) mod 2 == 0: - i = j - else: - a.add('"') - i = j+1 - else: - a.add(c[i]) - inc(i) - of '"': - inc(i) - if not inQuote: inQuote = true - elif i < c.len and c[i] == '"': - a.add(c[i]) - inc(i) - else: - inQuote = false - break - of ' ', '\t': - if not inQuote: break - a.add(c[i]) - inc(i) - else: - a.add(c[i]) - inc(i) - else: - case c[i] - of '\'', '\"': - var delim = c[i] - inc(i) # skip ' or " - while i < c.len and c[i] != delim: - add a, c[i] - inc(i) - if i < c.len: inc(i) - else: - while i < c.len and c[i] > ' ': - add(a, c[i]) - inc(i) - add(result, a) - -when defined(nimdoc): - # Common forward declaration docstring block for parameter retrieval procs. - proc paramCount*(): int {.tags: [ReadIOEffect].} = - ## Returns the number of `command line arguments`:idx: given to the - ## application. - ## - ## Unlike `argc`:idx: in C, if your binary was called without parameters this - ## will return zero. - ## You can query each individual parameter with `paramStr proc <#paramStr,int>`_ - ## or retrieve all of them in one go with `commandLineParams proc - ## <#commandLineParams>`_. - ## - ## **Availability**: When generating a dynamic library (see `--app:lib`) on - ## Posix this proc is not defined. - ## Test for availability using `declared() <system.html#declared,untyped>`_. - ## - ## See also: - ## * `parseopt module <parseopt.html>`_ - ## * `parseCmdLine proc <#parseCmdLine,string>`_ - ## * `paramStr proc <#paramStr,int>`_ - ## * `commandLineParams proc <#commandLineParams>`_ - ## - ## **Examples:** - ## - ## .. code-block:: nim - ## when declared(paramCount): - ## # Use paramCount() here - ## else: - ## # Do something else! - - proc paramStr*(i: int): string {.tags: [ReadIOEffect].} = - ## Returns the `i`-th `command line argument`:idx: given to the application. - ## - ## `i` should be in the range `1..paramCount()`, the `IndexDefect` - ## exception will be raised for invalid values. Instead of iterating - ## over `paramCount() <#paramCount>`_ with this proc you can - ## call the convenience `commandLineParams() <#commandLineParams>`_. - ## - ## Similarly to `argv`:idx: in C, - ## it is possible to call `paramStr(0)` but this will return OS specific - ## contents (usually the name of the invoked executable). You should avoid - ## this and call `getAppFilename() <#getAppFilename>`_ instead. - ## - ## **Availability**: When generating a dynamic library (see `--app:lib`) on - ## Posix this proc is not defined. - ## Test for availability using `declared() <system.html#declared,untyped>`_. - ## - ## See also: - ## * `parseopt module <parseopt.html>`_ - ## * `parseCmdLine proc <#parseCmdLine,string>`_ - ## * `paramCount proc <#paramCount>`_ - ## * `commandLineParams proc <#commandLineParams>`_ - ## * `getAppFilename proc <#getAppFilename>`_ - ## - ## **Examples:** - ## - ## .. code-block:: nim - ## when declared(paramStr): - ## # Use paramStr() here - ## else: - ## # Do something else! - -elif defined(nimscript): discard -elif defined(nodejs): - type Argv = object of JSRoot - let argv {.importjs: "process.argv".} : Argv - proc len(argv: Argv): int {.importjs: "#.length".} - proc `[]`(argv: Argv, i: int): cstring {.importjs: "#[#]".} - - proc paramCount*(): int {.tags: [ReadDirEffect].} = - result = argv.len - 2 - - proc paramStr*(i: int): string {.tags: [ReadIOEffect].} = - let i = i + 1 - if i < argv.len and i >= 0: - result = $argv[i] - else: - raise newException(IndexDefect, formatErrorIndexBound(i - 1, argv.len - 2)) -elif defined(nintendoswitch): - proc paramStr*(i: int): string {.tags: [ReadIOEffect].} = - raise newException(OSError, "paramStr is not implemented on Nintendo Switch") - - proc paramCount*(): int {.tags: [ReadIOEffect].} = - raise newException(OSError, "paramCount is not implemented on Nintendo Switch") - -elif defined(windows): - # Since we support GUI applications with Nim, we sometimes generate - # a WinMain entry proc. But a WinMain proc has no access to the parsed - # command line arguments. The way to get them differs. Thus we parse them - # ourselves. This has the additional benefit that the program's behaviour - # is always the same -- independent of the used C compiler. - var - ownArgv {.threadvar.}: seq[string] - ownParsedArgv {.threadvar.}: bool - - proc paramCount*(): int {.rtl, extern: "nos$1", tags: [ReadIOEffect].} = - # Docstring in nimdoc block. - if not ownParsedArgv: - ownArgv = parseCmdLine($getCommandLine()) - ownParsedArgv = true - result = ownArgv.len-1 - - proc paramStr*(i: int): string {.rtl, extern: "nos$1", - tags: [ReadIOEffect].} = - # Docstring in nimdoc block. - if not ownParsedArgv: - ownArgv = parseCmdLine($getCommandLine()) - ownParsedArgv = true - if i < ownArgv.len and i >= 0: - result = ownArgv[i] - else: - raise newException(IndexDefect, formatErrorIndexBound(i, ownArgv.len-1)) - -elif defined(genode): - proc paramStr*(i: int): string = - raise newException(OSError, "paramStr is not implemented on Genode") - - proc paramCount*(): int = - raise newException(OSError, "paramCount is not implemented on Genode") -elif weirdTarget: - proc paramStr*(i: int): string {.tags: [ReadIOEffect].} = - raise newException(OSError, "paramStr is not implemented on current platform") - - proc paramCount*(): int {.tags: [ReadIOEffect].} = - raise newException(OSError, "paramCount is not implemented on current platform") -elif not defined(createNimRtl) and - not(defined(posix) and appType == "lib"): - # On Posix, there is no portable way to get the command line from a DLL. - var - cmdCount {.importc: "cmdCount".}: cint - cmdLine {.importc: "cmdLine".}: cstringArray - - proc paramStr*(i: int): string {.tags: [ReadIOEffect].} = - # Docstring in nimdoc block. - if i < cmdCount and i >= 0: - result = $cmdLine[i] - else: - raise newException(IndexDefect, formatErrorIndexBound(i, cmdCount-1)) - - proc paramCount*(): int {.tags: [ReadIOEffect].} = - # Docstring in nimdoc block. - result = cmdCount-1 - -when declared(paramCount) or defined(nimdoc): - proc commandLineParams*(): seq[string] = - ## Convenience proc which returns the command line parameters. - ## - ## This returns **only** the parameters. If you want to get the application - ## executable filename, call `getAppFilename() <#getAppFilename>`_. - ## - ## **Availability**: On Posix there is no portable way to get the command - ## line from a DLL and thus the proc isn't defined in this environment. You - ## can test for its availability with `declared() - ## <system.html#declared,untyped>`_. - ## - ## See also: - ## * `parseopt module <parseopt.html>`_ - ## * `parseCmdLine proc <#parseCmdLine,string>`_ - ## * `paramCount proc <#paramCount>`_ - ## * `paramStr proc <#paramStr,int>`_ - ## * `getAppFilename proc <#getAppFilename>`_ - ## - ## **Examples:** - ## - ## .. code-block:: nim - ## when declared(commandLineParams): - ## # Use commandLineParams() here - ## else: - ## # Do something else! - result = @[] - for i in 1..paramCount(): - result.add(paramStr(i)) -else: - proc commandLineParams*(): seq[string] {.error: - "commandLineParams() unsupported by dynamic libraries".} = - discard - when not weirdTarget and (defined(freebsd) or defined(dragonfly) or defined(netbsd)): proc sysctl(name: ptr cint, namelen: cuint, oldp: pointer, oldplen: ptr csize_t, newp: pointer, newplen: csize_t): cint {.importc: "sysctl",header: """#include <sys/types.h> - #include <sys/sysctl.h>"""} + #include <sys/sysctl.h>""".} const CTL_KERN = 1 KERN_PROC = 14 @@ -3036,10 +540,10 @@ when not weirdTarget and (defined(freebsd) or defined(dragonfly) or defined(netb when not weirdTarget and (defined(linux) or defined(solaris) or defined(bsd) or defined(aix)): proc getApplAux(procPath: string): string = result = newString(maxSymlinkLen) - var len = readlink(procPath, result, maxSymlinkLen) + var len = readlink(procPath, result.cstring, maxSymlinkLen) if len > maxSymlinkLen: result = newString(len+1) - len = readlink(procPath, result, len) + len = readlink(procPath, result.cstring, len) setLen(result, len) when not weirdTarget and defined(openbsd): @@ -3108,7 +612,7 @@ when defined(haiku): B_FIND_PATH_IMAGE_PATH = 1000 proc find_path(codePointer: pointer, baseDirectory: cint, subPath: cstring, - pathBuffer: cstring, bufferSize: csize): int32 + pathBuffer: cstring, bufferSize: csize_t): int32 {.importc, header: "<FindDirectory.h>".} proc getApplHaiku(): string = @@ -3120,13 +624,15 @@ when defined(haiku): else: result = "" -proc getAppFilename*(): string {.rtl, extern: "nos$1", tags: [ReadIOEffect], noWeirdTarget.} = +proc getAppFilename*(): string {.rtl, extern: "nos$1", tags: [ReadIOEffect], noWeirdTarget, raises: [].} = ## Returns the filename of the application's executable. ## This proc will resolve symlinks. ## + ## Returns empty string when name is unavailable + ## ## See also: - ## * `getAppDir proc <#getAppDir>`_ - ## * `getCurrentCompilerExe proc <#getCurrentCompilerExe>`_ + ## * `getAppDir proc`_ + ## * `getCurrentCompilerExe proc`_ # Linux: /proc/<pid>/exe # Solaris: @@ -3134,68 +640,62 @@ proc getAppFilename*(): string {.rtl, extern: "nos$1", tags: [ReadIOEffect], noW # /proc/<pid>/path/a.out (complete pathname) when defined(windows): var bufsize = int32(MAX_PATH) - when useWinUnicode: - var buf = newWideCString("", bufsize) - while true: - var L = getModuleFileNameW(0, buf, bufsize) - if L == 0'i32: - result = "" # error! - break - elif L > bufsize: - buf = newWideCString("", L) - bufsize = L - else: - result = buf$L - break - else: - result = newString(bufsize) - while true: - var L = getModuleFileNameA(0, result, bufsize) - if L == 0'i32: - result = "" # error! - break - elif L > bufsize: - result = newString(L) - bufsize = L - else: - setLen(result, L) - break + var buf = newWideCString(bufsize) + while true: + var L = getModuleFileNameW(0, buf, bufsize) + if L == 0'i32: + result = "" # error! + break + elif L > bufsize: + buf = newWideCString(L) + bufsize = L + else: + result = buf$L + break elif defined(macosx): var size = cuint32(0) getExecPath1(nil, size) result = newString(int(size)) - if getExecPath2(result, size): + if getExecPath2(result.cstring, size): result = "" # error! if result.len > 0: - result = result.expandFilename + try: + result = result.expandFilename + except OSError: + result = "" else: when defined(linux) or defined(aix): result = getApplAux("/proc/self/exe") elif defined(solaris): result = getApplAux("/proc/" & $getpid() & "/path/a.out") - elif defined(genode) or defined(nintendoswitch): - raiseOSError(OSErrorCode(-1), "POSIX command line not supported") + elif defined(genode): + result = "" # Not supported elif defined(freebsd) or defined(dragonfly) or defined(netbsd): result = getApplFreebsd() elif defined(haiku): result = getApplHaiku() elif defined(openbsd): - result = getApplOpenBsd() + result = try: getApplOpenBsd() except OSError: "" + elif defined(nintendoswitch): + result = "" # little heuristic that may work on other POSIX-like systems: if result.len == 0: - result = getApplHeuristic() + result = try: getApplHeuristic() except OSError: "" proc getAppDir*(): string {.rtl, extern: "nos$1", tags: [ReadIOEffect], noWeirdTarget.} = ## Returns the directory of the application's executable. ## ## See also: - ## * `getAppFilename proc <#getAppFilename>`_ + ## * `getAppFilename proc`_ result = splitFile(getAppFilename()).dir proc sleep*(milsecs: int) {.rtl, extern: "nos$1", tags: [TimeEffect], noWeirdTarget.} = ## Sleeps `milsecs` milliseconds. + ## A negative `milsecs` causes sleep to return immediately. when defined(windows): + if milsecs < 0: + return # fixes #23732 winlean.sleep(int32(milsecs)) else: var a, b: Timespec @@ -3214,11 +714,10 @@ proc getFileSize*(file: string): BiggestInt {.rtl, extern: "nos$1", result = rdFileSize(a) findClose(resA) else: - var f: File - if open(f, file): - result = getFileSize(f) - close(f) - else: raiseOSError(osLastError(), file) + var rawInfo: Stat + if stat(file, rawInfo) < 0'i32: + raiseOSError(osLastError(), file) + rawInfo.st_size when defined(windows) or weirdTarget: type @@ -3234,9 +733,9 @@ type ## Contains information associated with a file object. ## ## See also: - ## * `getFileInfo(handle) proc <#getFileInfo,FileHandle>`_ - ## * `getFileInfo(file) proc <#getFileInfo,File>`_ - ## * `getFileInfo(path) proc <#getFileInfo,string>`_ + ## * `getFileInfo(handle) proc`_ + ## * `getFileInfo(file) proc`_ + ## * `getFileInfo(path, followSymlink) proc`_ id*: tuple[device: DeviceId, file: FileId] ## Device and file id. kind*: PathComponent ## Kind of file object - directory, symlink, etc. size*: BiggestInt ## Size of file. @@ -3247,21 +746,30 @@ type creationTime*: times.Time ## Time file was created. Not supported on all systems! blockSize*: int ## Preferred I/O block size for this object. ## In some filesystems, this may vary from file to file. + isSpecial*: bool ## Is file special? (on Unix some "files" + ## can be special=non-regular like FIFOs, + ## devices); for directories `isSpecial` + ## is always `false`, for symlinks it is + ## the same as for the link's target. template rawToFormalFileInfo(rawInfo, path, formalInfo): untyped = ## Transforms the native file info structure into the one nim uses. ## 'rawInfo' is either a 'BY_HANDLE_FILE_INFORMATION' structure on Windows, ## or a 'Stat' structure on posix when defined(windows): - template merge(a, b): untyped = a or (b shl 32) + template merge[T](a, b): untyped = + cast[T]( + (uint64(cast[uint32](a))) or + (uint64(cast[uint32](b)) shl 32) + ) formalInfo.id.device = rawInfo.dwVolumeSerialNumber - formalInfo.id.file = merge(rawInfo.nFileIndexLow, rawInfo.nFileIndexHigh) - formalInfo.size = merge(rawInfo.nFileSizeLow, rawInfo.nFileSizeHigh) + formalInfo.id.file = merge[FileId](rawInfo.nFileIndexLow, rawInfo.nFileIndexHigh) + formalInfo.size = merge[BiggestInt](rawInfo.nFileSizeLow, rawInfo.nFileSizeHigh) formalInfo.linkCount = rawInfo.nNumberOfLinks formalInfo.lastAccessTime = fromWinTime(rdFileTime(rawInfo.ftLastAccessTime)) formalInfo.lastWriteTime = fromWinTime(rdFileTime(rawInfo.ftLastWriteTime)) formalInfo.creationTime = fromWinTime(rdFileTime(rawInfo.ftCreationTime)) - formalInfo.blockSize = 8192 # xxx use windows API instead of hardcoding + formalInfo.blockSize = 8192 # xxx use Windows API instead of hardcoding # Retrieve basic permissions if (rawInfo.dwFileAttributes and FILE_ATTRIBUTE_READONLY) != 0'i32: @@ -3303,14 +811,14 @@ template rawToFormalFileInfo(rawInfo, path, formalInfo): untyped = checkAndIncludeMode(S_IWOTH, fpOthersWrite) checkAndIncludeMode(S_IXOTH, fpOthersExec) - formalInfo.kind = + (formalInfo.kind, formalInfo.isSpecial) = if S_ISDIR(rawInfo.st_mode): - pcDir + (pcDir, false) elif S_ISLNK(rawInfo.st_mode): assert(path != "") # symlinks can't occur for file handles getSymlinkFileKind(path) else: - pcFile + (pcFile, not S_ISREG(rawInfo.st_mode)) when defined(js): when not declared(FileHandle): @@ -3326,8 +834,8 @@ proc getFileInfo*(handle: FileHandle): FileInfo {.noWeirdTarget.} = ## is invalid, `OSError` is raised. ## ## See also: - ## * `getFileInfo(file) proc <#getFileInfo,File>`_ - ## * `getFileInfo(path) proc <#getFileInfo,string>`_ + ## * `getFileInfo(file) proc`_ + ## * `getFileInfo(path, followSymlink) proc`_ # Done: ID, Kind, Size, Permissions, Link Count when defined(windows): @@ -3348,8 +856,8 @@ proc getFileInfo*(file: File): FileInfo {.noWeirdTarget.} = ## Retrieves file information for the file object. ## ## See also: - ## * `getFileInfo(handle) proc <#getFileInfo,FileHandle>`_ - ## * `getFileInfo(path) proc <#getFileInfo,string>`_ + ## * `getFileInfo(handle) proc`_ + ## * `getFileInfo(path, followSymlink) proc`_ if file.isNil: raise newException(IOError, "File is nil") result = getFileInfo(file.getFileHandle()) @@ -3358,20 +866,21 @@ proc getFileInfo*(path: string, followSymlink = true): FileInfo {.noWeirdTarget. ## Retrieves file information for the file object pointed to by `path`. ## ## Due to intrinsic differences between operating systems, the information - ## contained by the returned `FileInfo object <#FileInfo>`_ will be slightly + ## contained by the returned `FileInfo object`_ will be slightly ## different across platforms, and in some cases, incomplete or inaccurate. ## ## When `followSymlink` is true (default), symlinks are followed and the ## information retrieved is information related to the symlink's target. - ## Otherwise, information on the symlink itself is retrieved. + ## Otherwise, information on the symlink itself is retrieved (however, + ## field `isSpecial` is still determined from the target on Unix). ## ## If the information cannot be retrieved, such as when the path doesn't ## exist, or when permission restrictions prevent the program from retrieving ## file information, `OSError` is raised. ## ## See also: - ## * `getFileInfo(handle) proc <#getFileInfo,FileHandle>`_ - ## * `getFileInfo(file) proc <#getFileInfo,File>`_ + ## * `getFileInfo(handle) proc`_ + ## * `getFileInfo(file) proc`_ when defined(windows): var handle = openHandle(path, followSymlink) @@ -3398,7 +907,7 @@ proc sameFileContent*(path1, path2: string): bool {.rtl, extern: "nos$1", ## binary content. ## ## See also: - ## * `sameFile proc <#sameFile,string,string>`_ + ## * `sameFile proc`_ var a, b: File if not open(a, path1): return false @@ -3445,10 +954,7 @@ proc isHidden*(path: string): bool {.noWeirdTarget.} = assert ".foo/".isHidden when defined(windows): - when useWinUnicode: - wrapUnary(attributes, getFileAttributesW, path) - else: - var attributes = getFileAttributesA(path) + wrapUnary(attributes, getFileAttributesW, path) if attributes != -1'i32: result = (attributes and FILE_ATTRIBUTE_HIDDEN) != 0'i32 else: @@ -3484,37 +990,43 @@ proc setLastModificationTime*(file: string, t: times.Time) {.noWeirdTarget.} = discard h.closeHandle if res == 0'i32: raiseOSError(osLastError(), file) + func isValidFilename*(filename: string, maxLen = 259.Positive): bool {.since: (1, 1).} = - ## Returns true if ``filename`` is valid for crossplatform use. + ## Returns `true` if `filename` is valid for crossplatform use. ## ## This is useful if you want to copy or save files across Windows, Linux, Mac, etc. - ## You can pass full paths as argument too, but func only checks filenames. - ## It uses ``invalidFilenameChars``, ``invalidFilenames`` and ``maxLen`` to verify the specified ``filename``. + ## It uses `invalidFilenameChars`, `invalidFilenames` and `maxLen` to verify the specified `filename`. + ## + ## See also: ## - ## .. code-block:: nim - ## assert not isValidFilename(" foo") ## Leading white space - ## assert not isValidFilename("foo ") ## Trailing white space - ## assert not isValidFilename("foo.") ## Ends with Dot - ## assert not isValidFilename("con.txt") ## "CON" is invalid (Windows) - ## assert not isValidFilename("OwO:UwU") ## ":" is invalid (Mac) - ## assert not isValidFilename("aux.bat") ## "AUX" is invalid (Windows) + ## * https://docs.microsoft.com/en-us/dotnet/api/system.io.pathtoolongexception + ## * https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file + ## * https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247%28v=vs.85%29.aspx ## - # https://docs.microsoft.com/en-us/dotnet/api/system.io.pathtoolongexception - # https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file - # https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247%28v=vs.85%29.aspx + ## .. warning:: This only checks filenames, not whole paths + ## (because basically you can mount anything as a path on Linux). + runnableExamples: + assert not isValidFilename(" foo") # Leading white space + assert not isValidFilename("foo ") # Trailing white space + assert not isValidFilename("foo.") # Ends with dot + assert not isValidFilename("con.txt") # "CON" is invalid (Windows) + assert not isValidFilename("OwO:UwU") # ":" is invalid (Mac) + assert not isValidFilename("aux.bat") # "AUX" is invalid (Windows) + assert not isValidFilename("") # Empty string + assert not isValidFilename("foo/") # Filename is empty + result = true let f = filename.splitFile() - if unlikely(f.name.len + f.ext.len > maxLen or + if unlikely(f.name.len + f.ext.len > maxLen or f.name.len == 0 or f.name[0] == ' ' or f.name[^1] == ' ' or f.name[^1] == '.' or find(f.name, invalidFilenameChars) != -1): return false for invalid in invalidFilenames: if cmpIgnoreCase(f.name, invalid) == 0: return false + # deprecated declarations -when not defined(nimscript): - when not defined(js): # `noNimJs` doesn't work with templates, this should improve. - template existsFile*(args: varargs[untyped]): untyped {.deprecated: "use fileExists".} = - fileExists(args) - template existsDir*(args: varargs[untyped]): untyped {.deprecated: "use dirExists".} = - dirExists(args) - # {.deprecated: [existsFile: fileExists].} # pending bug #14819; this would avoid above mentioned issue +when not weirdTarget: + template existsFile*(args: varargs[untyped]): untyped {.deprecated: "use fileExists".} = + fileExists(args) + template existsDir*(args: varargs[untyped]): untyped {.deprecated: "use dirExists".} = + dirExists(args) diff --git a/lib/pure/osproc.nim b/lib/pure/osproc.nim index 6aefb8d6c..c304ecca6 100644 --- a/lib/pure/osproc.nim +++ b/lib/pure/osproc.nim @@ -18,18 +18,24 @@ include "system/inclrtl" import - strutils, os, strtabs, streams, cpuinfo, streamwrapper, - std/private/since + std/[strutils, os, strtabs, streams, cpuinfo, streamwrapper, + private/since] export quoteShell, quoteShellWindows, quoteShellPosix when defined(windows): - import winlean + import std/winlean else: - import posix + import std/posix when defined(linux) and defined(useClone): - import linux + import std/linux + +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions] + when defined(windows): + import std/widestrs + type ProcessOption* = enum ## Options that can be passed to `startProcess proc @@ -66,16 +72,11 @@ type Process* = ref ProcessObj ## Represents an operating system process. -const poDemon* {.deprecated.} = poDaemon ## Nim versions before 0.20 - ## used the wrong spelling ("demon"). - ## Now `ProcessOption` uses the correct spelling ("daemon"), - ## and this is needed just for backward compatibility. - proc execProcess*(command: string, workingDir: string = "", args: openArray[string] = [], env: StringTableRef = nil, options: set[ProcessOption] = {poStdErrToStdOut, poUsePath, poEvalCommand}): - string {.rtl, extern: "nosp$1", + string {.rtl, extern: "nosp$1", raises: [OSError, IOError], tags: [ExecIOEffect, ReadIOEffect, RootEffect].} ## A convenience procedure that executes ``command`` with ``startProcess`` ## and returns its output as a string. @@ -90,12 +91,12 @@ proc execProcess*(command: string, workingDir: string = "", ## * `execCmd proc <#execCmd,string>`_ ## ## Example: - ## - ## .. code-block:: Nim - ## let outp = execProcess("nim", args=["c", "-r", "mytestfile.nim"], options={poUsePath}) - ## let outp_shell = execProcess("nim c -r mytestfile.nim") - ## # Note: outp may have an interleave of text from the nim compile - ## # and any output from mytestfile when it runs + ## ```Nim + ## let outp = execProcess("nim", args=["c", "-r", "mytestfile.nim"], options={poUsePath}) + ## let outp_shell = execProcess("nim c -r mytestfile.nim") + ## # Note: outp may have an interleave of text from the nim compile + ## # and any output from mytestfile when it runs + ## ``` proc execCmd*(command: string): int {.rtl, extern: "nosp$1", tags: [ExecIOEffect, ReadIOEffect, RootEffect].} @@ -112,14 +113,14 @@ proc execCmd*(command: string): int {.rtl, extern: "nosp$1", ## <#execProcess,string,string,openArray[string],StringTableRef,set[ProcessOption]>`_ ## ## Example: - ## - ## .. code-block:: Nim - ## let errC = execCmd("nim c -r mytestfile.nim") + ## ```Nim + ## let errC = execCmd("nim c -r mytestfile.nim") + ## ``` proc startProcess*(command: string, workingDir: string = "", args: openArray[string] = [], env: StringTableRef = nil, options: set[ProcessOption] = {poStdErrToStdOut}): - owned(Process) {.rtl, extern: "nosp$1", + owned(Process) {.rtl, extern: "nosp$1", raises: [OSError, IOError], tags: [ExecIOEffect, ReadEnvEffect, RootEffect].} ## Starts a process. `Command` is the executable file, `workingDir` is the ## process's working directory. If ``workingDir == ""`` the current directory @@ -151,7 +152,7 @@ proc startProcess*(command: string, workingDir: string = "", ## <#execProcess,string,string,openArray[string],StringTableRef,set[ProcessOption]>`_ ## * `execCmd proc <#execCmd,string>`_ -proc close*(p: Process) {.rtl, extern: "nosp$1", tags: [WriteIOEffect].} +proc close*(p: Process) {.rtl, extern: "nosp$1", raises: [IOError, OSError], tags: [WriteIOEffect].} ## When the process has finished executing, cleanup related handles. ## ## .. warning:: If the process has not finished executing, this will forcibly @@ -200,7 +201,7 @@ proc kill*(p: Process) {.rtl, extern: "nosp$1", tags: [].} ## * `terminate proc <#terminate,Process>`_ ## * `posix_utils.sendSignal(pid: Pid, signal: int) <posix_utils.html#sendSignal,Pid,int>`_ -proc running*(p: Process): bool {.rtl, extern: "nosp$1", tags: [].} +proc running*(p: Process): bool {.rtl, extern: "nosp$1", raises: [OSError], tags: [].} ## Returns true if the process `p` is still running. Returns immediately. proc processID*(p: Process): int {.rtl, extern: "nosp$1".} = @@ -211,7 +212,7 @@ proc processID*(p: Process): int {.rtl, extern: "nosp$1".} = return p.id proc waitForExit*(p: Process, timeout: int = -1): int {.rtl, - extern: "nosp$1", tags: [].} + extern: "nosp$1", raises: [OSError, ValueError], tags: [TimeEffect].} ## Waits for the process to finish and returns `p`'s error code. ## ## .. warning:: Be careful when using `waitForExit` for processes created without @@ -219,8 +220,12 @@ proc waitForExit*(p: Process, timeout: int = -1): int {.rtl, ## ## On posix, if the process has exited because of a signal, 128 + signal ## number will be returned. + ## + ## .. warning:: When working with `timeout` parameters, remember that the value is + ## typically expressed in milliseconds, and ensure that the correct unit of time + ## is used to avoid unexpected behavior. -proc peekExitCode*(p: Process): int {.rtl, extern: "nosp$1", tags: [].} +proc peekExitCode*(p: Process): int {.rtl, extern: "nosp$1", raises: [OSError], tags: [].} ## Return `-1` if the process is still running. Otherwise the process' exit code. ## ## On posix, if the process has exited because of a signal, 128 + signal @@ -236,7 +241,7 @@ proc inputStream*(p: Process): Stream {.rtl, extern: "nosp$1", tags: [].} ## * `outputStream proc <#outputStream,Process>`_ ## * `errorStream proc <#errorStream,Process>`_ -proc outputStream*(p: Process): Stream {.rtl, extern: "nosp$1", tags: [].} +proc outputStream*(p: Process): Stream {.rtl, extern: "nosp$1", raises: [IOError, OSError], tags: [].} ## Returns ``p``'s output stream for reading from. ## ## You cannot perform peek/write/setOption operations to this stream. @@ -288,7 +293,7 @@ proc peekableErrorStream*(p: Process): Stream {.rtl, extern: "nosp$1", tags: [], ## * `errorStream proc <#errorStream,Process>`_ ## * `peekableOutputStream proc <#peekableOutputStream,Process>`_ -proc inputHandle*(p: Process): FileHandle {.rtl, extern: "nosp$1", +proc inputHandle*(p: Process): FileHandle {.rtl, raises: [], extern: "nosp$1", tags: [].} = ## Returns ``p``'s input file handle for writing to. ## @@ -301,7 +306,7 @@ proc inputHandle*(p: Process): FileHandle {.rtl, extern: "nosp$1", result = p.inHandle proc outputHandle*(p: Process): FileHandle {.rtl, extern: "nosp$1", - tags: [].} = + raises: [], tags: [].} = ## Returns ``p``'s output file handle for reading from. ## ## .. warning:: The returned `FileHandle` should not be closed manually as @@ -313,7 +318,7 @@ proc outputHandle*(p: Process): FileHandle {.rtl, extern: "nosp$1", result = p.outHandle proc errorHandle*(p: Process): FileHandle {.rtl, extern: "nosp$1", - tags: [].} = + raises: [], tags: [].} = ## Returns ``p``'s error file handle for reading from. ## ## .. warning:: The returned `FileHandle` should not be closed manually as @@ -324,18 +329,23 @@ proc errorHandle*(p: Process): FileHandle {.rtl, extern: "nosp$1", ## * `outputHandle proc <#outputHandle,Process>`_ result = p.errHandle -proc countProcessors*(): int {.rtl, extern: "nosp$1".} = +proc countProcessors*(): int {.rtl, extern: "nosp$1", raises: [].} = ## Returns the number of the processors/cores the machine has. ## Returns 0 if it cannot be detected. ## It is implemented just calling `cpuinfo.countProcessors`. result = cpuinfo.countProcessors() +when not defined(nimHasEffectsOf): + {.pragma: effectsOf.} + proc execProcesses*(cmds: openArray[string], options = {poStdErrToStdOut, poParentStreams}, n = countProcessors(), beforeRunEvent: proc(idx: int) = nil, afterRunEvent: proc(idx: int, p: Process) = nil): int {.rtl, extern: "nosp$1", - tags: [ExecIOEffect, TimeEffect, ReadEnvEffect, RootEffect].} = + raises: [ValueError, OSError, IOError], + tags: [ExecIOEffect, TimeEffect, ReadEnvEffect, RootEffect], + effectsOf: [beforeRunEvent, afterRunEvent].} = ## Executes the commands `cmds` in parallel. ## Creates `n` processes that execute in parallel. ## @@ -447,7 +457,7 @@ proc execProcesses*(cmds: openArray[string], if afterRunEvent != nil: afterRunEvent(i, p) close(p) -iterator lines*(p: Process): string {.since: (1, 3), tags: [ReadIOEffect].} = +iterator lines*(p: Process, keepNewLines = false): string {.since: (1, 3), raises: [OSError, IOError, ValueError], tags: [ReadIOEffect, TimeEffect].} = ## Convenience iterator for working with `startProcess` to read data from a ## background process. ## @@ -455,8 +465,7 @@ iterator lines*(p: Process): string {.since: (1, 3), tags: [ReadIOEffect].} = ## * `readLines proc <#readLines,Process>`_ ## ## Example: - ## - ## .. code-block:: Nim + ## ```Nim ## const opts = {poUsePath, poDaemon, poStdErrToStdOut} ## var ps: seq[Process] ## for prog in ["a", "b"]: # run 2 progs in parallel @@ -468,15 +477,17 @@ iterator lines*(p: Process): string {.since: (1, 3), tags: [ReadIOEffect].} = ## i.inc ## if i > 100: break ## p.close + ## ``` var outp = p.outputStream var line = newStringOfCap(120) - while true: - if outp.readLine(line): - yield line - else: - if p.peekExitCode != -1: break - -proc readLines*(p: Process): (seq[string], int) {.since: (1, 3).} = + while outp.readLine(line): + if keepNewLines: + line.add("\n") + yield line + discard waitForExit(p) + +proc readLines*(p: Process): (seq[string], int) {.since: (1, 3), + raises: [OSError, IOError, ValueError], tags: [ReadIOEffect, TimeEffect].} = ## Convenience function for working with `startProcess` to read data from a ## background process. ## @@ -484,8 +495,7 @@ proc readLines*(p: Process): (seq[string], int) {.since: (1, 3).} = ## * `lines iterator <#lines.i,Process>`_ ## ## Example: - ## - ## .. code-block:: Nim + ## ```Nim ## const opts = {poUsePath, poDaemon, poStdErrToStdOut} ## var ps: seq[Process] ## for prog in ["a", "b"]: # run 2 progs in parallel @@ -495,6 +505,7 @@ proc readLines*(p: Process): (seq[string], int) {.since: (1, 3).} = ## if exCode != 0: ## for line in lines: echo line ## p.close + ## ``` for line in p.lines: result[0].add(line) result[1] = p.peekExitCode @@ -510,6 +521,7 @@ when not defined(useNimRtl): var outp = outputStream(p) result = "" var line = newStringOfCap(120) + # consider `p.lines(keepNewLines=true)` to circumvent `running` busy-wait while true: # FIXME: converts CR-LF to LF. if outp.readLine(line): @@ -562,12 +574,8 @@ when defined(windows) and not defined(useNimRtl): if a == 0: raiseOSError(osLastError()) proc newFileHandleStream(handle: Handle): owned FileHandleStream = - new(result) - result.handle = handle - result.closeImpl = hsClose - result.atEndImpl = hsAtEnd - result.readDataImpl = hsReadData - result.writeDataImpl = hsWriteData + result = FileHandleStream(handle: handle, closeImpl: hsClose, atEndImpl: hsAtEnd, + readDataImpl: hsReadData, writeDataImpl: hsWriteData) proc buildCommandLine(a: string, args: openArray[string]): string = result = quoteShell(a) @@ -710,22 +718,15 @@ when defined(windows) and not defined(useNimRtl): if len(workingDir) > 0: wd = workingDir if env != nil: e = buildEnv(env) if poEchoCmd in options: echo($cmdl) - when useWinUnicode: - var tmp = newWideCString(cmdl) - var ee = - if e.str.isNil: newWideCString(cstring(nil)) - else: newWideCString(e.str, e.len) - var wwd = newWideCString(wd) - var flags = NORMAL_PRIORITY_CLASS or CREATE_UNICODE_ENVIRONMENT - if poDaemon in options: flags = flags or CREATE_NO_WINDOW - success = winlean.createProcessW(nil, tmp, nil, nil, 1, flags, - ee, wwd, si, procInfo) - else: - var ee = - if e.str.isNil: cstring(nil) - else: cstring(e.str) - success = winlean.createProcessA(nil, - cmdl, nil, nil, 1, NORMAL_PRIORITY_CLASS, ee, wd, si, procInfo) + var tmp = newWideCString(cmdl) + var ee = + if e.str.isNil: newWideCString(cstring(nil)) + else: newWideCString(e.str, e.len) + var wwd = newWideCString(wd) + var flags = NORMAL_PRIORITY_CLASS or CREATE_UNICODE_ENVIRONMENT + if poDaemon in options: flags = flags or CREATE_NO_WINDOW + success = winlean.createProcessW(nil, tmp, nil, nil, 1, flags, + ee, wwd, si, procInfo) let lastError = osLastError() if poParentStreams notin options: @@ -741,7 +742,7 @@ when defined(windows) and not defined(useNimRtl): const errFileNotFound = 2.int if lastError.int in {errInvalidParameter, errFileNotFound}: raiseOSError(lastError, - "Requested command not found: '$1'. OS error:" % command) + "Requested command not found: '" & command & "'. OS error:") else: raiseOSError(lastError, command) result.fProcessHandle = procInfo.hProcess @@ -866,13 +867,9 @@ when defined(windows) and not defined(useNimRtl): si.hStdError = getStdHandle(STD_ERROR_HANDLE) si.hStdInput = getStdHandle(STD_INPUT_HANDLE) si.hStdOutput = getStdHandle(STD_OUTPUT_HANDLE) - when useWinUnicode: - var c = newWideCString(command) - var res = winlean.createProcessW(nil, c, nil, nil, 0, - NORMAL_PRIORITY_CLASS, nil, nil, si, procInfo) - else: - var res = winlean.createProcessA(nil, command, nil, nil, 0, - NORMAL_PRIORITY_CLASS, nil, nil, si, procInfo) + var c = newWideCString(command) + var res = winlean.createProcessW(nil, c, nil, nil, 0, + NORMAL_PRIORITY_CLASS, nil, nil, si, procInfo) if res == 0: raiseOSError(osLastError()) else: @@ -949,13 +946,13 @@ elif not defined(useNimRtl): not defined(useClone) and not defined(linux) when useProcessAuxSpawn: proc startProcessAuxSpawn(data: StartProcessData): Pid {. - tags: [ExecIOEffect, ReadEnvEffect, ReadDirEffect, RootEffect], gcsafe.} + raises: [OSError], tags: [ExecIOEffect, ReadEnvEffect, ReadDirEffect, RootEffect], gcsafe.} else: proc startProcessAuxFork(data: StartProcessData): Pid {. - tags: [ExecIOEffect, ReadEnvEffect, ReadDirEffect, RootEffect], gcsafe.} + raises: [OSError], tags: [ExecIOEffect, ReadEnvEffect, ReadDirEffect, RootEffect], gcsafe.} {.push stacktrace: off, profiler: off.} proc startProcessAfterFork(data: ptr StartProcessData) {. - tags: [ExecIOEffect, ReadEnvEffect, ReadDirEffect, RootEffect], cdecl, gcsafe.} + raises: [OSError], tags: [ExecIOEffect, ReadEnvEffect, ReadDirEffect, RootEffect], cdecl, gcsafe.} {.pop.} proc startProcess(command: string, workingDir: string = "", @@ -1056,13 +1053,15 @@ elif not defined(useNimRtl): var mask: Sigset chck sigemptyset(mask) chck posix_spawnattr_setsigmask(attr, mask) - if poDaemon in data.options: - chck posix_spawnattr_setpgroup(attr, 0'i32) + when not defined(nuttx): + if poDaemon in data.options: + chck posix_spawnattr_setpgroup(attr, 0'i32) var flags = POSIX_SPAWN_USEVFORK or POSIX_SPAWN_SETSIGMASK - if poDaemon in data.options: - flags = flags or POSIX_SPAWN_SETPGROUP + when not defined(nuttx): + if poDaemon in data.options: + flags = flags or POSIX_SPAWN_SETPGROUP chck posix_spawnattr_setflags(attr, flags) if not (poParentStreams in data.options): @@ -1082,9 +1081,9 @@ elif not defined(useNimRtl): var pid: Pid if (poUsePath in data.options): - res = posix_spawnp(pid, data.sysCommand, fops, attr, data.sysArgs, data.sysEnv) + res = posix_spawnp(pid, data.sysCommand.cstring, fops, attr, data.sysArgs, data.sysEnv) else: - res = posix_spawn(pid, data.sysCommand, fops, attr, data.sysArgs, data.sysEnv) + res = posix_spawn(pid, data.sysCommand.cstring, fops, attr, data.sysArgs, data.sysEnv) discard posix_spawn_file_actions_destroy(fops) discard posix_spawnattr_destroy(attr) @@ -1124,15 +1123,13 @@ elif not defined(useNimRtl): var error: cint let sizeRead = read(data.pErrorPipe[readIdx], addr error, sizeof(error)) if sizeRead == sizeof(error): - raiseOSError(osLastError(), - "Could not find command: '$1'. OS error: $2" % - [$data.sysCommand, $strerror(error)]) + raiseOSError(OSErrorCode(error), + "Could not find command: '" & $data.sysCommand & "'. OS error: " & $strerror(error)) return pid {.push stacktrace: off, profiler: off.} - proc startProcessFail(data: ptr StartProcessData) = - var error: cint = errno + proc startProcessFail(data: ptr StartProcessData, error: cint = errno) = discard write(data.pErrorPipe[writeIdx], addr error, sizeof(error)) exitnow(1) @@ -1169,15 +1166,19 @@ elif not defined(useNimRtl): if (poUsePath in data.options): when defined(uClibc) or defined(linux) or defined(haiku): # uClibc environment (OpenWrt included) doesn't have the full execvpe - let exe = findExe(data.sysCommand) - discard execve(exe, data.sysArgs, data.sysEnv) + var exe: string + try: + exe = findExe(data.sysCommand) + except OSError as e: + startProcessFail(data, e.errorCode) + discard execve(exe.cstring, data.sysArgs, data.sysEnv) else: # MacOSX doesn't have execvpe, so we need workaround. # On MacOSX we can arrive here only from fork, so this is safe: environ = data.sysEnv - discard execvp(data.sysCommand, data.sysArgs) + discard execvp(data.sysCommand.cstring, data.sysArgs) else: - discard execve(data.sysCommand, data.sysArgs, data.sysEnv) + discard execve(data.sysCommand.cstring, data.sysArgs, data.sysEnv) startProcessFail(data) {.pop.} @@ -1233,7 +1234,7 @@ elif not defined(useNimRtl): when defined(macosx) or defined(freebsd) or defined(netbsd) or defined(openbsd) or defined(dragonfly): - import kqueue + import std/kqueue proc waitForExit(p: Process, timeout: int = -1): int = if p.exitFlag: @@ -1350,119 +1351,68 @@ elif not defined(useNimRtl): p.exitStatus = status break else: - doAssert false, "unreachable!" + raiseAssert "unreachable!" result = exitStatusLikeShell(p.exitStatus) else: - import times - - const - hasThreadSupport = compileOption("threads") and not defined(nimscript) + import std/times except getTime + import std/monotimes proc waitForExit(p: Process, timeout: int = -1): int = - template adjustTimeout(t, s, e: Timespec) = - var diff: int - var b: Timespec - b.tv_sec = e.tv_sec - b.tv_nsec = e.tv_nsec - e.tv_sec = e.tv_sec - s.tv_sec - if e.tv_nsec >= s.tv_nsec: - e.tv_nsec -= s.tv_nsec - else: - if e.tv_sec == posix.Time(0): - raise newException(ValueError, "System time was modified") - else: - diff = s.tv_nsec - e.tv_nsec - e.tv_nsec = 1_000_000_000 - diff - t.tv_sec = t.tv_sec - e.tv_sec - if t.tv_nsec >= e.tv_nsec: - t.tv_nsec -= e.tv_nsec - else: - t.tv_sec = t.tv_sec - posix.Time(1) - diff = e.tv_nsec - t.tv_nsec - t.tv_nsec = 1_000_000_000 - diff - s.tv_sec = b.tv_sec - s.tv_nsec = b.tv_nsec - if p.exitFlag: return exitStatusLikeShell(p.exitStatus) - if timeout == -1: - var status: cint = 1 + if timeout < 0: + # Backwards compatibility with previous verison to + # handle cases where timeout == -1, but extend + # to handle cases where timeout < 0 + var status: cint if waitpid(p.id, status, 0) < 0: raiseOSError(osLastError()) p.exitFlag = true p.exitStatus = status else: - var nmask, omask: Sigset - var sinfo: SigInfo - var stspec, enspec, tmspec: Timespec - - discard sigemptyset(nmask) - discard sigemptyset(omask) - discard sigaddset(nmask, SIGCHLD) - - when hasThreadSupport: - if pthread_sigmask(SIG_BLOCK, nmask, omask) == -1: - raiseOSError(osLastError()) - else: - if sigprocmask(SIG_BLOCK, nmask, omask) == -1: - raiseOSError(osLastError()) - - if timeout >= 1000: - tmspec.tv_sec = posix.Time(timeout div 1_000) - tmspec.tv_nsec = (timeout %% 1_000) * 1_000_000 - else: - tmspec.tv_sec = posix.Time(0) - tmspec.tv_nsec = (timeout * 1_000_000) - - try: - if clock_gettime(CLOCK_REALTIME, stspec) == -1: - raiseOSError(osLastError()) - while true: - let res = sigtimedwait(nmask, sinfo, tmspec) - if res == SIGCHLD: - if sinfo.si_pid == p.id: - var status: cint = 1 - if waitpid(p.id, status, 0) < 0: - raiseOSError(osLastError()) - p.exitFlag = true - p.exitStatus = status - break - else: - # we have SIGCHLD, but not for process we are waiting, - # so we need to adjust timeout value and continue - if clock_gettime(CLOCK_REALTIME, enspec) == -1: - raiseOSError(osLastError()) - adjustTimeout(tmspec, stspec, enspec) - elif res < 0: - let err = osLastError() - if err.cint == EINTR: - # we have received another signal, so we need to - # adjust timeout and continue - if clock_gettime(CLOCK_REALTIME, enspec) == -1: - raiseOSError(osLastError()) - adjustTimeout(tmspec, stspec, enspec) - elif err.cint == EAGAIN: - # timeout expired, so we trying to kill process - if posix.kill(p.id, SIGKILL) == -1: - raiseOSError(osLastError()) - var status: cint = 1 - if waitpid(p.id, status, 0) < 0: - raiseOSError(osLastError()) - p.exitFlag = true - p.exitStatus = status - break - else: - raiseOSError(err) - finally: - when hasThreadSupport: - if pthread_sigmask(SIG_UNBLOCK, nmask, omask) == -1: - raiseOSError(osLastError()) + # Max 50ms delay + const maxWait = initDuration(milliseconds = 50) + let wait = initDuration(milliseconds = timeout) + let deadline = getMonoTime() + wait + # starting 50μs delay + var delay = initDuration(microseconds = 50) + + while true: + var status: cint + let pid = waitpid(p.id, status, WNOHANG) + if p.id == pid : + p.exitFlag = true + p.exitStatus = status + break + elif pid.int == -1: + raiseOsError(osLastError()) else: - if sigprocmask(SIG_UNBLOCK, nmask, omask) == -1: - raiseOSError(osLastError()) + # Continue waiting if needed + if getMonoTime() >= deadline: + # Previous version of `waitForExit` + # foricibly killed the process. + # We keep this so we don't break programs + # that depend on this behavior + if posix.kill(p.id, SIGKILL) < 0: + raiseOSError(osLastError()) + else: + const max = 1_000_000_000 + let + newWait = getMonoTime() + delay + ticks = newWait.ticks() + ns = ticks mod max + secs = ticks div max + var + waitSpec: TimeSpec + unused: Timespec + waitSpec.tv_sec = posix.Time(secs) + waitSpec.tv_nsec = clong ns + discard posix.clock_nanosleep(CLOCK_MONOTONIC, TIMER_ABSTIME, waitSpec, unused) + let remaining = deadline - getMonoTime() + delay = min([delay * 2, remaining, maxWait]) result = exitStatusLikeShell(p.exitStatus) @@ -1519,7 +1469,7 @@ elif not defined(useNimRtl): header: "<stdlib.h>".} proc execCmd(command: string): int = - when defined(linux): + when defined(posix): let tmp = csystem(command) result = if tmp == -1: tmp else: exitStatusLikeShell(tmp) else: @@ -1572,7 +1522,7 @@ proc execCmdEx*(command: string, options: set[ProcessOption] = { poStdErrToStdOut, poUsePath}, env: StringTableRef = nil, workingDir = "", input = ""): tuple[ output: string, - exitCode: int] {.tags: + exitCode: int] {.raises: [OSError, IOError], tags: [ExecIOEffect, ReadIOEffect, RootEffect], gcsafe.} = ## A convenience proc that runs the `command`, and returns its `output` and ## `exitCode`. `env` and `workingDir` params behave as for `startProcess`. @@ -1589,16 +1539,16 @@ proc execCmdEx*(command: string, options: set[ProcessOption] = { ## <#execProcess,string,string,openArray[string],StringTableRef,set[ProcessOption]>`_ ## ## Example: - ## - ## .. code-block:: Nim + ## ```Nim ## var result = execCmdEx("nim r --hints:off -", options = {}, input = "echo 3*4") ## import std/[strutils, strtabs] ## stripLineEnd(result[0]) ## portable way to remove trailing newline, if any ## doAssert result == ("12", 0) - ## doAssert execCmdEx("ls --nonexistant").exitCode != 0 + ## doAssert execCmdEx("ls --nonexistent").exitCode != 0 ## when defined(posix): ## assert execCmdEx("echo $FO", env = newStringTable({"FO": "B"})) == ("B\n", 0) ## assert execCmdEx("echo $PWD", workingDir = "/") == ("/\n", 0) + ## ``` when (NimMajor, NimMinor, NimPatch) < (1, 3, 5): doAssert input.len == 0 @@ -1618,6 +1568,7 @@ proc execCmdEx*(command: string, options: set[ProcessOption] = { inputStream(p).write(input) close inputStream(p) + # consider `p.lines(keepNewLines=true)` to avoid exit code test result = ("", -1) var line = newStringOfCap(120) while true: diff --git a/lib/pure/oswalkdir.nim b/lib/pure/oswalkdir.nim deleted file mode 100644 index 866f9ed70..000000000 --- a/lib/pure/oswalkdir.nim +++ /dev/null @@ -1,13 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## This module is deprecated, `import os` instead. -{.deprecated: "import os.nim instead".} -import os -export PathComponent, walkDir, walkDirRec diff --git a/lib/pure/parsecfg.nim b/lib/pure/parsecfg.nim index 23bb09ac4..8a43daf54 100644 --- a/lib/pure/parsecfg.nim +++ b/lib/pure/parsecfg.nim @@ -19,99 +19,97 @@ ## :literal: ## ## Here is an example of how to use the configuration file parser: -## -## .. code-block:: nim -## -## import std/[os, parsecfg, strutils, streams] -## -## var f = newFileStream(paramStr(1), fmRead) -## if f != nil: -## var p: CfgParser -## open(p, f, paramStr(1)) -## while true: -## var e = next(p) -## case e.kind -## of cfgEof: break -## of cfgSectionStart: ## a `[section]` has been parsed -## echo("new section: " & e.section) -## of cfgKeyValuePair: -## echo("key-value-pair: " & e.key & ": " & e.value) -## of cfgOption: -## echo("command: " & e.key & ": " & e.value) -## of cfgError: -## echo(e.msg) -## close(p) -## else: -## echo("cannot open: " & paramStr(1)) -## -## -## Examples -## ======== -## +runnableExamples("-r:off"): + import std/[strutils, streams] + + let configFile = "example.ini" + var f = newFileStream(configFile, fmRead) + assert f != nil, "cannot open " & configFile + var p: CfgParser + open(p, f, configFile) + while true: + var e = next(p) + case e.kind + of cfgEof: break + of cfgSectionStart: ## a `[section]` has been parsed + echo "new section: " & e.section + of cfgKeyValuePair: + echo "key-value-pair: " & e.key & ": " & e.value + of cfgOption: + echo "command: " & e.key & ": " & e.value + of cfgError: + echo e.msg + close(p) + +##[ ## Configuration file example -## -------------------------- -## -## .. code-block:: nim -## +]## + +## ```none ## charset = "utf-8" ## [Package] ## name = "hello" ## --threads:on ## [Author] -## name = "lihf8515" -## qq = "10214028" -## email = "lihaifeng@wxm.com" -## +## name = "nim-lang" +## website = "nim-lang.org" +## ``` + +##[ ## Creating a configuration file -## ----------------------------- -## .. code-block:: nim -## -## import std/parsecfg -## var dict=newConfig() -## dict.setSectionKey("","charset","utf-8") -## dict.setSectionKey("Package","name","hello") -## dict.setSectionKey("Package","--threads","on") -## dict.setSectionKey("Author","name","lihf8515") -## dict.setSectionKey("Author","qq","10214028") -## dict.setSectionKey("Author","email","lihaifeng@wxm.com") -## dict.writeConfig("config.ini") -## +]## + +runnableExamples: + var dict = newConfig() + dict.setSectionKey("","charset", "utf-8") + dict.setSectionKey("Package", "name", "hello") + dict.setSectionKey("Package", "--threads", "on") + dict.setSectionKey("Author", "name", "nim-lang") + dict.setSectionKey("Author", "website", "nim-lang.org") + assert $dict == """ +charset=utf-8 +[Package] +name=hello +--threads:on +[Author] +name=nim-lang +website=nim-lang.org +""" + +##[ ## Reading a configuration file -## ---------------------------- -## .. code-block:: nim -## -## import std/parsecfg -## var dict = loadConfig("config.ini") -## var charset = dict.getSectionValue("","charset") -## var threads = dict.getSectionValue("Package","--threads") -## var pname = dict.getSectionValue("Package","name") -## var name = dict.getSectionValue("Author","name") -## var qq = dict.getSectionValue("Author","qq") -## var email = dict.getSectionValue("Author","email") -## echo pname & "\n" & name & "\n" & qq & "\n" & email -## +]## + +runnableExamples("-r:off"): + let dict = loadConfig("config.ini") + let charset = dict.getSectionValue("","charset") + let threads = dict.getSectionValue("Package","--threads") + let pname = dict.getSectionValue("Package","name") + let name = dict.getSectionValue("Author","name") + let website = dict.getSectionValue("Author","website") + echo pname & "\n" & name & "\n" & website + +##[ ## Modifying a configuration file -## ------------------------------ -## .. code-block:: nim -## -## import std/parsecfg -## var dict = loadConfig("config.ini") -## dict.setSectionKey("Author","name","lhf") -## dict.writeConfig("config.ini") -## +]## + +runnableExamples("-r:off"): + var dict = loadConfig("config.ini") + dict.setSectionKey("Author", "name", "nim-lang") + dict.writeConfig("config.ini") + +##[ ## Deleting a section key in a configuration file -## ---------------------------------------------- -## .. code-block:: nim -## -## import std/parsecfg -## var dict = loadConfig("config.ini") -## dict.delSectionKey("Author","email") -## dict.writeConfig("config.ini") -## +]## + +runnableExamples("-r:off"): + var dict = loadConfig("config.ini") + dict.delSectionKey("Author", "website") + dict.writeConfig("config.ini") + +##[ ## Supported INI File structure -## ---------------------------- -## The examples below are supported: -## +]## # taken from https://docs.python.org/3/library/configparser.html#supported-ini-file-structure runnableExamples: @@ -150,33 +148,38 @@ runnableExamples: ) let section1 = "Simple Values" - doAssert dict.getSectionValue(section1, "key") == "value" - doAssert dict.getSectionValue(section1, "spaces in keys") == "allowed" - doAssert dict.getSectionValue(section1, "spaces in values") == "allowed as well" - doAssert dict.getSectionValue(section1, "spaces around the delimiter") == "obviously" - doAssert dict.getSectionValue(section1, "you can also use") == "to delimit keys from values" + assert dict.getSectionValue(section1, "key") == "value" + assert dict.getSectionValue(section1, "spaces in keys") == "allowed" + assert dict.getSectionValue(section1, "spaces in values") == "allowed as well" + assert dict.getSectionValue(section1, "spaces around the delimiter") == "obviously" + assert dict.getSectionValue(section1, "you can also use") == "to delimit keys from values" let section2 = "All Values Are Strings" - doAssert dict.getSectionValue(section2, "values like this") == "19990429" - doAssert dict.getSectionValue(section2, "or this") == "3.14159265359" - doAssert dict.getSectionValue(section2, "are they treated as numbers") == "no" - doAssert dict.getSectionValue(section2, "integers floats and booleans are held as") == "strings" - doAssert dict.getSectionValue(section2, "can use the API to get converted values directly") == "true" + assert dict.getSectionValue(section2, "values like this") == "19990429" + assert dict.getSectionValue(section2, "or this") == "3.14159265359" + assert dict.getSectionValue(section2, "are they treated as numbers") == "no" + assert dict.getSectionValue(section2, "integers floats and booleans are held as") == "strings" + assert dict.getSectionValue(section2, "can use the API to get converted values directly") == "true" let section3 = "Seletion A" - doAssert dict.getSectionValue(section3, + assert dict.getSectionValue(section3, "space around section name will be ignored", "not an empty value") == "" let section4 = "Sections Can Be Indented" - doAssert dict.getSectionValue(section4, "can_values_be_as_well") == "True" - doAssert dict.getSectionValue(section4, "does_that_mean_anything_special") == "False" - doAssert dict.getSectionValue(section4, "purpose") == "formatting for readability" + assert dict.getSectionValue(section4, "can_values_be_as_well") == "True" + assert dict.getSectionValue(section4, "does_that_mean_anything_special") == "False" + assert dict.getSectionValue(section4, "purpose") == "formatting for readability" -import strutils, lexbase, streams, tables +import std/[strutils, lexbase, streams, tables] import std/private/decode_helpers +import std/private/since + +when defined(nimPreviewSlimSystem): + import std/syncio include "system/inclrtl" + type CfgEventKind* = enum ## enumeration of all events that may occur when parsing cfgEof, ## end of file reached @@ -449,7 +452,7 @@ proc getKeyValPair(c: var CfgParser, kind: CfgEventKind): CfgEvent = if c.tok.kind == tkSymbol: case kind of cfgOption, cfgKeyValuePair: - result = CfgEvent(kind: kind, key: c.tok.literal, value: "") + result = CfgEvent(kind: kind, key: c.tok.literal.move, value: "") else: discard rawGetTok(c, c.tok) if c.tok.kind in {tkEquals, tkColon}: @@ -478,7 +481,7 @@ proc next*(c: var CfgParser): CfgEvent {.rtl, extern: "npc$1".} = of tkBracketLe: rawGetTok(c, c.tok) if c.tok.kind == tkSymbol: - result = CfgEvent(kind: cfgSectionStart, section: c.tok.literal) + result = CfgEvent(kind: cfgSectionStart, section: c.tok.literal.move) else: result = CfgEvent(kind: cfgError, msg: errorStr(c, "symbol expected, but found: " & c.tok.literal)) @@ -495,17 +498,17 @@ proc next*(c: var CfgParser): CfgEvent {.rtl, extern: "npc$1".} = # ---------------- Configuration file related operations ---------------- type - Config* = OrderedTableRef[string, <//>OrderedTableRef[string, string]] + Config* = OrderedTableRef[string, OrderedTableRef[string, string]] proc newConfig*(): Config = ## Creates a new configuration table. ## Useful when wanting to create a configuration file. - result = newOrderedTable[string, <//>OrderedTableRef[string, string]]() + result = newOrderedTable[string, OrderedTableRef[string, string]]() -proc loadConfig*(stream: Stream, filename: string = "[stream]"): <//>Config = +proc loadConfig*(stream: Stream, filename: string = "[stream]"): Config = ## Loads the specified configuration from stream into a new Config instance. ## `filename` parameter is only used for nicer error messages. - var dict = newOrderedTable[string, <//>OrderedTableRef[string, string]]() + var dict = newOrderedTable[string, OrderedTableRef[string, string]]() var curSection = "" ## Current section, ## the default value of the current section is "", ## which means that the current section is a common @@ -535,7 +538,7 @@ proc loadConfig*(stream: Stream, filename: string = "[stream]"): <//>Config = close(p) result = dict -proc loadConfig*(filename: string): <//>Config = +proc loadConfig*(filename: string): Config = ## Loads the specified configuration file into a new Config instance. let file = open(filename, fmRead) let fileStream = newFileStream(file) @@ -563,7 +566,7 @@ proc replace(s: string): string = proc writeConfig*(dict: Config, stream: Stream) = ## Writes the contents of the table to the specified stream. ## - ## **Note:** Comment statement will be ignored. + ## .. note:: Comment statement will be ignored. for section, sectionData in dict.pairs(): if section != "": ## Not general section if not allCharsInSet(section, SymChars): ## Non system character @@ -603,7 +606,7 @@ proc writeConfig*(dict: Config, stream: Stream) = proc `$`*(dict: Config): string = ## Writes the contents of the table to string. ## - ## **Note:** Comment statement will be ignored. + ## .. note:: Comment statement will be ignored. let stream = newStringStream() defer: stream.close() dict.writeConfig(stream) @@ -612,7 +615,7 @@ proc `$`*(dict: Config): string = proc writeConfig*(dict: Config, filename: string) = ## Writes the contents of the table to the specified configuration file. ## - ## **Note:** Comment statement will be ignored. + ## .. note:: Comment statement will be ignored. let file = open(filename, fmWrite) defer: file.close() let fileStream = newFileStream(file) @@ -649,3 +652,8 @@ proc delSectionKey*(dict: var Config, section, key: string) = dict.del(section) else: dict[section].del(key) + +iterator sections*(dict: Config): lent string {.since: (1, 5).} = + ## Iterates through the sections in the `dict`. + for section in dict.keys: + yield section diff --git a/lib/pure/parsecsv.nim b/lib/pure/parsecsv.nim index 1e34f3649..c7bf0c9c1 100644 --- a/lib/pure/parsecsv.nim +++ b/lib/pure/parsecsv.nim @@ -13,7 +13,7 @@ ## Basic usage ## =========== ## -## .. code-block:: nim +## ```nim ## import std/parsecsv ## from std/os import paramStr ## from std/streams import newFileStream @@ -29,11 +29,12 @@ ## for val in items(x.row): ## echo "##", val, "##" ## close(x) +## ``` ## ## For CSV files with a header row, the header can be read and then used as a ## reference for item access with `rowEntry <#rowEntry,CsvParser,string>`_: ## -## .. code-block:: nim +## ```nim ## import std/parsecsv ## ## # Prepare a file @@ -52,6 +53,7 @@ ## for col in items(p.headers): ## echo "##", col, ":", p.rowEntry(col), "##" ## p.close() +## ``` ## ## See also ## ======== @@ -65,8 +67,10 @@ ## * `parsesql module <parsesql.html>`_ for a SQL parser ## * `other parsers <lib.html#pure-libraries-parsers>`_ for other parsers -import - lexbase, streams +import std/[lexbase, streams] + +when defined(nimPreviewSlimSystem): + import std/syncio type CsvRow* = seq[string] ## A row in a CSV file. @@ -97,10 +101,10 @@ proc raiseEInvalidCsv(filename: string, line, col: int, e.msg = filename & "(" & $line & ", " & $col & ") Error: " & msg raise e -proc error(my: CsvParser, pos: int, msg: string) = - raiseEInvalidCsv(my.filename, my.lineNumber, getColNumber(my, pos), msg) +proc error(self: CsvParser, pos: int, msg: string) = + raiseEInvalidCsv(self.filename, self.lineNumber, getColNumber(self, pos), msg) -proc open*(my: var CsvParser, input: Stream, filename: string, +proc open*(self: var CsvParser, input: Stream, filename: string, separator = ',', quote = '"', escape = '\0', skipInitialSpace = false) = ## Initializes the parser with an input stream. `Filename` is only used @@ -127,16 +131,14 @@ proc open*(my: var CsvParser, input: Stream, filename: string, parser.close() strm.close() - lexbase.open(my, input) - my.filename = filename - my.sep = separator - my.quote = quote - my.esc = escape - my.skipWhite = skipInitialSpace - my.row = @[] - my.currRow = 0 + lexbase.open(self, input) + self.filename = filename + self.sep = separator + self.quote = quote + self.esc = escape + self.skipWhite = skipInitialSpace -proc open*(my: var CsvParser, filename: string, +proc open*(self: var CsvParser, filename: string, separator = ',', quote = '"', escape = '\0', skipInitialSpace = false) = ## Similar to the `other open proc<#open,CsvParser,Stream,string,char,char,char>`_, @@ -150,54 +152,54 @@ proc open*(my: var CsvParser, filename: string, removeFile("tmp.csv") var s = newFileStream(filename, fmRead) - if s == nil: my.error(0, "cannot open: " & filename) - open(my, s, filename, separator, + if s == nil: self.error(0, "cannot open: " & filename) + open(self, s, filename, separator, quote, escape, skipInitialSpace) -proc parseField(my: var CsvParser, a: var string) = - var pos = my.bufpos - if my.skipWhite: - while my.buf[pos] in {' ', '\t'}: inc(pos) +proc parseField(self: var CsvParser, a: var string) = + var pos = self.bufpos + if self.skipWhite: + while self.buf[pos] in {' ', '\t'}: inc(pos) setLen(a, 0) # reuse memory - if my.buf[pos] == my.quote and my.quote != '\0': + if self.buf[pos] == self.quote and self.quote != '\0': inc(pos) while true: - let c = my.buf[pos] + let c = self.buf[pos] if c == '\0': - my.bufpos = pos # can continue after exception? - error(my, pos, my.quote & " expected") + self.bufpos = pos # can continue after exception? + error(self, pos, self.quote & " expected") break - elif c == my.quote: - if my.esc == '\0' and my.buf[pos+1] == my.quote: - add(a, my.quote) + elif c == self.quote: + if self.esc == '\0' and self.buf[pos + 1] == self.quote: + add(a, self.quote) inc(pos, 2) else: inc(pos) break - elif c == my.esc: - add(a, my.buf[pos+1]) + elif c == self.esc: + add(a, self.buf[pos + 1]) inc(pos, 2) else: case c of '\c': - pos = handleCR(my, pos) + pos = handleCR(self, pos) add(a, "\n") of '\l': - pos = handleLF(my, pos) + pos = handleLF(self, pos) add(a, "\n") else: add(a, c) inc(pos) else: while true: - let c = my.buf[pos] - if c == my.sep: break + let c = self.buf[pos] + if c == self.sep: break if c in {'\c', '\l', '\0'}: break add(a, c) inc(pos) - my.bufpos = pos + self.bufpos = pos -proc processedRows*(my: var CsvParser): int = +proc processedRows*(self: var CsvParser): int {.inline.} = ## Returns number of the processed rows. ## ## But even if `readRow <#readRow,CsvParser,int>`_ arrived at EOF then @@ -220,9 +222,9 @@ proc processedRows*(my: var CsvParser): int = parser.close() strm.close() - return my.currRow + self.currRow -proc readRow*(my: var CsvParser, columns = 0): bool = +proc readRow*(self: var CsvParser, columns = 0): bool = ## Reads the next row; if `columns` > 0, it expects the row to have ## exactly this many columns. Returns false if the end of the file ## has been encountered else true. @@ -251,47 +253,47 @@ proc readRow*(my: var CsvParser, columns = 0): bool = strm.close() var col = 0 # current column - let oldpos = my.bufpos + let oldpos = self.bufpos # skip initial empty lines #8365 while true: - case my.buf[my.bufpos] - of '\c': my.bufpos = handleCR(my, my.bufpos) - of '\l': my.bufpos = handleLF(my, my.bufpos) + case self.buf[self.bufpos] + of '\c': self.bufpos = handleCR(self, self.bufpos) + of '\l': self.bufpos = handleLF(self, self.bufpos) else: break - while my.buf[my.bufpos] != '\0': - let oldlen = my.row.len - if oldlen < col+1: - setLen(my.row, col+1) - my.row[col] = "" - parseField(my, my.row[col]) + while self.buf[self.bufpos] != '\0': + let oldlen = self.row.len + if oldlen < col + 1: + setLen(self.row, col + 1) + self.row[col] = "" + parseField(self, self.row[col]) inc(col) - if my.buf[my.bufpos] == my.sep: - inc(my.bufpos) + if self.buf[self.bufpos] == self.sep: + inc(self.bufpos) else: - case my.buf[my.bufpos] + case self.buf[self.bufpos] of '\c', '\l': # skip empty lines: while true: - case my.buf[my.bufpos] - of '\c': my.bufpos = handleCR(my, my.bufpos) - of '\l': my.bufpos = handleLF(my, my.bufpos) + case self.buf[self.bufpos] + of '\c': self.bufpos = handleCR(self, self.bufpos) + of '\l': self.bufpos = handleLF(self, self.bufpos) else: break of '\0': discard - else: error(my, my.bufpos, my.sep & " expected") + else: error(self, self.bufpos, self.sep & " expected") break - setLen(my.row, col) + setLen(self.row, col) result = col > 0 if result and col != columns and columns > 0: - error(my, oldpos+1, $columns & " columns expected, but found " & + error(self, oldpos + 1, $columns & " columns expected, but found " & $col & " columns") - inc(my.currRow) + inc(self.currRow) -proc close*(my: var CsvParser) {.inline.} = - ## Closes the parser `my` and its associated input stream. - lexbase.close(my) +proc close*(self: var CsvParser) {.inline.} = + ## Closes the parser `self` and its associated input stream. + lexbase.close(self) -proc readHeaderRow*(my: var CsvParser) = +proc readHeaderRow*(self: var CsvParser) = ## Reads the first row and creates a look-up table for column numbers ## See also: ## * `rowEntry proc <#rowEntry,CsvParser,string>`_ @@ -313,16 +315,16 @@ proc readHeaderRow*(my: var CsvParser) = parser.close() strm.close() - let present = my.readRow() + let present = self.readRow() if present: - my.headers = my.row + self.headers = self.row -proc rowEntry*(my: var CsvParser, entry: string): var string = +proc rowEntry*(self: var CsvParser, entry: string): var string = ## Accesses a specified `entry` from the current row. ## ## Assumes that `readHeaderRow <#readHeaderRow,CsvParser>`_ has already been ## called. - ## + ## ## If specified `entry` does not exist, raises KeyError. runnableExamples: import std/streams @@ -340,14 +342,14 @@ proc rowEntry*(my: var CsvParser, entry: string): var string = parser.close() strm.close() - let index = my.headers.find(entry) + let index = self.headers.find(entry) if index >= 0: - result = my.row[index] + result = self.row[index] else: raise newException(KeyError, "Entry `" & entry & "` doesn't exist") when not defined(testing) and isMainModule: - import os + import std/os var s = newFileStream(paramStr(1), fmRead) if s == nil: quit("cannot open the file" & paramStr(1)) var x: CsvParser diff --git a/lib/pure/parsejson.nim b/lib/pure/parsejson.nim index 196d8c360..9292a8596 100644 --- a/lib/pure/parsejson.nim +++ b/lib/pure/parsejson.nim @@ -11,9 +11,12 @@ ## and exported by the `json` standard library ## module, but can also be used in its own right. -import strutils, lexbase, streams, unicode +import std/[strutils, lexbase, streams, unicode] import std/private/decode_helpers +when defined(nimPreviewSlimSystem): + import std/assertions + type JsonEventKind* = enum ## enumeration of all events that may occur when parsing jsonError, ## an error occurred during parsing @@ -218,7 +221,7 @@ proc parseString(my: var JsonParser): TokKind = add(my.a, 'u') inc(pos, 2) var pos2 = pos - var r = parseEscapedUTF16(my.buf, pos) + var r = parseEscapedUTF16(cstring(my.buf), pos) if r < 0: my.err = errInvalidToken break @@ -228,7 +231,7 @@ proc parseString(my: var JsonParser): TokKind = my.err = errInvalidToken break inc(pos, 2) - var s = parseEscapedUTF16(my.buf, pos) + var s = parseEscapedUTF16(cstring(my.buf), pos) if (s and 0xfc00) == 0xdc00 and s > 0: r = 0x10000 + (((r - 0xd800) shl 10) or (s - 0xdc00)) else: diff --git a/lib/pure/parseopt.nim b/lib/pure/parseopt.nim index b2d024a39..03f151b66 100644 --- a/lib/pure/parseopt.nim +++ b/lib/pure/parseopt.nim @@ -16,7 +16,7 @@ ## ## The following syntax is supported when arguments for the `shortNoVal` and ## `longNoVal` parameters, which are -## `described later<#shortnoval-and-longnoval>`_, are not provided: +## `described later<#nimshortnoval-and-nimlongnoval>`_, are not provided: ## ## 1. Short options: `-abcd`, `-e:5`, `-e=5` ## 2. Long options: `--foo:bar`, `--foo=bar`, `--foo` @@ -48,7 +48,7 @@ ## ## Here is an example: ## -## .. code-block:: +## ```Nim ## import std/parseopt ## ## var p = initOptParser("-ab -e:5 --foo --bar=20 file.txt") @@ -71,10 +71,34 @@ ## # Option: foo ## # Option and value: bar, 20 ## # Argument: file.txt +## ``` ## ## The `getopt iterator<#getopt.i,OptParser>`_, which is provided for ## convenience, can be used to iterate through all command line options as well. ## +## To set a default value for a variable assigned through `getopt` and accept arguments from the cmd line. +## Assign the default value to a variable before parsing. +## Then set the variable to the new value while parsing. +## +## Here is an example: +## +## ```Nim +## import std/parseopt +## +## var varName: string = "defaultValue" +## +## for kind, key, val in getopt(): +## case kind +## of cmdArgument: +## discard +## of cmdLongOption, cmdShortOption: +## case key: +## of "varName": # --varName:<value> in the console when executing +## varName = val # do input sanitization in production systems +## of cmdEnd: +## discard +## ``` +## ## `shortNoVal` and `longNoVal` ## ============================ ## @@ -98,7 +122,7 @@ ## `shortNoVal` and `longNoVal`, which is the default, and providing ## arguments for those two parameters: ## -## .. code-block:: +## ```Nim ## import std/parseopt ## ## proc printToken(kind: CmdLineKind, key: string, val: string) = @@ -132,6 +156,7 @@ ## # Output: ## # Option and value: j, 4 ## # Option and value: first, bar +## ``` ## ## See also ## ======== @@ -151,7 +176,8 @@ include "system/inclrtl" -import os +import std/strutils +import std/os type CmdLineKind* = enum ## The detected command line token. @@ -164,7 +190,7 @@ type ## ## To initialize it, use the ## `initOptParser proc<#initOptParser,string,set[char],seq[string]>`_. - pos*: int + pos: int inShortState: bool allowWhitespaceAfterColon: bool shortNoVal: set[char] @@ -192,26 +218,24 @@ proc parseWord(s: string, i: int, w: var string, add(w, s[result]) inc(result) -proc initOptParser*(cmdline = "", shortNoVal: set[char] = {}, +proc initOptParser*(cmdline: seq[string], shortNoVal: set[char] = {}, longNoVal: seq[string] = @[]; allowWhitespaceAfterColon = true): OptParser = ## Initializes the command line parser. ## - ## If `cmdline == ""`, the real command line as provided by the + ## If `cmdline.len == 0`, the real command line as provided by the ## `os` module is retrieved instead if it is available. If the ## command line is not available, a `ValueError` will be raised. - ## - ## `shortNoVal` and `longNoVal` are used to specify which options - ## do not take values. See the `documentation about these - ## parameters<#shortnoval-and-longnoval>`_ for more information on - ## how this affects parsing. + ## Behavior of the other parameters remains the same as in + ## `initOptParser(string, ...) + ## <#initOptParser,string,set[char],seq[string]>`_. ## ## See also: - ## * `getopt iterator<#getopt.i,OptParser>`_ + ## * `getopt iterator<#getopt.i,seq[string],set[char],seq[string]>`_ runnableExamples: var p = initOptParser() - p = initOptParser("--left --debug:3 -l -r:2") - p = initOptParser("--left --debug:3 -l -r:2", + p = initOptParser(@["--left", "--debug:3", "-l", "-r:2"]) + p = initOptParser(@["--left", "--debug:3", "-l", "-r:2"], shortNoVal = {'l'}, longNoVal = @["left"]) result.pos = 0 @@ -220,66 +244,60 @@ proc initOptParser*(cmdline = "", shortNoVal: set[char] = {}, result.shortNoVal = shortNoVal result.longNoVal = longNoVal result.allowWhitespaceAfterColon = allowWhitespaceAfterColon - if cmdline != "": - result.cmds = parseCmdLine(cmdline) + if cmdline.len != 0: + result.cmds = newSeq[string](cmdline.len) + for i in 0..<cmdline.len: + result.cmds[i] = cmdline[i] else: when declared(paramCount): - result.cmds = newSeq[string](paramCount()) - for i in countup(1, paramCount()): - result.cmds[i-1] = paramStr(i) + when defined(nimscript): + var ctr = 0 + var firstNimsFound = false + for i in countup(0, paramCount()): + if firstNimsFound: + result.cmds[ctr] = paramStr(i) + inc ctr, 1 + if paramStr(i).endsWith(".nims") and not firstNimsFound: + firstNimsFound = true + result.cmds = newSeq[string](paramCount()-i) + else: + result.cmds = newSeq[string](paramCount()) + for i in countup(1, paramCount()): + result.cmds[i-1] = paramStr(i) else: # we cannot provide this for NimRtl creation on Posix, because we can't # access the command line arguments then! - doAssert false, "empty command line given but" & + raiseAssert "empty command line given but" & " real command line is not accessible" - result.kind = cmdEnd result.key = "" result.val = "" -proc initOptParser*(cmdline: seq[string], shortNoVal: set[char] = {}, +proc initOptParser*(cmdline = "", shortNoVal: set[char] = {}, longNoVal: seq[string] = @[]; allowWhitespaceAfterColon = true): OptParser = ## Initializes the command line parser. ## - ## If `cmdline.len == 0`, the real command line as provided by the + ## If `cmdline == ""`, the real command line as provided by the ## `os` module is retrieved instead if it is available. If the ## command line is not available, a `ValueError` will be raised. - ## Behavior of the other parameters remains the same as in - ## `initOptParser(string, ...) - ## <#initOptParser,string,set[char],seq[string]>`_. + ## + ## `shortNoVal` and `longNoVal` are used to specify which options + ## do not take values. See the `documentation about these + ## parameters<#nimshortnoval-and-nimlongnoval>`_ for more information on + ## how this affects parsing. + ## + ## This does not provide a way of passing default values to arguments. ## ## See also: - ## * `getopt iterator<#getopt.i,seq[string],set[char],seq[string]>`_ + ## * `getopt iterator<#getopt.i,OptParser>`_ runnableExamples: var p = initOptParser() - p = initOptParser(@["--left", "--debug:3", "-l", "-r:2"]) - p = initOptParser(@["--left", "--debug:3", "-l", "-r:2"], + p = initOptParser("--left --debug:3 -l -r:2") + p = initOptParser("--left --debug:3 -l -r:2", shortNoVal = {'l'}, longNoVal = @["left"]) - result.pos = 0 - result.idx = 0 - result.inShortState = false - result.shortNoVal = shortNoVal - result.longNoVal = longNoVal - result.allowWhitespaceAfterColon = allowWhitespaceAfterColon - if cmdline.len != 0: - result.cmds = newSeq[string](cmdline.len) - for i in 0..<cmdline.len: - result.cmds[i] = cmdline[i] - else: - when declared(paramCount): - result.cmds = newSeq[string](paramCount()) - for i in countup(1, paramCount()): - result.cmds[i-1] = paramStr(i) - else: - # we cannot provide this for NimRtl creation on Posix, because we can't - # access the command line arguments then! - doAssert false, "empty command line given but" & - " real command line is not accessible" - result.kind = cmdEnd - result.key = "" - result.val = "" + initOptParser(parseCmdLine(cmdline), shortNoVal, longNoVal, allowWhitespaceAfterColon) proc handleShortOption(p: var OptParser; cmd: string) = var i = p.pos @@ -373,7 +391,7 @@ proc next*(p: var OptParser) {.rtl, extern: "npo$1".} = handleShortOption(p, p.cmds[p.idx]) else: p.kind = cmdArgument - p.key = p.cmds[p.idx] + p.key = p.cmds[p.idx] inc p.idx p.pos = 0 @@ -385,15 +403,14 @@ when declared(quoteShellCommand): ## * `remainingArgs proc<#remainingArgs,OptParser>`_ ## ## **Examples:** - ## - ## .. code-block:: + ## ```Nim ## var p = initOptParser("--left -r:2 -- foo.txt bar.txt") ## while true: ## p.next() ## if p.kind == cmdLongOption and p.key == "": # Look for "--" ## break - ## else: continue ## doAssert p.cmdLineRest == "foo.txt bar.txt" + ## ``` result = p.cmds[p.idx .. ^1].quoteShellCommand proc remainingArgs*(p: OptParser): seq[string] {.rtl, extern: "npo$1".} = @@ -403,15 +420,14 @@ proc remainingArgs*(p: OptParser): seq[string] {.rtl, extern: "npo$1".} = ## * `cmdLineRest proc<#cmdLineRest,OptParser>`_ ## ## **Examples:** - ## - ## .. code-block:: + ## ```Nim ## var p = initOptParser("--left -r:2 -- foo.txt bar.txt") ## while true: ## p.next() ## if p.kind == cmdLongOption and p.key == "": # Look for "--" ## break - ## else: continue ## doAssert p.remainingArgs == @["foo.txt", "bar.txt"] + ## ``` result = @[] for i in p.idx..<p.cmds.len: result.add p.cmds[i] @@ -420,14 +436,15 @@ iterator getopt*(p: var OptParser): tuple[kind: CmdLineKind, key, ## Convenience iterator for iterating over the given ## `OptParser<#OptParser>`_. ## - ## There is no need to check for `cmdEnd` while iterating. + ## There is no need to check for `cmdEnd` while iterating. If using `getopt` + ## with case switching, checking for `cmdEnd` is required. ## ## See also: ## * `initOptParser proc<#initOptParser,string,set[char],seq[string]>`_ ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## # these are placeholders, of course ## proc writeHelp() = discard ## proc writeVersion() = discard @@ -447,6 +464,7 @@ iterator getopt*(p: var OptParser): tuple[kind: CmdLineKind, key, ## if filename == "": ## # no filename has been given, so we show the help ## writeHelp() + ## ``` p.pos = 0 p.idx = 0 while true: @@ -465,18 +483,18 @@ iterator getopt*(cmdline: seq[string] = @[], ## ## `shortNoVal` and `longNoVal` are used to specify which options ## do not take values. See the `documentation about these - ## parameters<#shortnoval-and-longnoval>`_ for more information on + ## parameters<#nimshortnoval-and-nimlongnoval>`_ for more information on ## how this affects parsing. ## - ## There is no need to check for `cmdEnd` while iterating. + ## There is no need to check for `cmdEnd` while iterating. If using `getopt` + ## with case switching, checking for `cmdEnd` is required. ## ## See also: ## * `initOptParser proc<#initOptParser,seq[string],set[char],seq[string]>`_ ## ## **Examples:** ## - ## .. code-block:: - ## + ## ```Nim ## # these are placeholders, of course ## proc writeHelp() = discard ## proc writeVersion() = discard @@ -496,6 +514,7 @@ iterator getopt*(cmdline: seq[string] = @[], ## if filename == "": ## # no filename has been written, so we show the help ## writeHelp() + ## ``` var p = initOptParser(cmdline, shortNoVal = shortNoVal, longNoVal = longNoVal) while true: diff --git a/lib/pure/parsesql.nim b/lib/pure/parsesql.nim index eb5d7c2cc..a7c938d01 100644 --- a/lib/pure/parsesql.nim +++ b/lib/pure/parsesql.nim @@ -12,9 +12,12 @@ ## ## Unstable API. -import strutils, lexbase +import std/[strutils, lexbase] import std/private/decode_helpers +when defined(nimPreviewSlimSystem): + import std/assertions + # ------------------- scanner ------------------------------------------------- type @@ -57,7 +60,7 @@ const reservedKeywords = @[ # statements - "select", "from", "where", "group", "limit", "having", + "select", "from", "where", "group", "limit", "offset", "having", # functions "count", ] @@ -506,6 +509,7 @@ type nkFromItemPair, nkGroup, nkLimit, + nkOffset, nkHaving, nkOrder, nkJoin, @@ -982,7 +986,7 @@ proc parseInsert(p: var SqlParser): SqlNode = parseParIdentList(p, n) result.add n else: - result.add(nil) + result.add(newNode(nkNone)) if isKeyw(p, "default"): getTok(p) eat(p, "values") @@ -1017,7 +1021,7 @@ proc parseUpdate(p: var SqlParser): SqlNode = if isKeyw(p, "where"): result.add(parseWhere(p)) else: - result.add(nil) + result.add(newNode(nkNone)) proc parseDelete(p: var SqlParser): SqlNode = getTok(p) @@ -1029,7 +1033,7 @@ proc parseDelete(p: var SqlParser): SqlNode = if isKeyw(p, "where"): result.add(parseWhere(p)) else: - result.add(nil) + result.add(newNode(nkNone)) proc parseSelect(p: var SqlParser): SqlNode = getTok(p) @@ -1123,6 +1127,11 @@ proc parseSelect(p: var SqlParser): SqlNode = var l = newNode(nkLimit) l.add(parseExpr(p)) result.add(l) + if isKeyw(p, "offset"): + getTok(p) + var o = newNode(nkOffset) + o.add(parseExpr(p)) + result.add(o) proc parseStmt(p: var SqlParser; parent: SqlNode) = if isKeyw(p, "create"): @@ -1385,6 +1394,9 @@ proc ra(n: SqlNode, s: var SqlWriter) = of nkLimit: s.addKeyw("limit") s.addMulti(n) + of nkOffset: + s.addKeyw("offset") + s.addMulti(n) of nkHaving: s.addKeyw("having") s.addMulti(n) @@ -1451,7 +1463,7 @@ proc ra(n: SqlNode, s: var SqlWriter) = s.addKeyw("enum") rs(n, s) -proc renderSQL*(n: SqlNode, upperCase = false): string = +proc renderSql*(n: SqlNode, upperCase = false): string = ## Converts an SQL abstract syntax tree to its string representation. var s: SqlWriter s.buffer = "" @@ -1460,8 +1472,8 @@ proc renderSQL*(n: SqlNode, upperCase = false): string = return s.buffer proc `$`*(n: SqlNode): string = - ## an alias for `renderSQL`. - renderSQL(n) + ## an alias for `renderSql`. + renderSql(n) proc treeReprAux(s: SqlNode, level: int, result: var string) = result.add('\n') @@ -1479,7 +1491,7 @@ proc treeRepr*(s: SqlNode): string = result = newStringOfCap(128) treeReprAux(s, 0, result) -import streams +import std/streams proc open(L: var SqlLexer, input: Stream, filename: string) = lexbase.open(L, input) @@ -1493,7 +1505,7 @@ proc open(p: var SqlParser, input: Stream, filename: string) = p.tok.literal = "" getTok(p) -proc parseSQL*(input: Stream, filename: string): SqlNode = +proc parseSql*(input: Stream, filename: string): SqlNode = ## parses the SQL from `input` into an AST and returns the AST. ## `filename` is only used for error messages. ## Syntax errors raise an `SqlParseError` exception. @@ -1504,8 +1516,8 @@ proc parseSQL*(input: Stream, filename: string): SqlNode = finally: close(p) -proc parseSQL*(input: string, filename = ""): SqlNode = +proc parseSql*(input: string, filename = ""): SqlNode = ## parses the SQL from `input` into an AST and returns the AST. ## `filename` is only used for error messages. ## Syntax errors raise an `SqlParseError` exception. - parseSQL(newStringStream(input), "") + parseSql(newStringStream(input), "") diff --git a/lib/pure/parseutils.nim b/lib/pure/parseutils.nim index 64949428f..2ca255fa0 100644 --- a/lib/pure/parseutils.nim +++ b/lib/pure/parseutils.nim @@ -12,29 +12,28 @@ ## ## To unpack raw bytes look at the `streams <streams.html>`_ module. ## -## .. code-block:: nim -## :test: +## ```nim test +## let logs = @["2019-01-10: OK_", "2019-01-11: FAIL_", "2019-01: aaaa"] +## var outp: seq[string] ## -## let logs = @["2019-01-10: OK_", "2019-01-11: FAIL_", "2019-01: aaaa"] -## var outp: seq[string] +## for log in logs: +## var res: string +## if parseUntil(log, res, ':') == 10: # YYYY-MM-DD == 10 +## outp.add(res & " - " & captureBetween(log, ' ', '_')) +## doAssert outp == @["2019-01-10 - OK", "2019-01-11 - FAIL"] +## ``` ## -## for log in logs: -## var res: string -## if parseUntil(log, res, ':') == 10: # YYYY-MM-DD == 10 -## outp.add(res & " - " & captureBetween(log, ' ', '_')) -## doAssert outp == @["2019-01-10 - OK", "2019-01-11 - FAIL"] +## ```nim test +## from std/strutils import Digits, parseInt ## -## .. code-block:: nim -## :test: -## from std/strutils import Digits, parseInt -## -## let -## input1 = "2019 school start" -## input2 = "3 years back" -## startYear = input1[0 .. skipWhile(input1, Digits)-1] # 2019 -## yearsBack = input2[0 .. skipWhile(input2, Digits)-1] # 3 -## examYear = parseInt(startYear) + parseInt(yearsBack) -## doAssert "Examination is in " & $examYear == "Examination is in 2022" +## let +## input1 = "2019 school start" +## input2 = "3 years back" +## startYear = input1[0 .. skipWhile(input1, Digits)-1] # 2019 +## yearsBack = input2[0 .. skipWhile(input2, Digits)-1] # 3 +## examYear = parseInt(startYear) + parseInt(yearsBack) +## doAssert "Examination is in " & $examYear == "Examination is in 2022" +## ``` ## ## **See also:** ## * `strutils module<strutils.html>`_ for combined and identical parsing proc's @@ -50,6 +49,8 @@ include "system/inclrtl" +template toOa(s: string): openArray[char] = openArray[char](s) + const Whitespace = {' ', '\t', '\v', '\r', '\l', '\f'} IdentChars = {'a'..'z', 'A'..'Z', '0'..'9', '_'} @@ -59,8 +60,7 @@ const proc toLower(c: char): char {.inline.} = result = if c in {'A'..'Z'}: chr(ord(c)-ord('A')+ord('a')) else: c -proc parseBin*[T: SomeInteger](s: string, number: var T, start = 0, - maxLen = 0): int {.noSideEffect.} = +proc parseBin*[T: SomeInteger](s: openArray[char], number: var T, maxLen = 0): int {.noSideEffect.} = ## Parses a binary number and stores its value in ``number``. ## ## Returns the number of the parsed characters or 0 in case of an error. @@ -89,7 +89,7 @@ proc parseBin*[T: SomeInteger](s: string, number: var T, start = 0, var num64: int64 doAssert parseBin("0100111001101001111011010100111001101001", num64) == 40 doAssert num64 == 336784608873 - var i = start + var i = 0 var output = T(0) var foundDigit = false let last = min(s.len, if maxLen == 0: s.len else: i + maxLen) @@ -104,10 +104,9 @@ proc parseBin*[T: SomeInteger](s: string, number: var T, start = 0, inc(i) if foundDigit: number = output - result = i - start + result = i -proc parseOct*[T: SomeInteger](s: string, number: var T, start = 0, - maxLen = 0): int {.noSideEffect.} = +proc parseOct*[T: SomeInteger](s: openArray[char], number: var T, maxLen = 0): int {.noSideEffect.} = ## Parses an octal number and stores its value in ``number``. ## ## Returns the number of the parsed characters or 0 in case of an error. @@ -136,7 +135,7 @@ proc parseOct*[T: SomeInteger](s: string, number: var T, start = 0, var num64: int64 doAssert parseOct("2346475523464755", num64) == 16 doAssert num64 == 86216859871725 - var i = start + var i = 0 var output = T(0) var foundDigit = false let last = min(s.len, if maxLen == 0: s.len else: i + maxLen) @@ -151,10 +150,9 @@ proc parseOct*[T: SomeInteger](s: string, number: var T, start = 0, inc(i) if foundDigit: number = output - result = i - start + result = i -proc parseHex*[T: SomeInteger](s: string, number: var T, start = 0, - maxLen = 0): int {.noSideEffect.} = +proc parseHex*[T: SomeInteger](s: openArray[char], number: var T, maxLen = 0): int {.noSideEffect.} = ## Parses a hexadecimal number and stores its value in ``number``. ## ## Returns the number of the parsed characters or 0 in case of an error. @@ -184,7 +182,7 @@ proc parseHex*[T: SomeInteger](s: string, number: var T, start = 0, var num64: int64 doAssert parseHex("4E69ED4E69ED", num64) == 12 doAssert num64 == 86216859871725 - var i = start + var i = 0 var output = T(0) var foundDigit = false let last = min(s.len, if maxLen == 0: s.len else: i + maxLen) @@ -206,9 +204,9 @@ proc parseHex*[T: SomeInteger](s: string, number: var T, start = 0, inc(i) if foundDigit: number = output - result = i - start + result = i -proc parseIdent*(s: string, ident: var string, start = 0): int = +proc parseIdent*(s: openArray[char], ident: var string): int = ## Parses an identifier and stores it in ``ident``. Returns ## the number of the parsed characters or 0 in case of an error. ## If error, the value of `ident` is not changed. @@ -220,14 +218,14 @@ proc parseIdent*(s: string, ident: var string, start = 0): int = doAssert res == "ello" doAssert parseIdent("Hello World", res, 6) == 5 doAssert res == "World" - var i = start + var i = 0 if i < s.len and s[i] in IdentStartChars: inc(i) while i < s.len and s[i] in IdentChars: inc(i) - ident = substr(s, start, i-1) - result = i-start + ident = substr(s.toOpenArray(0, i-1)) + result = i -proc parseIdent*(s: string, start = 0): string = +proc parseIdent*(s: openArray[char]): string = ## Parses an identifier and returns it or an empty string in ## case of an error. runnableExamples: @@ -236,13 +234,13 @@ proc parseIdent*(s: string, start = 0): string = doAssert parseIdent("Hello World", 5) == "" doAssert parseIdent("Hello World", 6) == "World" result = "" - var i = start + var i = 0 if i < s.len and s[i] in IdentStartChars: inc(i) while i < s.len and s[i] in IdentChars: inc(i) - result = substr(s, start, i-1) + result = substr(s.toOpenArray(0, i - 1)) -proc parseChar*(s: string, c: var char, start = 0): int = +proc parseChar*(s: openArray[char], c: var char): int = ## Parses a single character, stores it in `c` and returns 1. ## In case of error (if start >= s.len) it returns 0 ## and the value of `c` is unchanged. @@ -252,11 +250,11 @@ proc parseChar*(s: string, c: var char, start = 0): int = doAssert c == '\0' doAssert "nim".parseChar(c, 0) == 1 doAssert c == 'n' - if start < s.len: - c = s[start] + if s.len > 0: + c = s[0] result = 1 -proc skipWhitespace*(s: string, start = 0): int {.inline.} = +proc skipWhitespace*(s: openArray[char]): int {.inline.} = ## Skips the whitespace starting at ``s[start]``. Returns the number of ## skipped characters. runnableExamples: @@ -265,9 +263,9 @@ proc skipWhitespace*(s: string, start = 0): int {.inline.} = doAssert skipWhitespace("Hello World", 5) == 1 doAssert skipWhitespace("Hello World", 5) == 2 result = 0 - while start+result < s.len and s[start+result] in Whitespace: inc(result) + while result < s.len and s[result] in Whitespace: inc(result) -proc skip*(s, token: string, start = 0): int {.inline.} = +proc skip*(s, token: openArray[char]): int {.inline.} = ## Skips the `token` starting at ``s[start]``. Returns the length of `token` ## or 0 if there was no `token` at ``s[start]``. runnableExamples: @@ -277,22 +275,22 @@ proc skip*(s, token: string, start = 0): int {.inline.} = doAssert skip("CAPlow", "CAP", 0) == 3 doAssert skip("CAPlow", "cap", 0) == 0 result = 0 - while start+result < s.len and result < token.len and - s[result+start] == token[result]: + while result < s.len and result < token.len and + s[result] == token[result]: inc(result) if result != token.len: result = 0 -proc skipIgnoreCase*(s, token: string, start = 0): int = +proc skipIgnoreCase*(s, token: openArray[char]): int = ## Same as `skip` but case is ignored for token matching. runnableExamples: doAssert skipIgnoreCase("CAPlow", "CAP", 0) == 3 doAssert skipIgnoreCase("CAPlow", "cap", 0) == 3 result = 0 - while start+result < s.len and result < token.len and - toLower(s[result+start]) == toLower(token[result]): inc(result) + while result < s.len and result < token.len and + toLower(s[result]) == toLower(token[result]): inc(result) if result != token.len: result = 0 -proc skipUntil*(s: string, until: set[char], start = 0): int {.inline.} = +proc skipUntil*(s: openArray[char], until: set[char]): int {.inline.} = ## Skips all characters until one char from the set `until` is found ## or the end is reached. ## Returns number of characters skipped. @@ -301,9 +299,9 @@ proc skipUntil*(s: string, until: set[char], start = 0): int {.inline.} = doAssert skipUntil("Hello World", {'W'}, 0) == 6 doAssert skipUntil("Hello World", {'W', 'd'}, 0) == 6 result = 0 - while start+result < s.len and s[result+start] notin until: inc(result) + while result < s.len and s[result] notin until: inc(result) -proc skipUntil*(s: string, until: char, start = 0): int {.inline.} = +proc skipUntil*(s: openArray[char], until: char): int {.inline.} = ## Skips all characters until the char `until` is found ## or the end is reached. ## Returns number of characters skipped. @@ -313,24 +311,23 @@ proc skipUntil*(s: string, until: char, start = 0): int {.inline.} = doAssert skipUntil("Hello World", 'W', 0) == 6 doAssert skipUntil("Hello World", 'w', 0) == 11 result = 0 - while start+result < s.len and s[result+start] != until: inc(result) + while result < s.len and s[result] != until: inc(result) -proc skipWhile*(s: string, toSkip: set[char], start = 0): int {.inline.} = - ## Skips all characters while one char from the set `token` is found. +proc skipWhile*(s: openArray[char], toSkip: set[char]): int {.inline.} = + ## Skips all characters while one char from the set `toSkip` is found. ## Returns number of characters skipped. runnableExamples: doAssert skipWhile("Hello World", {'H', 'e'}) == 2 doAssert skipWhile("Hello World", {'e'}) == 0 doAssert skipWhile("Hello World", {'W', 'o', 'r'}, 6) == 3 result = 0 - while start+result < s.len and s[result+start] in toSkip: inc(result) + while result < s.len and s[result] in toSkip: inc(result) -proc fastSubstr(s: string; token: var string; start, length: int) = +proc fastSubstr(s: openArray[char]; token: var string; length: int) = token.setLen length - for i in 0 ..< length: token[i] = s[i+start] + for i in 0 ..< length: token[i] = s[i] -proc parseUntil*(s: string, token: var string, until: set[char], - start = 0): int {.inline.} = +proc parseUntil*(s: openArray[char], token: var string, until: set[char]): int {.inline.} = ## Parses a token and stores it in ``token``. Returns ## the number of the parsed characters or 0 in case of an error. A token ## consists of the characters notin `until`. @@ -342,14 +339,13 @@ proc parseUntil*(s: string, token: var string, until: set[char], doAssert myToken == "Hello " doAssert parseUntil("Hello World", myToken, {'W', 'r'}, 3) == 3 doAssert myToken == "lo " - var i = start + var i = 0 while i < s.len and s[i] notin until: inc(i) - result = i-start - fastSubstr(s, token, start, result) + result = i + fastSubstr(s, token, result) #token = substr(s, start, i-1) -proc parseUntil*(s: string, token: var string, until: char, - start = 0): int {.inline.} = +proc parseUntil*(s: openArray[char], token: var string, until: char): int {.inline.} = ## Parses a token and stores it in ``token``. Returns ## the number of the parsed characters or 0 in case of an error. A token ## consists of any character that is not the `until` character. @@ -361,14 +357,13 @@ proc parseUntil*(s: string, token: var string, until: char, doAssert myToken == "Hell" doAssert parseUntil("Hello World", myToken, 'o', 2) == 2 doAssert myToken == "ll" - var i = start + var i = 0 while i < s.len and s[i] != until: inc(i) - result = i-start - fastSubstr(s, token, start, result) + result = i + fastSubstr(s, token, result) #token = substr(s, start, i-1) -proc parseUntil*(s: string, token: var string, until: string, - start = 0): int {.inline.} = +proc parseUntil*(s: openArray[char], token: var string, until: string): int {.inline.} = ## Parses a token and stores it in ``token``. Returns ## the number of the parsed characters or 0 in case of an error. A token ## consists of any character that comes before the `until` token. @@ -382,7 +377,7 @@ proc parseUntil*(s: string, token: var string, until: string, if until.len == 0: token.setLen(0) return 0 - var i = start + var i = 0 while i < s.len: if until.len > 0 and s[i] == until[0]: var u = 1 @@ -390,12 +385,11 @@ proc parseUntil*(s: string, token: var string, until: string, inc u if u >= until.len: break inc(i) - result = i-start - fastSubstr(s, token, start, result) + result = i + fastSubstr(s, token, result) #token = substr(s, start, i-1) -proc parseWhile*(s: string, token: var string, validChars: set[char], - start = 0): int {.inline.} = +proc parseWhile*(s: openArray[char], token: var string, validChars: set[char]): int {.inline.} = ## Parses a token and stores it in ``token``. Returns ## the number of the parsed characters or 0 in case of an error. A token ## consists of the characters in `validChars`. @@ -405,22 +399,22 @@ proc parseWhile*(s: string, token: var string, validChars: set[char], doAssert myToken.len() == 0 doAssert parseWhile("Hello World", myToken, {'W', 'o', 'r'}, 6) == 3 doAssert myToken == "Wor" - var i = start + var i = 0 while i < s.len and s[i] in validChars: inc(i) - result = i-start - fastSubstr(s, token, start, result) + result = i + fastSubstr(s, token, result) #token = substr(s, start, i-1) -proc captureBetween*(s: string, first: char, second = '\0', start = 0): string = +proc captureBetween*(s: openArray[char], first: char, second = '\0'): string = ## Finds the first occurrence of ``first``, then returns everything from there ## up to ``second`` (if ``second`` is '\0', then ``first`` is used). runnableExamples: doAssert captureBetween("Hello World", 'e') == "llo World" doAssert captureBetween("Hello World", 'e', 'r') == "llo Wo" - doAssert captureBetween("Hello World", 'l', start = 6) == "d" - var i = skipUntil(s, first, start)+1+start + doAssert captureBetween("Hello World".toOpenArray(6, "Hello World".high), 'l') == "d" + var i = skipUntil(s, first) + 1 result = "" - discard s.parseUntil(result, if second == '\0': first else: second, i) + discard parseUntil(s.toOpenArray(i, s.high), result, if second == '\0': first else: second) proc integerOutOfRangeError() {.noinline.} = raise newException(ValueError, "Parsed integer outside of valid range") @@ -429,10 +423,10 @@ proc integerOutOfRangeError() {.noinline.} = when defined(js): {.push overflowChecks: off.} -proc rawParseInt(s: string, b: var BiggestInt, start = 0): int = +proc rawParseInt(s: openArray[char], b: var BiggestInt): int = var sign: BiggestInt = -1 - i = start + i = 0 if i < s.len: if s[i] == '+': inc(i) elif s[i] == '-': @@ -452,47 +446,47 @@ proc rawParseInt(s: string, b: var BiggestInt, start = 0): int = integerOutOfRangeError() else: b = b * sign - result = i - start + result = i when defined(js): {.pop.} # overflowChecks: off -proc parseBiggestInt*(s: string, number: var BiggestInt, start = 0): int {. +proc parseBiggestInt*(s: openArray[char], number: var BiggestInt): int {. rtl, extern: "npuParseBiggestInt", noSideEffect, raises: [ValueError].} = - ## Parses an integer starting at `start` and stores the value into `number`. + ## Parses an integer and stores the value into `number`. ## Result is the number of processed chars or 0 if there is no integer. ## `ValueError` is raised if the parsed integer is out of the valid range. runnableExamples: var res: BiggestInt - doAssert parseBiggestInt("9223372036854775807", res, 0) == 19 + doAssert parseBiggestInt("9223372036854775807", res) == 19 doAssert res == 9223372036854775807 + doAssert parseBiggestInt("-2024_05_09", res) == 11 + doAssert res == -20240509 var res = BiggestInt(0) # use 'res' for exception safety (don't write to 'number' in case of an # overflow exception): - result = rawParseInt(s, res, start) + result = rawParseInt(s, res) if result != 0: number = res -proc parseInt*(s: string, number: var int, start = 0): int {. +proc parseInt*(s: openArray[char], number: var int): int {. rtl, extern: "npuParseInt", noSideEffect, raises: [ValueError].} = - ## Parses an integer starting at `start` and stores the value into `number`. + ## Parses an integer and stores the value into `number`. ## Result is the number of processed chars or 0 if there is no integer. ## `ValueError` is raised if the parsed integer is out of the valid range. runnableExamples: var res: int - doAssert parseInt("2019", res, 0) == 4 - doAssert res == 2019 - doAssert parseInt("2019", res, 2) == 2 - doAssert res == 19 + doAssert parseInt("-2024_05_02", res) == 11 + doAssert res == -20240502 var res = BiggestInt(0) - result = parseBiggestInt(s, res, start) + result = parseBiggestInt(s, res) when sizeof(int) <= 4: if res < low(int) or res > high(int): integerOutOfRangeError() if result != 0: number = int(res) -proc parseSaturatedNatural*(s: string, b: var int, start = 0): int {. +proc parseSaturatedNatural*(s: openArray[char], b: var int): int {. raises: [].} = ## Parses a natural number into ``b``. This cannot raise an overflow ## error. ``high(int)`` is returned for an overflow. @@ -502,7 +496,7 @@ proc parseSaturatedNatural*(s: string, b: var int, start = 0): int {. var res = 0 discard parseSaturatedNatural("848", res) doAssert res == 848 - var i = start + var i = 0 if i < s.len and s[i] == '+': inc(i) if i < s.len and s[i] in {'0'..'9'}: b = 0 @@ -514,13 +508,13 @@ proc parseSaturatedNatural*(s: string, b: var int, start = 0): int {. b = high(int) inc(i) while i < s.len and s[i] == '_': inc(i) # underscores are allowed and ignored - result = i - start + result = i -proc rawParseUInt(s: string, b: var BiggestUInt, start = 0): int = +proc rawParseUInt(s: openArray[char], b: var BiggestUInt): int = var res = 0.BiggestUInt prev = 0.BiggestUInt - i = start + i = 0 if i < s.len - 1 and s[i] == '-' and s[i + 1] in {'0'..'9'}: integerOutOfRangeError() if i < s.len and s[i] == '+': inc(i) # Allow @@ -534,11 +528,11 @@ proc rawParseUInt(s: string, b: var BiggestUInt, start = 0): int = inc(i) while i < s.len and s[i] == '_': inc(i) # underscores are allowed and ignored b = res - result = i - start + result = i -proc parseBiggestUInt*(s: string, number: var BiggestUInt, start = 0): int {. +proc parseBiggestUInt*(s: openArray[char], number: var BiggestUInt): int {. rtl, extern: "npuParseBiggestUInt", noSideEffect, raises: [ValueError].} = - ## Parses an unsigned integer starting at `start` and stores the value + ## Parses an unsigned integer and stores the value ## into `number`. ## `ValueError` is raised if the parsed integer is out of the valid range. runnableExamples: @@ -550,13 +544,13 @@ proc parseBiggestUInt*(s: string, number: var BiggestUInt, start = 0): int {. var res = BiggestUInt(0) # use 'res' for exception safety (don't write to 'number' in case of an # overflow exception): - result = rawParseUInt(s, res, start) + result = rawParseUInt(s, res) if result != 0: number = res -proc parseUInt*(s: string, number: var uint, start = 0): int {. +proc parseUInt*(s: openArray[char], number: var uint): int {. rtl, extern: "npuParseUInt", noSideEffect, raises: [ValueError].} = - ## Parses an unsigned integer starting at `start` and stores the value + ## Parses an unsigned integer and stores the value ## into `number`. ## `ValueError` is raised if the parsed integer is out of the valid range. runnableExamples: @@ -566,22 +560,22 @@ proc parseUInt*(s: string, number: var uint, start = 0): int {. doAssert parseUInt("3450", res, 2) == 2 doAssert res == 50 var res = BiggestUInt(0) - result = parseBiggestUInt(s, res, start) + result = parseBiggestUInt(s, res) when sizeof(BiggestUInt) > sizeof(uint) and sizeof(uint) <= 4: if res > 0xFFFF_FFFF'u64: integerOutOfRangeError() if result != 0: number = uint(res) -proc parseBiggestFloat*(s: string, number: var BiggestFloat, start = 0): int {. +proc parseBiggestFloat*(s: openArray[char], number: var BiggestFloat): int {. magic: "ParseBiggestFloat", importc: "nimParseBiggestFloat", noSideEffect.} - ## Parses a float starting at `start` and stores the value into `number`. + ## Parses a float and stores the value into `number`. ## Result is the number of processed chars or 0 if a parsing error ## occurred. -proc parseFloat*(s: string, number: var float, start = 0): int {. +proc parseFloat*(s: openArray[char], number: var float): int {. rtl, extern: "npuParseFloat", noSideEffect.} = - ## Parses a float starting at `start` and stores the value into `number`. + ## Parses a float and stores the value into `number`. ## Result is the number of processed chars or 0 if there occurred a parsing ## error. runnableExamples: @@ -593,10 +587,75 @@ proc parseFloat*(s: string, number: var float, start = 0): int {. doAssert parseFloat("32.57", res, 3) == 2 doAssert res == 57.00 var bf = BiggestFloat(0.0) - result = parseBiggestFloat(s, bf, start) + result = parseBiggestFloat(s, bf) if result != 0: number = bf +func toLowerAscii(c: char): char = + if c in {'A'..'Z'}: char(uint8(c) xor 0b0010_0000'u8) else: c + +func parseSize*(s: openArray[char], size: var int64, alwaysBin=false): int = + ## Parse a size qualified by binary or metric units into `size`. This format + ## is often called "human readable". Result is the number of processed chars + ## or 0 on parse errors and size is rounded to the nearest integer. Trailing + ## garbage like "/s" in "1k/s" is allowed and detected by `result < s.len`. + ## + ## To simplify use, following non-rare wild conventions, and since fractional + ## data like milli-bytes is so rare, unit matching is case-insensitive but for + ## the 'i' distinguishing binary-metric from metric (which cannot be 'I'). + ## + ## An optional trailing 'B|b' is ignored but processed. I.e., you must still + ## know if units are bytes | bits or infer this fact via the case of s[^1] (if + ## users can even be relied upon to use 'B' for byte and 'b' for bit or have + ## that be s[^1]). + ## + ## If `alwaysBin==true` then scales are always binary-metric, but e.g. "KiB" + ## is still accepted for clarity. If the value would exceed the range of + ## `int64`, `size` saturates to `int64.high`. Supported metric prefix chars + ## include k, m, g, t, p, e, z, y (but z & y saturate unless the number is a + ## small fraction). + ## + ## **See also:** + ## * https://en.wikipedia.org/wiki/Binary_prefix + ## * `formatSize module<strutils.html>`_ for formatting + runnableExamples: + var res: int64 # caller must still know if 'b' refers to bytes|bits + doAssert parseSize("10.5 MB", res) == 7 + doAssert res == 10_500_000 # decimal metric Mega prefix + doAssert parseSize("64 mib", res) == 6 + doAssert res == 67108864 # 64 shl 20 + doAssert parseSize("1G/h", res, true) == 2 # '/' stops parse + doAssert res == 1073741824 # 1 shl 30, forced binary metric + const prefix = "b" & "kmgtpezy" # byte|bit & lowCase metric-ish prefixes + const scaleM = [1.0, 1e3, 1e6, 1e9, 1e12, 1e15, 1e18, 1e21, 1e24] # 10^(3*idx) + const scaleB = [1.0, 1024, 1048576, 1073741824, 1099511627776.0, # 2^(10*idx) + 1125899906842624.0, 1152921504606846976.0, # ldexp? + 1.180591620717411303424e21, 1.208925819614629174706176e24] + var number: float + var scale = 1.0 + result = parseFloat(s, number) + if number < 0: # While parseFloat accepts negatives .. + result = 0 #.. we do not since sizes cannot be < 0 + if result > 0: + let start = result # Save spot to maybe unwind white to EOS + while result < s.len and s[result] in Whitespace: + inc result + if result < s.len: # Illegal starting char => unity + if (let si = prefix.find(s[result].toLowerAscii); si >= 0): + inc result # Now parse the scale + scale = if alwaysBin: scaleB[si] else: scaleM[si] + if result < s.len and s[result] == 'i': + scale = scaleB[si] # Switch from default to binary-metric + inc result + if result < s.len and s[result].toLowerAscii == 'b': + inc result # Skip optional '[bB]' + else: # Unwind result advancement when there.. + result = start #..is no unit to the end of `s`. + var sizeF = number * scale + 0.5 # Saturate to int64.high when too big + size = if sizeF > 9223372036854774784.0: int64.high else: sizeF.int64 +# Above constant=2^63-1024 avoids C UB; github.com/nim-lang/Nim/issues/20102 or +# stackoverflow.com/questions/20923556/math-pow2-63-1-math-pow2-63-512-is-true + type InterpolatedKind* = enum ## Describes for `interpolatedFragments` ## which part of the interpolated string is @@ -606,7 +665,7 @@ type ikVar, ## ``var`` part of the interpolated string ikExpr ## ``expr`` part of the interpolated string -iterator interpolatedFragments*(s: string): tuple[kind: InterpolatedKind, +iterator interpolatedFragments*(s: openArray[char]): tuple[kind: InterpolatedKind, value: string] = ## Tokenizes the string `s` into substrings for interpolation purposes. ## @@ -641,7 +700,7 @@ iterator interpolatedFragments*(s: string): tuple[kind: InterpolatedKind, else: discard inc j raise newException(ValueError, - "Expected closing '}': " & substr(s, i, s.high)) + "Expected closing '}': " & substr(s.toOpenArray(i, s.high))) inc i, 2 # skip ${ kind = ikExpr elif j+1 < s.len and s[j+1] in IdentStartChars: @@ -655,15 +714,374 @@ iterator interpolatedFragments*(s: string): tuple[kind: InterpolatedKind, kind = ikDollar else: raise newException(ValueError, - "Unable to parse a variable name at " & substr(s, i, s.high)) + "Unable to parse a variable name at " & substr(s.toOpenArray(i, s.high))) else: while j < s.len and s[j] != '$': inc j kind = ikStr if j > i: # do not copy the trailing } for ikExpr: - yield (kind, substr(s, i, j-1-ord(kind == ikExpr))) + yield (kind, substr(s.toOpenArray(i, j-1-ord(kind == ikExpr)))) else: break i = j {.pop.} + + +proc parseBin*[T: SomeInteger](s: string, number: var T, start = 0, + maxLen = 0): int {.noSideEffect.} = + ## Parses a binary number and stores its value in ``number``. + ## + ## Returns the number of the parsed characters or 0 in case of an error. + ## If error, the value of ``number`` is not changed. + ## + ## If ``maxLen == 0``, the parsing continues until the first non-bin character + ## or to the end of the string. Otherwise, no more than ``maxLen`` characters + ## are parsed starting from the ``start`` position. + ## + ## It does not check for overflow. If the value represented by the string is + ## too big to fit into ``number``, only the value of last fitting characters + ## will be stored in ``number`` without producing an error. + runnableExamples: + var num: int + doAssert parseBin("0100_1110_0110_1001_1110_1101", num) == 29 + doAssert num == 5138925 + doAssert parseBin("3", num) == 0 + var num8: int8 + doAssert parseBin("0b_0100_1110_0110_1001_1110_1101", num8) == 32 + doAssert num8 == 0b1110_1101'i8 + doAssert parseBin("0b_0100_1110_0110_1001_1110_1101", num8, 3, 9) == 9 + doAssert num8 == 0b0100_1110'i8 + var num8u: uint8 + doAssert parseBin("0b_0100_1110_0110_1001_1110_1101", num8u) == 32 + doAssert num8u == 237 + var num64: int64 + doAssert parseBin("0100111001101001111011010100111001101001", num64) == 40 + doAssert num64 == 336784608873 + parseBin(s.toOpenArray(start, s.high), number, maxLen) + +proc parseOct*[T: SomeInteger](s: string, number: var T, start = 0, + maxLen = 0): int {.noSideEffect.} = + ## Parses an octal number and stores its value in ``number``. + ## + ## Returns the number of the parsed characters or 0 in case of an error. + ## If error, the value of ``number`` is not changed. + ## + ## If ``maxLen == 0``, the parsing continues until the first non-oct character + ## or to the end of the string. Otherwise, no more than ``maxLen`` characters + ## are parsed starting from the ``start`` position. + ## + ## It does not check for overflow. If the value represented by the string is + ## too big to fit into ``number``, only the value of last fitting characters + ## will be stored in ``number`` without producing an error. + runnableExamples: + var num: int + doAssert parseOct("0o23464755", num) == 10 + doAssert num == 5138925 + doAssert parseOct("8", num) == 0 + var num8: int8 + doAssert parseOct("0o_1464_755", num8) == 11 + doAssert num8 == -19 + doAssert parseOct("0o_1464_755", num8, 3, 3) == 3 + doAssert num8 == 102 + var num8u: uint8 + doAssert parseOct("1464755", num8u) == 7 + doAssert num8u == 237 + var num64: int64 + doAssert parseOct("2346475523464755", num64) == 16 + doAssert num64 == 86216859871725 + parseOct(s.toOpenArray(start, s.high), number, maxLen) + +proc parseHex*[T: SomeInteger](s: string, number: var T, start = 0, + maxLen = 0): int {.noSideEffect.} = + ## Parses a hexadecimal number and stores its value in ``number``. + ## + ## Returns the number of the parsed characters or 0 in case of an error. + ## If error, the value of ``number`` is not changed. + ## + ## If ``maxLen == 0``, the parsing continues until the first non-hex character + ## or to the end of the string. Otherwise, no more than ``maxLen`` characters + ## are parsed starting from the ``start`` position. + ## + ## It does not check for overflow. If the value represented by the string is + ## too big to fit into ``number``, only the value of last fitting characters + ## will be stored in ``number`` without producing an error. + runnableExamples: + var num: int + doAssert parseHex("4E_69_ED", num) == 8 + doAssert num == 5138925 + doAssert parseHex("X", num) == 0 + doAssert parseHex("#ABC", num) == 4 + var num8: int8 + doAssert parseHex("0x_4E_69_ED", num8) == 11 + doAssert num8 == 0xED'i8 + doAssert parseHex("0x_4E_69_ED", num8, 3, 2) == 2 + doAssert num8 == 0x4E'i8 + var num8u: uint8 + doAssert parseHex("0x_4E_69_ED", num8u) == 11 + doAssert num8u == 237 + var num64: int64 + doAssert parseHex("4E69ED4E69ED", num64) == 12 + doAssert num64 == 86216859871725 + parseHex(s.toOpenArray(start, s.high), number, maxLen) + +proc parseIdent*(s: string, ident: var string, start = 0): int = + ## Parses an identifier and stores it in ``ident``. Returns + ## the number of the parsed characters or 0 in case of an error. + ## If error, the value of `ident` is not changed. + runnableExamples: + var res: string + doAssert parseIdent("Hello World", res, 0) == 5 + doAssert res == "Hello" + doAssert parseIdent("Hello World", res, 1) == 4 + doAssert res == "ello" + doAssert parseIdent("Hello World", res, 6) == 5 + doAssert res == "World" + parseIdent(s.toOpenArray(start, s.high), ident) + +proc parseIdent*(s: string, start = 0): string = + ## Parses an identifier and returns it or an empty string in + ## case of an error. + runnableExamples: + doAssert parseIdent("Hello World", 0) == "Hello" + doAssert parseIdent("Hello World", 1) == "ello" + doAssert parseIdent("Hello World", 5) == "" + doAssert parseIdent("Hello World", 6) == "World" + parseIdent(s.toOpenArray(start, s.high)) + +proc parseChar*(s: string, c: var char, start = 0): int = + ## Parses a single character, stores it in `c` and returns 1. + ## In case of error (if start >= s.len) it returns 0 + ## and the value of `c` is unchanged. + runnableExamples: + var c: char + doAssert "nim".parseChar(c, 3) == 0 + doAssert c == '\0' + doAssert "nim".parseChar(c, 0) == 1 + doAssert c == 'n' + parseChar(s.toOpenArray(start, s.high), c) + +proc skipWhitespace*(s: string, start = 0): int {.inline.} = + ## Skips the whitespace starting at ``s[start]``. Returns the number of + ## skipped characters. + runnableExamples: + doAssert skipWhitespace("Hello World", 0) == 0 + doAssert skipWhitespace(" Hello World", 0) == 1 + doAssert skipWhitespace("Hello World", 5) == 1 + doAssert skipWhitespace("Hello World", 5) == 2 + skipWhitespace(s.toOpenArray(start, s.high)) + +proc skip*(s, token: string, start = 0): int {.inline.} = + ## Skips the `token` starting at ``s[start]``. Returns the length of `token` + ## or 0 if there was no `token` at ``s[start]``. + runnableExamples: + doAssert skip("2019-01-22", "2019", 0) == 4 + doAssert skip("2019-01-22", "19", 0) == 0 + doAssert skip("2019-01-22", "19", 2) == 2 + doAssert skip("CAPlow", "CAP", 0) == 3 + doAssert skip("CAPlow", "cap", 0) == 0 + skip(s.toOpenArray(start, s.high), token) + +proc skipIgnoreCase*(s, token: string, start = 0): int = + ## Same as `skip` but case is ignored for token matching. + runnableExamples: + doAssert skipIgnoreCase("CAPlow", "CAP", 0) == 3 + doAssert skipIgnoreCase("CAPlow", "cap", 0) == 3 + skipIgnoreCase(s.toOpenArray(start, s.high), token) + +proc skipUntil*(s: string, until: set[char], start = 0): int {.inline.} = + ## Skips all characters until one char from the set `until` is found + ## or the end is reached. + ## Returns number of characters skipped. + runnableExamples: + doAssert skipUntil("Hello World", {'W', 'e'}, 0) == 1 + doAssert skipUntil("Hello World", {'W'}, 0) == 6 + doAssert skipUntil("Hello World", {'W', 'd'}, 0) == 6 + skipUntil(s.toOpenArray(start, s.high), until) + +proc skipUntil*(s: string, until: char, start = 0): int {.inline.} = + ## Skips all characters until the char `until` is found + ## or the end is reached. + ## Returns number of characters skipped. + runnableExamples: + doAssert skipUntil("Hello World", 'o', 0) == 4 + doAssert skipUntil("Hello World", 'o', 4) == 0 + doAssert skipUntil("Hello World", 'W', 0) == 6 + doAssert skipUntil("Hello World", 'w', 0) == 11 + skipUntil(s.toOpenArray(start, s.high), until) + +proc skipWhile*(s: string, toSkip: set[char], start = 0): int {.inline.} = + ## Skips all characters while one char from the set `toSkip` is found. + ## Returns number of characters skipped. + runnableExamples: + doAssert skipWhile("Hello World", {'H', 'e'}) == 2 + doAssert skipWhile("Hello World", {'e'}) == 0 + doAssert skipWhile("Hello World", {'W', 'o', 'r'}, 6) == 3 + skipWhile(s.toOpenArray(start, s.high), toSkip) + +proc parseUntil*(s: string, token: var string, until: set[char], + start = 0): int {.inline.} = + ## Parses a token and stores it in ``token``. Returns + ## the number of the parsed characters or 0 in case of an error. A token + ## consists of the characters notin `until`. + runnableExamples: + var myToken: string + doAssert parseUntil("Hello World", myToken, {'W', 'o', 'r'}) == 4 + doAssert myToken == "Hell" + doAssert parseUntil("Hello World", myToken, {'W', 'r'}) == 6 + doAssert myToken == "Hello " + doAssert parseUntil("Hello World", myToken, {'W', 'r'}, 3) == 3 + doAssert myToken == "lo " + parseUntil(s.toOpenArray(start, s.high), token, until) + +proc parseUntil*(s: string, token: var string, until: char, + start = 0): int {.inline.} = + ## Parses a token and stores it in ``token``. Returns + ## the number of the parsed characters or 0 in case of an error. A token + ## consists of any character that is not the `until` character. + runnableExamples: + var myToken: string + doAssert parseUntil("Hello World", myToken, 'W') == 6 + doAssert myToken == "Hello " + doAssert parseUntil("Hello World", myToken, 'o') == 4 + doAssert myToken == "Hell" + doAssert parseUntil("Hello World", myToken, 'o', 2) == 2 + doAssert myToken == "ll" + parseUntil(s.toOpenArray(start, s.high), token, until) + +proc parseUntil*(s: string, token: var string, until: string, + start = 0): int {.inline.} = + ## Parses a token and stores it in ``token``. Returns + ## the number of the parsed characters or 0 in case of an error. A token + ## consists of any character that comes before the `until` token. + runnableExamples: + var myToken: string + doAssert parseUntil("Hello World", myToken, "Wor") == 6 + doAssert myToken == "Hello " + doAssert parseUntil("Hello World", myToken, "Wor", 2) == 4 + doAssert myToken == "llo " + parseUntil(s.toOpenArray(start, s.high), token, until) + +proc parseWhile*(s: string, token: var string, validChars: set[char], + start = 0): int {.inline.} = + ## Parses a token and stores it in ``token``. Returns + ## the number of the parsed characters or 0 in case of an error. A token + ## consists of the characters in `validChars`. + runnableExamples: + var myToken: string + doAssert parseWhile("Hello World", myToken, {'W', 'o', 'r'}, 0) == 0 + doAssert myToken.len() == 0 + doAssert parseWhile("Hello World", myToken, {'W', 'o', 'r'}, 6) == 3 + doAssert myToken == "Wor" + parseWhile(s.toOpenArray(start, s.high), token, validChars) + +proc captureBetween*(s: string, first: char, second = '\0', start = 0): string = + ## Finds the first occurrence of ``first``, then returns everything from there + ## up to ``second`` (if ``second`` is '\0', then ``first`` is used). + runnableExamples: + doAssert captureBetween("Hello World", 'e') == "llo World" + doAssert captureBetween("Hello World", 'e', 'r') == "llo Wo" + doAssert captureBetween("Hello World", 'l', start = 6) == "d" + captureBetween(s.toOpenArray(start, s.high), first, second) + +proc parseBiggestInt*(s: string, number: var BiggestInt, start = 0): int {.noSideEffect, raises: [ValueError].} = + ## Parses an integer starting at `start` and stores the value into `number`. + ## Result is the number of processed chars or 0 if there is no integer. + ## `ValueError` is raised if the parsed integer is out of the valid range. + runnableExamples: + var res: BiggestInt + doAssert parseBiggestInt("9223372036854775807", res, 0) == 19 + doAssert res == 9223372036854775807 + doAssert parseBiggestInt("-2024_05_09", res) == 11 + doAssert res == -20240509 + doAssert parseBiggestInt("-2024_05_02", res, 7) == 4 + doAssert res == 502 + parseBiggestInt(s.toOpenArray(start, s.high), number) + +proc parseInt*(s: string, number: var int, start = 0): int {.noSideEffect, raises: [ValueError].} = + ## Parses an integer starting at `start` and stores the value into `number`. + ## Result is the number of processed chars or 0 if there is no integer. + ## `ValueError` is raised if the parsed integer is out of the valid range. + runnableExamples: + var res: int + doAssert parseInt("-2024_05_02", res) == 11 + doAssert res == -20240502 + doAssert parseInt("-2024_05_02", res, 7) == 4 + doAssert res == 502 + parseInt(s.toOpenArray(start, s.high), number) + + +proc parseSaturatedNatural*(s: string, b: var int, start = 0): int {. + raises: [].} = + ## Parses a natural number into ``b``. This cannot raise an overflow + ## error. ``high(int)`` is returned for an overflow. + ## The number of processed character is returned. + ## This is usually what you really want to use instead of `parseInt`:idx:. + runnableExamples: + var res = 0 + discard parseSaturatedNatural("848", res) + doAssert res == 848 + parseSaturatedNatural(s.toOpenArray(start, s.high), b) + + +proc parseBiggestUInt*(s: string, number: var BiggestUInt, start = 0): int {.noSideEffect, raises: [ValueError].} = + ## Parses an unsigned integer starting at `start` and stores the value + ## into `number`. + ## `ValueError` is raised if the parsed integer is out of the valid range. + runnableExamples: + var res: BiggestUInt + doAssert parseBiggestUInt("12", res, 0) == 2 + doAssert res == 12 + doAssert parseBiggestUInt("1111111111111111111", res, 0) == 19 + doAssert res == 1111111111111111111'u64 + parseBiggestUInt(s.toOpenArray(start, s.high), number) + +proc parseUInt*(s: string, number: var uint, start = 0): int {.noSideEffect, raises: [ValueError].} = + ## Parses an unsigned integer starting at `start` and stores the value + ## into `number`. + ## `ValueError` is raised if the parsed integer is out of the valid range. + runnableExamples: + var res: uint + doAssert parseUInt("3450", res) == 4 + doAssert res == 3450 + doAssert parseUInt("3450", res, 2) == 2 + doAssert res == 50 + parseUInt(s.toOpenArray(start, s.high), number) + +proc parseBiggestFloat*(s: string, number: var BiggestFloat, start = 0): int {.noSideEffect.} = + ## Parses a float starting at `start` and stores the value into `number`. + ## Result is the number of processed chars or 0 if a parsing error + ## occurred. + parseFloat(s.toOpenArray(start, s.high), number) + +proc parseFloat*(s: string, number: var float, start = 0): int {.noSideEffect.} = + ## Parses a float starting at `start` and stores the value into `number`. + ## Result is the number of processed chars or 0 if there occurred a parsing + ## error. + runnableExamples: + var res: float + doAssert parseFloat("32", res, 0) == 2 + doAssert res == 32.0 + doAssert parseFloat("32.57", res, 0) == 5 + doAssert res == 32.57 + doAssert parseFloat("32.57", res, 3) == 2 + doAssert res == 57.00 + parseFloat(s.toOpenArray(start, s.high), number) + +iterator interpolatedFragments*(s: string): tuple[kind: InterpolatedKind, + value: string] = + ## Tokenizes the string `s` into substrings for interpolation purposes. + ## + runnableExamples: + var outp: seq[tuple[kind: InterpolatedKind, value: string]] + for k, v in interpolatedFragments(" $this is ${an example} $$"): + outp.add (k, v) + doAssert outp == @[(ikStr, " "), + (ikVar, "this"), + (ikStr, " is "), + (ikExpr, "an example"), + (ikStr, " "), + (ikDollar, "$")] + for x in s.toOa.interpolatedFragments: + yield x + diff --git a/lib/pure/parsexml.nim b/lib/pure/parsexml.nim index 79a9cc730..c760799a2 100644 --- a/lib/pure/parsexml.nim +++ b/lib/pure/parsexml.nim @@ -36,43 +36,43 @@ The file ``examples/htmltitle.nim`` demonstrates how to use the XML parser to accomplish a simple task: To determine the title of an HTML document. -.. code-block:: nim + ```nim + # Example program to show the parsexml module + # This program reads an HTML file and writes its title to stdout. + # Errors and whitespace are ignored. - # Example program to show the parsexml module - # This program reads an HTML file and writes its title to stdout. - # Errors and whitespace are ignored. + import std/[os, streams, parsexml, strutils] - import os, streams, parsexml, strutils + if paramCount() < 1: + quit("Usage: htmltitle filename[.html]") - if paramCount() < 1: - quit("Usage: htmltitle filename[.html]") - - var filename = addFileExt(paramStr(1), "html") - var s = newFileStream(filename, fmRead) - if s == nil: quit("cannot open the file " & filename) - var x: XmlParser - open(x, s, filename) - while true: - x.next() - case x.kind - of xmlElementStart: - if cmpIgnoreCase(x.elementName, "title") == 0: - var title = "" - x.next() # skip "<title>" - while x.kind == xmlCharData: - title.add(x.charData) - x.next() - if x.kind == xmlElementEnd and cmpIgnoreCase(x.elementName, "title") == 0: - echo("Title: " & title) - quit(0) # Success! - else: - echo(x.errorMsgExpected("/title")) + var filename = addFileExt(paramStr(1), "html") + var s = newFileStream(filename, fmRead) + if s == nil: quit("cannot open the file " & filename) + var x: XmlParser + open(x, s, filename) + while true: + x.next() + case x.kind + of xmlElementStart: + if cmpIgnoreCase(x.elementName, "title") == 0: + var title = "" + x.next() # skip "<title>" + while x.kind == xmlCharData: + title.add(x.charData) + x.next() + if x.kind == xmlElementEnd and cmpIgnoreCase(x.elementName, "title") == 0: + echo("Title: " & title) + quit(0) # Success! + else: + echo(x.errorMsgExpected("/title")) - of xmlEof: break # end of file reached - else: discard # ignore other events + of xmlEof: break # end of file reached + else: discard # ignore other events - x.close() - quit("Could not determine title!") + x.close() + quit("Could not determine title!") + ``` ]## @@ -85,69 +85,72 @@ The file ``examples/htmlrefs.nim`` demonstrates how to use the XML parser to accomplish another simple task: To determine all the links an HTML document contains. -.. code-block:: nim + ```nim + # Example program to show the new parsexml module + # This program reads an HTML file and writes all its used links to stdout. + # Errors and whitespace are ignored. - # Example program to show the new parsexml module - # This program reads an HTML file and writes all its used links to stdout. - # Errors and whitespace are ignored. + import std/[os, streams, parsexml, strutils] - import os, streams, parsexml, strutils + proc `=?=` (a, b: string): bool = + # little trick: define our own comparator that ignores case + return cmpIgnoreCase(a, b) == 0 - proc `=?=` (a, b: string): bool = - # little trick: define our own comparator that ignores case - return cmpIgnoreCase(a, b) == 0 + if paramCount() < 1: + quit("Usage: htmlrefs filename[.html]") - if paramCount() < 1: - quit("Usage: htmlrefs filename[.html]") - - var links = 0 # count the number of links - var filename = addFileExt(paramStr(1), "html") - var s = newFileStream(filename, fmRead) - if s == nil: quit("cannot open the file " & filename) - var x: XmlParser - open(x, s, filename) - next(x) # get first event - block mainLoop: - while true: - case x.kind - of xmlElementOpen: - # the <a href = "xyz"> tag we are interested in always has an attribute, - # thus we search for ``xmlElementOpen`` and not for ``xmlElementStart`` - if x.elementName =?= "a": - x.next() - if x.kind == xmlAttribute: - if x.attrKey =?= "href": - var link = x.attrValue - inc(links) - # skip until we have an ``xmlElementClose`` event - while true: - x.next() - case x.kind - of xmlEof: break mainLoop - of xmlElementClose: break - else: discard - x.next() # skip ``xmlElementClose`` - # now we have the description for the ``a`` element - var desc = "" - while x.kind == xmlCharData: - desc.add(x.charData) - x.next() - echo(desc & ": " & link) - else: - x.next() - of xmlEof: break # end of file reached - of xmlError: - echo(errorMsg(x)) + var links = 0 # count the number of links + var filename = addFileExt(paramStr(1), "html") + var s = newFileStream(filename, fmRead) + if s == nil: quit("cannot open the file " & filename) + var x: XmlParser + open(x, s, filename) + next(x) # get first event + block mainLoop: + while true: + case x.kind + of xmlElementOpen: + # the <a href = "xyz"> tag we are interested in always has an attribute, + # thus we search for ``xmlElementOpen`` and not for ``xmlElementStart`` + if x.elementName =?= "a": x.next() - else: x.next() # skip other events + if x.kind == xmlAttribute: + if x.attrKey =?= "href": + var link = x.attrValue + inc(links) + # skip until we have an ``xmlElementClose`` event + while true: + x.next() + case x.kind + of xmlEof: break mainLoop + of xmlElementClose: break + else: discard + x.next() # skip ``xmlElementClose`` + # now we have the description for the ``a`` element + var desc = "" + while x.kind == xmlCharData: + desc.add(x.charData) + x.next() + echo(desc & ": " & link) + else: + x.next() + of xmlEof: break # end of file reached + of xmlError: + echo(errorMsg(x)) + x.next() + else: x.next() # skip other events - echo($links & " link(s) found!") - x.close() + echo($links & " link(s) found!") + x.close() + ``` ]## import - strutils, lexbase, streams, unicode + std/[strutils, lexbase, streams, unicode] + +when defined(nimPreviewSlimSystem): + import std/[assertions, syncio] # the parser treats ``<br />`` as ``<br></br>`` @@ -789,7 +792,7 @@ proc next*(my: var XmlParser) = my.state = stateNormal when not defined(testing) and isMainModule: - import os + import std/os var s = newFileStream(paramStr(1), fmRead) if s == nil: quit("cannot open the file" & paramStr(1)) var x: XmlParser diff --git a/lib/pure/pathnorm.nim b/lib/pure/pathnorm.nim index 10a2a0b67..4cdc02303 100644 --- a/lib/pure/pathnorm.nim +++ b/lib/pure/pathnorm.nim @@ -14,7 +14,7 @@ # Yes, this uses import here, not include so that # we don't end up exporting these symbols from pathnorm and os: -import includes/osseps +import std/private/osseps type PathIter* = object @@ -29,10 +29,6 @@ proc next*(it: var PathIter; x: string): (int, int) = if not it.notFirst and x[it.i] in {DirSep, AltSep}: # absolute path: inc it.i - when doslikeFileSystem: # UNC paths have leading `\\` - if hasNext(it, x) and x[it.i] == DirSep and - it.i+1 < x.len and x[it.i+1] != DirSep: - inc it.i else: while it.i < x.len and x[it.i] notin {DirSep, AltSep}: inc it.i if it.i > it.prev: @@ -56,10 +52,23 @@ proc isDotDot(x: string; bounds: (int, int)): bool = proc isSlash(x: string; bounds: (int, int)): bool = bounds[1] == bounds[0] and x[bounds[0]] in {DirSep, AltSep} +when doslikeFileSystem: + import std/private/ntpath + proc addNormalizePath*(x: string; result: var string; state: var int; dirSep = DirSep) = ## Low level proc. Undocumented. + when doslikeFileSystem: # Add Windows drive at start without normalization + var x = x + if result == "": + let (drive, file) = splitDrive(x) + x = file + result.add drive + for c in result.mitems: + if c in {DirSep, AltSep}: + c = dirSep + # state: 0th bit set if isAbsolute path. Other bits count # the number of path components. var it: PathIter diff --git a/lib/pure/pegs.nim b/lib/pure/pegs.nim index 06a77af57..2969fd6d7 100644 --- a/lib/pure/pegs.nim +++ b/lib/pure/pegs.nim @@ -16,15 +16,17 @@ ## include "system/inclrtl" +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions] const useUnicode = true ## change this to deactivate proper UTF-8 support -import strutils, macros +import std/[strutils, macros] import std/private/decode_helpers when useUnicode: - import unicode + import std/unicode export unicode.`==` const @@ -83,30 +85,30 @@ type of pkChar, pkGreedyRepChar: ch: char of pkCharChoice, pkGreedyRepSet: charChoice: ref set[char] of pkNonTerminal: nt: NonTerminal - of pkBackRef..pkBackRefIgnoreStyle: index: range[0..MaxSubpatterns] + of pkBackRef..pkBackRefIgnoreStyle: index: range[-MaxSubpatterns..MaxSubpatterns-1] else: sons: seq[Peg] NonTerminal* = ref NonTerminalObj -proc kind*(p: Peg): PegKind = p.kind +func kind*(p: Peg): PegKind = p.kind ## Returns the *PegKind* of a given *Peg* object. -proc term*(p: Peg): string = p.term +func term*(p: Peg): string = p.term ## Returns the *string* representation of a given *Peg* variant object ## where present. -proc ch*(p: Peg): char = p.ch +func ch*(p: Peg): char = p.ch ## Returns the *char* representation of a given *Peg* variant object ## where present. -proc charChoice*(p: Peg): ref set[char] = p.charChoice +func charChoice*(p: Peg): ref set[char] = p.charChoice ## Returns the *charChoice* field of a given *Peg* variant object ## where present. -proc nt*(p: Peg): NonTerminal = p.nt +func nt*(p: Peg): NonTerminal = p.nt ## Returns the *NonTerminal* object of a given *Peg* variant object ## where present. -proc index*(p: Peg): range[0..MaxSubpatterns] = p.index +func index*(p: Peg): range[-MaxSubpatterns..MaxSubpatterns-1] = p.index ## Returns the back-reference index of a captured sub-pattern in the ## *Captures* object for a given *Peg* variant object where present. @@ -120,59 +122,60 @@ iterator pairs*(p: Peg): (int, Peg) {.inline.} = for i in 0 ..< p.sons.len: yield (i, p.sons[i]) -proc name*(nt: NonTerminal): string = nt.name +func name*(nt: NonTerminal): string = nt.name ## Gets the name of the symbol represented by the parent *Peg* object variant ## of a given *NonTerminal*. -proc line*(nt: NonTerminal): int = nt.line +func line*(nt: NonTerminal): int = nt.line ## Gets the line number of the definition of the parent *Peg* object variant ## of a given *NonTerminal*. -proc col*(nt: NonTerminal): int = nt.col +func col*(nt: NonTerminal): int = nt.col ## Gets the column number of the definition of the parent *Peg* object variant ## of a given *NonTerminal*. -proc flags*(nt: NonTerminal): set[NonTerminalFlag] = nt.flags +func flags*(nt: NonTerminal): set[NonTerminalFlag] = nt.flags ## Gets the *NonTerminalFlag*-typed flags field of the parent *Peg* variant ## object of a given *NonTerminal*. -proc rule*(nt: NonTerminal): Peg = nt.rule +func rule*(nt: NonTerminal): Peg = nt.rule ## Gets the *Peg* object representing the rule definition of the parent *Peg* ## object variant of a given *NonTerminal*. -proc term*(t: string): Peg {.noSideEffect, rtl, extern: "npegs$1Str".} = +func term*(t: string): Peg {.rtl, extern: "npegs$1Str".} = ## constructs a PEG from a terminal string if t.len != 1: result = Peg(kind: pkTerminal, term: t) else: result = Peg(kind: pkChar, ch: t[0]) -proc termIgnoreCase*(t: string): Peg {. - noSideEffect, rtl, extern: "npegs$1".} = +func termIgnoreCase*(t: string): Peg {. + rtl, extern: "npegs$1".} = ## constructs a PEG from a terminal string; ignore case for matching result = Peg(kind: pkTerminalIgnoreCase, term: t) -proc termIgnoreStyle*(t: string): Peg {. - noSideEffect, rtl, extern: "npegs$1".} = +func termIgnoreStyle*(t: string): Peg {. + rtl, extern: "npegs$1".} = ## constructs a PEG from a terminal string; ignore style for matching result = Peg(kind: pkTerminalIgnoreStyle, term: t) -proc term*(t: char): Peg {.noSideEffect, rtl, extern: "npegs$1Char".} = +func term*(t: char): Peg {.rtl, extern: "npegs$1Char".} = ## constructs a PEG from a terminal char assert t != '\0' result = Peg(kind: pkChar, ch: t) -proc charSet*(s: set[char]): Peg {.noSideEffect, rtl, extern: "npegs$1".} = +func charSet*(s: set[char]): Peg {.rtl, extern: "npegs$1".} = ## constructs a PEG from a character set `s` assert '\0' notin s result = Peg(kind: pkCharChoice) - new(result.charChoice) - result.charChoice[] = s + {.cast(noSideEffect).}: + new(result.charChoice) + result.charChoice[] = s -proc len(a: Peg): int {.inline.} = return a.sons.len -proc add(d: var Peg, s: Peg) {.inline.} = add(d.sons, s) +func len(a: Peg): int {.inline.} = return a.sons.len +func add(d: var Peg, s: Peg) {.inline.} = add(d.sons, s) -proc addChoice(dest: var Peg, elem: Peg) = +func addChoice(dest: var Peg, elem: Peg) = var L = dest.len-1 if L >= 0 and dest.sons[L].kind == pkCharChoice: # caution! Do not introduce false aliasing here! @@ -195,12 +198,12 @@ template multipleOp(k: PegKind, localOpt: untyped) = if result.len == 1: result = result.sons[0] -proc `/`*(a: varargs[Peg]): Peg {. - noSideEffect, rtl, extern: "npegsOrderedChoice".} = +func `/`*(a: varargs[Peg]): Peg {. + rtl, extern: "npegsOrderedChoice".} = ## constructs an ordered choice with the PEGs in `a` multipleOp(pkOrderedChoice, addChoice) -proc addSequence(dest: var Peg, elem: Peg) = +func addSequence(dest: var Peg, elem: Peg) = var L = dest.len-1 if L >= 0 and dest.sons[L].kind == pkTerminal: # caution! Do not introduce false aliasing here! @@ -212,12 +215,12 @@ proc addSequence(dest: var Peg, elem: Peg) = else: add(dest, elem) else: add(dest, elem) -proc sequence*(a: varargs[Peg]): Peg {. - noSideEffect, rtl, extern: "npegs$1".} = +func sequence*(a: varargs[Peg]): Peg {. + rtl, extern: "npegs$1".} = ## constructs a sequence with all the PEGs from `a` multipleOp(pkSequence, addSequence) -proc `?`*(a: Peg): Peg {.noSideEffect, rtl, extern: "npegsOptional".} = +func `?`*(a: Peg): Peg {.rtl, extern: "npegsOptional".} = ## constructs an optional for the PEG `a` if a.kind in {pkOption, pkGreedyRep, pkGreedyAny, pkGreedyRepChar, pkGreedyRepSet}: @@ -227,7 +230,7 @@ proc `?`*(a: Peg): Peg {.noSideEffect, rtl, extern: "npegsOptional".} = else: result = Peg(kind: pkOption, sons: @[a]) -proc `*`*(a: Peg): Peg {.noSideEffect, rtl, extern: "npegsGreedyRep".} = +func `*`*(a: Peg): Peg {.rtl, extern: "npegsGreedyRep".} = ## constructs a "greedy repetition" for the PEG `a` case a.kind of pkGreedyRep, pkGreedyRepChar, pkGreedyRepSet, pkGreedyAny, pkOption: @@ -242,96 +245,99 @@ proc `*`*(a: Peg): Peg {.noSideEffect, rtl, extern: "npegsGreedyRep".} = else: result = Peg(kind: pkGreedyRep, sons: @[a]) -proc `!*`*(a: Peg): Peg {.noSideEffect, rtl, extern: "npegsSearch".} = +func `!*`*(a: Peg): Peg {.rtl, extern: "npegsSearch".} = ## constructs a "search" for the PEG `a` result = Peg(kind: pkSearch, sons: @[a]) -proc `!*\`*(a: Peg): Peg {.noSideEffect, rtl, +func `!*\`*(a: Peg): Peg {.rtl, extern: "npgegsCapturedSearch".} = ## constructs a "captured search" for the PEG `a` result = Peg(kind: pkCapturedSearch, sons: @[a]) -proc `+`*(a: Peg): Peg {.noSideEffect, rtl, extern: "npegsGreedyPosRep".} = +func `+`*(a: Peg): Peg {.rtl, extern: "npegsGreedyPosRep".} = ## constructs a "greedy positive repetition" with the PEG `a` return sequence(a, *a) -proc `&`*(a: Peg): Peg {.noSideEffect, rtl, extern: "npegsAndPredicate".} = +func `&`*(a: Peg): Peg {.rtl, extern: "npegsAndPredicate".} = ## constructs an "and predicate" with the PEG `a` result = Peg(kind: pkAndPredicate, sons: @[a]) -proc `!`*(a: Peg): Peg {.noSideEffect, rtl, extern: "npegsNotPredicate".} = +func `!`*(a: Peg): Peg {.rtl, extern: "npegsNotPredicate".} = ## constructs a "not predicate" with the PEG `a` result = Peg(kind: pkNotPredicate, sons: @[a]) -proc any*: Peg {.inline.} = +func any*: Peg {.inline.} = ## constructs the PEG `any character`:idx: (``.``) result = Peg(kind: pkAny) -proc anyRune*: Peg {.inline.} = +func anyRune*: Peg {.inline.} = ## constructs the PEG `any rune`:idx: (``_``) result = Peg(kind: pkAnyRune) -proc newLine*: Peg {.inline.} = +func newLine*: Peg {.inline.} = ## constructs the PEG `newline`:idx: (``\n``) result = Peg(kind: pkNewLine) -proc unicodeLetter*: Peg {.inline.} = +func unicodeLetter*: Peg {.inline.} = ## constructs the PEG ``\letter`` which matches any Unicode letter. result = Peg(kind: pkLetter) -proc unicodeLower*: Peg {.inline.} = +func unicodeLower*: Peg {.inline.} = ## constructs the PEG ``\lower`` which matches any Unicode lowercase letter. result = Peg(kind: pkLower) -proc unicodeUpper*: Peg {.inline.} = +func unicodeUpper*: Peg {.inline.} = ## constructs the PEG ``\upper`` which matches any Unicode uppercase letter. result = Peg(kind: pkUpper) -proc unicodeTitle*: Peg {.inline.} = +func unicodeTitle*: Peg {.inline.} = ## constructs the PEG ``\title`` which matches any Unicode title letter. result = Peg(kind: pkTitle) -proc unicodeWhitespace*: Peg {.inline.} = +func unicodeWhitespace*: Peg {.inline.} = ## constructs the PEG ``\white`` which matches any Unicode ## whitespace character. result = Peg(kind: pkWhitespace) -proc startAnchor*: Peg {.inline.} = +func startAnchor*: Peg {.inline.} = ## constructs the PEG ``^`` which matches the start of the input. result = Peg(kind: pkStartAnchor) -proc endAnchor*: Peg {.inline.} = +func endAnchor*: Peg {.inline.} = ## constructs the PEG ``$`` which matches the end of the input. result = !any() -proc capture*(a: Peg): Peg {.noSideEffect, rtl, extern: "npegsCapture".} = +func capture*(a: Peg = Peg(kind: pkEmpty)): Peg {.rtl, extern: "npegsCapture".} = ## constructs a capture with the PEG `a` result = Peg(kind: pkCapture, sons: @[a]) -proc backref*(index: range[1..MaxSubpatterns]): Peg {. - noSideEffect, rtl, extern: "npegs$1".} = +func backref*(index: range[1..MaxSubpatterns], reverse: bool = false): Peg {. + rtl, extern: "npegs$1".} = ## constructs a back reference of the given `index`. `index` starts counting - ## from 1. - result = Peg(kind: pkBackRef, index: index-1) + ## from 1. `reverse` specifies whether indexing starts from the end of the + ## capture list. + result = Peg(kind: pkBackRef, index: (if reverse: -index else: index - 1)) -proc backrefIgnoreCase*(index: range[1..MaxSubpatterns]): Peg {. - noSideEffect, rtl, extern: "npegs$1".} = +func backrefIgnoreCase*(index: range[1..MaxSubpatterns], reverse: bool = false): Peg {. + rtl, extern: "npegs$1".} = ## constructs a back reference of the given `index`. `index` starts counting - ## from 1. Ignores case for matching. - result = Peg(kind: pkBackRefIgnoreCase, index: index-1) + ## from 1. `reverse` specifies whether indexing starts from the end of the + ## capture list. Ignores case for matching. + result = Peg(kind: pkBackRefIgnoreCase, index: (if reverse: -index else: index - 1)) -proc backrefIgnoreStyle*(index: range[1..MaxSubpatterns]): Peg {. - noSideEffect, rtl, extern: "npegs$1".} = +func backrefIgnoreStyle*(index: range[1..MaxSubpatterns], reverse: bool = false): Peg {. + rtl, extern: "npegs$1".} = ## constructs a back reference of the given `index`. `index` starts counting - ## from 1. Ignores style for matching. - result = Peg(kind: pkBackRefIgnoreStyle, index: index-1) + ## from 1. `reverse` specifies whether indexing starts from the end of the + ## capture list. Ignores style for matching. + result = Peg(kind: pkBackRefIgnoreStyle, index: (if reverse: -index else: index - 1)) -proc spaceCost(n: Peg): int = +func spaceCost(n: Peg): int = case n.kind of pkEmpty: discard of pkTerminal, pkTerminalIgnoreCase, pkTerminalIgnoreStyle, pkChar, pkGreedyRepChar, pkCharChoice, pkGreedyRepSet, - pkAny..pkWhitespace, pkGreedyAny: + pkAny..pkWhitespace, pkGreedyAny, pkBackRef..pkBackRefIgnoreStyle: result = 1 of pkNonTerminal: # we cannot inline a rule with a non-terminal @@ -341,8 +347,8 @@ proc spaceCost(n: Peg): int = inc(result, spaceCost(n.sons[i])) if result >= InlineThreshold: break -proc nonterminal*(n: NonTerminal): Peg {. - noSideEffect, rtl, extern: "npegs$1".} = +func nonterminal*(n: NonTerminal): Peg {. + rtl, extern: "npegs$1".} = ## constructs a PEG that consists of the nonterminal symbol assert n != nil if ntDeclared in n.flags and spaceCost(n.rule) < InlineThreshold: @@ -351,8 +357,8 @@ proc nonterminal*(n: NonTerminal): Peg {. else: result = Peg(kind: pkNonTerminal, nt: n) -proc newNonTerminal*(name: string, line, column: int): NonTerminal {. - noSideEffect, rtl, extern: "npegs$1".} = +func newNonTerminal*(name: string, line, column: int): NonTerminal {. + rtl, extern: "npegs$1".} = ## constructs a nonterminal symbol result = NonTerminal(name: name, line: line, col: column) @@ -387,7 +393,7 @@ template natural*: Peg = # ------------------------- debugging ----------------------------------------- -proc esc(c: char, reserved = {'\0'..'\255'}): string = +func esc(c: char, reserved = {'\0'..'\255'}): string = case c of '\b': result = "\\b" of '\t': result = "\\t" @@ -403,14 +409,14 @@ proc esc(c: char, reserved = {'\0'..'\255'}): string = elif c in reserved: result = '\\' & c else: result = $c -proc singleQuoteEsc(c: char): string = return "'" & esc(c, {'\''}) & "'" +func singleQuoteEsc(c: char): string = return "'" & esc(c, {'\''}) & "'" -proc singleQuoteEsc(str: string): string = +func singleQuoteEsc(str: string): string = result = "'" for c in items(str): add result, esc(c, {'\''}) add result, '\'' -proc charSetEscAux(cc: set[char]): string = +func charSetEscAux(cc: set[char]): string = const reserved = {'^', '-', ']'} result = "" var c1 = 0 @@ -427,13 +433,13 @@ proc charSetEscAux(cc: set[char]): string = c1 = c2 inc(c1) -proc charSetEsc(cc: set[char]): string = +func charSetEsc(cc: set[char]): string = if card(cc) >= 128+64: result = "[^" & charSetEscAux({'\1'..'\xFF'} - cc) & ']' else: result = '[' & charSetEscAux(cc) & ']' -proc toStrAux(r: Peg, res: var string) = +func toStrAux(r: Peg, res: var string) = case r.kind of pkEmpty: add(res, "()") of pkAny: add(res, '.') @@ -519,7 +525,7 @@ proc toStrAux(r: Peg, res: var string) = of pkStartAnchor: add(res, '^') -proc `$` *(r: Peg): string {.noSideEffect, rtl, extern: "npegsToString".} = +func `$` *(r: Peg): string {.rtl, extern: "npegsToString".} = ## converts a PEG to its string representation result = "" toStrAux(r, result) @@ -532,7 +538,7 @@ type ml: int origStart: int -proc bounds*(c: Captures, +func bounds*(c: Captures, i: range[0..MaxSubpatterns-1]): tuple[first, last: int] = ## returns the bounds ``[first..last]`` of the `i`'th capture. result = c.matches[i] @@ -545,24 +551,26 @@ when not useUnicode: inc(i) template runeLenAt(s, i): untyped = 1 - proc isAlpha(a: char): bool {.inline.} = return a in {'a'..'z', 'A'..'Z'} - proc isUpper(a: char): bool {.inline.} = return a in {'A'..'Z'} - proc isLower(a: char): bool {.inline.} = return a in {'a'..'z'} - proc isTitle(a: char): bool {.inline.} = return false - proc isWhiteSpace(a: char): bool {.inline.} = return a in {' ', '\9'..'\13'} + func isAlpha(a: char): bool {.inline.} = return a in {'a'..'z', 'A'..'Z'} + func isUpper(a: char): bool {.inline.} = return a in {'A'..'Z'} + func isLower(a: char): bool {.inline.} = return a in {'a'..'z'} + func isTitle(a: char): bool {.inline.} = return false + func isWhiteSpace(a: char): bool {.inline.} = return a in {' ', '\9'..'\13'} template matchOrParse(mopProc: untyped) = # Used to make the main matcher proc *rawMatch* as well as event parser # procs. For the former, *enter* and *leave* event handler code generators # are provided which just return *discard*. - proc mopProc(s: string, p: Peg, start: int, c: var Captures): int = + proc mopProc(s: string, p: Peg, start: int, c: var Captures): int {.gcsafe, raises: [].} = proc matchBackRef(s: string, p: Peg, start: int, c: var Captures): int = # Parse handler code must run in an *of* clause of its own for each # *PegKind*, so we encapsulate the identical clause body for # *pkBackRef..pkBackRefIgnoreStyle* here. - if p.index >= c.ml: return -1 - var (a, b) = c.matches[p.index] + var index = p.index + if index < 0: index.inc(c.ml) + if index < 0 or index >= c.ml: return -1 + var (a, b) = c.matches[index] var n: Peg case p.kind of pkBackRef: @@ -822,15 +830,22 @@ template matchOrParse(mopProc: untyped) = leave(pkNotPredicate, s, p, start, result) of pkCapture: enter(pkCapture, s, p, start) - var idx = c.ml # reserve a slot for the subpattern - inc(c.ml) - result = mopProc(s, p.sons[0], start, c) - if result >= 0: - if idx < MaxSubpatterns: - c.matches[idx] = (start, start+result-1) - #else: silently ignore the capture + if p.sons.len == 0 or p.sons[0].kind == pkEmpty: + # empty capture removes last match + dec(c.ml) + c.matches[c.ml] = (0, 0) + result = 0 # match of length 0 else: - c.ml = idx + var idx = c.ml # reserve a slot for the subpattern + result = mopProc(s, p.sons[0], start, c) + if result >= 0: + if idx < MaxSubpatterns: + if idx != c.ml: + for i in countdown(c.ml, idx): + c.matches[i+1] = c.matches[i] + c.matches[idx] = (start, start+result-1) + #else: silently ignore the capture + inc(c.ml) leave(pkCapture, s, p, start, result) of pkBackRef: enter(pkBackRef, s, p, start) @@ -851,8 +866,8 @@ template matchOrParse(mopProc: untyped) = leave(pkStartAnchor, s, p, start, result) of pkRule, pkList: assert false -proc rawMatch*(s: string, p: Peg, start: int, c: var Captures): int - {.noSideEffect, rtl, extern: "npegs$1".} = +func rawMatch*(s: string, p: Peg, start: int, c: var Captures): int + {.rtl, extern: "npegs$1".} = ## low-level matching proc that implements the PEG interpreter. Use this ## for maximum efficiency (every other PEG operation ends up calling this ## proc). @@ -864,13 +879,17 @@ proc rawMatch*(s: string, p: Peg, start: int, c: var Captures): int template leave(pk, s, p, start, length) = discard matchOrParse(matchIt) - result = matchIt(s, p, start, c) + {.cast(noSideEffect).}: + # This cast is allowed because the `matchOrParse` template is used for + # both matching and parsing, but side effects are only possible when it's + # used by `eventParser`. + result = matchIt(s, p, start, c) macro mkHandlerTplts(handlers: untyped): untyped = # Transforms the handler spec in *handlers* into handler templates. # The AST structure of *handlers[0]*: # - # .. code-block:: + # ``` # StmtList # Call # Ident "pkNonTerminal" @@ -891,7 +910,8 @@ macro mkHandlerTplts(handlers: untyped): untyped = # StmtList # <handler code block> # ... - proc mkEnter(hdName, body: NimNode): NimNode = + # ``` + func mkEnter(hdName, body: NimNode): NimNode = template helper(hdName, body) {.dirty.} = template hdName(s, p, start) = let s {.inject.} = s @@ -940,60 +960,61 @@ template eventParser*(pegAst, handlers: untyped): (proc(s: string): int) = ## match, else the length of the total match. The following example code ## evaluates an arithmetic expression defined by a simple PEG: ## - ## .. code-block:: nim - ## import std/[strutils, pegs] + ## ```nim + ## import std/[strutils, pegs] ## - ## let - ## pegAst = """ - ## Expr <- Sum - ## Sum <- Product (('+' / '-')Product)* - ## Product <- Value (('*' / '/')Value)* - ## Value <- [0-9]+ / '(' Expr ')' - ## """.peg - ## txt = "(5+3)/2-7*22" + ## let + ## pegAst = """ + ## Expr <- Sum + ## Sum <- Product (('+' / '-')Product)* + ## Product <- Value (('*' / '/')Value)* + ## Value <- [0-9]+ / '(' Expr ')' + ## """.peg + ## txt = "(5+3)/2-7*22" ## - ## var - ## pStack: seq[string] = @[] - ## valStack: seq[float] = @[] - ## opStack = "" - ## let - ## parseArithExpr = pegAst.eventParser: - ## pkNonTerminal: - ## enter: - ## pStack.add p.nt.name - ## leave: - ## pStack.setLen pStack.high - ## if length > 0: - ## let matchStr = s.substr(start, start+length-1) - ## case p.nt.name - ## of "Value": - ## try: - ## valStack.add matchStr.parseFloat - ## echo valStack - ## except ValueError: - ## discard - ## of "Sum", "Product": - ## try: - ## let val = matchStr.parseFloat - ## except ValueError: - ## if valStack.len > 1 and opStack.len > 0: - ## valStack[^2] = case opStack[^1] - ## of '+': valStack[^2] + valStack[^1] - ## of '-': valStack[^2] - valStack[^1] - ## of '*': valStack[^2] * valStack[^1] - ## else: valStack[^2] / valStack[^1] - ## valStack.setLen valStack.high - ## echo valStack - ## opStack.setLen opStack.high - ## echo opStack - ## pkChar: - ## leave: - ## if length == 1 and "Value" != pStack[^1]: - ## let matchChar = s[start] - ## opStack.add matchChar - ## echo opStack + ## var + ## pStack: seq[string] = @[] + ## valStack: seq[float] = @[] + ## opStack = "" + ## let + ## parseArithExpr = pegAst.eventParser: + ## pkNonTerminal: + ## enter: + ## pStack.add p.nt.name + ## leave: + ## pStack.setLen pStack.high + ## if length > 0: + ## let matchStr = s.substr(start, start+length-1) + ## case p.nt.name + ## of "Value": + ## try: + ## valStack.add matchStr.parseFloat + ## echo valStack + ## except ValueError: + ## discard + ## of "Sum", "Product": + ## try: + ## let val = matchStr.parseFloat + ## except ValueError: + ## if valStack.len > 1 and opStack.len > 0: + ## valStack[^2] = case opStack[^1] + ## of '+': valStack[^2] + valStack[^1] + ## of '-': valStack[^2] - valStack[^1] + ## of '*': valStack[^2] * valStack[^1] + ## else: valStack[^2] / valStack[^1] + ## valStack.setLen valStack.high + ## echo valStack + ## opStack.setLen opStack.high + ## echo opStack + ## pkChar: + ## leave: + ## if length == 1 and "Value" != pStack[^1]: + ## let matchChar = s[start] + ## opStack.add matchChar + ## echo opStack ## - ## let pLen = parseArithExpr(txt) + ## let pLen = parseArithExpr(txt) + ## ``` ## ## The *handlers* parameter consists of code blocks for *PegKinds*, ## which define the grammar elements of interest. Each block can contain @@ -1008,7 +1029,7 @@ template eventParser*(pegAst, handlers: untyped): (proc(s: string): int) = ## Symbols declared in an *enter* handler can be made visible in the ## corresponding *leave* handler by annotating them with an *inject* pragma. proc rawParse(s: string, p: Peg, start: int, c: var Captures): int - {.genSym.} = + {.gensym.} = # binding from *macros* bind strVal @@ -1023,10 +1044,10 @@ template eventParser*(pegAst, handlers: untyped): (proc(s: string): int) = # by *mkHandlerTplts*. template mkDoEnter(hdPostf, s, pegNode, start) = when declared(`enter hdPostf`): - `enter hdPostf`(s, pegNode, start): + `enter hdPostf`(s, pegNode, start) else: discard - let hdPostf = ident(substr(strVal(pegKind), 2)) + let hdPostf = ident(substr($pegKind, 2)) getAst(mkDoEnter(hdPostf, s, pegNode, start)) macro leave(pegKind, s, pegNode, start, length: untyped): untyped = @@ -1034,16 +1055,16 @@ template eventParser*(pegAst, handlers: untyped): (proc(s: string): int) = # a grammar element of kind *pegKind*. template mkDoLeave(hdPostf, s, pegNode, start, length) = when declared(`leave hdPostf`): - `leave hdPostf`(s, pegNode, start, length): + `leave hdPostf`(s, pegNode, start, length) else: discard - let hdPostf = ident(substr(strVal(pegKind), 2)) + let hdPostf = ident(substr($pegKind, 2)) getAst(mkDoLeave(hdPostf, s, pegNode, start, length)) matchOrParse(parseIt) parseIt(s, p, start, c) - proc parser(s: string): int {.genSym.} = + proc parser(s: string): int {.gensym.} = # the proc to be returned var ms: array[MaxSubpatterns, (int, int)] @@ -1060,8 +1081,8 @@ template fillMatches(s, caps, c) = else: caps[k] = "" -proc matchLen*(s: string, pattern: Peg, matches: var openArray[string], - start = 0): int {.noSideEffect, rtl, extern: "npegs$1Capture".} = +func matchLen*(s: string, pattern: Peg, matches: var openArray[string], + start = 0): int {.rtl, extern: "npegs$1Capture".} = ## the same as ``match``, but it returns the length of the match, ## if there is no match, -1 is returned. Note that a match length ## of zero can happen. It's possible that a suffix of `s` remains @@ -1071,8 +1092,8 @@ proc matchLen*(s: string, pattern: Peg, matches: var openArray[string], result = rawMatch(s, pattern, start, c) if result >= 0: fillMatches(s, matches, c) -proc matchLen*(s: string, pattern: Peg, - start = 0): int {.noSideEffect, rtl, extern: "npegs$1".} = +func matchLen*(s: string, pattern: Peg, + start = 0): int {.rtl, extern: "npegs$1".} = ## the same as ``match``, but it returns the length of the match, ## if there is no match, -1 is returned. Note that a match length ## of zero can happen. It's possible that a suffix of `s` remains @@ -1081,22 +1102,22 @@ proc matchLen*(s: string, pattern: Peg, c.origStart = start result = rawMatch(s, pattern, start, c) -proc match*(s: string, pattern: Peg, matches: var openArray[string], - start = 0): bool {.noSideEffect, rtl, extern: "npegs$1Capture".} = +func match*(s: string, pattern: Peg, matches: var openArray[string], + start = 0): bool {.rtl, extern: "npegs$1Capture".} = ## returns ``true`` if ``s[start..]`` matches the ``pattern`` and ## the captured substrings in the array ``matches``. If it does not ## match, nothing is written into ``matches`` and ``false`` is ## returned. result = matchLen(s, pattern, matches, start) != -1 -proc match*(s: string, pattern: Peg, - start = 0): bool {.noSideEffect, rtl, extern: "npegs$1".} = +func match*(s: string, pattern: Peg, + start = 0): bool {.rtl, extern: "npegs$1".} = ## returns ``true`` if ``s`` matches the ``pattern`` beginning from ``start``. result = matchLen(s, pattern, start) != -1 -proc find*(s: string, pattern: Peg, matches: var openArray[string], - start = 0): int {.noSideEffect, rtl, extern: "npegs$1Capture".} = +func find*(s: string, pattern: Peg, matches: var openArray[string], + start = 0): int {.rtl, extern: "npegs$1Capture".} = ## returns the starting position of ``pattern`` in ``s`` and the captured ## substrings in the array ``matches``. If it does not match, nothing ## is written into ``matches`` and -1 is returned. @@ -1110,9 +1131,9 @@ proc find*(s: string, pattern: Peg, matches: var openArray[string], return -1 # could also use the pattern here: (!P .)* P -proc findBounds*(s: string, pattern: Peg, matches: var openArray[string], +func findBounds*(s: string, pattern: Peg, matches: var openArray[string], start = 0): tuple[first, last: int] {. - noSideEffect, rtl, extern: "npegs$1Capture".} = + rtl, extern: "npegs$1Capture".} = ## returns the starting position and end position of ``pattern`` in ``s`` ## and the captured ## substrings in the array ``matches``. If it does not match, nothing @@ -1127,8 +1148,8 @@ proc findBounds*(s: string, pattern: Peg, matches: var openArray[string], return (i, i+L-1) return (-1, 0) -proc find*(s: string, pattern: Peg, - start = 0): int {.noSideEffect, rtl, extern: "npegs$1".} = +func find*(s: string, pattern: Peg, + start = 0): int {.rtl, extern: "npegs$1".} = ## returns the starting position of ``pattern`` in ``s``. If it does not ## match, -1 is returned. var c: Captures @@ -1151,10 +1172,10 @@ iterator findAll*(s: string, pattern: Peg, start = 0): string = yield substr(s, i, i+L-1) inc(i, L) -proc findAll*(s: string, pattern: Peg, start = 0): seq[string] {. - noSideEffect, rtl, extern: "npegs$1".} = +func findAll*(s: string, pattern: Peg, start = 0): seq[string] {. + rtl, extern: "npegs$1".} = ## returns all matching *substrings* of `s` that match `pattern`. - ## If it does not match, @[] is returned. + ## If it does not match, `@[]` is returned. result = @[] for it in findAll(s, pattern, start): result.add it @@ -1162,8 +1183,7 @@ template `=~`*(s: string, pattern: Peg): bool = ## This calls ``match`` with an implicit declared ``matches`` array that ## can be used in the scope of the ``=~`` call: ## - ## .. code-block:: nim - ## + ## ```nim ## if line =~ peg"\s* {\w+} \s* '=' \s* {\w+}": ## # matches a key=value pair: ## echo("Key: ", matches[0]) @@ -1175,50 +1195,51 @@ template `=~`*(s: string, pattern: Peg): bool = ## echo("comment: ", matches[0]) ## else: ## echo("syntax error") - ## + ## ``` bind MaxSubpatterns when not declaredInScope(matches): - var matches {.inject.}: array[0..MaxSubpatterns-1, string] + var matches {.inject.} = default(array[0..MaxSubpatterns-1, string]) match(s, pattern, matches) # ------------------------- more string handling ------------------------------ -proc contains*(s: string, pattern: Peg, start = 0): bool {. - noSideEffect, rtl, extern: "npegs$1".} = +func contains*(s: string, pattern: Peg, start = 0): bool {. + rtl, extern: "npegs$1".} = ## same as ``find(s, pattern, start) >= 0`` return find(s, pattern, start) >= 0 -proc contains*(s: string, pattern: Peg, matches: var openArray[string], - start = 0): bool {.noSideEffect, rtl, extern: "npegs$1Capture".} = +func contains*(s: string, pattern: Peg, matches: var openArray[string], + start = 0): bool {.rtl, extern: "npegs$1Capture".} = ## same as ``find(s, pattern, matches, start) >= 0`` return find(s, pattern, matches, start) >= 0 -proc startsWith*(s: string, prefix: Peg, start = 0): bool {. - noSideEffect, rtl, extern: "npegs$1".} = +func startsWith*(s: string, prefix: Peg, start = 0): bool {. + rtl, extern: "npegs$1".} = ## returns true if `s` starts with the pattern `prefix` result = matchLen(s, prefix, start) >= 0 -proc endsWith*(s: string, suffix: Peg, start = 0): bool {. - noSideEffect, rtl, extern: "npegs$1".} = +func endsWith*(s: string, suffix: Peg, start = 0): bool {. + rtl, extern: "npegs$1".} = ## returns true if `s` ends with the pattern `suffix` var c: Captures c.origStart = start for i in start .. s.len-1: if rawMatch(s, suffix, i, c) == s.len - i: return true -proc replacef*(s: string, sub: Peg, by: string): string {. - noSideEffect, rtl, extern: "npegs$1".} = +func replacef*(s: string, sub: Peg, by: string): string {. + rtl, extern: "npegs$1".} = ## Replaces `sub` in `s` by the string `by`. Captures can be accessed in `by` ## with the notation ``$i`` and ``$#`` (see strutils.`%`). Examples: ## - ## .. code-block:: nim + ## ```nim ## "var1=key; var2=key2".replacef(peg"{\ident}'='{\ident}", "$1<-$2$2") + ## ``` ## ## Results in: ## - ## .. code-block:: nim - ## + ## ```nim ## "var1<-keykey; val2<-key2key2" + ## ``` result = "" var i = 0 var caps: array[0..MaxSubpatterns-1, string] @@ -1235,8 +1256,8 @@ proc replacef*(s: string, sub: Peg, by: string): string {. inc(i, x) add(result, substr(s, i)) -proc replace*(s: string, sub: Peg, by = ""): string {. - noSideEffect, rtl, extern: "npegs$1".} = +func replace*(s: string, sub: Peg, by = ""): string {. + rtl, extern: "npegs$1".} = ## Replaces `sub` in `s` by the string `by`. Captures cannot be accessed ## in `by`. result = "" @@ -1252,9 +1273,9 @@ proc replace*(s: string, sub: Peg, by = ""): string {. inc(i, x) add(result, substr(s, i)) -proc parallelReplace*(s: string, subs: varargs[ +func parallelReplace*(s: string, subs: varargs[ tuple[pattern: Peg, repl: string]]): string {. - noSideEffect, rtl, extern: "npegs$1".} = + rtl, extern: "npegs$1".} = ## Returns a modified copy of `s` with the substitutions in `subs` ## applied in parallel. result = "" @@ -1276,16 +1297,18 @@ proc parallelReplace*(s: string, subs: varargs[ # copy the rest: add(result, substr(s, i)) -proc replace*(s: string, sub: Peg, cb: proc( +when not defined(nimHasEffectsOf): + {.pragma: effectsOf.} + +func replace*(s: string, sub: Peg, cb: proc( match: int, cnt: int, caps: openArray[string]): string): string {. - rtl, extern: "npegs$1cb".} = + rtl, extern: "npegs$1cb", effectsOf: cb.} = ## Replaces `sub` in `s` by the resulting strings from the callback. ## The callback proc receives the index of the current match (starting with 0), ## the count of captures and an open array with the captures of each match. Examples: ## - ## .. code-block:: nim - ## - ## proc handleMatches*(m: int, n: int, c: openArray[string]): string = + ## ```nim + ## func handleMatches*(m: int, n: int, c: openArray[string]): string = ## result = "" ## if m > 0: ## result.add ", " @@ -1296,12 +1319,13 @@ proc replace*(s: string, sub: Peg, cb: proc( ## ## let s = "Var1=key1;var2=Key2; VAR3" ## echo s.replace(peg"{\ident}('='{\ident})* ';'* \s*", handleMatches) + ## ``` ## ## Results in: ## - ## .. code-block:: nim - ## + ## ```nim ## "var1: 'key1', var2: 'Key2', var3: ''" + ## ``` result = "" var i = 0 var caps: array[0..MaxSubpatterns-1, string] @@ -1339,18 +1363,19 @@ iterator split*(s: string, sep: Peg): string = ## Substrings are separated by the PEG `sep`. ## Examples: ## - ## .. code-block:: nim + ## ```nim ## for word in split("00232this02939is39an22example111", peg"\d+"): ## writeLine(stdout, word) + ## ``` ## ## Results in: ## - ## .. code-block:: nim + ## ```nim ## "this" ## "is" ## "an" ## "example" - ## + ## ``` var c: Captures var first = 0 @@ -1368,8 +1393,8 @@ iterator split*(s: string, sep: Peg): string = if first < last: yield substr(s, first, last-1) -proc split*(s: string, sep: Peg): seq[string] {. - noSideEffect, rtl, extern: "npegs$1".} = +func split*(s: string, sep: Peg): seq[string] {. + rtl, extern: "npegs$1".} = ## Splits the string `s` into substrings. result = @[] for it in split(s, sep): result.add it @@ -1395,6 +1420,7 @@ type tkCurlyLe, ## '{' tkCurlyRi, ## '}' tkCurlyAt, ## '{@}' + tkEmptyCurl, ## '{}' tkArrow, ## '<-' tkBar, ## '/' tkStar, ## '*' @@ -1427,25 +1453,25 @@ type const tokKindToStr: array[TokKind, string] = [ "invalid", "[EOF]", ".", "_", "identifier", "string literal", - "character set", "(", ")", "{", "}", "{@}", + "character set", "(", ")", "{", "}", "{@}", "{}", "<-", "/", "*", "+", "&", "!", "?", "@", "built-in", "escaped", "$", "$", "^" ] -proc handleCR(L: var PegLexer, pos: int): int = +func handleCR(L: var PegLexer, pos: int): int = assert(L.buf[pos] == '\c') inc(L.lineNumber) result = pos+1 if result < L.buf.len and L.buf[result] == '\L': inc(result) L.lineStart = result -proc handleLF(L: var PegLexer, pos: int): int = +func handleLF(L: var PegLexer, pos: int): int = assert(L.buf[pos] == '\L') inc(L.lineNumber) result = pos+1 L.lineStart = result -proc init(L: var PegLexer, input, filename: string, line = 1, col = 0) = +func init(L: var PegLexer, input, filename: string, line = 1, col = 0) = L.buf = input L.bufpos = 0 L.lineNumber = line @@ -1453,18 +1479,18 @@ proc init(L: var PegLexer, input, filename: string, line = 1, col = 0) = L.lineStart = 0 L.filename = filename -proc getColumn(L: PegLexer): int {.inline.} = +func getColumn(L: PegLexer): int {.inline.} = result = abs(L.bufpos - L.lineStart) + L.colOffset -proc getLine(L: PegLexer): int {.inline.} = +func getLine(L: PegLexer): int {.inline.} = result = L.lineNumber -proc errorStr(L: PegLexer, msg: string, line = -1, col = -1): string = +func errorStr(L: PegLexer, msg: string, line = -1, col = -1): string = var line = if line < 0: getLine(L) else: line var col = if col < 0: getColumn(L) else: col result = "$1($2, $3) Error: $4" % [L.filename, $line, $col, msg] -proc getEscapedChar(c: var PegLexer, tok: var Token) = +func getEscapedChar(c: var PegLexer, tok: var Token) = inc(c.bufpos) if c.bufpos >= len(c.buf): tok.kind = tkInvalid @@ -1524,7 +1550,7 @@ proc getEscapedChar(c: var PegLexer, tok: var Token) = add(tok.literal, c.buf[c.bufpos]) inc(c.bufpos) -proc skip(c: var PegLexer) = +func skip(c: var PegLexer) = var pos = c.bufpos while pos < c.buf.len: case c.buf[pos] @@ -1541,7 +1567,7 @@ proc skip(c: var PegLexer) = break # EndOfFile also leaves the loop c.bufpos = pos -proc getString(c: var PegLexer, tok: var Token) = +func getString(c: var PegLexer, tok: var Token) = tok.kind = tkStringLit var pos = c.bufpos + 1 var quote = c.buf[pos-1] @@ -1562,19 +1588,27 @@ proc getString(c: var PegLexer, tok: var Token) = inc(pos) c.bufpos = pos -proc getDollar(c: var PegLexer, tok: var Token) = +func getDollar(c: var PegLexer, tok: var Token) = var pos = c.bufpos + 1 + var neg = false + if pos < c.buf.len and c.buf[pos] == '^': + neg = true + inc(pos) if pos < c.buf.len and c.buf[pos] in {'0'..'9'}: tok.kind = tkBackref tok.index = 0 while pos < c.buf.len and c.buf[pos] in {'0'..'9'}: tok.index = tok.index * 10 + ord(c.buf[pos]) - ord('0') inc(pos) + if neg: + tok.index = -tok.index else: + if neg: + dec(pos) tok.kind = tkDollar c.bufpos = pos -proc getCharSet(c: var PegLexer, tok: var Token) = +func getCharSet(c: var PegLexer, tok: var Token) = tok.kind = tkCharSet tok.charset = {} var pos = c.bufpos + 1 @@ -1631,7 +1665,7 @@ proc getCharSet(c: var PegLexer, tok: var Token) = c.bufpos = pos if caret: tok.charset = {'\1'..'\xFF'} - tok.charset -proc getSymbol(c: var PegLexer, tok: var Token) = +func getSymbol(c: var PegLexer, tok: var Token) = var pos = c.bufpos while pos < c.buf.len: add(tok.literal, c.buf[pos]) @@ -1640,7 +1674,7 @@ proc getSymbol(c: var PegLexer, tok: var Token) = c.bufpos = pos tok.kind = tkIdentifier -proc getBuiltin(c: var PegLexer, tok: var Token) = +func getBuiltin(c: var PegLexer, tok: var Token) = if c.bufpos+1 < c.buf.len and c.buf[c.bufpos+1] in strutils.Letters: inc(c.bufpos) getSymbol(c, tok) @@ -1649,7 +1683,7 @@ proc getBuiltin(c: var PegLexer, tok: var Token) = tok.kind = tkEscaped getEscapedChar(c, tok) # may set tok.kind to tkInvalid -proc getTok(c: var PegLexer, tok: var Token) = +func getTok(c: var PegLexer, tok: var Token) = tok.kind = tkInvalid tok.modifier = modNone setLen(tok.literal, 0) @@ -1670,6 +1704,10 @@ proc getTok(c: var PegLexer, tok: var Token) = tok.kind = tkCurlyAt inc(c.bufpos, 2) add(tok.literal, "{@}") + elif c.buf[c.bufpos] == '}' and c.bufpos < c.buf.len: + tok.kind = tkEmptyCurl + inc(c.bufpos) + add(tok.literal, "{}") else: tok.kind = tkCurlyLe add(tok.literal, '{') @@ -1705,7 +1743,7 @@ proc getTok(c: var PegLexer, tok: var Token) = return if c.buf[c.bufpos] in {'\'', '"'} or c.buf[c.bufpos] == '$' and c.bufpos+1 < c.buf.len and - c.buf[c.bufpos+1] in {'0'..'9'}: + c.buf[c.bufpos+1] in {'^', '0'..'9'}: case tok.literal of "i": tok.modifier = modIgnoreCase of "y": tok.modifier = modIgnoreStyle @@ -1767,7 +1805,7 @@ proc getTok(c: var PegLexer, tok: var Token) = add(tok.literal, c.buf[c.bufpos]) inc(c.bufpos) -proc arrowIsNextTok(c: PegLexer): bool = +func arrowIsNextTok(c: PegLexer): bool = # the only look ahead we need var pos = c.bufpos while pos < c.buf.len and c.buf[pos] in {'\t', ' '}: inc(pos) @@ -1788,23 +1826,21 @@ type identIsVerbatim: bool skip: Peg -proc pegError(p: PegParser, msg: string, line = -1, col = -1) = - var e: ref EInvalidPeg - new(e) - e.msg = errorStr(p, msg, line, col) +func pegError(p: PegParser, msg: string, line = -1, col = -1) = + var e = (ref EInvalidPeg)(msg: errorStr(p, msg, line, col)) raise e -proc getTok(p: var PegParser) = +func getTok(p: var PegParser) = getTok(p, p.tok) if p.tok.kind == tkInvalid: pegError(p, "'" & p.tok.literal & "' is invalid token") -proc eat(p: var PegParser, kind: TokKind) = +func eat(p: var PegParser, kind: TokKind) = if p.tok.kind == kind: getTok(p) else: pegError(p, tokKindToStr[kind] & " expected") -proc parseExpr(p: var PegParser): Peg {.gcsafe.} +func parseExpr(p: var PegParser): Peg {.gcsafe.} -proc getNonTerminal(p: var PegParser, name: string): NonTerminal = +func getNonTerminal(p: var PegParser, name: string): NonTerminal = for i in 0..high(p.nonterms): result = p.nonterms[i] if cmpIgnoreStyle(result.name, name) == 0: return @@ -1812,19 +1848,22 @@ proc getNonTerminal(p: var PegParser, name: string): NonTerminal = result = newNonTerminal(name, getLine(p), getColumn(p)) add(p.nonterms, result) -proc modifiedTerm(s: string, m: Modifier): Peg = +func modifiedTerm(s: string, m: Modifier): Peg = case m of modNone, modVerbatim: result = term(s) of modIgnoreCase: result = termIgnoreCase(s) of modIgnoreStyle: result = termIgnoreStyle(s) -proc modifiedBackref(s: int, m: Modifier): Peg = +func modifiedBackref(s: int, m: Modifier): Peg = + var + reverse = s < 0 + index = if reverse: -s else: s case m - of modNone, modVerbatim: result = backref(s) - of modIgnoreCase: result = backrefIgnoreCase(s) - of modIgnoreStyle: result = backrefIgnoreStyle(s) + of modNone, modVerbatim: result = backref(index, reverse) + of modIgnoreCase: result = backrefIgnoreCase(index, reverse) + of modIgnoreStyle: result = backrefIgnoreStyle(index, reverse) -proc builtin(p: var PegParser): Peg = +func builtin(p: var PegParser): Peg = # do not use "y", "skip" or "i" as these would be ambiguous case p.tok.literal of "n": result = newLine() @@ -1844,11 +1883,11 @@ proc builtin(p: var PegParser): Peg = of "white": result = unicodeWhitespace() else: pegError(p, "unknown built-in: " & p.tok.literal) -proc token(terminal: Peg, p: PegParser): Peg = +func token(terminal: Peg, p: PegParser): Peg = if p.skip.kind == pkEmpty: result = terminal else: result = sequence(p.skip, terminal) -proc primary(p: var PegParser): Peg = +func primary(p: var PegParser): Peg = case p.tok.kind of tkAmp: getTok(p) @@ -1872,7 +1911,8 @@ proc primary(p: var PegParser): Peg = getTok(p) elif not arrowIsNextTok(p): var nt = getNonTerminal(p, p.tok.literal) - incl(nt.flags, ntUsed) + {.cast(noSideEffect).}: + incl(nt.flags, ntUsed) result = nonterminal(nt).token(p) getTok(p) else: @@ -1896,6 +1936,9 @@ proc primary(p: var PegParser): Peg = result = capture(parseExpr(p)).token(p) eat(p, tkCurlyRi) inc(p.captures) + of tkEmptyCurl: + result = capture() + getTok(p) of tkAny: result = any().token(p) getTok(p) @@ -1915,11 +1958,11 @@ proc primary(p: var PegParser): Peg = result = startAnchor() getTok(p) of tkBackref: + if abs(p.tok.index) > p.captures or p.tok.index == 0: + pegError(p, "invalid back reference index: " & $p.tok.index) var m = p.tok.modifier if m == modNone: m = p.modifier result = modifiedBackref(p.tok.index, m).token(p) - if p.tok.index < 0 or p.tok.index > p.captures: - pegError(p, "invalid back reference index: " & $p.tok.index) getTok(p) else: pegError(p, "expression expected, but found: " & p.tok.literal) @@ -1937,13 +1980,13 @@ proc primary(p: var PegParser): Peg = getTok(p) else: break -proc seqExpr(p: var PegParser): Peg = +func seqExpr(p: var PegParser): Peg = result = primary(p) while true: case p.tok.kind of tkAmp, tkNot, tkAt, tkStringLit, tkCharSet, tkParLe, tkCurlyLe, tkAny, tkAnyRune, tkBuiltin, tkEscaped, tkDollar, tkBackref, - tkHat, tkCurlyAt: + tkHat, tkCurlyAt, tkEmptyCurl: result = sequence(result, primary(p)) of tkIdentifier: if not arrowIsNextTok(p): @@ -1951,27 +1994,29 @@ proc seqExpr(p: var PegParser): Peg = else: break else: break -proc parseExpr(p: var PegParser): Peg = +func parseExpr(p: var PegParser): Peg = result = seqExpr(p) while p.tok.kind == tkBar: getTok(p) result = result / seqExpr(p) -proc parseRule(p: var PegParser): NonTerminal = +func parseRule(p: var PegParser): NonTerminal = if p.tok.kind == tkIdentifier and arrowIsNextTok(p): result = getNonTerminal(p, p.tok.literal) if ntDeclared in result.flags: pegError(p, "attempt to redefine: " & result.name) - result.line = getLine(p) - result.col = getColumn(p) + {.cast(noSideEffect).}: + result.line = getLine(p) + result.col = getColumn(p) getTok(p) eat(p, tkArrow) - result.rule = parseExpr(p) - incl(result.flags, ntDeclared) # NOW inlining may be attempted + {.cast(noSideEffect).}: + result.rule = parseExpr(p) + incl(result.flags, ntDeclared) # NOW inlining may be attempted else: pegError(p, "rule expected, but found: " & p.tok.literal) -proc rawParse(p: var PegParser): Peg = +func rawParse(p: var PegParser): Peg = ## parses a rule or a PEG expression while p.tok.kind == tkBuiltin: case p.tok.literal @@ -2001,7 +2046,7 @@ proc rawParse(p: var PegParser): Peg = elif ntUsed notin nt.flags and i > 0: pegError(p, "unused rule: " & nt.name, nt.line, nt.col) -proc parsePeg*(pattern: string, filename = "pattern", line = 1, col = 0): Peg = +func parsePeg*(pattern: string, filename = "pattern", line = 1, col = 0): Peg = ## constructs a Peg object from `pattern`. `filename`, `line`, `col` are ## used for error messages, but they only provide start offsets. `parsePeg` ## keeps track of line and column numbers within `pattern`. @@ -2016,14 +2061,14 @@ proc parsePeg*(pattern: string, filename = "pattern", line = 1, col = 0): Peg = getTok(p) result = rawParse(p) -proc peg*(pattern: string): Peg = +func peg*(pattern: string): Peg = ## constructs a Peg object from the `pattern`. The short name has been - ## chosen to encourage its use as a raw string modifier:: + ## chosen to encourage its use as a raw string modifier: ## - ## peg"{\ident} \s* '=' \s* {.*}" + ## peg"{\ident} \s* '=' \s* {.*}" result = parsePeg(pattern, "pattern") -proc escapePeg*(s: string): string = +func escapePeg*(s: string): string = ## escapes `s` so that it is matched verbatim when used as a peg. result = "" var inQuote = false @@ -2041,147 +2086,3 @@ proc escapePeg*(s: string): string = inQuote = true result.add(c) if inQuote: result.add('\'') - -when isMainModule: - proc pegsTest() = - assert escapePeg("abc''def'") == r"'abc'\x27\x27'def'\x27" - assert match("(a b c)", peg"'(' @ ')'") - assert match("W_HI_Le", peg"\y 'while'") - assert(not match("W_HI_L", peg"\y 'while'")) - assert(not match("W_HI_Le", peg"\y v'while'")) - assert match("W_HI_Le", peg"y'while'") - - assert($ +digits == $peg"\d+") - assert "0158787".match(peg"\d+") - assert "ABC 0232".match(peg"\w+\s+\d+") - assert "ABC".match(peg"\d+ / \w+") - - var accum: seq[string] = @[] - for word in split("00232this02939is39an22example111", peg"\d+"): - accum.add(word) - assert(accum == @["this", "is", "an", "example"]) - - assert matchLen("key", ident) == 3 - - var pattern = sequence(ident, *whitespace, term('='), *whitespace, ident) - assert matchLen("key1= cal9", pattern) == 11 - - var ws = newNonTerminal("ws", 1, 1) - ws.rule = *whitespace - - var expr = newNonTerminal("expr", 1, 1) - expr.rule = sequence(capture(ident), *sequence( - nonterminal(ws), term('+'), nonterminal(ws), nonterminal(expr))) - - var c: Captures - var s = "a+b + c +d+e+f" - assert rawMatch(s, expr.rule, 0, c) == len(s) - var a = "" - for i in 0..c.ml-1: - a.add(substr(s, c.matches[i][0], c.matches[i][1])) - assert a == "abcdef" - #echo expr.rule - - #const filename = "lib/devel/peg/grammar.txt" - #var grammar = parsePeg(newFileStream(filename, fmRead), filename) - #echo "a <- [abc]*?".match(grammar) - assert find("_____abc_______", term("abc"), 2) == 5 - assert match("_______ana", peg"A <- 'ana' / . A") - assert match("abcs%%%", peg"A <- ..A / .A / '%'") - - var matches: array[0..MaxSubpatterns-1, string] - if "abc" =~ peg"{'a'}'bc' 'xyz' / {\ident}": - assert matches[0] == "abc" - else: - assert false - - var g2 = peg"""S <- A B / C D - A <- 'a'+ - B <- 'b'+ - C <- 'c'+ - D <- 'd'+ - """ - assert($g2 == "((A B) / (C D))") - assert match("cccccdddddd", g2) - assert("var1=key; var2=key2".replacef(peg"{\ident}'='{\ident}", "$1<-$2$2") == - "var1<-keykey; var2<-key2key2") - assert("var1=key; var2=key2".replace(peg"{\ident}'='{\ident}", "$1<-$2$2") == - "$1<-$2$2; $1<-$2$2") - assert "var1=key; var2=key2".endsWith(peg"{\ident}'='{\ident}") - - if "aaaaaa" =~ peg"'aa' !. / ({'a'})+": - assert matches[0] == "a" - else: - assert false - - if match("abcdefg", peg"c {d} ef {g}", matches, 2): - assert matches[0] == "d" - assert matches[1] == "g" - else: - assert false - - accum = @[] - for x in findAll("abcdef", peg".", 3): - accum.add(x) - assert(accum == @["d", "e", "f"]) - - for x in findAll("abcdef", peg"^{.}", 3): - assert x == "d" - - if "f(a, b)" =~ peg"{[0-9]+} / ({\ident} '(' {@} ')')": - assert matches[0] == "f" - assert matches[1] == "a, b" - else: - assert false - - assert match("eine übersicht und außerdem", peg"(\letter \white*)+") - # ß is not a lower cased letter?! - assert match("eine übersicht und auerdem", peg"(\lower \white*)+") - assert match("EINE ÜBERSICHT UND AUSSERDEM", peg"(\upper \white*)+") - assert(not match("456678", peg"(\letter)+")) - - assert("var1 = key; var2 = key2".replacef( - peg"\skip(\s*) {\ident}'='{\ident}", "$1<-$2$2") == - "var1<-keykey;var2<-key2key2") - - assert match("prefix/start", peg"^start$", 7) - - if "foo" =~ peg"{'a'}?.*": - assert matches[0].len == 0 - else: assert false - - if "foo" =~ peg"{''}.*": - assert matches[0] == "" - else: assert false - - if "foo" =~ peg"{'foo'}": - assert matches[0] == "foo" - else: assert false - - let empty_test = peg"^\d*" - let str = "XYZ" - - assert(str.find(empty_test) == 0) - assert(str.match(empty_test)) - - proc handleMatches(m: int, n: int, c: openArray[string]): string = - result = "" - - if m > 0: - result.add ", " - - result.add case n: - of 2: toLowerAscii(c[0]) & ": '" & c[1] & "'" - of 1: toLowerAscii(c[0]) & ": ''" - else: "" - - assert("Var1=key1;var2=Key2; VAR3". - replace(peg"{\ident}('='{\ident})* ';'* \s*", - handleMatches) == "var1: 'key1', var2: 'Key2', var3: ''") - - - doAssert "test1".match(peg"""{@}$""") - doAssert "test2".match(peg"""{(!$ .)*} $""") - pegsTest() - static: - pegsTest() diff --git a/lib/pure/prelude.nim b/lib/pure/prelude.nim index f1728b5f7..9428f29eb 100644 --- a/lib/pure/prelude.nim +++ b/lib/pure/prelude.nim @@ -14,7 +14,7 @@ when defined(nimdoc) and isMainModule: runnableExamples: include std/prelude # same as: - # import std/[os, strutils, times, parseutils, hashes, tables, sets, sequtils, parseopt] + # import std/[os, strutils, times, parseutils, hashes, tables, sets, sequtils, parseopt, strformat] let x = 1 assert "foo $# $#" % [$x, "bar"] == "foo 1 bar" assert toSeq(1..3) == @[1, 2, 3] @@ -25,4 +25,4 @@ when defined(nimdoc) and isMainModule: # xxx `nim doc -b:js -d:nodejs --doccmd:-d:nodejs lib/pure/prelude.nim` fails for some reason # specific to `nim doc`, but the code otherwise works with nodejs. -import std/[os, strutils, times, parseutils, hashes, tables, sets, sequtils, parseopt] +import std/[os, strutils, times, parseutils, hashes, tables, sets, sequtils, parseopt, strformat] diff --git a/lib/pure/punycode.nim b/lib/pure/punycode.nim deleted file mode 100644 index dfeeea35e..000000000 --- a/lib/pure/punycode.nim +++ /dev/null @@ -1,208 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2016 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## Implements a representation of Unicode with the limited -## ASCII character subset. - -import strutils -import unicode - -# issue #3045 - -const - Base = 36 - TMin = 1 - TMax = 26 - Skew = 38 - Damp = 700 - InitialBias = 72 - InitialN = 128 - Delimiter = '-' - -type - PunyError* = object of ValueError - -proc decodeDigit(x: char): int {.raises: [PunyError].} = - if '0' <= x and x <= '9': - result = ord(x) - (ord('0') - 26) - elif 'A' <= x and x <= 'Z': - result = ord(x) - ord('A') - elif 'a' <= x and x <= 'z': - result = ord(x) - ord('a') - else: - raise newException(PunyError, "Bad input") - -proc encodeDigit(digit: int): Rune {.raises: [PunyError].} = - if 0 <= digit and digit < 26: - result = Rune(digit + ord('a')) - elif 26 <= digit and digit < 36: - result = Rune(digit + (ord('0') - 26)) - else: - raise newException(PunyError, "internal error in punycode encoding") - -proc isBasic(c: char): bool = ord(c) < 0x80 -proc isBasic(r: Rune): bool = int(r) < 0x80 - -proc adapt(delta, numPoints: int, first: bool): int = - var d = if first: delta div Damp else: delta div 2 - d += d div numPoints - var k = 0 - while d > ((Base-TMin)*TMax) div 2: - d = d div (Base - TMin) - k += Base - result = k + (Base - TMin + 1) * d div (d + Skew) - -proc encode*(prefix, s: string): string {.raises: [PunyError].} = - ## Encode a string that may contain Unicode. - ## Prepend `prefix` to the result - result = prefix - var (d, n, bias) = (0, InitialN, InitialBias) - var (b, remaining) = (0, 0) - for r in s.runes: - if r.isBasic: - # basic Ascii character - inc b - result.add($r) - else: - # special character - inc remaining - - var h = b - if b > 0: - result.add(Delimiter) # we have some Ascii chars - while remaining != 0: - var m: int = high(int32) - for r in s.runes: - if m > int(r) and int(r) >= n: - m = int(r) - d += (m - n) * (h + 1) - if d < 0: - raise newException(PunyError, "invalid label " & s) - n = m - for r in s.runes: - if int(r) < n: - inc d - if d < 0: - raise newException(PunyError, "invalid label " & s) - continue - if int(r) > n: - continue - var q = d - var k = Base - while true: - var t = k - bias - if t < TMin: - t = TMin - elif t > TMax: - t = TMax - if q < t: - break - result.add($encodeDigit(t + (q - t) mod (Base - t))) - q = (q - t) div (Base - t) - k += Base - result.add($encodeDigit(q)) - bias = adapt(d, h + 1, h == b) - d = 0 - inc h - dec remaining - inc d - inc n - -proc encode*(s: string): string {.raises: [PunyError].} = - ## Encode a string that may contain Unicode. Prefix is empty. - result = encode("", s) - -proc decode*(encoded: string): string {.raises: [PunyError].} = - ## Decode a Punycode-encoded string - var - n = InitialN - i = 0 - bias = InitialBias - var d = rfind(encoded, Delimiter) - result = "" - - if d > 0: - # found Delimiter - for j in 0..<d: - var c = encoded[j] # char - if not c.isBasic: - raise newException(PunyError, "Encoded contains a non-basic char") - result.add(c) # add the character - inc d - else: - d = 0 # set to first index - - while (d < len(encoded)): - var oldi = i - var w = 1 - var k = Base - while true: - if d == len(encoded): - raise newException(PunyError, "Bad input: " & encoded) - var c = encoded[d]; inc d - var digit = int(decodeDigit(c)) - if digit > (high(int32) - i) div w: - raise newException(PunyError, "Too large a value: " & $digit) - i += digit * w - var t: int - if k <= bias: - t = TMin - elif k >= bias + TMax: - t = TMax - else: - t = k - bias - if digit < t: - break - w *= Base - t - k += Base - bias = adapt(i - oldi, runeLen(result) + 1, oldi == 0) - - if i div (runeLen(result) + 1) > high(int32) - n: - raise newException(PunyError, "Value too large") - - n += i div (runeLen(result) + 1) - i = i mod (runeLen(result) + 1) - insert(result, $Rune(n), i) - inc i - - -runnableExamples: - static: - block: - doAssert encode("") == "" - doAssert encode("a") == "a-" - doAssert encode("A") == "A-" - doAssert encode("3") == "3-" - doAssert encode("-") == "--" - doAssert encode("--") == "---" - doAssert encode("abc") == "abc-" - doAssert encode("London") == "London-" - doAssert encode("Lloyd-Atkinson") == "Lloyd-Atkinson-" - doAssert encode("This has spaces") == "This has spaces-" - doAssert encode("ü") == "tda" - doAssert encode("München") == "Mnchen-3ya" - doAssert encode("Mnchen-3ya") == "Mnchen-3ya-" - doAssert encode("München-Ost") == "Mnchen-Ost-9db" - doAssert encode("Bahnhof München-Ost") == "Bahnhof Mnchen-Ost-u6b" - block: - doAssert decode("") == "" - doAssert decode("a-") == "a" - doAssert decode("A-") == "A" - doAssert decode("3-") == "3" - doAssert decode("--") == "-" - doAssert decode("---") == "--" - doAssert decode("abc-") == "abc" - doAssert decode("London-") == "London" - doAssert decode("Lloyd-Atkinson-") == "Lloyd-Atkinson" - doAssert decode("This has spaces-") == "This has spaces" - doAssert decode("tda") == "ü" - doAssert decode("Mnchen-3ya") == "München" - doAssert decode("Mnchen-3ya-") == "Mnchen-3ya" - doAssert decode("Mnchen-Ost-9db") == "München-Ost" - doAssert decode("Bahnhof Mnchen-Ost-u6b") == "Bahnhof München-Ost" diff --git a/lib/pure/random.nim b/lib/pure/random.nim index f91d92731..3ec77d37e 100644 --- a/lib/pure/random.nim +++ b/lib/pure/random.nim @@ -66,26 +66,41 @@ runnableExamples: ## See also ## ======== ## * `std/sysrand module <sysrand.html>`_ for a cryptographically secure pseudorandom number generator -## * `mersenne module <mersenne.html>`_ for the Mersenne Twister random number generator ## * `math module <math.html>`_ for basic math routines ## * `stats module <stats.html>`_ for statistical analysis ## * `list of cryptographic and hashing modules <lib.html#pure-libraries-hashing>`_ ## in the standard library import std/[algorithm, math] -import std/private/since +import std/private/[since, jsutils] + +when defined(nimPreviewSlimSystem): + import std/[assertions] include system/inclrtl {.push debugger: off.} +template whenHasBigInt64(yes64, no64): untyped = + when defined(js): + when compiles(compileOption("jsbigint64")): + when compileOption("jsbigint64"): + yes64 + else: + no64 + else: + no64 + else: + yes64 -when defined(js): - type Ui = uint32 - const randMax = 4_294_967_295u32 -else: +whenHasBigInt64: type Ui = uint64 const randMax = 18_446_744_073_709_551_615u64 +do: + type Ui = uint32 + + const randMax = 4_294_967_295u32 + type Rand* = object ## State of a random number generator. @@ -103,15 +118,24 @@ type ## generator are **not** thread-safe! a0, a1: Ui -when defined(js): +whenHasBigInt64: + const DefaultRandSeed = Rand( + a0: 0x69B4C98CB8530805u64, + a1: 0xFED1DD3004688D67CAu64) + + # racy for multi-threading but good enough for now: + var state = DefaultRandSeed # global for backwards compatibility +do: var state = Rand( a0: 0x69B4C98Cu32, a1: 0xFED1DD30u32) # global for backwards compatibility -else: - # racy for multi-threading but good enough for now: - var state = Rand( - a0: 0x69B4C98CB8530805u64, - a1: 0xFED1DD3004688D67CAu64) # global for backwards compatibility + +func isValid(r: Rand): bool {.inline.} = + ## Check whether state of `r` is valid. + ## + ## In `xoroshiro128+`, if all bits of `a0` and `a1` are zero, + ## they are always zero after calling `next(r: var Rand)`. + not (r.a0 == 0 and r.a1 == 0) since (1, 5): template randState*(): untyped = @@ -133,11 +157,10 @@ proc next*(r: var Rand): uint64 = ## that accepts a slice ## * `rand proc<#rand,typedesc[T]>`_ that accepts an integer or range type ## * `skipRandomNumbers proc<#skipRandomNumbers,Rand>`_ - runnableExamples: + runnableExamples("-r:off"): var r = initRand(2019) - doAssert r.next() == 138_744_656_611_299'u64 - doAssert r.next() == 979_810_537_855_049_344'u64 - doAssert r.next() == 3_628_232_584_225_300_704'u64 + assert r.next() == 13223559681708962501'u64 # implementation defined + assert r.next() == 7229677234260823147'u64 # ditto let s0 = r.a0 var s1 = r.a1 @@ -170,29 +193,38 @@ proc skipRandomNumbers*(s: var Rand) = ## **See also:** ## * `next proc<#next,Rand>`_ runnableExamples("--threads:on"): - import std/[random, threadpool] + import std/random - const spawns = 4 const numbers = 100000 - proc randomSum(r: Rand): int = - var r = r + var + thr: array[0..3, Thread[(Rand, int)]] + vals: array[0..3, int] + + proc randomSum(params: tuple[r: Rand, index: int]) {.thread.} = + var r = params.r + var s = 0 # avoid cache thrashing for i in 1..numbers: - result += r.rand(0..10) + s += r.rand(0..10) + vals[params.index] = s var r = initRand(2019) - var vals: array[spawns, FlowVar[int]] - for val in vals.mitems: - val = spawn randomSum(r) + for i in 0..<thr.len: + createThread(thr[i], randomSum, (r, i)) r.skipRandomNumbers() + joinThreads(thr) + for val in vals: - doAssert abs(^val - numbers * 5) / numbers < 0.1 + doAssert abs(val - numbers * 5) / numbers < 0.1 - when defined(js): - const helper = [0xbeac0467u32, 0xd86b048bu32] - else: + doAssert vals == [501737, 497901, 500683, 500157] + + + whenHasBigInt64: const helper = [0xbeac0467eba5facbu64, 0xd86b048b86aa9922u64] + do: + const helper = [0xbeac0467u32, 0xd86b048bu32] var s0 = Ui 0 s1 = Ui 0 @@ -205,6 +237,22 @@ proc skipRandomNumbers*(s: var Rand) = s.a0 = s0 s.a1 = s1 +proc rand[T: uint | uint64](r: var Rand; max: T): T = + # xxx export in future work + if max == 0: return + else: + let max = uint64(max) + when T.high.uint64 == uint64.high: + if max == uint64.high: return T(next(r)) + var iters = 0 + while true: + let x = next(r) + # avoid `mod` bias + if x <= randMax - (randMax mod max) or iters > 20: + return T(x mod (max + 1)) + else: + inc iters + proc rand*(r: var Rand; max: Natural): int {.benign.} = ## Returns a random integer in the range `0..max` using the given state. ## @@ -216,15 +264,11 @@ proc rand*(r: var Rand; max: Natural): int {.benign.} = ## * `rand proc<#rand,typedesc[T]>`_ that accepts an integer or range type runnableExamples: var r = initRand(123) - doAssert r.rand(100) == 0 - doAssert r.rand(100) == 96 - doAssert r.rand(100) == 66 - - if max == 0: return - while true: - let x = next(r) - if x <= randMax - (randMax mod Ui(max)): - return int(x mod (uint64(max) + 1u64)) + if false: + assert r.rand(100) == 96 # implementation defined + # bootstrap: can't use `runnableExamples("-r:off")` + cast[int](rand(r, uint64(max))) + # xxx toUnsigned pending https://github.com/nim-lang/Nim/pull/18445 proc rand*(max: int): int {.benign.} = ## Returns a random integer in the range `0..max`. @@ -241,11 +285,9 @@ proc rand*(max: int): int {.benign.} = ## * `rand proc<#rand,HSlice[T: Ordinal or float or float32 or float64,T: Ordinal or float or float32 or float64]>`_ ## that accepts a slice ## * `rand proc<#rand,typedesc[T]>`_ that accepts an integer or range type - runnableExamples: + runnableExamples("-r:off"): randomize(123) - doAssert rand(100) == 0 - doAssert rand(100) == 96 - doAssert rand(100) == 66 + assert [rand(100), rand(100)] == [96, 63] # implementation defined rand(state, max) @@ -265,7 +307,13 @@ proc rand*(r: var Rand; max: range[0.0 .. high(float)]): float {.benign.} = let x = next(r) when defined(js): - result = (float(x) / float(high(uint32))) * max + when compiles(compileOption("jsbigint64")): + when compileOption("jsbigint64"): + result = (float(x) / float(high(uint64))) * max + else: + result = (float(x) / float(high(uint32))) * max + else: + result = (float(x) / float(high(uint32))) * max else: let u = (0x3FFu64 shl 52u64) or (x shr 12u64) result = (cast[float](u) - 1.0) * max @@ -305,15 +353,16 @@ proc rand*[T: Ordinal or SomeFloat](r: var Rand; x: HSlice[T, T]): T = ## * `rand proc<#rand,typedesc[T]>`_ that accepts an integer or range type runnableExamples: var r = initRand(345) - doAssert r.rand(1..6) == 4 - doAssert r.rand(1..6) == 4 - doAssert r.rand(1..6) == 6 - let f = r.rand(-1.0 .. 1.0) # 0.8741183448756229 + assert r.rand(1..5) <= 5 + assert r.rand(-1.1 .. 1.2) >= -1.1 assert x.a <= x.b when T is SomeFloat: result = rand(r, x.b - x.a) + x.a else: # Integers and Enum types - result = T(rand(r, int(x.b) - int(x.a)) + int(x.a)) + whenJsNoBigInt64: + result = cast[T](rand(r, cast[uint](x.b) - cast[uint](x.a)) + cast[uint](x.a)) + do: + result = cast[T](rand(r, cast[uint64](x.b) - cast[uint64](x.a)) + cast[uint64](x.a)) proc rand*[T: Ordinal or SomeFloat](x: HSlice[T, T]): T = ## For a slice `a..b`, returns a value in the range `a..b`. @@ -333,14 +382,33 @@ proc rand*[T: Ordinal or SomeFloat](x: HSlice[T, T]): T = ## * `rand proc<#rand,typedesc[T]>`_ that accepts an integer or range type runnableExamples: randomize(345) - doAssert rand(1..6) == 4 - doAssert rand(1..6) == 4 - doAssert rand(1..6) == 6 + assert rand(1..6) <= 6 result = rand(state, x) -proc rand*[T: SomeInteger](t: typedesc[T]): T = - ## Returns a random integer in the range `low(T)..high(T)`. +proc rand*[T: Ordinal](r: var Rand; t: typedesc[T]): T {.since: (1, 7, 1).} = + ## Returns a random Ordinal in the range `low(T)..high(T)`. + ## + ## If `randomize <#randomize>`_ has not been called, the sequence of random + ## numbers returned from this proc will always be the same. + ## + ## **See also:** + ## * `rand proc<#rand,int>`_ that returns an integer + ## * `rand proc<#rand,float>`_ that returns a floating point number + ## * `rand proc<#rand,HSlice[T: Ordinal or float or float32 or float64,T: Ordinal or float or float32 or float64]>`_ + ## that accepts a slice + when T is range or T is enum: + result = rand(r, low(T)..high(T)) + elif T is bool: + result = r.next < randMax div 2 + else: + whenJsNoBigInt64: + result = cast[T](r.next shr (sizeof(uint)*8 - sizeof(T)*8)) + do: + result = cast[T](r.next shr (sizeof(uint64)*8 - sizeof(T)*8)) + +proc rand*[T: Ordinal](t: typedesc[T]): T = + ## Returns a random Ordinal in the range `low(T)..high(T)`. ## ## If `randomize <#randomize>`_ has not been called, the sequence of random ## numbers returned from this proc will always be the same. @@ -354,20 +422,15 @@ proc rand*[T: SomeInteger](t: typedesc[T]): T = ## that accepts a slice runnableExamples: randomize(567) - doAssert rand(int8) == 55 - doAssert rand(int8) == -42 - doAssert rand(int8) == 43 - doAssert rand(uint32) == 578980729'u32 - doAssert rand(uint32) == 4052940463'u32 - doAssert rand(uint32) == 2163872389'u32 - doAssert rand(range[1..16]) == 11 - doAssert rand(range[1..16]) == 4 - doAssert rand(range[1..16]) == 16 - - when T is range: - result = rand(state, low(T)..high(T)) - else: - result = cast[T](state.next) + type E = enum a, b, c, d + + assert rand(E) in a..d + assert rand(char) in low(char)..high(char) + assert rand(int8) in low(int8)..high(int8) + assert rand(uint32) in low(uint32)..high(uint32) + assert rand(range[1..16]) in 1..16 + + result = rand(state, t) proc sample*[T](r: var Rand; s: set[T]): T = ## Returns a random element from the set `s` using the given state. @@ -380,9 +443,7 @@ proc sample*[T](r: var Rand; s: set[T]): T = runnableExamples: var r = initRand(987) let s = {1, 3, 5, 7, 9} - doAssert r.sample(s) == 5 - doAssert r.sample(s) == 7 - doAssert r.sample(s) == 1 + assert r.sample(s) in s assert card(s) != 0 var i = rand(r, card(s) - 1) @@ -406,9 +467,7 @@ proc sample*[T](s: set[T]): T = runnableExamples: randomize(987) let s = {1, 3, 5, 7, 9} - doAssert sample(s) == 5 - doAssert sample(s) == 7 - doAssert sample(s) == 1 + assert sample(s) in s sample(state, s) @@ -423,13 +482,11 @@ proc sample*[T](r: var Rand; a: openArray[T]): T = runnableExamples: let marbles = ["red", "blue", "green", "yellow", "purple"] var r = initRand(456) - doAssert r.sample(marbles) == "blue" - doAssert r.sample(marbles) == "yellow" - doAssert r.sample(marbles) == "red" + assert r.sample(marbles) in marbles result = a[r.rand(a.low..a.high)] -proc sample*[T](a: openArray[T]): T = +proc sample*[T](a: openArray[T]): lent T = ## Returns a random element from `a`. ## ## If `randomize <#randomize>`_ has not been called, the order of outcomes @@ -445,9 +502,7 @@ proc sample*[T](a: openArray[T]): T = runnableExamples: let marbles = ["red", "blue", "green", "yellow", "purple"] randomize(456) - doAssert sample(marbles) == "blue" - doAssert sample(marbles) == "yellow" - doAssert sample(marbles) == "red" + assert sample(marbles) in marbles result = a[rand(a.low..a.high)] @@ -476,9 +531,7 @@ proc sample*[T, U](r: var Rand; a: openArray[T]; cdf: openArray[U]): T = let count = [1, 6, 8, 3, 4] let cdf = count.cumsummed var r = initRand(789) - doAssert r.sample(marbles, cdf) == "red" - doAssert r.sample(marbles, cdf) == "green" - doAssert r.sample(marbles, cdf) == "blue" + assert r.sample(marbles, cdf) in marbles assert(cdf.len == a.len) # Two basic sanity checks. assert(float(cdf[^1]) > 0.0) @@ -512,9 +565,7 @@ proc sample*[T, U](a: openArray[T]; cdf: openArray[U]): T = let count = [1, 6, 8, 3, 4] let cdf = count.cumsummed randomize(789) - doAssert sample(marbles, cdf) == "red" - doAssert sample(marbles, cdf) == "green" - doAssert sample(marbles, cdf) == "blue" + assert sample(marbles, cdf) in marbles state.sample(a, cdf) @@ -547,10 +598,10 @@ proc gauss*(mu = 0.0, sigma = 1.0): float {.since: (1, 3).} = proc initRand*(seed: int64): Rand = ## Initializes a new `Rand <#Rand>`_ state using the given seed. ## - ## `seed` must not be zero. Providing a specific seed will produce - ## the same results for that seed each time. + ## Providing a specific seed will produce the same results for that seed each time. ## - ## The resulting state is independent of the default RNG's state. + ## The resulting state is independent of the default RNG's state. When `seed == 0`, + ## we internally set the seed to an implementation defined non-zero value. ## ## **See also:** ## * `initRand proc<#initRand>`_ that uses the current time @@ -560,20 +611,22 @@ proc initRand*(seed: int64): Rand = from std/times import getTime, toUnix, nanosecond var r1 = initRand(123) - let now = getTime() var r2 = initRand(now.toUnix * 1_000_000_000 + now.nanosecond) - - doAssert seed != 0 # 0 causes `rand(int)` to always return 0 for example. + const seedFallback0 = int32.high # arbitrary + let seed = if seed != 0: seed else: seedFallback0 # because 0 is a fixed point result.a0 = Ui(seed shr 16) result.a1 = Ui(seed and 0xffff) + when not defined(nimLegacyRandomInitRand): + # calling `discard next(result)` (even a few times) would still produce + # skewed numbers for the 1st call to `rand()`. + skipRandomNumbers(result) discard next(result) proc randomize*(seed: int64) {.benign.} = ## Initializes the default random number generator with the given seed. ## - ## `seed` must not be zero. Providing a specific seed will produce - ## the same results for that seed each time. + ## Providing a specific seed will produce the same results for that seed each time. ## ## **See also:** ## * `initRand proc<#initRand,int64>`_ that initializes a Rand state @@ -600,7 +653,8 @@ proc shuffle*[T](r: var Rand; x: var openArray[T]) = var cards = ["Ace", "King", "Queen", "Jack", "Ten"] var r = initRand(678) r.shuffle(cards) - doAssert cards == ["King", "Ace", "Queen", "Ten", "Jack"] + import std/algorithm + assert cards.sorted == @["Ace", "Jack", "King", "Queen", "Ten"] for i in countdown(x.high, 1): let j = r.rand(i) @@ -620,19 +674,33 @@ proc shuffle*[T](x: var openArray[T]) = var cards = ["Ace", "King", "Queen", "Jack", "Ten"] randomize(678) shuffle(cards) - doAssert cards == ["King", "Ace", "Queen", "Ten", "Jack"] + import std/algorithm + assert cards.sorted == @["Ace", "Jack", "King", "Queen", "Ten"] shuffle(state, x) -when not defined(nimscript) and not defined(standalone): - import std/times +when not defined(standalone): + when defined(js): + import std/times + else: + when defined(nimscript): + import std/hashes + else: + import std/[hashes, os, sysrand, monotimes] + + when compileOption("threads"): + import std/locks + var baseSeedLock: Lock + baseSeedLock.initLock + + var baseState: Rand proc initRand(): Rand = - ## Initializes a new Rand state with a seed based on the current time. + ## Initializes a new Rand state. ## ## The resulting state is independent of the default RNG's state. ## - ## **Note:** Does not work for NimScript or the compile-time VM. + ## **Note:** Does not work for the compile-time VM. ## ## See also: ## * `initRand proc<#initRand,int64>`_ that accepts a seed for a new Rand state @@ -642,20 +710,50 @@ when not defined(nimscript) and not defined(standalone): let time = int64(times.epochTime() * 1000) and 0x7fff_ffff result = initRand(time) else: - let now = times.getTime() - result = initRand(convert(Seconds, Nanoseconds, now.toUnix) + now.nanosecond) + proc getRandomState(): Rand = + when defined(nimscript): + result = Rand( + a0: CompileTime.hash.Ui, + a1: CompileDate.hash.Ui) + if not result.isValid: + result = DefaultRandSeed + else: + var urand: array[sizeof(Rand), byte] + + for i in 0 .. 7: + if sysrand.urandom(urand): + copyMem(result.addr, urand[0].addr, sizeof(Rand)) + if result.isValid: + break + + if not result.isValid: + # Don't try to get alternative random values from other source like time or process/thread id, + # because such code would be never tested and is a liability for security. + quit("Failed to initializes baseState in random module as sysrand.urandom doesn't work.") + + when compileOption("threads"): + baseSeedLock.withLock: + if not baseState.isValid: + baseState = getRandomState() + result = baseState + baseState.skipRandomNumbers + else: + if not baseState.isValid: + baseState = getRandomState() + result = baseState + baseState.skipRandomNumbers since (1, 5, 1): export initRand proc randomize*() {.benign.} = ## Initializes the default random number generator with a seed based on - ## the current time. + ## random number source. ## ## This proc only needs to be called once, and it should be called before ## the first usage of procs from this module that use the default RNG. ## - ## **Note:** Does not work for NimScript or the compile-time VM. + ## **Note:** Does not work for the compile-time VM. ## ## **See also:** ## * `randomize proc<#randomize,int64>`_ that accepts a seed diff --git a/lib/pure/rationals.nim b/lib/pure/rationals.nim index a059651bb..5f806bd70 100644 --- a/lib/pure/rationals.nim +++ b/lib/pure/rationals.nim @@ -22,6 +22,8 @@ runnableExamples: doAssert r1 / r2 == -2 // 3 import std/[math, hashes] +when defined(nimPreviewSlimSystem): + import std/assertions type Rational*[T] = object ## A rational number, consisting of a numerator `num` and a denominator `den`. @@ -38,16 +40,16 @@ func reduce*[T: SomeInteger](x: var Rational[T]) = reduce(r) doAssert r.num == 1 doAssert r.den == 2 - + if x.den == 0: + raise newException(DivByZeroDefect, "division by zero") let common = gcd(x.num, x.den) if x.den > 0: x.num = x.num div common x.den = x.den div common - elif x.den < 0: - x.num = -x.num div common - x.den = -x.den div common - else: - raise newException(DivByZeroDefect, "division by zero") + when T isnot SomeUnsignedInt: + if x.den < 0: + x.num = -x.num div common + x.den = -x.den div common func initRational*[T: SomeInteger](num, den: T): Rational[T] = ## Creates a new rational number with numerator `num` and denominator `den`. @@ -316,3 +318,23 @@ func hash*[T](x: Rational[T]): Hash = h = h !& hash(copy.num) h = h !& hash(copy.den) result = !$h + +func `^`*[T: SomeInteger](x: Rational[T], y: T): Rational[T] = + ## Computes `x` to the power of `y`. + ## + ## The exponent `y` must be an integer. Negative exponents are supported + ## but floating point exponents are not. + runnableExamples: + doAssert (-3 // 5) ^ 0 == (1 // 1) + doAssert (-3 // 5) ^ 1 == (-3 // 5) + doAssert (-3 // 5) ^ 2 == (9 // 25) + doAssert (-3 // 5) ^ -2 == (25 // 9) + + if y >= 0: + result.num = x.num ^ y + result.den = x.den ^ y + else: + result.num = x.den ^ -y + result.den = x.num ^ -y + # Note that all powers of reduced rationals are already reduced, + # so we don't need to call reduce() here diff --git a/lib/pure/reservedmem.nim b/lib/pure/reservedmem.nim index 232a2b383..ffa0128dc 100644 --- a/lib/pure/reservedmem.nim +++ b/lib/pure/reservedmem.nim @@ -9,7 +9,7 @@ ## :Authors: Zahary Karadjov ## -## This module provides utilities for reserving a portions of the +## This module provides utilities for reserving portions of the ## address space of a program without consuming physical memory. ## It can be used to implement a dynamically resizable buffer that ## is guaranteed to remain in the same memory location. The buffer @@ -18,7 +18,10 @@ ## ## Unstable API. -from os import raiseOSError, osLastError +from std/oserrors import raiseOSError, osLastError + +when defined(nimPreviewSlimSystem): + import std/assertions template distance*(lhs, rhs: pointer): int = cast[int](rhs) - cast[int](lhs) @@ -41,26 +44,11 @@ type mem: ReservedMem when defined(windows): - import winlean - - type - SYSTEM_INFO {.final, pure.} = object - u1: uint32 - dwPageSize: uint32 - lpMinimumApplicationAddress: pointer - lpMaximumApplicationAddress: pointer - dwActiveProcessorMask: ptr uint32 - dwNumberOfProcessors: uint32 - dwProcessorType: uint32 - dwAllocationGranularity: uint32 - wProcessorLevel: uint16 - wProcessorRevision: uint16 - - proc getSystemInfo(lpSystemInfo: ptr SYSTEM_INFO) {.stdcall, - dynlib: "kernel32", importc: "GetSystemInfo".} + import std/winlean + import std/private/win_getsysteminfo proc getAllocationGranularity: uint = - var sysInfo: SYSTEM_INFO + var sysInfo: SystemInfo getSystemInfo(addr sysInfo) return uint(sysInfo.dwAllocationGranularity) @@ -80,7 +68,7 @@ when defined(windows): raiseOSError(osLastError()) else: - import posix + import std/posix let allocationGranularity = sysconf(SC_PAGESIZE) diff --git a/lib/pure/ropes.nim b/lib/pure/ropes.nim index 42550af1d..8750aca87 100644 --- a/lib/pure/ropes.nim +++ b/lib/pure/ropes.nim @@ -19,6 +19,9 @@ include system/inclrtl import std/streams +when defined(nimPreviewSlimSystem): + import std/[syncio, formatfloat, assertions] + {.push debugger: off.} # the user does not want to trace a part # of the standard library! diff --git a/lib/pure/segfaults.nim b/lib/pure/segfaults.nim index e0da8b81d..65b059e86 100644 --- a/lib/pure/segfaults.nim +++ b/lib/pure/segfaults.nim @@ -26,11 +26,11 @@ se.msg = "Could not access value because it is nil." when defined(windows): include "../system/ansi_c" - import winlean + import std/winlean const EXCEPTION_ACCESS_VIOLATION = DWORD(0xc0000005'i32) - EXCEPTION_CONTINUE_SEARCH = Long(0) + EXCEPTION_CONTINUE_SEARCH = LONG(0) type PEXCEPTION_RECORD = ptr object @@ -65,7 +65,7 @@ when defined(windows): c_signal(SIGSEGV, segfaultHandler) else: - import posix + import std/posix var sa: Sigaction diff --git a/lib/pure/selectors.nim b/lib/pure/selectors.nim index e78219aec..ac180e2bd 100644 --- a/lib/pure/selectors.nim +++ b/lib/pure/selectors.nim @@ -27,13 +27,17 @@ ## ## TODO: `/dev/poll`, `event ports` and filesystem events. -import os, nativesockets +import std/nativesockets +import std/oserrors + +when defined(nimPreviewSlimSystem): + import std/assertions const hasThreadSupport = compileOption("threads") and defined(threadsafe) const ioselSupportedPlatform* = defined(macosx) or defined(freebsd) or defined(netbsd) or defined(openbsd) or - defined(dragonfly) or + defined(dragonfly) or defined(nuttx) or (defined(linux) and not defined(android) and not defined(emscripten)) ## This constant is used to determine whether the destination platform is ## fully supported by `ioselectors` module. @@ -201,12 +205,11 @@ when defined(nimdoc): ## to `value`. This `value` can be modified in the scope of ## the `withData` call. ## - ## .. code-block:: nim - ## + ## ```nim ## s.withData(fd, value) do: ## # block is executed only if `fd` registered in selector `s` ## value.uid = 1000 - ## + ## ``` template withData*[T](s: Selector[T], fd: SocketHandle|int, value, body1, body2: untyped) = @@ -214,15 +217,14 @@ when defined(nimdoc): ## to `value`. This `value` can be modified in the scope of ## the `withData` call. ## - ## .. code-block:: nim - ## + ## ```nim ## s.withData(fd, value) do: ## # block is executed only if `fd` registered in selector `s`. ## value.uid = 1000 ## do: ## # block is executed if `fd` not registered in selector `s`. ## raise - ## + ## ``` proc contains*[T](s: Selector[T], fd: SocketHandle|int): bool {.inline.} = ## Determines whether selector contains a file descriptor. @@ -233,9 +235,9 @@ when defined(nimdoc): ## For *poll* and *select* selectors `-1` is returned. else: - import strutils + import std/strutils when hasThreadSupport: - import locks + import std/locks type SharedArray[T] = UncheckedArray[T] @@ -243,8 +245,8 @@ else: proc allocSharedArray[T](nsize: int): ptr SharedArray[T] = result = cast[ptr SharedArray[T]](allocShared0(sizeof(T) * nsize)) - proc reallocSharedArray[T](sa: ptr SharedArray[T], nsize: int): ptr SharedArray[T] = - result = cast[ptr SharedArray[T]](reallocShared(sa, sizeof(T) * nsize)) + proc reallocSharedArray[T](sa: ptr SharedArray[T], oldsize, nsize: int): ptr SharedArray[T] = + result = cast[ptr SharedArray[T]](reallocShared0(sa, oldsize * sizeof(T), sizeof(T) * nsize)) proc deallocSharedArray[T](sa: ptr SharedArray[T]) = deallocShared(cast[pointer](sa)) @@ -286,7 +288,7 @@ else: setBlocking(fd.SocketHandle, false) when not defined(windows): - import posix + import std/posix template setKey(s, pident, pevents, pparam, pdata: untyped) = var skey = addr(s.fds[pident]) @@ -321,9 +323,37 @@ else: proc verifySelectParams(timeout: int) = # Timeout of -1 means: wait forever # Anything higher is the time to wait in milliseconds. - doAssert(timeout >= -1, "Cannot select with a negative value, got " & $timeout) - - when defined(linux) and not defined(emscripten): + doAssert(timeout >= -1, "Cannot select with a negative value, got: " & $timeout) + + when defined(linux) or defined(windows) or defined(macosx) or defined(bsd) or + defined(solaris) or defined(zephyr) or defined(freertos) or defined(nuttx) or defined(haiku): + template maxDescriptors*(): int = + ## Returns the maximum number of active file descriptors for the current + ## process. This involves a system call. For now `maxDescriptors` is + ## supported on the following OSes: Windows, Linux, OSX, BSD, Solaris. + when defined(windows): + 16_700_000 + elif defined(zephyr) or defined(freertos): + FD_MAX + else: + var fdLim: RLimit + var res = int(getrlimit(RLIMIT_NOFILE, fdLim)) + if res >= 0: + res = int(fdLim.rlim_cur) - 1 + res + + when defined(nimIoselector): + when nimIoselector == "epoll": + include ioselects/ioselectors_epoll + elif nimIoselector == "kqueue": + include ioselects/ioselectors_kqueue + elif nimIoselector == "poll": + include ioselects/ioselectors_poll + elif nimIoselector == "select": + include ioselects/ioselectors_select + else: + {.fatal: "Unknown nimIoselector specified by define.".} + elif defined(linux) and not defined(emscripten): include ioselects/ioselectors_epoll elif bsdPlatform: include ioselects/ioselectors_kqueue @@ -337,5 +367,9 @@ else: include ioselects/ioselectors_select elif defined(freertos) or defined(lwip): include ioselects/ioselectors_select + elif defined(zephyr): + include ioselects/ioselectors_poll + elif defined(nuttx): + include ioselects/ioselectors_epoll else: include ioselects/ioselectors_poll diff --git a/lib/pure/smtp.nim b/lib/pure/smtp.nim deleted file mode 100644 index 5ba048608..000000000 --- a/lib/pure/smtp.nim +++ /dev/null @@ -1,343 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2012 Dominik Picheta -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## This module implements the SMTP client protocol as specified by RFC 5321, -## this can be used to send mail to any SMTP Server. -## -## This module also implements the protocol used to format messages, -## as specified by RFC 2822. -## -## Example gmail use: -## -## -## .. code-block:: Nim -## var msg = createMessage("Hello from Nim's SMTP", -## "Hello!.\n Is this awesome or what?", -## @["foo@gmail.com"]) -## let smtpConn = newSmtp(useSsl = true, debug=true) -## smtpConn.connect("smtp.gmail.com", Port 465) -## smtpConn.auth("username", "password") -## smtpConn.sendmail("username@gmail.com", @["foo@gmail.com"], $msg) -## -## -## Example for startTls use: -## -## -## .. code-block:: Nim -## var msg = createMessage("Hello from Nim's SMTP", -## "Hello!.\n Is this awesome or what?", -## @["foo@gmail.com"]) -## let smtpConn = newSmtp(debug=true) -## smtpConn.connect("smtp.mailtrap.io", Port 2525) -## smtpConn.startTls() -## smtpConn.auth("username", "password") -## smtpConn.sendmail("username@gmail.com", @["foo@gmail.com"], $msg) -## -## -## For SSL support this module relies on OpenSSL. If you want to -## enable SSL, compile with `-d:ssl`. - -import net, strutils, strtabs, base64, os, strutils -import asyncnet, asyncdispatch - -export Port - -type - Message* = object - msgTo: seq[string] - msgCc: seq[string] - msgSubject: string - msgOtherHeaders: StringTableRef - msgBody: string - - ReplyError* = object of IOError - - SmtpBase[SocketType] = ref object - sock: SocketType - address: string - debug: bool - - Smtp* = SmtpBase[Socket] - AsyncSmtp* = SmtpBase[AsyncSocket] - -proc containsNewline(xs: seq[string]): bool = - for x in xs: - if x.contains({'\c', '\L'}): - return true - -proc debugSend*(smtp: Smtp | AsyncSmtp, cmd: string) {.multisync.} = - ## Sends `cmd` on the socket connected to the SMTP server. - ## - ## If the `smtp` object was created with `debug` enabled, - ## debugSend will invoke `echo("C:" & cmd)` before sending. - ## - ## This is a lower level proc and not something that you typically - ## would need to call when using this module. One exception to - ## this is if you are implementing any - ## `SMTP extensions<https://en.wikipedia.org/wiki/Extended_SMTP>`_. - - if smtp.debug: - echo("C:" & cmd) - await smtp.sock.send(cmd) - -proc debugRecv*(smtp: Smtp | AsyncSmtp): Future[string] {.multisync.} = - ## Receives a line of data from the socket connected to the - ## SMTP server. - ## - ## If the `smtp` object was created with `debug` enabled, - ## debugRecv will invoke `echo("S:" & result.string)` after - ## the data is received. - ## - ## This is a lower level proc and not something that you typically - ## would need to call when using this module. One exception to - ## this is if you are implementing any - ## `SMTP extensions<https://en.wikipedia.org/wiki/Extended_SMTP>`_. - ## - ## See `checkReply(reply)<#checkReply,AsyncSmtp,string>`_. - - result = await smtp.sock.recvLine() - if smtp.debug: - echo("S:" & result) - -proc quitExcpt(smtp: Smtp, msg: string) = - smtp.debugSend("QUIT") - raise newException(ReplyError, msg) - -const compiledWithSsl = defined(ssl) - -when not defined(ssl): - let defaultSSLContext: SslContext = nil -else: - var defaultSSLContext {.threadvar.}: SslContext - - proc getSSLContext(): SslContext = - if defaultSSLContext == nil: - defaultSSLContext = newContext(verifyMode = CVerifyNone) - result = defaultSSLContext - -proc createMessage*(mSubject, mBody: string, mTo, mCc: seq[string], - otherHeaders: openArray[tuple[name, value: string]]): Message = - ## Creates a new MIME compliant message. - ## - ## You need to make sure that `mSubject`, `mTo` and `mCc` don't contain - ## any newline characters. Failing to do so will raise `AssertionDefect`. - doAssert(not mSubject.contains({'\c', '\L'}), - "'mSubject' shouldn't contain any newline characters") - doAssert(not (mTo.containsNewline() or mCc.containsNewline()), - "'mTo' and 'mCc' shouldn't contain any newline characters") - - result.msgTo = mTo - result.msgCc = mCc - result.msgSubject = mSubject - result.msgBody = mBody - result.msgOtherHeaders = newStringTable() - for n, v in items(otherHeaders): - result.msgOtherHeaders[n] = v - -proc createMessage*(mSubject, mBody: string, mTo, - mCc: seq[string] = @[]): Message = - ## Alternate version of the above. - ## - ## You need to make sure that `mSubject`, `mTo` and `mCc` don't contain - ## any newline characters. Failing to do so will raise `AssertionDefect`. - doAssert(not mSubject.contains({'\c', '\L'}), - "'mSubject' shouldn't contain any newline characters") - doAssert(not (mTo.containsNewline() or mCc.containsNewline()), - "'mTo' and 'mCc' shouldn't contain any newline characters") - result.msgTo = mTo - result.msgCc = mCc - result.msgSubject = mSubject - result.msgBody = mBody - result.msgOtherHeaders = newStringTable() - -proc `$`*(msg: Message): string = - ## stringify for `Message`. - result = "" - if msg.msgTo.len() > 0: - result = "TO: " & msg.msgTo.join(", ") & "\c\L" - if msg.msgCc.len() > 0: - result.add("CC: " & msg.msgCc.join(", ") & "\c\L") - # TODO: Folding? i.e when a line is too long, shorten it... - result.add("Subject: " & msg.msgSubject & "\c\L") - for key, value in pairs(msg.msgOtherHeaders): - result.add(key & ": " & value & "\c\L") - - result.add("\c\L") - result.add(msg.msgBody) - -proc newSmtp*(useSsl = false, debug = false, - sslContext: SslContext = nil): Smtp = - ## Creates a new `Smtp` instance. - new result - result.debug = debug - result.sock = newSocket() - if useSsl: - when compiledWithSsl: - if sslContext == nil: - getSSLContext().wrapSocket(result.sock) - else: - sslContext.wrapSocket(result.sock) - else: - {.error: "SMTP module compiled without SSL support".} - -proc newAsyncSmtp*(useSsl = false, debug = false, - sslContext: SslContext = nil): AsyncSmtp = - ## Creates a new `AsyncSmtp` instance. - new result - result.debug = debug - - result.sock = newAsyncSocket() - if useSsl: - when compiledWithSsl: - if sslContext == nil: - getSSLContext().wrapSocket(result.sock) - else: - sslContext.wrapSocket(result.sock) - else: - {.error: "SMTP module compiled without SSL support".} - -proc quitExcpt(smtp: AsyncSmtp, msg: string): Future[void] = - var retFuture = newFuture[void]() - var sendFut = smtp.debugSend("QUIT") - sendFut.callback = - proc () = - retFuture.fail(newException(ReplyError, msg)) - return retFuture - -proc checkReply*(smtp: Smtp | AsyncSmtp, reply: string) {.multisync.} = - ## Calls `debugRecv<#debugRecv,AsyncSmtp>`_ and checks that the received - ## data starts with `reply`. If the received data does not start - ## with `reply`, then a `QUIT` command will be sent to the SMTP - ## server and a `ReplyError` exception will be raised. - ## - ## This is a lower level proc and not something that you typically - ## would need to call when using this module. One exception to - ## this is if you are implementing any - ## `SMTP extensions<https://en.wikipedia.org/wiki/Extended_SMTP>`_. - - var line = await smtp.debugRecv() - if not line.startsWith(reply): - await quitExcpt(smtp, "Expected " & reply & " reply, got: " & line) - -proc helo*(smtp: Smtp | AsyncSmtp) {.multisync.} = - # Sends the HELO request - await smtp.debugSend("HELO " & smtp.address & "\c\L") - await smtp.checkReply("250") - -proc connect*(smtp: Smtp | AsyncSmtp, - address: string, port: Port) {.multisync.} = - ## Establishes a connection with a SMTP server. - ## May fail with ReplyError or with a socket error. - smtp.address = address - await smtp.sock.connect(address, port) - await smtp.checkReply("220") - await smtp.helo() - -proc startTls*(smtp: Smtp | AsyncSmtp, sslContext: SslContext = nil) {.multisync.} = - ## Put the SMTP connection in TLS (Transport Layer Security) mode. - ## May fail with ReplyError - await smtp.debugSend("STARTTLS\c\L") - await smtp.checkReply("220") - when compiledWithSsl: - if sslContext == nil: - getSSLContext().wrapConnectedSocket(smtp.sock, handshakeAsClient) - else: - sslContext.wrapConnectedSocket(smtp.sock, handshakeAsClient) - await smtp.helo() - else: - {.error: "SMTP module compiled without SSL support".} - -proc auth*(smtp: Smtp | AsyncSmtp, username, password: string) {.multisync.} = - ## Sends an AUTH command to the server to login as the `username` - ## using `password`. - ## May fail with ReplyError. - - await smtp.debugSend("AUTH LOGIN\c\L") - await smtp.checkReply("334") # TODO: Check whether it's asking for the "Username:" - # i.e "334 VXNlcm5hbWU6" - await smtp.debugSend(encode(username) & "\c\L") - await smtp.checkReply("334") # TODO: Same as above, only "Password:" (I think?) - - await smtp.debugSend(encode(password) & "\c\L") - await smtp.checkReply("235") # Check whether the authentication was successful. - -proc sendMail*(smtp: Smtp | AsyncSmtp, fromAddr: string, - toAddrs: seq[string], msg: string) {.multisync.} = - ## Sends `msg` from `fromAddr` to the addresses specified in `toAddrs`. - ## Messages may be formed using `createMessage` by converting the - ## Message into a string. - ## - ## You need to make sure that `fromAddr` and `toAddrs` don't contain - ## any newline characters. Failing to do so will raise `AssertionDefect`. - doAssert(not (toAddrs.containsNewline() or fromAddr.contains({'\c', '\L'})), - "'toAddrs' and 'fromAddr' shouldn't contain any newline characters") - - await smtp.debugSend("MAIL FROM:<" & fromAddr & ">\c\L") - await smtp.checkReply("250") - for address in items(toAddrs): - await smtp.debugSend("RCPT TO:<" & address & ">\c\L") - await smtp.checkReply("250") - - # Send the message - await smtp.debugSend("DATA " & "\c\L") - await smtp.checkReply("354") - await smtp.sock.send(msg & "\c\L") - await smtp.debugSend(".\c\L") - await smtp.checkReply("250") - -proc close*(smtp: Smtp | AsyncSmtp) {.multisync.} = - ## Disconnects from the SMTP server and closes the socket. - await smtp.debugSend("QUIT\c\L") - smtp.sock.close() - -when not defined(testing) and isMainModule: - # To test with a real SMTP service, create a smtp.ini file, e.g.: - # username = "" - # password = "" - # smtphost = "smtp.gmail.com" - # port = 465 - # use_tls = true - # sender = "" - # recipient = "" - - import parsecfg - - proc `[]`(c: Config, key: string): string = c.getSectionValue("", key) - - let - conf = loadConfig("smtp.ini") - msg = createMessage("Hello from Nim's SMTP!", - "Hello!\n Is this awesome or what?", @[conf["recipient"]]) - - assert conf["smtphost"] != "" - - proc async_test() {.async.} = - let client = newAsyncSmtp( - conf["use_tls"].parseBool, - debug = true - ) - await client.connect(conf["smtphost"], conf["port"].parseInt.Port) - await client.auth(conf["username"], conf["password"]) - await client.sendMail(conf["sender"], @[conf["recipient"]], $msg) - await client.close() - echo "async email sent" - - proc sync_test() = - var smtpConn = newSmtp( - conf["use_tls"].parseBool, - debug = true - ) - smtpConn.connect(conf["smtphost"], conf["port"].parseInt.Port) - smtpConn.auth(conf["username"], conf["password"]) - smtpConn.sendMail(conf["sender"], @[conf["recipient"]], $msg) - smtpConn.close() - echo "sync email sent" - - waitFor async_test() - sync_test() diff --git a/lib/pure/smtp.nim.cfg b/lib/pure/smtp.nim.cfg deleted file mode 100644 index 521e21de4..000000000 --- a/lib/pure/smtp.nim.cfg +++ /dev/null @@ -1 +0,0 @@ --d:ssl diff --git a/lib/pure/ssl_certs.nim b/lib/pure/ssl_certs.nim index 2d2644ebe..d60cd22eb 100644 --- a/lib/pure/ssl_certs.nim +++ b/lib/pure/ssl_certs.nim @@ -10,7 +10,7 @@ ## The default locations can be overridden using the SSL_CERT_FILE and ## SSL_CERT_DIR environment variables. -import os, strutils +import std/[os, strutils] # FWIW look for files before scanning entire dirs. @@ -34,6 +34,7 @@ elif defined(linux): # Fedora/RHEL "/etc/pki/tls/certs", # Android + "/data/data/com.termux/files/usr/etc/tls/cert.pem", "/system/etc/security/cacerts", ] elif defined(bsd): @@ -87,7 +88,7 @@ when defined(haiku): proc find_paths_etc(architecture: cstring, baseDirectory: cint, subPath: cstring, flags: uint32, paths: var ptr UncheckedArray[cstring], - pathCount: var csize): int32 + pathCount: var csize_t): int32 {.importc, header: "<FindDirectory.h>".} proc free(p: pointer) {.importc, header: "<stdlib.h>".} @@ -125,12 +126,18 @@ iterator scanSSLCertificates*(useEnvVars = false): string = if fileExists(p): yield p elif dirExists(p): + # check if it's a dir where each cert is one file + # named by it's hasg + for fn in joinPath(p, "*.0").walkFiles: + yield p.normalizePathEnd(true) + break for fn in joinPath(p, "*").walkFiles(): + yield fn else: var paths: ptr UncheckedArray[cstring] - size: csize + size: csize_t let err = find_paths_etc( nil, B_FIND_PATH_DATA_DIRECTORY, "ssl/CARootCertificates.pem", B_FIND_PATH_EXISTING_ONLY, paths, size @@ -143,7 +150,7 @@ iterator scanSSLCertificates*(useEnvVars = false): string = # Certificates management on windows # when defined(windows) or defined(nimdoc): # -# import openssl +# import std/openssl # # type # PCCertContext {.final, pure.} = pointer diff --git a/lib/pure/stats.nim b/lib/pure/stats.nim index 6e2d5fecd..6a4fd8f01 100644 --- a/lib/pure/stats.nim +++ b/lib/pure/stats.nim @@ -42,7 +42,7 @@ runnableExamples: template `~=`(a, b: float): bool = almostEqual(a, b) - var statistics: RunningStat ## Must be var + var statistics: RunningStat # must be var statistics.push(@[1.0, 2.0, 1.0, 4.0, 1.0, 4.0, 1.0, 2.0]) doAssert statistics.n == 8 doAssert statistics.mean() ~= 2.0 @@ -55,6 +55,9 @@ runnableExamples: from std/math import FloatClass, sqrt, pow, round +when defined(nimPreviewSlimSystem): + import std/[assertions, formatfloat] + {.push debugger: off.} # the user does not want to trace a part # of the standard library! {.push checks: off, line_dir: off, stack_trace: off.} @@ -76,7 +79,7 @@ type proc clear*(s: var RunningStat) = ## Resets `s`. s.n = 0 - s.min = toBiggestFloat(int.high) + s.min = 0.0 s.max = 0.0 s.sum = 0.0 s.mom1 = 0.0 @@ -86,11 +89,14 @@ proc clear*(s: var RunningStat) = proc push*(s: var RunningStat, x: float) = ## Pushes a value `x` for processing. - if s.n == 0: s.min = x + if s.n == 0: + s.min = x + s.max = x + else: + if s.min > x: s.min = x + if s.max < x: s.max = x inc(s.n) # See Knuth TAOCP vol 2, 3rd edition, page 232 - if s.min > x: s.min = x - if s.max < x: s.max = x s.sum += x let n = toFloat(s.n) let delta = x - s.mom1 diff --git a/lib/pure/streams.nim b/lib/pure/streams.nim index eb1c9cc14..56f49d7b1 100644 --- a/lib/pure/streams.nim +++ b/lib/pure/streams.nim @@ -15,6 +15,11 @@ ## Other modules may provide other implementations for this standard ## stream interface. ## +## .. warning:: Due to the use of `pointer`, the `readData`, `peekData` and +## `writeData` interfaces are not available on the compile-time VM, and must +## be cast from a `ptr string` on the JS backend. However, `readDataStr` is +## available generally in place of `readData`. +## ## Basic usage ## =========== ## @@ -27,75 +32,79 @@ ## StringStream example ## -------------------- ## -## .. code-block:: Nim -## -## import std/streams +## ```Nim +## import std/streams ## -## var strm = newStringStream("""The first line -## the second line -## the third line""") +## var strm = newStringStream("""The first line +## the second line +## the third line""") ## -## var line = "" +## var line = "" ## -## while strm.readLine(line): -## echo line +## while strm.readLine(line): +## echo line ## -## # Output: -## # The first line -## # the second line -## # the third line +## # Output: +## # The first line +## # the second line +## # the third line ## -## strm.close() +## strm.close() +## ``` ## ## FileStream example ## ------------------ ## ## Write file stream example: ## -## .. code-block:: Nim -## -## import std/streams +## ```Nim +## import std/streams ## -## var strm = newFileStream("somefile.txt", fmWrite) -## var line = "" +## var strm = newFileStream("somefile.txt", fmWrite) +## var line = "" ## -## if not isNil(strm): -## strm.writeLine("The first line") -## strm.writeLine("the second line") -## strm.writeLine("the third line") -## strm.close() +## if not isNil(strm): +## strm.writeLine("The first line") +## strm.writeLine("the second line") +## strm.writeLine("the third line") +## strm.close() ## -## # Output (somefile.txt): -## # The first line -## # the second line -## # the third line +## # Output (somefile.txt): +## # The first line +## # the second line +## # the third line +## ``` ## ## Read file stream example: ## -## .. code-block:: Nim +## ```Nim +## import std/streams ## -## import std/streams +## var strm = newFileStream("somefile.txt", fmRead) +## var line = "" ## -## var strm = newFileStream("somefile.txt", fmRead) -## var line = "" +## if not isNil(strm): +## while strm.readLine(line): +## echo line +## strm.close() ## -## if not isNil(strm): -## while strm.readLine(line): -## echo line -## strm.close() -## -## # Output: -## # The first line -## # the second line -## # the third line +## # Output: +## # The first line +## # the second line +## # the third line +## ``` ## ## See also ## ======== ## * `asyncstreams module <asyncstreams.html>`_ -## * `io module <io.html>`_ for `FileMode enum <io.html#FileMode>`_ +## * `io module <syncio.html>`_ for `FileMode enum <syncio.html#FileMode>`_ import std/private/since +when defined(nimPreviewSlimSystem): + import std/syncio + export FileMode + proc newEIO(msg: string): owned(ref IOError) = new(result) result.msg = msg @@ -111,7 +120,7 @@ type ## * That these fields here shouldn't be used directly. ## They are accessible so that a stream implementation can override them. closeImpl*: proc (s: Stream) - {.nimcall, raises: [Exception, IOError, OSError], tags: [WriteIOEffect], gcsafe.} + {.nimcall, raises: [IOError, OSError], tags: [WriteIOEffect], gcsafe.} atEndImpl*: proc (s: Stream): bool {.nimcall, raises: [Defect, IOError, OSError], tags: [], gcsafe.} setPositionImpl*: proc (s: Stream, pos: int) @@ -170,10 +179,20 @@ proc close*(s: Stream) = ## See also: ## * `flush proc <#flush,Stream>`_ runnableExamples: - var strm = newStringStream("The first line\nthe second line\nthe third line") - ## do something... - strm.close() - if not isNil(s.closeImpl): s.closeImpl(s) + block: + let strm = newStringStream("The first line\nthe second line\nthe third line") + ## do something... + strm.close() + + block: + let strm = newFileStream("amissingfile.txt") + # deferring works even if newFileStream fails + defer: strm.close() + if not isNil(strm): + ## do something... + + if not isNil(s) and not isNil(s.closeImpl): + s.closeImpl(s) proc atEnd*(s: Stream): bool = ## Checks if more data can be read from `s`. Returns ``true`` if all data has @@ -240,6 +259,9 @@ proc readDataStr*(s: Stream, buffer: var string, slice: Slice[int]): int = result = s.readDataStrImpl(s, buffer, slice) else: # fallback + when declared(prepareMutation): + # buffer might potentially be a CoW literal with ARC + prepareMutation(buffer) result = s.readData(addr buffer[slice.a], slice.b + 1 - slice.a) template jsOrVmBlock(caseJsOrVm, caseElse: untyped): untyped = @@ -331,9 +353,9 @@ proc write*[T](s: Stream, x: T) = ## **Note:** Not available for JS backend. Use `write(Stream, string) ## <#write,Stream,string>`_ for now. ## - ## .. code-block:: Nim - ## - ## s.writeData(s, unsafeAddr(x), sizeof(x)) + ## ```Nim + ## s.writeData(s, unsafeAddr(x), sizeof(x)) + ## ``` runnableExamples: var strm = newStringStream("") strm.write("abcde") @@ -921,10 +943,14 @@ proc peekFloat64*(s: Stream): float64 = proc readStrPrivate(s: Stream, length: int, str: var string) = if length > len(str): setLen(str, length) - when defined(js): - let L = readData(s, addr(str), length) + var L: int + when nimvm: + L = readDataStr(s, str, 0..length-1) else: - let L = readData(s, cstring(str), length) + when defined(js): + L = readData(s, addr(str), length) + else: + L = readData(s, cstring(str), length) if L != len(str): setLen(str, L) proc readStr*(s: Stream, length: int, str: var string) {.since: (1, 3).} = @@ -1191,6 +1217,11 @@ else: # after 1.3 or JS not defined proc ssReadDataStr(s: Stream, buffer: var string, slice: Slice[int]): int = var s = StringStream(s) + when nimvm: + discard + else: + when declared(prepareMutation): + prepareMutation(buffer) # buffer might potentially be a CoW literal with ARC result = min(slice.b + 1 - slice.a, s.data.len - s.pos) if result > 0: jsOrVmBlock: @@ -1252,7 +1283,7 @@ else: # after 1.3 or JS not defined var s = StringStream(s) s.data = "" - proc newStringStream*(s: string = ""): owned StringStream = + proc newStringStream*(s: sink string = ""): owned StringStream = ## Creates a new stream from the string `s`. ## ## See also: @@ -1271,6 +1302,11 @@ else: # after 1.3 or JS not defined new(result) result.data = s + when nimvm: + discard + else: + when declared(prepareMutation): + prepareMutation(result.data) # Allows us to mutate using `addr` logic like `copyMem`, otherwise it errors. result.pos = 0 result.closeImpl = ssClose result.atEndImpl = ssAtEnd @@ -1331,7 +1367,7 @@ proc newFileStream*(f: File): owned FileStream = ## * `newStringStream proc <#newStringStream,string>`_ creates a new stream ## from string. ## * `newFileStream proc <#newFileStream,string,FileMode,int>`_ is the same - ## as using `open proc <io.html#open,File,string,FileMode,int>`_ + ## as using `open proc <syncio.html#open,File,string,FileMode,int>`_ ## on Examples. ## * `openFileStream proc <#openFileStream,string,FileMode,int>`_ creates a ## file stream from the file name and the mode. @@ -1370,7 +1406,7 @@ proc newFileStream*(filename: string, mode: FileMode = fmRead, ## Creates a new stream from the file named `filename` with the mode `mode`. ## ## If the file cannot be opened, `nil` is returned. See the `io module - ## <io.html>`_ for a list of available `FileMode enums <io.html#FileMode>`_. + ## <syncio.html>`_ for a list of available `FileMode enums <syncio.html#FileMode>`_. ## ## **Note:** ## * **This function returns nil in case of failure.** @@ -1455,7 +1491,7 @@ when false: # do not import windows as this increases compile times: discard else: - import posix + import std/posix proc hsSetPosition(s: FileHandleStream, pos: int) = discard lseek(s.handle, pos, SEEK_SET) @@ -1503,6 +1539,7 @@ when false: of fmReadWrite: flags = O_RDWR or int(O_CREAT) of fmReadWriteExisting: flags = O_RDWR of fmAppend: flags = O_WRONLY or int(O_CREAT) or O_APPEND + static: raiseAssert "unreachable" # handle bug #17888 var handle = open(filename, flags) if handle < 0: raise newEOS("posix.open() call failed") result = newFileHandleStream(handle) diff --git a/lib/pure/streamwrapper.nim b/lib/pure/streamwrapper.nim index 7a501760b..99752a9ab 100644 --- a/lib/pure/streamwrapper.nim +++ b/lib/pure/streamwrapper.nim @@ -11,7 +11,11 @@ ## ## **Since** version 1.2. -import deques, streams +import std/[deques, streams] + +when defined(nimPreviewSlimSystem): + import std/assertions + type PipeOutStream*[T] = ref object of T @@ -87,14 +91,14 @@ proc newPipeOutStream*[T](s: sink (ref T)): owned PipeOutStream[T] = ## when setPosition/getPosition is called or write operation is performed. ## ## Example: - ## - ## .. code-block:: Nim + ## ```Nim ## import std/[osproc, streamwrapper] ## var ## p = startProcess(exePath) ## outStream = p.outputStream().newPipeOutStream() ## echo outStream.peekChar ## p.close() + ## ``` assert s.readDataImpl != nil diff --git a/lib/pure/strformat.nim b/lib/pure/strformat.nim index 7ab359038..7d093ebb3 100644 --- a/lib/pure/strformat.nim +++ b/lib/pure/strformat.nim @@ -74,6 +74,20 @@ runnableExamples: assert fmt"{123.456:13e}" == " 1.234560e+02" ##[ +# Expressions +]## +runnableExamples: + let x = 3.14 + assert fmt"{(if x!=0: 1.0/x else: 0):.5}" == "0.31847" + assert fmt"""{(block: + var res: string + for i in 1..15: + res.add (if i mod 15 == 0: "FizzBuzz" + elif i mod 5 == 0: "Buzz" + elif i mod 3 == 0: "Fizz" + else: $i) & " " + res)}""" == "1 2 Fizz 4 Buzz Fizz 7 8 Fizz Buzz 11 Fizz 13 14 FizzBuzz " +##[ # Debugging strings `fmt"{expr=}"` expands to `fmt"expr={expr}"` namely the text of the expression, @@ -119,17 +133,31 @@ runnableExamples: An expression like `&"{key} is {value:arg} {{z}}"` is transformed into: -.. code-block:: nim + ```nim var temp = newStringOfCap(educatedCapGuess) temp.formatValue(key, "") temp.add(" is ") temp.formatValue(value, arg) temp.add(" {z}") temp + ``` Parts of the string that are enclosed in the curly braces are interpreted -as Nim code, to escape a `{` or `}`, double it. +as Nim code. To escape a `{` or `}`, double it. + +Within a curly expression, however, `{`, `}`, must be escaped with a backslash. + +To enable evaluating Nim expressions within curlies, colons inside parentheses +do not need to be escaped. +]## +runnableExamples: + let x = "hello" + assert fmt"""{ "\{(" & x & ")\}" }""" == "{(hello)}" + assert fmt"""{{({ x })}}""" == "{(hello)}" + assert fmt"""{ $(\{x:1,"world":2\}) }""" == """[("hello", 1), ("world", 2)]""" + +##[ `&` delegates most of the work to an open overloaded set of `formatValue` procs. The required signature for a type `T` that supports formatting is usually `proc formatValue(result: var string; x: T; specifier: string)`. @@ -144,34 +172,34 @@ For strings and numeric types the optional argument is a so-called # Standard format specifiers for strings, integers and floats -The general form of a standard format specifier is:: +The general form of a standard format specifier is: - [[fill]align][sign][#][0][minimumwidth][.precision][type] + [[fill]align][sign][#][0][minimumwidth][.precision][type] The square brackets `[]` indicate an optional element. -The optional 'align' flag can be one of the following: +The optional `align` flag can be one of the following: -'<' - Forces the field to be left-aligned within the available +`<` +: Forces the field to be left-aligned within the available space. (This is the default for strings.) -'>' - Forces the field to be right-aligned within the available space. +`>` +: Forces the field to be right-aligned within the available space. (This is the default for numbers.) -'^' - Forces the field to be centered within the available space. +`^` +: Forces the field to be centered within the available space. Note that unless a minimum field width is defined, the field width will always be the same size as the data to fill it, so that the alignment option has no meaning in this case. -The optional 'fill' character defines the character to be used to pad +The optional `fill` character defines the character to be used to pad the field to the minimum width. The fill character, if present, must be followed by an alignment flag. -The 'sign' option is only valid for numeric types, and can be one of the following: +The `sign` option is only valid for numeric types, and can be one of the following: ================= ==================================================== Sign Meaning @@ -184,22 +212,22 @@ The 'sign' option is only valid for numeric types, and can be one of the followi positive numbers. ================= ==================================================== -If the '#' character is present, integers use the 'alternate form' for formatting. +If the `#` character is present, integers use the 'alternate form' for formatting. This means that binary, octal and hexadecimal output will be prefixed -with '0b', '0o' and '0x', respectively. +with `0b`, `0o` and `0x`, respectively. -'width' is a decimal integer defining the minimum field width. If not specified, +`width` is a decimal integer defining the minimum field width. If not specified, then the field width will be determined by the content. -If the width field is preceded by a zero ('0') character, this enables +If the width field is preceded by a zero (`0`) character, this enables zero-padding. -The 'precision' is a decimal number indicating how many digits should be displayed +The `precision` is a decimal number indicating how many digits should be displayed after the decimal point in a floating point conversion. For non-numeric types the field indicates the maximum field size - in other words, how many characters will be used from the field content. The precision is ignored for integer conversions. -Finally, the 'type' determines how the data should be presented. +Finally, the `type` determines how the data should be presented. The available integer presentation types are: @@ -213,7 +241,7 @@ The available integer presentation types are: lower-case letters for the digits above 9. `X` Hex format. Outputs the number in base 16, using uppercase letters for the digits above 9. -(None) The same as 'd'. +(None) The same as `d`. ================= ==================================================== The available floating point presentation types are: @@ -222,21 +250,27 @@ The available floating point presentation types are: Type Result ================= ==================================================== `e` Exponent notation. Prints the number in scientific - notation using the letter 'e' to indicate the + notation using the letter `e` to indicate the exponent. -`E` Exponent notation. Same as 'e' except it converts +`E` Exponent notation. Same as `e` except it converts the number to uppercase. `f` Fixed point. Displays the number as a fixed-point number. -`F` Fixed point. Same as 'f' except it converts the +`F` Fixed point. Same as `f` except it converts the number to uppercase. `g` General format. This prints the number as a fixed-point number, unless the number is too - large, in which case it switches to 'e' + large, in which case it switches to `e` exponent notation. -`G` General format. Same as 'g' except it switches to 'E' +`G` General format. Same as `g` except it switches to `E` if the number gets to large. -(None) Similar to 'g', except that it prints at least one +`i` Complex General format. This is only supported for + complex numbers, which it prints using the mathematical + (RE+IMj) format. The real and imaginary parts are printed + using the general format `g` by default, but it is + possible to combine this format with one of the other + formats (e.g `jf`). +(None) Similar to `g`, except that it prints at least one digit after the decimal point. ================= ==================================================== @@ -245,13 +279,14 @@ The available floating point presentation types are: Because of the well defined order how templates and macros are expanded, strformat cannot expand template arguments: -.. code-block:: nim + ```nim template myTemplate(arg: untyped): untyped = echo "arg is: ", arg echo &"--- {arg} ---" let x = "abc" myTemplate(x) + ``` First the template `myTemplate` is expanded, where every identifier `arg` is substituted with its argument. The `arg` inside the @@ -262,12 +297,13 @@ identifier that cannot be resolved anymore. The workaround for this is to bind the template argument to a new local variable. -.. code-block:: nim + ```nim template myTemplate(arg: untyped): untyped = block: let arg1 {.inject.} = arg echo "arg is: ", arg1 echo &"--- {arg1} ---" + ``` The use of `{.inject.}` here is necessary again because of template expansion order and hygienic templates. But since we generally want to @@ -275,6 +311,7 @@ keep the hygiene of `myTemplate`, and we do not want `arg1` to be injected into the context where `myTemplate` is expanded, everything is wrapped in a `block`. + # Future directions A curly expression with commas in it like `{x, argA, argB}` could be @@ -288,6 +325,10 @@ single letter DSLs. import std/[macros, parseutils, unicode] import std/strutils except format +when defined(nimPreviewSlimSystem): + import std/assertions + + proc mkDigit(v: int, typ: char): string {.inline.} = assert(v < 26) if v < 10: @@ -391,9 +432,9 @@ proc formatInt(n: SomeNumber; radix: int; spec: StandardFormatSpecifier): string proc parseStandardFormatSpecifier*(s: string; start = 0; ignoreUnknownSuffix = false): StandardFormatSpecifier = ## An exported helper proc that parses the "standard format specifiers", - ## as specified by the grammar:: + ## as specified by the grammar: ## - ## [[fill]align][sign][#][0][minimumwidth][.precision][type] + ## [[fill]align][sign][#][0][minimumwidth][.precision][type] ## ## This is only of interest if you want to write a custom `format` proc that ## should support the standard format specifiers. If `ignoreUnknownSuffix` is true, @@ -440,50 +481,48 @@ proc parseStandardFormatSpecifier*(s: string; start = 0; raise newException(ValueError, "invalid format string, cannot parse: " & s[i..^1]) +proc toRadix(typ: char): int = + case typ + of 'x', 'X': 16 + of 'd', '\0': 10 + of 'o': 8 + of 'b': 2 + else: + raise newException(ValueError, + "invalid type in format string for number, expected one " & + " of 'x', 'X', 'b', 'd', 'o' but got: " & typ) + proc formatValue*[T: SomeInteger](result: var string; value: T; - specifier: string) = + specifier: static string) = ## Standard format implementation for `SomeInteger`. It makes little ## sense to call this directly, but it is required to exist ## by the `&` macro. - if specifier.len == 0: + when specifier.len == 0: result.add $value - return - let spec = parseStandardFormatSpecifier(specifier) - var radix = 10 - case spec.typ - of 'x', 'X': radix = 16 - of 'd', '\0': discard - of 'b': radix = 2 - of 'o': radix = 8 else: - raise newException(ValueError, - "invalid type in format string for number, expected one " & - " of 'x', 'X', 'b', 'd', 'o' but got: " & spec.typ) - result.add formatInt(value, radix, spec) + const + spec = parseStandardFormatSpecifier(specifier) + radix = toRadix(spec.typ) -proc formatValue*(result: var string; value: SomeFloat; specifier: string) = - ## Standard format implementation for `SomeFloat`. It makes little + result.add formatInt(value, radix, spec) + +proc formatValue*[T: SomeInteger](result: var string; value: T; + specifier: string) = + ## Standard format implementation for `SomeInteger`. It makes little ## sense to call this directly, but it is required to exist ## by the `&` macro. if specifier.len == 0: result.add $value - return - let spec = parseStandardFormatSpecifier(specifier) - - var fmode = ffDefault - case spec.typ - of 'e', 'E': - fmode = ffScientific - of 'f', 'F': - fmode = ffDecimal - of 'g', 'G': - fmode = ffDefault - of '\0': discard else: - raise newException(ValueError, - "invalid type in format string for number, expected one " & - " of 'e', 'E', 'f', 'F', 'g', 'G' but got: " & spec.typ) + let + spec = parseStandardFormatSpecifier(specifier) + radix = toRadix(spec.typ) + + result.add formatInt(value, radix, spec) +proc formatFloat( + result: var string, value: SomeFloat, fmode: FloatFormatMode, + spec: StandardFormatSpecifier) = var f = formatBiggestFloat(value, fmode, spec.precision) var sign = false if value >= 0.0: @@ -518,23 +557,83 @@ proc formatValue*(result: var string; value: SomeFloat; specifier: string) = else: result.add res +proc toFloatFormatMode(typ: char): FloatFormatMode = + case typ + of 'e', 'E': ffScientific + of 'f', 'F': ffDecimal + of 'g', 'G': ffDefault + of '\0': ffDefault + else: + raise newException(ValueError, + "invalid type in format string for number, expected one " & + " of 'e', 'E', 'f', 'F', 'g', 'G' but got: " & typ) + +proc formatValue*(result: var string; value: SomeFloat; specifier: static string) = + ## Standard format implementation for `SomeFloat`. It makes little + ## sense to call this directly, but it is required to exist + ## by the `&` macro. + when specifier.len == 0: + result.add $value + else: + const + spec = parseStandardFormatSpecifier(specifier) + fmode = toFloatFormatMode(spec.typ) + + formatFloat(result, value, fmode, spec) + +proc formatValue*(result: var string; value: SomeFloat; specifier: string) = + ## Standard format implementation for `SomeFloat`. It makes little + ## sense to call this directly, but it is required to exist + ## by the `&` macro. + if specifier.len == 0: + result.add $value + else: + let + spec = parseStandardFormatSpecifier(specifier) + fmode = toFloatFormatMode(spec.typ) + + formatFloat(result, value, fmode, spec) + +proc formatValue*(result: var string; value: string; specifier: static string) = + ## Standard format implementation for `string`. It makes little + ## sense to call this directly, but it is required to exist + ## by the `&` macro. + const spec = parseStandardFormatSpecifier(specifier) + var value = + when spec.typ in {'s', '\0'}: value + else: static: + raise newException(ValueError, + "invalid type in format string for string, expected 's', but got " & + spec.typ) + when spec.precision != -1: + if spec.precision < runeLen(value): + const precision = cast[Natural](spec.precision) + setLen(value, Natural(runeOffset(value, precision))) + + result.add alignString(value, spec.minimumWidth, spec.align, spec.fill) + proc formatValue*(result: var string; value: string; specifier: string) = ## Standard format implementation for `string`. It makes little ## sense to call this directly, but it is required to exist ## by the `&` macro. let spec = parseStandardFormatSpecifier(specifier) - var value = value - case spec.typ - of 's', '\0': discard - else: - raise newException(ValueError, - "invalid type in format string for string, expected 's', but got " & - spec.typ) + var value = + if spec.typ in {'s', '\0'}: value + else: + raise newException(ValueError, + "invalid type in format string for string, expected 's', but got " & + spec.typ) if spec.precision != -1: if spec.precision < runeLen(value): - setLen(value, runeOffset(value, spec.precision)) + let precision = cast[Natural](spec.precision) + setLen(value, Natural(runeOffset(value, precision))) + result.add alignString(value, spec.minimumWidth, spec.align, spec.fill) +proc formatValue[T: not SomeInteger](result: var string; value: T; specifier: static string) = + mixin `$` + formatValue(result, $value, specifier) + proc formatValue[T: not SomeInteger](result: var string; value: T; specifier: string) = mixin `$` formatValue(result, $value, specifier) @@ -545,19 +644,20 @@ template formatValue(result: var string; value: char; specifier: string) = template formatValue(result: var string; value: cstring; specifier: string) = result.add value -proc strformatImpl(pattern: NimNode; openChar, closeChar: char): NimNode = - if pattern.kind notin {nnkStrLit..nnkTripleStrLit}: - error "string formatting (fmt(), &) only works with string literals", pattern +proc strformatImpl(f: string; openChar, closeChar: char, + lineInfoNode: NimNode = nil): NimNode = + template missingCloseChar = + error("invalid format string: missing closing character '" & closeChar & "'") + if openChar == ':' or closeChar == ':': error "openChar and closeChar must not be ':'" - let f = pattern.strVal var i = 0 let res = genSym(nskVar, "fmtRes") - result = newNimNode(nnkStmtListExpr, lineInfoFrom = pattern) + result = newNimNode(nnkStmtListExpr, lineInfoNode) # XXX: https://github.com/nim-lang/Nim/issues/8405 # When compiling with -d:useNimRtl, certain procs such as `count` from the strutils # module are not accessible at compile-time: - let expectedGrowth = when defined(useNimRtl): 0 else: count(f, '{') * 10 + let expectedGrowth = when defined(useNimRtl): 0 else: count(f, openChar) * 10 result.add newVarStmt(res, newCall(bindSym"newStringOfCap", newLit(f.len + expectedGrowth))) var strlit = "" @@ -573,28 +673,46 @@ proc strformatImpl(pattern: NimNode; openChar, closeChar: char): NimNode = strlit = "" var subexpr = "" - while i < f.len and f[i] != closeChar and f[i] != ':': - if f[i] == '=': + var inParens = 0 + var inSingleQuotes = false + var inDoubleQuotes = false + template notEscaped:bool = f[i-1]!='\\' + while i < f.len and f[i] != closeChar and (f[i] != ':' or inParens != 0): + case f[i] + of '\\': + if i < f.len-1 and f[i+1] in {openChar,closeChar,':'}: inc i + of '\'': + if not inDoubleQuotes and notEscaped: inSingleQuotes = not inSingleQuotes + of '\"': + if notEscaped: inDoubleQuotes = not inDoubleQuotes + of '(': + if not (inSingleQuotes or inDoubleQuotes): inc inParens + of ')': + if not (inSingleQuotes or inDoubleQuotes): dec inParens + of '=': let start = i inc i i += f.skipWhitespace(i) + if i == f.len: + missingCloseChar if f[i] == closeChar or f[i] == ':': result.add newCall(bindSym"add", res, newLit(subexpr & f[start ..< i])) else: subexpr.add f[start ..< i] - else: - subexpr.add f[i] - inc i + continue + else: discard + subexpr.add f[i] + inc i + + if i == f.len: + missingCloseChar var x: NimNode try: x = parseExpr(subexpr) - except ValueError: - when declared(getCurrentExceptionMsg): - let msg = getCurrentExceptionMsg() - error("could not parse `" & subexpr & "`.\n" & msg, pattern) - else: - error("could not parse `" & subexpr & "`.\n", pattern) + except ValueError as e: + error("could not parse `$#` in `$#`.\n$#" % [subexpr, f, e.msg]) + x.copyLineInfo(lineInfoNode) let formatSym = bindSym("formatValue", brOpen) var options = "" if f[i] == ':': @@ -602,40 +720,71 @@ proc strformatImpl(pattern: NimNode; openChar, closeChar: char): NimNode = while i < f.len and f[i] != closeChar: options.add f[i] inc i + if i == f.len: + missingCloseChar if f[i] == closeChar: inc i - else: - doAssert false, "invalid format string: missing '}'" result.add newCall(formatSym, res, x, newLit(options)) elif f[i] == closeChar: - if f[i+1] == closeChar: + if i<f.len-1 and f[i+1] == closeChar: strlit.add closeChar inc i, 2 else: - doAssert false, "invalid format string: '}' instead of '}}'" - inc i + raiseAssert "invalid format string: '$1' instead of '$1$1'" % $closeChar else: strlit.add f[i] inc i if strlit.len > 0: result.add newCall(bindSym"add", res, newLit(strlit)) result.add res + # workaround for #20381 + var blockExpr = newNimNode(nnkBlockExpr, lineInfoNode) + blockExpr.add(newEmptyNode()) + blockExpr.add(result) + result = blockExpr when defined(debugFmtDsl): echo repr result -macro `&`*(pattern: string): untyped = strformatImpl(pattern, '{', '}') - ## For a specification of the `&` macro, see the module level documentation. +macro fmt(pattern: static string; openChar: static char, closeChar: static char, lineInfoNode: untyped): string = + ## version of `fmt` with dummy untyped param for line info + strformatImpl(pattern, openChar, closeChar, lineInfoNode) -macro fmt*(pattern: string): untyped = strformatImpl(pattern, '{', '}') - ## An alias for `& <#&.m,string>`_. +when not defined(nimHasCallsitePragma): + {.pragma: callsite.} -macro fmt*(pattern: string; openChar, closeChar: char): untyped = - ## The same as `fmt <#fmt.m,string>`_, but uses `openChar` instead of `'{'` - ## and `closeChar` instead of `'}'`. +template fmt*(pattern: static string; openChar: static char, closeChar: static char): string {.callsite.} = + ## Interpolates `pattern` using symbols in scope. runnableExamples: - let testInt = 123 - assert "<testInt>".fmt('<', '>') == "123" - assert """(()"foo" & "bar"())""".fmt(')', '(') == "(foobar)" - assert """ ""{"123+123"}"" """.fmt('"', '"') == " \"{246}\" " - - strformatImpl(pattern, openChar.intVal.char, closeChar.intVal.char) + let x = 7 + assert "var is {x * 2}".fmt == "var is 14" + assert "var is {{x}}".fmt == "var is {x}" # escape via doubling + const s = "foo: {x}" + assert s.fmt == "foo: 7" # also works with const strings + + assert fmt"\n" == r"\n" # raw string literal + assert "\n".fmt == "\n" # regular literal (likewise with `fmt("\n")` or `fmt "\n"`) + runnableExamples: + # custom `openChar`, `closeChar` + let x = 7 + assert "<x>".fmt('<', '>') == "7" + assert "<<<x>>>".fmt('<', '>') == "<7>" + assert "`x`".fmt('`', '`') == "7" + fmt(pattern, openChar, closeChar, dummyForLineInfo) + +template fmt*(pattern: static string): untyped {.callsite.} = + ## Alias for `fmt(pattern, '{', '}')`. + fmt(pattern, '{', '}', dummyForLineInfo) + +template `&`*(pattern: string{lit}): string {.callsite.} = + ## `&pattern` is the same as `pattern.fmt`. + ## For a specification of the `&` macro, see the module level documentation. + # pending bug #18275, bug #18278, use `pattern: static string` + # consider deprecating this, it's redundant with `fmt` and `fmt` is strictly + # more flexible, readable (no confusion with the binary `&`), self-documenting, + # not to mention #18275, bug #18278. + runnableExamples: + let x = 7 + assert &"{x}\n" == "7\n" # regular string literal + assert &"{x}\n" == "{x}\n".fmt # `fmt` can be used instead + assert &"{x}\n" != fmt"{x}\n" # see `fmt` docs, this would use a raw string literal + fmt(pattern, '{', '}', dummyForLineInfo) diff --git a/lib/pure/strmisc.nim b/lib/pure/strmisc.nim index c8cd839be..a3e539e7e 100644 --- a/lib/pure/strmisc.nim +++ b/lib/pure/strmisc.nim @@ -27,20 +27,17 @@ func expandTabs*(s: string, tabSize: int = 8): string = doAssert expandTabs("a\tb\n\txy\t", 3) == "a b\n xy " result = newStringOfCap(s.len + s.len shr 2) - var pos = 0 template addSpaces(n) = - for j in 0 ..< n: + for _ in 1..n: result.add(' ') - pos += 1 + pos += n - for i in 0 ..< len(s): - let c = s[i] + var pos = 0 + let denominator = if tabSize > 0: tabSize else: 1 + for c in s: if c == '\t': - let - denominator = if tabSize > 0: tabSize else: 1 - numSpaces = tabSize - pos mod denominator - + let numSpaces = tabSize - pos mod denominator addSpaces(numSpaces) else: result.add(c) diff --git a/lib/pure/strscans.nim b/lib/pure/strscans.nim index 73b53e3d6..16ef9e642 100644 --- a/lib/pure/strscans.nim +++ b/lib/pure/strscans.nim @@ -12,7 +12,7 @@ This module contains a `scanf`:idx: macro that can be used for extracting substrings from an input string. This is often easier than regular expressions. Some examples as an appetizer: -.. code-block:: nim + ```nim # check if input string matches a triple of integers: const input = "(1,2,4)" var x, y, z: int @@ -26,6 +26,7 @@ Some examples as an appetizer: var myfloat: float if scanf(input, "$i-$i-$i $w$s$f", year, month, day, identifier, myfloat): echo "yes, we have a match!" + ``` As can be seen from the examples, strings are matched verbatim except for substrings starting with ``$``. These constructions are available: @@ -35,7 +36,7 @@ substrings starting with ``$``. These constructions are available: ``$o`` Matches an octal integer. This uses ``parseutils.parseOct``. ``$i`` Matches a decimal integer. This uses ``parseutils.parseInt``. ``$h`` Matches a hex integer. This uses ``parseutils.parseHex``. -``$f`` Matches a floating pointer number. Uses ``parseFloat``. +``$f`` Matches a floating-point number. Uses ``parseFloat``. ``$w`` Matches an ASCII identifier: ``[A-Za-z_][A-Za-z_0-9]*``. ``$c`` Matches a single ASCII character. ``$s`` Skips optional whitespace. @@ -52,7 +53,7 @@ substrings starting with ``$``. These constructions are available: ================= ======================================================== Even though ``$*`` and ``$+`` look similar to the regular expressions ``.*`` -and ``.+`` they work quite differently, there is no non-deterministic +and ``.+``, they work quite differently. There is no non-deterministic state machine involved and the matches are non-greedy. ``[$*]`` matches ``[xyz]`` via ``parseutils.parseUntil``. @@ -83,8 +84,7 @@ matches optional tokens without any result binding. In this example, we define a helper proc ``someSep`` that skips some separators which we then use in our scanf pattern to help us in the matching process: -.. code-block:: nim - + ```nim proc someSep(input: string; start: int; seps: set[char] = {':','-','.'}): int = # Note: The parameters and return value must match to what ``scanf`` requires result = 0 @@ -92,11 +92,11 @@ which we then use in our scanf pattern to help us in the matching process: if scanf(input, "$w$[someSep]$w", key, value): ... + ``` -It also possible to pass arguments to a user definable matcher: - -.. code-block:: nim +It is also possible to pass arguments to a user definable matcher: + ```nim proc ndigits(input: string; intVal: var int; start: int; n: int): int = # matches exactly ``n`` digits. Matchers need to return 0 if nothing # matched or otherwise the number of processed chars. @@ -115,6 +115,7 @@ It also possible to pass arguments to a user definable matcher: var year, month, day: int if scanf("2013-01-03", "${ndigits(4)}-${ndigits(2)}-${ndigits(2)}$.", year, month, day): ... + ``` The scanp macro @@ -145,8 +146,7 @@ not implemented. Simple example that parses the ``/etc/passwd`` file line by line: -.. code-block:: nim - + ```nim const etc_passwd = """root:x:0:0:root:/root:/bin/bash daemon:x:1:1:daemon:/usr/sbin:/bin/sh @@ -165,16 +165,16 @@ Simple example that parses the ``/etc/passwd`` file line by line: result.add entry else: break + ``` The ``scanp`` maps the grammar code into Nim code that performs the parsing. The parsing is performed with the help of 3 helper templates that that can be implemented for a custom type. These templates need to be named ``atom`` and ``nxt``. ``atom`` should be -overloaded to handle both single characters and sets of character. - -.. code-block:: nim +overloaded to handle both `char` and `set[char]`. + ```nim import std/streams template atom(input: Stream; idx: int; c: char): bool = @@ -190,11 +190,11 @@ overloaded to handle both single characters and sets of character. if scanp(content, idx, +( ~{'\L', '\0'} -> entry.add(peekChar($input))), '\L'): result.add entry + ``` Calling ordinary Nim procs inside the macro is possible: -.. code-block:: nim - + ```nim proc digits(s: string; intVal: var int; start: int): int = var x = 0 while result+start < s.len and s[result+start] in {'0'..'9'} and s[result+start] != ':': @@ -220,12 +220,12 @@ Calling ordinary Nim procs inside the macro is possible: result.add login & " " & homedir else: break + ``` When used for matching, keep in mind that likewise scanf, no backtracking is performed. -.. code-block:: nim - + ```nim proc skipUntil(s: string; until: string; unless = '\0'; start: int): int = # Skips all characters until the string `until` is found. Returns 0 # if the char `unless` is found first or the end is reached. @@ -256,12 +256,12 @@ is performed. for r in collectLinks(body): echo r + ``` In this example both macros are combined seamlessly in order to maximise efficiency and perform different checks. -.. code-block:: nim - + ```nim iterator parseIps*(soup: string): string = ## ipv4 only! const digits = {'0'..'9'} @@ -279,13 +279,17 @@ efficiency and perform different checks. yield buf buf.setLen(0) # need to clear `buf` each time, cause it might contain garbage idx.inc - + ``` ]## -import macros, parseutils +import std/[macros, parseutils] import std/private/since +when defined(nimPreviewSlimSystem): + import std/assertions + + proc conditionsToIfChain(n, idx, res: NimNode; start: int): NimNode = assert n.kind == nnkStmtList if start >= n.len: return newAssignment(res, newLit true) @@ -468,7 +472,7 @@ macro scanf*(input: string; pattern: static[string]; results: varargs[typed]): b macro scanTuple*(input: untyped; pattern: static[string]; matcherTypes: varargs[untyped]): untyped {.since: (1, 5).}= ## Works identically as scanf, but instead of predeclaring variables it returns a tuple. - ## Tuple is started with a bool which indicates if the scan was successful + ## Tuple is started with a bool which indicates if the scan was successful ## followed by the requested data. ## If using a user defined matcher, provide the types in order they appear after pattern: ## `line.scanTuple("${yourMatcher()}", int)` @@ -508,7 +512,7 @@ macro scanTuple*(input: untyped; pattern: static[string]; matcherTypes: varargs[ inc userMatches else: discard inc p - result.add newPar(newCall(ident("scanf"), input, newStrLitNode(pattern))) + result.add nnkTupleConstr.newTree(newCall(ident("scanf"), input, newStrLitNode(pattern))) for arg in arguments: result[^1][0].add arg result[^1].add arg @@ -584,7 +588,7 @@ macro scanp*(input, idx: typed; pattern: varargs[untyped]): bool = action # (x) a # bind action a to (x) - if it[0].kind == nnkPar and it.len == 2: + if it[0].kind in {nnkPar, nnkTupleConstr} and it.len == 2: result = atm(it[0], input, idx, placeholder(it[1], input, idx)) elif it.kind == nnkInfix and it[0].eqIdent"->": # bind matching to some action: diff --git a/lib/pure/strtabs.nim b/lib/pure/strtabs.nim index 3b90fea50..4b07aca5a 100644 --- a/lib/pure/strtabs.nim +++ b/lib/pure/strtabs.nim @@ -51,13 +51,17 @@ runnableExamples: import std/private/since import - hashes, strutils + std/[hashes, strutils] + +when defined(nimPreviewSlimSystem): + import std/assertions + when defined(js) or defined(nimscript) or defined(Standalone): {.pragma: rtlFunc.} else: {.pragma: rtlFunc, rtl.} - import os + import std/envvars include "system/inclrtl" @@ -257,10 +261,7 @@ proc newStringTable*(mode: StringTableMode): owned(StringTableRef) {. ## See also: ## * `newStringTable(keyValuePairs) proc ## <#newStringTable,varargs[tuple[string,string]],StringTableMode>`_ - new(result) - result.mode = mode - result.counter = 0 - newSeq(result.data, startSize) + result = StringTableRef(mode: mode, counter: 0, data: newSeq[KeyValuePair](startSize)) proc newStringTable*(keyValuePairs: varargs[string], mode: StringTableMode): owned(StringTableRef) {. diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index 4098749ce..81be7db17 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -70,17 +70,21 @@ runnableExamples: ## easier substring extraction than regular expressions -import parseutils -from math import pow, floor, log10 -from algorithm import reverse +import std/parseutils +from std/math import pow, floor, log10 +from std/algorithm import fill, reverse import std/enumutils -from unicode import toLower, toUpper +from std/unicode import toLower, toUpper export toLower, toUpper include "system/inclrtl" -import std/private/since -from std/private/strimpl import cmpIgnoreStyleImpl, cmpIgnoreCaseImpl, startsWithImpl, endsWithImpl +import std/private/[since, jsutils] +from std/private/strimpl import cmpIgnoreStyleImpl, cmpIgnoreCaseImpl, + startsWithImpl, endsWithImpl + +when defined(nimPreviewSlimSystem): + import std/assertions const @@ -91,6 +95,15 @@ const Letters* = {'A'..'Z', 'a'..'z'} ## The set of letters. + UppercaseLetters* = {'A'..'Z'} + ## The set of uppercase ASCII letters. + + LowercaseLetters* = {'a'..'z'} + ## The set of lowercase ASCII letters. + + PunctuationChars* = {'!'..'/', ':'..'@', '['..'`', '{'..'~'} + ## The set of all ASCII punctuation characters. + Digits* = {'0'..'9'} ## The set of digits. @@ -107,17 +120,20 @@ const ## The set of characters a newline terminator can start with (carriage ## return, line feed). + PrintableChars* = Letters + Digits + PunctuationChars + Whitespace + ## The set of all printable ASCII characters (letters, digits, whitespace, and punctuation characters). + AllChars* = {'\x00'..'\xFF'} ## A set with all the possible characters. ## ## Not very useful by its own, you can use it to create *inverted* sets to ## make the `find func<#find,string,set[char],Natural,int>`_ ## find **invalid** characters in strings. Example: - ## - ## .. code-block:: nim + ## ```nim ## let invalid = AllChars - Digits ## doAssert "01234".find(invalid) == -1 ## doAssert "01A34".find(invalid) == 2 + ## ``` func isAlphaAscii*(c: char): bool {.rtl, extern: "nsuIsAlphaAsciiChar".} = ## Checks whether or not character `c` is alphabetical. @@ -169,7 +185,7 @@ func isLowerAscii*(c: char): bool {.rtl, extern: "nsuIsLowerAsciiChar".} = doAssert isLowerAscii('e') == true doAssert isLowerAscii('E') == false doAssert isLowerAscii('7') == false - return c in {'a'..'z'} + return c in LowercaseLetters func isUpperAscii*(c: char): bool {.rtl, extern: "nsuIsUpperAsciiChar".} = ## Checks whether or not `c` is an upper case character. @@ -183,8 +199,7 @@ func isUpperAscii*(c: char): bool {.rtl, extern: "nsuIsUpperAsciiChar".} = doAssert isUpperAscii('e') == false doAssert isUpperAscii('E') == true doAssert isUpperAscii('7') == false - return c in {'A'..'Z'} - + return c in UppercaseLetters func toLowerAscii*(c: char): char {.rtl, extern: "nsuToLowerAsciiChar".} = ## Returns the lower case version of character `c`. @@ -199,7 +214,7 @@ func toLowerAscii*(c: char): char {.rtl, extern: "nsuToLowerAsciiChar".} = runnableExamples: doAssert toLowerAscii('A') == 'a' doAssert toLowerAscii('e') == 'e' - if c in {'A'..'Z'}: + if c in UppercaseLetters: result = char(uint8(c) xor 0b0010_0000'u8) else: result = c @@ -236,7 +251,7 @@ func toUpperAscii*(c: char): char {.rtl, extern: "nsuToUpperAsciiChar".} = runnableExamples: doAssert toUpperAscii('a') == 'A' doAssert toUpperAscii('E') == 'E' - if c in {'a'..'z'}: + if c in LowercaseLetters: result = char(uint8(c) xor 0b0010_0000'u8) else: result = c @@ -273,14 +288,20 @@ func nimIdentNormalize*(s: string): string = ## ## That means to convert to lower case and remove any '_' on all characters ## except first one. + ## + ## .. Warning:: Backticks (`) are not handled: they remain *as is* and + ## spaces are preserved. See `nimIdentBackticksNormalize + ## <dochelpers.html#nimIdentBackticksNormalize,string>`_ for + ## an alternative approach. runnableExamples: doAssert nimIdentNormalize("Foo_bar") == "Foobar" result = newString(s.len) - if s.len > 0: - result[0] = s[0] + if s.len == 0: + return + result[0] = s[0] var j = 1 for i in 1..len(s) - 1: - if s[i] in {'A'..'Z'}: + if s[i] in UppercaseLetters: result[j] = chr(ord(s[i]) + (ord('a') - ord('A'))) inc j elif s[i] != '_': @@ -302,7 +323,7 @@ func normalize*(s: string): string {.rtl, extern: "nsuNormalize".} = result = newString(s.len) var j = 0 for i in 0..len(s) - 1: - if s[i] in {'A'..'Z'}: + if s[i] in UppercaseLetters: result[j] = chr(ord(s[i]) + (ord('a') - ord('A'))) inc j elif s[i] != '_': @@ -313,9 +334,9 @@ func normalize*(s: string): string {.rtl, extern: "nsuNormalize".} = func cmpIgnoreCase*(a, b: string): int {.rtl, extern: "nsuCmpIgnoreCase".} = ## Compares two strings in a case insensitive manner. Returns: ## - ## | 0 if a == b - ## | < 0 if a < b - ## | > 0 if a > b + ## | `0` if a == b + ## | `< 0` if a < b + ## | `> 0` if a > b runnableExamples: doAssert cmpIgnoreCase("FooBar", "foobar") == 0 doAssert cmpIgnoreCase("bar", "Foo") < 0 @@ -333,9 +354,9 @@ func cmpIgnoreStyle*(a, b: string): int {.rtl, extern: "nsuCmpIgnoreStyle".} = ## ## Returns: ## - ## | 0 if a == b - ## | < 0 if a < b - ## | > 0 if a > b + ## | `0` if a == b + ## | `< 0` if a < b + ## | `> 0` if a > b runnableExamples: doAssert cmpIgnoreStyle("foo_bar", "FooBar") == 0 doAssert cmpIgnoreStyle("foo_bar_5", "FooBar4") > 0 @@ -345,11 +366,14 @@ func cmpIgnoreStyle*(a, b: string): int {.rtl, extern: "nsuCmpIgnoreStyle".} = # --------- Private templates for different split separators ----------- func substrEq(s: string, pos: int, substr: string): bool = - var i = 0 + # Always returns false for empty `substr` var length = substr.len - while i < length and pos+i < s.len and s[pos+i] == substr[i]: - inc i - return i == length + if length > 0: + var i = 0 + while i < length and pos+i < s.len and s[pos+i] == substr[i]: + inc i + i == length + else: false template stringHasSep(s: string, index: int, seps: set[char]): bool = s[index] in seps @@ -399,14 +423,12 @@ iterator split*(s: string, sep: char, maxsplit: int = -1): string = ## ## Substrings are separated by the character `sep`. ## The code: - ## - ## .. code-block:: nim + ## ```nim ## for word in split(";;this;is;an;;example;;;", ';'): ## writeLine(stdout, word) - ## + ## ``` ## Results in: - ## - ## .. code-block:: + ## ``` ## "" ## "" ## "this" @@ -417,6 +439,7 @@ iterator split*(s: string, sep: char, maxsplit: int = -1): string = ## "" ## "" ## "" + ## ``` ## ## See also: ## * `rsplit iterator<#rsplit.i,string,char,int>`_ @@ -431,41 +454,49 @@ iterator split*(s: string, seps: set[char] = Whitespace, ## ## Substrings are separated by a substring containing only `seps`. ## - ## .. code-block:: nim + ## ```nim ## for word in split("this\lis an\texample"): ## writeLine(stdout, word) + ## ``` ## ## ...generates this output: ## - ## .. code-block:: + ## ``` ## "this" ## "is" ## "an" ## "example" + ## ``` ## ## And the following code: ## - ## .. code-block:: nim + ## ```nim ## for word in split("this:is;an$example", {';', ':', '$'}): ## writeLine(stdout, word) + ## ``` ## ## ...produces the same output as the first example. The code: ## - ## .. code-block:: nim + ## ```nim ## let date = "2012-11-20T22:08:08.398990" ## let separators = {' ', '-', ':', 'T'} ## for number in split(date, separators): ## writeLine(stdout, number) + ## ``` ## ## ...results in: ## - ## .. code-block:: + ## ``` ## "2012" ## "11" ## "20" ## "22" ## "08" ## "08.398990" + ## ``` + ## + ## .. note:: Empty separator set results in returning an original string, + ## following the interpretation "split by no element". ## ## See also: ## * `rsplit iterator<#rsplit.i,string,set[char],int>`_ @@ -480,23 +511,30 @@ iterator split*(s: string, sep: string, maxsplit: int = -1): string = ## Substrings are separated by the string `sep`. ## The code: ## - ## .. code-block:: nim + ## ```nim ## for word in split("thisDATAisDATAcorrupted", "DATA"): ## writeLine(stdout, word) + ## ``` ## ## Results in: ## - ## .. code-block:: + ## ``` ## "this" ## "is" ## "corrupted" + ## ``` + ## + ## .. note:: Empty separator string results in returning an original string, + ## following the interpretation "split by no element". ## ## See also: ## * `rsplit iterator<#rsplit.i,string,string,int,bool>`_ ## * `splitLines iterator<#splitLines.i,string>`_ ## * `splitWhitespace iterator<#splitWhitespace.i,string,int>`_ ## * `split func<#split,string,string,int>`_ - splitCommon(s, sep, maxsplit, sep.len) + let sepLen = if sep.len == 0: 1 # prevents infinite loop + else: sep.len + splitCommon(s, sep, maxsplit, sepLen) template rsplitCommon(s, sep, maxsplit, sepLen) = @@ -527,17 +565,19 @@ iterator rsplit*(s: string, sep: char, maxsplit: int = -1): string = ## Splits the string `s` into substrings from the right using a ## string separator. Works exactly the same as `split iterator - ## <#split.i,string,char,int>`_ except in reverse order. + ## <#split.i,string,char,int>`_ except in **reverse** order. ## - ## .. code-block:: nim + ## ```nim ## for piece in "foo:bar".rsplit(':'): ## echo piece + ## ``` ## ## Results in: ## - ## .. code-block:: nim + ## ``` ## "bar" ## "foo" + ## ``` ## ## Substrings are separated from the right by the char `sep`. ## @@ -552,20 +592,25 @@ iterator rsplit*(s: string, seps: set[char] = Whitespace, maxsplit: int = -1): string = ## Splits the string `s` into substrings from the right using a ## string separator. Works exactly the same as `split iterator - ## <#split.i,string,char,int>`_ except in reverse order. + ## <#split.i,string,char,int>`_ except in **reverse** order. ## - ## .. code-block:: nim + ## ```nim ## for piece in "foo bar".rsplit(WhiteSpace): ## echo piece + ## ``` ## ## Results in: ## - ## .. code-block:: nim + ## ``` ## "bar" ## "foo" + ## ``` ## ## Substrings are separated from the right by the set of chars `seps` ## + ## .. note:: Empty separator set results in returning an original string, + ## following the interpretation "split by no element". + ## ## See also: ## * `split iterator<#split.i,string,set[char],int>`_ ## * `splitLines iterator<#splitLines.i,string>`_ @@ -577,44 +622,52 @@ iterator rsplit*(s: string, sep: string, maxsplit: int = -1, keepSeparators: bool = false): string = ## Splits the string `s` into substrings from the right using a ## string separator. Works exactly the same as `split iterator - ## <#split.i,string,string,int>`_ except in reverse order. + ## <#split.i,string,string,int>`_ except in **reverse** order. ## - ## .. code-block:: nim + ## ```nim ## for piece in "foothebar".rsplit("the"): ## echo piece + ## ``` ## ## Results in: ## - ## .. code-block:: nim + ## ``` ## "bar" ## "foo" + ## ``` ## ## Substrings are separated from the right by the string `sep` ## + ## .. note:: Empty separator string results in returning an original string, + ## following the interpretation "split by no element". + ## ## See also: ## * `split iterator<#split.i,string,string,int>`_ ## * `splitLines iterator<#splitLines.i,string>`_ ## * `splitWhitespace iterator<#splitWhitespace.i,string,int>`_ ## * `rsplit func<#rsplit,string,string,int>`_ - rsplitCommon(s, sep, maxsplit, sep.len) + let sepLen = if sep.len == 0: 1 # prevents infinite loop + else: sep.len + rsplitCommon(s, sep, maxsplit, sepLen) iterator splitLines*(s: string, keepEol = false): string = ## Splits the string `s` into its containing lines. ## ## Every `character literal <manual.html#lexical-analysis-character-literals>`_ ## newline combination (CR, LF, CR-LF) is supported. The result strings - ## contain no trailing end of line characters unless parameter `keepEol` + ## contain no trailing end of line characters unless the parameter `keepEol` ## is set to `true`. ## ## Example: ## - ## .. code-block:: nim + ## ```nim ## for line in splitLines("\nthis\nis\nan\n\nexample\n"): ## writeLine(stdout, line) + ## ``` ## ## Results in: ## - ## .. code-block:: nim + ## ```nim ## "" ## "this" ## "is" @@ -622,6 +675,7 @@ iterator splitLines*(s: string, keepEol = false): string = ## "" ## "example" ## "" + ## ``` ## ## See also: ## * `splitWhitespace iterator<#splitWhitespace.i,string,int>`_ @@ -654,16 +708,17 @@ iterator splitWhitespace*(s: string, maxsplit: int = -1): string = ## ## The following code: ## - ## .. code-block:: nim + ## ```nim ## let s = " foo \t bar baz " ## for ms in [-1, 1, 2, 3]: ## echo "------ maxsplit = ", ms, ":" ## for item in s.splitWhitespace(maxsplit=ms): ## echo '"', item, '"' + ## ``` ## ## ...results in: ## - ## .. code-block:: + ## ``` ## ------ maxsplit = -1: ## "foo" ## "bar" @@ -679,6 +734,7 @@ iterator splitWhitespace*(s: string, maxsplit: int = -1): string = ## "foo" ## "bar" ## "baz" + ## ``` ## ## See also: ## * `splitLines iterator<#splitLines.i,string>`_ @@ -707,6 +763,9 @@ func split*(s: string, seps: set[char] = Whitespace, maxsplit: int = -1): seq[ ## The same as the `split iterator <#split.i,string,set[char],int>`_ (see its ## documentation), but is a func that returns a sequence of substrings. ## + ## .. note:: Empty separator set results in returning an original string, + ## following the interpretation "split by no element". + ## ## See also: ## * `split iterator <#split.i,string,set[char],int>`_ ## * `rsplit func<#rsplit,string,set[char],int>`_ @@ -715,6 +774,7 @@ func split*(s: string, seps: set[char] = Whitespace, maxsplit: int = -1): seq[ runnableExamples: doAssert "a,b;c".split({',', ';'}) == @["a", "b", "c"] doAssert "".split({' '}) == @[""] + doAssert "empty seps return unsplit s".split({}) == @["empty seps return unsplit s"] accResult(split(s, seps, maxsplit)) func split*(s: string, sep: string, maxsplit: int = -1): seq[string] {.rtl, @@ -724,6 +784,9 @@ func split*(s: string, sep: string, maxsplit: int = -1): seq[string] {.rtl, ## Substrings are separated by the string `sep`. This is a wrapper around the ## `split iterator <#split.i,string,string,int>`_. ## + ## .. note:: Empty separator string results in returning an original string, + ## following the interpretation "split by no element". + ## ## See also: ## * `split iterator <#split.i,string,string,int>`_ ## * `rsplit func<#rsplit,string,string,int>`_ @@ -736,14 +799,13 @@ func split*(s: string, sep: string, maxsplit: int = -1): seq[string] {.rtl, doAssert "a largely spaced sentence".split(" ") == @["a", "", "largely", "", "", "", "spaced", "sentence"] doAssert "a largely spaced sentence".split(" ", maxsplit = 1) == @["a", " largely spaced sentence"] - doAssert(sep.len > 0) - + doAssert "empty sep returns unsplit s".split("") == @["empty sep returns unsplit s"] accResult(split(s, sep, maxsplit)) func rsplit*(s: string, sep: char, maxsplit: int = -1): seq[string] {.rtl, extern: "nsuRSplitChar".} = ## The same as the `rsplit iterator <#rsplit.i,string,char,int>`_, but is a func - ## that returns a sequence of substrings. + ## that returns a sequence of substrings in original order. ## ## A possible common use case for `rsplit` is path manipulation, ## particularly on systems that don't use a common delimiter. @@ -751,13 +813,15 @@ func rsplit*(s: string, sep: char, maxsplit: int = -1): seq[string] {.rtl, ## For example, if a system had `#` as a delimiter, you could ## do the following to get the tail of the path: ## - ## .. code-block:: nim + ## ```nim ## var tailSplit = rsplit("Root#Object#Method#Index", '#', maxsplit=1) + ## ``` ## ## Results in `tailSplit` containing: ## - ## .. code-block:: nim + ## ```nim ## @["Root#Object#Method", "Index"] + ## ``` ## ## See also: ## * `rsplit iterator <#rsplit.i,string,char,int>`_ @@ -771,7 +835,7 @@ func rsplit*(s: string, seps: set[char] = Whitespace, maxsplit: int = -1): seq[string] {.rtl, extern: "nsuRSplitCharSet".} = ## The same as the `rsplit iterator <#rsplit.i,string,set[char],int>`_, but is a - ## func that returns a sequence of substrings. + ## func that returns a sequence of substrings in original order. ## ## A possible common use case for `rsplit` is path manipulation, ## particularly on systems that don't use a common delimiter. @@ -779,13 +843,18 @@ func rsplit*(s: string, seps: set[char] = Whitespace, ## For example, if a system had `#` as a delimiter, you could ## do the following to get the tail of the path: ## - ## .. code-block:: nim + ## ```nim ## var tailSplit = rsplit("Root#Object#Method#Index", {'#'}, maxsplit=1) + ## ``` ## ## Results in `tailSplit` containing: ## - ## .. code-block:: nim + ## ```nim ## @["Root#Object#Method", "Index"] + ## ``` + ## + ## .. note:: Empty separator set results in returning an original string, + ## following the interpretation "split by no element". ## ## See also: ## * `rsplit iterator <#rsplit.i,string,set[char],int>`_ @@ -798,7 +867,7 @@ func rsplit*(s: string, seps: set[char] = Whitespace, func rsplit*(s: string, sep: string, maxsplit: int = -1): seq[string] {.rtl, extern: "nsuRSplitString".} = ## The same as the `rsplit iterator <#rsplit.i,string,string,int,bool>`_, but is a func - ## that returns a sequence of substrings. + ## that returns a sequence of substrings in original order. ## ## A possible common use case for `rsplit` is path manipulation, ## particularly on systems that don't use a common delimiter. @@ -806,13 +875,18 @@ func rsplit*(s: string, sep: string, maxsplit: int = -1): seq[string] {.rtl, ## For example, if a system had `#` as a delimiter, you could ## do the following to get the tail of the path: ## - ## .. code-block:: nim + ## ```nim ## var tailSplit = rsplit("Root#Object#Method#Index", "#", maxsplit=1) + ## ``` ## ## Results in `tailSplit` containing: ## - ## .. code-block:: nim + ## ```nim ## @["Root#Object#Method", "Index"] + ## ``` + ## + ## .. note:: Empty separator string results in returning an original string, + ## following the interpretation "split by no element". ## ## See also: ## * `rsplit iterator <#rsplit.i,string,string,int,bool>`_ @@ -828,6 +902,7 @@ func rsplit*(s: string, sep: string, maxsplit: int = -1): seq[string] {.rtl, doAssert "".rsplit("Elon Musk") == @[""] doAssert "a largely spaced sentence".rsplit(" ") == @["a", "", "largely", "", "", "", "spaced", "sentence"] + doAssert "empty sep returns unsplit s".rsplit("") == @["empty sep returns unsplit s"] accResult(rsplit(s, sep, maxsplit)) result.reverse() @@ -923,14 +998,26 @@ func toHex*[T: SomeInteger](x: T, len: Positive): string = doAssert b.toHex(4) == "1001" doAssert toHex(62, 3) == "03E" doAssert toHex(-8, 6) == "FFFFF8" - toHexImpl(cast[BiggestUInt](x), len, x < 0) + whenJsNoBigInt64: + toHexImpl(cast[BiggestUInt](x), len, x < 0) + do: + when T is SomeSignedInt: + toHexImpl(cast[BiggestUInt](BiggestInt(x)), len, x < 0) + else: + toHexImpl(BiggestUInt(x), len, x < 0) func toHex*[T: SomeInteger](x: T): string = ## Shortcut for `toHex(x, T.sizeof * 2)` runnableExamples: doAssert toHex(1984'i64) == "00000000000007C0" doAssert toHex(1984'i16) == "07C0" - toHexImpl(cast[BiggestUInt](x), 2*sizeof(T), x < 0) + whenJsNoBigInt64: + toHexImpl(cast[BiggestUInt](x), 2*sizeof(T), x < 0) + do: + when T is SomeSignedInt: + toHexImpl(cast[BiggestUInt](BiggestInt(x)), 2*sizeof(T), x < 0) + else: + toHexImpl(BiggestUInt(x), 2*sizeof(T), x < 0) func toHex*(s: string): string {.rtl.} = ## Converts a bytes string to its hexadecimal representation. @@ -1220,7 +1307,7 @@ func parseEnum*[T: enum](s: string): T = ## type contains multiple fields with the same string value. ## ## Raises `ValueError` for an invalid value in `s`. The comparison is - ## done in a style insensitive way. + ## done in a style insensitive way (first letter is still case-sensitive). runnableExamples: type MyEnum = enum @@ -1240,7 +1327,7 @@ func parseEnum*[T: enum](s: string, default: T): T = ## type contains multiple fields with the same string value. ## ## Uses `default` for an invalid value in `s`. The comparison is done in a - ## style insensitive way. + ## style insensitive way (first letter is still case-sensitive). runnableExamples: type MyEnum = enum @@ -1476,12 +1563,40 @@ func dedent*(s: string, count: Natural = indentation(s)): string {.rtl, doAssert x == "Hello\n There\n" unindent(s, count, " ") -func delete*(s: var string, first, last: int) {.rtl, extern: "nsuDelete".} = - ## Deletes in `s` (must be declared as `var`) the characters at positions - ## `first .. last` (both ends included). - ## - ## This modifies `s` itself, it does not return a copy. - runnableExamples: +func delete*(s: var string, slice: Slice[int]) = + ## Deletes the items `s[slice]`, raising `IndexDefect` if the slice contains + ## elements out of range. + ## + ## This operation moves all elements after `s[slice]` in linear time, and + ## is the string analog to `sequtils.delete`. + runnableExamples: + var a = "abcde" + doAssertRaises(IndexDefect): a.delete(4..5) + assert a == "abcde" + a.delete(4..4) + assert a == "abcd" + a.delete(1..2) + assert a == "ad" + a.delete(1..<1) # empty slice + assert a == "ad" + when compileOption("boundChecks"): + if not (slice.a < s.len and slice.a >= 0 and slice.b < s.len): + raise newException(IndexDefect, $(slice: slice, len: s.len)) + if slice.b >= slice.a: + var i = slice.a + var j = slice.b + 1 + var newLen = s.len - j + i + # if j < s.len: moveMem(addr s[i], addr s[j], s.len - j) # pending benchmark + while i < newLen: + s[i] = s[j] + inc(i) + inc(j) + setLen(s, newLen) + +func delete*(s: var string, first, last: int) {.rtl, extern: "nsuDelete", + deprecated: "use `delete(s, first..last)`".} = + ## Deletes in `s` the characters at positions `first .. last` (both ends included). + runnableExamples("--warning:deprecated:off"): var a = "abracadabra" a.delete(4, 5) @@ -1502,7 +1617,6 @@ func delete*(s: var string, first, last: int) {.rtl, extern: "nsuDelete".} = inc(j) setLen(s, newLen) - func startsWith*(s: string, prefix: char): bool {.inline.} = ## Returns true if `s` starts with character `prefix`. ## @@ -1600,7 +1714,7 @@ func removePrefix*(s: var string, chars: set[char] = Newlines) {.rtl, var start = 0 while start < s.len and s[start] in chars: start += 1 - if start > 0: s.delete(0, start - 1) + if start > 0: s.delete(0..start - 1) func removePrefix*(s: var string, c: char) {.rtl, extern: "nsuRemovePrefixChar".} = @@ -1627,8 +1741,8 @@ func removePrefix*(s: var string, prefix: string) {.rtl, var answers = "yesyes" answers.removePrefix("yes") doAssert answers == "yes" - if s.startsWith(prefix): - s.delete(0, prefix.len - 1) + if s.startsWith(prefix) and prefix.len > 0: + s.delete(0..prefix.len - 1) func removeSuffix*(s: var string, chars: set[char] = Newlines) {.rtl, extern: "nsuRemoveSuffixCharSet".} = @@ -1694,8 +1808,9 @@ func addSep*(dest: var string, sep = ", ", startLen: Natural = 0) {.inline.} = ## ## A shorthand for: ## - ## .. code-block:: nim + ## ```nim ## if dest.len > startLen: add(dest, sep) + ## ``` ## ## This is often useful for generating some code where the items need to ## be *separated* by `sep`. `sep` is only added if `dest` is longer than @@ -1760,7 +1875,7 @@ func join*(a: openArray[string], sep: string = ""): string {.rtl, else: result = "" -func join*[T: not string](a: openArray[T], sep: string = ""): string {.rtl.} = +proc join*[T: not string](a: openArray[T], sep: string = ""): string = ## Converts all elements in the container `a` to strings using `$`, ## and concatenates them with `sep`. runnableExamples: @@ -1773,36 +1888,44 @@ func join*[T: not string](a: openArray[T], sep: string = ""): string {.rtl.} = add(result, $x) type - SkipTable* = array[char, int] + SkipTable* = array[char, int] ## Character table for efficient substring search. func initSkipTable*(a: var SkipTable, sub: string) {.rtl, extern: "nsuInitSkipTable".} = - ## Preprocess table `a` for `sub`. + ## Initializes table `a` for efficient search of substring `sub`. + ## + ## See also: + ## * `initSkipTable func<#initSkipTable,string>`_ + ## * `find func<#find,SkipTable,string,string,Natural,int>`_ + # TODO: this should be the `default()` initializer for the type. let m = len(sub) - var i = 0 - while i <= 0xff-7: - a[chr(i + 0)] = m - a[chr(i + 1)] = m - a[chr(i + 2)] = m - a[chr(i + 3)] = m - a[chr(i + 4)] = m - a[chr(i + 5)] = m - a[chr(i + 6)] = m - a[chr(i + 7)] = m - i += 8 + fill(a, m) for i in 0 ..< m - 1: a[sub[i]] = m - 1 - i -func find*(a: SkipTable, s, sub: string, start: Natural = 0, last = 0): int {. +func initSkipTable*(sub: string): SkipTable {.noinit, rtl, + extern: "nsuInitNewSkipTable".} = + ## Returns a new table initialized for `sub`. + ## + ## See also: + ## * `initSkipTable func<#initSkipTable,SkipTable,string>`_ + ## * `find func<#find,SkipTable,string,string,Natural,int>`_ + initSkipTable(result, sub) + +func find*(a: SkipTable, s, sub: string, start: Natural = 0, last = -1): int {. rtl, extern: "nsuFindStrA".} = ## Searches for `sub` in `s` inside range `start..last` using preprocessed ## table `a`. If `last` is unspecified, it defaults to `s.high` (the last ## element). ## ## Searching is case-sensitive. If `sub` is not in `s`, -1 is returned. + ## + ## See also: + ## * `initSkipTable func<#initSkipTable,string>`_ + ## * `initSkipTable func<#initSkipTable,SkipTable,string>`_ let - last = if last == 0: s.high else: last + last = if last < 0: s.high else: last subLast = sub.len - 1 if subLast == -1: @@ -1812,6 +1935,7 @@ func find*(a: SkipTable, s, sub: string, start: Natural = 0, last = 0): int {. # This is an implementation of the Boyer-Moore Horspool algorithms # https://en.wikipedia.org/wiki/Boyer%E2%80%93Moore%E2%80%93Horspool_algorithm + result = -1 var skip = start while last - skip >= subLast: @@ -1821,7 +1945,6 @@ func find*(a: SkipTable, s, sub: string, start: Natural = 0, last = 0): int {. return skip dec i inc skip, a[s[skip + subLast]] - return -1 when not (defined(js) or defined(nimdoc) or defined(nimscript)): func c_memchr(cstr: pointer, c: char, n: csize_t): pointer {. @@ -1830,68 +1953,97 @@ when not (defined(js) or defined(nimdoc) or defined(nimscript)): else: const hasCStringBuiltin = false -func find*(s: string, sub: char, start: Natural = 0, last = 0): int {.rtl, +func find*(s: string, sub: char, start: Natural = 0, last = -1): int {.rtl, extern: "nsuFindChar".} = ## Searches for `sub` in `s` inside range `start..last` (both ends included). - ## If `last` is unspecified, it defaults to `s.high` (the last element). + ## If `last` is unspecified or negative, it defaults to `s.high` (the last element). ## ## Searching is case-sensitive. If `sub` is not in `s`, -1 is returned. ## Otherwise the index returned is relative to `s[0]`, not `start`. - ## Use `s[start..last].rfind` for a `start`-origin index. + ## Subtract `start` from the result for a `start`-origin index. ## ## See also: - ## * `rfind func<#rfind,string,char,Natural>`_ + ## * `rfind func<#rfind,string,char,Natural,int>`_ ## * `replace func<#replace,string,char,char>`_ - let last = if last == 0: s.high else: last - when nimvm: + result = -1 + let last = if last < 0: s.high else: last + + template findImpl = for i in int(start)..last: - if sub == s[i]: return i + if s[i] == sub: + return i + + when nimvm: + findImpl() else: when hasCStringBuiltin: - let L = last-start+1 - if L > 0: - let found = c_memchr(s[start].unsafeAddr, sub, cast[csize_t](L)) + let length = last-start+1 + if length > 0: + let found = c_memchr(s[start].unsafeAddr, sub, cast[csize_t](length)) if not found.isNil: - return cast[ByteAddress](found) -% cast[ByteAddress](s.cstring) + return cast[int](found) -% cast[int](s.cstring) else: - for i in int(start)..last: - if sub == s[i]: return i - return -1 + findImpl() -func find*(s: string, chars: set[char], start: Natural = 0, last = 0): int {. +func find*(s: string, chars: set[char], start: Natural = 0, last = -1): int {. rtl, extern: "nsuFindCharSet".} = ## Searches for `chars` in `s` inside range `start..last` (both ends included). - ## If `last` is unspecified, it defaults to `s.high` (the last element). + ## If `last` is unspecified or negative, it defaults to `s.high` (the last element). ## ## If `s` contains none of the characters in `chars`, -1 is returned. ## Otherwise the index returned is relative to `s[0]`, not `start`. - ## Use `s[start..last].find` for a `start`-origin index. + ## Subtract `start` from the result for a `start`-origin index. ## ## See also: - ## * `rfind func<#rfind,string,set[char],Natural>`_ + ## * `rfind func<#rfind,string,set[char],Natural,int>`_ ## * `multiReplace func<#multiReplace,string,varargs[]>`_ - let last = if last == 0: s.high else: last + result = -1 + let last = if last < 0: s.high else: last for i in int(start)..last: - if s[i] in chars: return i - return -1 - -func find*(s, sub: string, start: Natural = 0, last = 0): int {.rtl, + if s[i] in chars: + return i + +when defined(linux): + proc memmem(haystack: pointer, haystacklen: csize_t, + needle: pointer, needlelen: csize_t): pointer {.importc, header: """#define _GNU_SOURCE +#include <string.h>""".} +elif defined(bsd) or (defined(macosx) and not defined(ios)): + proc memmem(haystack: pointer, haystacklen: csize_t, + needle: pointer, needlelen: csize_t): pointer {.importc, header: "#include <string.h>".} + +func find*(s, sub: string, start: Natural = 0, last = -1): int {.rtl, extern: "nsuFindStr".} = ## Searches for `sub` in `s` inside range `start..last` (both ends included). - ## If `last` is unspecified, it defaults to `s.high` (the last element). + ## If `last` is unspecified or negative, it defaults to `s.high` (the last element). ## ## Searching is case-sensitive. If `sub` is not in `s`, -1 is returned. ## Otherwise the index returned is relative to `s[0]`, not `start`. - ## Use `s[start..last].find` for a `start`-origin index. + ## Subtract `start` from the result for a `start`-origin index. ## ## See also: - ## * `rfind func<#rfind,string,string,Natural>`_ + ## * `rfind func<#rfind,string,string,Natural,int>`_ ## * `replace func<#replace,string,string,string>`_ - if sub.len > s.len: return -1 + if sub.len > s.len - start: return -1 if sub.len == 1: return find(s, sub[0], start, last) - var a {.noinit.}: SkipTable - initSkipTable(a, sub) - result = find(a, s, sub, start, last) + + template useSkipTable = + result = find(initSkipTable(sub), s, sub, start, last) + + when nimvm: + useSkipTable() + else: + when declared(memmem): + let subLen = sub.len + if last < 0 and start < s.len and subLen != 0: + let found = memmem(s[start].unsafeAddr, csize_t(s.len - start), sub.cstring, csize_t(subLen)) + result = if not found.isNil: + cast[int](found) -% cast[int](s.cstring) + else: + -1 + else: + useSkipTable() + else: + useSkipTable() func rfind*(s: string, sub: char, start: Natural = 0, last = -1): int {.rtl, extern: "nsuRFindChar".} = @@ -1902,7 +2054,7 @@ func rfind*(s: string, sub: char, start: Natural = 0, last = -1): int {.rtl, ## ## Searching is case-sensitive. If `sub` is not in `s`, -1 is returned. ## Otherwise the index returned is relative to `s[0]`, not `start`. - ## Use `s[start..last].find` for a `start`-origin index. + ## Subtract `start` from the result for a `start`-origin index. ## ## See also: ## * `find func<#find,string,char,Natural,int>`_ @@ -1920,7 +2072,7 @@ func rfind*(s: string, chars: set[char], start: Natural = 0, last = -1): int {. ## ## If `s` contains none of the characters in `chars`, -1 is returned. ## Otherwise the index returned is relative to `s[0]`, not `start`. - ## Use `s[start..last].rfind` for a `start`-origin index. + ## Subtract `start` from the result for a `start`-origin index. ## ## See also: ## * `find func<#find,string,set[char],Natural,int>`_ @@ -1938,11 +2090,14 @@ func rfind*(s, sub: string, start: Natural = 0, last = -1): int {.rtl, ## ## Searching is case-sensitive. If `sub` is not in `s`, -1 is returned. ## Otherwise the index returned is relative to `s[0]`, not `start`. - ## Use `s[start..last].rfind` for a `start`-origin index. + ## Subtract `start` from the result for a `start`-origin index. ## ## See also: ## * `find func<#find,string,string,Natural,int>`_ if sub.len == 0: + let rightIndex: Natural = if last < 0: s.len else: last + return max(start, rightIndex) + if sub.len > s.len - start: return -1 let last = if last == -1: s.high else: last result = 0 @@ -1998,7 +2153,7 @@ func countLines*(s: string): int {.rtl, extern: "nsuCountLines".} = ## Returns the number of lines in the string `s`. ## ## This is the same as `len(splitLines(s))`, but much more efficient - ## because it doesn't modify the string creating temporal objects. Every + ## because it doesn't modify the string creating temporary objects. Every ## `character literal <manual.html#lexical-analysis-character-literals>`_ ## newline combination (CR, LF, CR-LF) is supported. ## @@ -2064,8 +2219,7 @@ func replace*(s, sub: string, by = ""): string {.rtl, # copy the rest: add result, substr(s, i) else: - var a {.noinit.}: SkipTable - initSkipTable(a, sub) + var a = initSkipTable(sub) let last = s.high var i = 0 while true: @@ -2105,9 +2259,8 @@ func replaceWord*(s, sub: string, by = ""): string {.rtl, ## replaced. if sub.len == 0: return s const wordChars = {'a'..'z', 'A'..'Z', '0'..'9', '_', '\128'..'\255'} - var a {.noinit.}: SkipTable result = "" - initSkipTable(a, sub) + var a = initSkipTable(sub) var i = 0 let last = s.high let sublen = sub.len @@ -2128,17 +2281,31 @@ func replaceWord*(s, sub: string, by = ""): string {.rtl, add result, substr(s, i) func multiReplace*(s: string, replacements: varargs[(string, string)]): string = - ## Same as replace, but specialized for doing multiple replacements in a single - ## pass through the input string. + ## Same as `replace<#replace,string,string,string>`_, but specialized for + ## doing multiple replacements in a single pass through the input string. + ## + ## `multiReplace` scans the input string from left to right and replaces the + ## matching substrings in the same order as passed in the argument list. ## - ## `multiReplace` performs all replacements in a single pass, this means it - ## can be used to swap the occurrences of "a" and "b", for instance. + ## The implications of the order of scanning the string and matching the + ## replacements: + ## - In case of multiple matches at a given position, the earliest + ## replacement is applied. + ## - Overlaps are not handled. After performing a replacement, the scan + ## continues from the character after the matched substring. If the + ## resulting string then contains a possible match starting in a newly + ## placed substring, the additional replacement is not performed. ## ## If the resulting string is not longer than the original input string, ## only a single memory allocation is required. ## - ## The order of the replacements does matter. Earlier replacements are - ## preferred over later replacements in the argument list. + runnableExamples: + # Swapping occurrences of 'a' and 'b': + doAssert multireplace("abba", [("a", "b"), ("b", "a")]) == "baab" + + # The second replacement ("ab") is matched and performed first, the scan then + # continues from 'c', so the "bc" replacement is never matched and thus skipped. + doAssert multireplace("abc", [("bc", "x"), ("ab", "_b")]) == "_bc" result = newStringOfCap(s.len) var i = 0 var fastChk: set[char] = {} @@ -2173,7 +2340,7 @@ func insertSep*(s: string, sep = '_', digits = 3): string {.rtl, doAssert insertSep("1000000") == "1_000_000" result = newStringOfCap(s.len) let hasPrefix = isDigit(s[s.low]) == false - var idx:int + var idx: int if hasPrefix: result.add s[s.low] for i in (s.low + 1)..s.high: @@ -2187,7 +2354,7 @@ func insertSep*(s: string, sep = '_', digits = 3): string {.rtl, result.setLen(L + idx) var j = 0 dec(L) - for i in countdown(partsLen-1,0): + for i in countdown(partsLen-1, 0): if j == digits: result[L + idx] = sep dec(L) @@ -2198,13 +2365,21 @@ func insertSep*(s: string, sep = '_', digits = 3): string {.rtl, func escape*(s: string, prefix = "\"", suffix = "\""): string {.rtl, extern: "nsuEscape".} = - ## Escapes a string `s`. See `system.addEscapedChar - ## <system.html#addEscapedChar,string,char>`_ for the escaping scheme. + ## Escapes a string `s`. + ## + ## .. note:: The escaping scheme is different from + ## `system.addEscapedChar`. + ## + ## * replaces `'\0'..'\31'` and `'\127'..'\255'` by `\xHH` where `HH` is its hexadecimal value + ## * replaces ``\`` by `\\` + ## * replaces `'` by `\'` + ## * replaces `"` by `\"` ## ## The resulting string is prefixed with `prefix` and suffixed with `suffix`. ## Both may be empty strings. ## ## See also: + ## * `addEscapedChar proc<system.html#addEscapedChar,string,char>`_ ## * `unescape func<#unescape,string,string,string>`_ for the opposite ## operation result = newStringOfCap(s.len + s.len shr 2) @@ -2279,8 +2454,8 @@ func validIdentifier*(s: string): bool {.rtl, extern: "nsuValidIdentifier".} = # floating point formatting: when not defined(js): - func c_sprintf(buf, frmt: cstring): cint {.header: "<stdio.h>", - importc: "sprintf", varargs} + func c_snprintf(buf: cstring, n: csize_t, frmt: cstring): cint {.header: "<stdio.h>", + importc: "snprintf", varargs.} type FloatFormatMode* = enum @@ -2307,60 +2482,63 @@ func formatBiggestFloat*(f: BiggestFloat, format: FloatFormatMode = ffDefault, doAssert x.formatBiggestFloat() == "123.4560000000000" doAssert x.formatBiggestFloat(ffDecimal, 4) == "123.4560" doAssert x.formatBiggestFloat(ffScientific, 2) == "1.23e+02" - when defined(js): - var precision = precision - if precision == -1: - # use the same default precision as c_sprintf - precision = 6 - var res: cstring - case format - of ffDefault: - {.emit: "`res` = `f`.toString();".} - of ffDecimal: - {.emit: "`res` = `f`.toFixed(`precision`);".} - of ffScientific: - {.emit: "`res` = `f`.toExponential(`precision`);".} - result = $res - if 1.0 / f == -Inf: - # JavaScript removes the "-" from negative Zero, add it back here - result = "-" & $res - for i in 0 ..< result.len: - # Depending on the locale either dot or comma is produced, - # but nothing else is possible: - if result[i] in {'.', ','}: result[i] = decimalSep + when nimvm: + discard "implemented in the vmops" else: - const floatFormatToChar: array[FloatFormatMode, char] = ['g', 'f', 'e'] - var - frmtstr {.noinit.}: array[0..5, char] - buf {.noinit.}: array[0..2500, char] - L: cint - frmtstr[0] = '%' - if precision >= 0: - frmtstr[1] = '#' - frmtstr[2] = '.' - frmtstr[3] = '*' - frmtstr[4] = floatFormatToChar[format] - frmtstr[5] = '\0' - L = c_sprintf(addr buf, addr frmtstr, precision, f) + when defined(js): + var precision = precision + if precision == -1: + # use the same default precision as c_snprintf + precision = 6 + var res: cstring + case format + of ffDefault: + {.emit: "`res` = `f`.toString();".} + of ffDecimal: + {.emit: "`res` = `f`.toFixed(`precision`);".} + of ffScientific: + {.emit: "`res` = `f`.toExponential(`precision`);".} + result = $res + if 1.0 / f == -Inf: + # JavaScript removes the "-" from negative Zero, add it back here + result = "-" & $res + for i in 0 ..< result.len: + # Depending on the locale either dot or comma is produced, + # but nothing else is possible: + if result[i] in {'.', ','}: result[i] = decimalSep else: - frmtstr[1] = floatFormatToChar[format] - frmtstr[2] = '\0' - L = c_sprintf(addr buf, addr frmtstr, f) - result = newString(L) - for i in 0 ..< L: - # Depending on the locale either dot or comma is produced, - # but nothing else is possible: - if buf[i] in {'.', ','}: result[i] = decimalSep - else: result[i] = buf[i] - when defined(windows): - # VS pre 2015 violates the C standard: "The exponent always contains at - # least two digits, and only as many more digits as necessary to - # represent the exponent." [C11 §7.21.6.1] - # The following post-processing fixes this behavior. - if result.len > 4 and result[^4] == '+' and result[^3] == '0': - result[^3] = result[^2] - result[^2] = result[^1] - result.setLen(result.len - 1) + const floatFormatToChar: array[FloatFormatMode, char] = ['g', 'f', 'e'] + var + frmtstr {.noinit.}: array[0..5, char] + buf {.noinit.}: array[0..2500, char] + L: cint + frmtstr[0] = '%' + if precision >= 0: + frmtstr[1] = '#' + frmtstr[2] = '.' + frmtstr[3] = '*' + frmtstr[4] = floatFormatToChar[format] + frmtstr[5] = '\0' + L = c_snprintf(cast[cstring](addr buf), csize_t(2501), cast[cstring](addr frmtstr), precision, f) + else: + frmtstr[1] = floatFormatToChar[format] + frmtstr[2] = '\0' + L = c_snprintf(cast[cstring](addr buf), csize_t(2501), cast[cstring](addr frmtstr), f) + result = newString(L) + for i in 0 ..< L: + # Depending on the locale either dot or comma is produced, + # but nothing else is possible: + if buf[i] in {'.', ','}: result[i] = decimalSep + else: result[i] = buf[i] + when defined(windows): + # VS pre 2015 violates the C standard: "The exponent always contains at + # least two digits, and only as many more digits as necessary to + # represent the exponent." [C11 §7.21.6.1] + # The following post-processing fixes this behavior. + if result.len > 4 and result[^4] == '+' and result[^3] == '0': + result[^3] = result[^2] + result[^2] = result[^1] + result.setLen(result.len - 1) func formatFloat*(f: float, format: FloatFormatMode = ffDefault, precision: range[-1..32] = 16; decimalSep = '.'): string {. @@ -2400,7 +2578,8 @@ func trimZeros*(x: var string; decimalSep = '.') = var pos = last while pos >= 0 and x[pos] == '0': dec(pos) if pos > sPos: inc(pos) - x.delete(pos, last) + if last >= pos: + x.delete(pos..last) type BinaryPrefixMode* = enum ## The different names for binary prefixes. @@ -2492,13 +2671,13 @@ func formatEng*(f: BiggestFloat, ## decimal point or (if `trim` is true) the maximum number of digits to be ## shown. ## - ## .. code-block:: nim - ## + ## ```nim ## formatEng(0, 2, trim=false) == "0.00" ## formatEng(0, 2) == "0" ## formatEng(0.053, 0) == "53e-3" ## formatEng(52731234, 2) == "52.73e6" ## formatEng(-52731234, 2) == "-52.73e6" + ## ``` ## ## If `siPrefix` is set to true, the number will be displayed with the SI ## prefix corresponding to the exponent. For example 4100 will be displayed @@ -2513,8 +2692,7 @@ func formatEng*(f: BiggestFloat, ## different to appending the unit to the result as the location of the space ## is altered depending on whether there is an exponent. ## - ## .. code-block:: nim - ## + ## ```nim ## formatEng(4100, siPrefix=true, unit="V") == "4.1 kV" ## formatEng(4.1, siPrefix=true, unit="V") == "4.1 V" ## formatEng(4.1, siPrefix=true) == "4.1" # Note lack of space @@ -2524,6 +2702,7 @@ func formatEng*(f: BiggestFloat, ## formatEng(4100) == "4.1e3" ## formatEng(4100, unit="V") == "4.1e3 V" ## formatEng(4100, unit="", useUnitSpace=true) == "4.1e3 " # Space with useUnitSpace=true + ## ``` ## ## `decimalSep` is used as the decimal separator. ## @@ -2611,8 +2790,8 @@ func findNormalized(x: string, inArray: openArray[string]): int = # security hole... return -1 -func invalidFormatString() {.noinline.} = - raise newException(ValueError, "invalid format string") +func invalidFormatString(formatstr: string) {.noinline.} = + raise newException(ValueError, "invalid format string: " & formatstr) func addf*(s: var string, formatstr: string, a: varargs[string, `$`]) {.rtl, extern: "nsuAddf".} = @@ -2624,7 +2803,7 @@ func addf*(s: var string, formatstr: string, a: varargs[string, `$`]) {.rtl, if formatstr[i] == '$' and i+1 < len(formatstr): case formatstr[i+1] of '#': - if num > a.high: invalidFormatString() + if num > a.high: invalidFormatString(formatstr) add s, a[num] inc i, 2 inc num @@ -2640,7 +2819,7 @@ func addf*(s: var string, formatstr: string, a: varargs[string, `$`]) {.rtl, j = j * 10 + ord(formatstr[i]) - ord('0') inc(i) let idx = if not negative: j-1 else: a.len-j - if idx < 0 or idx > a.high: invalidFormatString() + if idx < 0 or idx > a.high: invalidFormatString(formatstr) add s, a[idx] of '{': var j = i+2 @@ -2657,27 +2836,27 @@ func addf*(s: var string, formatstr: string, a: varargs[string, `$`]) {.rtl, inc(j) if isNumber == 1: let idx = if not negative: k-1 else: a.len-k - if idx < 0 or idx > a.high: invalidFormatString() + if idx < 0 or idx > a.high: invalidFormatString(formatstr) add s, a[idx] else: var x = findNormalized(substr(formatstr, i+2, j-1), a) if x >= 0 and x < high(a): add s, a[x+1] - else: invalidFormatString() + else: invalidFormatString(formatstr) i = j+1 of 'a'..'z', 'A'..'Z', '\128'..'\255', '_': var j = i+1 while j < formatstr.len and formatstr[j] in PatternChars: inc(j) var x = findNormalized(substr(formatstr, i+1, j-1), a) if x >= 0 and x < high(a): add s, a[x+1] - else: invalidFormatString() + else: invalidFormatString(formatstr) i = j else: - invalidFormatString() + invalidFormatString(formatstr) else: add s, formatstr[i] inc(i) -func `%` *(formatstr: string, a: openArray[string]): string {.rtl, +func `%`*(formatstr: string, a: openArray[string]): string {.rtl, extern: "nsuFormatOpenArray".} = ## Interpolates a format string with the values from `a`. ## @@ -2687,13 +2866,15 @@ func `%` *(formatstr: string, a: openArray[string]): string {.rtl, ## ## This is best explained by an example: ## - ## .. code-block:: nim + ## ```nim ## "$1 eats $2." % ["The cat", "fish"] + ## ``` ## ## Results in: ## - ## .. code-block:: nim + ## ```nim ## "The cat eats fish." + ## ``` ## ## The substitution variables (the thing after the `$`) are enumerated ## from 1 to `a.len`. @@ -2701,21 +2882,24 @@ func `%` *(formatstr: string, a: openArray[string]): string {.rtl, ## The notation `$#` can be used to refer to the next substitution ## variable: ## - ## .. code-block:: nim + ## ```nim ## "$# eats $#." % ["The cat", "fish"] + ## ``` ## ## Substitution variables can also be words (that is ## `[A-Za-z_]+[A-Za-z0-9_]*`) in which case the arguments in `a` with even ## indices are keys and with odd indices are the corresponding values. ## An example: ## - ## .. code-block:: nim + ## ```nim ## "$animal eats $food." % ["animal", "The cat", "food", "fish"] + ## ``` ## ## Results in: ## - ## .. code-block:: nim + ## ```nim ## "The cat eats fish." + ## ``` ## ## The variables are compared with `cmpIgnoreStyle`. `ValueError` is ## raised if an ill-formed format string has been passed to the `%` operator. @@ -2725,7 +2909,7 @@ func `%` *(formatstr: string, a: openArray[string]): string {.rtl, result = newStringOfCap(formatstr.len + a.len shl 4) addf(result, formatstr, a) -func `%` *(formatstr, a: string): string {.rtl, +func `%`*(formatstr, a: string): string {.rtl, extern: "nsuFormatSingleElem".} = ## This is the same as `formatstr % [a]` (see ## `% func<#%25,string,openArray[string]>`_). @@ -2781,7 +2965,7 @@ func strip*(s: string, leading = true, trailing = true, result = substr(s, first, last) func stripLineEnd*(s: var string) = - ## Returns `s` stripped from one of these suffixes: + ## Strips one of these suffixes from `s` in-place: ## `\r, \n, \r\n, \f, \v` (at most once instance). ## For example, can be useful in conjunction with `osproc.execCmdEx`. ## aka: `chomp`:idx: @@ -2813,13 +2997,14 @@ iterator tokenize*(s: string, seps: set[char] = Whitespace): tuple[ ## Substrings are separated by a substring containing only `seps`. ## Example: ## - ## .. code-block:: nim + ## ```nim ## for word in tokenize(" this is an example "): ## writeLine(stdout, word) + ## ``` ## ## Results in: ## - ## .. code-block:: nim + ## ```nim ## (" ", true) ## ("this", false) ## (" ", true) @@ -2829,6 +3014,7 @@ iterator tokenize*(s: string, seps: set[char] = Whitespace): tuple[ ## (" ", true) ## ("example", false) ## (" ", true) + ## ``` var i = 0 while true: var j = i diff --git a/lib/pure/sugar.nim b/lib/pure/sugar.nim index 239ed9cf1..90ba20c13 100644 --- a/lib/pure/sugar.nim +++ b/lib/pure/sugar.nim @@ -54,6 +54,8 @@ proc createProcType(p, b: NimNode): NimNode = macro `=>`*(p, b: untyped): untyped = ## Syntax sugar for anonymous procedures. It also supports pragmas. + ## + ## .. warning:: Semicolons can not be used to separate procedure arguments. runnableExamples: proc passTwoAndTwo(f: (int, int) -> int): int = f(2, 2) @@ -119,9 +121,9 @@ macro `=>`*(p, b: untyped): untyped = else: error("Incorrect procedure parameter.", c) params.add(identDefs) - of nnkIdent: + of nnkIdent, nnkOpenSymChoice, nnkClosedSymChoice, nnkSym: var identDefs = newNimNode(nnkIdentDefs) - identDefs.add(p) + identDefs.add(ident $p) identDefs.add(ident"auto") identDefs.add(newEmptyNode()) params.add(identDefs) @@ -133,6 +135,8 @@ macro `=>`*(p, b: untyped): untyped = macro `->`*(p, b: untyped): untyped = ## Syntax sugar for procedure types. It also supports pragmas. + ## + ## .. warning:: Semicolons can not be used to separate procedure arguments. runnableExamples: proc passTwoAndTwo(f: (int, int) -> int): int = f(2, 2) # is the same as: @@ -186,7 +190,7 @@ macro dumpToString*(x: untyped): string = assert dumpToString(a + x) == "a + x: 1 + x = 11" template square(x): untyped = x * x assert dumpToString(square(x)) == "square(x): x * x = 100" - assert not compiles dumpToString(1 + nonexistant) + assert not compiles dumpToString(1 + nonexistent) import std/strutils assert "failedAssertImpl" in dumpToString(assert true) # example with a statement result = newCall(bindSym"dumpToStringImpl") @@ -199,7 +203,7 @@ proc freshIdentNodes(ast: NimNode): NimNode = # see also https://github.com/nim-lang/Nim/pull/8531#issuecomment-410436458 proc inspect(node: NimNode): NimNode = case node.kind: - of nnkIdent, nnkSym: + of nnkIdent, nnkSym, nnkOpenSymChoice, nnkClosedSymChoice, nnkOpenSym: result = ident($node) of nnkEmpty, nnkLiterals: result = node @@ -227,9 +231,19 @@ macro capture*(locals: varargs[typed], body: untyped): untyped {.since: (1, 1).} let locals = if locals.len == 1 and locals[0].kind == nnkBracket: locals[0] else: locals for arg in locals: - if arg.strVal == "result": - error("The variable name cannot be `result`!", arg) - params.add(newIdentDefs(ident(arg.strVal), freshIdentNodes getTypeInst arg)) + proc getIdent(n: NimNode): NimNode = + case n.kind + of nnkIdent, nnkSym: + let nStr = n.strVal + if nStr == "result": + error("The variable name cannot be `result`!", n) + result = ident(nStr) + of nnkHiddenDeref: result = n[0].getIdent() + else: + error("The argument to be captured `" & n.repr & "` is not a pure identifier. " & + "It is an unsupported `" & $n.kind & "` node.", n) + let argName = getIdent(arg) + params.add(newIdentDefs(argName, freshIdentNodes getTypeInst arg)) result = newNimNode(nnkCall) result.add(newProc(newEmptyNode(), params, body, nnkLambda)) for arg in locals: result.add(arg) @@ -331,9 +345,10 @@ proc collectImpl(init, body: NimNode): NimNode {.since: (1, 1).} = let res = genSym(nskVar, "collectResult") var bracketExpr: NimNode if init != nil: - expectKind init, {nnkCall, nnkIdent, nnkSym} + expectKind init, {nnkCall, nnkIdent, nnkSym, nnkClosedSymChoice, nnkOpenSymChoice, nnkOpenSym} bracketExpr = newTree(nnkBracketExpr, - if init.kind == nnkCall: freshIdentNodes(init[0]) else: freshIdentNodes(init)) + if init.kind in {nnkCall, nnkClosedSymChoice, nnkOpenSymChoice, nnkOpenSym}: + freshIdentNodes(init[0]) else: freshIdentNodes(init)) else: bracketExpr = newTree(nnkBracketExpr) let (resBody, keyType, valueType) = trans(body, res, bracketExpr) @@ -387,6 +402,10 @@ macro collect*(init, body: untyped): untyped {.since: (1, 1).} = macro collect*(body: untyped): untyped {.since: (1, 5).} = ## Same as `collect` but without an `init` parameter. + ## + ## **See also:** + ## * `sequtils.toSeq proc<sequtils.html#toSeq.t%2Cuntyped>`_ + ## * `sequtils.mapIt template<sequtils.html#mapIt.t%2Ctyped%2Cuntyped>`_ runnableExamples: import std/[sets, tables] let data = @["bird", "word"] @@ -407,10 +426,4 @@ macro collect*(body: untyped): untyped {.since: (1, 5).} = for i, d in data.pairs: {i: d} assert m == {0: "bird", 1: "word"}.toTable - # avoid `collect` when `sequtils.toSeq` suffices: - assert collect(for i in 1..3: i*i) == @[1, 4, 9] # ok in this case - assert collect(for i in 1..3: i) == @[1, 2, 3] # overkill in this case - from std/sequtils import toSeq - assert toSeq(1..3) == @[1, 2, 3] # simpler - result = collectImpl(nil, body) diff --git a/lib/pure/terminal.nim b/lib/pure/terminal.nim index cada72196..53b3d61da 100644 --- a/lib/pure/terminal.nim +++ b/lib/pure/terminal.nim @@ -15,14 +15,59 @@ ## code `exitprocs.addExitProc(resetAttributes)` to restore the defaults. ## Similarly, if you hide the cursor, make sure to unhide it with ## `showCursor` before quitting. +## +## Progress bar +## ============ +## +## Basic progress bar example: +runnableExamples("-r:off"): + import std/[os, strutils] + + for i in 0..100: + stdout.styledWriteLine(fgRed, "0% ", fgWhite, '#'.repeat i, if i > 50: fgGreen else: fgYellow, "\t", $i , "%") + sleep 42 + cursorUp 1 + eraseLine() -import macros -import strformat -from strutils import toLowerAscii, `%` -import colors + stdout.resetAttributes() + +##[ +## Playing with colorful and styled text +]## + +## Procs like `styledWriteLine`, `styledEcho` etc. have a temporary effect on +## text parameters. Style parameters only affect the text parameter right after them. +## After being called, these procs will reset the default style of the terminal. +## While `setBackGroundColor`, `setForeGroundColor` etc. have a lasting +## influence on the terminal, you can use `resetAttributes` to +## reset the default style of the terminal. +runnableExamples("-r:off"): + stdout.styledWriteLine({styleBright, styleBlink, styleUnderscore}, "styled text ") + stdout.styledWriteLine(fgRed, "red text ") + stdout.styledWriteLine(fgWhite, bgRed, "white text in red background") + stdout.styledWriteLine(" ordinary text without style ") + + stdout.setBackGroundColor(bgCyan, true) + stdout.setForeGroundColor(fgBlue) + stdout.write("blue text in cyan background") + stdout.resetAttributes() + + # You can specify multiple text parameters. Style parameters + # only affect the text parameter right after them. + styledEcho styleBright, fgGreen, "[PASS]", resetStyle, fgGreen, " Yay!" + + stdout.styledWriteLine(fgRed, "red text ", styleBright, "bold red", fgDefault, " bold text") + +import std/macros +import std/strformat +from std/strutils import toLowerAscii, `%`, parseInt +import std/colors when defined(windows): - import winlean + import std/winlean + +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions] type PTerminal = ref object @@ -51,10 +96,11 @@ const fgPrefix = "\e[38;2;" bgPrefix = "\e[48;2;" ansiResetCode* = "\e[0m" + getPos = "\e[6n" stylePrefix = "\e[" when defined(windows): - import winlean, os + import std/[winlean, os] const DUPLICATE_SAME_ACCESS = 2 @@ -128,6 +174,7 @@ when defined(windows): return 0 proc terminalWidth*(): int = + ## Returns the terminal width in columns. var w: int = 0 w = terminalWidthIoctl([getStdHandle(STD_INPUT_HANDLE), getStdHandle(STD_OUTPUT_HANDLE), @@ -136,6 +183,7 @@ when defined(windows): return 80 proc terminalHeight*(): int = + ## Returns the terminal height in rows. var h: int = 0 h = terminalHeightIoctl([getStdHandle(STD_INPUT_HANDLE), getStdHandle(STD_OUTPUT_HANDLE), @@ -173,6 +221,9 @@ when defined(windows): raiseOSError(osLastError()) return (int(c.dwCursorPosition.x), int(c.dwCursorPosition.y)) + proc getCursorPos*(): tuple [x, y: int] {.raises: [ValueError, IOError, OSError].} = + return getCursorPos(getStdHandle(STD_OUTPUT_HANDLE)) + proc setCursorPos(h: Handle, x, y: int) = var c: COORD c.x = int16(x) @@ -206,7 +257,7 @@ when defined(windows): if f == stderr: term.hStderr else: term.hStdout else: - import termios, posix, os, parseutils + import std/[termios, posix, os, parseutils] proc setRaw(fd: FileHandle, time: cint = TCSAFLUSH) = var mode: Termios @@ -220,6 +271,48 @@ else: mode.c_cc[VTIME] = 0.cuchar discard fd.tcSetAttr(time, addr mode) + proc getCursorPos*(): tuple [x, y: int] {.raises: [ValueError, IOError].} = + ## Returns cursor position (x, y) + ## writes to stdout and expects the terminal to respond via stdin + var + xStr = "" + yStr = "" + ch: char + ct: int + readX = false + + # use raw mode to ask terminal for cursor position + let fd = getFileHandle(stdin) + var oldMode: Termios + discard fd.tcGetAttr(addr oldMode) + fd.setRaw() + stdout.write(getPos) + flushFile(stdout) + + try: + # parse response format: [yyy;xxxR + while true: + let n = readBuffer(stdin, addr ch, 1) + if n == 0 or ch == 'R': + if xStr == "" or yStr == "": + raise newException(ValueError, "Got character position message that was missing data") + break + ct += 1 + if ct > 16: + raise newException(ValueError, "Got unterminated character position message from terminal") + if ch == ';': + readX = true + elif ch in {'0'..'9'}: + if readX: + xStr.add(ch) + else: + yStr.add(ch) + finally: + # restore previous terminal mode + discard fd.tcSetAttr(TCSADRAIN, addr oldMode) + + return (parseInt(xStr), parseInt(yStr)) + proc terminalWidthIoctl*(fds: openArray[int]): int = ## Returns terminal width from first fd that supports the ioctl. @@ -244,25 +337,57 @@ else: ## Returns some reasonable terminal width from either standard file ## descriptors, controlling terminal, environment variables or tradition. - var w = terminalWidthIoctl([0, 1, 2]) #Try standard file descriptors + # POSIX environment variable takes precendence. + # _COLUMNS_: This variable shall represent a decimal integer >0 used + # to indicate the user's preferred width in column positions for + # the terminal screen or window. If this variable is unset or null, + # the implementation determines the number of columns, appropriate + # for the terminal or window, in an unspecified manner. + # When COLUMNS is set, any terminal-width information implied by TERM + # is overridden. Users and conforming applications should not set COLUMNS + # unless they wish to override the system selection and produce output + # unrelated to the terminal characteristics. + # See POSIX Base Definitions Section 8.1 Environment Variable Definition + + var w: int + var s = getEnv("COLUMNS") # Try standard env var + if len(s) > 0 and parseSaturatedNatural(s, w) > 0 and w > 0: + return w + w = terminalWidthIoctl([0, 1, 2]) # Try standard file descriptors if w > 0: return w - var cterm = newString(L_ctermid) #Try controlling tty + var cterm = newString(L_ctermid) # Try controlling tty var fd = open(ctermid(cstring(cterm)), O_RDONLY) if fd != -1: w = terminalWidthIoctl([int(fd)]) discard close(fd) if w > 0: return w - var s = getEnv("COLUMNS") #Try standard env var - if len(s) > 0 and parseInt(s, w) > 0 and w > 0: - return w - return 80 #Finally default to venerable value + return 80 # Finally default to venerable value proc terminalHeight*(): int = ## Returns some reasonable terminal height from either standard file ## descriptors, controlling terminal, environment variables or tradition. ## Zero is returned if the height could not be determined. - var h = terminalHeightIoctl([0, 1, 2]) # Try standard file descriptors + # POSIX environment variable takes precendence. + # _LINES_: This variable shall represent a decimal integer >0 used + # to indicate the user's preferred number of lines on a page or + # the vertical screen or window size in lines. A line in this case + # is a vertical measure large enough to hold the tallest character + # in the character set being displayed. If this variable is unset or null, + # the implementation determines the number of lines, appropriate + # for the terminal or window (size, terminal baud rate, and so on), + # in an unspecified manner. + # When LINES is set, any terminal-height information implied by TERM + # is overridden. Users and conforming applications should not set LINES + # unless they wish to override the system selection and produce output + # unrelated to the terminal characteristics. + # See POSIX Base Definitions Section 8.1 Environment Variable Definition + + var h: int + var s = getEnv("LINES") # Try standard env var + if len(s) > 0 and parseSaturatedNatural(s, h) > 0 and h > 0: + return h + h = terminalHeightIoctl([0, 1, 2]) # Try standard file descriptors if h > 0: return h var cterm = newString(L_ctermid) # Try controlling tty var fd = open(ctermid(cstring(cterm)), O_RDONLY) @@ -270,9 +395,6 @@ else: h = terminalHeightIoctl([int(fd)]) discard close(fd) if h > 0: return h - var s = getEnv("LINES") # Try standard env var - if len(s) > 0 and parseInt(s, h) > 0 and h > 0: - return h return 0 # Could not determine height proc terminalSize*(): tuple[w, h: int] = @@ -347,6 +469,9 @@ when defined(windows): proc cursorUp*(f: File, count = 1) = ## Moves the cursor up by `count` rows. + runnableExamples("-r:off"): + stdout.cursorUp(2) + write(stdout, "Hello World!") # anything written at that location will be erased/replaced with this when defined(windows): let h = conHandle(f) var p = getCursorPos(h) @@ -357,6 +482,9 @@ proc cursorUp*(f: File, count = 1) = proc cursorDown*(f: File, count = 1) = ## Moves the cursor down by `count` rows. + runnableExamples("-r:off"): + stdout.cursorDown(2) + write(stdout, "Hello World!") # anything written at that location will be erased/replaced with this when defined(windows): let h = conHandle(f) var p = getCursorPos(h) @@ -367,6 +495,9 @@ proc cursorDown*(f: File, count = 1) = proc cursorForward*(f: File, count = 1) = ## Moves the cursor forward by `count` columns. + runnableExamples("-r:off"): + stdout.cursorForward(2) + write(stdout, "Hello World!") # anything written at that location will be erased/replaced with this when defined(windows): let h = conHandle(f) var p = getCursorPos(h) @@ -377,6 +508,9 @@ proc cursorForward*(f: File, count = 1) = proc cursorBackward*(f: File, count = 1) = ## Moves the cursor backward by `count` columns. + runnableExamples("-r:off"): + stdout.cursorBackward(2) + write(stdout, "Hello World!") # anything written at that location will be erased/replaced with this when defined(windows): let h = conHandle(f) var p = getCursorPos(h) @@ -418,6 +552,9 @@ else: proc eraseLine*(f: File) = ## Erases the entire current line. + runnableExamples("-r:off"): + write(stdout, "never mind") + stdout.eraseLine() # nothing will be printed on the screen when defined(windows): let h = conHandle(f) var scrbuf: CONSOLE_SCREEN_BUFFER_INFO @@ -480,7 +617,7 @@ proc resetAttributes*(f: File) = gBG = 0 type - Style* = enum ## different styles for text output + Style* = enum ## Different styles for text output. styleBright = 1, ## bright text styleDim, ## dim text styleItalic, ## italic (or reverse on terminals not supporting) @@ -534,7 +671,7 @@ proc writeStyled*(txt: string, style: set[Style] = {styleBright}) = stdout.write(ansiStyleCode(gBG)) type - ForegroundColor* = enum ## terminal's foreground colors + ForegroundColor* = enum ## Terminal's foreground colors. fgBlack = 30, ## black fgRed, ## red fgGreen, ## green @@ -546,7 +683,7 @@ type fg8Bit, ## 256-color (not supported, see `enableTrueColors` instead.) fgDefault ## default terminal foreground color - BackgroundColor* = enum ## terminal's background colors + BackgroundColor* = enum ## Terminal's background colors. bgBlack = 40, ## black bgRed, ## red bgGreen, ## green @@ -582,9 +719,9 @@ proc setForegroundColor*(f: File, fg: ForegroundColor, bright = false) = 0, # fg8Bit not supported, see `enableTrueColors` instead. 0] # unused if fg == fgDefault: - discard setConsoleTextAttribute(h, toU16(old or defaultForegroundColor)) + discard setConsoleTextAttribute(h, cast[int16](cast[uint16](old) or cast[uint16](defaultForegroundColor))) else: - discard setConsoleTextAttribute(h, toU16(old or lookup[fg])) + discard setConsoleTextAttribute(h, cast[int16](cast[uint16](old) or cast[uint16](lookup[fg]))) else: gFG = ord(fg) if bright: inc(gFG, 60) @@ -611,9 +748,9 @@ proc setBackgroundColor*(f: File, bg: BackgroundColor, bright = false) = 0, # bg8Bit not supported, see `enableTrueColors` instead. 0] # unused if bg == bgDefault: - discard setConsoleTextAttribute(h, toU16(old or defaultBackgroundColor)) + discard setConsoleTextAttribute(h, cast[int16](cast[uint16](old) or cast[uint16](defaultBackgroundColor))) else: - discard setConsoleTextAttribute(h, toU16(old or lookup[bg])) + discard setConsoleTextAttribute(h, cast[int16](cast[uint16](old) or cast[uint16](lookup[bg]))) else: gBG = ord(bg) if bright: inc(gBG, 60) @@ -701,14 +838,10 @@ macro styledWrite*(f: File, m: varargs[typed]): untyped = ## When some argument is `Style`, `set[Style]`, `ForegroundColor`, ## `BackgroundColor` or `TerminalCmd` then it is not sent directly to ## `f`, but instead corresponding terminal style proc is called. - ## - ## Example: - ## - ## .. code-block:: nim - ## - ## stdout.styledWrite(fgRed, "red text ") - ## stdout.styledWrite(fgGreen, "green text") - ## + runnableExamples("-r:off"): + stdout.styledWrite(fgRed, "red text ") + stdout.styledWrite(fgGreen, "green text") + var reset = false result = newNimNode(nnkStmtList) @@ -731,14 +864,10 @@ macro styledWrite*(f: File, m: varargs[typed]): untyped = template styledWriteLine*(f: File, args: varargs[untyped]) = ## Calls `styledWrite` and appends a newline at the end. - ## - ## Example: - ## - ## .. code-block:: nim - ## - ## proc error(msg: string) = - ## styledWriteLine(stderr, fgRed, "Error: ", resetStyle, msg) - ## + runnableExamples: + proc error(msg: string) = + styledWriteLine(stderr, fgRed, "Error: ", resetStyle, msg) + styledWrite(f, args) write(f, "\n") @@ -747,7 +876,7 @@ template styledEcho*(args: varargs[untyped]) = stdout.styledWriteLine(args) proc getch*(): char = - ## Read a single character from the terminal, blocking until it is entered. + ## Reads a single character from the terminal, blocking until it is entered. ## The character is not printed to the terminal. when defined(windows): let fd = getStdHandle(STD_INPUT_HANDLE) @@ -793,7 +922,7 @@ when defined(windows): stdout.write "\n" else: - import termios + import std/termios proc readPasswordFromStdin*(prompt: string, password: var string): bool {.tags: [ReadIOEffect, WriteIOEffect].} = @@ -849,10 +978,10 @@ proc isTrueColorSupported*(): bool = return getTerminal().trueColorIsSupported when defined(windows): - import os + import std/os proc enableTrueColors*() = - ## Enable true color. + ## Enables true color. var term = getTerminal() when defined(windows): var @@ -885,7 +1014,7 @@ proc enableTrueColors*() = term.trueColorIsEnabled = term.trueColorIsSupported proc disableTrueColors*() = - ## Disable true color. + ## Disables true color. var term = getTerminal() when defined(windows): if term.trueColorIsSupported: @@ -902,51 +1031,3 @@ proc newTerminal(): owned(PTerminal) = new result when defined(windows): initTerminal(result) - -when not defined(testing) and isMainModule: - assert ansiStyleCode(styleBright) == "\e[1m" - assert ansiStyleCode(styleStrikethrough) == "\e[9m" - # exitprocs.addExitProc(resetAttributes) - write(stdout, "never mind") - stdout.eraseLine() - stdout.styledWriteLine({styleBright, styleBlink, styleUnderscore}, "styled text ") - stdout.styledWriteLine("italic text ", {styleItalic}) - stdout.setBackGroundColor(bgCyan, true) - stdout.setForeGroundColor(fgBlue) - stdout.write("blue text in cyan background") - stdout.resetAttributes() - echo "" - stdout.writeLine("ordinary text") - echo "more ordinary text" - styledEcho styleBright, fgGreen, "[PASS]", resetStyle, fgGreen, " Yay!" - echo "ordinary text again" - styledEcho styleBright, fgRed, "[FAIL]", resetStyle, fgRed, " Nay :(" - echo "ordinary text again" - setForeGroundColor(fgGreen) - echo "green text" - echo "more green text" - setForeGroundColor(fgBlue) - echo "blue text" - resetAttributes() - echo "ordinary text" - - stdout.styledWriteLine(fgRed, "red text ") - stdout.styledWriteLine(fgWhite, bgRed, "white text in red background") - stdout.styledWriteLine(" ordinary text ") - stdout.styledWriteLine(fgGreen, "green text") - - writeStyled("underscored text", {styleUnderscore}) - stdout.styledWrite(fgRed, " red text ") - writeStyled("bright text ", {styleBright}) - echo "ordinary text" - - stdout.styledWrite(fgRed, "red text ") - stdout.styledWrite(fgWhite, bgRed, "white text in red background") - stdout.styledWrite(" ordinary text ") - stdout.styledWrite(fgGreen, "green text") - echo "" - echo "ordinary text" - stdout.styledWriteLine(fgRed, "red text ", styleBright, "bold red", fgDefault, " bold text") - stdout.styledWriteLine(bgYellow, "text in yellow bg", styleBright, - " bold text in yellow bg", bgDefault, " bold text") - echo "ordinary text" diff --git a/lib/pure/times.nim b/lib/pure/times.nim index e763b17bb..e59153455 100644 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -20,7 +20,7 @@ Examples ======== - .. code-block:: nim + ```nim import std/[times, os] # Simple benchmarking let time = cpuTime() @@ -37,6 +37,7 @@ # Arithmetic using TimeInterval echo "One year from now : ", now() + 1.years echo "One month from now : ", now() + 1.months + ``` Parsing and Formatting Dates ============================ @@ -44,10 +45,10 @@ The `DateTime` type can be parsed and formatted using the different `parse` and `format` procedures. - .. code-block:: nim - + ```nim let dt = parse("2000-01-01", "yyyy-MM-dd") echo dt.format("yyyy-MM-dd") + ``` The different format patterns that are supported are documented below. @@ -62,6 +63,8 @@ | `Monday -> Mon` `dddd` Full string for the day of the week. | `Saturday -> Saturday` | `Monday -> Monday` + `GG` The last two digits of the Iso Week-Year | `30/12/2012 -> 13` + `GGGG` The Iso week-calendar year padded to four digits | `30/12/2012 -> 2013` `h` The hours in one digit if possible. Ranging from 1-12. | `5pm -> 5` | `2am -> 2` `hh` The hours in two digits always. If the hour is one digit, 0 is prepended. | `5pm -> 05` @@ -104,6 +107,10 @@ | `24 AD -> 24` | `24 BC -> -23` | `12345 AD -> 12345` + `V` The Iso Week-Number as one or two digits | `3/2/2012 -> 5` + | `1/4/2012 -> 13` + `VV` The Iso Week-Number as two digits always. 0 is prepended if one digit. | `3/2/2012 -> 05` + | `1/4/2012 -> 13` `z` Displays the timezone offset from UTC. | `UTC+7 -> +7` | `UTC-5 -> -5` `zz` Same as above but with leading 0. | `UTC+7 -> +07` @@ -124,9 +131,10 @@ =========== ================================================================================= ============================================== Other strings can be inserted by putting them in `''`. For example - `hh'->'mm` will give `01->56`. The following characters can be - inserted without quoting them: `:` `-` `(` `)` `/` `[` `]` - `,`. A literal `'` can be specified with `''`. + `hh'->'mm` will give `01->56`. In addition to spaces, + the following characters can be inserted without quoting them: + `:` `-` `,` `.` `(` `)` `/` `[` `]`. + A literal `'` can be specified with `''`. However you don't need to necessarily separate format patterns, as an unambiguous format string like `yyyyMMddhhmmss` is also valid (although @@ -196,13 +204,17 @@ * `monotimes module <monotimes.html>`_ ]## -import strutils, math, options +import std/[strutils, math, options] import std/private/since include "system/inclrtl" +when defined(nimPreviewSlimSystem): + import std/assertions + + when defined(js): - import jscore + import std/jscore # This is really bad, but overflow checks are broken badly for # ints on the JS backend. See #6752. @@ -226,7 +238,7 @@ when defined(js): {.pop.} elif defined(posix): - import posix + import std/posix type CTime = posix.Time @@ -235,7 +247,7 @@ elif defined(posix): {.importc: "gettimeofday", header: "<sys/time.h>", sideEffect.} elif defined(windows): - import winlean, std/time_t + import std/winlean, std/time_t type CTime = time_t.Time @@ -288,6 +300,13 @@ type YeardayRange* = range[0..365] NanosecondRange* = range[0..999_999_999] + IsoWeekRange* = range[1 .. 53] + ## An ISO 8601 calendar week number. + IsoYear* = distinct int + ## An ISO 8601 calendar year number. + ## + ## .. warning:: The ISO week-based year can correspond to the following or previous year from 29 December to January 3. + Time* = object ## Represents a point in time. seconds: int64 nanosecond: NanosecondRange @@ -313,8 +332,11 @@ type Duration* = object ## Represents a fixed duration of time, meaning a duration ## that has constant length independent of the context. ## - ## To create a new `Duration`, use `initDuration proc + ## To create a new `Duration`, use `initDuration ## <#initDuration,int64,int64,int64,int64,int64,int64,int64,int64>`_. + ## Instead of trying to access the private attributes, use + ## `inSeconds <#inSeconds,Duration>`_ for converting to seconds and + ## `inNanoseconds <#inNanoseconds,Duration>`_ for converting to nanoseconds. seconds: int64 nanosecond: NanosecondRange @@ -356,7 +378,7 @@ type Timezone* = ref object ## \ ## Timezone interface for supporting `DateTime <#DateTime>`_\s of arbitrary ## timezones. The `times` module only supplies implementations for the - ## systems local time and UTC. + ## system's local time and UTC. zonedTimeFromTimeImpl: proc (x: Time): ZonedTime {.tags: [], raises: [], benign.} zonedTimeFromAdjTimeImpl: proc (x: Time): ZonedTime @@ -394,6 +416,16 @@ const unitWeights: array[FixedTimeUnit, int64] = [ 7 * secondsInDay * 1e9.int64, ] +when (NimMajor, NimMinor) >= (1, 4): + # Newer versions of Nim don't track defects + {.pragma: parseFormatRaises, raises: [TimeParseError, TimeFormatParseError].} + {.pragma: parseRaises, raises: [TimeParseError].} +else: + # Still track when using older versions + {.pragma: parseFormatRaises, raises: [TimeParseError, TimeFormatParseError, Defect].} + {.pragma: parseRaises, raises: [TimeParseError, Defect].} + + # # Helper procs # @@ -485,7 +517,7 @@ proc fromEpochDay(epochday: int64): proc getDayOfYear*(monthday: MonthdayRange, month: Month, year: int): YeardayRange {.tags: [], raises: [], benign.} = ## Returns the day of the year. - ## Equivalent with `initDateTime(monthday, month, year, 0, 0, 0).yearday`. + ## Equivalent with `dateTime(year, month, monthday, 0, 0, 0, 0).yearday`. runnableExamples: doAssert getDayOfYear(1, mJan, 2000) == 0 doAssert getDayOfYear(10, mJan, 2000) == 9 @@ -505,7 +537,7 @@ proc getDayOfYear*(monthday: MonthdayRange, month: Month, year: int): proc getDayOfWeek*(monthday: MonthdayRange, month: Month, year: int): WeekDay {.tags: [], raises: [], benign.} = ## Returns the day of the week enum from day, month and year. - ## Equivalent with `initDateTime(monthday, month, year, 0, 0, 0).weekday`. + ## Equivalent with `dateTime(year, month, monthday, 0, 0, 0, 0).weekday`. runnableExamples: doAssert getDayOfWeek(13, mJun, 1990) == dWed doAssert $getDayOfWeek(13, mJun, 1990) == "Wednesday" @@ -513,7 +545,7 @@ proc getDayOfWeek*(monthday: MonthdayRange, month: Month, year: int): WeekDay assertValidDate monthday, month, year # 1970-01-01 is a Thursday, we adjust to the previous Monday let days = toEpochDay(monthday, month, year) - 3 - let weeks = floorDiv(days, 7) + let weeks = floorDiv(days, 7'i64) let wd = days - weeks * 7 # The value of d is 0 for a Sunday, 1 for a Monday, 2 for a Tuesday, etc. # so we must correct for the WeekDay type. @@ -526,12 +558,60 @@ proc getDaysInYear*(year: int): int = doAssert getDaysInYear(2001) == 365 result = 365 + (if isLeapYear(year): 1 else: 0) +proc `==`*(a, b: IsoYear): bool {.borrow.} +proc `$`*(p: IsoYear): string {.borrow.} + +proc getWeeksInIsoYear*(y: IsoYear): IsoWeekRange {.since: (1, 5).} = + ## Returns the number of weeks in the specified ISO 8601 week-based year, which can be + ## either 53 or 52. + runnableExamples: + assert getWeeksInIsoYear(IsoYear(2019)) == 52 + assert getWeeksInIsoYear(IsoYear(2020)) == 53 + + var y = int(y) + + # support negative years + y = if y < 0: 400 + y mod 400 else: y + + # source: https://webspace.science.uu.nl/~gent0113/calendar/isocalendar.htm + let p = (y + (y div 4) - (y div 100) + (y div 400)) mod 7 + let y1 = y - 1 + let p1 = (y1 + (y1 div 4) - (y1 div 100) + (y1 div 400)) mod 7 + if p == 4 or p1 == 3: 53 else: 52 + +proc getIsoWeekAndYear*(dt: DateTime): + tuple[isoweek: IsoWeekRange, isoyear: IsoYear] {.since: (1, 5).} = + ## Returns the ISO 8601 week and year. + ## + ## .. warning:: The ISO week-based year can correspond to the following or previous year from 29 December to January 3. + runnableExamples: + assert getIsoWeekAndYear(initDateTime(21, mApr, 2018, 00, 00, 00)) == (isoweek: 16.IsoWeekRange, isoyear: 2018.IsoYear) + block: + let (w, y) = getIsoWeekAndYear(initDateTime(30, mDec, 2019, 00, 00, 00)) + assert w == 01.IsoWeekRange + assert y == 2020.IsoYear + assert getIsoWeekAndYear(initDateTime(13, mSep, 2020, 00, 00, 00)) == (isoweek: 37.IsoWeekRange, isoyear: 2020.IsoYear) + block: + let (w, y) = getIsoWeekAndYear(initDateTime(2, mJan, 2021, 00, 00, 00)) + assert w.int > 52 + assert w.int < 54 + assert y.int mod 100 == 20 + + # source: https://webspace.science.uu.nl/~gent0113/calendar/isocalendar.htm + var w = (dt.yearday.int - dt.weekday.int + 10) div 7 + if w < 1: + (isoweek: getWeeksInIsoYear(IsoYear(dt.year - 1)), isoyear: IsoYear(dt.year - 1)) + elif (w > getWeeksInIsoYear(IsoYear(dt.year))): + (isoweek: IsoWeekRange(1), isoyear: IsoYear(dt.year + 1)) + else: + (isoweek: IsoWeekRange(w), isoyear: IsoYear(dt.year)) + proc stringifyUnit(value: int | int64, unit: TimeUnit): string = ## Stringify time unit with it's name, lowercased let strUnit = $unit result = "" - result.add($value) - result.add(" ") + result.addInt value + result.add ' ' if abs(value) != 1: result.add(strUnit.toLowerAscii()) else: @@ -574,11 +654,10 @@ template eqImpl(a: Duration|Time, b: Duration|Time): bool = const DurationZero* = Duration() ## \ ## Zero value for durations. Useful for comparisons. - ## - ## .. code-block:: nim - ## + ## ```nim ## doAssert initDuration(seconds = 1) > DurationZero ## doAssert initDuration(seconds = 0) == DurationZero + ## ``` proc initDuration*(nanoseconds, microseconds, milliseconds, seconds, minutes, hours, days, weeks: int64 = 0): Duration = @@ -622,56 +701,56 @@ template convert(dur: Duration, unit: static[FixedTimeUnit]): int64 = convert(Nanoseconds, unit, dur.nanosecond) proc inWeeks*(dur: Duration): int64 = - ## Convert the duration to the number of whole weeks. + ## Converts the duration to the number of whole weeks. runnableExamples: let dur = initDuration(days = 8) doAssert dur.inWeeks == 1 dur.convert(Weeks) proc inDays*(dur: Duration): int64 = - ## Convert the duration to the number of whole days. + ## Converts the duration to the number of whole days. runnableExamples: let dur = initDuration(hours = -50) doAssert dur.inDays == -2 dur.convert(Days) proc inHours*(dur: Duration): int64 = - ## Convert the duration to the number of whole hours. + ## Converts the duration to the number of whole hours. runnableExamples: let dur = initDuration(minutes = 60, days = 2) doAssert dur.inHours == 49 dur.convert(Hours) proc inMinutes*(dur: Duration): int64 = - ## Convert the duration to the number of whole minutes. + ## Converts the duration to the number of whole minutes. runnableExamples: let dur = initDuration(hours = 2, seconds = 10) doAssert dur.inMinutes == 120 dur.convert(Minutes) proc inSeconds*(dur: Duration): int64 = - ## Convert the duration to the number of whole seconds. + ## Converts the duration to the number of whole seconds. runnableExamples: let dur = initDuration(hours = 2, milliseconds = 10) doAssert dur.inSeconds == 2 * 60 * 60 dur.convert(Seconds) proc inMilliseconds*(dur: Duration): int64 = - ## Convert the duration to the number of whole milliseconds. + ## Converts the duration to the number of whole milliseconds. runnableExamples: let dur = initDuration(seconds = -2) doAssert dur.inMilliseconds == -2000 dur.convert(Milliseconds) proc inMicroseconds*(dur: Duration): int64 = - ## Convert the duration to the number of whole microseconds. + ## Converts the duration to the number of whole microseconds. runnableExamples: let dur = initDuration(seconds = -2) doAssert dur.inMicroseconds == -2000000 dur.convert(Microseconds) proc inNanoseconds*(dur: Duration): int64 = - ## Convert the duration to the number of whole nanoseconds. + ## Converts the duration to the number of whole nanoseconds. runnableExamples: let dur = initDuration(seconds = -2) doAssert dur.inNanoseconds == -2000000000 @@ -877,6 +956,7 @@ since((1, 1)): export fromUnixFloat export toUnixFloat + proc fromWinTime*(win: int64): Time = ## Convert a Windows file time (100-nanosecond intervals since ## `1601-01-01T00:00:00Z`) to a `Time`. @@ -890,27 +970,33 @@ proc toWinTime*(t: Time): int64 = ## since `1601-01-01T00:00:00Z`). result = t.seconds * rateDiff + epochDiff + t.nanosecond div 100 +proc getTimeImpl(typ: typedesc[Time]): Time = + discard "implemented in the vm" + proc getTime*(): Time {.tags: [TimeEffect], benign.} = ## Gets the current time as a `Time` with up to nanosecond resolution. - when defined(js): - let millis = newDate().getTime() - let seconds = convert(Milliseconds, Seconds, millis) - let nanos = convert(Milliseconds, Nanoseconds, - millis mod convert(Seconds, Milliseconds, 1).int) - result = initTime(seconds, nanos) - elif defined(macosx): - var a {.noinit.}: Timeval - gettimeofday(a) - result = initTime(a.tv_sec.int64, - convert(Microseconds, Nanoseconds, a.tv_usec.int)) - elif defined(posix): - var ts {.noinit.}: Timespec - discard clock_gettime(CLOCK_REALTIME, ts) - result = initTime(ts.tv_sec.int64, ts.tv_nsec.int) - elif defined(windows): - var f {.noinit.}: FILETIME - getSystemTimeAsFileTime(f) - result = fromWinTime(rdFileTime(f)) + when nimvm: + result = getTimeImpl(Time) + else: + when defined(js): + let millis = newDate().getTime() + let seconds = convert(Milliseconds, Seconds, millis) + let nanos = convert(Milliseconds, Nanoseconds, + millis mod convert(Seconds, Milliseconds, 1).int) + result = initTime(seconds, nanos) + elif defined(macosx): + var a {.noinit.}: Timeval + gettimeofday(a) + result = initTime(a.tv_sec.int64, + convert(Microseconds, Nanoseconds, a.tv_usec.int)) + elif defined(posix): + var ts {.noinit.}: Timespec + discard clock_gettime(CLOCK_REALTIME, ts) + result = initTime(ts.tv_sec.int64, ts.tv_nsec.int) + elif defined(windows): + var f {.noinit.}: FILETIME + getSystemTimeAsFileTime(f) + result = fromWinTime(rdFileTime(f)) proc `-`*(a, b: Time): Duration {.operator, extern: "ntDiffTime".} = ## Computes the duration between two points in time. @@ -955,7 +1041,7 @@ proc high*(typ: typedesc[Time]): Time = initTime(high(int64), high(NanosecondRange)) proc low*(typ: typedesc[Time]): Time = - initTime(low(int64), 0) + initTime(0, 0) # # DateTime & Timezone @@ -1056,13 +1142,13 @@ proc isLeapDay*(dt: DateTime): bool {.since: (1, 1).} = ## Returns whether `t` is a leap day, i.e. Feb 29 in a leap year. This matters ## as it affects time offset calculations. runnableExamples: - let dt = initDateTime(29, mFeb, 2020, 00, 00, 00, utc()) + let dt = dateTime(2020, mFeb, 29, 00, 00, 00, 00, utc()) doAssert dt.isLeapDay doAssert dt+1.years-1.years != dt - let dt2 = initDateTime(28, mFeb, 2020, 00, 00, 00, utc()) + let dt2 = dateTime(2020, mFeb, 28, 00, 00, 00, 00, utc()) doAssert not dt2.isLeapDay doAssert dt2+1.years-1.years == dt2 - doAssertRaises(Exception): discard initDateTime(29, mFeb, 2021, 00, 00, 00, utc()) + doAssertRaises(Exception): discard dateTime(2021, mFeb, 29, 00, 00, 00, 00, utc()) assertDateTimeInitialized dt dt.year.isLeapYear and dt.month == mFeb and dt.monthday == 29 @@ -1141,7 +1227,7 @@ proc name*(zone: Timezone): string = ## If the timezone doesn't exist in the tz database, or if the timezone ## name is unknown, then any string that describes the timezone ## unambiguously might be used. For example, the string "LOCAL" is used - ## for the systems local timezone. + ## for the system's local timezone. ## ## See also: https://en.wikipedia.org/wiki/Tz_database zone.name @@ -1329,14 +1415,13 @@ proc now*(): DateTime {.tags: [TimeEffect], benign.} = ## `cpuTime` instead, depending on the use case. getTime().local -proc initDateTime*(monthday: MonthdayRange, month: Month, year: int, - hour: HourRange, minute: MinuteRange, second: SecondRange, - nanosecond: NanosecondRange, - zone: Timezone = local()): DateTime = +proc dateTime*(year: int, month: Month, monthday: MonthdayRange, + hour: HourRange = 0, minute: MinuteRange = 0, second: SecondRange = 0, + nanosecond: NanosecondRange = 0, + zone: Timezone = local()): DateTime = ## Create a new `DateTime <#DateTime>`_ in the specified timezone. runnableExamples: - let dt1 = initDateTime(30, mMar, 2017, 00, 00, 00, 00, utc()) - doAssert $dt1 == "2017-03-30T00:00:00Z" + assert $dateTime(2017, mMar, 30, zone = utc()) == "2017-03-30T00:00:00Z" assertValidDate monthday, month, year let dt = DateTime( @@ -1352,16 +1437,24 @@ proc initDateTime*(monthday: MonthdayRange, month: Month, year: int, proc initDateTime*(monthday: MonthdayRange, month: Month, year: int, hour: HourRange, minute: MinuteRange, second: SecondRange, - zone: Timezone = local()): DateTime = + nanosecond: NanosecondRange, + zone: Timezone = local()): DateTime {.deprecated: "use `dateTime`".} = ## Create a new `DateTime <#DateTime>`_ in the specified timezone. - runnableExamples: - let dt1 = initDateTime(30, mMar, 2017, 00, 00, 00, utc()) - doAssert $dt1 == "2017-03-30T00:00:00Z" - initDateTime(monthday, month, year, hour, minute, second, 0, zone) + runnableExamples("--warning:deprecated:off"): + assert $initDateTime(30, mMar, 2017, 00, 00, 00, 00, utc()) == "2017-03-30T00:00:00Z" + dateTime(year, month, monthday, hour, minute, second, nanosecond, zone) + +proc initDateTime*(monthday: MonthdayRange, month: Month, year: int, + hour: HourRange, minute: MinuteRange, second: SecondRange, + zone: Timezone = local()): DateTime {.deprecated: "use `dateTime`".} = + ## Create a new `DateTime <#DateTime>`_ in the specified timezone. + runnableExamples("--warning:deprecated:off"): + assert $initDateTime(30, mMar, 2017, 00, 00, 00, utc()) == "2017-03-30T00:00:00Z" + dateTime(year, month, monthday, hour, minute, second, 0, zone) proc `+`*(dt: DateTime, dur: Duration): DateTime = runnableExamples: - let dt = initDateTime(30, mMar, 2017, 00, 00, 00, utc()) + let dt = dateTime(2017, mMar, 30, 00, 00, 00, 00, utc()) let dur = initDuration(hours = 5) doAssert $(dt + dur) == "2017-03-30T05:00:00Z" @@ -1369,7 +1462,7 @@ proc `+`*(dt: DateTime, dur: Duration): DateTime = proc `-`*(dt: DateTime, dur: Duration): DateTime = runnableExamples: - let dt = initDateTime(30, mMar, 2017, 00, 00, 00, utc()) + let dt = dateTime(2017, mMar, 30, 00, 00, 00, 00, utc()) let dur = initDuration(days = 5) doAssert $(dt - dur) == "2017-03-25T00:00:00Z" @@ -1378,8 +1471,8 @@ proc `-`*(dt: DateTime, dur: Duration): DateTime = proc `-`*(dt1, dt2: DateTime): Duration = ## Compute the duration between `dt1` and `dt2`. runnableExamples: - let dt1 = initDateTime(30, mMar, 2017, 00, 00, 00, utc()) - let dt2 = initDateTime(25, mMar, 2017, 00, 00, 00, utc()) + let dt1 = dateTime(2017, mMar, 30, 00, 00, 00, 00, utc()) + let dt2 = dateTime(2017, mMar, 25, 00, 00, 00, 00, utc()) doAssert dt1 - dt2 == initDuration(days = 5) @@ -1406,20 +1499,42 @@ proc `-=`*(a: var DateTime, b: Duration) = a = a - b proc getDateStr*(dt = now()): string {.rtl, extern: "nt$1", tags: [TimeEffect].} = - ## Gets the current local date as a string of the format `YYYY-MM-DD`. + ## Gets the current local date as a string of the format `YYYY-MM-dd`. runnableExamples: echo getDateStr(now() - 1.months) assertDateTimeInitialized dt - result = $dt.year & '-' & intToStr(dt.monthZero, 2) & - '-' & intToStr(dt.monthday, 2) + result = newStringOfCap(10) # len("YYYY-MM-dd") == 10 + result.addInt dt.year + result.add '-' + result.add intToStr(dt.monthZero, 2) + result.add '-' + result.add intToStr(dt.monthday, 2) proc getClockStr*(dt = now()): string {.rtl, extern: "nt$1", tags: [TimeEffect].} = ## Gets the current local clock time as a string of the format `HH:mm:ss`. runnableExamples: echo getClockStr(now() - 1.hours) assertDateTimeInitialized dt - result = intToStr(dt.hour, 2) & ':' & intToStr(dt.minute, 2) & - ':' & intToStr(dt.second, 2) + result = newStringOfCap(8) # len("HH:mm:ss") == 8 + result.add intToStr(dt.hour, 2) + result.add ':' + result.add intToStr(dt.minute, 2) + result.add ':' + result.add intToStr(dt.second, 2) + + +# +# Iso week forward declarations +# + +proc initDateTime*(weekday: WeekDay, isoweek: IsoWeekRange, isoyear: IsoYear, + hour: HourRange, minute: MinuteRange, second: SecondRange, + nanosecond: NanosecondRange, + zone: Timezone = local()): DateTime {.gcsafe, raises: [], tags: [], since: (1, 5).} + +proc initDateTime*(weekday: WeekDay, isoweek: IsoWeekRange, isoyear: IsoYear, + hour: HourRange, minute: MinuteRange, second: SecondRange, + zone: Timezone = local()): DateTime {.gcsafe, raises: [], tags: [], since: (1, 5).} # # TimeFormat @@ -1451,6 +1566,9 @@ type year: Option[int] month: Option[int] monthday: Option[int] + isoyear: Option[int] + yearweek: Option[int] + weekday: Option[WeekDay] utcOffset: Option[int] # '0' as default for these work fine @@ -1465,6 +1583,7 @@ type FormatPattern {.pure.} = enum d, dd, ddd, dddd + GG, GGGG h, hh, H, HH m, mm, M, MM, MMM, MMMM s, ss @@ -1474,6 +1593,7 @@ type YYYY uuuu UUUU + V, VV z, zz, zzz, zzzz ZZZ, ZZZZ g @@ -1512,7 +1632,7 @@ const "Sunday"], ) - FormatLiterals = {' ', '-', '/', ':', '(', ')', '[', ']', ','} + FormatLiterals = {' ', '-', '/', ':', '(', ')', '[', ']', ',', '.'} proc `$`*(f: TimeFormat): string = ## Returns the format string that was used to construct `f`. @@ -1602,6 +1722,8 @@ proc stringToPattern(str: string): FormatPattern = of "dd": result = dd of "ddd": result = ddd of "dddd": result = dddd + of "GG": result = GG + of "GGGG": result = GGGG of "h": result = h of "hh": result = hh of "H": result = H @@ -1624,6 +1746,8 @@ proc stringToPattern(str: string): FormatPattern = of "YYYY": result = YYYY of "uuuu": result = uuuu of "UUUU": result = UUUU + of "V": result = V + of "VV": result = VV of "z": result = z of "zz": result = zz of "zzz": result = zzz @@ -1673,6 +1797,10 @@ proc formatPattern(dt: DateTime, pattern: FormatPattern, result: var string, result.add loc.ddd[dt.weekday] of dddd: result.add loc.dddd[dt.weekday] + of GG: + result.add (dt.getIsoWeekAndYear.isoyear.int mod 100).intToStr(2) + of GGGG: + result.add $dt.getIsoWeekAndYear.isoyear of h: result.add( if dt.hour == 0: "12" @@ -1736,6 +1864,10 @@ proc formatPattern(dt: DateTime, pattern: FormatPattern, result: var string, result.add '+' & $year of UUUU: result.add $dt.year + of V: + result.add $dt.getIsoWeekAndYear.isoweek + of VV: + result.add dt.getIsoWeekAndYear.isoweek.intToStr(2) of z, zz, zzz, zzzz, ZZZ, ZZZZ: if dt.timezone != nil and dt.timezone.name == "Etc/UTC": result.add 'Z' @@ -1790,18 +1922,30 @@ proc parsePattern(input: string, pattern: FormatPattern, i: var int, result = monthday in MonthdayRange of ddd: result = false - for v in loc.ddd: + for d, v in loc.ddd: if input.substr(i, i+v.len-1).cmpIgnoreCase(v) == 0: + parsed.weekday = some(d.WeekDay) result = true i.inc v.len break of dddd: result = false - for v in loc.dddd: + for d, v in loc.dddd: if input.substr(i, i+v.len-1).cmpIgnoreCase(v) == 0: + parsed.weekday = some(d.WeekDay) result = true i.inc v.len break + of GG: + # Assumes current century + var isoyear = takeInt(2..2) + var thisCen = now().year div 100 + parsed.isoyear = some(thisCen*100 + isoyear) + result = isoyear > 0 + of GGGG: + let isoyear = takeInt(1..high(int)) + parsed.isoyear = some(isoyear) + result = isoyear > 0 of h, H: parsed.hour = takeInt(1..2) result = parsed.hour in HourRange @@ -1892,6 +2036,14 @@ proc parsePattern(input: string, pattern: FormatPattern, i: var int, parsed.year = some(year) of UUUU: parsed.year = some(takeInt(1..high(int), allowSign = true)) + of V: + let yearweek = takeInt(1..2) + parsed.yearweek = some(yearweek) + result = yearweek in IsoWeekRange + of VV: + let yearweek = takeInt(2..2) + parsed.yearweek = some(yearweek) + result = yearweek in IsoWeekRange of z, zz, zzz, zzzz, ZZZ, ZZZZ: case input[i] of '+', '-': @@ -1938,7 +2090,7 @@ proc parsePattern(input: string, pattern: FormatPattern, i: var int, i.inc 2 else: result = false - of Lit: doAssert false, "Can't happen" + of Lit: raiseAssert "Can't happen" proc toDateTime(p: ParsedTime, zone: Timezone, f: TimeFormat, input: string): DateTime = @@ -1987,10 +2139,42 @@ proc toDateTime(p: ParsedTime, zone: Timezone, f: TimeFormat, if p.utcOffset.isNone: # No timezone parsed - assume timezone is `zone` - result = initDateTime(monthday, month, year, hour, minute, second, nanosecond, zone) + result = dateTime(year, month, monthday, hour, minute, second, nanosecond, zone) else: # Otherwise convert to `zone` - result = (initDateTime(monthday, month, year, hour, minute, second, nanosecond, utc()).toTime + + result = (dateTime(year, month, monthday, hour, minute, second, nanosecond, utc()).toTime + + initDuration(seconds = p.utcOffset.get())).inZone(zone) + +proc toDateTimeByWeek(p: ParsedTime, zone: Timezone, f: TimeFormat, + input: string): DateTime = + var isoyear = p.isoyear.get(0) + var yearweek = p.yearweek.get(1) + var weekday = p.weekday.get(dMon) + + if p.amPm != apUnknown: + raiseParseException(f, input, "Parsing iso weekyear dates does not support am/pm") + + if p.year.isSome: + raiseParseException(f, input, "Use iso-year GG or GGGG as year with iso week number") + + if p.month.isSome: + raiseParseException(f, input, "Use either iso week number V or VV or month") + + if p.monthday.isSome: + raiseParseException(f, input, "Use weekday ddd or dddd as day with with iso week number") + + if p.isoyear.isNone: + raiseParseException(f, input, "Need iso-year with week number") + + let hour = p.hour + let minute = p.minute + let second = p.second + let nanosecond = p.nanosecond + + if p.utcOffset.isNone: + result = initDateTime(weekday, yearweek.IsoWeekRange, isoyear.IsoYear, hour, minute, second, nanosecond, zone) + else: + result = (initDateTime(weekday, yearweek.IsoWeekRange, isoyear.IsoYear, hour, minute, second, nanosecond, zone).toTime + initDuration(seconds = p.utcOffset.get())).inZone(zone) proc format*(dt: DateTime, f: TimeFormat, @@ -1998,7 +2182,7 @@ proc format*(dt: DateTime, f: TimeFormat, ## Format `dt` using the format specified by `f`. runnableExamples: let f = initTimeFormat("yyyy-MM-dd") - let dt = initDateTime(01, mJan, 2000, 00, 00, 00, utc()) + let dt = dateTime(2000, mJan, 01, 00, 00, 00, 00, utc()) doAssert "2000-01-01" == dt.format(f) assertDateTimeInitialized dt result = "" @@ -2023,7 +2207,7 @@ proc format*(dt: DateTime, f: string, loc: DateTimeLocale = DefaultLocale): stri ## See `Parsing and formatting dates`_ for documentation of the ## `format` argument. runnableExamples: - let dt = initDateTime(01, mJan, 2000, 00, 00, 00, utc()) + let dt = dateTime(2000, mJan, 01, 00, 00, 00, 00, utc()) doAssert "2000-01-01" == format(dt, "yyyy-MM-dd") let dtFormat = initTimeFormat(f) result = dt.format(dtFormat, loc) @@ -2033,7 +2217,7 @@ proc format*(dt: DateTime, f: static[string]): string {.raises: [].} = const f2 = initTimeFormat(f) result = dt.format(f2) -proc formatValue*(result: var string; value: DateTime, specifier: string) = +proc formatValue*(result: var string; value: DateTime | Time, specifier: string) = ## adapter for strformat. Not intended to be called directly. result.add format(value, if specifier.len == 0: "yyyy-MM-dd'T'HH:mm:sszzz" else: specifier) @@ -2046,7 +2230,7 @@ proc format*(time: Time, f: string, zone: Timezone = local()): string ## See `Parsing and formatting dates`_ for documentation of the ## `f` argument. runnableExamples: - var dt = initDateTime(01, mJan, 1970, 00, 00, 00, utc()) + var dt = dateTime(1970, mJan, 01, 00, 00, 00, 00, utc()) var tm = dt.toTime() doAssert format(tm, "yyyy-MM-dd'T'HH:mm:ss", utc()) == "1970-01-01T00:00:00" time.inZone(zone).format(f) @@ -2057,13 +2241,8 @@ proc format*(time: Time, f: static[string], zone: Timezone = local()): string const f2 = initTimeFormat(f) result = time.inZone(zone).format(f2) -template formatValue*(result: var string; value: Time, specifier: string) = - ## adapter for `strformat`. Not intended to be called directly. - result.add format(value, specifier) - proc parse*(input: string, f: TimeFormat, zone: Timezone = local(), - loc: DateTimeLocale = DefaultLocale): DateTime - {.raises: [TimeParseError, Defect].} = + loc: DateTimeLocale = DefaultLocale): DateTime {.parseRaises.} = ## Parses `input` as a `DateTime` using the format specified by `f`. ## If no UTC offset was parsed, then `input` is assumed to be specified in ## the `zone` timezone. If a UTC offset was parsed, the result will be @@ -2072,7 +2251,7 @@ proc parse*(input: string, f: TimeFormat, zone: Timezone = local(), ## Month and day names from the passed in `loc` are used. runnableExamples: let f = initTimeFormat("yyyy-MM-dd") - let dt = initDateTime(01, mJan, 2000, 00, 00, 00, utc()) + let dt = dateTime(2000, mJan, 01, 00, 00, 00, 00, utc()) doAssert dt == "2000-01-01".parse(f, utc()) var inpIdx = 0 # Input index var patIdx = 0 # Pattern index @@ -2103,31 +2282,33 @@ proc parse*(input: string, f: TimeFormat, zone: Timezone = local(), raiseParseException(f, input, "Parsing ended but there was still patterns remaining") - result = toDateTime(parsed, zone, f, input) + if parsed.yearweek.isSome: + result = toDateTimeByWeek(parsed, zone, f, input) + elif parsed.isoyear.isSome: + raiseParseException(f, input, "Iso year GG or GGGG require iso week V or VV") + else: + result = toDateTime(parsed, zone, f, input) proc parse*(input, f: string, tz: Timezone = local(), - loc: DateTimeLocale = DefaultLocale): DateTime - {.raises: [TimeParseError, TimeFormatParseError, Defect].} = + loc: DateTimeLocale = DefaultLocale): DateTime {.parseFormatRaises.} = ## Shorthand for constructing a `TimeFormat` and using it to parse ## `input` as a `DateTime`. ## ## See `Parsing and formatting dates`_ for documentation of the ## `f` argument. runnableExamples: - let dt = initDateTime(01, mJan, 2000, 00, 00, 00, utc()) + let dt = dateTime(2000, mJan, 01, 00, 00, 00, 00, utc()) doAssert dt == parse("2000-01-01", "yyyy-MM-dd", utc()) let dtFormat = initTimeFormat(f) result = input.parse(dtFormat, tz, loc = loc) proc parse*(input: string, f: static[string], zone: Timezone = local(), - loc: DateTimeLocale = DefaultLocale): - DateTime {.raises: [TimeParseError, Defect].} = + loc: DateTimeLocale = DefaultLocale): DateTime {.parseRaises.} = ## Overload that validates `f` at compile time. const f2 = initTimeFormat(f) result = input.parse(f2, zone, loc = loc) -proc parseTime*(input, f: string, zone: Timezone): Time - {.raises: [TimeParseError, TimeFormatParseError, Defect].} = +proc parseTime*(input, f: string, zone: Timezone): Time {.parseFormatRaises.} = ## Shorthand for constructing a `TimeFormat` and using it to parse ## `input` as a `DateTime`, then converting it a `Time`. ## @@ -2139,7 +2320,7 @@ proc parseTime*(input, f: string, zone: Timezone): Time parse(input, f, zone).toTime() proc parseTime*(input: string, f: static[string], zone: Timezone): Time - {.raises: [TimeParseError, Defect].} = + {.parseRaises.} = ## Overload that validates `format` at compile time. const f2 = initTimeFormat(f) result = input.parse(f2, zone).toTime() @@ -2148,7 +2329,7 @@ proc `$`*(dt: DateTime): string {.tags: [], raises: [], benign.} = ## Converts a `DateTime` object to a string representation. ## It uses the format `yyyy-MM-dd'T'HH:mm:sszzz`. runnableExamples: - let dt = initDateTime(01, mJan, 2000, 12, 00, 00, utc()) + let dt = dateTime(2000, mJan, 01, 12, 00, 00, 00, utc()) doAssert $dt == "2000-01-01T12:00:00Z" doAssert $default(DateTime) == "Uninitialized DateTime" if not dt.isInitialized: @@ -2160,7 +2341,7 @@ proc `$`*(time: Time): string {.tags: [], raises: [], benign.} = ## Converts a `Time` value to a string representation. It will use the local ## time zone and use the format `yyyy-MM-dd'T'HH:mm:sszzz`. runnableExamples: - let dt = initDateTime(01, mJan, 1970, 00, 00, 00, local()) + let dt = dateTime(1970, mJan, 01, 00, 00, 00, 00, local()) let tm = dt.toTime() doAssert $tm == "1970-01-01T00:00:00" & format(dt, "zzz") $time.local @@ -2182,7 +2363,7 @@ proc initTimeInterval*(nanoseconds, microseconds, milliseconds, ## `seconds`, `minutes`, `hours`, `days`, `months`, and `years`. runnableExamples: let day = initTimeInterval(hours = 24) - let dt = initDateTime(01, mJan, 2000, 12, 00, 00, utc()) + let dt = dateTime(2000, mJan, 01, 12, 00, 00, 00, utc()) doAssert $(dt + day) == "2000-01-02T12:00:00Z" doAssert initTimeInterval(hours = 24) != initTimeInterval(days = 1) result.nanoseconds = nanoseconds @@ -2268,8 +2449,8 @@ proc between*(startDt, endDt: DateTime): TimeInterval = ## - If `startDt.timezone != endDt.timezone`, then the result will be ## equivalent to `between(startDt.utc, endDt.utc)`. runnableExamples: - var a = initDateTime(25, mMar, 2015, 12, 0, 0, utc()) - var b = initDateTime(1, mApr, 2017, 15, 0, 15, utc()) + var a = dateTime(2015, mMar, 25, 12, 0, 0, 00, utc()) + var b = dateTime(2017, mApr, 1, 15, 0, 15, 00, utc()) var ti = initTimeInterval(years = 2, weeks = 1, hours = 3, seconds = 15) doAssert between(a, b) == ti doAssert between(a, b) == -between(b, a) @@ -2349,8 +2530,8 @@ proc between*(startDt, endDt: DateTime): TimeInterval = startDate = endDate # Handle hours, minutes, seconds, milliseconds, microseconds and nanoseconds - let newStartDt = initDateTime(startDate.monthday, startDate.month.Month, - startDate.year, startDt.hour, startDt.minute, startDt.second, + let newStartDt = dateTime(startDate.year, startDate.month.Month, + startDate.monthday, startDt.hour, startDt.minute, startDt.second, startDt.nanosecond, startDt.timezone) let dur = endDt - newStartDt let parts = toParts(dur) @@ -2501,7 +2682,7 @@ proc `+`*(dt: DateTime, interval: TimeInterval): DateTime = ## So adding one month to `31 October` will result in `31 November`, which ## will overflow and result in `1 December`. runnableExamples: - let dt = initDateTime(30, mMar, 2017, 00, 00, 00, utc()) + let dt = dateTime(2017, mMar, 30, 00, 00, 00, 00, utc()) doAssert $(dt + 1.months) == "2017-04-30T00:00:00Z" # This is correct and happens due to monthday overflow. doAssert $(dt - 1.months) == "2017-03-02T00:00:00Z" @@ -2524,7 +2705,7 @@ proc `-`*(dt: DateTime, interval: TimeInterval): DateTime = ## then the `months` component and so on. The returned `DateTime` will ## have the same timezone as the input. runnableExamples: - let dt = initDateTime(30, mMar, 2017, 00, 00, 00, utc()) + let dt = dateTime(2017, mMar, 30, 00, 00, 00, 00, utc()) doAssert $(dt - 5.days) == "2017-03-25T00:00:00Z" dt + (-interval) @@ -2568,6 +2749,33 @@ proc `-=`*(t: var Time, b: TimeInterval) = t = t - b # +# Iso week +# + +proc initDateTime*(weekday: WeekDay, isoweek: IsoWeekRange, isoyear: IsoYear, + hour: HourRange, minute: MinuteRange, second: SecondRange, + nanosecond: NanosecondRange, + zone: Timezone = local()): DateTime {.raises: [], tags: [], since: (1, 5).} = + ## Create a new `DateTime <#DateTime>`_ from a weekday and an ISO 8601 week number and year + ## in the specified timezone. + ## + ## .. warning:: The ISO week-based year can correspond to the following or previous year from 29 December to January 3. + runnableExamples: + assert initDateTime(21, mApr, 2018, 00, 00, 00) == initDateTime(dSat, 16, 2018.IsoYear, 00, 00, 00) + assert initDateTime(30, mDec, 2019, 00, 00, 00) == initDateTime(dMon, 01, 2020.IsoYear, 00, 00, 00) + assert initDateTime(13, mSep, 2020, 00, 00, 00) == initDateTime(dSun, 37, 2020.IsoYear, 00, 00, 00) + assert initDateTime(2, mJan, 2021, 00, 00, 00) == initDateTime(dSat, 53, 2020.IsoYear, 00, 00, 00) + + # source https://webspace.science.uu.nl/~gent0113/calendar/isocalendar.htm + let d = isoweek * 7 + weekday.int - initDateTime(4, mJan, isoyear.int, 00, 00, 00, zone).weekday.int - 4 + initDateTime(1, mJan, isoyear.int, hour, minute, second, nanosecond, zone) + initTimeInterval(days=d) + +proc initDateTime*(weekday: WeekDay, isoweek: IsoWeekRange, isoyear: IsoYear, + hour: HourRange, minute: MinuteRange, second: SecondRange, + zone: Timezone = local()): DateTime {.raises: [], tags: [], since: (1, 5).} = + initDateTime(weekday, isoweek, isoyear, hour, minute, second, 0, zone) + +# # Other # @@ -2580,7 +2788,9 @@ proc epochTime*(): float {.tags: [TimeEffect].} = ## ## .. warning:: Unsuitable for benchmarking (but still better than `now`), ## use `monotimes.getMonoTime` or `cpuTime` instead, depending on the use case. - when defined(macosx): + when defined(js): + result = newDate().getTime() / 1000 + elif defined(macosx): var a {.noinit.}: Timeval gettimeofday(a) result = toBiggestFloat(a.tv_sec.int64) + toBiggestFloat( @@ -2597,8 +2807,6 @@ proc epochTime*(): float {.tags: [TimeEffect].} = var secs = i64 div rateDiff var subsecs = i64 mod rateDiff result = toFloat(int(secs)) + toFloat(int(subsecs)) * 0.0000001 - elif defined(js): - result = newDate().getTime() / 1000 else: {.error: "unknown OS".} diff --git a/lib/pure/typetraits.nim b/lib/pure/typetraits.nim index 6827e23b1..78af84fdd 100644 --- a/lib/pure/typetraits.nim +++ b/lib/pure/typetraits.nim @@ -15,6 +15,10 @@ import std/private/since export system.`$` # for backward compatibility +when defined(nimPreviewSlimSystem): + import std/assertions + + type HoleyEnum* = (not Ordinal) and enum ## Enum with holes. type OrdinalEnum* = Ordinal and enum ## Enum without holes. @@ -31,7 +35,7 @@ runnableExamples: assert C[float] is HoleyEnum proc name*(t: typedesc): string {.magic: "TypeTrait".} = - ## Returns the name of the given type. + ## Returns the name of `t`. ## ## Alias for `system.\`$\`(t) <dollars.html#$,typedesc>`_ since Nim v0.20. runnableExamples: @@ -39,7 +43,7 @@ proc name*(t: typedesc): string {.magic: "TypeTrait".} = doAssert name(seq[string]) == "seq[string]" proc arity*(t: typedesc): int {.magic: "TypeTrait".} = - ## Returns the arity of the given type. This is the number of "type" + ## Returns the arity of `t`. This is the number of "type" ## components or the number of generic parameters a given type `t` has. runnableExamples: doAssert arity(int) == 0 @@ -88,11 +92,27 @@ proc stripGenericParams*(t: typedesc): typedesc {.magic: "TypeTrait".} = doAssert stripGenericParams(int) is int proc supportsCopyMem*(t: typedesc): bool {.magic: "TypeTrait".} - ## This trait returns true if the type `t` is safe to use for - ## `copyMem`:idx:. + ## Returns true if `t` is safe to use for `copyMem`:idx:. ## ## Other languages name a type like these `blob`:idx:. +proc hasDefaultValue*(t: typedesc): bool {.magic: "TypeTrait".} = + ## Returns true if `t` has a valid default value. + runnableExamples: + {.experimental: "strictNotNil".} + type + NilableObject = ref object + a: int + Object = NilableObject not nil + RequiresInit[T] = object + a {.requiresInit.}: T + + assert hasDefaultValue(NilableObject) + assert not hasDefaultValue(Object) + assert hasDefaultValue(string) + assert not hasDefaultValue(var string) + assert not hasDefaultValue(RequiresInit[int]) + proc isNamedTuple*(T: typedesc): bool {.magic: "TypeTrait".} = ## Returns true for named tuples, false for any other type. runnableExamples: @@ -100,25 +120,76 @@ proc isNamedTuple*(T: typedesc): bool {.magic: "TypeTrait".} = doAssert not isNamedTuple((string, int)) doAssert isNamedTuple(tuple[name: string, age: int]) -proc distinctBase*(T: typedesc): typedesc {.magic: "TypeTrait".} = +template pointerBase*[T](_: typedesc[ptr T | ref T]): typedesc = + ## Returns `T` for `ref T | ptr T`. + runnableExamples: + assert (ref int).pointerBase is int + type A = ptr seq[float] + assert A.pointerBase is seq[float] + assert (ref A).pointerBase is A # not seq[float] + assert (var s = "abc"; s[0].addr).typeof.pointerBase is char + T + +proc rangeBase*(T: typedesc[range]): typedesc {.magic: "TypeTrait".} = + ## Returns the base type for range types, or the type itself otherwise. + ## + ## **See also:** + ## * `rangeBase template <#rangeBase.t,T>`_ + runnableExamples: + type MyRange = range[0..5] + type MyEnum = enum a, b, c + type MyEnumRange = range[b..c] + doAssert rangeBase(MyRange) is int + doAssert rangeBase(MyEnumRange) is MyEnum + doAssert rangeBase(range['a'..'z']) is char + +template rangeBase*[T: range](a: T): untyped = + ## Overload of `rangeBase <#rangeBase,typedesc,static[bool]>`_ for values. + runnableExamples: + type MyRange = range[0..5] + type MyEnum = enum a, b, c + type MyEnumRange = range[b..c] + let x = MyRange(3) + doAssert rangeBase(x) is int + doAssert $typeof(rangeBase(x)) == "int" + doAssert rangeBase(x) == 3 + let y: set[MyEnumRange] = {c} + for e in y: + doAssert rangeBase(e) is MyEnum + doAssert $typeof(rangeBase(e)) == "MyEnum" + doAssert rangeBase(e) == c + let z: seq[range['a'..'z']] = @['c'] + doAssert rangeBase(z[0]) is char + doAssert $typeof(rangeBase(z[0])) == "char" + doAssert rangeBase(z[0]) == 'c' + rangeBase(typeof(T))(a) + +proc distinctBase*(T: typedesc, recursive: static bool = true): typedesc {.magic: "TypeTrait".} = ## Returns the base type for distinct types, or the type itself otherwise. + ## If `recursive` is false, only the immediate distinct base will be returned. ## ## **See also:** - ## * `distinctBase template <#distinctBase.t,T>`_ + ## * `distinctBase template <#distinctBase.t,T,static[bool]>`_ runnableExamples: type MyInt = distinct int + type MyOtherInt = distinct MyInt doAssert distinctBase(MyInt) is int + doAssert distinctBase(MyOtherInt) is int + doAssert distinctBase(MyOtherInt, false) is MyInt doAssert distinctBase(int) is int since (1, 1): - template distinctBase*[T](a: T): untyped = - ## Overload of `distinctBase <#distinctBase,typedesc>`_ for values. + template distinctBase*[T](a: T, recursive: static bool = true): untyped = + ## Overload of `distinctBase <#distinctBase,typedesc,static[bool]>`_ for values. runnableExamples: type MyInt = distinct int + type MyOtherInt = distinct MyInt doAssert 12.MyInt.distinctBase == 12 + doAssert 12.MyOtherInt.distinctBase == 12 + doAssert 12.MyOtherInt.distinctBase(false) is MyInt doAssert 12.distinctBase == 12 when T is distinct: - distinctBase(typeof(a))(a) + distinctBase(typeof(a), recursive)(a) else: # avoids hint ConvFromXtoItselfNotNeeded a @@ -205,7 +276,7 @@ macro genericParamsImpl(T: typedesc): untyped = case ai.typeKind of ntyTypeDesc: ret = ai - of ntyStatic: doAssert false + of ntyStatic: raiseAssert "unreachable" else: # getType from a resolved symbol might return a typedesc symbol. # If so, use it directly instead of wrapping it in StaticParam. @@ -263,3 +334,43 @@ since (1, 1): type T2 = T genericParamsImpl(T2) + + +proc hasClosureImpl(n: NimNode): bool = discard "see compiler/vmops.nim" + +proc hasClosure*(fn: NimNode): bool {.since: (1, 5, 1).} = + ## Returns true if the func/proc/etc `fn` has `closure`. + ## `fn` has to be a resolved symbol of kind `nnkSym`. This + ## implies that the macro that calls this proc should accept `typed` + ## arguments and not `untyped` arguments. + expectKind fn, nnkSym + result = hasClosureImpl(fn) + +template toUnsigned*(T: typedesc[SomeInteger and not range]): untyped = + ## Returns an unsigned type with same bit size as `T`. + runnableExamples: + assert int8.toUnsigned is uint8 + assert uint.toUnsigned is uint + assert int.toUnsigned is uint + # range types are currently unsupported: + assert not compiles(toUnsigned(range[0..7])) + when T is int8: uint8 + elif T is int16: uint16 + elif T is int32: uint32 + elif T is int64: uint64 + elif T is int: uint + else: T + +template toSigned*(T: typedesc[SomeInteger and not range]): untyped = + ## Returns a signed type with same bit size as `T`. + runnableExamples: + assert int8.toSigned is int8 + assert uint16.toSigned is int16 + # range types are currently unsupported: + assert not compiles(toSigned(range[0..7])) + when T is uint8: int8 + elif T is uint16: int16 + elif T is uint32: int32 + elif T is uint64: int64 + elif T is uint: int + else: T diff --git a/lib/pure/unicode.nim b/lib/pure/unicode.nim index 903f01fb4..8cbe117bb 100644 --- a/lib/pure/unicode.nim +++ b/lib/pure/unicode.nim @@ -21,6 +21,16 @@ ## * `encodings module <encodings.html>`_ include "system/inclrtl" +import std/strbasics +template toOa(s: string): auto = s.toOpenArray(0, s.high) + +proc substr(s: openArray[char] , first, last: int): string = + # Copied substr from system + let first = max(first, 0) + let L = max(min(last, high(s)) - first + 1, 0) + result = newString(L) + for i in 0 .. L-1: + result[i] = s[i+first] type RuneImpl = int32 # underlying type of Rune @@ -32,7 +42,7 @@ type template ones(n: untyped): untyped = ((1 shl n)-1) -proc runeLen*(s: string): int {.rtl, extern: "nuc$1".} = +proc runeLen*(s: openArray[char]): int {.rtl, extern: "nuc$1".} = ## Returns the number of runes of the string ``s``. runnableExamples: let a = "añyóng" @@ -51,7 +61,7 @@ proc runeLen*(s: string): int {.rtl, extern: "nuc$1".} = else: inc i inc(result) -proc runeLenAt*(s: string, i: Natural): int = +proc runeLenAt*(s: openArray[char], i: Natural): int = ## Returns the number of bytes the rune starting at ``s[i]`` takes. ## ## See also: @@ -71,7 +81,7 @@ proc runeLenAt*(s: string, i: Natural): int = const replRune = Rune(0xFFFD) -template fastRuneAt*(s: string, i: int, result: untyped, doInc = true) = +template fastRuneAt*(s: openArray[char] or string, i: int, result: untyped, doInc = true) = ## Returns the rune ``s[i]`` in ``result``. ## ## If ``doInc == true`` (default), ``i`` is incremented by the number @@ -149,7 +159,7 @@ template fastRuneAt*(s: string, i: int, result: untyped, doInc = true) = result = Rune(uint(s[i])) when doInc: inc(i) -proc runeAt*(s: string, i: Natural): Rune = +proc runeAt*(s: openArray[char], i: Natural): Rune = ## Returns the rune in ``s`` at **byte index** ``i``. ## ## See also: @@ -163,7 +173,7 @@ proc runeAt*(s: string, i: Natural): Rune = doAssert a.runeAt(3) == "y".runeAt(0) fastRuneAt(s, i, result, false) -proc validateUtf8*(s: string): int = +proc validateUtf8*(s: openArray[char]): int = ## Returns the position of the invalid byte in ``s`` if the string ``s`` does ## not hold valid UTF-8 data. Otherwise ``-1`` is returned. ## @@ -300,7 +310,7 @@ proc `$`*(runes: seq[Rune]): string = for rune in runes: result.add rune -proc runeOffset*(s: string, pos: Natural, start: Natural = 0): int = +proc runeOffset*(s: openArray[char], pos: Natural, start: Natural = 0): int = ## Returns the byte position of rune ## at position ``pos`` in ``s`` with an optional start byte position. ## Returns the special value -1 if it runs out of the string. @@ -327,13 +337,13 @@ proc runeOffset*(s: string, pos: Natural, start: Natural = 0): int = inc i return o -proc runeReverseOffset*(s: string, rev: Positive): (int, int) = +proc runeReverseOffset*(s: openArray[char], rev: Positive): (int, int) = ## Returns a tuple with the byte offset of the ## rune at position ``rev`` in ``s``, counting ## from the end (starting with 1) and the total ## number of runes in the string. ## - ## Returns a negative value for offset if there are to few runes in + ## Returns a negative value for offset if there are too few runes in ## the string to satisfy the request. ## ## **Beware:** This can lead to unoptimized code and slow execution! @@ -346,18 +356,16 @@ proc runeReverseOffset*(s: string, rev: Positive): (int, int) = a = rev.int o = 0 x = 0 + let times = 2*rev.int-s.runeLen # transformed from rev.int - a < s.runeLen - rev.int while o < s.len: let r = runeLenAt(s, o) o += r - if a < 0: + if a > times: x += r dec a + result = if a > 0: (-a, rev.int-a) else: (x, -a+rev.int) - if a > 0: - return (-a, rev.int-a) - return (x, -a+rev.int) - -proc runeAtPos*(s: string, pos: int): Rune = +proc runeAtPos*(s: openArray[char], pos: int): Rune = ## Returns the rune at position ``pos``. ## ## **Beware:** This can lead to unoptimized code and slow execution! @@ -370,7 +378,7 @@ proc runeAtPos*(s: string, pos: int): Rune = ## * `fastRuneAt template <#fastRuneAt.t,string,int,untyped>`_ fastRuneAt(s, runeOffset(s, pos), result, false) -proc runeStrAtPos*(s: string, pos: Natural): string = +proc runeStrAtPos*(s: openArray[char], pos: Natural): string = ## Returns the rune at position ``pos`` as UTF8 String. ## ## **Beware:** This can lead to unoptimized code and slow execution! @@ -382,9 +390,9 @@ proc runeStrAtPos*(s: string, pos: Natural): string = ## * `runeAtPos proc <#runeAtPos,string,int>`_ ## * `fastRuneAt template <#fastRuneAt.t,string,int,untyped>`_ let o = runeOffset(s, pos) - s[o .. (o+runeLenAt(s, o)-1)] + substr(s.toOpenArray(o, (o+runeLenAt(s, o)-1))) -proc runeSubStr*(s: string, pos: int, len: int = int.high): string = +proc runeSubStr*(s: openArray[char], pos: int, len: int = int.high): string = ## Returns the UTF-8 substring starting at code point ``pos`` ## with ``len`` code points. ## @@ -403,7 +411,7 @@ proc runeSubStr*(s: string, pos: int, len: int = int.high): string = if pos < 0: let (o, rl) = runeReverseOffset(s, -pos) if len >= rl: - result = s.substr(o, s.len-1) + result = s.substr(o, s.high) elif len < 0: let e = rl + len if e < 0: @@ -456,7 +464,7 @@ proc `==`*(a, b: Rune): bool = include "includes/unicode_ranges" -proc binarySearch(c: RuneImpl, tab: openArray[int], len, stride: int): int = +proc binarySearch(c: RuneImpl, tab: openArray[int32], len, stride: int): int = var n = len var t = 0 while n > 1: @@ -628,7 +636,7 @@ template runeCheck(s, runeProc) = fastRuneAt(s, i, rune, doInc = true) result = runeProc(rune) and result -proc isAlpha*(s: string): bool {.noSideEffect, +proc isAlpha*(s: openArray[char]): bool {.noSideEffect, rtl, extern: "nuc$1Str".} = ## Returns true if ``s`` contains all alphabetic runes. runnableExamples: @@ -636,7 +644,7 @@ proc isAlpha*(s: string): bool {.noSideEffect, doAssert a.isAlpha runeCheck(s, isAlpha) -proc isSpace*(s: string): bool {.noSideEffect, +proc isSpace*(s: openArray[char]): bool {.noSideEffect, rtl, extern: "nuc$1Str".} = ## Returns true if ``s`` contains all whitespace runes. runnableExamples: @@ -657,21 +665,21 @@ template convertRune(s, runeProc) = rune = runeProc(rune) fastToUTF8Copy(rune, result, resultIndex, doInc = true) -proc toUpper*(s: string): string {.noSideEffect, +proc toUpper*(s: openArray[char]): string {.noSideEffect, rtl, extern: "nuc$1Str".} = ## Converts ``s`` into upper-case runes. runnableExamples: doAssert toUpper("abγ") == "ABΓ" convertRune(s, toUpper) -proc toLower*(s: string): string {.noSideEffect, +proc toLower*(s: openArray[char]): string {.noSideEffect, rtl, extern: "nuc$1Str".} = ## Converts ``s`` into lower-case runes. runnableExamples: doAssert toLower("ABΓ") == "abγ" convertRune(s, toLower) -proc swapCase*(s: string): string {.noSideEffect, +proc swapCase*(s: openArray[char]): string {.noSideEffect, rtl, extern: "nuc$1".} = ## Swaps the case of runes in ``s``. ## @@ -693,7 +701,7 @@ proc swapCase*(s: string): string {.noSideEffect, rune = rune.toUpper() fastToUTF8Copy(rune, result, resultIndex, doInc = true) -proc capitalize*(s: string): string {.noSideEffect, +proc capitalize*(s: openArray[char]): string {.noSideEffect, rtl, extern: "nuc$1".} = ## Converts the first character of ``s`` into an upper-case rune. runnableExamples: @@ -705,10 +713,13 @@ proc capitalize*(s: string): string {.noSideEffect, rune: Rune i = 0 fastRuneAt(s, i, rune, doInc = true) - result = $toUpper(rune) & substr(s, i) + result = $toUpper(rune) & substr(s.toOpenArray(i, s.high)) -proc translate*(s: string, replacements: proc(key: string): string): string {. - rtl, extern: "nuc$1".} = +when not defined(nimHasEffectsOf): + {.pragma: effectsOf.} + +proc translate*(s: openArray[char], replacements: proc(key: string): string): string {. + rtl, extern: "nuc$1", effectsOf: replacements.} = ## Translates words in a string using the ``replacements`` proc to substitute ## words inside ``s`` with their replacements. ## @@ -742,7 +753,7 @@ proc translate*(s: string, replacements: proc(key: string): string): string {. if whiteSpace and inWord: # If we've reached the end of a word - let word = s[wordStart ..< lastIndex] + let word = substr(s.toOpenArray(wordStart, lastIndex - 1)) result.add(replacements(word)) result.add($rune) inWord = false @@ -757,10 +768,10 @@ proc translate*(s: string, replacements: proc(key: string): string): string {. if wordStart < len(s) and inWord: # Get the trailing word at the end - let word = s[wordStart .. ^1] + let word = substr(s.toOpenArray(wordStart, s.high)) result.add(replacements(word)) -proc title*(s: string): string {.noSideEffect, +proc title*(s: openArray[char]): string {.noSideEffect, rtl, extern: "nuc$1".} = ## Converts ``s`` to a unicode title. ## @@ -786,7 +797,7 @@ proc title*(s: string): string {.noSideEffect, fastToUTF8Copy(rune, result, resultIndex, doInc = true) -iterator runes*(s: string): Rune = +iterator runes*(s: openArray[char]): Rune = ## Iterates over any rune of the string ``s`` returning runes. var i = 0 @@ -795,7 +806,7 @@ iterator runes*(s: string): Rune = fastRuneAt(s, i, result, true) yield result -iterator utf8*(s: string): string = +iterator utf8*(s: openArray[char]): string = ## Iterates over any rune of the string ``s`` returning utf8 values. ## ## See also: @@ -806,10 +817,10 @@ iterator utf8*(s: string): string = var o = 0 while o < s.len: let n = runeLenAt(s, o) - yield s[o .. (o+n-1)] + yield substr(s.toOpenArray(o, (o+n-1))) o += n -proc toRunes*(s: string): seq[Rune] = +proc toRunes*(s: openArray[char]): seq[Rune] = ## Obtains a sequence containing the Runes in ``s``. ## ## See also: @@ -822,12 +833,12 @@ proc toRunes*(s: string): seq[Rune] = for r in s.runes: result.add(r) -proc cmpRunesIgnoreCase*(a, b: string): int {.rtl, extern: "nuc$1".} = +proc cmpRunesIgnoreCase*(a, b: openArray[char]): int {.rtl, extern: "nuc$1".} = ## Compares two UTF-8 strings and ignores the case. Returns: ## - ## | 0 if a == b - ## | < 0 if a < b - ## | > 0 if a > b + ## | `0` if a == b + ## | `< 0` if a < b + ## | `> 0` if a > b var i = 0 var j = 0 var ar, br: Rune @@ -835,11 +846,16 @@ proc cmpRunesIgnoreCase*(a, b: string): int {.rtl, extern: "nuc$1".} = # slow path: fastRuneAt(a, i, ar) fastRuneAt(b, j, br) - result = RuneImpl(toLower(ar)) - RuneImpl(toLower(br)) + when sizeof(int) < 4: + const lo = low(int).int32 + const hi = high(int).int32 + result = clamp(RuneImpl(toLower(ar)) - RuneImpl(toLower(br)), lo, hi).int + else: + result = RuneImpl(toLower(ar)) - RuneImpl(toLower(br)) if result != 0: return result = a.len - b.len -proc reversed*(s: string): string = +proc reversed*(s: openArray[char]): string = ## Returns the reverse of ``s``, interpreting it as runes. ## ## Unicode combining characters are correctly interpreted as well. @@ -874,9 +890,9 @@ proc reversed*(s: string): string = reverseUntil(len(s)) -proc graphemeLen*(s: string; i: Natural): Natural = +proc graphemeLen*(s: openArray[char]; i: Natural): Natural = ## The number of bytes belonging to byte index ``s[i]``, - ## including following combining code unit. + ## including following combining code units. runnableExamples: let a = "añyóng" doAssert a.graphemeLen(1) == 2 ## ñ @@ -893,7 +909,7 @@ proc graphemeLen*(s: string; i: Natural): Natural = if not isCombining(r2): break result = j-i -proc lastRune*(s: string; last: int): (Rune, int) = +proc lastRune*(s: openArray[char]; last: int): (Rune, int) = ## Length of the last rune in ``s[0..last]``. Returns the rune and its length ## in bytes. if s[last] <= chr(127): @@ -922,12 +938,12 @@ proc size*(r: Rune): int {.noSideEffect.} = else: result = 1 # --------- Private templates for different split separators ----------- -proc stringHasSep(s: string, index: int, seps: openArray[Rune]): bool = +proc stringHasSep(s: openArray[char], index: int, seps: openArray[Rune]): bool = var rune: Rune fastRuneAt(s, index, rune, false) return seps.contains(rune) -proc stringHasSep(s: string, index: int, sep: Rune): bool = +proc stringHasSep(s: openArray[char], index: int, sep: Rune): bool = var rune: Rune fastRuneAt(s, index, rune, false) return sep == rune @@ -945,12 +961,12 @@ template splitCommon(s, sep, maxsplit: untyped) = while last < sLen and not stringHasSep(s, last, sep): inc(last, runeLenAt(s, last)) if splits == 0: last = sLen - yield s[first .. (last - 1)] + yield substr(s.toOpenArray(first, (last - 1))) if splits == 0: break dec(splits) inc(last, if last < sLen: runeLenAt(s, last) else: 1) -iterator split*(s: string, seps: openArray[Rune] = unicodeSpaces, +iterator split*(s: openArray[char], seps: openArray[Rune] = unicodeSpaces, maxsplit: int = -1): string = ## Splits the unicode string ``s`` into substrings using a group of separators. ## @@ -976,7 +992,7 @@ iterator split*(s: string, seps: openArray[Rune] = unicodeSpaces, splitCommon(s, seps, maxsplit) -iterator splitWhitespace*(s: string): string = +iterator splitWhitespace*(s: openArray[char]): string = ## Splits a unicode string at whitespace runes. splitCommon(s, unicodeSpaces, -1) @@ -984,13 +1000,13 @@ template accResult(iter: untyped) = result = @[] for x in iter: add(result, x) -proc splitWhitespace*(s: string): seq[string] {.noSideEffect, +proc splitWhitespace*(s: openArray[char]): seq[string] {.noSideEffect, rtl, extern: "ncuSplitWhitespace".} = ## The same as the `splitWhitespace <#splitWhitespace.i,string>`_ ## iterator, but is a proc that returns a sequence of substrings. accResult(splitWhitespace(s)) -iterator split*(s: string, sep: Rune, maxsplit: int = -1): string = +iterator split*(s: openArray[char], sep: Rune, maxsplit: int = -1): string = ## Splits the unicode string ``s`` into substrings using a single separator. ## Substrings are separated by the rune ``sep``. runnableExamples: @@ -1001,19 +1017,19 @@ iterator split*(s: string, sep: Rune, maxsplit: int = -1): string = splitCommon(s, sep, maxsplit) -proc split*(s: string, seps: openArray[Rune] = unicodeSpaces, maxsplit: int = -1): +proc split*(s: openArray[char], seps: openArray[Rune] = unicodeSpaces, maxsplit: int = -1): seq[string] {.noSideEffect, rtl, extern: "nucSplitRunes".} = ## The same as the `split iterator <#split.i,string,openArray[Rune],int>`_, ## but is a proc that returns a sequence of substrings. accResult(split(s, seps, maxsplit)) -proc split*(s: string, sep: Rune, maxsplit: int = -1): seq[string] {.noSideEffect, +proc split*(s: openArray[char], sep: Rune, maxsplit: int = -1): seq[string] {.noSideEffect, rtl, extern: "nucSplitRune".} = ## The same as the `split iterator <#split.i,string,Rune,int>`_, but is a proc ## that returns a sequence of substrings. accResult(split(s, sep, maxsplit)) -proc strip*(s: string, leading = true, trailing = true, +proc strip*(s: openArray[char], leading = true, trailing = true, runes: openArray[Rune] = unicodeSpaces): string {.noSideEffect, rtl, extern: "nucStrip".} = ## Strips leading or trailing ``runes`` from ``s`` and returns @@ -1068,7 +1084,7 @@ proc strip*(s: string, leading = true, trailing = true, let newLen = eI - sI + 1 result = newStringOfCap(newLen) if newLen > 0: - result.add s[sI .. eI] + result.add substr(s.toOpenArray(sI, eI)) proc repeat*(c: Rune, count: Natural): string {.noSideEffect, rtl, extern: "nucRepeatRune".} = @@ -1084,7 +1100,7 @@ proc repeat*(c: Rune, count: Natural): string {.noSideEffect, for i in 0 ..< count: result.add s -proc align*(s: string, count: Natural, padding = ' '.Rune): string {. +proc align*(s: openArray[char], count: Natural, padding = ' '.Rune): string {. noSideEffect, rtl, extern: "nucAlignString".} = ## Aligns a unicode string ``s`` with ``padding``, so that it has a rune-length ## of ``count``. @@ -1109,9 +1125,9 @@ proc align*(s: string, count: Natural, padding = ' '.Rune): string {. for i in 0 ..< spaces: result.add padStr result.add s else: - result = s + result = s.substr -proc alignLeft*(s: string, count: Natural, padding = ' '.Rune): string {. +proc alignLeft*(s: openArray[char], count: Natural, padding = ' '.Rune): string {. noSideEffect.} = ## Left-aligns a unicode string ``s`` with ``padding``, so that it has a ## rune-length of ``count``. @@ -1135,4 +1151,365 @@ proc alignLeft*(s: string, count: Natural, padding = ' '.Rune): string {. for i in sLen ..< count: result.add padStr else: - result = s + result = s.substr + + +proc runeLen*(s: string): int {.inline.} = + ## Returns the number of runes of the string ``s``. + runnableExamples: + let a = "añyóng" + doAssert a.runeLen == 6 + ## note: a.len == 8 + runeLen(toOa(s)) + +proc runeLenAt*(s: string, i: Natural): int {.inline.} = + ## Returns the number of bytes the rune starting at ``s[i]`` takes. + ## + ## See also: + ## * `fastRuneAt template <#fastRuneAt.t,string,int,untyped>`_ + runnableExamples: + let a = "añyóng" + doAssert a.runeLenAt(0) == 1 + doAssert a.runeLenAt(1) == 2 + runeLenAt(toOa(s), i) + +proc runeAt*(s: string, i: Natural): Rune {.inline.} = + ## Returns the rune in ``s`` at **byte index** ``i``. + ## + ## See also: + ## * `runeAtPos proc <#runeAtPos,string,int>`_ + ## * `runeStrAtPos proc <#runeStrAtPos,string,Natural>`_ + ## * `fastRuneAt template <#fastRuneAt.t,string,int,untyped>`_ + runnableExamples: + let a = "añyóng" + doAssert a.runeAt(1) == "ñ".runeAt(0) + doAssert a.runeAt(2) == "ñ".runeAt(1) + doAssert a.runeAt(3) == "y".runeAt(0) + fastRuneAt(s, i, result, false) + +proc validateUtf8*(s: string): int {.inline.} = + ## Returns the position of the invalid byte in ``s`` if the string ``s`` does + ## not hold valid UTF-8 data. Otherwise ``-1`` is returned. + ## + ## See also: + ## * `toUTF8 proc <#toUTF8,Rune>`_ + ## * `$ proc <#$,Rune>`_ alias for `toUTF8` + ## * `fastToUTF8Copy template <#fastToUTF8Copy.t,Rune,string,int>`_ + validateUtf8(toOa(s)) + +proc runeOffset*(s: string, pos: Natural, start: Natural = 0): int {.inline.} = + ## Returns the byte position of rune + ## at position ``pos`` in ``s`` with an optional start byte position. + ## Returns the special value -1 if it runs out of the string. + ## + ## **Beware:** This can lead to unoptimized code and slow execution! + ## Most problems can be solved more efficiently by using an iterator + ## or conversion to a seq of Rune. + ## + ## See also: + ## * `runeReverseOffset proc <#runeReverseOffset,string,Positive>`_ + runnableExamples: + let a = "añyóng" + doAssert a.runeOffset(1) == 1 + doAssert a.runeOffset(3) == 4 + doAssert a.runeOffset(4) == 6 + runeOffset(toOa(s), pos, start) + +proc runeReverseOffset*(s: string, rev: Positive): (int, int) {.inline.} = + ## Returns a tuple with the byte offset of the + ## rune at position ``rev`` in ``s``, counting + ## from the end (starting with 1) and the total + ## number of runes in the string. + ## + ## Returns a negative value for offset if there are too few runes in + ## the string to satisfy the request. + ## + ## **Beware:** This can lead to unoptimized code and slow execution! + ## Most problems can be solved more efficiently by using an iterator + ## or conversion to a seq of Rune. + ## + ## See also: + ## * `runeOffset proc <#runeOffset,string,Natural,Natural>`_ + runeReverseOffset(toOa(s), rev) + +proc runeAtPos*(s: string, pos: int): Rune {.inline.} = + ## Returns the rune at position ``pos``. + ## + ## **Beware:** This can lead to unoptimized code and slow execution! + ## Most problems can be solved more efficiently by using an iterator + ## or conversion to a seq of Rune. + ## + ## See also: + ## * `runeAt proc <#runeAt,string,Natural>`_ + ## * `runeStrAtPos proc <#runeStrAtPos,string,Natural>`_ + ## * `fastRuneAt template <#fastRuneAt.t,string,int,untyped>`_ + fastRuneAt(toOa(s), runeOffset(s, pos), result, false) + +proc runeStrAtPos*(s: string, pos: Natural): string {.inline.} = + ## Returns the rune at position ``pos`` as UTF8 String. + ## + ## **Beware:** This can lead to unoptimized code and slow execution! + ## Most problems can be solved more efficiently by using an iterator + ## or conversion to a seq of Rune. + ## + ## See also: + ## * `runeAt proc <#runeAt,string,Natural>`_ + ## * `runeAtPos proc <#runeAtPos,string,int>`_ + ## * `fastRuneAt template <#fastRuneAt.t,string,int,untyped>`_ + let o = runeOffset(s, pos) + substr(s.toOpenArray(o, (o+runeLenAt(s, o)-1))) + +proc runeSubStr*(s: string, pos: int, len: int = int.high): string {.inline.} = + ## Returns the UTF-8 substring starting at code point ``pos`` + ## with ``len`` code points. + ## + ## If ``pos`` or ``len`` is negative they count from + ## the end of the string. If ``len`` is not given it means the longest + ## possible string. + runnableExamples: + let s = "Hänsel ««: 10,00€" + doAssert(runeSubStr(s, 0, 2) == "Hä") + doAssert(runeSubStr(s, 10, 1) == ":") + doAssert(runeSubStr(s, -6) == "10,00€") + doAssert(runeSubStr(s, 10) == ": 10,00€") + doAssert(runeSubStr(s, 12, 5) == "10,00") + doAssert(runeSubStr(s, -6, 3) == "10,") + runeSubStr(toOa(s), pos, len) + + +proc isAlpha*(s: string): bool {.noSideEffect, inline.} = + ## Returns true if ``s`` contains all alphabetic runes. + runnableExamples: + let a = "añyóng" + doAssert a.isAlpha + isAlpha(toOa(s)) + +proc isSpace*(s: string): bool {.noSideEffect, inline.} = + ## Returns true if ``s`` contains all whitespace runes. + runnableExamples: + let a = "\t\l \v\r\f" + doAssert a.isSpace + isSpace(toOa(s)) + + +proc toUpper*(s: string): string {.noSideEffect, inline.} = + ## Converts ``s`` into upper-case runes. + runnableExamples: + doAssert toUpper("abγ") == "ABΓ" + toUpper(toOa(s)) + +proc toLower*(s: string): string {.noSideEffect, inline.} = + ## Converts ``s`` into lower-case runes. + runnableExamples: + doAssert toLower("ABΓ") == "abγ" + toLower(toOa(s)) + +proc swapCase*(s: string): string {.noSideEffect, inline.} = + ## Swaps the case of runes in ``s``. + ## + ## Returns a new string such that the cases of all runes + ## are swapped if possible. + runnableExamples: + doAssert swapCase("Αlpha Βeta Γamma") == "αLPHA βETA γAMMA" + swapCase(toOa(s)) + +proc capitalize*(s: string): string {.noSideEffect.} = + ## Converts the first character of ``s`` into an upper-case rune. + runnableExamples: + doAssert capitalize("βeta") == "Βeta" + capitalize(toOa(s)) + + +proc translate*(s: string, replacements: proc(key: string): string): string {.effectsOf: replacements, inline.} = + ## Translates words in a string using the ``replacements`` proc to substitute + ## words inside ``s`` with their replacements. + ## + ## ``replacements`` is any proc that takes a word and returns + ## a new word to fill it's place. + runnableExamples: + proc wordToNumber(s: string): string = + case s + of "one": "1" + of "two": "2" + else: s + let a = "one two three four" + doAssert a.translate(wordToNumber) == "1 2 three four" + translate(toOa(s), replacements) + +proc title*(s: string): string {.noSideEffect, inline.} = + ## Converts ``s`` to a unicode title. + ## + ## Returns a new string such that the first character + ## in each word inside ``s`` is capitalized. + runnableExamples: + doAssert title("αlpha βeta γamma") == "Αlpha Βeta Γamma" + title(toOa(s)) + + +iterator runes*(s: string): Rune = + ## Iterates over any rune of the string ``s`` returning runes. + for rune in runes(toOa(s)): + yield rune + +iterator utf8*(s: string): string = + ## Iterates over any rune of the string ``s`` returning utf8 values. + ## + ## See also: + ## * `validateUtf8 proc <#validateUtf8,string>`_ + ## * `toUTF8 proc <#toUTF8,Rune>`_ + ## * `$ proc <#$,Rune>`_ alias for `toUTF8` + ## * `fastToUTF8Copy template <#fastToUTF8Copy.t,Rune,string,int>`_ + for str in utf8(toOa(s)): + yield str + +proc toRunes*(s: string): seq[Rune] {.inline.} = + ## Obtains a sequence containing the Runes in ``s``. + ## + ## See also: + ## * `$ proc <#$,Rune>`_ for a reverse operation + runnableExamples: + let a = toRunes("aáä") + doAssert a == @["a".runeAt(0), "á".runeAt(0), "ä".runeAt(0)] + toRunes(toOa(s)) + +proc cmpRunesIgnoreCase*(a, b: string): int {.inline.} = + ## Compares two UTF-8 strings and ignores the case. Returns: + ## + ## | `0` if a == b + ## | `< 0` if a < b + ## | `> 0` if a > b + cmpRunesIgnoreCase(a.toOa(), b.toOa()) + +proc reversed*(s: string): string {.inline.} = + ## Returns the reverse of ``s``, interpreting it as runes. + ## + ## Unicode combining characters are correctly interpreted as well. + runnableExamples: + assert reversed("Reverse this!") == "!siht esreveR" + assert reversed("先秦兩漢") == "漢兩秦先" + assert reversed("as⃝df̅") == "f̅ds⃝a" + assert reversed("a⃞b⃞c⃞") == "c⃞b⃞a⃞" + reversed(toOa(s)) + +proc graphemeLen*(s: string; i: Natural): Natural {.inline.} = + ## The number of bytes belonging to byte index ``s[i]``, + ## including following combining code unit. + runnableExamples: + let a = "añyóng" + doAssert a.graphemeLen(1) == 2 ## ñ + doAssert a.graphemeLen(2) == 1 + doAssert a.graphemeLen(4) == 2 ## ó + graphemeLen(toOa(s), i) + +proc lastRune*(s: string; last: int): (Rune, int) {.inline.} = + ## Length of the last rune in ``s[0..last]``. Returns the rune and its length + ## in bytes. + lastRune(toOa(s), last) + +iterator split*(s: string, seps: openArray[Rune] = unicodeSpaces, + maxsplit: int = -1): string = + ## Splits the unicode string ``s`` into substrings using a group of separators. + ## + ## Substrings are separated by a substring containing only ``seps``. + runnableExamples: + import std/sequtils + + assert toSeq("hÃllo\lthis\lis an\texample\l是".split) == + @["hÃllo", "this", "is", "an", "example", "是"] + + # And the following code splits the same string using a sequence of Runes. + assert toSeq(split("añyóng:hÃllo;是$example", ";:$".toRunes)) == + @["añyóng", "hÃllo", "是", "example"] + + # example with a `Rune` separator and unused one `;`: + assert toSeq(split("ab是de:f:", ";:是".toRunes)) == @["ab", "de", "f", ""] + + # Another example that splits a string containing a date. + let date = "2012-11-20T22:08:08.398990" + + assert toSeq(split(date, " -:T".toRunes)) == + @["2012", "11", "20", "22", "08", "08.398990"] + + splitCommon(toOa(s), seps, maxsplit) + +iterator splitWhitespace*(s: string): string = + ## Splits a unicode string at whitespace runes. + splitCommon(s.toOa(), unicodeSpaces, -1) + + +proc splitWhitespace*(s: string): seq[string] {.noSideEffect, inline.}= + ## The same as the `splitWhitespace <#splitWhitespace.i,string>`_ + ## iterator, but is a proc that returns a sequence of substrings. + accResult(splitWhitespace(toOa(s))) + +iterator split*(s: string, sep: Rune, maxsplit: int = -1): string = + ## Splits the unicode string ``s`` into substrings using a single separator. + ## Substrings are separated by the rune ``sep``. + runnableExamples: + import std/sequtils + + assert toSeq(split(";;hÃllo;this;is;an;;example;;;是", ";".runeAt(0))) == + @["", "", "hÃllo", "this", "is", "an", "", "example", "", "", "是"] + + splitCommon(toOa(s), sep, maxsplit) + +proc split*(s: string, seps: openArray[Rune] = unicodeSpaces, maxsplit: int = -1): + seq[string] {.noSideEffect, inline.} = + ## The same as the `split iterator <#split.i,string,openArray[Rune],int>`_, + ## but is a proc that returns a sequence of substrings. + accResult(split(toOa(s), seps, maxsplit)) + +proc split*(s: string, sep: Rune, maxsplit: int = -1): seq[string] {.noSideEffect, inline.} = + ## The same as the `split iterator <#split.i,string,Rune,int>`_, but is a proc + ## that returns a sequence of substrings. + accResult(split(toOa(s), sep, maxsplit)) + +proc strip*(s: string, leading = true, trailing = true, + runes: openArray[Rune] = unicodeSpaces): string {.noSideEffect, inline.} = + ## Strips leading or trailing ``runes`` from ``s`` and returns + ## the resulting string. + ## + ## If ``leading`` is true (default), leading ``runes`` are stripped. + ## If ``trailing`` is true (default), trailing ``runes`` are stripped. + ## If both are false, the string is returned unchanged. + runnableExamples: + let a = "\táñyóng " + doAssert a.strip == "áñyóng" + doAssert a.strip(leading = false) == "\táñyóng" + doAssert a.strip(trailing = false) == "áñyóng " + strip(toOa(s), leading, trailing, runes) + + +proc align*(s: string, count: Natural, padding = ' '.Rune): string {.noSideEffect, inline.} = + ## Aligns a unicode string ``s`` with ``padding``, so that it has a rune-length + ## of ``count``. + ## + ## ``padding`` characters (by default spaces) are added before ``s`` resulting in + ## right alignment. If ``s.runelen >= count``, no spaces are added and ``s`` is + ## returned unchanged. If you need to left align a string use the `alignLeft + ## proc <#alignLeft,string,Natural>`_. + runnableExamples: + assert align("abc", 4) == " abc" + assert align("a", 0) == "a" + assert align("1232", 6) == " 1232" + assert align("1232", 6, '#'.Rune) == "##1232" + assert align("Åge", 5) == " Åge" + assert align("×", 4, '_'.Rune) == "___×" + align(toOa(s), count, padding) + +proc alignLeft*(s: string, count: Natural, padding = ' '.Rune): string {.noSideEffect, inline.} = + ## Left-aligns a unicode string ``s`` with ``padding``, so that it has a + ## rune-length of ``count``. + ## + ## ``padding`` characters (by default spaces) are added after ``s`` resulting in + ## left alignment. If ``s.runelen >= count``, no spaces are added and ``s`` is + ## returned unchanged. If you need to right align a string use the `align + ## proc <#align,string,Natural>`_. + runnableExamples: + assert alignLeft("abc", 4) == "abc " + assert alignLeft("a", 0) == "a" + assert alignLeft("1232", 6) == "1232 " + assert alignLeft("1232", 6, '#'.Rune) == "1232##" + assert alignLeft("Åge", 5) == "Åge " + assert alignLeft("×", 4, '_'.Rune) == "×___" + alignLeft(toOa(s), count, padding) diff --git a/lib/pure/unidecode/gen.py b/lib/pure/unidecode/gen.py index 0b180a381..2fb69f7b2 100644 --- a/lib/pure/unidecode/gen.py +++ b/lib/pure/unidecode/gen.py @@ -1,4 +1,4 @@ -#! usr/bin/env python3 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Generates the unidecode.dat module diff --git a/lib/pure/unittest.nim b/lib/pure/unittest.nim index 3cff833e4..cfb762258 100644 --- a/lib/pure/unittest.nim +++ b/lib/pure/unittest.nim @@ -34,9 +34,9 @@ ## ## Specify the test name as a command line argument. ## -## .. code:: -## +## ```cmd ## nim c -r test "my test name" "another test" +## ``` ## ## Multiple arguments can be used. ## @@ -45,9 +45,9 @@ ## ## Specify the suite name delimited by `"::"`. ## -## .. code:: -## +## ```cmd ## nim c -r test "my test name::" +## ``` ## ## Selecting tests by pattern ## ========================== @@ -58,19 +58,18 @@ ## ## Tests matching **any** of the arguments are executed. ## -## .. code:: -## +## ```cmd ## nim c -r test fast_suite::mytest1 fast_suite::mytest2 ## nim c -r test "fast_suite::mytest*" ## nim c -r test "auth*::" "crypto::hashing*" ## # Run suites starting with 'bug #' and standalone tests starting with '#' ## nim c -r test 'bug #*::' '::#*' +## ``` ## ## Examples ## ======== ## -## .. code:: nim -## +## ```nim ## suite "description for this stuff": ## echo "suite setup: run once before the tests" ## @@ -96,6 +95,7 @@ ## discard v[4] ## ## echo "suite teardown: run once after the tests" +## ``` ## ## Limitations/Bugs ## ================ @@ -108,6 +108,9 @@ import std/private/since import std/exitprocs +when defined(nimPreviewSlimSystem): + import std/assertions + import std/[macros, strutils, streams, times, sets, sequtils] when declared(stdout): @@ -218,7 +221,7 @@ proc resetOutputFormatters* {.since: (1, 1).} = formatters = @[] proc newConsoleOutputFormatter*(outputLevel: OutputLevel = outputLevelDefault, - colorOutput = true): <//>ConsoleOutputFormatter = + colorOutput = true): ConsoleOutputFormatter = ConsoleOutputFormatter( outputLevel: outputLevel, colorOutput: colorOutput @@ -232,7 +235,7 @@ proc colorOutput(): bool = else: result = false of "on": result = true of "off": result = false - else: doAssert false, $color + else: raiseAssert $color when declared(stdout): if existsEnv("NIMTEST_COLOR"): @@ -246,7 +249,7 @@ proc colorOutput(): bool = deprecateEnvVarHere() result = false -proc defaultConsoleFormatter*(): <//>ConsoleOutputFormatter = +proc defaultConsoleFormatter*(): ConsoleOutputFormatter = var colorOutput = colorOutput() var outputLevel = nimUnittestOutputLevel.parseEnum[:OutputLevel] when declared(stdout): @@ -315,7 +318,7 @@ proc xmlEscape(s: string): string = else: result.add(c) -proc newJUnitOutputFormatter*(stream: Stream): <//>JUnitOutputFormatter = +proc newJUnitOutputFormatter*(stream: Stream): JUnitOutputFormatter = ## Creates a formatter that writes report to the specified stream in ## JUnit format. ## The ``stream`` is NOT closed automatically when the test are finished, @@ -430,8 +433,6 @@ proc matchFilter(suiteName, testName, filter: string): bool = return glob(suiteName, suiteAndTestFilters[0]) and glob(testName, suiteAndTestFilters[1]) -when defined(testing): export matchFilter - proc shouldRun(currentSuiteName, testName: string): bool = ## Check if a test should be run by matching suiteName and testName against ## test filters. @@ -472,27 +473,26 @@ template suite*(name, body) {.dirty.} = ## common fixture (``setup``, ``teardown``). The fixture is executed ## for EACH test. ## - ## .. code-block:: nim - ## suite "test suite for addition": - ## setup: - ## let result = 4 + ## ```nim + ## suite "test suite for addition": + ## setup: + ## let result = 4 ## - ## test "2 + 2 = 4": - ## check(2+2 == result) + ## test "2 + 2 = 4": + ## check(2+2 == result) ## - ## test "(2 + -2) != 4": - ## check(2 + -2 != result) + ## test "(2 + -2) != 4": + ## check(2 + -2 != result) ## - ## # No teardown needed + ## # No teardown needed + ## ``` ## ## The suite will run the individual test cases in the order in which ## they were listed. With default global settings the above code prints: ## - ## .. code-block:: - ## - ## [Suite] test suite for addition - ## [OK] 2 + 2 = 4 - ## [OK] (2 + -2) != 4 + ## [Suite] test suite for addition + ## [OK] 2 + 2 = 4 + ## [OK] (2 + -2) != 4 bind formatters, ensureInitialized, suiteEnded block: @@ -518,20 +518,24 @@ proc exceptionTypeName(e: ref Exception): string {.inline.} = if e == nil: "<foreign exception>" else: $e.name +when not declared(setProgramResult): + {.warning: "setProgramResult not available on platform, unittest will not" & + " give failing exit code on test failure".} + template setProgramResult(a: int) = + discard + template test*(name, body) {.dirty.} = ## Define a single test case identified by `name`. ## - ## .. code-block:: nim - ## - ## test "roses are red": - ## let roses = "red" - ## check(roses == "red") + ## ```nim + ## test "roses are red": + ## let roses = "red" + ## check(roses == "red") + ## ``` ## ## The above code outputs: ## - ## .. code-block:: - ## - ## [OK] roses are red + ## [OK] roses are red bind shouldRun, checkpoints, formatters, ensureInitialized, testEnded, exceptionTypeName, setProgramResult ensureInitialized() @@ -543,11 +547,14 @@ template test*(name, body) {.dirty.} = for formatter in formatters: formatter.testStarted(name) + {.push warning[BareExcept]:off.} try: when declared(testSetupIMPLFlag): testSetupIMPL() when declared(testTeardownIMPLFlag): defer: testTeardownIMPL() + {.push warning[BareExcept]:on.} body + {.pop.} except: let e = getCurrentException() @@ -569,16 +576,17 @@ template test*(name, body) {.dirty.} = ) testEnded(testResult) checkpoints = @[] + {.pop.} proc checkpoint*(msg: string) = ## Set a checkpoint identified by `msg`. Upon test failure all ## checkpoints encountered so far are printed out. Example: ## - ## .. code-block:: nim - ## - ## checkpoint("Checkpoint A") - ## check((42, "the Answer to life and everything") == (1, "a")) - ## checkpoint("Checkpoint B") + ## ```nim + ## checkpoint("Checkpoint A") + ## check((42, "the Answer to life and everything") == (1, "a")) + ## checkpoint("Checkpoint B") + ## ``` ## ## outputs "Checkpoint A" once it fails. checkpoints.add(msg) @@ -590,11 +598,11 @@ template fail* = ## failed (change exit code and test status). This template is useful ## for debugging, but is otherwise mostly used internally. Example: ## - ## .. code-block:: nim - ## - ## checkpoint("Checkpoint A") - ## complicatedProcInThread() - ## fail() + ## ```nim + ## checkpoint("Checkpoint A") + ## complicatedProcInThread() + ## fail() + ## ``` ## ## outputs "Checkpoint A" before quitting. bind ensureInitialized, setProgramResult @@ -622,11 +630,10 @@ template skip* = ## for reasons depending on outer environment, ## or certain application logic conditions or configurations. ## The test code is still executed. - ## - ## .. code-block:: nim - ## - ## if not isGLContextCreated(): - ## skip() + ## ```nim + ## if not isGLContextCreated(): + ## skip() + ## ``` bind checkpoints testStatusIMPL = TestStatus.SKIPPED @@ -704,7 +711,9 @@ macro check*(conditions: untyped): untyped = result = quote do: block: `assigns` - if not `check`: + if `check`: + discard + else: checkpoint(`lineinfo` & ": Check failed: " & `callLit`) `printOuts` fail() @@ -720,7 +729,9 @@ macro check*(conditions: untyped): untyped = let callLit = checked.toStrLit result = quote do: - if not `checked`: + if `checked`: + discard + else: checkpoint(`lineinfo` & ": Check failed: " & `callLit`) fail() @@ -747,20 +758,25 @@ macro expect*(exceptions: varargs[typed], body: untyped): untyped = of 2: discard parseInt("Hello World!") of 3: raise newException(IOError, "I can't do that Dave.") else: assert 2 + 2 == 5 - + expect IOError, OSError, ValueError, AssertionDefect: defectiveRobot() template expectBody(errorTypes, lineInfoLit, body): NimNode {.dirty.} = + {.push warning[BareExcept]:off.} try: + {.push warning[BareExcept]:on.} body + {.pop.} checkpoint(lineInfoLit & ": Expect Failed, no exception was thrown.") fail() except errorTypes: discard except: - checkpoint(lineInfoLit & ": Expect Failed, unexpected exception was thrown.") + let err = getCurrentException() + checkpoint(lineInfoLit & ": Expect Failed, " & $err.name & " was thrown.") fail() + {.pop.} var errorTypes = newNimNode(nnkBracket) for exp in exceptions: diff --git a/lib/pure/uri.nim b/lib/pure/uri.nim index 920529ecf..725d5bbd9 100644 --- a/lib/pure/uri.nim +++ b/lib/pure/uri.nim @@ -14,34 +14,36 @@ ## as a locator, a name, or both. The term "Uniform Resource Locator" ## (URL) refers to the subset of URIs. ## +## .. warning:: URI parsers in this module do not perform security validation. +## ## # Basic usage ## ## Combine URIs runnableExamples: let host = parseUri("https://nim-lang.org") - let blog = "/blog.html" - let bloguri = host / blog assert $host == "https://nim-lang.org" - assert $bloguri == "https://nim-lang.org/blog.html" + assert $(host / "/blog.html") == "https://nim-lang.org/blog.html" + assert $(host / "blog2.html") == "https://nim-lang.org/blog2.html" ## ## Access URI item runnableExamples: let res = parseUri("sftp://127.0.0.1:4343") - if isAbsolute(res): - assert res.port == "4343" - else: - echo "Wrong format" + assert isAbsolute(res) + assert res.port == "4343" ## ## Data URI Base64 runnableExamples: - doAssert getDataUri("Hello World", "text/plain") == "data:text/plain;charset=utf-8;base64,SGVsbG8gV29ybGQ=" - doAssert getDataUri("Nim", "text/plain") == "data:text/plain;charset=utf-8;base64,Tmlt" + assert getDataUri("Hello World", "text/plain") == "data:text/plain;charset=utf-8;base64,SGVsbG8gV29ybGQ=" + assert getDataUri("Nim", "text/plain") == "data:text/plain;charset=utf-8;base64,Tmlt" import std/[strutils, parseutils, base64] import std/private/[since, decode_helpers] +when defined(nimPreviewSlimSystem): + import std/assertions + type Url* = distinct string @@ -50,7 +52,7 @@ type scheme*, username*, password*: string hostname*, port*, path*, query*, anchor*: string opaque*: bool - isIpv6: bool # not expose it for compatibility. + isIpv6*: bool UriParseError* = object of ValueError @@ -126,13 +128,13 @@ func decodeUrl*(s: string, decodePlus = true): string = setLen(result, j) func encodeQuery*(query: openArray[(string, string)], usePlus = true, - omitEq = true): string = + omitEq = true, sep = '&'): string = ## Encodes a set of (key, value) parameters into a URL query string. ## ## Every (key, value) pair is URL-encoded and written as `key=value`. If the ## value is an empty string then the `=` is omitted, unless `omitEq` is ## false. - ## The pairs are joined together by a `&` character. + ## The pairs are joined together by the `sep` character. ## ## The `usePlus` parameter is passed down to the `encodeUrl` function that ## is used for the URL encoding of the string values. @@ -143,9 +145,10 @@ func encodeQuery*(query: openArray[(string, string)], usePlus = true, assert encodeQuery({: }) == "" assert encodeQuery({"a": "1", "b": "2"}) == "a=1&b=2" assert encodeQuery({"a": "1", "b": ""}) == "a=1&b" + assert encodeQuery({"a": "1", "b": ""}, omitEq = false, sep = ';') == "a=1;b=" for elem in query: - # Encode the `key = value` pairs and separate them with a '&' - if result.len > 0: result.add('&') + # Encode the `key = value` pairs and separate them with 'sep' + if result.len > 0: result.add(sep) let (key, val) = elem result.add(encodeUrl(key, usePlus)) # Omit the '=' if the value string is empty @@ -153,15 +156,16 @@ func encodeQuery*(query: openArray[(string, string)], usePlus = true, result.add('=') result.add(encodeUrl(val, usePlus)) -iterator decodeQuery*(data: string): tuple[key, value: string] = - ## Reads and decodes query string `data` and yields the `(key, value)` pairs - ## the data consists of. If compiled with `-d:nimLegacyParseQueryStrict`, an - ## error is raised when there is an unencoded `=` character in a decoded - ## value, which was the behavior in Nim < 1.5.1 +iterator decodeQuery*(data: string, sep = '&'): tuple[key, value: string] = + ## Reads and decodes the query string `data` and yields the `(key, value)` pairs + ## the data consists of. If compiled with `-d:nimLegacyParseQueryStrict`, + ## a `UriParseError` is raised when there is an unencoded `=` character in a decoded + ## value, which was the behavior in Nim < 1.5.1. runnableExamples: import std/sequtils - doAssert toSeq(decodeQuery("foo=1&bar=2=3")) == @[("foo", "1"), ("bar", "2=3")] - doAssert toSeq(decodeQuery("&a&=b&=&&")) == @[("", ""), ("a", ""), ("", "b"), ("", ""), ("", "")] + assert toSeq(decodeQuery("foo=1&bar=2=3")) == @[("foo", "1"), ("bar", "2=3")] + assert toSeq(decodeQuery("foo=1;bar=2=3", ';')) == @[("foo", "1"), ("bar", "2=3")] + assert toSeq(decodeQuery("&a&=b&=&&")) == @[("", ""), ("a", ""), ("", "b"), ("", ""), ("", "")] proc parseData(data: string, i: int, field: var string, sep: char): int = result = i @@ -189,7 +193,7 @@ iterator decodeQuery*(data: string): tuple[key, value: string] = when defined(nimLegacyParseQueryStrict): i = parseData(data, i, value, '=') else: - i = parseData(data, i, value, '&') + i = parseData(data, i, value, sep) yield (name, value) if i < data.len: when defined(nimLegacyParseQueryStrict): @@ -227,7 +231,6 @@ func parseAuthority(authority: string, result: var Uri) = i.inc func parsePath(uri: string, i: var int, result: var Uri) = - i.inc parseUntil(uri, result.path, {'?', '#'}, i) # The 'mailto' scheme's PATH actually contains the hostname/username @@ -243,19 +246,7 @@ func parsePath(uri: string, i: var int, result: var Uri) = i.inc # Skip '#' i.inc parseUntil(uri, result.anchor, {}, i) -func initUri*(): Uri = - ## Initializes a URI with `scheme`, `username`, `password`, - ## `hostname`, `port`, `path`, `query` and `anchor`. - ## - ## **See also:** - ## * `Uri type <#Uri>`_ for available fields in the URI type - runnableExamples: - var uri2: Uri - assert initUri() == uri2 - result = Uri(scheme: "", username: "", password: "", hostname: "", port: "", - path: "", query: "", anchor: "") - -func initUri*(isIpv6: bool): Uri {.since: (1, 3, 5).} = +func initUri*(isIpv6 = false): Uri = ## Initializes a URI with `scheme`, `username`, `password`, ## `hostname`, `port`, `path`, `query`, `anchor` and `isIpv6`. ## @@ -294,7 +285,7 @@ func parseUri*(uri: string, result: var Uri) = var i = 0 # Check if this is a reference URI (relative URI) - let doubleSlash = uri.len > 1 and uri[1] == '/' + let doubleSlash = uri.len > 1 and uri[0] == '/' and uri[1] == '/' if i < uri.len and uri[i] == '/': # Make sure `uri` doesn't begin with '//'. if not doubleSlash: @@ -455,10 +446,8 @@ func combine*(uris: varargs[Uri]): Uri = func isAbsolute*(uri: Uri): bool = ## Returns true if URI is absolute, false otherwise. runnableExamples: - let foo = parseUri("https://nim-lang.org") - assert isAbsolute(foo) == true - let bar = parseUri("nim-lang") - assert isAbsolute(bar) == false + assert parseUri("https://nim-lang.org").isAbsolute + assert not parseUri("nim-lang").isAbsolute return uri.scheme != "" and (uri.hostname != "" or uri.path != "") func `/`*(x: Uri, path: string): Uri = @@ -506,44 +495,62 @@ func `?`*(u: Uri, query: openArray[(string, string)]): Uri = func `$`*(u: Uri): string = ## Returns the string representation of the specified URI object. runnableExamples: - let foo = parseUri("https://nim-lang.org") - assert $foo == "https://nim-lang.org" - result = "" - if u.scheme.len > 0: - result.add(u.scheme) - if u.opaque: - result.add(":") - else: - result.add("://") - if u.username.len > 0: - result.add(u.username) - if u.password.len > 0: - result.add(":") - result.add(u.password) - result.add("@") + assert $parseUri("https://nim-lang.org") == "https://nim-lang.org" + # Get the len of all the parts. + let schemeLen = u.scheme.len + let usernameLen = u.username.len + let passwordLen = u.password.len + let hostnameLen = u.hostname.len + let portLen = u.port.len + let pathLen = u.path.len + let queryLen = u.query.len + let anchorLen = u.anchor.len + # Prepare a string that fits all the parts and all punctuation chars. + # 12 is the max len required by all possible punctuation chars. + result = newStringOfCap( + schemeLen + usernameLen + passwordLen + hostnameLen + portLen + pathLen + queryLen + anchorLen + 12 + ) + # Insert to result. + if schemeLen > 0: + result.add u.scheme + result.add ':' + if not u.opaque: + result.add '/' + result.add '/' + if usernameLen > 0: + result.add u.username + if passwordLen > 0: + result.add ':' + result.add u.password + result.add '@' if u.hostname.endsWith('/'): if u.isIpv6: - result.add("[" & u.hostname[0 .. ^2] & "]") + result.add '[' + result.add u.hostname[0 .. ^2] + result.add ']' else: - result.add(u.hostname[0 .. ^2]) + result.add u.hostname[0 .. ^2] else: if u.isIpv6: - result.add("[" & u.hostname & "]") + result.add '[' + result.add u.hostname + result.add ']' else: - result.add(u.hostname) - if u.port.len > 0: - result.add(":") - result.add(u.port) - if u.path.len > 0: - if u.hostname.len > 0 and u.path[0] != '/': - result.add('/') - result.add(u.path) - if u.query.len > 0: - result.add("?") - result.add(u.query) - if u.anchor.len > 0: - result.add("#") - result.add(u.anchor) + result.add u.hostname + if portLen > 0: + result.add ':' + result.add u.port + if pathLen > 0: + if hostnameLen > 0 and u.path[0] != '/': + result.add '/' + result.add u.path + if queryLen > 0: + result.add '?' + result.add u.query + if anchorLen > 0: + result.add '#' + result.add u.anchor + proc getDataUri*(data, mime: string, encoding = "utf-8"): string {.since: (1, 3).} = ## Convenience proc for `base64.encode` returns a standard Base64 Data URI (RFC-2397) @@ -552,28 +559,14 @@ proc getDataUri*(data, mime: string, encoding = "utf-8"): string {.since: (1, 3) ## * `mimetypes <mimetypes.html>`_ for `mime` argument ## * https://tools.ietf.org/html/rfc2397 ## * https://en.wikipedia.org/wiki/Data_URI_scheme - runnableExamples: static: doAssert getDataUri("Nim", "text/plain") == "data:text/plain;charset=utf-8;base64,Tmlt" + runnableExamples: static: assert getDataUri("Nim", "text/plain") == "data:text/plain;charset=utf-8;base64,Tmlt" assert encoding.len > 0 and mime.len > 0 # Must *not* be URL-Safe, see RFC-2397 - result = "data:" & mime & ";charset=" & encoding & ";base64," & base64.encode(data) - -when isMainModule and defined(testing): - # needed (pending https://github.com/nim-lang/Nim/pull/11865) because - # `removeDotSegments` is private, the other tests are in `turi`. - block: # removeDotSegments - # `removeDotSegments` is exported for -d:testing only - doAssert removeDotSegments("/foo/bar/baz") == "/foo/bar/baz" - doAssert removeDotSegments("") == "" # empty test - doAssert removeDotSegments(".") == "." # trailing period - doAssert removeDotSegments("a1/a2/../a3/a4/a5/./a6/a7/././") == "a1/a3/a4/a5/a6/a7/" - doAssert removeDotSegments("https://a1/a2/../a3/a4/a5/./a6/a7/././") == "https://a1/a3/a4/a5/a6/a7/" - doAssert removeDotSegments("http://a1/a2") == "http://a1/a2" - doAssert removeDotSegments("http://www.ai.") == "http://www.ai." - when false: # xxx these cases are buggy - # this should work, refs https://webmasters.stackexchange.com/questions/73934/how-can-urls-have-a-dot-at-the-end-e-g-www-bla-de - doAssert removeDotSegments("http://www.ai./") == "http://www.ai./" # fails - echo removeDotSegments("http://www.ai./") # http://www.ai/ - echo removeDotSegments("a/b.../c") # b.c - echo removeDotSegments("a/b../c") # bc - echo removeDotSegments("a/.../c") # .c - echo removeDotSegments("a//../b") # a/b - echo removeDotSegments("a/b/c//") # a/b/c// + let base64encoded: string = base64.encode(data) + # ("data:".len + ";charset=".len + ";base64,".len) == 22 + result = newStringOfCap(22 + mime.len + encoding.len + base64encoded.len) + result.add "data:" + result.add mime + result.add ";charset=" + result.add encoding + result.add ";base64," + result.add base64encoded diff --git a/lib/pure/volatile.nim b/lib/pure/volatile.nim index f30d899df..a38247c7d 100644 --- a/lib/pure/volatile.nim +++ b/lib/pure/volatile.nim @@ -10,20 +10,18 @@ ## This module contains code for generating volatile loads and stores, ## which are useful in embedded and systems programming. -template volatileLoad*[T](src: ptr T): T = +proc volatileLoad*[T](src: ptr T): T {.inline, noinit.} = ## Generates a volatile load of the value stored in the container `src`. ## Note that this only effects code generation on `C` like backends. when nimvm: - src[] + result = src[] else: when defined(js): - src[] + result = src[] else: - var res: T - {.emit: [res, " = (*(", typeof(src[]), " volatile*)", src, ");"].} - res + {.emit: [result, " = (*(", typeof(src[]), " volatile*)", src, ");"].} -template volatileStore*[T](dest: ptr T, val: T) = +proc volatileStore*[T](dest: ptr T, val: T) {.inline.} = ## Generates a volatile store into the container `dest` of the value ## `val`. Note that this only effects code generation on `C` like ## backends. diff --git a/lib/pure/xmlparser.nim b/lib/pure/xmlparser.nim index 3d9c288ed..2c1e4e37c 100644 --- a/lib/pure/xmlparser.nim +++ b/lib/pure/xmlparser.nim @@ -9,7 +9,10 @@ ## This module parses an XML document and creates its XML tree representation. -import streams, parsexml, strtabs, xmltree +import std/[streams, parsexml, strtabs, xmltree] + +when defined(nimPreviewSlimSystem): + import std/syncio type XmlError* = object of ValueError ## Exception that is raised @@ -148,7 +151,7 @@ proc loadXml*(path: string, options: set[XmlParseOption] = {reportComments}): Xm when isMainModule: when not defined(testing): - import os + import std/os var errors: seq[string] = @[] var x = loadXml(paramStr(1), errors) diff --git a/lib/pure/xmltree.nim b/lib/pure/xmltree.nim index af7e4d508..5c0cbc5e4 100644 --- a/lib/pure/xmltree.nim +++ b/lib/pure/xmltree.nim @@ -31,7 +31,11 @@ runnableExamples: ## * `htmlgen module <htmlgen.html>`_ for html code generator import std/private/since -import macros, strtabs, strutils +import std/[macros, strtabs, strutils, sequtils] + +when defined(nimPreviewSlimSystem): + import std/assertions + type XmlNode* = ref XmlNodeObj ## An XML tree consisting of XML nodes. @@ -66,11 +70,19 @@ const xmlHeader* = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n" ## Header to use for complete XML output. +template expect(node: XmlNode, kind: set[XmlNodeKind]) = + ## Check the node's kind is within a set of values + assert node.k in kind, "Got " & $node.k + +template expect(node: XmlNode, kind: XmlNodeKind) = + ## Check the node's kind equals a value + assert node.k == kind, "Got " & $node.k + proc newXmlNode(kind: XmlNodeKind): XmlNode = ## Creates a new ``XmlNode``. result = XmlNode(k: kind) -proc newElement*(tag: string): XmlNode = +proc newElement*(tag: sink string): XmlNode = ## Creates a new ``XmlNode`` of kind ``xnElement`` with the given `tag`. ## ## See also: @@ -89,7 +101,7 @@ proc newElement*(tag: string): XmlNode = result.s = @[] # init attributes lazily to save memory -proc newText*(text: string): XmlNode = +proc newText*(text: sink string): XmlNode = ## Creates a new ``XmlNode`` of kind ``xnText`` with the text `text`. runnableExamples: var b = newText("my text") @@ -99,13 +111,13 @@ proc newText*(text: string): XmlNode = result = newXmlNode(xnText) result.fText = text -proc newVerbatimText*(text: string): XmlNode {.since: (1, 3).} = +proc newVerbatimText*(text: sink string): XmlNode {.since: (1, 3).} = ## Creates a new ``XmlNode`` of kind ``xnVerbatimText`` with the text `text`. ## **Since**: Version 1.3. result = newXmlNode(xnVerbatimText) result.fText = text -proc newComment*(comment: string): XmlNode = +proc newComment*(comment: sink string): XmlNode = ## Creates a new ``XmlNode`` of kind ``xnComment`` with the text `comment`. runnableExamples: var c = newComment("my comment") @@ -115,7 +127,7 @@ proc newComment*(comment: string): XmlNode = result = newXmlNode(xnComment) result.fText = comment -proc newCData*(cdata: string): XmlNode = +proc newCData*(cdata: sink string): XmlNode = ## Creates a new ``XmlNode`` of kind ``xnCData`` with the text `cdata`. runnableExamples: var d = newCData("my cdata") @@ -135,7 +147,7 @@ proc newEntity*(entity: string): XmlNode = result = newXmlNode(xnEntity) result.fText = entity -proc newXmlTree*(tag: string, children: openArray[XmlNode], +proc newXmlTree*(tag: sink string, children: openArray[XmlNode], attributes: XmlAttributes = nil): XmlNode = ## Creates a new XML tree with `tag`, `children` and `attributes`. ## @@ -151,7 +163,7 @@ proc newXmlTree*(tag: string, children: openArray[XmlNode], h.add newEntity("some entity") let att = {"key1": "first value", "key2": "second value"}.toXmlAttributes let k = newXmlTree("treeTag", [g, h], att) - + doAssert $k == """<treeTag key1="first value" key2="second value"> <myTag>some text<!-- this is comment --></myTag> <secondTag>&some entity;</secondTag> @@ -163,7 +175,7 @@ proc newXmlTree*(tag: string, children: openArray[XmlNode], for i in 0..children.len-1: result.s[i] = children[i] result.fAttr = attributes -proc text*(n: XmlNode): string {.inline.} = +proc text*(n: XmlNode): lent string {.inline.} = ## Gets the associated text with the node `n`. ## ## `n` can be a CDATA, Text, comment, or entity node. @@ -178,10 +190,10 @@ proc text*(n: XmlNode): string {.inline.} = assert $c == "<!-- my comment -->" assert c.text == "my comment" - assert n.k in {xnText, xnComment, xnCData, xnEntity} + n.expect {xnText, xnVerbatimText, xnComment, xnCData, xnEntity} result = n.fText -proc `text=`*(n: XmlNode, text: string){.inline.} = +proc `text=`*(n: XmlNode, text: sink string) {.inline.} = ## Sets the associated text with the node `n`. ## ## `n` can be a CDATA, Text, comment, or entity node. @@ -196,10 +208,10 @@ proc `text=`*(n: XmlNode, text: string){.inline.} = e.text = "a new entity text" assert $e == "&a new entity text;" - assert n.k in {xnText, xnComment, xnCData, xnEntity} + n.expect {xnText, xnVerbatimText, xnComment, xnCData, xnEntity} n.fText = text -proc tag*(n: XmlNode): string {.inline.} = +proc tag*(n: XmlNode): lent string {.inline.} = ## Gets the tag name of `n`. ## ## `n` has to be an ``xnElement`` node. @@ -217,10 +229,10 @@ proc tag*(n: XmlNode): string {.inline.} = </firstTag>""" assert a.tag == "firstTag" - assert n.k == xnElement + n.expect xnElement result = n.fTag -proc `tag=`*(n: XmlNode, tag: string) {.inline.} = +proc `tag=`*(n: XmlNode, tag: sink string) {.inline.} = ## Sets the tag name of `n`. ## ## `n` has to be an ``xnElement`` node. @@ -240,7 +252,7 @@ proc `tag=`*(n: XmlNode, tag: string) {.inline.} = <childTag /> </newTag>""" - assert n.k == xnElement + n.expect xnElement n.fTag = tag proc rawText*(n: XmlNode): string {.inline.} = @@ -294,26 +306,60 @@ proc innerText*(n: XmlNode): string = proc add*(father, son: XmlNode) {.inline.} = ## Adds the child `son` to `father`. + ## `father` must be of `xnElement` type ## ## See also: + ## * `add proc <#add,XmlNode,openArray[XmlNode]>`_ ## * `insert proc <#insert,XmlNode,XmlNode,int>`_ + ## * `insert proc <#insert,XmlNode,openArray[XmlNode],int>`_ ## * `delete proc <#delete,XmlNode,Natural>`_ + ## * `delete proc <#delete.XmlNode,Slice[int]>`_ + ## * `replace proc <#replace.XmlNode,int,openArray[XmlNode]>`_ + ## * `replace proc <#replace.XmlNode,Slice[int],openArray[XmlNode]>`_ runnableExamples: var f = newElement("myTag") f.add newText("my text") f.add newElement("sonTag") f.add newEntity("my entity") assert $f == "<myTag>my text<sonTag />&my entity;</myTag>" + + father.expect xnElement add(father.s, son) +proc add*(father: XmlNode, sons: openArray[XmlNode]) {.inline.} = + ## Adds the children `sons` to `father`. + ## `father` must be of `xnElement` type + ## + ## See also: + ## * `add proc <#add,XmlNode,XmlNode>`_ + ## * `insert proc <#insert,XmlNode,XmlNode,int>`_ + ## * `insert proc <#insert,XmlNode,openArray[XmlNode],int>`_ + ## * `delete proc <#delete,XmlNode,Natural>`_ + ## * `delete proc <#delete.XmlNode,Slice[int]>`_ + ## * `replace proc <#replace.XmlNode,int,openArray[XmlNode]>`_ + ## * `replace proc <#replace.XmlNode,Slice[int],openArray[XmlNode]>`_ + runnableExamples: + var f = newElement("myTag") + f.add(@[newText("my text"), newElement("sonTag"), newEntity("my entity")]) + assert $f == "<myTag>my text<sonTag />&my entity;</myTag>" + + father.expect xnElement + add(father.s, sons) + + proc insert*(father, son: XmlNode, index: int) {.inline.} = ## Inserts the child `son` to a given position in `father`. ## - ## `father` and `son` must be of `xnElement` kind. + ## `father` must be of `xnElement` kind. ## ## See also: + ## * `insert proc <#insert,XmlNode,openArray[XmlNode],int>`_ ## * `add proc <#add,XmlNode,XmlNode>`_ + ## * `add proc <#add,XmlNode,openArray[XmlNode]>`_ ## * `delete proc <#delete,XmlNode,Natural>`_ + ## * `delete proc <#delete.XmlNode,Slice[int]>`_ + ## * `replace proc <#replace.XmlNode,int,openArray[XmlNode]>`_ + ## * `replace proc <#replace.XmlNode,Slice[int],openArray[XmlNode]>`_ runnableExamples: var f = newElement("myTag") f.add newElement("first") @@ -323,18 +369,52 @@ proc insert*(father, son: XmlNode, index: int) {.inline.} = <first /> </myTag>""" - assert father.k == xnElement and son.k == xnElement + father.expect xnElement if len(father.s) > index: insert(father.s, son, index) else: insert(father.s, son, len(father.s)) +proc insert*(father: XmlNode, sons: openArray[XmlNode], index: int) {.inline.} = + ## Inserts the children openArray[`sons`] to a given position in `father`. + ## + ## `father` must be of `xnElement` kind. + ## + ## See also: + ## * `insert proc <#insert,XmlNode,XmlNode,int>`_ + ## * `add proc <#add,XmlNode,XmlNode>`_ + ## * `add proc <#add,XmlNode,openArray[XmlNode]>`_ + ## * `delete proc <#delete,XmlNode,Natural>`_ + ## * `delete proc <#delete.XmlNode,Slice[int]>`_ + ## * `replace proc <#replace.XmlNode,int,openArray[XmlNode]>`_ + ## * `replace proc <#replace.XmlNode,Slice[int],openArray[XmlNode]>`_ + runnableExamples: + var f = newElement("myTag") + f.add newElement("first") + f.insert([newElement("second"), newElement("third")], 0) + assert $f == """<myTag> + <second /> + <third /> + <first /> +</myTag>""" + + father.expect xnElement + if len(father.s) > index: + insert(father.s, sons, index) + else: + insert(father.s, sons, len(father.s)) + proc delete*(n: XmlNode, i: Natural) = ## Deletes the `i`'th child of `n`. ## ## See also: + ## * `delete proc <#delete.XmlNode,Slice[int]>`_ ## * `add proc <#add,XmlNode,XmlNode>`_ + ## * `add proc <#add,XmlNode,openArray[XmlNode]>`_ ## * `insert proc <#insert,XmlNode,XmlNode,int>`_ + ## * `insert proc <#insert,XmlNode,openArray[XmlNode],int>`_ + ## * `replace proc <#replace.XmlNode,int,openArray[XmlNode]>`_ + ## * `replace proc <#replace.XmlNode,Slice[int],openArray[XmlNode]>`_ runnableExamples: var f = newElement("myTag") f.add newElement("first") @@ -344,8 +424,87 @@ proc delete*(n: XmlNode, i: Natural) = <first /> </myTag>""" - assert n.k == xnElement + n.expect xnElement + n.s.delete(i) + +proc delete*(n: XmlNode, slice: Slice[int]) = + ## Deletes the items `n[slice]` of `n`. + ## + ## See also: + ## * `delete proc <#delete.XmlNode,int>`_ + ## * `add proc <#add,XmlNode,XmlNode>`_ + ## * `add proc <#add,XmlNode,openArray[XmlNode]>`_ + ## * `insert proc <#insert,XmlNode,XmlNode,int>`_ + ## * `insert proc <#insert,XmlNode,openArray[XmlNode],int>`_ + ## * `replace proc <#replace.XmlNode,int,openArray[XmlNode]>`_ + ## * `replace proc <#replace.XmlNode,Slice[int],openArray[XmlNode]>`_ + runnableExamples: + var f = newElement("myTag") + f.add newElement("first") + f.insert([newElement("second"), newElement("third")], 0) + f.delete(0..1) + assert $f == """<myTag> + <first /> +</myTag>""" + + n.expect xnElement + n.s.delete(slice) + +proc replace*(n: XmlNode, i: Natural, replacement: openArray[XmlNode]) = + ## Replaces the `i`'th child of `n` with `replacement` openArray. + ## + ## `n` must be of `xnElement` kind. + ## + ## See also: + ## * `replace proc <#replace.XmlNode,Slice[int],openArray[XmlNode]>`_ + ## * `add proc <#add,XmlNode,XmlNode>`_ + ## * `add proc <#add,XmlNode,openArray[XmlNode]>`_ + ## * `delete proc <#delete,XmlNode,Natural>`_ + ## * `delete proc <#delete.XmlNode,Slice[int]>`_ + ## * `insert proc <#insert,XmlNode,XmlNode,int>`_ + ## * `insert proc <#insert,XmlNode,openArray[XmlNode],int>`_ + runnableExamples: + var f = newElement("myTag") + f.add newElement("first") + f.insert(newElement("second"), 0) + f.replace(0, @[newElement("third"), newElement("fourth")]) + assert $f == """<myTag> + <third /> + <fourth /> + <first /> +</myTag>""" + + n.expect xnElement n.s.delete(i) + n.s.insert(replacement, i) + +proc replace*(n: XmlNode, slice: Slice[int], replacement: openArray[XmlNode]) = + ## Deletes the items `n[slice]` of `n`. + ## + ## `n` must be of `xnElement` kind. + ## + ## See also: + ## * `replace proc <#replace.XmlNode,int,openArray[XmlNode]>`_ + ## * `add proc <#add,XmlNode,XmlNode>`_ + ## * `add proc <#add,XmlNode,openArray[XmlNode]>`_ + ## * `delete proc <#delete,XmlNode,Natural>`_ + ## * `delete proc <#delete.XmlNode,Slice[int]>`_ + ## * `insert proc <#insert,XmlNode,XmlNode,int>`_ + ## * `insert proc <#insert,XmlNode,openArray[XmlNode],int>`_ + runnableExamples: + var f = newElement("myTag") + f.add newElement("first") + f.insert([newElement("second"), newElement("fifth")], 0) + f.replace(0..1, @[newElement("third"), newElement("fourth")]) + assert $f == """<myTag> + <third /> + <fourth /> + <first /> +</myTag>""" + + n.expect xnElement + n.s.delete(slice) + n.s.insert(replacement, slice.a) proc len*(n: XmlNode): int {.inline.} = ## Returns the number of `n`'s children. @@ -374,12 +533,12 @@ proc `[]`*(n: XmlNode, i: int): XmlNode {.inline.} = assert $f[1] == "<first />" assert $f[0] == "<second />" - assert n.k == xnElement + n.expect xnElement result = n.s[i] proc `[]`*(n: var XmlNode, i: int): var XmlNode {.inline.} = ## Returns the `i`'th child of `n` so that it can be modified. - assert n.k == xnElement + n.expect xnElement result = n.s[i] proc clear*(n: var XmlNode) = @@ -389,13 +548,13 @@ proc clear*(n: var XmlNode) = var g = newElement("myTag") g.add newText("some text") g.add newComment("this is comment") - + var h = newElement("secondTag") h.add newEntity("some entity") - + let att = {"key1": "first value", "key2": "second value"}.toXmlAttributes var k = newXmlTree("treeTag", [g, h], att) - + doAssert $k == """<treeTag key1="first value" key2="second value"> <myTag>some text<!-- this is comment --></myTag> <secondTag>&some entity;</secondTag> @@ -431,12 +590,12 @@ iterator items*(n: XmlNode): XmlNode {.inline.} = # <!-- this is comment --> # <secondTag>&some entity;<![CDATA[some cdata]]></secondTag> - assert n.k == xnElement + n.expect xnElement for i in 0 .. n.len-1: yield n[i] iterator mitems*(n: var XmlNode): var XmlNode {.inline.} = ## Iterates over all direct children of `n` so that they can be modified. - assert n.k == xnElement + n.expect xnElement for i in 0 .. n.len-1: yield n[i] proc toXmlAttributes*(keyValuePairs: varargs[tuple[key, @@ -447,7 +606,7 @@ proc toXmlAttributes*(keyValuePairs: varargs[tuple[key, let att = {"key1": "first value", "key2": "second value"}.toXmlAttributes var j = newElement("myTag") j.attrs = att - + doAssert $j == """<myTag key1="first value" key2="second value" />""" newStringTable(keyValuePairs) @@ -468,7 +627,7 @@ proc attrs*(n: XmlNode): XmlAttributes {.inline.} = j.attrs = att assert j.attrs == att - assert n.k == xnElement + n.expect xnElement result = n.fAttr proc `attrs=`*(n: XmlNode, attr: XmlAttributes) {.inline.} = @@ -485,7 +644,7 @@ proc `attrs=`*(n: XmlNode, attr: XmlAttributes) {.inline.} = j.attrs = att assert j.attrs == att - assert n.k == xnElement + n.expect xnElement n.fAttr = attr proc attrsLen*(n: XmlNode): int {.inline.} = @@ -502,7 +661,7 @@ proc attrsLen*(n: XmlNode): int {.inline.} = j.attrs = att assert j.attrsLen == 2 - assert n.k == xnElement + n.expect xnElement if not isNil(n.fAttr): result = len(n.fAttr) proc attr*(n: XmlNode, name: string): string = @@ -520,7 +679,7 @@ proc attr*(n: XmlNode, name: string): string = assert j.attr("key1") == "first value" assert j.attr("key2") == "second value" - assert n.kind == xnElement + n.expect xnElement if n.attrs == nil: return "" return n.attrs.getOrDefault(name) @@ -552,15 +711,15 @@ proc escape*(s: string): string = ## ## Escapes these characters: ## - ## ------------ ------------------- + ## ============ =================== ## char is converted to - ## ------------ ------------------- + ## ============ =================== ## ``<`` ``<`` ## ``>`` ``>`` ## ``&`` ``&`` ## ``"`` ``"`` ## ``'`` ``'`` - ## ------------ ------------------- + ## ============ =================== ## ## You can also use `addEscaped proc <#addEscaped,string,string>`_. result = newStringOfCap(s.len) @@ -572,23 +731,11 @@ proc addIndent(result: var string, indent: int, addNewLines: bool) = for i in 1 .. indent: result.add(' ') -proc add*(result: var string, n: XmlNode, indent = 0, indWidth = 2, - addNewLines = true) = - ## Adds the textual representation of `n` to string `result`. - runnableExamples: - var - a = newElement("firstTag") - b = newText("my text") - c = newComment("my comment") - s = "" - s.add(c) - s.add(a) - s.add(b) - assert s == "<!-- my comment --><firstTag />my text" - +proc addImpl(result: var string, n: XmlNode, indent = 0, indWidth = 2, + addNewLines = true, lastNodeIsText = false) = proc noWhitespace(n: XmlNode): bool = for i in 0 ..< n.len: - if n[i].kind in {xnText, xnEntity}: return true + if n[i].kind in {xnText, xnVerbatimText, xnEntity}: return true proc addEscapedAttr(result: var string, s: string) = # `addEscaped` alternative with less escaped characters. @@ -605,7 +752,7 @@ proc add*(result: var string, n: XmlNode, indent = 0, indWidth = 2, case n.k of xnElement: - if indent > 0: + if indent > 0 and not lastNodeIsText: result.addIndent(indent, addNewLines) let @@ -634,8 +781,10 @@ proc add*(result: var string, n: XmlNode, indent = 0, indWidth = 2, else: indent+indWidth result.add('>') + var lastNodeIsText = false for i in 0 ..< n.len: - result.add(n[i], indentNext, indWidth, addNewLines) + result.addImpl(n[i], indentNext, indWidth, addNewLines, lastNodeIsText) + lastNodeIsText = (n[i].kind == xnText) or (n[i].kind == xnVerbatimText) if not n.noWhitespace(): result.addIndent(indent, addNewLines) @@ -660,6 +809,21 @@ proc add*(result: var string, n: XmlNode, indent = 0, indWidth = 2, result.add(n.fText) result.add(';') +proc add*(result: var string, n: XmlNode, indent = 0, indWidth = 2, + addNewLines = true) {.inline.} = + ## Adds the textual representation of `n` to string `result`. + runnableExamples: + var + a = newElement("firstTag") + b = newText("my text") + c = newComment("my comment") + s = "" + s.add(c) + s.add(a) + s.add(b) + assert s == "<!-- my comment --><firstTag />my text" + result.addImpl(n, indent, indWidth, addNewLines) + proc `$`*(n: XmlNode): string = ## Converts `n` into its string representation. ## @@ -678,7 +842,7 @@ proc child*(n: XmlNode, name: string): XmlNode = f.add newElement("thirdSon") assert $(f.child("secondSon")) == "<secondSon />" - assert n.kind == xnElement + n.expect xnElement for i in items(n): if i.kind == xnElement: if i.tag == name: @@ -714,7 +878,7 @@ proc findAll*(n: XmlNode, tag: string, result: var seq[XmlNode], a.findAll("BAD", s, caseInsensitive = true) assert $s == "@[<bad>c text</bad>, <BAD>d text</BAD>]" - assert n.k == xnElement + n.expect xnElement for child in n.items(): if child.k != xnElement: continue @@ -771,11 +935,12 @@ proc xmlConstructor(a: NimNode): NimNode = macro `<>`*(x: untyped): untyped = ## Constructor macro for XML. Example usage: ## - ## .. code-block:: nim + ## ```nim ## <>a(href="http://nim-lang.org", newText("Nim rules.")) + ## ``` ## - ## Produces an XML tree for:: + ## Produces an XML tree for: ## - ## <a href="http://nim-lang.org">Nim rules.</a> + ## <a href="http://nim-lang.org">Nim rules.</a> ## result = xmlConstructor(x) |