diff options
Diffstat (limited to 'lib/pure')
109 files changed, 7828 insertions, 10610 deletions
diff --git a/lib/pure/algorithm.nim b/lib/pure/algorithm.nim index fc0eceac3..b12ed7cdd 100644 --- a/lib/pure/algorithm.nim +++ b/lib/pure/algorithm.nim @@ -379,22 +379,22 @@ func sort*[T](a: var openArray[T], ## `cmp`, you may use `system.cmp` or instead call the overloaded ## version of `sort`, which uses `system.cmp`. ## - ## .. code-block:: nim - ## - ## sort(myIntArray, system.cmp[int]) - ## # do not use cmp[string] here as we want to use the specialized - ## # overload: - ## sort(myStrArray, system.cmp) + ## ```nim + ## sort(myIntArray, system.cmp[int]) + ## # do not use cmp[string] here as we want to use the specialized + ## # overload: + ## sort(myStrArray, system.cmp) + ## ``` ## ## You can inline adhoc comparison procs with the `do notation - ## <manual_experimental.html#do-notation>`_. Example: - ## - ## .. code-block:: nim + ## <manual.html#procedures-do-notation>`_. Example: ## + ## ```nim ## people.sort do (x, y: Person) -> int: ## result = cmp(x.surname, y.surname) ## if result == 0: ## result = cmp(x.name, y.name) + ## ``` ## ## **See also:** ## * `sort proc<#sort,openArray[T]>`_ diff --git a/lib/pure/async.nim b/lib/pure/async.nim index 482ab32c6..e4d8d41c3 100644 --- a/lib/pure/async.nim +++ b/lib/pure/async.nim @@ -1,6 +1,9 @@ +## Exports [asyncmacro](asyncmacro.html) and [asyncfutures](asyncfutures.html) for native backends, +## and [asyncjs](asyncjs.html) on the JS backend. + when defined(js): - import asyncjs + import std/asyncjs export asyncjs else: - import asyncmacro, asyncfutures + import std/[asyncmacro, asyncfutures] export asyncmacro, asyncfutures diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index 92ad9c5ff..126db7a7f 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -41,13 +41,13 @@ ## requested amount of data is read **or** an exception occurs. ## ## Code to read some data from a socket may look something like this: -## -## .. code-block:: Nim -## var future = socket.recv(100) -## future.addCallback( -## proc () = -## echo(future.read) -## ) +## ```Nim +## var future = socket.recv(100) +## future.addCallback( +## proc () = +## echo(future.read) +## ) +## ``` ## ## All asynchronous functions returning a `Future` will not block. They ## will not however return immediately. An asynchronous function will have @@ -108,24 +108,24 @@ ## You can handle exceptions in the same way as in ordinary Nim code; ## by using the try statement: ## -## -## .. code-block:: Nim +## ```Nim ## try: ## let data = await sock.recv(100) ## echo("Received ", data) ## except: ## # Handle exception -## -## +## ``` ## ## An alternative approach to handling exceptions is to use `yield` on a future ## then check the future's `failed` property. For example: ## -## .. code-block:: Nim +## ```Nim ## var future = sock.recv(100) ## yield future ## if future.failed: ## # Handle exception +## ``` +## ## ## Discarding futures ## ================== @@ -226,11 +226,11 @@ ## ``none`` can be used when a library supports both a synchronous and ## asynchronous API, to disable the latter. -import os, tables, strutils, times, heapqueue, options, asyncstreams -import options, math, std/monotimes -import asyncfutures except callSoon +import std/[os, tables, strutils, times, heapqueue, options, asyncstreams] +import std/[math, monotimes] +import std/asyncfutures except callSoon -import nativesockets, net, deques +import std/[nativesockets, net, deques] when defined(nimPreviewSlimSystem): import std/[assertions, syncio] @@ -281,6 +281,8 @@ proc adjustTimeout( result = max(nextTimer.get(), 0) result = min(pollTimeout, result) +proc runOnce(timeout: int): bool {.gcsafe.} + proc callSoon*(cbproc: proc () {.gcsafe.}) {.gcsafe.} ## Schedule `cbproc` to be called as soon as possible. ## The callback is called when control returns to the event loop. @@ -300,7 +302,7 @@ template implementSetInheritable() {.dirty.} = fd.FileHandle.setInheritable(inheritable) when defined(windows) or defined(nimdoc): - import winlean, sets, hashes + import std/[winlean, sets, hashes] type CompletionKey = ULONG_PTR @@ -390,7 +392,7 @@ when defined(windows) or defined(nimdoc): let p = getGlobalDispatcher() p.handles.len != 0 or p.timers.len != 0 or p.callbacks.len != 0 - proc runOnce(timeout = 500): bool = + proc runOnce(timeout: int): bool = let p = getGlobalDispatcher() if p.handles.len == 0 and p.timers.len == 0 and p.callbacks.len == 0: raise newException(ValueError, @@ -531,7 +533,7 @@ when defined(windows) or defined(nimdoc): if flags.isDisconnectionError(errcode): retFuture.complete("") else: - retFuture.fail(newException(OSError, osErrorMsg(errcode))) + retFuture.fail(newOSError(errcode)) if dataBuf.buf != nil: dealloc dataBuf.buf dataBuf.buf = nil @@ -549,7 +551,7 @@ when defined(windows) or defined(nimdoc): if flags.isDisconnectionError(err): retFuture.complete("") else: - retFuture.fail(newException(OSError, osErrorMsg(err))) + retFuture.fail(newOSError(err)) elif ret == 0: # Request completed immediately. if bytesReceived != 0: @@ -601,7 +603,7 @@ when defined(windows) or defined(nimdoc): if flags.isDisconnectionError(errcode): retFuture.complete(0) else: - retFuture.fail(newException(OSError, osErrorMsg(errcode))) + retFuture.fail(newOSError(errcode)) if dataBuf.buf != nil: dataBuf.buf = nil ) @@ -617,7 +619,7 @@ when defined(windows) or defined(nimdoc): if flags.isDisconnectionError(err): retFuture.complete(0) else: - retFuture.fail(newException(OSError, osErrorMsg(err))) + retFuture.fail(newOSError(err)) elif ret == 0: # Request completed immediately. if bytesReceived != 0: @@ -665,7 +667,7 @@ when defined(windows) or defined(nimdoc): if flags.isDisconnectionError(err): retFuture.complete() else: - retFuture.fail(newException(OSError, osErrorMsg(err))) + retFuture.fail(newOSError(err)) else: retFuture.complete() # We don't deallocate `ol` here because even though this completed @@ -700,7 +702,7 @@ when defined(windows) or defined(nimdoc): if errcode == OSErrorCode(-1): retFuture.complete() else: - retFuture.fail(newException(OSError, osErrorMsg(errcode))) + retFuture.fail(newOSError(errcode)) ) let ret = WSASendTo(socket.SocketHandle, addr dataBuf, 1, addr bytesSent, @@ -710,7 +712,7 @@ when defined(windows) or defined(nimdoc): let err = osLastError() if err.int32 != ERROR_IO_PENDING: GC_unref(ol) - retFuture.fail(newException(OSError, osErrorMsg(err))) + retFuture.fail(newOSError(err)) else: retFuture.complete() # We don't deallocate `ol` here because even though this completed @@ -744,7 +746,7 @@ when defined(windows) or defined(nimdoc): else: # datagram sockets don't have disconnection, # so we can just raise an exception - retFuture.fail(newException(OSError, osErrorMsg(errcode))) + retFuture.fail(newOSError(errcode)) ) let res = WSARecvFrom(socket.SocketHandle, addr dataBuf, 1, @@ -755,7 +757,7 @@ when defined(windows) or defined(nimdoc): let err = osLastError() if err.int32 != ERROR_IO_PENDING: GC_unref(ol) - retFuture.fail(newException(OSError, osErrorMsg(err))) + retFuture.fail(newOSError(err)) else: # Request completed immediately. if bytesReceived != 0: @@ -806,7 +808,7 @@ when defined(windows) or defined(nimdoc): else: retFuture.complete(newAcceptFut.read) else: - retFuture.fail(newException(OSError, osErrorMsg(errcode))) + retFuture.fail(newOSError(errcode)) template completeAccept() {.dirty.} = var listenSock = socket @@ -1164,11 +1166,14 @@ when defined(windows) or defined(nimdoc): initAll() else: - import selectors - from posix import EINTR, EAGAIN, EINPROGRESS, EWOULDBLOCK, MSG_PEEK, + import std/selectors + from std/posix import EINTR, EAGAIN, EINPROGRESS, EWOULDBLOCK, MSG_PEEK, MSG_NOSIGNAL when declared(posix.accept4): - from posix import accept4, SOCK_CLOEXEC + from std/posix import accept4, SOCK_CLOEXEC + when defined(genode): + import genode/env # get the implicit Genode env + import genode/signals const InitCallbackListSize = 4 # initial size of callbacks sequence, @@ -1187,6 +1192,8 @@ else: PDispatcher* = ref object of PDispatcherBase selector: Selector[AsyncData] + when defined(genode): + signalHandler: SignalHandler proc `==`*(x, y: AsyncFD): bool {.borrow.} proc `==`*(x, y: AsyncEvent): bool {.borrow.} @@ -1202,9 +1209,22 @@ else: result.selector = newSelector[AsyncData]() result.timers.clear() result.callbacks = initDeque[proc () {.closure, gcsafe.}](InitDelayedCallbackListSize) + when defined(genode): + let entrypoint = ep(cast[GenodeEnv](runtimeEnv)) + result.signalHandler = newSignalHandler(entrypoint): + discard runOnce(0) var gDisp{.threadvar.}: owned PDispatcher ## Global dispatcher + when defined(nuttx): + import std/exitprocs + + proc cleanDispatcher() {.noconv.} = + gDisp = nil + + proc addFinalyzer() = + addExitProc(cleanDispatcher) + proc setGlobalDispatcher*(disp: owned PDispatcher) = if not gDisp.isNil: assert gDisp.callbacks.len == 0 @@ -1214,6 +1234,8 @@ else: proc getGlobalDispatcher*(): PDispatcher = if gDisp.isNil: setGlobalDispatcher(newDispatcher()) + when defined(nuttx): + addFinalyzer() result = gDisp proc getIoHandler*(disp: PDispatcher): Selector[AsyncData] = @@ -1371,10 +1393,11 @@ else: ValueError, "Expecting async operations to stop when fd has closed." ) - - proc runOnce(timeout = 500): bool = + proc runOnce(timeout: int): bool = let p = getGlobalDispatcher() if p.selector.isEmpty() and p.timers.len == 0 and p.callbacks.len == 0: + when defined(genode): + if timeout == 0: return raise newException(ValueError, "No handles or timers registered in dispatcher.") @@ -1445,7 +1468,7 @@ else: if flags.isDisconnectionError(lastError): retFuture.complete("") else: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) + retFuture.fail(newOSError(lastError)) else: result = false # We still want this callback to be called. elif res == 0: @@ -1474,7 +1497,7 @@ else: if flags.isDisconnectionError(lastError): retFuture.complete(0) else: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) + retFuture.fail(newOSError(lastError)) else: result = false # We still want this callback to be called. else: @@ -1540,7 +1563,7 @@ else: let lastError = osLastError() if lastError.int32 != EINTR and lastError.int32 != EWOULDBLOCK and lastError.int32 != EAGAIN: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) + retFuture.fail(newOSError(lastError)) else: result = false # We still want this callback to be called. else: @@ -1566,7 +1589,7 @@ else: let lastError = osLastError() if lastError.int32 != EINTR and lastError.int32 != EWOULDBLOCK and lastError.int32 != EAGAIN: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) + retFuture.fail(newOSError(lastError)) else: result = false else: @@ -1579,7 +1602,7 @@ else: owned(Future[tuple[address: string, client: AsyncFD]]) = var retFuture = newFuture[tuple[address: string, client: AsyncFD]]("acceptAddr") - proc cb(sock: AsyncFD): bool = + proc cb(sock: AsyncFD): bool {.gcsafe.} = result = true var sockAddress: Sockaddr_storage var addrLen = sizeof(sockAddress).SockLen @@ -1607,7 +1630,7 @@ else: if flags.isDisconnectionError(lastError): return false else: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) + retFuture.fail(newOSError(lastError)) else: try: let address = getAddrString(cast[ptr SockAddr](addr sockAddress)) @@ -1740,7 +1763,7 @@ when defined(windows) or defined(nimdoc): socket.SocketHandle.setSockOptInt(SOL_SOCKET, SO_UPDATE_CONNECT_CONTEXT, 1) # 15022 retFuture.complete() else: - retFuture.fail(newException(OSError, osErrorMsg(errcode))) + retFuture.fail(newOSError(errcode)) ) let ret = connectEx(socket.SocketHandle, addrInfo.ai_addr, @@ -1758,7 +1781,7 @@ when defined(windows) or defined(nimdoc): # With ERROR_IO_PENDING `ol` will be deallocated in `poll`, # and the future will be completed/failed there, too. GC_unref(ol) - retFuture.fail(newException(OSError, osErrorMsg(lastError))) + retFuture.fail(newOSError(lastError)) else: proc doConnect(socket: AsyncFD, addrInfo: ptr AddrInfo): owned(Future[void]) = let retFuture = newFuture[void]("doConnect") @@ -1775,7 +1798,7 @@ else: # interrupted, keep waiting return false else: - retFuture.fail(newException(OSError, osErrorMsg(OSErrorCode(ret)))) + retFuture.fail(newOSError(OSErrorCode(ret))) return true let ret = connect(socket.SocketHandle, @@ -1789,7 +1812,7 @@ else: if lastError.int32 == EINTR or lastError.int32 == EINPROGRESS: addWrite(socket, cb) else: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) + retFuture.fail(newOSError(lastError)) template asyncAddrInfoLoop(addrInfo: ptr AddrInfo, fd: untyped, protocol: Protocol = IPPROTO_RAW) = @@ -1971,7 +1994,8 @@ proc send*(socket: AsyncFD, data: string, return retFuture # -- Await Macro -include asyncmacro +import std/asyncmacro +export asyncmacro proc readAll*(future: FutureStream[string]): owned(Future[string]) {.async.} = ## Returns a future that will complete when all the string data from the @@ -2008,10 +2032,10 @@ proc activeDescriptors*(): int {.inline.} = result = getGlobalDispatcher().selector.count when defined(posix): - import posix + import std/posix when defined(linux) or defined(windows) or defined(macosx) or defined(bsd) or - defined(solaris) or defined(zephyr) or defined(freertos): + defined(solaris) or defined(zephyr) or defined(freertos) or defined(nuttx) or defined(haiku): proc maxDescriptors*(): int {.raises: OSError.} = ## Returns the maximum number of active file descriptors for the current ## process. This involves a system call. For now `maxDescriptors` is @@ -2025,3 +2049,17 @@ when defined(linux) or defined(windows) or defined(macosx) or defined(bsd) or if getrlimit(RLIMIT_NOFILE, fdLim) < 0: raiseOSError(osLastError()) result = int(fdLim.rlim_cur) - 1 + +when defined(genode): + proc scheduleCallbacks*(): bool {.discardable.} = + ## *Genode only.* + ## Schedule callback processing and return immediately. + ## Returns `false` if there is nothing to schedule. + ## RPC servers should call this to dispatch `callSoon` + ## bodies after retiring an RPC to its client. + ## This is effectively a non-blocking `poll(…)` and is + ## equivalent to scheduling a momentary no-op timeout + ## but faster and with less overhead. + let dis = getGlobalDispatcher() + result = dis.callbacks.len > 0 + if result: submit(dis.signalHandler.cap) diff --git a/lib/pure/asyncfile.nim b/lib/pure/asyncfile.nim index 9cc9f5b48..0f6504342 100644 --- a/lib/pure/asyncfile.nim +++ b/lib/pure/asyncfile.nim @@ -9,30 +9,33 @@ ## This module implements asynchronous file reading and writing. ## -## .. code-block:: Nim -## import std/[asyncfile, asyncdispatch, os] +## ```Nim +## import std/[asyncfile, asyncdispatch, os] ## -## proc main() {.async.} = -## var file = openAsync(getTempDir() / "foobar.txt", fmReadWrite) -## await file.write("test") -## file.setFilePos(0) -## let data = await file.readAll() -## doAssert data == "test" -## file.close() +## proc main() {.async.} = +## var file = openAsync(getTempDir() / "foobar.txt", fmReadWrite) +## await file.write("test") +## file.setFilePos(0) +## let data = await file.readAll() +## doAssert data == "test" +## file.close() ## -## waitFor main() +## waitFor main() +## ``` -import asyncdispatch, os +import std/[asyncdispatch, os] when defined(nimPreviewSlimSystem): import std/[assertions, syncio] + when defined(windows) or defined(nimdoc): + import std/widestrs # TODO: Fix duplication introduced by PR #4683. when defined(windows) or defined(nimdoc): - import winlean + import std/winlean else: - import posix + import std/posix type AsyncFile* = ref object @@ -99,14 +102,9 @@ proc openAsync*(filename: string, mode = fmRead): AsyncFile = let flags = FILE_FLAG_OVERLAPPED or FILE_ATTRIBUTE_NORMAL let desiredAccess = getDesiredAccess(mode) let creationDisposition = getCreationDisposition(mode, filename) - when useWinUnicode: - let fd = createFileW(newWideCString(filename), desiredAccess, - FILE_SHARE_READ, - nil, creationDisposition, flags, 0) - else: - let fd = createFileA(filename, desiredAccess, - FILE_SHARE_READ, - nil, creationDisposition, flags, 0) + let fd = createFileW(newWideCString(filename), desiredAccess, + FILE_SHARE_READ, + nil, creationDisposition, flags, 0) if fd == INVALID_HANDLE_VALUE: raiseOSError(osLastError()) @@ -148,7 +146,7 @@ proc readBuffer*(f: AsyncFile, buf: pointer, size: int): Future[int] = if errcode.int32 == ERROR_HANDLE_EOF: retFuture.complete(0) else: - retFuture.fail(newException(OSError, osErrorMsg(errcode))) + retFuture.fail(newOSError(errcode)) ) ol.offset = DWORD(f.offset and 0xffffffff) ol.offsetHigh = DWORD(f.offset shr 32) @@ -164,7 +162,7 @@ proc readBuffer*(f: AsyncFile, buf: pointer, size: int): Future[int] = # This happens in Windows Server 2003 retFuture.complete(0) else: - retFuture.fail(newException(OSError, osErrorMsg(err))) + retFuture.fail(newOSError(err)) else: # Request completed immediately. var bytesRead: DWORD @@ -175,7 +173,7 @@ proc readBuffer*(f: AsyncFile, buf: pointer, size: int): Future[int] = if err.int32 == ERROR_HANDLE_EOF: retFuture.complete(0) else: - retFuture.fail(newException(OSError, osErrorMsg(osLastError()))) + retFuture.fail(newOSError(osLastError())) else: assert bytesRead > 0 assert bytesRead <= size @@ -188,7 +186,7 @@ proc readBuffer*(f: AsyncFile, buf: pointer, size: int): Future[int] = if res < 0: let lastError = osLastError() if lastError.int32 != EAGAIN: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) + retFuture.fail(newOSError(lastError)) else: result = false # We still want this callback to be called. elif res == 0: @@ -230,7 +228,7 @@ proc read*(f: AsyncFile, size: int): Future[string] = if errcode.int32 == ERROR_HANDLE_EOF: retFuture.complete("") else: - retFuture.fail(newException(OSError, osErrorMsg(errcode))) + retFuture.fail(newOSError(errcode)) if buffer != nil: dealloc buffer buffer = nil @@ -253,7 +251,7 @@ proc read*(f: AsyncFile, size: int): Future[string] = # This happens in Windows Server 2003 retFuture.complete("") else: - retFuture.fail(newException(OSError, osErrorMsg(err))) + retFuture.fail(newOSError(err)) else: # Request completed immediately. var bytesRead: DWORD @@ -264,7 +262,7 @@ proc read*(f: AsyncFile, size: int): Future[string] = if err.int32 == ERROR_HANDLE_EOF: retFuture.complete("") else: - retFuture.fail(newException(OSError, osErrorMsg(osLastError()))) + retFuture.fail(newOSError(osLastError())) else: assert bytesRead > 0 assert bytesRead <= size @@ -281,7 +279,7 @@ proc read*(f: AsyncFile, size: int): Future[string] = if res < 0: let lastError = osLastError() if lastError.int32 != EAGAIN: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) + retFuture.fail(newOSError(lastError)) else: result = false # We still want this callback to be called. elif res == 0: @@ -303,6 +301,8 @@ proc readLine*(f: AsyncFile): Future[string] {.async.} = result = "" while true: var c = await read(f, 1) + if c.len == 0: + break if c[0] == '\c': c = await read(f, 1) break @@ -350,7 +350,7 @@ proc writeBuffer*(f: AsyncFile, buf: pointer, size: int): Future[void] = assert bytesCount == size.int32 retFuture.complete() else: - retFuture.fail(newException(OSError, osErrorMsg(errcode))) + retFuture.fail(newOSError(errcode)) ) # passing -1 here should work according to MSDN, but doesn't. For more # information see @@ -367,14 +367,14 @@ proc writeBuffer*(f: AsyncFile, buf: pointer, size: int): Future[void] = let err = osLastError() if err.int32 != ERROR_IO_PENDING: GC_unref(ol) - retFuture.fail(newException(OSError, osErrorMsg(err))) + retFuture.fail(newOSError(err)) else: # Request completed immediately. var bytesWritten: DWORD let overlappedRes = getOverlappedResult(f.fd.Handle, cast[POVERLAPPED](ol), bytesWritten, false.WINBOOL) if not overlappedRes.bool: - retFuture.fail(newException(OSError, osErrorMsg(osLastError()))) + retFuture.fail(newOSError(osLastError())) else: assert bytesWritten == size.int32 retFuture.complete() @@ -389,7 +389,7 @@ proc writeBuffer*(f: AsyncFile, buf: pointer, size: int): Future[void] = if res < 0: let lastError = osLastError() if lastError.int32 != EAGAIN: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) + retFuture.fail(newOSError(lastError)) else: result = false # We still want this callback to be called. else: @@ -423,7 +423,7 @@ proc write*(f: AsyncFile, data: string): Future[void] = assert bytesCount == data.len.int32 retFuture.complete() else: - retFuture.fail(newException(OSError, osErrorMsg(errcode))) + retFuture.fail(newOSError(errcode)) if buffer != nil: dealloc buffer buffer = nil @@ -442,14 +442,14 @@ proc write*(f: AsyncFile, data: string): Future[void] = dealloc buffer buffer = nil GC_unref(ol) - retFuture.fail(newException(OSError, osErrorMsg(err))) + retFuture.fail(newOSError(err)) else: # Request completed immediately. var bytesWritten: DWORD let overlappedRes = getOverlappedResult(f.fd.Handle, cast[POVERLAPPED](ol), bytesWritten, false.WINBOOL) if not overlappedRes.bool: - retFuture.fail(newException(OSError, osErrorMsg(osLastError()))) + retFuture.fail(newOSError(osLastError())) else: assert bytesWritten == data.len.int32 retFuture.complete() @@ -470,7 +470,7 @@ proc write*(f: AsyncFile, data: string): Future[void] = if res < 0: let lastError = osLastError() if lastError.int32 != EAGAIN: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) + retFuture.fail(newOSError(lastError)) else: result = false # We still want this callback to be called. else: diff --git a/lib/pure/asyncftpclient.nim b/lib/pure/asyncftpclient.nim deleted file mode 100644 index 0d2ee80c5..000000000 --- a/lib/pure/asyncftpclient.nim +++ /dev/null @@ -1,451 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Dominik Picheta -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## This module implements an asynchronous FTP client. It allows you to connect -## to an FTP server and perform operations on it such as for example: -## -## * The upload of new files. -## * The removal of existing files. -## * Download of files. -## * Changing of files' permissions. -## * Navigation through the FTP server's directories. -## -## Connecting to an FTP server -## =========================== -## -## In order to begin any sort of transfer of files you must first -## connect to an FTP server. You can do so with the `connect` procedure. -## -## .. code-block:: Nim -## import std/[asyncdispatch, asyncftpclient] -## proc main() {.async.} = -## var ftp = newAsyncFtpClient("example.com", user = "test", pass = "test") -## await ftp.connect() -## echo("Connected") -## waitFor(main()) -## -## A new `main` async procedure must be declared to allow the use of the -## `await` keyword. The connection will complete asynchronously and the -## client will be connected after the `await ftp.connect()` call. -## -## Uploading a new file -## ==================== -## -## After a connection is made you can use the `store` procedure to upload -## a new file to the FTP server. Make sure to check you are in the correct -## working directory before you do so with the `pwd` procedure, you can also -## instead specify an absolute path. -## -## .. code-block:: Nim -## import std/[asyncdispatch, asyncftpclient] -## proc main() {.async.} = -## var ftp = newAsyncFtpClient("example.com", user = "test", pass = "test") -## await ftp.connect() -## let currentDir = await ftp.pwd() -## assert currentDir == "/home/user/" -## await ftp.store("file.txt", "file.txt") -## echo("File finished uploading") -## waitFor(main()) -## -## Checking the progress of a file transfer -## ======================================== -## -## The progress of either a file upload or a file download can be checked -## by specifying a `onProgressChanged` procedure to the `store` or -## `retrFile` procedures. -## -## Procs that take an `onProgressChanged` callback will call this every -## `progressInterval` milliseconds. -## -## .. code-block:: Nim -## import std/[asyncdispatch, asyncftpclient] -## -## proc onProgressChanged(total, progress: BiggestInt, -## speed: float) {.async.} = -## echo("Uploaded ", progress, " of ", total, " bytes") -## echo("Current speed: ", speed, " kb/s") -## -## proc main() {.async.} = -## var ftp = newAsyncFtpClient("example.com", user = "test", pass = "test", progressInterval = 500) -## await ftp.connect() -## await ftp.store("file.txt", "/home/user/file.txt", onProgressChanged) -## echo("File finished uploading") -## waitFor(main()) - - -import asyncdispatch, asyncnet, nativesockets, strutils, parseutils, os, times -from net import BufferSize - -when defined(nimPreviewSlimSystem): - import std/assertions - -type - AsyncFtpClient* = ref object - csock*: AsyncSocket - dsock*: AsyncSocket - user*, pass*: string - address*: string - port*: Port - progressInterval: int - jobInProgress*: bool - job*: FtpJob - dsockConnected*: bool - - FtpJobType* = enum - JRetrText, JRetr, JStore - - FtpJob = ref object - prc: proc (ftp: AsyncFtpClient, async: bool): bool {.nimcall, gcsafe.} - case typ*: FtpJobType - of JRetrText: - lines: string - of JRetr, JStore: - file: File - filename: string - total: BiggestInt # In bytes. - progress: BiggestInt # In bytes. - oneSecond: BiggestInt # Bytes transferred in one second. - lastProgressReport: float # Time - toStore: string # Data left to upload (Only used with async) - - FtpEventType* = enum - EvTransferProgress, EvLines, EvRetr, EvStore - - FtpEvent* = object ## Event - filename*: string - case typ*: FtpEventType - of EvLines: - lines*: string ## Lines that have been transferred. - of EvRetr, EvStore: ## Retr/Store operation finished. - nil - of EvTransferProgress: - bytesTotal*: BiggestInt ## Bytes total. - bytesFinished*: BiggestInt ## Bytes transferred. - speed*: BiggestInt ## Speed in bytes/s - currentJob*: FtpJobType ## The current job being performed. - - ReplyError* = object of IOError - - ProgressChangedProc* = - proc (total, progress: BiggestInt, speed: float): - Future[void] {.closure, gcsafe.} - -const multiLineLimit = 10000 - -proc expectReply(ftp: AsyncFtpClient): Future[string] {.async.} = - var line = await ftp.csock.recvLine() - result = line - var count = 0 - while line.len > 3 and line[3] == '-': - ## Multi-line reply. - line = await ftp.csock.recvLine() - result.add("\n" & line) - count.inc() - if count >= multiLineLimit: - raise newException(ReplyError, "Reached maximum multi-line reply count.") - -proc send*(ftp: AsyncFtpClient, m: string): Future[string] {.async.} = - ## Send a message to the server, and wait for a primary reply. - ## `\c\L` is added for you. - ## - ## You need to make sure that the message `m` doesn't contain any newline - ## characters. Failing to do so will raise `AssertionDefect`. - ## - ## **Note:** The server may return multiple lines of coded replies. - doAssert(not m.contains({'\c', '\L'}), "message shouldn't contain any newline characters") - await ftp.csock.send(m & "\c\L") - return await ftp.expectReply() - -proc assertReply(received: string, expected: varargs[string]) = - for i in items(expected): - if received.startsWith(i): return - raise newException(ReplyError, - "Expected reply '$1' got: $2" % - [expected.join("' or '"), received]) - -proc pasv(ftp: AsyncFtpClient) {.async.} = - ## Negotiate a data connection. - ftp.dsock = newAsyncSocket() - - var pasvMsg = (await ftp.send("PASV")).strip - assertReply(pasvMsg, "227") - var betweenParens = captureBetween(pasvMsg, '(', ')') - var nums = betweenParens.split(',') - var ip = nums[0 .. ^3] - var port = nums[^2 .. ^1] - var properPort = port[0].parseInt()*256+port[1].parseInt() - await ftp.dsock.connect(ip.join("."), Port(properPort)) - ftp.dsockConnected = true - -proc normalizePathSep(path: string): string = - return replace(path, '\\', '/') - -proc connect*(ftp: AsyncFtpClient) {.async.} = - ## Connect to the FTP server specified by `ftp`. - await ftp.csock.connect(ftp.address, ftp.port) - - var reply = await ftp.expectReply() - if reply.startsWith("120"): - # 120 Service ready in nnn minutes. - # We wait until we receive 220. - reply = await ftp.expectReply() - - # Handle 220 messages from the server - assertReply(reply, "220") - - if ftp.user != "": - assertReply(await(ftp.send("USER " & ftp.user)), "230", "331") - - if ftp.pass != "": - assertReply(await(ftp.send("PASS " & ftp.pass)), "230") - -proc pwd*(ftp: AsyncFtpClient): Future[string] {.async.} = - ## Returns the current working directory. - let wd = await ftp.send("PWD") - assertReply wd, "257" - return wd.captureBetween('"') # " - -proc cd*(ftp: AsyncFtpClient, dir: string) {.async.} = - ## Changes the current directory on the remote FTP server to `dir`. - assertReply(await(ftp.send("CWD " & dir.normalizePathSep)), "250") - -proc cdup*(ftp: AsyncFtpClient) {.async.} = - ## Changes the current directory to the parent of the current directory. - assertReply(await(ftp.send("CDUP")), "200") - -proc getLines(ftp: AsyncFtpClient): Future[string] {.async.} = - ## Downloads text data in ASCII mode - result = "" - assert ftp.dsockConnected - while ftp.dsockConnected: - let r = await ftp.dsock.recvLine() - if r == "": - ftp.dsockConnected = false - else: - result.add(r & "\n") - - assertReply(await(ftp.expectReply()), "226") - -proc listDirs*(ftp: AsyncFtpClient, dir = ""): Future[seq[string]] {.async.} = - ## Returns a list of filenames in the given directory. If `dir` is "", - ## the current directory is used. If `async` is true, this - ## function will return immediately and it will be your job to - ## use asyncdispatch's `poll` to progress this operation. - await ftp.pasv() - - assertReply(await(ftp.send("NLST " & dir.normalizePathSep)), ["125", "150"]) - - result = splitLines(await ftp.getLines()) - -proc fileExists*(ftp: AsyncFtpClient, file: string): Future[bool] {.async.} = - ## Determines whether `file` exists. - var files = await ftp.listDirs() - for f in items(files): - if f.normalizePathSep == file.normalizePathSep: return true - -proc createDir*(ftp: AsyncFtpClient, dir: string, recursive = false){.async.} = - ## Creates a directory `dir`. If `recursive` is true, the topmost - ## subdirectory of `dir` will be created first, following the secondmost... - ## etc. this allows you to give a full path as the `dir` without worrying - ## about subdirectories not existing. - if not recursive: - assertReply(await(ftp.send("MKD " & dir.normalizePathSep)), "257") - else: - var reply = "" - var previousDirs = "" - for p in split(dir, {os.DirSep, os.AltSep}): - if p != "": - previousDirs.add(p) - reply = await ftp.send("MKD " & previousDirs) - previousDirs.add('/') - assertReply reply, "257" - -proc chmod*(ftp: AsyncFtpClient, path: string, - permissions: set[FilePermission]) {.async.} = - ## Changes permission of `path` to `permissions`. - var userOctal = 0 - var groupOctal = 0 - var otherOctal = 0 - for i in items(permissions): - case i - of fpUserExec: userOctal.inc(1) - of fpUserWrite: userOctal.inc(2) - of fpUserRead: userOctal.inc(4) - of fpGroupExec: groupOctal.inc(1) - of fpGroupWrite: groupOctal.inc(2) - of fpGroupRead: groupOctal.inc(4) - of fpOthersExec: otherOctal.inc(1) - of fpOthersWrite: otherOctal.inc(2) - of fpOthersRead: otherOctal.inc(4) - - var perm = $userOctal & $groupOctal & $otherOctal - assertReply(await(ftp.send("SITE CHMOD " & perm & - " " & path.normalizePathSep)), "200") - -proc list*(ftp: AsyncFtpClient, dir = ""): Future[string] {.async.} = - ## Lists all files in `dir`. If `dir` is `""`, uses the current - ## working directory. - await ftp.pasv() - - let reply = await ftp.send("LIST" & " " & dir.normalizePathSep) - assertReply(reply, ["125", "150"]) - - result = await ftp.getLines() - -proc retrText*(ftp: AsyncFtpClient, file: string): Future[string] {.async.} = - ## Retrieves `file`. File must be ASCII text. - await ftp.pasv() - let reply = await ftp.send("RETR " & file.normalizePathSep) - assertReply(reply, ["125", "150"]) - - result = await ftp.getLines() - -proc getFile(ftp: AsyncFtpClient, file: File, total: BiggestInt, - onProgressChanged: ProgressChangedProc) {.async.} = - assert ftp.dsockConnected - var progress = 0 - var progressInSecond = 0 - var countdownFut = sleepAsync(ftp.progressInterval) - var dataFut = ftp.dsock.recv(BufferSize) - while ftp.dsockConnected: - await dataFut or countdownFut - if countdownFut.finished: - asyncCheck onProgressChanged(total, progress, - progressInSecond.float) - progressInSecond = 0 - countdownFut = sleepAsync(ftp.progressInterval) - - if dataFut.finished: - let data = dataFut.read - if data != "": - progress.inc(data.len) - progressInSecond.inc(data.len) - file.write(data) - dataFut = ftp.dsock.recv(BufferSize) - else: - ftp.dsockConnected = false - - assertReply(await(ftp.expectReply()), "226") - -proc defaultOnProgressChanged*(total, progress: BiggestInt, - speed: float): Future[void] {.nimcall, gcsafe.} = - ## Default FTP `onProgressChanged` handler. Does nothing. - result = newFuture[void]() - #echo(total, " ", progress, " ", speed) - result.complete() - -proc retrFile*(ftp: AsyncFtpClient, file, dest: string, - onProgressChanged: ProgressChangedProc = defaultOnProgressChanged) {.async.} = - ## Downloads `file` and saves it to `dest`. - ## The `EvRetr` event is passed to the specified `handleEvent` function - ## when the download is finished. The event's `filename` field will be equal - ## to `file`. - var destFile = open(dest, mode = fmWrite) - await ftp.pasv() - var reply = await ftp.send("RETR " & file.normalizePathSep) - assertReply reply, ["125", "150"] - if {'(', ')'} notin reply: - raise newException(ReplyError, "Reply has no file size.") - var fileSize: BiggestInt - if reply.captureBetween('(', ')').parseBiggestInt(fileSize) == 0: - raise newException(ReplyError, "Reply has no file size.") - - await getFile(ftp, destFile, fileSize, onProgressChanged) - destFile.close() - -proc doUpload(ftp: AsyncFtpClient, file: File, - onProgressChanged: ProgressChangedProc) {.async.} = - assert ftp.dsockConnected - - let total = file.getFileSize() - var data = newString(4000) - var progress = 0 - var progressInSecond = 0 - var countdownFut = sleepAsync(ftp.progressInterval) - var sendFut: Future[void] = nil - while ftp.dsockConnected: - if sendFut == nil or sendFut.finished: - # TODO: Async file reading. - let len = file.readBuffer(addr(data[0]), 4000) - setLen(data, len) - if len == 0: - # File finished uploading. - ftp.dsock.close() - ftp.dsockConnected = false - - assertReply(await(ftp.expectReply()), "226") - else: - progress.inc(len) - progressInSecond.inc(len) - sendFut = ftp.dsock.send(data) - - if countdownFut.finished: - asyncCheck onProgressChanged(total, progress, progressInSecond.float) - progressInSecond = 0 - countdownFut = sleepAsync(ftp.progressInterval) - - await countdownFut or sendFut - -proc store*(ftp: AsyncFtpClient, file, dest: string, - onProgressChanged: ProgressChangedProc = defaultOnProgressChanged) {.async.} = - ## Uploads `file` to `dest` on the remote FTP server. Usage of this - ## function asynchronously is recommended to view the progress of - ## the download. - ## The `EvStore` event is passed to the specified `handleEvent` function - ## when the upload is finished, and the `filename` field will be - ## equal to `file`. - var destFile = open(file) - await ftp.pasv() - - let reply = await ftp.send("STOR " & dest.normalizePathSep) - assertReply reply, ["125", "150"] - - await doUpload(ftp, destFile, onProgressChanged) - -proc rename*(ftp: AsyncFtpClient, nameFrom: string, nameTo: string) {.async.} = - ## Rename a file or directory on the remote FTP Server from current name - ## `name_from` to new name `name_to` - assertReply(await ftp.send("RNFR " & nameFrom), "350") - assertReply(await ftp.send("RNTO " & nameTo), "250") - -proc removeFile*(ftp: AsyncFtpClient, filename: string) {.async.} = - ## Delete a file `filename` on the remote FTP server - assertReply(await ftp.send("DELE " & filename), "250") - -proc removeDir*(ftp: AsyncFtpClient, dir: string) {.async.} = - ## Delete a directory `dir` on the remote FTP server - assertReply(await ftp.send("RMD " & dir), "250") - -proc newAsyncFtpClient*(address: string, port = Port(21), - user, pass = "", progressInterval: int = 1000): AsyncFtpClient = - ## Creates a new `AsyncFtpClient` object. - new result - result.user = user - result.pass = pass - result.address = address - result.port = port - result.progressInterval = progressInterval - result.dsockConnected = false - result.csock = newAsyncSocket() - -when not defined(testing) and isMainModule: - var ftp = newAsyncFtpClient("example.com", user = "test", pass = "test") - proc main(ftp: AsyncFtpClient) {.async.} = - await ftp.connect() - echo await ftp.pwd() - echo await ftp.listDirs() - await ftp.store("payload.jpg", "payload.jpg") - await ftp.retrFile("payload.jpg", "payload2.jpg") - await ftp.rename("payload.jpg", "payload_renamed.jpg") - await ftp.store("payload.jpg", "payload_remove.jpg") - await ftp.removeFile("payload_remove.jpg") - await ftp.createDir("deleteme") - await ftp.removeDir("deleteme") - echo("Finished") - - waitFor main(ftp) diff --git a/lib/pure/asyncfutures.nim b/lib/pure/asyncfutures.nim index 035b6182d..29ebf8f89 100644 --- a/lib/pure/asyncfutures.nim +++ b/lib/pure/asyncfutures.nim @@ -7,7 +7,7 @@ # distribution, for details about the copyright. # -import os, tables, strutils, times, heapqueue, options, deques, cstrutils +import std/[os, tables, strutils, times, heapqueue, options, deques, cstrutils, typetraits] import system/stacktraces @@ -29,7 +29,7 @@ type finished: bool error*: ref Exception ## Stored exception errorStackTrace*: string - when not defined(release): + when not defined(release) or defined(futureLogging): stackTrace: seq[StackTraceEntry] ## For debugging purposes only. id: int fromProc: string @@ -51,7 +51,7 @@ const NimAsyncContinueSuffix* = "NimAsyncContinue" ## For internal usage. Do not use. when isFutureLoggingEnabled: - import hashes + import std/hashes type FutureInfo* = object stackTrace*: seq[StackTraceEntry] @@ -193,7 +193,7 @@ proc add(callbacks: var CallbackList, function: CallbackFunc) = last = last.next last.next = newCallback -proc completeImpl[T, U](future: Future[T], val: U, isVoid: static bool) = +proc completeImpl[T, U](future: Future[T], val: sink U, isVoid: static bool) = #assert(not future.finished, "Future already finished, cannot finish twice.") checkFinished(future) assert(future.error == nil) @@ -203,7 +203,7 @@ proc completeImpl[T, U](future: Future[T], val: U, isVoid: static bool) = future.callbacks.call() when isFutureLoggingEnabled: logFutureFinish(future) -proc complete*[T](future: Future[T], val: T) = +proc complete*[T](future: Future[T], val: sink T) = ## Completes `future` with value `val`. completeImpl(future, val, false) @@ -219,7 +219,7 @@ proc complete*[T](future: FutureVar[T]) = fut.callbacks.call() when isFutureLoggingEnabled: logFutureFinish(Future[T](future)) -proc complete*[T](future: FutureVar[T], val: T) = +proc complete*[T](future: FutureVar[T], val: sink T) = ## Completes a `FutureVar` with value `val`. ## ## Any previously stored value will be overwritten. @@ -329,29 +329,21 @@ proc `$`*(stackTraceEntries: seq[StackTraceEntry]): string = if leftLen > longestLeft: longestLeft = leftLen - var indent = 2 # Format the entries. for entry in entries: let (filename, procname) = getFilenameProcname(entry) - if procname == "": - if entry.line == reraisedFromBegin: - result.add(spaces(indent) & "#[\n") - indent.inc(2) - elif entry.line == reraisedFromEnd: - indent.dec(2) - result.add(spaces(indent) & "]#\n") - continue + if procname == "" and entry.line == reraisedFromBegin: + break let left = "$#($#)" % [filename, $entry.line] - result.add((spaces(indent) & "$#$# $#\n") % [ + result.add((spaces(2) & "$# $#\n") % [ left, - spaces(longestLeft - left.len + 2), procname ]) let hint = getHint(entry) if hint.len > 0: - result.add(spaces(indent+2) & "## " & hint & "\n") + result.add(spaces(4) & "## " & hint & "\n") proc injectStacktrace[T](future: Future[T]) = when not defined(release): @@ -378,11 +370,7 @@ proc injectStacktrace[T](future: Future[T]) = # newMsg.add "\n" & $entry future.error.msg = newMsg -proc read*[T](future: Future[T] | FutureVar[T]): T = - ## Retrieves the value of `future`. Future must be finished otherwise - ## this function will fail with a `ValueError` exception. - ## - ## If the result of the future is an error then that error will be raised. +template readImpl(future, T) = when future is Future[T]: let fut {.cursor.} = future else: @@ -392,11 +380,21 @@ proc read*[T](future: Future[T] | FutureVar[T]): T = injectStacktrace(fut) raise fut.error when T isnot void: - result = fut.value + result = distinctBase(future).value else: # TODO: Make a custom exception type for this? raise newException(ValueError, "Future still in progress.") +proc read*[T](future: Future[T] | FutureVar[T]): lent T = + ## Retrieves the value of `future`. Future must be finished otherwise + ## this function will fail with a `ValueError` exception. + ## + ## If the result of the future is an error then that error will be raised. + readImpl(future, T) + +proc read*(future: Future[void] | FutureVar[void]) = + readImpl(future, void) + proc readError*[T](future: Future[T]): ref Exception = ## Retrieves the exception stored in `future`. ## diff --git a/lib/pure/asynchttpserver.nim b/lib/pure/asynchttpserver.nim index 6694c4bc2..39e945d5e 100644 --- a/lib/pure/asynchttpserver.nim +++ b/lib/pure/asynchttpserver.nim @@ -39,9 +39,9 @@ runnableExamples("-r:off"): waitFor main() -import asyncnet, asyncdispatch, parseutils, uri, strutils -import httpcore -from nativesockets import getLocalAddr, Domain, AF_INET, AF_INET6 +import std/[asyncnet, asyncdispatch, parseutils, uri, strutils] +import std/httpcore +from std/nativesockets import getLocalAddr, Domain, AF_INET, AF_INET6 import std/private/since when defined(nimPreviewSlimSystem): @@ -110,16 +110,16 @@ proc respond*(req: Request, code: HttpCode, content: string, ## This procedure will **not** close the client socket. ## ## Example: - ## - ## .. code-block:: Nim - ## import std/json - ## proc handler(req: Request) {.async.} = - ## if req.url.path == "/hello-world": - ## let msg = %* {"message": "Hello World"} - ## let headers = newHttpHeaders([("Content-Type","application/json")]) - ## await req.respond(Http200, $msg, headers) - ## else: - ## await req.respond(Http404, "Not Found") + ## ```Nim + ## import std/json + ## proc handler(req: Request) {.async.} = + ## if req.url.path == "/hello-world": + ## let msg = %* {"message": "Hello World"} + ## let headers = newHttpHeaders([("Content-Type","application/json")]) + ## await req.respond(Http200, $msg, headers) + ## else: + ## await req.respond(Http404, "Not Found") + ## ``` var msg = "HTTP/1.1 " & $code & "\c\L" if headers != nil: @@ -158,7 +158,7 @@ proc parseProtocol(protocol: string): tuple[orig: string, major, minor: int] = proc sendStatus(client: AsyncSocket, status: string): Future[void] = client.send("HTTP/1.1 " & status & "\c\L\c\L") -func hasChunkedEncoding(request: Request): bool = +func hasChunkedEncoding(request: Request): bool = ## Searches for a chunked transfer encoding const transferEncoding = "Transfer-Encoding" @@ -187,7 +187,7 @@ proc processRequest( # \n request.headers.clear() request.body = "" - when defined(gcArc) or defined(gcOrc): + when defined(gcArc) or defined(gcOrc) or defined(gcAtomicArc): request.hostname = address else: request.hostname.shallowCopy(address) @@ -300,7 +300,7 @@ proc processRequest( while true: lineFut.mget.setLen(0) lineFut.clean() - + # The encoding format alternates between specifying a number of bytes to read # and the data to be read, of the previously specified size if sizeOrData mod 2 == 0: @@ -387,8 +387,9 @@ proc listen*(server: AsyncHttpServer; port: Port; address = ""; domain = AF_INET server.socket = newAsyncSocket(domain) if server.reuseAddr: server.socket.setSockOpt(OptReuseAddr, true) - if server.reusePort: - server.socket.setSockOpt(OptReusePort, true) + when not defined(nuttx): + if server.reusePort: + server.socket.setSockOpt(OptReusePort, true) server.socket.bindAddr(port, address) server.socket.listen() diff --git a/lib/pure/asyncmacro.nim b/lib/pure/asyncmacro.nim index d85e3e621..d4e72c28a 100644 --- a/lib/pure/asyncmacro.nim +++ b/lib/pure/asyncmacro.nim @@ -7,9 +7,14 @@ # distribution, for details about the copyright. # -## `asyncdispatch` module depends on the `asyncmacro` module to work properly. +## Implements the `async` and `multisync` macros for `asyncdispatch`. -import macros, strutils, asyncfutures +import std/[macros, strutils, asyncfutures] + +type + Context = ref object + inTry: int + hasRet: bool # TODO: Ref https://github.com/nim-lang/Nim/issues/5617 # TODO: Add more line infos @@ -20,9 +25,8 @@ proc newCallWithLineInfo(fromNode: NimNode; theProc: NimNode, args: varargs[NimN template createCb(retFutureSym, iteratorNameSym, strName, identName, futureVarCompletions: untyped) = bind finished - var nameIterVar = iteratorNameSym - proc identName {.closure.} = + proc identName {.closure, stackTrace: off.} = try: if not nameIterVar.finished: var next = nameIterVar() @@ -64,7 +68,7 @@ proc createFutureVarCompletions(futureVarIdents: seq[NimNode], fromNode: NimNode ) ) -proc processBody(node, retFutureSym: NimNode, futureVarIdents: seq[NimNode]): NimNode = +proc processBody(ctx: Context; node, needsCompletionSym, retFutureSym: NimNode, futureVarIdents: seq[NimNode]): NimNode = result = node case node.kind of nnkReturnStmt: @@ -73,23 +77,53 @@ proc processBody(node, retFutureSym: NimNode, futureVarIdents: seq[NimNode]): Ni # As I've painfully found out, the order here really DOES matter. result.add createFutureVarCompletions(futureVarIdents, node) + ctx.hasRet = true if node[0].kind == nnkEmpty: - result.add newCall(newIdentNode("complete"), retFutureSym, newIdentNode("result")) + if ctx.inTry == 0: + result.add newCallWithLineInfo(node, newIdentNode("complete"), retFutureSym, newIdentNode("result")) + else: + result.add newAssignment(needsCompletionSym, newLit(true)) else: - let x = node[0].processBody(retFutureSym, futureVarIdents) + let x = processBody(ctx, node[0], needsCompletionSym, retFutureSym, futureVarIdents) if x.kind == nnkYieldStmt: result.add x + elif ctx.inTry == 0: + result.add newCallWithLineInfo(node, newIdentNode("complete"), retFutureSym, x) else: - result.add newCall(newIdentNode("complete"), retFutureSym, x) + result.add newAssignment(newIdentNode("result"), x) + result.add newAssignment(needsCompletionSym, newLit(true)) result.add newNimNode(nnkReturnStmt, node).add(newNilLit()) return # Don't process the children of this return stmt of RoutineNodes-{nnkTemplateDef}: # skip all the nested procedure definitions return - else: discard - - for i in 0 ..< result.len: - result[i] = processBody(result[i], retFutureSym, futureVarIdents) + of nnkTryStmt: + if result[^1].kind == nnkFinally: + inc ctx.inTry + result[0] = processBody(ctx, result[0], needsCompletionSym, retFutureSym, futureVarIdents) + dec ctx.inTry + for i in 1 ..< result.len: + result[i] = processBody(ctx, result[i], needsCompletionSym, retFutureSym, futureVarIdents) + if ctx.inTry == 0 and ctx.hasRet: + let finallyNode = copyNimNode(result[^1]) + let stmtNode = newNimNode(nnkStmtList) + for child in result[^1]: + stmtNode.add child + stmtNode.add newIfStmt( + ( needsCompletionSym, + newCallWithLineInfo(node, newIdentNode("complete"), retFutureSym, + newIdentNode("result") + ) + ) + ) + finallyNode.add stmtNode + result[^1] = finallyNode + else: + for i in 0 ..< result.len: + result[i] = processBody(ctx, result[i], needsCompletionSym, retFutureSym, futureVarIdents) + else: + for i in 0 ..< result.len: + result[i] = processBody(ctx, result[i], needsCompletionSym, retFutureSym, futureVarIdents) # echo result.repr @@ -151,6 +185,10 @@ proc asyncSingleProc(prc: NimNode): NimNode = result[0][0] = quote do: Future[void] return result + if prc.kind in RoutineNodes and prc.name.kind != nnkEmpty: + # Only non anonymous functions need/can have stack trace disabled + prc.addPragma(nnkExprColonExpr.newTree(ident"stackTrace", ident"off")) + if prc.kind notin {nnkProcDef, nnkLambda, nnkMethodDef, nnkDo}: error("Cannot transform this node kind into an async proc." & " proc/method definition or lambda node expected.", prc) @@ -209,14 +247,23 @@ proc asyncSingleProc(prc: NimNode): NimNode = # -> {.pop.} # -> <proc_body> # -> complete(retFuture, result) - var iteratorNameSym = genSym(nskIterator, $prcName & "Iter") - var procBody = prc.body.processBody(retFutureSym, futureVarIdents) + var iteratorNameSym = genSym(nskIterator, $prcName & " (Async)") + var needsCompletionSym = genSym(nskVar, "needsCompletion") + var ctx = Context() + var procBody = processBody(ctx, prc.body, needsCompletionSym, retFutureSym, futureVarIdents) # don't do anything with forward bodies (empty) if procBody.kind != nnkEmpty: # fix #13899, defer should not escape its original scope - procBody = newStmtList(newTree(nnkBlockStmt, newEmptyNode(), procBody)) - procBody.add(createFutureVarCompletions(futureVarIdents, nil)) + let blockStmt = newStmtList(newTree(nnkBlockStmt, newEmptyNode(), procBody)) + procBody = newStmtList() let resultIdent = ident"result" + procBody.add quote do: + # Check whether there is an implicit return + when typeof(`blockStmt`) is void: + `blockStmt` + else: + `resultIdent` = `blockStmt` + procBody.add(createFutureVarCompletions(futureVarIdents, nil)) procBody.insert(0): quote do: {.push warning[resultshadowed]: off.} when `subRetType` isnot void: @@ -224,6 +271,8 @@ proc asyncSingleProc(prc: NimNode): NimNode = else: var `resultIdent`: Future[void] {.pop.} + + var `needsCompletionSym` = false procBody.add quote do: complete(`retFutureSym`, `resultIdent`) @@ -242,9 +291,10 @@ proc asyncSingleProc(prc: NimNode): NimNode = # friendlier stack traces: var cbName = genSym(nskProc, prcName & NimAsyncContinueSuffix) var procCb = getAst createCb(retFutureSym, iteratorNameSym, - newStrLitNode(prcName), - cbName, - createFutureVarCompletions(futureVarIdents, nil)) + newStrLitNode(prcName), + cbName, + createFutureVarCompletions(futureVarIdents, nil) + ) outerProcBody.add procCb # -> return retFuture @@ -277,8 +327,8 @@ macro async*(prc: untyped): untyped = proc splitParamType(paramType: NimNode, async: bool): NimNode = result = paramType if paramType.kind == nnkInfix and paramType[0].strVal in ["|", "or"]: - let firstAsync = "async" in paramType[1].strVal.normalize - let secondAsync = "async" in paramType[2].strVal.normalize + let firstAsync = "async" in paramType[1].toStrLit().strVal.normalize + let secondAsync = "async" in paramType[2].toStrLit().strVal.normalize if firstAsync: result = paramType[if async: 1 else: 2] diff --git a/lib/pure/asyncnet.nim b/lib/pure/asyncnet.nim index b61eaa902..ee07e599e 100644 --- a/lib/pure/asyncnet.nim +++ b/lib/pure/asyncnet.nim @@ -65,8 +65,7 @@ ## ## The following example demonstrates a simple chat server. ## -## .. code-block:: Nim -## +## ```Nim ## import std/[asyncnet, asyncdispatch] ## ## var clients {.threadvar.}: seq[AsyncSocket] @@ -93,24 +92,25 @@ ## ## asyncCheck serve() ## runForever() -## +## ``` import std/private/since when defined(nimPreviewSlimSystem): import std/[assertions, syncio] -import asyncdispatch, nativesockets, net, os +import std/[asyncdispatch, nativesockets, net, os] export SOBool # TODO: Remove duplication introduced by PR #4683. const defineSsl = defined(ssl) or defined(nimdoc) -const useNimNetLite = defined(nimNetLite) or defined(freertos) or defined(zephyr) +const useNimNetLite = defined(nimNetLite) or defined(freertos) or defined(zephyr) or + defined(nuttx) when defineSsl: - import openssl + import std/openssl type # TODO: I would prefer to just do: @@ -230,7 +230,7 @@ when defineSsl: let len = bioCtrlPending(socket.bioOut) if len > 0: var data = newString(len) - let read = bioRead(socket.bioOut, addr data[0], len) + let read = bioRead(socket.bioOut, cast[cstring](addr data[0]), len) assert read != 0 if read < 0: raiseSSLError() @@ -248,7 +248,7 @@ when defineSsl: var data = await recv(socket.fd.AsyncFD, BufferSize, flags) let length = len(data) if length > 0: - let ret = bioWrite(socket.bioIn, addr data[0], length.cint) + let ret = bioWrite(socket.bioIn, cast[cstring](addr data[0]), length.cint) if ret < 0: raiseSSLError() elif length == 0: @@ -265,14 +265,17 @@ when defineSsl: ErrClearError() # Call the desired operation. opResult = op - + let err = + if opResult < 0: + getSslError(socket, opResult.cint) + else: + SSL_ERROR_NONE # Send any remaining pending SSL data. await sendPendingSslData(socket, flags) # If the operation failed, try to see if SSL has some data to read # or write. if opResult < 0: - let err = getSslError(socket, opResult.cint) let fut = appeaseSsl(socket, flags, err.cint) yield fut if not fut.read(): @@ -460,7 +463,7 @@ proc send*(socket: AsyncSocket, data: string, when defineSsl: var copy = data sslLoop(socket, flags, - sslWrite(socket.sslHandle, addr copy[0], copy.len.cint)) + sslWrite(socket.sslHandle, cast[cstring](addr copy[0]), copy.len.cint)) await sendPendingSslData(socket, flags) else: await send(socket.fd.AsyncFD, data, flags) @@ -679,12 +682,12 @@ when defined(posix) and not useNimNetLite: elif ret == EINTR: return false else: - retFuture.fail(newException(OSError, osErrorMsg(OSErrorCode(ret)))) + retFuture.fail(newOSError(OSErrorCode(ret))) return true var socketAddr = makeUnixAddr(path) let ret = socket.fd.connect(cast[ptr SockAddr](addr socketAddr), - (sizeof(socketAddr.sun_family) + path.len).SockLen) + (offsetOf(socketAddr, sun_path) + path.len + 1).SockLen) if ret == 0: # Request to connect completed immediately. retFuture.complete() @@ -693,7 +696,7 @@ when defined(posix) and not useNimNetLite: if lastError.int32 == EINTR or lastError.int32 == EINPROGRESS: addWrite(AsyncFD(socket.fd), cb) else: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) + retFuture.fail(newOSError(lastError)) proc bindUnix*(socket: AsyncSocket, path: string) {. tags: [ReadIOEffect].} = @@ -702,7 +705,7 @@ when defined(posix) and not useNimNetLite: when not defined(nimdoc): var socketAddr = makeUnixAddr(path) if socket.fd.bindAddr(cast[ptr SockAddr](addr socketAddr), - (sizeof(socketAddr.sun_family) + path.len).SockLen) != 0'i32: + (offsetOf(socketAddr, sun_path) + path.len + 1).SockLen) != 0'i32: raiseOSError(osLastError()) elif defined(nimdoc): diff --git a/lib/pure/asyncstreams.nim b/lib/pure/asyncstreams.nim index 3f7774ed8..c97b98d55 100644 --- a/lib/pure/asyncstreams.nim +++ b/lib/pure/asyncstreams.nim @@ -9,12 +9,12 @@ ## Unstable API. -import asyncfutures +import std/asyncfutures when defined(nimPreviewSlimSystem): import std/assertions -import deques +import std/deques type FutureStream*[T] = ref object ## Special future that acts as diff --git a/lib/pure/base64.nim b/lib/pure/base64.nim index bc01a9164..591d22cc0 100644 --- a/lib/pure/base64.nim +++ b/lib/pure/base64.nim @@ -31,7 +31,7 @@ runnableExamples: ## runnableExamples: - let encodedInts = encode([1,2,3]) + let encodedInts = encode([1'u8,2,3]) assert encodedInts == "AQID" let encodedChars = encode(['h','e','y']) assert encodedChars == "aGV5" @@ -66,15 +66,11 @@ template cbBase(a, b): untyped = [ 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', a, b] -let +const cb64 = cbBase('+', '/') cb64safe = cbBase('-', '_') const - cb64VM = cbBase('+', '/') - cb64safeVM = cbBase('-', '_') - -const invalidChar = 255 template encodeSize(size: int): int = (size * 4 div 3) + 6 @@ -84,10 +80,13 @@ template encodeInternal(s, alphabet: typed): untyped = result.setLen(encodeSize(s.len)) + let + padding = s.len mod 3 + inputEnds = s.len - padding + var inputIndex = 0 outputIndex = 0 - inputEnds = s.len - s.len mod 3 n: uint32 b: uint32 @@ -113,7 +112,6 @@ template encodeInternal(s, alphabet: typed): untyped = outputChar(n shr 6) outputChar(n shr 0) - var padding = s.len mod 3 if padding == 1: inputByte(b shl 16) outputChar(n shr 18) @@ -132,21 +130,14 @@ template encodeInternal(s, alphabet: typed): untyped = result.setLen(outputIndex) template encodeImpl() {.dirty.} = - when nimvm: - block: - let lookupTableVM = if safe: cb64safeVM else: cb64VM - encodeInternal(s, lookupTableVM) + if safe: + encodeInternal(s, cb64safe) else: - block: - let lookupTable = if safe: unsafeAddr(cb64safe) else: unsafeAddr(cb64) - encodeInternal(s, lookupTable) + encodeInternal(s, cb64) -proc encode*[T: SomeInteger|char](s: openArray[T], safe = false): string = +proc encode*[T: byte|char](s: openArray[T], safe = false): string = ## Encodes `s` into base64 representation. ## - ## This procedure encodes an openarray (array or sequence) of either integers - ## or characters. - ## ## If `safe` is `true` then it will encode using the ## URL-Safe and Filesystem-safe standard alphabet characters, ## which substitutes `-` instead of `+` and `_` instead of `/`. @@ -154,18 +145,24 @@ proc encode*[T: SomeInteger|char](s: openArray[T], safe = false): string = ## * https://tools.ietf.org/html/rfc4648#page-7 ## ## **See also:** - ## * `encode proc<#encode,string>`_ for encoding a string ## * `decode proc<#decode,string>`_ for decoding a string runnableExamples: + assert encode("Hello World") == "SGVsbG8gV29ybGQ=" assert encode(['n', 'i', 'm']) == "bmlt" assert encode(@['n', 'i', 'm']) == "bmlt" - assert encode([1, 2, 3, 4, 5]) == "AQIDBAU=" + assert encode([1'u8, 2, 3, 4, 5]) == "AQIDBAU=" encodeImpl() -proc encode*(s: string, safe = false): string = - ## Encodes `s` into base64 representation. +proc encode*[T: SomeInteger and not byte](s: openArray[T], safe = false): string + {.deprecated: "use `byte` or `char` instead".} = + encodeImpl() + +proc encodeMime*(s: string, lineLen = 75.Positive, newLine = "\r\n", + safe = false): string = + ## Encodes `s` into base64 representation as lines. + ## Used in email MIME format, use `lineLen` and `newline`. ## - ## This procedure encodes a string. + ## This procedure encodes a string according to MIME spec. ## ## If `safe` is `true` then it will encode using the ## URL-Safe and Filesystem-safe standard alphabet characters, @@ -174,20 +171,7 @@ proc encode*(s: string, safe = false): string = ## * https://tools.ietf.org/html/rfc4648#page-7 ## ## **See also:** - ## * `encode proc<#encode,openArray[T]>`_ for encoding an openarray - ## * `decode proc<#decode,string>`_ for decoding a string - runnableExamples: - assert encode("Hello World") == "SGVsbG8gV29ybGQ=" - encodeImpl() - -proc encodeMime*(s: string, lineLen = 75.Positive, newLine = "\r\n"): string = - ## Encodes `s` into base64 representation as lines. - ## Used in email MIME format, use `lineLen` and `newline`. - ## - ## This procedure encodes a string according to MIME spec. - ## - ## **See also:** - ## * `encode proc<#encode,string>`_ for encoding a string + ## * `encode proc<#encode,openArray[T]>`_ for encoding an openArray ## * `decode proc<#decode,string>`_ for decoding a string runnableExamples: assert encodeMime("Hello World", 4, "\n") == "SGVs\nbG8g\nV29y\nbGQ=" @@ -199,7 +183,7 @@ proc encodeMime*(s: string, lineLen = 75.Positive, newLine = "\r\n"): string = inc idx if s.len == 0: return - let e = encode(s) + let e = encode(s, safe) if e.len <= lineLen or newLine.len == 0: return e result = newString(e.len + newLine.len * ((e.len div lineLen) - int(e.len mod lineLen == 0))) @@ -232,7 +216,6 @@ proc decode*(s: string): string = ## ## **See also:** ## * `encode proc<#encode,openArray[T]>`_ for encoding an openarray - ## * `encode proc<#encode,string>`_ for encoding a string runnableExamples: assert decode("SGVsbG8gV29ybGQ=") == "Hello World" assert decode(" SGVsbG8gV29ybGQ=") == "Hello World" @@ -261,7 +244,7 @@ proc decode*(s: string): string = inputLen = s.len inputEnds = 0 # strip trailing characters - while s[inputLen - 1] in {'\n', '\r', ' ', '='}: + while inputLen > 0 and s[inputLen - 1] in {'\n', '\r', ' ', '='}: dec inputLen # hot loop: read 4 characters at at time inputEnds = inputLen - 4 diff --git a/lib/pure/bitops.nim b/lib/pure/bitops.nim index a518c25d2..0d3351ee5 100644 --- a/lib/pure/bitops.nim +++ b/lib/pure/bitops.nim @@ -25,7 +25,7 @@ ## At this time only `fastLog2`, `firstSetBit`, `countLeadingZeroBits` and `countTrailingZeroBits` ## may return undefined and/or platform dependent values if given invalid input. -import macros +import std/macros import std/private/since from std/private/bitops_utils import forwardImpl, castToUnsigned @@ -63,6 +63,12 @@ macro bitxor*[T: SomeInteger](x, y: T; z: varargs[T]): T = type BitsRange*[T] = range[0..sizeof(T)*8-1] ## A range with all bit positions for type `T`. +template typeMasked[T: SomeInteger](x: T): T = + when defined(js): + T(x and ((0xffffffff_ffffffff'u shr (64 - sizeof(T) * 8)))) + else: + x + func bitsliced*[T: SomeInteger](v: T; slice: Slice[int]): T {.inline, since: (1, 3).} = ## Returns an extracted (and shifted) slice of bits from `v`. runnableExamples: @@ -73,7 +79,7 @@ func bitsliced*[T: SomeInteger](v: T; slice: Slice[int]): T {.inline, since: (1, let upmost = sizeof(T) * 8 - 1 uv = v.castToUnsigned - (uv shl (upmost - slice.b) shr (upmost - slice.b + slice.a)).T + ((uv shl (upmost - slice.b)).typeMasked shr (upmost - slice.b + slice.a)).T proc bitslice*[T: SomeInteger](v: var T; slice: Slice[int]) {.inline, since: (1, 3).} = ## Mutates `v` into an extracted (and shifted) slice of bits from `v`. @@ -85,7 +91,7 @@ proc bitslice*[T: SomeInteger](v: var T; slice: Slice[int]) {.inline, since: (1, let upmost = sizeof(T) * 8 - 1 uv = v.castToUnsigned - v = (uv shl (upmost - slice.b) shr (upmost - slice.b + slice.a)).T + v = ((uv shl (upmost - slice.b)).typeMasked shr (upmost - slice.b + slice.a)).T func toMask*[T: SomeInteger](slice: Slice[int]): T {.inline, since: (1, 3).} = ## Creates a bitmask based on a slice of bits. @@ -96,7 +102,7 @@ func toMask*[T: SomeInteger](slice: Slice[int]): T {.inline, since: (1, 3).} = let upmost = sizeof(T) * 8 - 1 bitmask = bitnot(0.T).castToUnsigned - (bitmask shl (upmost - slice.b + slice.a) shr (upmost - slice.b)).T + ((bitmask shl (upmost - slice.b + slice.a)).typeMasked shr (upmost - slice.b)).T proc masked*[T: SomeInteger](v, mask :T): T {.inline, since: (1, 3).} = ## Returns `v`, with only the `1` bits from `mask` matching those of @@ -457,7 +463,7 @@ elif useVCC_builtins: importc: "_BitScanForward64", header: "<intrin.h>".} template vcc_scan_impl(fnc: untyped; v: untyped): int = - var index: culong + var index {.inject.}: culong = 0 discard fnc(index.addr, v) index.int diff --git a/lib/pure/browsers.nim b/lib/pure/browsers.nim index c36e31b11..59e2078df 100644 --- a/lib/pure/browsers.nim +++ b/lib/pure/browsers.nim @@ -12,20 +12,22 @@ ## ## Unstable API. -import std/private/since +import std/private/since # used by the deprecated `openDefaultBrowser()` -import strutils +import std/strutils when defined(nimPreviewSlimSystem): import std/assertions when defined(windows): - import winlean - from os import absolutePath + import std/winlean + when defined(nimPreviewSlimSystem): + import std/widestrs + from std/os import absolutePath else: - import os + import std/os when not defined(osx): - import osproc + import std/osproc const osOpenCmd* = when defined(macos) or defined(macosx) or defined(windows): "open" else: "xdg-open" ## \ @@ -38,15 +40,17 @@ proc prepare(s: string): string = else: result = "file://" & absolutePath(s) -proc openDefaultBrowserImpl(url: string) = +proc openDefaultBrowserRaw(url: string) = + ## note the url argument should be alreadly prepared, i.e. the url is passed "AS IS" + when defined(windows): var o = newWideCString(osOpenCmd) - var u = newWideCString(prepare url) + var u = newWideCString(url) discard shellExecuteW(0'i32, o, u, nil, nil, SW_SHOWNORMAL) elif defined(macosx): - discard execShellCmd(osOpenCmd & " " & quoteShell(prepare url)) + discard execShellCmd(osOpenCmd & " " & quoteShell(url)) else: - var u = quoteShell(prepare url) + var u = quoteShell(url) if execShellCmd(osOpenCmd & " " & u) == 0: return for b in getEnv("BROWSER").split(PathSep): try: @@ -67,26 +71,42 @@ proc openDefaultBrowser*(url: string) = ## ## This proc doesn't raise an exception on error, beware. ## - ## .. code-block:: nim + ## ```nim ## block: openDefaultBrowser("https://nim-lang.org") + ## ``` doAssert url.len > 0, "URL must not be empty string" - openDefaultBrowserImpl(url) + openDefaultBrowserRaw(url) -proc openDefaultBrowser*() {.since: (1, 1).} = - ## Opens the user's default browser without any `url` (blank page). This does not block. - ## Implements IETF RFC-6694 Section 3, "about:blank" must be reserved for a blank page. +proc openDefaultBrowser*() {.since: (1, 1), deprecated: + "not implemented, please open with a specific url instead".} = + ## Intends to open the user's default browser without any `url` (blank page). + ## This does not block. + ## Intends to implement IETF RFC-6694 Section 3, + ## ("about:blank" is reserved for a blank page). ## - ## Under Windows, `ShellExecute` is used. Under Mac OS X the `open` - ## command is used. Under Unix, it is checked if `xdg-open` exists and - ## used if it does. Otherwise the environment variable `BROWSER` is - ## used to determine the default browser to use. + ## Beware that this intended behavior is **not** implemented and + ## considered not worthy to implement here. + ## + ## The following describes the behavior of current implementation: + ## + ## - Under Windows, this will only cause a pop-up dialog \ + ## asking the assocated application with `about` \ + ## (as Windows simply treats `about:` as a protocol like `http`). + ## - Under Mac OS X the `open "about:blank"` command is used. + ## - Under Unix, it is checked if `xdg-open` exists and used \ + ## if it does and open the application assocated with `text/html` mime \ + ## (not `x-scheme-handler/http`, so maybe html-viewer \ + ## other than your default browser is opened). \ + ## Otherwise the environment variable `BROWSER` is used \ + ## to determine the default browser to use. ## ## This proc doesn't raise an exception on error, beware. ## + ## ```nim + ## block: openDefaultBrowser() + ## ``` + ## ## **See also:** ## ## * https://tools.ietf.org/html/rfc6694#section-3 - ## - ## .. code-block:: nim - ## block: openDefaultBrowser() - openDefaultBrowserImpl("http:about:blank") # See IETF RFC-6694 Section 3. + openDefaultBrowserRaw("about:blank") # See IETF RFC-6694 Section 3. diff --git a/lib/pure/cgi.nim b/lib/pure/cgi.nim index 0ab8f4c95..034f224ac 100644 --- a/lib/pure/cgi.nim +++ b/lib/pure/cgi.nim @@ -9,27 +9,27 @@ ## This module implements helper procs for CGI applications. Example: ## -## .. code-block:: Nim +## ```Nim +## import std/[strtabs, cgi] ## -## import std/[strtabs, cgi] -## -## # Fill the values when debugging: -## when debug: -## setTestData("name", "Klaus", "password", "123456") -## # read the data into `myData` -## var myData = readData() -## # check that the data's variable names are "name" or "password" -## validateData(myData, "name", "password") -## # start generating content: -## writeContentType() -## # generate content: -## write(stdout, "<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.01//EN\">\n") -## write(stdout, "<html><head><title>Test</title></head><body>\n") -## writeLine(stdout, "your name: " & myData["name"]) -## writeLine(stdout, "your password: " & myData["password"]) -## writeLine(stdout, "</body></html>") - -import strutils, os, strtabs, cookies, uri +## # Fill the values when debugging: +## when debug: +## setTestData("name", "Klaus", "password", "123456") +## # read the data into `myData` +## var myData = readData() +## # check that the data's variable names are "name" or "password" +## validateData(myData, "name", "password") +## # start generating content: +## writeContentType() +## # generate content: +## write(stdout, "<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.01//EN\">\n") +## write(stdout, "<html><head><title>Test</title></head><body>\n") +## writeLine(stdout, "your name: " & myData["name"]) +## writeLine(stdout, "your password: " & myData["password"]) +## writeLine(stdout, "</body></html>") +## ``` + +import std/[strutils, os, strtabs, cookies, uri] export uri.encodeUrl, uri.decodeUrl when defined(nimPreviewSlimSystem): @@ -252,9 +252,9 @@ proc setTestData*(keysvalues: varargs[string]) = ## Fills the appropriate environment variables to test your CGI application. ## This can only simulate the 'GET' request method. `keysvalues` should ## provide embedded (name, value)-pairs. Example: - ## - ## .. code-block:: Nim - ## setTestData("name", "Hanz", "password", "12345") + ## ```Nim + ## setTestData("name", "Hanz", "password", "12345") + ## ``` putEnv("REQUEST_METHOD", "GET") var i = 0 var query = "" @@ -269,9 +269,9 @@ proc setTestData*(keysvalues: varargs[string]) = proc writeContentType*() = ## Calls this before starting to send your HTML data to `stdout`. This ## implements this part of the CGI protocol: - ## - ## .. code-block:: Nim - ## write(stdout, "Content-type: text/html\n\n") + ## ```Nim + ## write(stdout, "Content-type: text/html\n\n") + ## ``` write(stdout, "Content-type: text/html\n\n") proc resetForStacktrace() = diff --git a/lib/pure/collections/deques.nim b/lib/pure/collections/deques.nim index c4eb5331e..d2b0099f2 100644 --- a/lib/pure/collections/deques.nim +++ b/lib/pure/collections/deques.nim @@ -47,11 +47,10 @@ runnableExamples: ## See also ## ======== ## * `lists module <lists.html>`_ for singly and doubly linked lists and rings -## * `channels module <channels_builtin.html>`_ for inter-thread communication import std/private/since -import math +import std/[assertions, hashes, math] type Deque*[T] = object @@ -60,20 +59,28 @@ type ## To initialize an empty deque, ## use the `initDeque proc <#initDeque,int>`_. data: seq[T] - head, tail, count, mask: int + + # `head` and `tail` are masked only when accessing an element of `data` + # so that `tail - head == data.len` when the deque is full. + # They are uint so that incrementing/decrementing them doesn't cause + # over/underflow. You can get a number of items with `tail - head` + # even if `tail` or `head` is wraps around and `tail < head`, because + # `tail - head == (uint.high + 1 + tail) - head` when `tail < head`. + head, tail: uint const defaultInitialSize* = 4 template initImpl(result: typed, initialSize: int) = let correctSize = nextPowerOfTwo(initialSize) - result.mask = correctSize - 1 newSeq(result.data, correctSize) template checkIfInitialized(deq: typed) = - when compiles(defaultInitialSize): - if deq.mask == 0: - initImpl(deq, defaultInitialSize) + if deq.data.len == 0: + initImpl(deq, defaultInitialSize) + +func mask[T](deq: Deque[T]): uint {.inline.} = + uint(deq.data.len) - 1 proc initDeque*[T](initialSize: int = defaultInitialSize): Deque[T] = ## Creates a new empty deque. @@ -87,22 +94,22 @@ proc initDeque*[T](initialSize: int = defaultInitialSize): Deque[T] = ## * `toDeque proc <#toDeque,openArray[T]>`_ result.initImpl(initialSize) -proc len*[T](deq: Deque[T]): int {.inline.} = +func len*[T](deq: Deque[T]): int {.inline.} = ## Returns the number of elements of `deq`. - result = deq.count + int(deq.tail - deq.head) template emptyCheck(deq) = # Bounds check for the regular deque access. when compileOption("boundChecks"): - if unlikely(deq.count < 1): + if unlikely(deq.len < 1): raise newException(IndexDefect, "Empty deque.") template xBoundsCheck(deq, i) = # Bounds check for the array like accesses. when compileOption("boundChecks"): # `-d:danger` or `--checks:off` should disable this. - if unlikely(i >= deq.count): # x < deq.low is taken care by the Natural parameter + if unlikely(i >= deq.len): # x < deq.low is taken care by the Natural parameter raise newException(IndexDefect, - "Out of bounds: " & $i & " > " & $(deq.count - 1)) + "Out of bounds: " & $i & " > " & $(deq.len - 1)) if unlikely(i < 0): # when used with BackwardsIndex raise newException(IndexDefect, "Out of bounds: " & $i & " < 0") @@ -116,7 +123,7 @@ proc `[]`*[T](deq: Deque[T], i: Natural): lent T {.inline.} = doAssertRaises(IndexDefect, echo a[8]) xBoundsCheck(deq, i) - return deq.data[(deq.head + i) and deq.mask] + return deq.data[(deq.head + i.uint) and deq.mask] proc `[]`*[T](deq: var Deque[T], i: Natural): var T {.inline.} = ## Accesses the `i`-th element of `deq` and returns a mutable @@ -127,7 +134,7 @@ proc `[]`*[T](deq: var Deque[T], i: Natural): var T {.inline.} = assert a[0] == 11 xBoundsCheck(deq, i) - return deq.data[(deq.head + i) and deq.mask] + return deq.data[(deq.head + i.uint) and deq.mask] proc `[]=`*[T](deq: var Deque[T], i: Natural, val: sink T) {.inline.} = ## Sets the `i`-th element of `deq` to `val`. @@ -139,7 +146,7 @@ proc `[]=`*[T](deq: var Deque[T], i: Natural, val: sink T) {.inline.} = checkIfInitialized(deq) xBoundsCheck(deq, i) - deq.data[(deq.head + i) and deq.mask] = val + deq.data[(deq.head + i.uint) and deq.mask] = val proc `[]`*[T](deq: Deque[T], i: BackwardsIndex): lent T {.inline.} = ## Accesses the backwards indexed `i`-th element. @@ -192,10 +199,8 @@ iterator items*[T](deq: Deque[T]): lent T = let a = [10, 20, 30, 40, 50].toDeque assert toSeq(a.items) == @[10, 20, 30, 40, 50] - var i = deq.head - for c in 0 ..< deq.count: - yield deq.data[i] - i = (i + 1) and deq.mask + for c in 0 ..< deq.len: + yield deq.data[(deq.head + c.uint) and deq.mask] iterator mitems*[T](deq: var Deque[T]): var T = ## Yields every element of `deq`, which can be modified. @@ -209,10 +214,8 @@ iterator mitems*[T](deq: var Deque[T]): var T = x = 5 * x - 1 assert $a == "[49, 99, 149, 199, 249]" - var i = deq.head - for c in 0 ..< deq.count: - yield deq.data[i] - i = (i + 1) and deq.mask + for c in 0 ..< deq.len: + yield deq.data[(deq.head + c.uint) and deq.mask] iterator pairs*[T](deq: Deque[T]): tuple[key: int, val: T] = ## Yields every `(position, value)`-pair of `deq`. @@ -222,10 +225,8 @@ iterator pairs*[T](deq: Deque[T]): tuple[key: int, val: T] = let a = [10, 20, 30].toDeque assert toSeq(a.pairs) == @[(0, 10), (1, 20), (2, 30)] - var i = deq.head - for c in 0 ..< deq.count: - yield (c, deq.data[i]) - i = (i + 1) and deq.mask + for c in 0 ..< deq.len: + yield (c, deq.data[(deq.head + c.uint) and deq.mask]) proc contains*[T](deq: Deque[T], item: T): bool {.inline.} = ## Returns true if `item` is in `deq` or false if not found. @@ -244,8 +245,9 @@ proc contains*[T](deq: Deque[T], item: T): bool {.inline.} = proc expandIfNeeded[T](deq: var Deque[T]) = checkIfInitialized(deq) - var cap = deq.mask + 1 - if unlikely(deq.count >= cap): + let cap = deq.data.len + assert deq.len <= cap + if unlikely(deq.len == cap): var n = newSeq[T](cap * 2) var i = 0 for x in mitems(deq): @@ -253,8 +255,7 @@ proc expandIfNeeded[T](deq: var Deque[T]) = else: n[i] = move(x) inc i deq.data = move(n) - deq.mask = cap * 2 - 1 - deq.tail = deq.count + deq.tail = cap.uint deq.head = 0 proc addFirst*[T](deq: var Deque[T], item: sink T) = @@ -269,9 +270,8 @@ proc addFirst*[T](deq: var Deque[T], item: sink T) = assert $a == "[50, 40, 30, 20, 10]" expandIfNeeded(deq) - inc deq.count - deq.head = (deq.head - 1) and deq.mask - deq.data[deq.head] = item + dec deq.head + deq.data[deq.head and deq.mask] = item proc addLast*[T](deq: var Deque[T], item: sink T) = ## Adds an `item` to the end of `deq`. @@ -285,9 +285,8 @@ proc addLast*[T](deq: var Deque[T], item: sink T) = assert $a == "[10, 20, 30, 40, 50]" expandIfNeeded(deq) - inc deq.count - deq.data[deq.tail] = item - deq.tail = (deq.tail + 1) and deq.mask + deq.data[deq.tail and deq.mask] = item + inc deq.tail proc toDeque*[T](x: openArray[T]): Deque[T] {.since: (1, 3).} = ## Creates a new deque that contains the elements of `x` (in the same order). @@ -316,7 +315,7 @@ proc peekFirst*[T](deq: Deque[T]): lent T {.inline.} = assert len(a) == 5 emptyCheck(deq) - result = deq.data[deq.head] + result = deq.data[deq.head and deq.mask] proc peekLast*[T](deq: Deque[T]): lent T {.inline.} = ## Returns the last element of `deq`, but does not remove it from the deque. @@ -346,7 +345,7 @@ proc peekFirst*[T](deq: var Deque[T]): var T {.inline, since: (1, 3).} = assert $a == "[99, 20, 30, 40, 50]" emptyCheck(deq) - result = deq.data[deq.head] + result = deq.data[deq.head and deq.mask] proc peekLast*[T](deq: var Deque[T]): var T {.inline, since: (1, 3).} = ## Returns a mutable reference to the last element of `deq`, @@ -379,9 +378,8 @@ proc popFirst*[T](deq: var Deque[T]): T {.inline, discardable.} = assert $a == "[20, 30, 40, 50]" emptyCheck(deq) - dec deq.count - result = move deq.data[deq.head] - deq.head = (deq.head + 1) and deq.mask + result = move deq.data[deq.head and deq.mask] + inc deq.head proc popLast*[T](deq: var Deque[T]): T {.inline, discardable.} = ## Removes and returns the last element of the `deq`. @@ -396,9 +394,8 @@ proc popLast*[T](deq: var Deque[T]): T {.inline, discardable.} = assert $a == "[10, 20, 30, 40]" emptyCheck(deq) - dec deq.count - deq.tail = (deq.tail - 1) and deq.mask - result = move deq.data[deq.tail] + dec deq.tail + result = move deq.data[deq.tail and deq.mask] proc clear*[T](deq: var Deque[T]) {.inline.} = ## Resets the deque so that it is empty. @@ -412,7 +409,6 @@ proc clear*[T](deq: var Deque[T]) {.inline.} = assert len(a) == 0 for el in mitems(deq): destroy(el) - deq.count = 0 deq.tail = deq.head proc shrink*[T](deq: var Deque[T], fromFirst = 0, fromLast = 0) = @@ -432,19 +428,17 @@ proc shrink*[T](deq: var Deque[T], fromFirst = 0, fromLast = 0) = a.shrink(fromFirst = 2, fromLast = 1) assert $a == "[30, 40]" - if fromFirst + fromLast > deq.count: + if fromFirst + fromLast > deq.len: clear(deq) return for i in 0 ..< fromFirst: - destroy(deq.data[deq.head]) - deq.head = (deq.head + 1) and deq.mask + destroy(deq.data[deq.head and deq.mask]) + inc deq.head for i in 0 ..< fromLast: - destroy(deq.data[deq.tail]) - deq.tail = (deq.tail - 1) and deq.mask - - dec deq.count, fromFirst + fromLast + destroy(deq.data[(deq.tail - 1) and deq.mask]) + dec deq.tail proc `$`*[T](deq: Deque[T]): string = ## Turns a deque into its string representation. @@ -457,3 +451,30 @@ proc `$`*[T](deq: Deque[T]): string = if result.len > 1: result.add(", ") result.addQuoted(x) result.add("]") + +func `==`*[T](deq1, deq2: Deque[T]): bool = + ## The `==` operator for Deque. + ## Returns `true` if both deques contains the same values in the same order. + runnableExamples: + var a, b = initDeque[int]() + a.addFirst(2) + a.addFirst(1) + b.addLast(1) + b.addLast(2) + doAssert a == b + + if deq1.len != deq2.len: + return false + + for i in 0 ..< deq1.len: + if deq1.data[(deq1.head + i.uint) and deq1.mask] != deq2.data[(deq2.head + i.uint) and deq2.mask]: + return false + + true + +func hash*[T](deq: Deque[T]): Hash = + ## Hashing of Deque. + var h: Hash = 0 + for x in deq: + h = h !& hash(x) + !$h diff --git a/lib/pure/collections/hashcommon.nim b/lib/pure/collections/hashcommon.nim index deff8fa21..17785c8c7 100644 --- a/lib/pure/collections/hashcommon.nim +++ b/lib/pure/collections/hashcommon.nim @@ -13,6 +13,7 @@ when defined(nimPreviewSlimSystem): import std/assertions +import std / outparams const growthFactor = 2 @@ -37,16 +38,6 @@ proc slotsNeeded(count: Natural): int {.inline.} = # Make sure to synchronize with `mustRehash` above result = nextPowerOfTwo(count * 3 div 2 + 4) -proc rightSize*(count: Natural): int {.inline, deprecated: "Deprecated since 1.4.0".} = - ## It is not needed anymore because - ## picking the correct size is done internally. - ## - ## Returns the value of `initialSize` to support `count` items. - ## - ## If more items are expected to be added, simply add that - ## expected extra amount to the parameter before calling this. - result = count - template rawGetKnownHCImpl() {.dirty.} = if t.dataLen == 0: return -1 @@ -67,7 +58,10 @@ proc rawGetKnownHC[X, A](t: X, key: A, hc: Hash): int {.inline.} = template genHashImpl(key, hc: typed) = hc = hash(key) if hc == 0: # This almost never taken branch should be very predictable. - hc = 314159265 # Value doesn't matter; Any non-zero favorite is fine. + when sizeof(int) < 4: + hc = 31415 # Value doesn't matter; Any non-zero favorite is fine <= 16-bit. + else: + hc = 314159265 # Value doesn't matter; Any non-zero favorite is fine. template genHash(key: typed): Hash = var res: Hash @@ -78,5 +72,5 @@ template rawGetImpl() {.dirty.} = genHashImpl(key, hc) rawGetKnownHCImpl() -proc rawGet[X, A](t: X, key: A, hc: var Hash): int {.inline.} = +proc rawGet[X, A](t: X, key: A, hc: var Hash): int {.inline, outParamsAt: [3].} = rawGetImpl() diff --git a/lib/pure/collections/heapqueue.nim b/lib/pure/collections/heapqueue.nim index 89e532951..96f9b4430 100644 --- a/lib/pure/collections/heapqueue.nim +++ b/lib/pure/collections/heapqueue.nim @@ -47,6 +47,9 @@ runnableExamples: import std/private/since +when defined(nimPreviewSlimSystem): + import std/assertions + type HeapQueue*[T] = object ## A heap queue, commonly known as a priority queue. data: seq[T] @@ -59,7 +62,7 @@ proc initHeapQueue*[T](): HeapQueue[T] = ## ## **See also:** ## * `toHeapQueue proc <#toHeapQueue,openArray[T]>`_ - discard + result = default(HeapQueue[T]) proc len*[T](heap: HeapQueue[T]): int {.inline.} = ## Returns the number of elements of `heap`. @@ -73,6 +76,13 @@ proc `[]`*[T](heap: HeapQueue[T], i: Natural): lent T {.inline.} = ## Accesses the i-th element of `heap`. heap.data[i] +iterator items*[T](heap: HeapQueue[T]): lent T {.inline, since: (2, 1, 1).} = + ## Iterates over each item of `heap`. + let L = len(heap) + for i in 0 .. high(heap.data): + yield heap.data[i] + assert(len(heap) == L, "the length of the HeapQueue changed while iterating over it") + proc heapCmp[T](x, y: T): bool {.inline.} = x < y proc siftup[T](heap: var HeapQueue[T], startpos, p: int) = @@ -178,6 +188,11 @@ proc find*[T](heap: HeapQueue[T], x: T): int {.since: (1, 3).} = for i in 0 ..< heap.len: if heap[i] == x: return i +proc contains*[T](heap: HeapQueue[T], x: T): bool {.since: (2, 1, 1).} = + ## Returns true if `x` is in `heap` or false if not found. This is a shortcut + ## for `find(heap, x) >= 0`. + result = find(heap, x) >= 0 + proc del*[T](heap: var HeapQueue[T], index: Natural) = ## Removes the element at `index` from `heap`, maintaining the heap invariant. runnableExamples: diff --git a/lib/pure/collections/lists.nim b/lib/pure/collections/lists.nim index 829ec2ccb..6b88747ef 100644 --- a/lib/pure/collections/lists.nim +++ b/lib/pure/collections/lists.nim @@ -62,9 +62,6 @@ import std/private/since when defined(nimPreviewSlimSystem): import std/assertions -when not defined(nimHasCursor): - {.pragma: cursor.} - type DoublyLinkedNodeObj*[T] = object ## A node of a doubly linked list. @@ -168,36 +165,14 @@ proc newSinglyLinkedNode*[T](value: T): SinglyLinkedNode[T] = new(result) result.value = value -func toSinglyLinkedList*[T](elems: openArray[T]): SinglyLinkedList[T] {.since: (1, 5, 1).} = - ## Creates a new `SinglyLinkedList` from the members of `elems`. - runnableExamples: - from std/sequtils import toSeq - let a = [1, 2, 3, 4, 5].toSinglyLinkedList - assert a.toSeq == [1, 2, 3, 4, 5] - - result = initSinglyLinkedList[T]() - for elem in elems.items: - result.add(elem) - -func toDoublyLinkedList*[T](elems: openArray[T]): DoublyLinkedList[T] {.since: (1, 5, 1).} = - ## Creates a new `DoublyLinkedList` from the members of `elems`. - runnableExamples: - from std/sequtils import toSeq - let a = [1, 2, 3, 4, 5].toDoublyLinkedList - assert a.toSeq == [1, 2, 3, 4, 5] - - result = initDoublyLinkedList[T]() - for elem in elems.items: - result.add(elem) - template itemsListImpl() {.dirty.} = - var it = L.head + var it {.cursor.} = L.head while it != nil: yield it.value it = it.next template itemsRingImpl() {.dirty.} = - var it = L.head + var it {.cursor.} = L.head if it != nil: while true: yield it.value @@ -289,7 +264,7 @@ iterator nodes*[T](L: SomeLinkedList[T]): SomeLinkedNode[T] = x.value = 5 * x.value - 1 assert $a == "[49, 99, 199, 249]" - var it = L.head + var it {.cursor.} = L.head while it != nil: let nxt = it.next yield it @@ -314,7 +289,7 @@ iterator nodes*[T](L: SomeLinkedRing[T]): SomeLinkedNode[T] = x.value = 5 * x.value - 1 assert $a == "[49, 99, 199, 249]" - var it = L.head + var it {.cursor.} = L.head if it != nil: while true: let nxt = it.next @@ -409,9 +384,7 @@ proc prependMoved*[T: SomeLinkedList](a, b: var T) {.since: (1, 5, 1).} = assert s == [0, 1, 0, 1, 0, 1] b.addMoved(a) - when defined(js): # XXX: swap broken in js; bug #16771 - (b, a) = (a, b) - else: swap a, b + swap a, b proc add*[T](L: var SinglyLinkedList[T], n: SinglyLinkedNode[T]) {.inline.} = ## Appends (adds to the end) a node `n` to `L`. Efficiency: O(1). @@ -736,7 +709,7 @@ proc remove*[T](L: var SinglyLinkedList[T], n: SinglyLinkedNode[T]): bool {.disc if L.tail.next == n: L.tail.next = L.head # restore cycle else: - var prev = L.head + var prev {.cursor.} = L.head while prev.next != n and prev.next != nil: prev = prev.next if prev.next == nil: @@ -996,3 +969,47 @@ proc appendMoved*[T: SomeLinkedList](a, b: var T) {.since: (1, 5, 1).} = ## * `addMoved proc <#addMoved,SinglyLinkedList[T],SinglyLinkedList[T]>`_ ## * `addMoved proc <#addMoved,DoublyLinkedList[T],DoublyLinkedList[T]>`_ a.addMoved(b) + +func toSinglyLinkedList*[T](elems: openArray[T]): SinglyLinkedList[T] {.since: (1, 5, 1).} = + ## Creates a new `SinglyLinkedList` from the members of `elems`. + runnableExamples: + from std/sequtils import toSeq + let a = [1, 2, 3, 4, 5].toSinglyLinkedList + assert a.toSeq == [1, 2, 3, 4, 5] + + result = initSinglyLinkedList[T]() + for elem in elems.items: + result.add(elem) + +func toSinglyLinkedRing*[T](elems: openArray[T]): SinglyLinkedRing[T] = + ## Creates a new `SinglyLinkedRing` from the members of `elems`. + runnableExamples: + from std/sequtils import toSeq + let a = [1, 2, 3, 4, 5].toSinglyLinkedRing + assert a.toSeq == [1, 2, 3, 4, 5] + + result = initSinglyLinkedRing[T]() + for elem in elems.items: + result.add(elem) + +func toDoublyLinkedList*[T](elems: openArray[T]): DoublyLinkedList[T] {.since: (1, 5, 1).} = + ## Creates a new `DoublyLinkedList` from the members of `elems`. + runnableExamples: + from std/sequtils import toSeq + let a = [1, 2, 3, 4, 5].toDoublyLinkedList + assert a.toSeq == [1, 2, 3, 4, 5] + + result = initDoublyLinkedList[T]() + for elem in elems.items: + result.add(elem) + +func toDoublyLinkedRing*[T](elems: openArray[T]): DoublyLinkedRing[T] = + ## Creates a new `DoublyLinkedRing` from the members of `elems`. + runnableExamples: + from std/sequtils import toSeq + let a = [1, 2, 3, 4, 5].toDoublyLinkedRing + assert a.toSeq == [1, 2, 3, 4, 5] + + result = initDoublyLinkedRing[T]() + for elem in elems.items: + result.add(elem) diff --git a/lib/pure/collections/sequtils.nim b/lib/pure/collections/sequtils.nim index b86977539..3c0d8dc0e 100644 --- a/lib/pure/collections/sequtils.nim +++ b/lib/pure/collections/sequtils.nim @@ -82,7 +82,8 @@ runnableExamples: import std/private/since -import macros +import std/macros +from std/typetraits import supportsCopyMem when defined(nimPreviewSlimSystem): import std/assertions @@ -140,6 +141,22 @@ func concat*[T](seqs: varargs[seq[T]]): seq[T] = result[i] = itm inc(i) +func addUnique*[T](s: var seq[T], x: sink T) = + ## Adds `x` to the container `s` if it is not already present. + ## Uses `==` to check if the item is already present. + runnableExamples: + var a = @[1, 2, 3] + a.addUnique(4) + a.addUnique(4) + assert a == @[1, 2, 3, 4] + + for i in 0..high(s): + if s[i] == x: return + when declared(ensureMove): + s.add ensureMove(x) + else: + s.add x + func count*[T](s: openArray[T], x: T): int = ## Returns the number of occurrences of the item `x` in the container `s`. ## @@ -248,6 +265,15 @@ func maxIndex*[T](s: openArray[T]): int {.since: (1, 1).} = for i in 1..high(s): if s[i] > s[result]: result = i +func minmax*[T](x: openArray[T]): (T, T) = + ## The minimum and maximum values of `x`. `T` needs to have a `<` operator. + var l = x[0] + var h = x[0] + for i in 1..high(x): + if x[i] < l: l = x[i] + if h < x[i]: h = x[i] + result = (l, h) + template zipImpl(s1, s2, retType: untyped): untyped = proc zip*[S, T](s1: openArray[S], s2: openArray[T]): retType = @@ -302,8 +328,7 @@ proc unzip*[S, T](s: openArray[(S, T)]): (seq[S], seq[T]) {.since: (1, 1).} = unzipped2 = @['a', 'b', 'c'] assert zipped.unzip() == (unzipped1, unzipped2) assert zip(unzipped1, unzipped2).unzip() == (unzipped1, unzipped2) - result[0] = newSeq[S](s.len) - result[1] = newSeq[T](s.len) + result = (newSeq[S](s.len), newSeq[T](s.len)) for i in 0..<s.len: result[0][i] = s[i][0] result[1][i] = s[i][1] @@ -852,7 +877,7 @@ template toSeq*(iter: untyped): untyped = inc i result else: - var result: seq[typeof(iter)]# = @[] + var result: seq[typeof(iter)] = @[] for x in iter: result.add(x) result @@ -1078,9 +1103,13 @@ template newSeqWith*(len: int, init: untyped): untyped = import std/random var seqRand = newSeqWith(20, rand(1.0)) assert seqRand[0] != seqRand[1] - - var result = newSeq[typeof(init)](len) - for i in 0 ..< len: + type T = typeof(init) + let newLen = len + when supportsCopyMem(T) and declared(newSeqUninit): + var result = newSeqUninit[T](newLen) + else: # TODO: use `newSeqUnsafe` when that's available + var result = newSeq[T](newLen) + for i in 0 ..< newLen: result[i] = init move(result) # refs bug #7295 diff --git a/lib/pure/collections/setimpl.nim b/lib/pure/collections/setimpl.nim index 7ebd22760..360a075d6 100644 --- a/lib/pure/collections/setimpl.nim +++ b/lib/pure/collections/setimpl.nim @@ -62,6 +62,7 @@ template containsOrInclImpl() {.dirty.} = if index >= 0: result = true else: + result = false if mustRehash(s): enlarge(s) index = rawGetKnownHC(s, key, hc) @@ -86,7 +87,9 @@ proc exclImpl[A](s: var HashSet[A], key: A): bool {.inline.} = var j = i # The correctness of this depends on (h+1) in nextTry, var r = j # though may be adaptable to other simple sequences. s.data[i].hcode = 0 # mark current EMPTY - s.data[i].key = default(typeof(s.data[i].key)) + {.push warning[UnsafeDefault]:off.} + reset(s.data[i].key) + {.pop.} doWhile((i >= r and r > j) or (r > j and j > i) or (j > i and i >= r)): i = (i + 1) and msk # increment mod table size if isEmpty(s.data[i].hcode): # end of collision cluster; So all done diff --git a/lib/pure/collections/sets.nim b/lib/pure/collections/sets.nim index 114e4582a..af13135aa 100644 --- a/lib/pure/collections/sets.nim +++ b/lib/pure/collections/sets.nim @@ -25,7 +25,9 @@ ## `difference <#difference,HashSet[A],HashSet[A]>`_, and ## `symmetric difference <#symmetricDifference,HashSet[A],HashSet[A]>`_ ## -## .. code-block:: +## **Examples:** +## +## ```Nim ## echo toHashSet([9, 5, 1]) # {9, 1, 5} ## echo toOrderedSet([9, 5, 1]) # {9, 5, 1} ## @@ -37,7 +39,7 @@ ## echo s1 - s2 # {1, 9} ## echo s1 * s2 # {5} ## echo s1 -+- s2 # {9, 1, 3, 7} -## +## ``` ## ## Note: The data types declared here have *value semantics*: This means ## that `=` performs a copy of the set. @@ -48,7 +50,10 @@ import - hashes, math + std/[hashes, math] + +when not defined(nimHasEffectsOf): + {.pragma: effectsOf.} {.pragma: myShallow.} # For "integer-like A" that are too big for intsets/bit-vectors to be practical, @@ -125,7 +130,7 @@ proc initHashSet*[A](initialSize = defaultInitialSize): HashSet[A] = var a = initHashSet[int]() a.incl(3) assert len(a) == 1 - + result = default(HashSet[A]) result.init(initialSize) proc `[]`*[A](s: var HashSet[A], key: A): var A = @@ -246,7 +251,7 @@ iterator items*[A](s: HashSet[A]): A = ## If you need a sequence with the elements you can use `sequtils.toSeq ## template <sequtils.html#toSeq.t,untyped>`_. ## - ## .. code-block:: + ## ```Nim ## type ## pair = tuple[a, b: int] ## var @@ -259,6 +264,7 @@ iterator items*[A](s: HashSet[A]): A = ## assert a.len == 2 ## echo b ## # --> {(a: 1, b: 3), (a: 0, b: 4)} + ## ``` let length = s.len for h in 0 .. high(s.data): if isFilled(s.data[h].hcode): @@ -344,7 +350,7 @@ proc missingOrExcl*[A](s: var HashSet[A], key: A): bool = proc pop*[A](s: var HashSet[A]): A = ## Removes and returns an arbitrary element from the set `s`. ## - ## Raises KeyError if the set `s` is empty. + ## Raises `KeyError` if the set `s` is empty. ## ## See also: ## * `clear proc <#clear,HashSet[A]>`_ @@ -376,7 +382,9 @@ proc clear*[A](s: var HashSet[A]) = s.counter = 0 for i in 0 ..< s.data.len: s.data[i].hcode = 0 - s.data[i].key = default(typeof(s.data[i].key)) + {.push warning[UnsafeDefault]:off.} + reset(s.data[i].key) + {.pop.} proc union*[A](s1, s2: HashSet[A]): HashSet[A] = @@ -422,7 +430,7 @@ proc intersection*[A](s1, s2: HashSet[A]): HashSet[A] = assert c == toHashSet(["b"]) result = initHashSet[A](max(min(s1.data.len, s2.data.len), 2)) - + # iterate over the elements of the smaller set if s1.data.len < s2.data.len: for item in s1: @@ -430,7 +438,7 @@ proc intersection*[A](s1, s2: HashSet[A]): HashSet[A] = else: for item in s2: if item in s1: incl(result, item) - + proc difference*[A](s1, s2: HashSet[A]): HashSet[A] = ## Returns the difference of the sets `s1` and `s2`. @@ -556,7 +564,7 @@ proc `==`*[A](s, t: HashSet[A]): bool = s.counter == t.counter and s <= t -proc map*[A, B](data: HashSet[A], op: proc (x: A): B {.closure.}): HashSet[B] = +proc map*[A, B](data: HashSet[A], op: proc (x: A): B {.closure.}): HashSet[B] {.effectsOf: op.} = ## Returns a new set after applying `op` proc on each of the elements of ##`data` set. ## @@ -583,12 +591,12 @@ proc `$`*[A](s: HashSet[A]): string = ## any moment and values are not escaped. ## ## **Examples:** - ## - ## .. code-block:: + ## ```Nim ## echo toHashSet([2, 4, 5]) ## # --> {2, 4, 5} ## echo toHashSet(["no", "esc'aping", "is \" provided"]) ## # --> {no, esc'aping, is " provided} + ## ``` dollarImpl() @@ -810,7 +818,9 @@ proc clear*[A](s: var OrderedSet[A]) = for i in 0 ..< s.data.len: s.data[i].hcode = 0 s.data[i].next = 0 - s.data[i].key = default(typeof(s.data[i].key)) + {.push warning[UnsafeDefault]:off.} + reset(s.data[i].key) + {.pop.} proc len*[A](s: OrderedSet[A]): int {.inline.} = ## Returns the number of elements in `s`. @@ -871,12 +881,12 @@ proc `$`*[A](s: OrderedSet[A]): string = ## any moment and values are not escaped. ## ## **Examples:** - ## - ## .. code-block:: + ## ```Nim ## echo toOrderedSet([2, 4, 5]) ## # --> {2, 4, 5} ## echo toOrderedSet(["no", "esc'aping", "is \" provided"]) ## # --> {no, esc'aping, is " provided} + ## ``` dollarImpl() @@ -887,7 +897,7 @@ iterator items*[A](s: OrderedSet[A]): A = ## If you need a sequence with the elements you can use `sequtils.toSeq ## template <sequtils.html#toSeq.t,untyped>`_. ## - ## .. code-block:: + ## ```Nim ## var a = initOrderedSet[int]() ## for value in [9, 2, 1, 5, 1, 8, 4, 2]: ## a.incl(value) @@ -899,6 +909,7 @@ iterator items*[A](s: OrderedSet[A]): A = ## # --> Got 5 ## # --> Got 8 ## # --> Got 4 + ## ``` let length = s.len forAllOrderedPairs: yield s.data[h].key diff --git a/lib/pure/collections/sharedlist.nim b/lib/pure/collections/sharedlist.nim index e61883220..ec8f1cd86 100644 --- a/lib/pure/collections/sharedlist.nim +++ b/lib/pure/collections/sharedlist.nim @@ -16,7 +16,7 @@ {.push stackTrace: off.} import - locks + std/locks const ElemsPerNode = 100 diff --git a/lib/pure/collections/sharedtables.nim b/lib/pure/collections/sharedtables.nim index 816ab49ab..b474ecd31 100644 --- a/lib/pure/collections/sharedtables.nim +++ b/lib/pure/collections/sharedtables.nim @@ -17,7 +17,7 @@ {.deprecated.} import - hashes, math, locks + std/[hashes, math, locks] type KeyValuePair[A, B] = tuple[hcode: Hash, key: A, val: B] @@ -191,8 +191,7 @@ proc withKey*[A, B](t: var SharedTable[A, B], key: A, ## ## Example usage: ## - ## .. code-block:: nim - ## + ## ```nim ## # If value exists, decrement it. ## # If it becomes zero or less, delete the key ## t.withKey(1'i64) do (k: int64, v: var int, pairExists: var bool): @@ -200,6 +199,7 @@ proc withKey*[A, B](t: var SharedTable[A, B], key: A, ## dec v ## if v <= 0: ## pairExists = false + ## ``` withLock t: var hc: Hash var index = rawGet(t, key, hc) diff --git a/lib/pure/collections/tableimpl.nim b/lib/pure/collections/tableimpl.nim index 27c339177..3542741fa 100644 --- a/lib/pure/collections/tableimpl.nim +++ b/lib/pure/collections/tableimpl.nim @@ -11,6 +11,9 @@ include hashcommon +const + defaultInitialSize* = 32 + template rawGetDeepImpl() {.dirty.} = # Search algo for unconditional add genHashImpl(key, hc) var h: Hash = hc and maxHash(t) @@ -23,7 +26,7 @@ template rawInsertImpl() {.dirty.} = data[h].val = val data[h].hcode = hc -proc rawGetDeep[X, A](t: X, key: A, hc: var Hash): int {.inline.} = +proc rawGetDeep[X, A](t: X, key: A, hc: var Hash): int {.inline, outParamsAt: [3].} = rawGetDeepImpl() proc rawInsert[X, A, B](t: var X, data: var KeyValuePairSeq[A, B], @@ -31,9 +34,8 @@ proc rawInsert[X, A, B](t: var X, data: var KeyValuePairSeq[A, B], rawInsertImpl() template checkIfInitialized() = - when compiles(defaultInitialSize): - if t.dataLen == 0: - initImpl(t, defaultInitialSize) + if t.dataLen == 0: + initImpl(t, defaultInitialSize) template addImpl(enlarge) {.dirty.} = checkIfInitialized() @@ -43,7 +45,7 @@ template addImpl(enlarge) {.dirty.} = rawInsert(t, t.data, key, val, hc, j) inc(t.counter) -template maybeRehashPutImpl(enlarge) {.dirty.} = +template maybeRehashPutImpl(enlarge, val) {.dirty.} = checkIfInitialized() if mustRehash(t): enlarge(t) @@ -54,28 +56,41 @@ template maybeRehashPutImpl(enlarge) {.dirty.} = template putImpl(enlarge) {.dirty.} = checkIfInitialized() - var hc: Hash + var hc: Hash = default(Hash) var index = rawGet(t, key, hc) if index >= 0: t.data[index].val = val - else: maybeRehashPutImpl(enlarge) + else: maybeRehashPutImpl(enlarge, val) template mgetOrPutImpl(enlarge) {.dirty.} = checkIfInitialized() - var hc: Hash + var hc: Hash = default(Hash) var index = rawGet(t, key, hc) if index < 0: # not present: insert (flipping index) - maybeRehashPutImpl(enlarge) + when declared(val): + maybeRehashPutImpl(enlarge, val) + else: + maybeRehashPutImpl(enlarge, default(B)) # either way return modifiable val result = t.data[index].val +# template mgetOrPutDefaultImpl(enlarge) {.dirty.} = +# checkIfInitialized() +# var hc: Hash = default(Hash) +# var index = rawGet(t, key, hc) +# if index < 0: +# # not present: insert (flipping index) +# maybeRehashPutImpl(enlarge, default(B)) +# # either way return modifiable val +# result = t.data[index].val + template hasKeyOrPutImpl(enlarge) {.dirty.} = checkIfInitialized() - var hc: Hash + var hc: Hash = default(Hash) var index = rawGet(t, key, hc) if index < 0: result = false - maybeRehashPutImpl(enlarge) + maybeRehashPutImpl(enlarge, val) else: result = true # delImplIdx is KnuthV3 Algo6.4R adapted to i=i+1 (from i=i-1) which has come to @@ -117,8 +132,10 @@ template delImplIdx(t, i, makeEmpty, cellEmpty, cellHash) = var j = i # The correctness of this depends on (h+1) in nextTry var r = j # though may be adaptable to other simple sequences. makeEmpty(i) # mark current EMPTY - t.data[i].key = default(typeof(t.data[i].key)) - t.data[i].val = default(typeof(t.data[i].val)) + {.push warning[UnsafeDefault]:off.} + reset(t.data[i].key) + reset(t.data[i].val) + {.pop.} while true: i = (i + 1) and msk # increment mod table size if cellEmpty(i): # end of collision cluster; So all done @@ -149,8 +166,10 @@ template clearImpl() {.dirty.} = for i in 0 ..< t.dataLen: when compiles(t.data[i].hcode): # CountTable records don't contain a hcode t.data[i].hcode = 0 - t.data[i].key = default(typeof(t.data[i].key)) - t.data[i].val = default(typeof(t.data[i].val)) + {.push warning[UnsafeDefault]:off.} + reset(t.data[i].key) + reset(t.data[i].val) + {.pop.} t.counter = 0 template ctAnd(a, b): bool = @@ -208,3 +227,5 @@ template equalsImpl(s, t: typed) = if not t.hasKey(key): return false if t.getOrDefault(key) != val: return false return true + else: + return false diff --git a/lib/pure/collections/tables.nim b/lib/pure/collections/tables.nim index 5c62ed56a..d414caeed 100644 --- a/lib/pure/collections/tables.nim +++ b/lib/pure/collections/tables.nim @@ -136,14 +136,11 @@ runnableExamples: ## a more complex object as a key you will be greeted by a strange compiler ## error: ## -## .. code:: -## -## Error: type mismatch: got (Person) -## but expected one of: -## hashes.hash(x: openArray[A]): Hash -## hashes.hash(x: int): Hash -## hashes.hash(x: float): Hash -## … +## Error: type mismatch: got (Person) +## but expected one of: +## hashes.hash(x: openArray[A]): Hash +## hashes.hash(x: int): Hash +## hashes.hash(x: float): Hash ## ## What is happening here is that the types used for table keys require to have ## a `hash()` proc which will convert them to a `Hash <hashes.html#Hash>`_ @@ -197,7 +194,11 @@ runnableExamples: import std/private/since -import hashes, math, algorithm +import std/[hashes, math, algorithm] + + +when not defined(nimHasEffectsOf): + {.pragma: effectsOf.} type KeyValuePair[A, B] = tuple[hcode: Hash, key: A, val: B] @@ -216,8 +217,6 @@ type ## For creating a new empty TableRef, use `newTable proc ## <#newTable>`_. -const - defaultInitialSize* = 32 # ------------------------------ helpers --------------------------------- @@ -279,6 +278,7 @@ proc initTable*[A, B](initialSize = defaultInitialSize): Table[A, B] = let a = initTable[int, string]() b = initTable[char, seq[int]]() + result = default(Table[A, B]) initImpl(result, initialSize) proc `[]=`*[A, B](t: var Table[A, B], key: A, val: sink B) = @@ -313,7 +313,7 @@ proc toTable*[A, B](pairs: openArray[(A, B)]): Table[A, B] = result = initTable[A, B](pairs.len) for key, val in items(pairs): result[key] = val -proc `[]`*[A, B](t: Table[A, B], key: A): B = +proc `[]`*[A, B](t: Table[A, B], key: A): lent B = ## Retrieves the value at `t[key]`. ## ## If `key` is not in `t`, the `KeyError` exception is raised. @@ -416,7 +416,7 @@ proc getOrDefault*[A, B](t: Table[A, B], key: A): B = let a = {'a': 5, 'b': 9}.toTable doAssert a.getOrDefault('a') == 5 doAssert a.getOrDefault('z') == 0 - + result = default(B) getOrDefaultImpl(t, key) proc getOrDefault*[A, B](t: Table[A, B], key: A, default: B): B = @@ -434,7 +434,7 @@ proc getOrDefault*[A, B](t: Table[A, B], key: A, default: B): B = let a = {'a': 5, 'b': 9}.toTable doAssert a.getOrDefault('a', 99) == 5 doAssert a.getOrDefault('z', 99) == 99 - + result = default(B) getOrDefaultImpl(t, key, default) proc mgetOrPut*[A, B](t: var Table[A, B], key: A, val: B): var B = @@ -443,7 +443,7 @@ proc mgetOrPut*[A, B](t: var Table[A, B], key: A, val: B): var B = ## ## ## Note that while the value returned is of type `var B`, - ## it is easy to accidentally create an copy of the value at `t[key]`. + ## it is easy to accidentally create a copy of the value at `t[key]`. ## Remember that seqs and strings are value types, and therefore ## cannot be copied into a separate variable for modification. ## See the example below. @@ -475,6 +475,18 @@ proc mgetOrPut*[A, B](t: var Table[A, B], key: A, val: B): var B = mgetOrPutImpl(enlarge) +proc mgetOrPut*[A, B](t: var Table[A, B], key: A): var B = + ## Retrieves the value at `t[key]` or puts the + ## default initialization value for type `B` (e.g. 0 for any + ## integer type). + runnableExamples: + var a = {'a': 5}.newTable + doAssert a.mgetOrPut('a') == 5 + a.mgetOrPut('z').inc + doAssert a == {'a': 5, 'z': 1}.newTable + + mgetOrPutImpl(enlarge) + proc len*[A, B](t: Table[A, B]): int = ## Returns the number of keys in `t`. runnableExamples: @@ -676,7 +688,7 @@ iterator pairs*[A, B](t: Table[A, B]): (A, B) = ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## let a = { ## 'o': [1, 5, 7, 9], ## 'e': [2, 4, 6, 8] @@ -690,6 +702,7 @@ iterator pairs*[A, B](t: Table[A, B]): (A, B) = ## # value: [2, 4, 6, 8] ## # key: o ## # value: [1, 5, 7, 9] + ## ``` let L = len(t) for h in 0 .. high(t.data): if isFilled(t.data[h].hcode): @@ -824,7 +837,8 @@ proc newTable*[A, B](initialSize = defaultInitialSize): TableRef[A, B] = b = newTable[char, seq[int]]() new(result) - result[] = initTable[A, B](initialSize) + {.noSideEffect.}: + result[] = initTable[A, B](initialSize) proc newTable*[A, B](pairs: openArray[(A, B)]): TableRef[A, B] = ## Creates a new ref hash table that contains the given `pairs`. @@ -840,14 +854,16 @@ proc newTable*[A, B](pairs: openArray[(A, B)]): TableRef[A, B] = assert b == {'a': 5, 'b': 9}.newTable new(result) - result[] = toTable[A, B](pairs) + {.noSideEffect.}: + result[] = toTable[A, B](pairs) proc newTableFrom*[A, B, C](collection: A, index: proc(x: B): C): TableRef[C, B] = ## Index the collection with the proc provided. # TODO: As soon as supported, change collection: A to collection: A[B] result = newTable[C, B]() - for item in collection: - result[index(item)] = item + {.noSideEffect.}: + for item in collection: + result[index(item)] = item proc `[]`*[A, B](t: TableRef[A, B], key: A): var B = ## Retrieves the value at `t[key]`. @@ -1010,6 +1026,18 @@ proc mgetOrPut*[A, B](t: TableRef[A, B], key: A, val: B): var B = doAssert t[25] == @[25, 35] t[].mgetOrPut(key, val) +proc mgetOrPut*[A, B](t: TableRef[A, B], key: A): var B = + ## Retrieves the value at `t[key]` or puts the + ## default initialization value for type `B` (e.g. 0 for any + ## integer type). + runnableExamples: + var a = {'a': 5}.newTable + doAssert a.mgetOrPut('a') == 5 + a.mgetOrPut('z').inc + doAssert a == {'a': 5, 'z': 1}.newTable + + t[].mgetOrPut(key) + proc len*[A, B](t: TableRef[A, B]): int = ## Returns the number of keys in `t`. runnableExamples: @@ -1122,7 +1150,7 @@ iterator pairs*[A, B](t: TableRef[A, B]): (A, B) = ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## let a = { ## 'o': [1, 5, 7, 9], ## 'e': [2, 4, 6, 8] @@ -1136,6 +1164,7 @@ iterator pairs*[A, B](t: TableRef[A, B]): (A, B) = ## # value: [2, 4, 6, 8] ## # key: o ## # value: [1, 5, 7, 9] + ## ``` let L = len(t) for h in 0 .. high(t.data): if isFilled(t.data[h].hcode): @@ -1318,6 +1347,7 @@ proc initOrderedTable*[A, B](initialSize = defaultInitialSize): OrderedTable[A, let a = initOrderedTable[int, string]() b = initOrderedTable[char, seq[int]]() + result = default(OrderedTable[A, B]) initImpl(result, initialSize) proc `[]=`*[A, B](t: var OrderedTable[A, B], key: A, val: sink B) = @@ -1353,7 +1383,7 @@ proc toOrderedTable*[A, B](pairs: openArray[(A, B)]): OrderedTable[A, B] = result = initOrderedTable[A, B](pairs.len) for key, val in items(pairs): result[key] = val -proc `[]`*[A, B](t: OrderedTable[A, B], key: A): B = +proc `[]`*[A, B](t: OrderedTable[A, B], key: A): lent B = ## Retrieves the value at `t[key]`. ## ## If `key` is not in `t`, the `KeyError` exception is raised. @@ -1409,7 +1439,7 @@ proc hasKey*[A, B](t: OrderedTable[A, B], key: A): bool = doAssert a.hasKey('a') == true doAssert a.hasKey('z') == false - var hc: Hash + var hc: Hash = default(Hash) result = rawGet(t, key, hc) >= 0 proc contains*[A, B](t: OrderedTable[A, B], key: A): bool = @@ -1458,7 +1488,7 @@ proc getOrDefault*[A, B](t: OrderedTable[A, B], key: A): B = let a = {'a': 5, 'b': 9}.toOrderedTable doAssert a.getOrDefault('a') == 5 doAssert a.getOrDefault('z') == 0 - + result = default(B) getOrDefaultImpl(t, key) proc getOrDefault*[A, B](t: OrderedTable[A, B], key: A, default: B): B = @@ -1476,7 +1506,7 @@ proc getOrDefault*[A, B](t: OrderedTable[A, B], key: A, default: B): B = let a = {'a': 5, 'b': 9}.toOrderedTable doAssert a.getOrDefault('a', 99) == 5 doAssert a.getOrDefault('z', 99) == 99 - + result = default(B) getOrDefaultImpl(t, key, default) proc mgetOrPut*[A, B](t: var OrderedTable[A, B], key: A, val: B): var B = @@ -1499,6 +1529,18 @@ proc mgetOrPut*[A, B](t: var OrderedTable[A, B], key: A, val: B): var B = mgetOrPutImpl(enlarge) +proc mgetOrPut*[A, B](t: var OrderedTable[A, B], key: A): var B = + ## Retrieves the value at `t[key]` or puts the + ## default initialization value for type `B` (e.g. 0 for any + ## integer type). + runnableExamples: + var a = {'a': 5}.toOrderedTable + doAssert a.mgetOrPut('a') == 5 + a.mgetOrPut('z').inc + doAssert a == {'a': 5, 'z': 1}.toOrderedTable + + mgetOrPutImpl(enlarge) + proc len*[A, B](t: OrderedTable[A, B]): int {.inline.} = ## Returns the number of keys in `t`. runnableExamples: @@ -1597,7 +1639,7 @@ proc clear*[A, B](t: var OrderedTable[A, B]) = t.last = -1 proc sort*[A, B](t: var OrderedTable[A, B], cmp: proc (x, y: (A, B)): int, - order = SortOrder.Ascending) = + order = SortOrder.Ascending) {.effectsOf: cmp.} = ## Sorts `t` according to the function `cmp`. ## ## This modifies the internal list @@ -1698,7 +1740,7 @@ iterator pairs*[A, B](t: OrderedTable[A, B]): (A, B) = ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## let a = { ## 'o': [1, 5, 7, 9], ## 'e': [2, 4, 6, 8] @@ -1712,6 +1754,7 @@ iterator pairs*[A, B](t: OrderedTable[A, B]): (A, B) = ## # value: [1, 5, 7, 9] ## # key: e ## # value: [2, 4, 6, 8] + ## ``` let L = len(t) forAllOrderedPairs: @@ -1821,7 +1864,8 @@ proc newOrderedTable*[A, B](initialSize = defaultInitialSize): OrderedTableRef[A a = newOrderedTable[int, string]() b = newOrderedTable[char, seq[int]]() new(result) - result[] = initOrderedTable[A, B](initialSize) + {.noSideEffect.}: + result[] = initOrderedTable[A, B](initialSize) proc newOrderedTable*[A, B](pairs: openArray[(A, B)]): OrderedTableRef[A, B] = ## Creates a new ordered ref hash table that contains the given `pairs`. @@ -1838,7 +1882,8 @@ proc newOrderedTable*[A, B](pairs: openArray[(A, B)]): OrderedTableRef[A, B] = assert b == {'a': 5, 'b': 9}.newOrderedTable result = newOrderedTable[A, B](pairs.len) - for key, val in items(pairs): result[key] = val + {.noSideEffect.}: + for key, val in items(pairs): result[key] = val proc `[]`*[A, B](t: OrderedTableRef[A, B], key: A): var B = @@ -1985,6 +2030,18 @@ proc mgetOrPut*[A, B](t: OrderedTableRef[A, B], key: A, val: B): var B = result = t[].mgetOrPut(key, val) +proc mgetOrPut*[A, B](t: OrderedTableRef[A, B], key: A): var B = + ## Retrieves the value at `t[key]` or puts the + ## default initialization value for type `B` (e.g. 0 for any + ## integer type). + runnableExamples: + var a = {'a': 5}.toOrderedTable + doAssert a.mgetOrPut('a') == 5 + a.mgetOrPut('z').inc + doAssert a == {'a': 5, 'z': 1}.toOrderedTable + + t[].mgetOrPut(key) + proc len*[A, B](t: OrderedTableRef[A, B]): int {.inline.} = ## Returns the number of keys in `t`. runnableExamples: @@ -2054,7 +2111,7 @@ proc clear*[A, B](t: OrderedTableRef[A, B]) = clear(t[]) proc sort*[A, B](t: OrderedTableRef[A, B], cmp: proc (x, y: (A, B)): int, - order = SortOrder.Ascending) = + order = SortOrder.Ascending) {.effectsOf: cmp.} = ## Sorts `t` according to the function `cmp`. ## ## This modifies the internal list @@ -2106,7 +2163,7 @@ iterator pairs*[A, B](t: OrderedTableRef[A, B]): (A, B) = ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## let a = { ## 'o': [1, 5, 7, 9], ## 'e': [2, 4, 6, 8] @@ -2120,6 +2177,7 @@ iterator pairs*[A, B](t: OrderedTableRef[A, B]): (A, B) = ## # value: [1, 5, 7, 9] ## # key: e ## # value: [2, 4, 6, 8] + ## ``` let L = len(t) forAllOrderedPairs: @@ -2280,6 +2338,7 @@ proc initCountTable*[A](initialSize = defaultInitialSize): CountTable[A] = ## * `toCountTable proc<#toCountTable,openArray[A]>`_ ## * `newCountTable proc<#newCountTable>`_ for creating a ## `CountTableRef` + result = default(CountTable[A]) initImpl(result, initialSize) proc toCountTable*[A](keys: openArray[A]): CountTable[A] = @@ -2389,7 +2448,7 @@ proc contains*[A](t: CountTable[A], key: A): bool = return hasKey[A](t, key) proc getOrDefault*[A](t: CountTable[A], key: A; default: int = 0): int = - ## Retrieves the value at `t[key]` if`key` is in `t`. Otherwise, the + ## Retrieves the value at `t[key]` if `key` is in `t`. Otherwise, the ## integer value of `default` is returned. ## ## See also: @@ -2519,7 +2578,7 @@ iterator pairs*[A](t: CountTable[A]): (A, int) = ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## let a = toCountTable("abracadabra") ## ## for k, v in pairs(a): @@ -2536,6 +2595,7 @@ iterator pairs*[A](t: CountTable[A]): (A, int) = ## # value: 1 ## # key: r ## # value: 2 + ## ``` let L = len(t) for h in 0 .. high(t.data): if t.data[h].val != 0: @@ -2637,13 +2697,15 @@ proc newCountTable*[A](initialSize = defaultInitialSize): CountTableRef[A] = ## * `initCountTable proc<#initCountTable>`_ for creating a ## `CountTable` new(result) - result[] = initCountTable[A](initialSize) + {.noSideEffect.}: + result[] = initCountTable[A](initialSize) proc newCountTable*[A](keys: openArray[A]): CountTableRef[A] = ## Creates a new ref count table with every member of a container `keys` ## having a count of how many times it occurs in that container. result = newCountTable[A](keys.len) - for key in items(keys): result.inc(key) + {.noSideEffect.}: + for key in items(keys): result.inc(key) proc `[]`*[A](t: CountTableRef[A], key: A): int = ## Retrieves the value at `t[key]` if `key` is in `t`. @@ -2667,7 +2729,8 @@ proc `[]=`*[A](t: CountTableRef[A], key: A, val: int) = ## * `inc proc<#inc,CountTableRef[A],A,int>`_ for incrementing a ## value of a key assert val > 0 - t[][key] = val + {.noSideEffect.}: + t[][key] = val proc inc*[A](t: CountTableRef[A], key: A, val = 1) = ## Increments `t[key]` by `val` (default: 1). @@ -2676,7 +2739,8 @@ proc inc*[A](t: CountTableRef[A], key: A, val = 1) = a.inc('a') a.inc('b', 10) doAssert a == newCountTable("aaabbbbbbbbbbb") - t[].inc(key, val) + {.noSideEffect.}: + t[].inc(key, val) proc smallest*[A](t: CountTableRef[A]): tuple[key: A, val: int] = ## Returns the `(key, value)` pair with the smallest `val`. Efficiency: O(n) @@ -2709,7 +2773,7 @@ proc contains*[A](t: CountTableRef[A], key: A): bool = return hasKey[A](t, key) proc getOrDefault*[A](t: CountTableRef[A], key: A, default: int): int = - ## Retrieves the value at `t[key]` if`key` is in `t`. Otherwise, the + ## Retrieves the value at `t[key]` if `key` is in `t`. Otherwise, the ## integer value of `default` is returned. ## ## See also: @@ -2795,7 +2859,7 @@ iterator pairs*[A](t: CountTableRef[A]): (A, int) = ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## let a = newCountTable("abracadabra") ## ## for k, v in pairs(a): @@ -2812,6 +2876,7 @@ iterator pairs*[A](t: CountTableRef[A]): (A, int) = ## # value: 1 ## # key: r ## # value: 2 + ## ``` let L = len(t) for h in 0 .. high(t.data): if t.data[h].val != 0: @@ -2890,3 +2955,18 @@ iterator mvalues*[A](t: CountTableRef[A]): var int = if t.data[h].val != 0: yield t.data[h].val assert(len(t) == L, "the length of the table changed while iterating over it") + +proc hash*[K,V](s: Table[K,V]): Hash = + for p in pairs(s): + result = result xor hash(p) + result = !$result + +proc hash*[K,V](s: OrderedTable[K,V]): Hash = + for p in pairs(s): + result = result !& hash(p) + result = !$result + +proc hash*[V](s: CountTable[V]): Hash = + for p in pairs(s): + result = result xor hash(p) + result = !$result diff --git a/lib/pure/colors.nim b/lib/pure/colors.nim index 8cda2fab8..d3e6dc063 100644 --- a/lib/pure/colors.nim +++ b/lib/pure/colors.nim @@ -9,22 +9,23 @@ ## This module implements color handling for Nim, ## namely color mixing and parsing the CSS color names. -import strutils -from algorithm import binarySearch +import std/strutils +from std/algorithm import binarySearch type Color* = distinct int ## A color stored as RGB, e.g. `0xff00cc`. -proc `==` *(a, b: Color): bool {.borrow.} +proc `==`*(a, b: Color): bool {.borrow.} ## Compares two colors. ## - ## .. code-block:: + ## ```Nim ## var ## a = Color(0xff_00_ff) ## b = colFuchsia ## c = Color(0x00_ff_cc) ## assert a == b - ## assert not a == c + ## assert not (a == c) + ## ``` template extract(a: Color, r, g, b: untyped) = var r = a.int shr 16 and 0xff @@ -200,8 +201,8 @@ const colGoldenRod* = Color(0xDAA520) colGray* = Color(0x808080) colGreen* = Color(0x008000) - colGrey* = Color(0x808080) colGreenYellow* = Color(0xADFF2F) + colGrey* = Color(0x808080) colHoneyDew* = Color(0xF0FFF0) colHotPink* = Color(0xFF69B4) colIndianRed* = Color(0xCD5C5C) @@ -350,8 +351,8 @@ const "goldenrod": colGoldenRod, "gray": colGray, "green": colGreen, - "grey": colGrey, "greenyellow": colGreenYellow, + "grey": colGrey, "honeydew": colHoneyDew, "hotpink": colHotPink, "indianred": colIndianRed, diff --git a/lib/pure/complex.nim b/lib/pure/complex.nim index 1e8349ef6..b48811eae 100644 --- a/lib/pure/complex.nim +++ b/lib/pure/complex.nim @@ -15,9 +15,6 @@ runnableExamples: from std/math import almostEqual, sqrt - func almostEqual(a, b: Complex): bool = - almostEqual(a.re, b.re) and almostEqual(a.im, b.im) - let z1 = complex(1.0, 2.0) z2 = complex(3.0, -4.0) @@ -36,7 +33,7 @@ runnableExamples: {.push checks: off, line_dir: off, stack_trace: off, debugger: off.} # the user does not want to trace a part of the standard library! -import math +import std/[math, strformat] type Complex*[T: SomeFloat] = object @@ -81,7 +78,7 @@ func abs2*[T](z: Complex[T]): T = ## that is the squared distance from (0, 0) to `z`. ## This is more efficient than `abs(z) ^ 2`. result = z.re * z.re + z.im * z.im - + func sgn*[T](z: Complex[T]): Complex[T] = ## Returns the phase of `z` as a unit complex number, ## or 0 if `z` is 0. @@ -249,10 +246,31 @@ func pow*[T](x, y: Complex[T]): Complex[T] = else: result.re = 0.0 result.im = 0.0 - elif y.re == 1.0 and y.im == 0.0: - result = x - elif y.re == -1.0 and y.im == 0.0: - result = T(1.0) / x + elif y.im == 0.0: + if y.re == 1.0: + result = x + elif y.re == -1.0: + result = T(1.0) / x + elif y.re == 2.0: + result = x * x + elif y.re == 0.5: + result = sqrt(x) + elif x.im == 0.0: + # Revert to real pow when both base and exponent are real + result.re = pow(x.re, y.re) + result.im = 0.0 + else: + # Special case when the exponent is real + let + rho = abs(x) + theta = arctan2(x.im, x.re) + s = pow(rho, y.re) + r = y.re * theta + result.re = s * cos(r) + result.im = s * sin(r) + elif x.im == 0.0 and x.re == E: + # Special case Euler's formula + result = exp(y) else: let rho = abs(x) @@ -391,6 +409,24 @@ func rect*[T](r, phi: T): Complex[T] = ## * `polar func<#polar,Complex[T]>`_ for the inverse operation complex(r * cos(phi), r * sin(phi)) +func almostEqual*[T: SomeFloat](x, y: Complex[T]; unitsInLastPlace: Natural = 4): bool = + ## Checks if two complex values are almost equal, using the + ## [machine epsilon](https://en.wikipedia.org/wiki/Machine_epsilon). + ## + ## Two complex values are considered almost equal if their real and imaginary + ## components are almost equal. + ## + ## `unitsInLastPlace` is the max number of + ## [units in the last place](https://en.wikipedia.org/wiki/Unit_in_the_last_place) + ## difference tolerated when comparing two numbers. The larger the value, the + ## more error is allowed. A `0` value means that two numbers must be exactly the + ## same to be considered equal. + ## + ## The machine epsilon has to be scaled to the magnitude of the values used + ## and multiplied by the desired precision in ULPs unless the difference is + ## subnormal. + almostEqual(x.re, y.re, unitsInLastPlace = unitsInLastPlace) and + almostEqual(x.im, y.im, unitsInLastPlace = unitsInLastPlace) func `$`*(z: Complex): string = ## Returns `z`'s string representation as `"(re, im)"`. @@ -399,4 +435,39 @@ func `$`*(z: Complex): string = result = "(" & $z.re & ", " & $z.im & ")" +proc formatValueAsTuple(result: var string; value: Complex; specifier: string) = + ## Format implementation for `Complex` representing the value as a (real, imaginary) tuple. + result.add "(" + formatValue(result, value.re, specifier) + result.add ", " + formatValue(result, value.im, specifier) + result.add ")" + +proc formatValueAsComplexNumber(result: var string; value: Complex; specifier: string) = + ## Format implementation for `Complex` representing the value as a (RE+IMj) number + ## By default, the real and imaginary parts are formatted using the general ('g') format + let specifier = if specifier.contains({'e', 'E', 'f', 'F', 'g', 'G'}): + specifier.replace("j") + else: + specifier.replace('j', 'g') + result.add "(" + formatValue(result, value.re, specifier) + if value.im >= 0 and not specifier.contains({'+', '-'}): + result.add "+" + formatValue(result, value.im, specifier) + result.add "j)" + +proc formatValue*(result: var string; value: Complex; specifier: string) = + ## Standard format implementation for `Complex`. It makes little + ## sense to call this directly, but it is required to exist + ## by the `&` macro. + ## For complex numbers, we add a specific 'j' specifier, which formats + ## the value as (A+Bj) like in mathematics. + if specifier.len == 0: + result.add $value + elif 'j' in specifier: + formatValueAsComplexNumber(result, value, specifier) + else: + formatValueAsTuple(result, value, specifier) + {.pop.} diff --git a/lib/pure/concurrency/atomics.nim b/lib/pure/concurrency/atomics.nim index 315f44f33..818f1b37a 100644 --- a/lib/pure/concurrency/atomics.nim +++ b/lib/pure/concurrency/atomics.nim @@ -10,6 +10,9 @@ ## Types and operations for atomic operations and lockless algorithms. ## ## Unstable API. +## +## By default, C++ uses C11 atomic primitives. To use C++ `std::atomic`, +## `-d:nimUseCppAtomics` can be defined. runnableExamples: # Atomic @@ -50,8 +53,7 @@ runnableExamples: flag.clear(moRelaxed) assert not flag.testAndSet - -when defined(cpp) or defined(nimdoc): +when (defined(cpp) and defined(nimUseCppAtomics)) or defined(nimdoc): # For the C++ backend, types and operations map directly to C++11 atomics. {.push, header: "<atomic>".} @@ -211,8 +213,8 @@ else: # MSVC intrinsics proc interlockedExchange(location: pointer; desired: int8): int8 {.importc: "_InterlockedExchange8".} - proc interlockedExchange(location: pointer; desired: int16): int16 {.importc: "_InterlockedExchange".} - proc interlockedExchange(location: pointer; desired: int32): int32 {.importc: "_InterlockedExchange16".} + proc interlockedExchange(location: pointer; desired: int16): int16 {.importc: "_InterlockedExchange16".} + proc interlockedExchange(location: pointer; desired: int32): int32 {.importc: "_InterlockedExchange".} proc interlockedExchange(location: pointer; desired: int64): int64 {.importc: "_InterlockedExchange64".} proc interlockedCompareExchange(location: pointer; desired, expected: int8): int8 {.importc: "_InterlockedCompareExchange8".} @@ -274,10 +276,17 @@ else: cast[T](interlockedXor(addr(location.value), cast[nonAtomicType(T)](value))) else: - {.push, header: "<stdatomic.h>".} + when defined(cpp): + {.push, header: "<atomic>".} + template maybeWrapStd(x: string): string = + "std::" & x + else: + {.push, header: "<stdatomic.h>".} + template maybeWrapStd(x: string): string = + x type - MemoryOrder* {.importc: "memory_order".} = enum + MemoryOrder* {.importc: "memory_order".maybeWrapStd.} = enum moRelaxed moConsume moAcquire @@ -285,16 +294,25 @@ else: moAcquireRelease moSequentiallyConsistent - type - # Atomic*[T] {.importcpp: "_Atomic('0)".} = object + when defined(cpp): + type + # Atomic*[T] {.importcpp: "_Atomic('0)".} = object + + AtomicInt8 {.importc: "std::atomic<NI8>".} = int8 + AtomicInt16 {.importc: "std::atomic<NI16>".} = int16 + AtomicInt32 {.importc: "std::atomic<NI32>".} = int32 + AtomicInt64 {.importc: "std::atomic<NI64>".} = int64 + else: + type + # Atomic*[T] {.importcpp: "_Atomic('0)".} = object - AtomicInt8 {.importc: "_Atomic NI8".} = int8 - AtomicInt16 {.importc: "_Atomic NI16".} = int16 - AtomicInt32 {.importc: "_Atomic NI32".} = int32 - AtomicInt64 {.importc: "_Atomic NI64".} = int64 + AtomicInt8 {.importc: "_Atomic NI8".} = int8 + AtomicInt16 {.importc: "_Atomic NI16".} = int16 + AtomicInt32 {.importc: "_Atomic NI32".} = int32 + AtomicInt64 {.importc: "_Atomic NI64".} = int64 type - AtomicFlag* {.importc: "atomic_flag", size: 1.} = object + AtomicFlag* {.importc: "atomic_flag".maybeWrapStd, size: 1.} = object Atomic*[T] = object when T is Trivial: @@ -308,27 +326,27 @@ else: guard: AtomicFlag #proc init*[T](location: var Atomic[T]; value: T): T {.importcpp: "atomic_init(@)".} - proc atomic_load_explicit[T, A](location: ptr A; order: MemoryOrder): T {.importc.} - proc atomic_store_explicit[T, A](location: ptr A; desired: T; order: MemoryOrder = moSequentiallyConsistent) {.importc.} - proc atomic_exchange_explicit[T, A](location: ptr A; desired: T; order: MemoryOrder = moSequentiallyConsistent): T {.importc.} - proc atomic_compare_exchange_strong_explicit[T, A](location: ptr A; expected: ptr T; desired: T; success, failure: MemoryOrder): bool {.importc.} - proc atomic_compare_exchange_weak_explicit[T, A](location: ptr A; expected: ptr T; desired: T; success, failure: MemoryOrder): bool {.importc.} + proc atomic_load_explicit[T, A](location: ptr A; order: MemoryOrder): T {.importc: "atomic_load_explicit".maybeWrapStd.} + proc atomic_store_explicit[T, A](location: ptr A; desired: T; order: MemoryOrder = moSequentiallyConsistent) {.importc: "atomic_store_explicit".maybeWrapStd.} + proc atomic_exchange_explicit[T, A](location: ptr A; desired: T; order: MemoryOrder = moSequentiallyConsistent): T {.importc: "atomic_exchange_explicit".maybeWrapStd.} + proc atomic_compare_exchange_strong_explicit[T, A](location: ptr A; expected: ptr T; desired: T; success, failure: MemoryOrder): bool {.importc: "atomic_compare_exchange_strong_explicit".maybeWrapStd.} + proc atomic_compare_exchange_weak_explicit[T, A](location: ptr A; expected: ptr T; desired: T; success, failure: MemoryOrder): bool {.importc: "atomic_compare_exchange_weak_explicit".maybeWrapStd.} # Numerical operations - proc atomic_fetch_add_explicit[T, A](location: ptr A; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.importc.} - proc atomic_fetch_sub_explicit[T, A](location: ptr A; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.importc.} - proc atomic_fetch_and_explicit[T, A](location: ptr A; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.importc.} - proc atomic_fetch_or_explicit[T, A](location: ptr A; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.importc.} - proc atomic_fetch_xor_explicit[T, A](location: ptr A; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.importc.} + proc atomic_fetch_add_explicit[T, A](location: ptr A; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.importc: "atomic_fetch_add_explicit".maybeWrapStd.} + proc atomic_fetch_sub_explicit[T, A](location: ptr A; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.importc: "atomic_fetch_sub_explicit".maybeWrapStd.} + proc atomic_fetch_and_explicit[T, A](location: ptr A; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.importc: "atomic_fetch_and_explicit".maybeWrapStd.} + proc atomic_fetch_or_explicit[T, A](location: ptr A; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.importc: "atomic_fetch_or_explicit".maybeWrapStd.} + proc atomic_fetch_xor_explicit[T, A](location: ptr A; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.importc: "atomic_fetch_xor_explicit".maybeWrapStd.} # Flag operations # var ATOMIC_FLAG_INIT {.importc, nodecl.}: AtomicFlag # proc init*(location: var AtomicFlag) {.inline.} = location = ATOMIC_FLAG_INIT - proc testAndSet*(location: var AtomicFlag; order: MemoryOrder = moSequentiallyConsistent): bool {.importc: "atomic_flag_test_and_set_explicit".} - proc clear*(location: var AtomicFlag; order: MemoryOrder = moSequentiallyConsistent) {.importc: "atomic_flag_clear_explicit".} + proc testAndSet*(location: var AtomicFlag; order: MemoryOrder = moSequentiallyConsistent): bool {.importc: "atomic_flag_test_and_set_explicit".maybeWrapStd.} + proc clear*(location: var AtomicFlag; order: MemoryOrder = moSequentiallyConsistent) {.importc: "atomic_flag_clear_explicit".maybeWrapStd.} - proc fence*(order: MemoryOrder) {.importc: "atomic_thread_fence".} - proc signalFence*(order: MemoryOrder) {.importc: "atomic_signal_fence".} + proc fence*(order: MemoryOrder) {.importc: "atomic_thread_fence".maybeWrapStd.} + proc signalFence*(order: MemoryOrder) {.importc: "atomic_signal_fence".maybeWrapStd.} {.pop.} diff --git a/lib/pure/concurrency/cpuinfo.nim b/lib/pure/concurrency/cpuinfo.nim index 16d32002d..9bc3fd579 100644 --- a/lib/pure/concurrency/cpuinfo.nim +++ b/lib/pure/concurrency/cpuinfo.nim @@ -15,75 +15,96 @@ runnableExamples: include "system/inclrtl" -when defined(posix) and not (defined(macosx) or defined(bsd)): - import posix +when defined(js): + import std/jsffi + proc countProcessorsImpl(): int = + when defined(nodejs): + let jsOs = require("os") + let jsObj = jsOs.cpus().length + else: + # `navigator.hardwareConcurrency` + # works on browser as well as deno. + let navigator{.importcpp.}: JsObject + let jsObj = navigator.hardwareConcurrency + result = jsObj.to int +else: + when defined(posix) and not (defined(macosx) or defined(bsd)): + import std/posix -when defined(windows): - import std/private/win_getsysteminfo + when defined(windows): + import std/private/win_getsysteminfo + + when defined(freebsd) or defined(macosx): + {.emit: "#include <sys/types.h>".} + + when defined(openbsd) or defined(netbsd): + {.emit: "#include <sys/param.h>".} -when defined(freebsd) or defined(macosx): - {.emit: "#include <sys/types.h>".} + when defined(macosx) or defined(bsd): + # we HAVE to emit param.h before sysctl.h so we cannot use .header here + # either. The amount of archaic bullshit in Poonix based OSes is just insane. + {.emit: "#include <sys/sysctl.h>".} + {.push nodecl.} + when defined(macosx): + proc sysctlbyname(name: cstring, + oldp: pointer, oldlenp: var csize_t, + newp: pointer, newlen: csize_t): cint {.importc.} + let + CTL_HW{.importc.}: cint + HW_NCPU{.importc.}: cint + proc sysctl[I: static[int]](name: var array[I, cint], namelen: cuint, + oldp: pointer, oldlenp: var csize_t, + newp: pointer, newlen: csize_t): cint {.importc.} + {.pop.} -when defined(openbsd) or defined(netbsd): - {.emit: "#include <sys/param.h>".} + when defined(genode): + import genode/env -when defined(macosx) or defined(bsd): - # we HAVE to emit param.h before sysctl.h so we cannot use .header here - # either. The amount of archaic bullshit in Poonix based OSes is just insane. - {.emit: "#include <sys/sysctl.h>".} - const - CTL_HW = 6 - HW_AVAILCPU = 25 - HW_NCPU = 3 - proc sysctl(x: ptr array[0..3, cint], y: cint, z: pointer, - a: var csize_t, b: pointer, c: csize_t): cint {. - importc: "sysctl", nodecl.} + proc affinitySpaceTotal(env: GenodeEnvPtr): cuint {. + importcpp: "@->cpu().affinity_space().total()".} -when defined(genode): - include genode/env + when defined(haiku): + type + SystemInfo {.importc: "system_info", header: "<OS.h>".} = object + cpuCount {.importc: "cpu_count".}: uint32 - proc affinitySpaceTotal(env: GenodeEnvPtr): cuint {. - importcpp: "@->cpu().affinity_space().total()".} + proc getSystemInfo(info: ptr SystemInfo): int32 {.importc: "get_system_info", + header: "<OS.h>".} + + proc countProcessorsImpl(): int {.inline.} = + when defined(windows): + var + si: SystemInfo + getSystemInfo(addr si) + result = int(si.dwNumberOfProcessors) + elif defined(macosx) or defined(bsd): + let dest = addr result + var len = sizeof(result).csize_t + when defined(macosx): + # alias of "hw.activecpu" + if sysctlbyname("hw.logicalcpu", dest, len, nil, 0) == 0: + return + var mib = [CTL_HW, HW_NCPU] + if sysctl(mib, 2, dest, len, nil, 0) == 0: + return + elif defined(hpux): + result = mpctl(MPC_GETNUMSPUS, nil, nil) + elif defined(irix): + var SC_NPROC_ONLN {.importc: "_SC_NPROC_ONLN", header: "<unistd.h>".}: cint + result = sysconf(SC_NPROC_ONLN) + elif defined(genode): + result = runtimeEnv.affinitySpaceTotal().int + elif defined(haiku): + var sysinfo: SystemInfo + if getSystemInfo(addr sysinfo) == 0: + result = sysinfo.cpuCount.int + else: + result = sysconf(SC_NPROCESSORS_ONLN) + if result < 0: result = 0 -when defined(haiku): - type - SystemInfo {.importc: "system_info", header: "<OS.h>".} = object - cpuCount {.importc: "cpu_count".}: uint32 - proc getSystemInfo(info: ptr SystemInfo): int32 {.importc: "get_system_info", - header: "<OS.h>".} proc countProcessors*(): int {.rtl, extern: "ncpi$1".} = ## Returns the number of the processors/cores the machine has. ## Returns 0 if it cannot be detected. - when defined(windows): - var - si: SystemInfo - getSystemInfo(addr si) - result = int(si.dwNumberOfProcessors) - elif defined(macosx) or defined(bsd): - var - mib: array[0..3, cint] - numCPU: int - mib[0] = CTL_HW - mib[1] = HW_AVAILCPU - var len = sizeof(numCPU).csize_t - discard sysctl(addr(mib), 2, addr(numCPU), len, nil, 0) - if numCPU < 1: - mib[1] = HW_NCPU - discard sysctl(addr(mib), 2, addr(numCPU), len, nil, 0) - result = numCPU - elif defined(hpux): - result = mpctl(MPC_GETNUMSPUS, nil, nil) - elif defined(irix): - var SC_NPROC_ONLN {.importc: "_SC_NPROC_ONLN", header: "<unistd.h>".}: cint - result = sysconf(SC_NPROC_ONLN) - elif defined(genode): - result = runtimeEnv.affinitySpaceTotal().int - elif defined(haiku): - var sysinfo: SystemInfo - if getSystemInfo(addr sysinfo) == 0: - result = sysinfo.cpuCount.int - else: - result = sysconf(SC_NPROCESSORS_ONLN) - if result <= 0: result = 0 + countProcessorsImpl() diff --git a/lib/pure/concurrency/cpuload.nim b/lib/pure/concurrency/cpuload.nim index 137dd67ad..bfbf16721 100644 --- a/lib/pure/concurrency/cpuload.nim +++ b/lib/pure/concurrency/cpuload.nim @@ -13,11 +13,11 @@ ## Unstable API. when defined(windows): - import winlean, os, strutils, math + import std/[winlean, os, strutils, math] proc `-`(a, b: FILETIME): int64 = a.rdFileTime - b.rdFileTime elif defined(linux): - from cpuinfo import countProcessors + from std/cpuinfo import countProcessors when defined(nimPreviewSlimSystem): import std/syncio @@ -87,7 +87,7 @@ proc advice*(s: var ThreadPoolState): ThreadPoolAdvice = inc s.calls when not defined(testing) and isMainModule and not defined(nimdoc): - import random + import std/random proc busyLoop() = while true: diff --git a/lib/pure/concurrency/threadpool.nim b/lib/pure/concurrency/threadpool.nim index 5d9e7452b..06ed2fe54 100644 --- a/lib/pure/concurrency/threadpool.nim +++ b/lib/pure/concurrency/threadpool.nim @@ -7,24 +7,25 @@ # distribution, for details about the copyright. # +{.deprecated: "use the nimble packages `malebolgia`, `taskpools` or `weave` instead".} + ## Implements Nim's `parallel & spawn statements <manual_experimental.html#parallel-amp-spawn>`_. ## ## Unstable API. ## ## See also ## ======== -## * `threads module <threads.html>`_ for basic thread support -## * `channels module <channels_builtin.html>`_ for message passing support +## * `threads module <typedthreads.html>`_ for basic thread support ## * `locks module <locks.html>`_ for locks and condition variables ## * `asyncdispatch module <asyncdispatch.html>`_ for asynchronous IO when not compileOption("threads"): {.error: "Threadpool requires --threads:on option.".} -import cpuinfo, cpuload, locks, os +import std/[cpuinfo, cpuload, locks, os] when defined(nimPreviewSlimSystem): - import std/assertions + import std/[assertions, typedthreads, sysatomics] {.push stackTrace:off.} @@ -103,7 +104,7 @@ type idx: int FlowVarBase* = ref FlowVarBaseObj ## Untyped base class for `FlowVar[T] <#FlowVar>`_. - FlowVarBaseObj = object of RootObj + FlowVarBaseObj {.acyclic.} = object of RootObj ready, usesSemaphore, awaited: bool cv: Semaphore # for 'blockUntilAny' support ai: ptr AwaitInfo @@ -112,7 +113,7 @@ type # be RootRef here otherwise the wrong GC keeps track of it! owner: pointer # ptr Worker - FlowVarObj[T] = object of FlowVarBaseObj + FlowVarObj[T] {.acyclic.} = object of FlowVarBaseObj blob: T FlowVar*[T] {.compilerproc.} = ref FlowVarObj[T] ## A data flow variable. diff --git a/lib/pure/cookies.nim b/lib/pure/cookies.nim index 22704e434..f628aaf6b 100644 --- a/lib/pure/cookies.nim +++ b/lib/pure/cookies.nim @@ -9,7 +9,7 @@ ## This module implements helper procs for parsing Cookies. -import strtabs, times, options +import std/[strtabs, times, options] when defined(nimPreviewSlimSystem): import std/assertions diff --git a/lib/pure/coro.nim b/lib/pure/coro.nim index aaf442a83..24836e316 100644 --- a/lib/pure/coro.nim +++ b/lib/pure/coro.nim @@ -29,12 +29,14 @@ when not nimCoroutines and not defined(nimdoc): else: {.error: "Coroutines require -d:nimCoroutines".} -import os -import lists +import std/[os, lists] include system/timers +when defined(nimPreviewSlimSystem): + import std/assertions + const defaultStackSize = 512 * 1024 -const useOrcArc = defined(gcArc) or defined(gcOrc) +const useOrcArc = defined(gcArc) or defined(gcOrc) or defined(gcAtomicArc) when useOrcArc: proc nimGC_setStackBottom*(theStackBottom: pointer) = discard @@ -65,7 +67,7 @@ else: const coroBackend = CORO_BACKEND_UCONTEXT when coroBackend == CORO_BACKEND_FIBERS: - import windows/winlean + import std/winlean type Context = pointer @@ -221,7 +223,7 @@ proc switchTo(current, to: CoroutinePtr) = elif to.state == CORO_CREATED: # Coroutine is started. coroExecWithStack(runCurrentTask, to.stack.bottom) - #doAssert false + #raiseAssert "unreachable" else: {.error: "Invalid coroutine backend set.".} # Execution was just resumed. Restore frame information and set active stack. @@ -263,7 +265,7 @@ proc runCurrentTask() = current.state = CORO_FINISHED nimGC_setStackBottom(ctx.ncbottom) suspend(0) # Exit coroutine without returning from coroExecWithStack() - doAssert false + raiseAssert "unreachable" proc start*(c: proc(), stacksize: int = defaultStackSize): CoroutineRef {.discardable.} = ## Schedule coroutine for execution. It does not run immediately. @@ -278,8 +280,8 @@ proc start*(c: proc(), stacksize: int = defaultStackSize): CoroutineRef {.discar (proc(p: pointer) {.stdcall.} = runCurrentTask()), nil) else: coro = cast[CoroutinePtr](alloc0(sizeof(Coroutine) + stacksize)) - coro.stack.top = cast[pointer](cast[ByteAddress](coro) + sizeof(Coroutine)) - coro.stack.bottom = cast[pointer](cast[ByteAddress](coro.stack.top) + stacksize) + coro.stack.top = cast[pointer](cast[int](coro) + sizeof(Coroutine)) + coro.stack.bottom = cast[pointer](cast[int](coro.stack.top) + stacksize) when coroBackend == CORO_BACKEND_UCONTEXT: discard getcontext(coro.execContext) coro.execContext.uc_stack.ss_sp = coro.stack.top diff --git a/lib/pure/cstrutils.nim b/lib/pure/cstrutils.nim index 0eae00fba..c907e54d8 100644 --- a/lib/pure/cstrutils.nim +++ b/lib/pure/cstrutils.nim @@ -75,7 +75,7 @@ func cmpIgnoreStyle*(a, b: cstring): int {.rtl, extern: "csuCmpIgnoreStyle".} = ## for that. Returns: ## * 0 if `a == b` ## * < 0 if `a < b` - ## * > 0 if `a > b` + ## * \> 0 if `a > b` runnableExamples: assert cmpIgnoreStyle(cstring"hello", cstring"H_e_L_Lo") == 0 @@ -101,7 +101,7 @@ func cmpIgnoreCase*(a, b: cstring): int {.rtl, extern: "csuCmpIgnoreCase".} = ## Compares two strings in a case insensitive manner. Returns: ## * 0 if `a == b` ## * < 0 if `a < b` - ## * > 0 if `a > b` + ## * \> 0 if `a > b` runnableExamples: assert cmpIgnoreCase(cstring"hello", cstring"HeLLo") == 0 assert cmpIgnoreCase(cstring"echo", cstring"hello") < 0 diff --git a/lib/pure/db_common.nim b/lib/pure/db_common.nim deleted file mode 100644 index 852c8e1c3..000000000 --- a/lib/pure/db_common.nim +++ /dev/null @@ -1,100 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## Common datatypes and definitions for all `db_*.nim` ( -## `db_mysql <db_mysql.html>`_, `db_postgres <db_postgres.html>`_, -## and `db_sqlite <db_sqlite.html>`_) modules. - -type - DbError* = object of IOError ## exception that is raised if a database error occurs - - SqlQuery* = distinct string ## an SQL query string - - - DbEffect* = object of IOEffect ## effect that denotes a database operation - ReadDbEffect* = object of DbEffect ## effect that denotes a read operation - WriteDbEffect* = object of DbEffect ## effect that denotes a write operation - - DbTypeKind* = enum ## a superset of datatypes that might be supported. - dbUnknown, ## unknown datatype - dbSerial, ## datatype used for primary auto-increment keys - dbNull, ## datatype used for the NULL value - dbBit, ## bit datatype - dbBool, ## boolean datatype - dbBlob, ## blob datatype - dbFixedChar, ## string of fixed length - dbVarchar, ## string datatype - dbJson, ## JSON datatype - dbXml, ## XML datatype - dbInt, ## some integer type - dbUInt, ## some unsigned integer type - dbDecimal, ## decimal numbers (fixed-point number) - dbFloat, ## some floating point type - dbDate, ## a year-month-day description - dbTime, ## HH:MM:SS information - dbDatetime, ## year-month-day and HH:MM:SS information, - ## plus optional time or timezone information - dbTimestamp, ## Timestamp values are stored as the number of seconds - ## since the epoch ('1970-01-01 00:00:00' UTC). - dbTimeInterval, ## an interval [a,b] of times - dbEnum, ## some enum - dbSet, ## set of enum values - dbArray, ## an array of values - dbComposite, ## composite type (record, struct, etc) - dbUrl, ## a URL - dbUuid, ## a UUID - dbInet, ## an IP address - dbMacAddress, ## a MAC address - dbGeometry, ## some geometric type - dbPoint, ## Point on a plane (x,y) - dbLine, ## Infinite line ((x1,y1),(x2,y2)) - dbLseg, ## Finite line segment ((x1,y1),(x2,y2)) - dbBox, ## Rectangular box ((x1,y1),(x2,y2)) - dbPath, ## Closed or open path (similar to polygon) ((x1,y1),...) - dbPolygon, ## Polygon (similar to closed path) ((x1,y1),...) - dbCircle, ## Circle <(x,y),r> (center point and radius) - dbUser1, ## user definable datatype 1 (for unknown extensions) - dbUser2, ## user definable datatype 2 (for unknown extensions) - dbUser3, ## user definable datatype 3 (for unknown extensions) - dbUser4, ## user definable datatype 4 (for unknown extensions) - dbUser5 ## user definable datatype 5 (for unknown extensions) - - DbType* = object ## describes a database type - kind*: DbTypeKind ## the kind of the described type - notNull*: bool ## does the type contain NULL? - name*: string ## the name of the type - size*: Natural ## the size of the datatype; 0 if of variable size - maxReprLen*: Natural ## maximal length required for the representation - precision*, scale*: Natural ## precision and scale of the number - min*, max*: BiggestInt ## the minimum and maximum of allowed values - validValues*: seq[string] ## valid values of an enum or a set - - DbColumn* = object ## information about a database column - name*: string ## name of the column - tableName*: string ## name of the table the column belongs to (optional) - typ*: DbType ## type of the column - primaryKey*: bool ## is this a primary key? - foreignKey*: bool ## is this a foreign key? - DbColumns* = seq[DbColumn] - -template sql*(query: string): SqlQuery = - ## constructs a SqlQuery from the string `query`. This is supposed to be - ## used as a raw-string-literal modifier: - ## `sql"update user set counter = counter + 1"` - ## - ## If assertions are turned off, it does nothing. If assertions are turned - ## on, later versions will check the string for valid syntax. - SqlQuery(query) - -proc dbError*(msg: string) {.noreturn, noinline.} = - ## raises an DbError exception with message `msg`. - var e: ref DbError - new(e) - e.msg = msg - raise e diff --git a/lib/pure/distros.nim b/lib/pure/distros.nim index 797698d61..9e71d4ce0 100644 --- a/lib/pure/distros.nim +++ b/lib/pure/distros.nim @@ -9,21 +9,20 @@ ## This module implements the basics for Linux distribution ("distro") ## detection and the OS's native package manager. Its primary purpose is to -## produce output for Nimble packages, like:: +## produce output for Nimble packages, like: ## -## To complete the installation, run: +## To complete the installation, run: ## -## sudo apt-get install libblas-dev -## sudo apt-get install libvoodoo +## sudo apt-get install libblas-dev +## sudo apt-get install libvoodoo ## ## The above output could be the result of a code snippet like: ## -## .. code-block:: nim -## +## ```nim ## if detectOs(Ubuntu): ## foreignDep "lbiblas-dev" ## foreignDep "libvoodoo" -## +## ``` ## ## See `packaging <packaging.html>`_ for hints on distributing Nim using OS packages. @@ -31,7 +30,7 @@ from std/strutils import contains, toLowerAscii when not defined(nimscript): from std/osproc import execProcess - from std/os import existsEnv + from std/envvars import existsEnv type Distribution* {.pure.} = enum ## the list of known distributions @@ -128,6 +127,7 @@ type BSD FreeBSD + NetBSD OpenBSD DragonFlyBSD @@ -169,7 +169,7 @@ proc detectOsImpl(d: Distribution): bool = else: when defined(bsd): case d - of Distribution.FreeBSD, Distribution.OpenBSD: + of Distribution.FreeBSD, Distribution.NetBSD, Distribution.OpenBSD: result = $d in uname() else: result = false @@ -252,7 +252,7 @@ proc foreignDepInstallCmd*(foreignPackageName: string): (string, bool) = result = ("nix-env -i " & p, false) elif detectOs(Solaris) or detectOs(FreeBSD): result = ("pkg install " & p, true) - elif detectOs(OpenBSD): + elif detectOs(NetBSD) or detectOs(OpenBSD): result = ("pkg_add " & p, true) elif detectOs(PCLinuxOS): result = ("rpm -ivh " & p, true) diff --git a/lib/pure/dynlib.nim b/lib/pure/dynlib.nim index 48fd91b8f..a162fe37f 100644 --- a/lib/pure/dynlib.nim +++ b/lib/pure/dynlib.nim @@ -105,7 +105,7 @@ when defined(posix) and not defined(nintendoswitch): # as an emulation layer on top of native functions. # ========================================================================= # - import posix + import std/posix proc loadLib(path: string, globalSymbols = false): LibHandle = let flags = @@ -148,7 +148,7 @@ elif defined(genode): # template raiseErr(prc: string) = - raise newException(OSError, prc & " not implemented, compile with POSIX suport") + raise newException(OSError, prc & " not implemented, compile with POSIX support") proc dlclose(lib: LibHandle) = raiseErr(OSError, "dlclose") diff --git a/lib/pure/encodings.nim b/lib/pure/encodings.nim index 26b6e1f22..bbadca655 100644 --- a/lib/pure/encodings.nim +++ b/lib/pure/encodings.nim @@ -39,7 +39,7 @@ runnableExamples: assert fromGB2312.convert(second) == "有白头如新，倾盖如故" -import os +import std/os when defined(nimPreviewSlimSystem): import std/assertions @@ -59,7 +59,7 @@ type ## for encoding errors. when defined(windows): - import parseutils, strutils + import std/[parseutils, strutils] proc eqEncodingNames(a, b: string): bool = var i = 0 var j = 0 @@ -339,10 +339,10 @@ proc getCurrentEncoding*(uiApp = false): string = proc open*(destEncoding = "UTF-8", srcEncoding = "CP1252"): EncodingConverter = ## Opens a converter that can convert from `srcEncoding` to `destEncoding`. - ## Raises `IOError` if it cannot fulfill the request. + ## Raises `EncodingError` if it cannot fulfill the request. when not defined(windows): result = iconvOpen(destEncoding, srcEncoding) - if result == nil: + if result == cast[EncodingConverter](-1): raise newException(EncodingError, "cannot create encoding converter from " & srcEncoding & " to " & destEncoding) diff --git a/lib/pure/endians.nim b/lib/pure/endians.nim index a0dc97c1d..4c1d45ae5 100644 --- a/lib/pure/endians.nim +++ b/lib/pure/endians.nim @@ -10,7 +10,7 @@ ## This module contains helpers that deal with different byte orders ## (`endian`:idx:). ## -## Endianess is the order of bytes of a value in memory. Big-endian means that +## Endianness is the order of bytes of a value in memory. Big-endian means that ## the most significant byte is stored at the smallest memory address, ## while little endian means that the least-significant byte is stored ## at the smallest address. See also https://en.wikipedia.org/wiki/Endianness. diff --git a/lib/pure/fenv.nim b/lib/pure/fenv.nim index ddb6a1a0c..1d96fd6be 100644 --- a/lib/pure/fenv.nim +++ b/lib/pure/fenv.nim @@ -12,7 +12,7 @@ ## The types, vars and procs are bindings for the C standard library ## [<fenv.h>](https://en.cppreference.com/w/c/numeric/fenv) header. -when defined(posix) and not defined(genode): +when defined(posix) and not defined(genode) and not defined(macosx): {.passl: "-lm".} var diff --git a/lib/pure/hashes.nim b/lib/pure/hashes.nim index e88210757..1038d55a1 100644 --- a/lib/pure/hashes.nim +++ b/lib/pure/hashes.nim @@ -62,7 +62,7 @@ runnableExamples: ## ======== ## * `md5 module <md5.html>`_ for the MD5 checksum algorithm ## * `base64 module <base64.html>`_ for a Base64 encoder and decoder -## * `std/sha1 module <sha1.html>`_ for the SHA-1 checksum algorithm +## * `sha1 module <sha1.html>`_ for the SHA-1 checksum algorithm ## * `tables module <tables.html>`_ for hash tables import std/private/since @@ -190,7 +190,7 @@ proc hashData*(data: pointer, size: int): Hash = var h: Hash = 0 when defined(js): var p: cstring - asm """`p` = `Data`""" + {.emit: """`p` = `Data`;""".} else: var p = cast[cstring](data) var i = 0 @@ -217,7 +217,7 @@ else: when defined(js): var objectID = 0 proc getObjectId(x: pointer): int = - asm """ + {.emit: """ if (typeof `x` == "object") { if ("_NimID" in `x`) `result` = `x`["_NimID"]; @@ -226,7 +226,7 @@ when defined(js): `x`["_NimID"] = `result`; } } - """ + """.} proc hash*(x: pointer): Hash {.inline.} = ## Efficient `hash` overload. @@ -247,7 +247,7 @@ proc hash*[T](x: ptr[T]): Hash {.inline.} = when defined(nimPreviewHashRef) or defined(nimdoc): proc hash*[T](x: ref[T]): Hash {.inline.} = ## Efficient `hash` overload. - ## + ## ## .. important:: Use `-d:nimPreviewHashRef` to ## enable hashing `ref`s. It is expected that this behavior ## becomes the new default in upcoming versions. @@ -319,16 +319,24 @@ proc murmurHash(x: openArray[byte]): Hash = h1: uint32 i = 0 + + template impl = + var j = stepSize + while j > 0: + dec j + k1 = (k1 shl 8) or (ord(x[i+j])).uint32 + # body while i < n * stepSize: var k1: uint32 - when defined(js) or defined(sparc) or defined(sparc64): - var j = stepSize - while j > 0: - dec j - k1 = (k1 shl 8) or (ord(x[i+j])).uint32 + + when nimvm: + impl() else: - k1 = cast[ptr uint32](unsafeAddr x[i])[] + when declared(copyMem): + copyMem(addr k1, addr x[i], 4) + else: + impl() inc i, stepSize k1 = imul(k1, c1) @@ -360,16 +368,168 @@ proc murmurHash(x: openArray[byte]): Hash = return cast[Hash](h1) proc hashVmImpl(x: cstring, sPos, ePos: int): Hash = - doAssert false, "implementation override in compiler/vmops.nim" + raiseAssert "implementation override in compiler/vmops.nim" proc hashVmImpl(x: string, sPos, ePos: int): Hash = - doAssert false, "implementation override in compiler/vmops.nim" + raiseAssert "implementation override in compiler/vmops.nim" proc hashVmImplChar(x: openArray[char], sPos, ePos: int): Hash = - doAssert false, "implementation override in compiler/vmops.nim" + raiseAssert "implementation override in compiler/vmops.nim" proc hashVmImplByte(x: openArray[byte], sPos, ePos: int): Hash = - doAssert false, "implementation override in compiler/vmops.nim" + raiseAssert "implementation override in compiler/vmops.nim" + +const k0 = 0xc3a5c85c97cb3127u64 # Primes on (2^63, 2^64) for various uses +const k1 = 0xb492b66fbe98f273u64 +const k2 = 0x9ae16a3b2f90404fu64 + +proc load4e(s: openArray[byte], o=0): uint32 {.inline.} = + uint32(s[o + 3]) shl 24 or uint32(s[o + 2]) shl 16 or + uint32(s[o + 1]) shl 8 or uint32(s[o + 0]) + +proc load8e(s: openArray[byte], o=0): uint64 {.inline.} = + uint64(s[o + 7]) shl 56 or uint64(s[o + 6]) shl 48 or + uint64(s[o + 5]) shl 40 or uint64(s[o + 4]) shl 32 or + uint64(s[o + 3]) shl 24 or uint64(s[o + 2]) shl 16 or + uint64(s[o + 1]) shl 8 or uint64(s[o + 0]) + +proc load4(s: openArray[byte], o=0): uint32 {.inline.} = + when nimvm: result = load4e(s, o) + else: + when declared copyMem: copyMem result.addr, s[o].addr, result.sizeof + else: result = load4e(s, o) + +proc load8(s: openArray[byte], o=0): uint64 {.inline.} = + when nimvm: result = load8e(s, o) + else: + when declared copyMem: copyMem result.addr, s[o].addr, result.sizeof + else: result = load8e(s, o) + +proc lenU(s: openArray[byte]): uint64 {.inline.} = s.len.uint64 + +proc shiftMix(v: uint64): uint64 {.inline.} = v xor (v shr 47) + +proc rotR(v: uint64; bits: cint): uint64 {.inline.} = + (v shr bits) or (v shl (64 - bits)) + +proc len16(u: uint64; v: uint64; mul: uint64): uint64 {.inline.} = + var a = (u xor v)*mul + a = a xor (a shr 47) + var b = (v xor a)*mul + b = b xor (b shr 47) + b*mul + +proc len0_16(s: openArray[byte]): uint64 {.inline.} = + if s.len >= 8: + let mul = k2 + 2*s.lenU + let a = load8(s) + k2 + let b = load8(s, s.len - 8) + let c = rotR(b, 37)*mul + a + let d = (rotR(a, 25) + b)*mul + len16 c, d, mul + elif s.len >= 4: + let mul = k2 + 2*s.lenU + let a = load4(s).uint64 + len16 s.lenU + (a shl 3), load4(s, s.len - 4), mul + elif s.len > 0: + let a = uint32(s[0]) + let b = uint32(s[s.len shr 1]) + let c = uint32(s[s.len - 1]) + let y = a + (b shl 8) + let z = s.lenU + (c shl 2) + shiftMix(y*k2 xor z*k0)*k2 + else: k2 # s.len == 0 + +proc len17_32(s: openArray[byte]): uint64 {.inline.} = + let mul = k2 + 2*s.lenU + let a = load8(s)*k1 + let b = load8(s, 8) + let c = load8(s, s.len - 8)*mul + let d = load8(s, s.len - 16)*k2 + len16 rotR(a + b, 43) + rotR(c, 30) + d, a + rotR(b + k2, 18) + c, mul + +proc len33_64(s: openArray[byte]): uint64 {.inline.} = + let mul = k2 + 2*s.lenU + let a = load8(s)*k2 + let b = load8(s, 8) + let c = load8(s, s.len - 8)*mul + let d = load8(s, s.len - 16)*k2 + let y = rotR(a + b, 43) + rotR(c, 30) + d + let z = len16(y, a + rotR(b + k2, 18) + c, mul) + let e = load8(s, 16)*mul + let f = load8(s, 24) + let g = (y + load8(s, s.len - 32))*mul + let h = (z + load8(s, s.len - 24))*mul + len16 rotR(e + f, 43) + rotR(g, 30) + h, e + rotR(f + a, 18) + g, mul + +type Pair = tuple[first, second: uint64] + +proc weakLen32withSeeds2(w, x, y, z, a, b: uint64): Pair {.inline.} = + var a = a + w + var b = rotR(b + a + z, 21) + let c = a + a += x + a += y + b += rotR(a, 44) + result[0] = a + z + result[1] = b + c + +proc weakLen32withSeeds(s: openArray[byte]; o: int; a,b: uint64): Pair {.inline.} = + weakLen32withSeeds2 load8(s, o ), load8(s, o + 8), + load8(s, o + 16), load8(s, o + 24), a, b + +proc hashFarm(s: openArray[byte]): uint64 {.inline.} = + if s.len <= 16: return len0_16(s) + if s.len <= 32: return len17_32(s) + if s.len <= 64: return len33_64(s) + const seed = 81u64 # not const to use input `h` + var + o = 0 # s[] ptr arith -> variable origin variable `o` + x = seed + y = seed*k1 + 113 + z = shiftMix(y*k2 + 113)*k2 + v, w: Pair + x = x*k2 + load8(s) + let eos = ((s.len - 1) div 64)*64 + let last64 = eos + ((s.len - 1) and 63) - 63 + while true: + x = rotR(x + y + v[0] + load8(s, o+8), 37)*k1 + y = rotR(y + v[1] + load8(s, o+48), 42)*k1 + x = x xor w[1] + y += v[0] + load8(s, o+40) + z = rotR(z + w[0], 33)*k1 + v = weakLen32withSeeds(s, o+0 , v[1]*k1, x + w[0]) + w = weakLen32withSeeds(s, o+32, z + w[1], y + load8(s, o+16)) + swap z, x + inc o, 64 + if o == eos: break + let mul = k1 + ((z and 0xff) shl 1) + o = last64 + w[0] += (s.lenU - 1) and 63 + v[0] += w[0] + w[0] += v[0] + x = rotR(x + y + v[0] + load8(s, o+8), 37)*mul + y = rotR(y + v[1] + load8(s, o+48), 42)*mul + x = x xor w[1]*9 + y += v[0]*9 + load8(s, o+40) + z = rotR(z + w[0], 33)*mul + v = weakLen32withSeeds(s, o+0 , v[1]*mul, x + w[0]) + w = weakLen32withSeeds(s, o+32, z + w[1], y + load8(s, o+16)) + swap z, x + len16 len16(v[0],w[0],mul) + shiftMix(y)*k0 + z, len16(v[1],w[1],mul) + x, mul + +template jsNoInt64: untyped = + when defined js: + when compiles(compileOption("jsbigint64")): + when not compileOption("jsbigint64"): true + else: false + else: false + else: false +const sHash2 = (when defined(nimStringHash2) or jsNoInt64(): true else: false) + +template maybeFailJS_Number = + when jsNoInt64() and not defined(nimStringHash2): + {.error: "Must use `-d:nimStringHash2` when using `--jsbigint64:off`".} proc hash*(x: string): Hash = ## Efficient hashing of strings. @@ -379,16 +539,13 @@ proc hash*(x: string): Hash = ## * `hashIgnoreCase <#hashIgnoreCase,string>`_ runnableExamples: doAssert hash("abracadabra") != hash("AbracadabrA") - - when not defined(nimToOpenArrayCString): - result = 0 - for c in x: - result = result !& ord(c) - result = !$result + maybeFailJS_Number() + when not sHash2: + result = cast[Hash](hashFarm(toOpenArrayByte(x, 0, x.high))) else: - when nimvm: - result = hashVmImpl(x, 0, high(x)) - else: + #when nimvm: + # result = hashVmImpl(x, 0, high(x)) + when true: result = murmurHash(toOpenArrayByte(x, 0, high(x))) proc hash*(x: cstring): Hash = @@ -398,22 +555,22 @@ proc hash*(x: cstring): Hash = doAssert hash(cstring"AbracadabrA") == hash("AbracadabrA") doAssert hash(cstring"abracadabra") != hash(cstring"AbracadabrA") - when not defined(nimToOpenArrayCString): - result = 0 - var i = 0 - while x[i] != '\0': - result = result !& ord(x[i]) - inc i - result = !$result - else: - when nimvm: - hashVmImpl(x, 0, high(x)) + maybeFailJS_Number() + when not sHash2: + when defined js: + let xx = $x + result = cast[Hash](hashFarm(toOpenArrayByte(xx, 0, xx.high))) else: - when not defined(js) and defined(nimToOpenArrayCString): - murmurHash(toOpenArrayByte(x, 0, x.high)) + result = cast[Hash](hashFarm(toOpenArrayByte(x, 0, x.high))) + else: + #when nimvm: + # result = hashVmImpl(x, 0, high(x)) + when true: + when not defined(js): + result = murmurHash(toOpenArrayByte(x, 0, x.high)) else: let xx = $x - murmurHash(toOpenArrayByte(xx, 0, high(xx))) + result = murmurHash(toOpenArrayByte(xx, 0, high(xx))) proc hash*(sBuf: string, sPos, ePos: int): Hash = ## Efficient hashing of a string buffer, from starting @@ -424,11 +581,9 @@ proc hash*(sBuf: string, sPos, ePos: int): Hash = var a = "abracadabra" doAssert hash(a, 0, 3) == hash(a, 7, 10) - when not defined(nimToOpenArrayCString): - result = 0 - for i in sPos..ePos: - result = result !& ord(sBuf[i]) - result = !$result + maybeFailJS_Number() + when not sHash2: + result = cast[Hash](hashFarm(toOpenArrayByte(sBuf, sPos, ePos))) else: murmurHash(toOpenArrayByte(sBuf, sPos, ePos)) @@ -521,7 +676,7 @@ proc hashIgnoreCase*(sBuf: string, sPos, ePos: int): Hash = h = h !& ord(c) result = !$h -proc hash*[T: tuple | object | proc](x: T): Hash = +proc hash*[T: tuple | object | proc | iterator {.closure.}](x: T): Hash = ## Efficient `hash` overload. runnableExamples: # for `tuple|object`, `hash` must be defined for each component of `x`. @@ -554,8 +709,9 @@ proc hash*[T: tuple | object | proc](x: T): Hash = when T is "closure": result = hash((rawProc(x), rawEnv(x))) elif T is (proc): - result = hash(pointer(x)) + result = hash(cast[pointer](x)) else: + result = 0 for f in fields(x): result = result !& hash(f) result = !$result @@ -564,13 +720,20 @@ proc hash*[A](x: openArray[A]): Hash = ## Efficient hashing of arrays and sequences. ## There must be a `hash` proc defined for the element type `A`. when A is byte: - result = murmurHash(x) + when not sHash2: + result = cast[Hash](hashFarm(x)) + else: + result = murmurHash(x) elif A is char: - when nimvm: - result = hashVmImplChar(x, 0, x.high) + when not sHash2: + result = cast[Hash](hashFarm(toOpenArrayByte(x, 0, x.high))) else: - result = murmurHash(toOpenArrayByte(x, 0, x.high)) + #when nimvm: + # result = hashVmImplChar(x, 0, x.high) + when true: + result = murmurHash(toOpenArrayByte(x, 0, x.high)) else: + result = 0 for a in x: result = result !& hash(a) result = !$result @@ -584,17 +747,24 @@ proc hash*[A](aBuf: openArray[A], sPos, ePos: int): Hash = runnableExamples: let a = [1, 2, 5, 1, 2, 6] doAssert hash(a, 0, 1) == hash(a, 3, 4) - when A is byte: - when nimvm: - result = hashVmImplByte(aBuf, sPos, ePos) + maybeFailJS_Number() + when not sHash2: + result = cast[Hash](hashFarm(toOpenArray(aBuf, sPos, ePos))) else: - result = murmurHash(toOpenArray(aBuf, sPos, ePos)) + #when nimvm: + # result = hashVmImplByte(aBuf, sPos, ePos) + when true: + result = murmurHash(toOpenArray(aBuf, sPos, ePos)) elif A is char: - when nimvm: - result = hashVmImplChar(aBuf, sPos, ePos) + maybeFailJS_Number() + when not sHash2: + result = cast[Hash](hashFarm(toOpenArrayByte(aBuf, sPos, ePos))) else: - result = murmurHash(toOpenArrayByte(aBuf, sPos, ePos)) + #when nimvm: + # result = hashVmImplChar(aBuf, sPos, ePos) + when true: + result = murmurHash(toOpenArrayByte(aBuf, sPos, ePos)) else: for i in sPos .. ePos: result = result !& hash(aBuf[i]) @@ -603,6 +773,7 @@ proc hash*[A](aBuf: openArray[A], sPos, ePos: int): Hash = proc hash*[A](x: set[A]): Hash = ## Efficient hashing of sets. ## There must be a `hash` proc defined for the element type `A`. + result = 0 for it in items(x): result = result !& hash(it) result = !$result diff --git a/lib/pure/htmlgen.nim b/lib/pure/htmlgen.nim index 89eb24bb9..fafa72463 100644 --- a/lib/pure/htmlgen.nim +++ b/lib/pure/htmlgen.nim @@ -8,7 +8,7 @@ # ## Do yourself a favor and import the module -## as `from htmlgen import nil` and then fully qualify the macros. +## as `from std/htmlgen import nil` and then fully qualify the macros. ## ## *Note*: The Karax project (`nimble install karax`) has a better ## way to achieve the same, see https://github.com/pragmagic/karax/blob/master/tests/nativehtmlgen.nim @@ -30,17 +30,18 @@ ## Examples ## ======== ## -## .. code-block:: Nim +## ```Nim ## var nim = "Nim" ## echo h1(a(href="https://nim-lang.org", nim)) +## ``` ## -## Writes the string:: +## Writes the string: ## -## <h1><a href="https://nim-lang.org">Nim</a></h1> +## <h1><a href="https://nim-lang.org">Nim</a></h1> ## import - macros, strutils + std/[macros, strutils] const coreAttr* = " accesskey class contenteditable dir hidden id lang " & diff --git a/lib/pure/htmlparser.nim b/lib/pure/htmlparser.nim index 6b1300f11..62919546f 100644 --- a/lib/pure/htmlparser.nim +++ b/lib/pure/htmlparser.nim @@ -12,10 +12,9 @@ ## ## It can be used to parse a wild HTML document and output it as valid XHTML ## document (well, if you are lucky): -## -## .. code-block:: Nim -## +## ```Nim ## echo loadHtml("mydirty.html") +## ``` ## ## Every tag in the resulting tree is in lower case. ## @@ -29,9 +28,7 @@ ## and write back the modified version. In this case we look for hyperlinks ## ending with the extension `.rst` and convert them to `.html`. ## -## .. code-block:: Nim -## :test: -## +## ```Nim test ## import std/htmlparser ## import std/xmltree # To use '$' for XmlNode ## import std/strtabs # To access XmlAttributes @@ -48,8 +45,11 @@ ## a.attrs["href"] = dir / filename & ".html" ## ## writeFile("output.html", $html) +## ``` + +{.deprecated: "use `nimble install htmlparser` and import `pkg/htmlparser` instead".} -import strutils, streams, parsexml, xmltree, unicode, strtabs +import std/[strutils, streams, parsexml, xmltree, unicode, strtabs] when defined(nimPreviewSlimSystem): import std/syncio @@ -394,10 +394,10 @@ proc entityToRune*(entity: string): Rune = case entity[1] of '0'..'9': try: runeValue = parseInt(entity[1..^1]) - except: discard + except ValueError: discard of 'x', 'X': # not case sensitive here try: runeValue = parseHexInt(entity[2..^1]) - except: discard + except ValueError: discard else: discard # other entities are not defined with prefix `#` if runeValue notin 0..0x10FFFF: runeValue = 0 # only return legal values return Rune(runeValue) @@ -2064,7 +2064,7 @@ proc loadHtml*(path: string): XmlNode = result = loadHtml(path, errors) when not defined(testing) and isMainModule: - import os + import std/os var errors: seq[string] = @[] var x = loadHtml(paramStr(1), errors) diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index 5c7f538b5..08ea99627 100644 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -10,34 +10,38 @@ ## This module implements a simple HTTP client that can be used to retrieve ## webpages and other data. ## +## .. warning:: Validate untrusted inputs: URI parsers and getters are not detecting malicious URIs. +## ## Retrieving a website ## ==================== ## ## This example uses HTTP GET to retrieve ## `http://google.com`: ## -## .. code-block:: Nim +## ```Nim ## import std/httpclient ## var client = newHttpClient() ## try: ## echo client.getContent("http://google.com") ## finally: ## client.close() +## ``` ## ## The same action can also be performed asynchronously, simply use the ## `AsyncHttpClient`: ## -## .. code-block:: Nim +## ```Nim ## import std/[asyncdispatch, httpclient] ## ## proc asyncProc(): Future[string] {.async.} = ## var client = newAsyncHttpClient() ## try: -## return await client.getContent("http://example.com") +## return await client.getContent("http://google.com") ## finally: ## client.close() ## ## echo waitFor asyncProc() +## ``` ## ## The functionality implemented by `HttpClient` and `AsyncHttpClient` ## is the same, so you can use whichever one suits you best in the examples @@ -46,6 +50,10 @@ ## **Note:** You need to run asynchronous examples in an async proc ## otherwise you will get an `Undeclared identifier: 'await'` error. ## +## **Note:** An asynchronous client instance can only deal with one +## request at a time. To send multiple requests in parallel, use +## multiple client instances. +## ## Using HTTP POST ## =============== ## @@ -53,7 +61,7 @@ ## uses `multipart/form-data` as the `Content-Type` to send the HTML to be ## validated to the server. ## -## .. code-block:: Nim +## ```Nim ## var client = newHttpClient() ## var data = newMultipartData() ## data["output"] = "soap12" @@ -63,13 +71,14 @@ ## echo client.postContent("http://validator.w3.org/check", multipart=data) ## finally: ## client.close() +## ``` ## ## To stream files from disk when performing the request, use `addFiles`. ## ## **Note:** This will allocate a new `Mimetypes` database every time you call ## it, you can pass your own via the `mimeDb` parameter to avoid this. ## -## .. code-block:: Nim +## ```Nim ## let mimes = newMimetypes() ## var client = newHttpClient() ## var data = newMultipartData() @@ -78,12 +87,13 @@ ## echo client.postContent("http://validator.w3.org/check", multipart=data) ## finally: ## client.close() +## ``` ## ## You can also make post requests with custom headers. ## This example sets `Content-Type` to `application/json` ## and uses a json object for the body ## -## .. code-block:: Nim +## ```Nim ## import std/[httpclient, json] ## ## let client = newHttpClient() @@ -96,6 +106,7 @@ ## echo response.status ## finally: ## client.close() +## ``` ## ## Progress reporting ## ================== @@ -104,27 +115,29 @@ ## This callback will be executed every second with information about the ## progress of the HTTP request. ## -## .. code-block:: Nim -## import std/[asyncdispatch, httpclient] +## ```Nim +## import std/[asyncdispatch, httpclient] ## -## proc onProgressChanged(total, progress, speed: BiggestInt) {.async.} = -## echo("Downloaded ", progress, " of ", total) -## echo("Current rate: ", speed div 1000, "kb/s") +## proc onProgressChanged(total, progress, speed: BiggestInt) {.async.} = +## echo("Downloaded ", progress, " of ", total) +## echo("Current rate: ", speed div 1000, "kb/s") ## -## proc asyncProc() {.async.} = -## var client = newAsyncHttpClient() -## client.onProgressChanged = onProgressChanged -## try: -## discard await client.getContent("http://speedtest-ams2.digitalocean.com/100mb.test") -## finally: -## client.close() +## proc asyncProc() {.async.} = +## var client = newAsyncHttpClient() +## client.onProgressChanged = onProgressChanged +## try: +## discard await client.getContent("http://speedtest-ams2.digitalocean.com/100mb.test") +## finally: +## client.close() ## -## waitFor asyncProc() +## waitFor asyncProc() +## ``` ## ## If you would like to remove the callback simply set it to `nil`. ## -## .. code-block:: Nim +## ```Nim ## client.onProgressChanged = nil +## ``` ## ## .. warning:: The `total` reported by httpclient may be 0 in some cases. ## @@ -146,9 +159,10 @@ ## ## Example of setting SSL verification parameters in a new client: ## -## .. code-block:: Nim -## import httpclient -## var client = newHttpClient(sslContext=newContext(verifyMode=CVerifyPeer)) +## ```Nim +## import httpclient +## var client = newHttpClient(sslContext=newContext(verifyMode=CVerifyPeer)) +## ``` ## ## There are three options for verify mode: ## @@ -177,10 +191,11 @@ ## ## Here is how to set a timeout when creating an `HttpClient` instance: ## -## .. code-block:: Nim -## import std/httpclient +## ```Nim +## import std/httpclient ## -## let client = newHttpClient(timeout = 42) +## let client = newHttpClient(timeout = 42) +## ``` ## ## Proxy ## ===== @@ -191,28 +206,39 @@ ## ## Some examples on how to configure a Proxy for `HttpClient`: ## -## .. code-block:: Nim -## import std/httpclient +## ```Nim +## import std/httpclient +## +## let myProxy = newProxy("http://myproxy.network") +## let client = newHttpClient(proxy = myProxy) +## ``` +## +## Use proxies with basic authentication: ## -## let myProxy = newProxy("http://myproxy.network") -## let client = newHttpClient(proxy = myProxy) +## ```Nim +## import std/httpclient +## +## let myProxy = newProxy("http://myproxy.network", auth="user:password") +## let client = newHttpClient(proxy = myProxy) +## ``` ## ## Get Proxy URL from environment variables: ## -## .. code-block:: Nim -## import std/httpclient +## ```Nim +## import std/httpclient ## -## var url = "" -## try: -## if existsEnv("http_proxy"): -## url = getEnv("http_proxy") -## elif existsEnv("https_proxy"): -## url = getEnv("https_proxy") -## except ValueError: -## echo "Unable to parse proxy from environment variables." +## var url = "" +## try: +## if existsEnv("http_proxy"): +## url = getEnv("http_proxy") +## elif existsEnv("https_proxy"): +## url = getEnv("https_proxy") +## except ValueError: +## echo "Unable to parse proxy from environment variables." ## -## let myProxy = newProxy(url = url) -## let client = newHttpClient(proxy = myProxy) +## let myProxy = newProxy(url = url) +## let client = newHttpClient(proxy = myProxy) +## ``` ## ## Redirects ## ========= @@ -223,10 +249,11 @@ ## ## Here you can see an example about how to set the `maxRedirects` of `HttpClient`: ## -## .. code-block:: Nim -## import std/httpclient +## ```Nim +## import std/httpclient ## -## let client = newHttpClient(maxRedirects = 0) +## let client = newHttpClient(maxRedirects = 0) +## ``` ## import std/private/since @@ -278,7 +305,7 @@ proc contentLength*(response: Response | AsyncResponse): int = ## ## A `ValueError` exception will be raised if the value is not an integer. ## If the Content-Length header is not set in the response, ContentLength is set to the value -1. - var contentLengthHeader = response.headers.getOrDefault("Content-Length", @["-1"]) + var contentLengthHeader = response.headers.getOrDefault("Content-Length", HttpHeaderValues(@["-1"])) result = contentLengthHeader.parseInt() proc lastModified*(response: Response | AsyncResponse): DateTime = @@ -332,7 +359,7 @@ type ## and `postContent` proc, ## when the server returns an error -const defUserAgent* = "Nim httpclient/" & NimVersion +const defUserAgent* = "Nim-httpclient/" & NimVersion proc httpError(msg: string) = var e: ref ProtocolError @@ -415,8 +442,9 @@ proc add*(p: MultipartData, xs: MultipartEntries): MultipartData ## Add a list of multipart entries to the multipart data `p`. All values are ## added without a filename and without a content type. ## - ## .. code-block:: Nim + ## ```Nim ## data.add({"action": "login", "format": "json"}) + ## ``` for name, content in xs.items: p.add(name, content) result = p @@ -425,8 +453,9 @@ proc newMultipartData*(xs: MultipartEntries): MultipartData = ## Create a new multipart data object and fill it with the entries `xs` ## directly. ## - ## .. code-block:: Nim + ## ```Nim ## var data = newMultipartData({"action": "login", "format": "json"}) + ## ``` result = MultipartData() for entry in xs: result.add(entry.name, entry.content) @@ -441,8 +470,9 @@ proc addFiles*(p: MultipartData, xs: openArray[tuple[name, file: string]], ## Raises an `IOError` if the file cannot be opened or reading fails. To ## manually specify file content, filename and MIME type, use `[]=` instead. ## - ## .. code-block:: Nim + ## ```Nim ## data.addFiles({"uploaded_file": "public/test.html"}) + ## ``` for name, file in xs.items: var contentType: string let (_, fName, ext) = splitFile(file) @@ -456,8 +486,9 @@ proc `[]=`*(p: MultipartData, name, content: string) {.inline.} = ## Add a multipart entry to the multipart data `p`. The value is added ## without a filename and without a content type. ## - ## .. code-block:: Nim + ## ```Nim ## data["username"] = "NimUser" + ## ``` p.add(name, content) proc `[]=`*(p: MultipartData, name: string, @@ -465,9 +496,10 @@ proc `[]=`*(p: MultipartData, name: string, ## Add a file to the multipart data `p`, specifying filename, contentType ## and content manually. ## - ## .. code-block:: Nim + ## ```Nim ## data["uploaded_file"] = ("test.html", "text/html", ## "<html><head></head><body><p>test</p></body></html>") + ## ``` p.add(name, file.content, file.name, file.contentType, useStream = false) proc getBoundary(p: MultipartData): string = @@ -674,15 +706,15 @@ proc close*(client: HttpClient | AsyncHttpClient) = client.connected = false proc getSocket*(client: HttpClient): Socket {.inline.} = - ## Get network socket, useful if you want to find out more details about the connection + ## Get network socket, useful if you want to find out more details about the connection. ## - ## this example shows info about local and remote endpoints + ## This example shows info about local and remote endpoints: ## - ## .. code-block:: Nim + ## ```Nim ## if client.connected: ## echo client.getSocket.getLocalAddr ## echo client.getSocket.getPeerAddr - ## + ## ``` return client.socket proc getSocket*(client: AsyncHttpClient): AsyncSocket {.inline.} = @@ -824,6 +856,7 @@ proc parseResponse(client: HttpClient | AsyncHttpClient, var parsedStatus = false var linei = 0 var fullyRead = false + var lastHeaderName = "" var line = "" result.headers = newHttpHeaders() while true: @@ -858,16 +891,29 @@ proc parseResponse(client: HttpClient | AsyncHttpClient, parsedStatus = true else: # Parse headers - var name = "" - var le = parseUntil(line, name, ':', linei) - if le <= 0: httpError("invalid headers") - inc(linei, le) - if line[linei] != ':': httpError("invalid headers") - inc(linei) # Skip : - - result.headers.add(name, line[linei .. ^1].strip()) - if result.headers.len > headerLimit: - httpError("too many headers") + # There's at least one char because empty lines are handled above (with client.close) + if line[0] in {' ', '\t'}: + # Check if it's a multiline header value, if so, append to the header we're currently parsing + # This works because a line with a header must start with the header name without any leading space + # See https://datatracker.ietf.org/doc/html/rfc7230, section 3.2 and 3.2.4 + # Multiline headers are deprecated in the spec, but it's better to parse them than crash + if lastHeaderName == "": + # Some extra unparsable lines in the HTTP output - we ignore them + discard + else: + result.headers.table[result.headers.toCaseInsensitive(lastHeaderName)][^1].add "\n" & line + else: + var name = "" + var le = parseUntil(line, name, ':', linei) + if le <= 0: httpError("Invalid headers - received empty header name") + if line.len == le: httpError("Invalid headers - no colon after header name") + inc(linei, le) # Skip the parsed header name + inc(linei) # Skip : + # If we want to be HTTP spec compliant later, error on linei == line.len (for empty header value) + lastHeaderName = name # Remember the header name for the possible multi-line header + result.headers.add(name, line[linei .. ^1].strip()) + if result.headers.len > headerLimit: + httpError("too many headers") if not fullyRead: httpError("Connection was closed before full request has been made") @@ -1186,7 +1232,7 @@ proc responseContent(resp: Response | AsyncResponse): Future[string] {.multisync ## A `HttpRequestError` will be raised if the server responds with a ## client error (status code 4xx) or a server error (status code 5xx). if resp.code.is4xx or resp.code.is5xx: - raise newException(HttpRequestError, resp.status) + raise newException(HttpRequestError, resp.status.move) else: return await resp.bodyStream.readAll() diff --git a/lib/pure/httpcore.nim b/lib/pure/httpcore.nim index 7e995ac46..5ccab379c 100644 --- a/lib/pure/httpcore.nim +++ b/lib/pure/httpcore.nim @@ -12,7 +12,7 @@ ## ## Unstable API. import std/private/since -import tables, strutils, parseutils +import std/[tables, strutils, parseutils] type HttpHeaders* = ref object @@ -126,23 +126,20 @@ func toTitleCase(s: string): string = result[i] = if upper: toUpperAscii(s[i]) else: toLowerAscii(s[i]) upper = s[i] == '-' -func toCaseInsensitive(headers: HttpHeaders, s: string): string {.inline.} = +func toCaseInsensitive*(headers: HttpHeaders, s: string): string {.inline.} = + ## For internal usage only. Do not use. return if headers.isTitleCase: toTitleCase(s) else: toLowerAscii(s) func newHttpHeaders*(titleCase=false): HttpHeaders = ## Returns a new `HttpHeaders` object. if `titleCase` is set to true, ## headers are passed to the server in title case (e.g. "Content-Length") - new result - result.table = newTable[string, seq[string]]() - result.isTitleCase = titleCase + result = HttpHeaders(table: newTable[string, seq[string]](), isTitleCase: titleCase) func newHttpHeaders*(keyValuePairs: openArray[tuple[key: string, val: string]], titleCase=false): HttpHeaders = ## Returns a new `HttpHeaders` object from an array. if `titleCase` is set to true, ## headers are passed to the server in title case (e.g. "Content-Length") - new result - result.table = newTable[string, seq[string]]() - result.isTitleCase = titleCase + result = HttpHeaders(table: newTable[string, seq[string]](), isTitleCase: titleCase) for pair in keyValuePairs: let key = result.toCaseInsensitive(pair.key) @@ -167,7 +164,8 @@ func `[]`*(headers: HttpHeaders, key: string): HttpHeaderValues = ## To access multiple values of a key, use the overloaded `[]` below or ## to get all of them access the `table` field directly. {.cast(noSideEffect).}: - return headers.table[headers.toCaseInsensitive(key)].HttpHeaderValues + let tmp = headers.table[headers.toCaseInsensitive(key)] + return HttpHeaderValues(tmp) converter toString*(values: HttpHeaderValues): string = return seq[string](values)[0] @@ -237,10 +235,9 @@ func parseList(line: string, list: var seq[string], start: int): int = while start+i < line.len and line[start + i] notin {'\c', '\l'}: i += line.skipWhitespace(start + i) i += line.parseUntil(current, {'\c', '\l', ','}, start + i) - list.add(current) + list.add(move current) # implicit current.setLen(0) if start+i < line.len and line[start + i] == ',': i.inc # Skip , - current.setLen(0) func parseHeader*(line: string): tuple[key: string, value: seq[string]] = ## Parses a single raw header HTTP line into key value pairs. @@ -352,20 +349,20 @@ func is1xx*(code: HttpCode): bool {.inline, since: (1, 5).} = runnableExamples: doAssert is1xx(HttpCode(103)) - code.int in {100 .. 199} + code.int in 100 .. 199 func is2xx*(code: HttpCode): bool {.inline.} = ## Determines whether `code` is a 2xx HTTP status code. - code.int in {200 .. 299} + code.int in 200 .. 299 func is3xx*(code: HttpCode): bool {.inline.} = ## Determines whether `code` is a 3xx HTTP status code. - code.int in {300 .. 399} + code.int in 300 .. 399 func is4xx*(code: HttpCode): bool {.inline.} = ## Determines whether `code` is a 4xx HTTP status code. - code.int in {400 .. 499} + code.int in 400 .. 499 func is5xx*(code: HttpCode): bool {.inline.} = ## Determines whether `code` is a 5xx HTTP status code. - code.int in {500 .. 599} + code.int in 500 .. 599 diff --git a/lib/pure/includes/osenv.nim b/lib/pure/includes/osenv.nim deleted file mode 100644 index b7200a8e2..000000000 --- a/lib/pure/includes/osenv.nim +++ /dev/null @@ -1,207 +0,0 @@ -# Include file that implements 'getEnv' and friends. Do not import it! - -when not declared(os): - {.error: "This is an include file for os.nim!".} - -when not defined(nimscript): - when defined(nodejs): - proc getEnv*(key: string, default = ""): string {.tags: [ReadEnvEffect].} = - var ret = default.cstring - let key2 = key.cstring - {.emit: "const value = process.env[`key2`];".} - {.emit: "if (value !== undefined) { `ret` = value };".} - result = $ret - - proc existsEnv*(key: string): bool {.tags: [ReadEnvEffect].} = - var key2 = key.cstring - var ret: bool - {.emit: "`ret` = `key2` in process.env;".} - result = ret - - proc putEnv*(key, val: string) {.tags: [WriteEnvEffect].} = - var key2 = key.cstring - var val2 = val.cstring - {.emit: "process.env[`key2`] = `val2`;".} - - proc delEnv*(key: string) {.tags: [WriteEnvEffect].} = - var key2 = key.cstring - {.emit: "delete process.env[`key2`];".} - - iterator envPairsImpl(): tuple[key, value: string] {.tags: [ReadEnvEffect].} = - var num: int - var keys: RootObj - {.emit: "`keys` = Object.keys(process.env); `num` = `keys`.length;".} - for i in 0..<num: - var key, value: cstring - {.emit: "`key` = `keys`[`i`]; `value` = process.env[`key`];".} - yield ($key, $value) - - # commented because it must keep working with js+VM - # elif defined(js): - # {.error: "requires -d:nodejs".} - - else: - - when defined(windows): - proc c_putenv(envstring: cstring): cint {.importc: "_putenv", header: "<stdlib.h>".} - from std/private/win_setenv import setEnvImpl - proc c_wgetenv(varname: WideCString): WideCString {.importc: "_wgetenv", - header: "<stdlib.h>".} - proc getEnvImpl(env: cstring): WideCString = c_wgetenv(env.newWideCString) - else: - proc c_getenv(env: cstring): cstring {. - importc: "getenv", header: "<stdlib.h>".} - proc c_setenv(envname: cstring, envval: cstring, overwrite: cint): cint {.importc: "setenv", header: "<stdlib.h>".} - proc c_unsetenv(env: cstring): cint {.importc: "unsetenv", header: "<stdlib.h>".} - proc getEnvImpl(env: cstring): cstring = c_getenv(env) - - proc getEnv*(key: string, default = ""): string {.tags: [ReadEnvEffect].} = - ## Returns the value of the `environment variable`:idx: named `key`. - ## - ## If the variable does not exist, `""` is returned. To distinguish - ## whether a variable exists or it's value is just `""`, call - ## `existsEnv(key) proc`_. - ## - ## See also: - ## * `existsEnv proc`_ - ## * `putEnv proc`_ - ## * `delEnv proc`_ - ## * `envPairs iterator`_ - runnableExamples: - assert getEnv("unknownEnv") == "" - assert getEnv("unknownEnv", "doesn't exist") == "doesn't exist" - - let env = getEnvImpl(key) - if env == nil: return default - result = $env - - proc existsEnv*(key: string): bool {.tags: [ReadEnvEffect].} = - ## Checks whether the environment variable named `key` exists. - ## Returns true if it exists, false otherwise. - ## - ## See also: - ## * `getEnv proc`_ - ## * `putEnv proc`_ - ## * `delEnv proc`_ - ## * `envPairs iterator`_ - runnableExamples: - assert not existsEnv("unknownEnv") - - return getEnvImpl(key) != nil - - proc putEnv*(key, val: string) {.tags: [WriteEnvEffect].} = - ## Sets the value of the `environment variable`:idx: named `key` to `val`. - ## If an error occurs, `OSError` is raised. - ## - ## See also: - ## * `getEnv proc`_ - ## * `existsEnv proc`_ - ## * `delEnv proc`_ - ## * `envPairs iterator`_ - when defined(windows): - if key.len == 0 or '=' in key: - raise newException(OSError, "invalid key, got: " & $(key, val)) - if setEnvImpl(key, val, 1'i32) != 0'i32: - raiseOSError(osLastError(), $(key, val)) - else: - if c_setenv(key, val, 1'i32) != 0'i32: - raiseOSError(osLastError(), $(key, val)) - - proc delEnv*(key: string) {.tags: [WriteEnvEffect].} = - ## Deletes the `environment variable`:idx: named `key`. - ## If an error occurs, `OSError` is raised. - ## - ## See also:ven - ## * `getEnv proc`_ - ## * `existsEnv proc`_ - ## * `putEnv proc`_ - ## * `envPairs iterator`_ - template bail = raiseOSError(osLastError(), key) - when defined(windows): - #[ - # https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/putenv-s-wputenv-s?view=msvc-160 - > You can remove a variable from the environment by specifying an empty string (that is, "") for value_string - note that nil is not legal - ]# - if key.len == 0 or '=' in key: - raise newException(OSError, "invalid key, got: " & key) - let envToDel = key & "=" - if c_putenv(cstring envToDel) != 0'i32: bail - else: - if c_unsetenv(key) != 0'i32: bail - - when defined(windows): - when useWinUnicode: - when defined(cpp): - proc strEnd(cstr: WideCString, c = 0'i32): WideCString {.importcpp: "(NI16*)wcschr((const wchar_t *)#, #)", - header: "<string.h>".} - else: - proc strEnd(cstr: WideCString, c = 0'i32): WideCString {.importc: "wcschr", - header: "<string.h>".} - else: - proc strEnd(cstr: cstring, c = 0'i32): cstring {.importc: "strchr", - header: "<string.h>".} - elif defined(macosx) and not defined(ios) and not defined(emscripten): - # From the manual: - # Shared libraries and bundles don't have direct access to environ, - # which is only available to the loader ld(1) when a complete program - # is being linked. - # The environment routines can still be used, but if direct access to - # environ is needed, the _NSGetEnviron() routine, defined in - # <crt_externs.h>, can be used to retrieve the address of environ - # at runtime. - proc NSGetEnviron(): ptr cstringArray {.importc: "_NSGetEnviron", - header: "<crt_externs.h>".} - elif defined(haiku): - var gEnv {.importc: "environ", header: "<stdlib.h>".}: cstringArray - else: - var gEnv {.importc: "environ".}: cstringArray - - iterator envPairsImpl(): tuple[key, value: string] {.tags: [ReadEnvEffect].} = - when defined(windows): - block: - template impl(get_fun, typ, size, zero, free_fun) = - let env = get_fun() - var e = env - if e == nil: break - while true: - let eend = strEnd(e) - let kv = $e - let p = find(kv, '=') - yield (substr(kv, 0, p-1), substr(kv, p+1)) - e = cast[typ](cast[ByteAddress](eend)+size) - if typeof(zero)(eend[1]) == zero: break - discard free_fun(env) - when useWinUnicode: - impl(getEnvironmentStringsW, WideCString, 2, 0, freeEnvironmentStringsW) - else: - impl(getEnvironmentStringsA, cstring, 1, '\0', freeEnvironmentStringsA) - else: - var i = 0 - when defined(macosx) and not defined(ios) and not defined(emscripten): - var gEnv = NSGetEnviron()[] - while gEnv[i] != nil: - let kv = $gEnv[i] - inc(i) - let p = find(kv, '=') - yield (substr(kv, 0, p-1), substr(kv, p+1)) - -proc envPairsImplSeq(): seq[tuple[key, value: string]] = discard # vmops - -iterator envPairs*(): tuple[key, value: string] {.tags: [ReadEnvEffect].} = - ## Iterate over all `environments variables`:idx:. - ## - ## In the first component of the tuple is the name of the current variable stored, - ## in the second its value. - ## - ## Works in native backends, nodejs and vm, like the following APIs: - ## * `getEnv proc`_ - ## * `existsEnv proc`_ - ## * `putEnv proc`_ - ## * `delEnv proc`_ - when nimvm: - for ai in envPairsImplSeq(): yield ai - else: - when defined(nimscript): discard - else: - for ai in envPairsImpl(): yield ai diff --git a/lib/pure/includes/oserr.nim b/lib/pure/includes/oserr.nim deleted file mode 100644 index a6eba84ba..000000000 --- a/lib/pure/includes/oserr.nim +++ /dev/null @@ -1,119 +0,0 @@ -# Include file that implements 'osErrorMsg' and friends. Do not import it! - -when not declared(os): - {.error: "This is an include file for os.nim!".} - -when not defined(nimscript): - var errno {.importc, header: "<errno.h>".}: cint - - proc c_strerror(errnum: cint): cstring {. - importc: "strerror", header: "<string.h>".} - - when defined(windows): - import winlean - -proc `==`*(err1, err2: OSErrorCode): bool {.borrow.} -proc `$`*(err: OSErrorCode): string {.borrow.} - -proc osErrorMsg*(errorCode: OSErrorCode): string = - ## Converts an OS error code into a human readable string. - ## - ## The error code can be retrieved using the `osLastError proc`_. - ## - ## If conversion fails, or `errorCode` is `0` then `""` will be - ## returned. - ## - ## On Windows, the `-d:useWinAnsi` compilation flag can be used to - ## make this procedure use the non-unicode Win API calls to retrieve the - ## message. - ## - ## See also: - ## * `raiseOSError proc`_ - ## * `osLastError proc`_ - runnableExamples: - when defined(linux): - assert osErrorMsg(OSErrorCode(0)) == "" - assert osErrorMsg(OSErrorCode(1)) == "Operation not permitted" - assert osErrorMsg(OSErrorCode(2)) == "No such file or directory" - - result = "" - when defined(nimscript): - discard - elif defined(windows): - if errorCode != OSErrorCode(0'i32): - when useWinUnicode: - var msgbuf: WideCString - if formatMessageW(0x00000100 or 0x00001000 or 0x00000200, - nil, errorCode.int32, 0, addr(msgbuf), 0, nil) != 0'i32: - result = $msgbuf - if msgbuf != nil: localFree(cast[pointer](msgbuf)) - else: - var msgbuf: cstring - if formatMessageA(0x00000100 or 0x00001000 or 0x00000200, - nil, errorCode.int32, 0, addr(msgbuf), 0, nil) != 0'i32: - result = $msgbuf - if msgbuf != nil: localFree(msgbuf) - else: - if errorCode != OSErrorCode(0'i32): - result = $c_strerror(errorCode.int32) - -proc newOSError*( - errorCode: OSErrorCode, additionalInfo = "" -): owned(ref OSError) {.noinline.} = - ## Creates a new `OSError exception <system.html#OSError>`_. - ## - ## The `errorCode` will determine the - ## message, `osErrorMsg proc`_ will be used - ## to get this message. - ## - ## The error code can be retrieved using the `osLastError proc`_. - ## - ## If the error code is `0` or an error message could not be retrieved, - ## the message `unknown OS error` will be used. - ## - ## See also: - ## * `osErrorMsg proc`_ - ## * `osLastError proc`_ - var e: owned(ref OSError); new(e) - e.errorCode = errorCode.int32 - e.msg = osErrorMsg(errorCode) - if additionalInfo.len > 0: - if e.msg.len > 0 and e.msg[^1] != '\n': e.msg.add '\n' - e.msg.add "Additional info: " - e.msg.add additionalInfo - # don't add trailing `.` etc, which negatively impacts "jump to file" in IDEs. - if e.msg == "": - e.msg = "unknown OS error" - return e - -proc raiseOSError*(errorCode: OSErrorCode, additionalInfo = "") {.noinline.} = - ## Raises an `OSError exception <system.html#OSError>`_. - ## - ## Read the description of the `newOSError proc`_ to learn - ## how the exception object is created. - raise newOSError(errorCode, additionalInfo) - -{.push stackTrace:off.} -proc osLastError*(): OSErrorCode {.sideEffect.} = - ## Retrieves the last operating system error code. - ## - ## This procedure is useful in the event when an OS call fails. In that case - ## this procedure will return the error code describing the reason why the - ## OS call failed. The `OSErrorMsg` procedure can then be used to convert - ## this code into a string. - ## - ## .. warning:: The behaviour of this procedure varies between Windows and POSIX systems. - ## On Windows some OS calls can reset the error code to `0` causing this - ## procedure to return `0`. It is therefore advised to call this procedure - ## immediately after an OS call fails. On POSIX systems this is not a problem. - ## - ## See also: - ## * `osErrorMsg proc`_ - ## * `raiseOSError proc`_ - when defined(nimscript): - discard - elif defined(windows): - result = cast[OSErrorCode](getLastError()) - else: - result = OSErrorCode(errno) -{.pop.} diff --git a/lib/pure/includes/osseps.nim b/lib/pure/includes/osseps.nim deleted file mode 100644 index 1ea587e3c..000000000 --- a/lib/pure/includes/osseps.nim +++ /dev/null @@ -1,111 +0,0 @@ -# Include file that implements 'DirSep' and friends. Do not import this when -# you also import `os.nim`! - -# Improved based on info in 'compiler/platform.nim' - -const - doslikeFileSystem* = defined(windows) or defined(OS2) or defined(DOS) - -const - CurDir* = - when defined(macos): ':' - elif defined(genode): '/' - else: '.' - ## The constant character used by the operating system to refer to the - ## current directory. - ## - ## For example: `'.'` for POSIX or `':'` for the classic Macintosh. - - ParDir* = - when defined(macos): "::" - else: ".." - ## The constant string used by the operating system to refer to the - ## parent directory. - ## - ## For example: `".."` for POSIX or `"::"` for the classic Macintosh. - - DirSep* = - when defined(macos): ':' - elif doslikeFileSystem or defined(vxworks): '\\' - elif defined(RISCOS): '.' - else: '/' - ## The character used by the operating system to separate pathname - ## components, for example: `'/'` for POSIX, `':'` for the classic - ## Macintosh, and `'\\'` on Windows. - - AltSep* = - when doslikeFileSystem: '/' - else: DirSep - ## An alternative character used by the operating system to separate - ## pathname components, or the same as DirSep_ if only one separator - ## character exists. This is set to `'/'` on Windows systems - ## where DirSep_ is a backslash (`'\\'`). - - PathSep* = - when defined(macos) or defined(RISCOS): ',' - elif doslikeFileSystem or defined(vxworks): ';' - elif defined(PalmOS) or defined(MorphOS): ':' # platform has ':' but osseps has ';' - else: ':' - ## The character conventionally used by the operating system to separate - ## search path components (as in PATH), such as `':'` for POSIX - ## or `';'` for Windows. - - FileSystemCaseSensitive* = - when defined(macos) or defined(macosx) or doslikeFileSystem or defined(vxworks) or - defined(PalmOS) or defined(MorphOS): false - else: true - ## True if the file system is case sensitive, false otherwise. Used by - ## `cmpPaths proc`_ to compare filenames properly. - - ExeExt* = - when doslikeFileSystem: "exe" - elif defined(atari): "tpp" - elif defined(netware): "nlm" - elif defined(vxworks): "vxe" - elif defined(nintendoswitch): "elf" - else: "" - ## The file extension of native executables. For example: - ## `""` for POSIX, `"exe"` on Windows (without a dot). - - ScriptExt* = - when doslikeFileSystem: "bat" - else: "" - ## The file extension of a script file. For example: `""` for POSIX, - ## `"bat"` on Windows. - - DynlibFormat* = - when defined(macos): "$1.dylib" # platform has $1Lib - elif defined(macosx): "lib$1.dylib" - elif doslikeFileSystem or defined(atari): "$1.dll" - elif defined(MorphOS): "$1.prc" - elif defined(PalmOS): "$1.prc" # platform has lib$1.so - elif defined(genode): "$1.lib.so" - elif defined(netware): "$1.nlm" - elif defined(amiga): "$1.Library" - else: "lib$1.so" - ## The format string to turn a filename into a `DLL`:idx: file (also - ## called `shared object`:idx: on some operating systems). - - ExtSep* = '.' - ## The character which separates the base filename from the extension; - ## for example, the `'.'` in `os.nim`. - - # MacOS paths - # =========== - # MacOS directory separator is a colon ":" which is the only character not - # allowed in filenames. - # - # A path containing no colon or which begins with a colon is a partial - # path. - # E.g. ":kalle:petter" ":kalle" "kalle" - # - # All other paths are full (absolute) paths. E.g. "HD:kalle:" "HD:" - # When generating paths, one is safe if one ensures that all partial paths - # begin with a colon, and all full paths end with a colon. - # In full paths the first name (e g HD above) is the name of a mounted - # volume. - # These names are not unique, because, for instance, two diskettes with the - # same names could be inserted. This means that paths on MacOS are not - # waterproof. In case of equal names the first volume found will do. - # Two colons "::" are the relative path to the parent. Three is to the - # grandparent etc. diff --git a/lib/pure/includes/unicode_ranges.nim b/lib/pure/includes/unicode_ranges.nim index 74c8c6399..04ccfb747 100644 --- a/lib/pure/includes/unicode_ranges.nim +++ b/lib/pure/includes/unicode_ranges.nim @@ -2,1988 +2,1979 @@ const toLowerRanges = [ - 0x00041, 0x0005A, 532, - 0x000C0, 0x000D6, 532, - 0x000D8, 0x000DE, 532, - 0x00189, 0x0018A, 705, - 0x001B1, 0x001B2, 717, - 0x00388, 0x0038A, 537, - 0x0038E, 0x0038F, 563, - 0x00391, 0x003A1, 532, - 0x003A3, 0x003AB, 532, - 0x003FD, 0x003FF, 370, - 0x00400, 0x0040F, 580, - 0x00410, 0x0042F, 532, - 0x00531, 0x00556, 548, - 0x010A0, 0x010C5, 7764, - 0x013A0, 0x013EF, 39364, - 0x013F0, 0x013F5, 508, - 0x01C90, 0x01CBA, -2508, - 0x01CBD, 0x01CBF, -2508, - 0x01F08, 0x01F0F, 492, - 0x01F18, 0x01F1D, 492, - 0x01F28, 0x01F2F, 492, - 0x01F38, 0x01F3F, 492, - 0x01F48, 0x01F4D, 492, - 0x01F68, 0x01F6F, 492, - 0x01F88, 0x01F8F, 492, - 0x01F98, 0x01F9F, 492, - 0x01FA8, 0x01FAF, 492, - 0x01FB8, 0x01FB9, 492, - 0x01FBA, 0x01FBB, 426, - 0x01FC8, 0x01FCB, 414, - 0x01FD8, 0x01FD9, 492, - 0x01FDA, 0x01FDB, 400, - 0x01FE8, 0x01FE9, 492, - 0x01FEA, 0x01FEB, 388, - 0x01FF8, 0x01FF9, 372, - 0x01FFA, 0x01FFB, 374, - 0x02C00, 0x02C2E, 548, - 0x02C7E, 0x02C7F, -10315, - 0x0FF21, 0x0FF3A, 532, - 0x10400, 0x10427, 540, - 0x104B0, 0x104D3, 540, - 0x10C80, 0x10CB2, 564, - 0x118A0, 0x118BF, 532, - 0x16E40, 0x16E5F, 532, - 0x1E900, 0x1E921, 534, + 0x00041'i32, 0x0005A'i32, 532, + 0x000C0'i32, 0x000D6'i32, 532, + 0x000D8'i32, 0x000DE'i32, 532, + 0x00189'i32, 0x0018A'i32, 705, + 0x001B1'i32, 0x001B2'i32, 717, + 0x00388'i32, 0x0038A'i32, 537, + 0x0038E'i32, 0x0038F'i32, 563, + 0x00391'i32, 0x003A1'i32, 532, + 0x003A3'i32, 0x003AB'i32, 532, + 0x003FD'i32, 0x003FF'i32, 370, + 0x00400'i32, 0x0040F'i32, 580, + 0x00410'i32, 0x0042F'i32, 532, + 0x00531'i32, 0x00556'i32, 548, + 0x010A0'i32, 0x010C5'i32, 7764, + 0x013A0'i32, 0x013EF'i32, 39364, + 0x013F0'i32, 0x013F5'i32, 508, + 0x01C90'i32, 0x01CBA'i32, -2508, + 0x01CBD'i32, 0x01CBF'i32, -2508, + 0x01F08'i32, 0x01F0F'i32, 492, + 0x01F18'i32, 0x01F1D'i32, 492, + 0x01F28'i32, 0x01F2F'i32, 492, + 0x01F38'i32, 0x01F3F'i32, 492, + 0x01F48'i32, 0x01F4D'i32, 492, + 0x01F68'i32, 0x01F6F'i32, 492, + 0x01F88'i32, 0x01F8F'i32, 492, + 0x01F98'i32, 0x01F9F'i32, 492, + 0x01FA8'i32, 0x01FAF'i32, 492, + 0x01FB8'i32, 0x01FB9'i32, 492, + 0x01FBA'i32, 0x01FBB'i32, 426, + 0x01FC8'i32, 0x01FCB'i32, 414, + 0x01FD8'i32, 0x01FD9'i32, 492, + 0x01FDA'i32, 0x01FDB'i32, 400, + 0x01FE8'i32, 0x01FE9'i32, 492, + 0x01FEA'i32, 0x01FEB'i32, 388, + 0x01FF8'i32, 0x01FF9'i32, 372, + 0x01FFA'i32, 0x01FFB'i32, 374, + 0x02C00'i32, 0x02C2E'i32, 548, + 0x02C7E'i32, 0x02C7F'i32, -10315, + 0x0FF21'i32, 0x0FF3A'i32, 532, + 0x10400'i32, 0x10427'i32, 540, + 0x104B0'i32, 0x104D3'i32, 540, + 0x10C80'i32, 0x10CB2'i32, 564, + 0x118A0'i32, 0x118BF'i32, 532, + 0x16E40'i32, 0x16E5F'i32, 532, + 0x1E900'i32, 0x1E921'i32, 534, ] toLowerSinglets = [ - 0x00100, 501, - 0x00102, 501, - 0x00104, 501, - 0x00106, 501, - 0x00108, 501, - 0x0010A, 501, - 0x0010C, 501, - 0x0010E, 501, - 0x00110, 501, - 0x00112, 501, - 0x00114, 501, - 0x00116, 501, - 0x00118, 501, - 0x0011A, 501, - 0x0011C, 501, - 0x0011E, 501, - 0x00120, 501, - 0x00122, 501, - 0x00124, 501, - 0x00126, 501, - 0x00128, 501, - 0x0012A, 501, - 0x0012C, 501, - 0x0012E, 501, - 0x00130, 301, - 0x00132, 501, - 0x00134, 501, - 0x00136, 501, - 0x00139, 501, - 0x0013B, 501, - 0x0013D, 501, - 0x0013F, 501, - 0x00141, 501, - 0x00143, 501, - 0x00145, 501, - 0x00147, 501, - 0x0014A, 501, - 0x0014C, 501, - 0x0014E, 501, - 0x00150, 501, - 0x00152, 501, - 0x00154, 501, - 0x00156, 501, - 0x00158, 501, - 0x0015A, 501, - 0x0015C, 501, - 0x0015E, 501, - 0x00160, 501, - 0x00162, 501, - 0x00164, 501, - 0x00166, 501, - 0x00168, 501, - 0x0016A, 501, - 0x0016C, 501, - 0x0016E, 501, - 0x00170, 501, - 0x00172, 501, - 0x00174, 501, - 0x00176, 501, - 0x00178, 379, - 0x00179, 501, - 0x0017B, 501, - 0x0017D, 501, - 0x00181, 710, - 0x00182, 501, - 0x00184, 501, - 0x00186, 706, - 0x00187, 501, - 0x0018B, 501, - 0x0018E, 579, - 0x0018F, 702, - 0x00190, 703, - 0x00191, 501, - 0x00193, 705, - 0x00194, 707, - 0x00196, 711, - 0x00197, 709, - 0x00198, 501, - 0x0019C, 711, - 0x0019D, 713, - 0x0019F, 714, - 0x001A0, 501, - 0x001A2, 501, - 0x001A4, 501, - 0x001A6, 718, - 0x001A7, 501, - 0x001A9, 718, - 0x001AC, 501, - 0x001AE, 718, - 0x001AF, 501, - 0x001B3, 501, - 0x001B5, 501, - 0x001B7, 719, - 0x001B8, 501, - 0x001BC, 501, - 0x001C4, 502, - 0x001C5, 501, - 0x001C7, 502, - 0x001C8, 501, - 0x001CA, 502, - 0x001CB, 501, - 0x001CD, 501, - 0x001CF, 501, - 0x001D1, 501, - 0x001D3, 501, - 0x001D5, 501, - 0x001D7, 501, - 0x001D9, 501, - 0x001DB, 501, - 0x001DE, 501, - 0x001E0, 501, - 0x001E2, 501, - 0x001E4, 501, - 0x001E6, 501, - 0x001E8, 501, - 0x001EA, 501, - 0x001EC, 501, - 0x001EE, 501, - 0x001F1, 502, - 0x001F2, 501, - 0x001F4, 501, - 0x001F6, 403, - 0x001F7, 444, - 0x001F8, 501, - 0x001FA, 501, - 0x001FC, 501, - 0x001FE, 501, - 0x00200, 501, - 0x00202, 501, - 0x00204, 501, - 0x00206, 501, - 0x00208, 501, - 0x0020A, 501, - 0x0020C, 501, - 0x0020E, 501, - 0x00210, 501, - 0x00212, 501, - 0x00214, 501, - 0x00216, 501, - 0x00218, 501, - 0x0021A, 501, - 0x0021C, 501, - 0x0021E, 501, - 0x00220, 370, - 0x00222, 501, - 0x00224, 501, - 0x00226, 501, - 0x00228, 501, - 0x0022A, 501, - 0x0022C, 501, - 0x0022E, 501, - 0x00230, 501, - 0x00232, 501, - 0x0023A, 11295, - 0x0023B, 501, - 0x0023D, 337, - 0x0023E, 11292, - 0x00241, 501, - 0x00243, 305, - 0x00244, 569, - 0x00245, 571, - 0x00246, 501, - 0x00248, 501, - 0x0024A, 501, - 0x0024C, 501, - 0x0024E, 501, - 0x00370, 501, - 0x00372, 501, - 0x00376, 501, - 0x0037F, 616, - 0x00386, 538, - 0x0038C, 564, - 0x003CF, 508, - 0x003D8, 501, - 0x003DA, 501, - 0x003DC, 501, - 0x003DE, 501, - 0x003E0, 501, - 0x003E2, 501, - 0x003E4, 501, - 0x003E6, 501, - 0x003E8, 501, - 0x003EA, 501, - 0x003EC, 501, - 0x003EE, 501, - 0x003F4, 440, - 0x003F7, 501, - 0x003F9, 493, - 0x003FA, 501, - 0x00460, 501, - 0x00462, 501, - 0x00464, 501, - 0x00466, 501, - 0x00468, 501, - 0x0046A, 501, - 0x0046C, 501, - 0x0046E, 501, - 0x00470, 501, - 0x00472, 501, - 0x00474, 501, - 0x00476, 501, - 0x00478, 501, - 0x0047A, 501, - 0x0047C, 501, - 0x0047E, 501, - 0x00480, 501, - 0x0048A, 501, - 0x0048C, 501, - 0x0048E, 501, - 0x00490, 501, - 0x00492, 501, - 0x00494, 501, - 0x00496, 501, - 0x00498, 501, - 0x0049A, 501, - 0x0049C, 501, - 0x0049E, 501, - 0x004A0, 501, - 0x004A2, 501, - 0x004A4, 501, - 0x004A6, 501, - 0x004A8, 501, - 0x004AA, 501, - 0x004AC, 501, - 0x004AE, 501, - 0x004B0, 501, - 0x004B2, 501, - 0x004B4, 501, - 0x004B6, 501, - 0x004B8, 501, - 0x004BA, 501, - 0x004BC, 501, - 0x004BE, 501, - 0x004C0, 515, - 0x004C1, 501, - 0x004C3, 501, - 0x004C5, 501, - 0x004C7, 501, - 0x004C9, 501, - 0x004CB, 501, - 0x004CD, 501, - 0x004D0, 501, - 0x004D2, 501, - 0x004D4, 501, - 0x004D6, 501, - 0x004D8, 501, - 0x004DA, 501, - 0x004DC, 501, - 0x004DE, 501, - 0x004E0, 501, - 0x004E2, 501, - 0x004E4, 501, - 0x004E6, 501, - 0x004E8, 501, - 0x004EA, 501, - 0x004EC, 501, - 0x004EE, 501, - 0x004F0, 501, - 0x004F2, 501, - 0x004F4, 501, - 0x004F6, 501, - 0x004F8, 501, - 0x004FA, 501, - 0x004FC, 501, - 0x004FE, 501, - 0x00500, 501, - 0x00502, 501, - 0x00504, 501, - 0x00506, 501, - 0x00508, 501, - 0x0050A, 501, - 0x0050C, 501, - 0x0050E, 501, - 0x00510, 501, - 0x00512, 501, - 0x00514, 501, - 0x00516, 501, - 0x00518, 501, - 0x0051A, 501, - 0x0051C, 501, - 0x0051E, 501, - 0x00520, 501, - 0x00522, 501, - 0x00524, 501, - 0x00526, 501, - 0x00528, 501, - 0x0052A, 501, - 0x0052C, 501, - 0x0052E, 501, - 0x010C7, 7764, - 0x010CD, 7764, - 0x01E00, 501, - 0x01E02, 501, - 0x01E04, 501, - 0x01E06, 501, - 0x01E08, 501, - 0x01E0A, 501, - 0x01E0C, 501, - 0x01E0E, 501, - 0x01E10, 501, - 0x01E12, 501, - 0x01E14, 501, - 0x01E16, 501, - 0x01E18, 501, - 0x01E1A, 501, - 0x01E1C, 501, - 0x01E1E, 501, - 0x01E20, 501, - 0x01E22, 501, - 0x01E24, 501, - 0x01E26, 501, - 0x01E28, 501, - 0x01E2A, 501, - 0x01E2C, 501, - 0x01E2E, 501, - 0x01E30, 501, - 0x01E32, 501, - 0x01E34, 501, - 0x01E36, 501, - 0x01E38, 501, - 0x01E3A, 501, - 0x01E3C, 501, - 0x01E3E, 501, - 0x01E40, 501, - 0x01E42, 501, - 0x01E44, 501, - 0x01E46, 501, - 0x01E48, 501, - 0x01E4A, 501, - 0x01E4C, 501, - 0x01E4E, 501, - 0x01E50, 501, - 0x01E52, 501, - 0x01E54, 501, - 0x01E56, 501, - 0x01E58, 501, - 0x01E5A, 501, - 0x01E5C, 501, - 0x01E5E, 501, - 0x01E60, 501, - 0x01E62, 501, - 0x01E64, 501, - 0x01E66, 501, - 0x01E68, 501, - 0x01E6A, 501, - 0x01E6C, 501, - 0x01E6E, 501, - 0x01E70, 501, - 0x01E72, 501, - 0x01E74, 501, - 0x01E76, 501, - 0x01E78, 501, - 0x01E7A, 501, - 0x01E7C, 501, - 0x01E7E, 501, - 0x01E80, 501, - 0x01E82, 501, - 0x01E84, 501, - 0x01E86, 501, - 0x01E88, 501, - 0x01E8A, 501, - 0x01E8C, 501, - 0x01E8E, 501, - 0x01E90, 501, - 0x01E92, 501, - 0x01E94, 501, - 0x01E9E, -7115, - 0x01EA0, 501, - 0x01EA2, 501, - 0x01EA4, 501, - 0x01EA6, 501, - 0x01EA8, 501, - 0x01EAA, 501, - 0x01EAC, 501, - 0x01EAE, 501, - 0x01EB0, 501, - 0x01EB2, 501, - 0x01EB4, 501, - 0x01EB6, 501, - 0x01EB8, 501, - 0x01EBA, 501, - 0x01EBC, 501, - 0x01EBE, 501, - 0x01EC0, 501, - 0x01EC2, 501, - 0x01EC4, 501, - 0x01EC6, 501, - 0x01EC8, 501, - 0x01ECA, 501, - 0x01ECC, 501, - 0x01ECE, 501, - 0x01ED0, 501, - 0x01ED2, 501, - 0x01ED4, 501, - 0x01ED6, 501, - 0x01ED8, 501, - 0x01EDA, 501, - 0x01EDC, 501, - 0x01EDE, 501, - 0x01EE0, 501, - 0x01EE2, 501, - 0x01EE4, 501, - 0x01EE6, 501, - 0x01EE8, 501, - 0x01EEA, 501, - 0x01EEC, 501, - 0x01EEE, 501, - 0x01EF0, 501, - 0x01EF2, 501, - 0x01EF4, 501, - 0x01EF6, 501, - 0x01EF8, 501, - 0x01EFA, 501, - 0x01EFC, 501, - 0x01EFE, 501, - 0x01F59, 492, - 0x01F5B, 492, - 0x01F5D, 492, - 0x01F5F, 492, - 0x01FBC, 491, - 0x01FCC, 491, - 0x01FEC, 493, - 0x01FFC, 491, - 0x02126, -7017, - 0x0212A, -7883, - 0x0212B, -7762, - 0x02132, 528, - 0x02183, 501, - 0x02C60, 501, - 0x02C62, -10243, - 0x02C63, -3314, - 0x02C64, -10227, - 0x02C67, 501, - 0x02C69, 501, - 0x02C6B, 501, - 0x02C6D, -10280, - 0x02C6E, -10249, - 0x02C6F, -10283, - 0x02C70, -10282, - 0x02C72, 501, - 0x02C75, 501, - 0x02C80, 501, - 0x02C82, 501, - 0x02C84, 501, - 0x02C86, 501, - 0x02C88, 501, - 0x02C8A, 501, - 0x02C8C, 501, - 0x02C8E, 501, - 0x02C90, 501, - 0x02C92, 501, - 0x02C94, 501, - 0x02C96, 501, - 0x02C98, 501, - 0x02C9A, 501, - 0x02C9C, 501, - 0x02C9E, 501, - 0x02CA0, 501, - 0x02CA2, 501, - 0x02CA4, 501, - 0x02CA6, 501, - 0x02CA8, 501, - 0x02CAA, 501, - 0x02CAC, 501, - 0x02CAE, 501, - 0x02CB0, 501, - 0x02CB2, 501, - 0x02CB4, 501, - 0x02CB6, 501, - 0x02CB8, 501, - 0x02CBA, 501, - 0x02CBC, 501, - 0x02CBE, 501, - 0x02CC0, 501, - 0x02CC2, 501, - 0x02CC4, 501, - 0x02CC6, 501, - 0x02CC8, 501, - 0x02CCA, 501, - 0x02CCC, 501, - 0x02CCE, 501, - 0x02CD0, 501, - 0x02CD2, 501, - 0x02CD4, 501, - 0x02CD6, 501, - 0x02CD8, 501, - 0x02CDA, 501, - 0x02CDC, 501, - 0x02CDE, 501, - 0x02CE0, 501, - 0x02CE2, 501, - 0x02CEB, 501, - 0x02CED, 501, - 0x02CF2, 501, - 0x0A640, 501, - 0x0A642, 501, - 0x0A644, 501, - 0x0A646, 501, - 0x0A648, 501, - 0x0A64A, 501, - 0x0A64C, 501, - 0x0A64E, 501, - 0x0A650, 501, - 0x0A652, 501, - 0x0A654, 501, - 0x0A656, 501, - 0x0A658, 501, - 0x0A65A, 501, - 0x0A65C, 501, - 0x0A65E, 501, - 0x0A660, 501, - 0x0A662, 501, - 0x0A664, 501, - 0x0A666, 501, - 0x0A668, 501, - 0x0A66A, 501, - 0x0A66C, 501, - 0x0A680, 501, - 0x0A682, 501, - 0x0A684, 501, - 0x0A686, 501, - 0x0A688, 501, - 0x0A68A, 501, - 0x0A68C, 501, - 0x0A68E, 501, - 0x0A690, 501, - 0x0A692, 501, - 0x0A694, 501, - 0x0A696, 501, - 0x0A698, 501, - 0x0A69A, 501, - 0x0A722, 501, - 0x0A724, 501, - 0x0A726, 501, - 0x0A728, 501, - 0x0A72A, 501, - 0x0A72C, 501, - 0x0A72E, 501, - 0x0A732, 501, - 0x0A734, 501, - 0x0A736, 501, - 0x0A738, 501, - 0x0A73A, 501, - 0x0A73C, 501, - 0x0A73E, 501, - 0x0A740, 501, - 0x0A742, 501, - 0x0A744, 501, - 0x0A746, 501, - 0x0A748, 501, - 0x0A74A, 501, - 0x0A74C, 501, - 0x0A74E, 501, - 0x0A750, 501, - 0x0A752, 501, - 0x0A754, 501, - 0x0A756, 501, - 0x0A758, 501, - 0x0A75A, 501, - 0x0A75C, 501, - 0x0A75E, 501, - 0x0A760, 501, - 0x0A762, 501, - 0x0A764, 501, - 0x0A766, 501, - 0x0A768, 501, - 0x0A76A, 501, - 0x0A76C, 501, - 0x0A76E, 501, - 0x0A779, 501, - 0x0A77B, 501, - 0x0A77D, -34832, - 0x0A77E, 501, - 0x0A780, 501, - 0x0A782, 501, - 0x0A784, 501, - 0x0A786, 501, - 0x0A78B, 501, - 0x0A78D, -41780, - 0x0A790, 501, - 0x0A792, 501, - 0x0A796, 501, - 0x0A798, 501, - 0x0A79A, 501, - 0x0A79C, 501, - 0x0A79E, 501, - 0x0A7A0, 501, - 0x0A7A2, 501, - 0x0A7A4, 501, - 0x0A7A6, 501, - 0x0A7A8, 501, - 0x0A7AA, -41808, - 0x0A7AB, -41819, - 0x0A7AC, -41815, - 0x0A7AD, -41805, - 0x0A7AE, -41808, - 0x0A7B0, -41758, - 0x0A7B1, -41782, - 0x0A7B2, -41761, - 0x0A7B3, 1428, - 0x0A7B4, 501, - 0x0A7B6, 501, - 0x0A7B8, 501, - 0x0A7BA, 501, - 0x0A7BC, 501, - 0x0A7BE, 501, - 0x0A7C2, 501, - 0x0A7C4, 452, - 0x0A7C5, -41807, - 0x0A7C6, -34884, + 0x00100'i32, 501, + 0x00102'i32, 501, + 0x00104'i32, 501, + 0x00106'i32, 501, + 0x00108'i32, 501, + 0x0010A'i32, 501, + 0x0010C'i32, 501, + 0x0010E'i32, 501, + 0x00110'i32, 501, + 0x00112'i32, 501, + 0x00114'i32, 501, + 0x00116'i32, 501, + 0x00118'i32, 501, + 0x0011A'i32, 501, + 0x0011C'i32, 501, + 0x0011E'i32, 501, + 0x00120'i32, 501, + 0x00122'i32, 501, + 0x00124'i32, 501, + 0x00126'i32, 501, + 0x00128'i32, 501, + 0x0012A'i32, 501, + 0x0012C'i32, 501, + 0x0012E'i32, 501, + 0x00130'i32, 301, + 0x00132'i32, 501, + 0x00134'i32, 501, + 0x00136'i32, 501, + 0x00139'i32, 501, + 0x0013B'i32, 501, + 0x0013D'i32, 501, + 0x0013F'i32, 501, + 0x00141'i32, 501, + 0x00143'i32, 501, + 0x00145'i32, 501, + 0x00147'i32, 501, + 0x0014A'i32, 501, + 0x0014C'i32, 501, + 0x0014E'i32, 501, + 0x00150'i32, 501, + 0x00152'i32, 501, + 0x00154'i32, 501, + 0x00156'i32, 501, + 0x00158'i32, 501, + 0x0015A'i32, 501, + 0x0015C'i32, 501, + 0x0015E'i32, 501, + 0x00160'i32, 501, + 0x00162'i32, 501, + 0x00164'i32, 501, + 0x00166'i32, 501, + 0x00168'i32, 501, + 0x0016A'i32, 501, + 0x0016C'i32, 501, + 0x0016E'i32, 501, + 0x00170'i32, 501, + 0x00172'i32, 501, + 0x00174'i32, 501, + 0x00176'i32, 501, + 0x00178'i32, 379, + 0x00179'i32, 501, + 0x0017B'i32, 501, + 0x0017D'i32, 501, + 0x00181'i32, 710, + 0x00182'i32, 501, + 0x00184'i32, 501, + 0x00186'i32, 706, + 0x00187'i32, 501, + 0x0018B'i32, 501, + 0x0018E'i32, 579, + 0x0018F'i32, 702, + 0x00190'i32, 703, + 0x00191'i32, 501, + 0x00193'i32, 705, + 0x00194'i32, 707, + 0x00196'i32, 711, + 0x00197'i32, 709, + 0x00198'i32, 501, + 0x0019C'i32, 711, + 0x0019D'i32, 713, + 0x0019F'i32, 714, + 0x001A0'i32, 501, + 0x001A2'i32, 501, + 0x001A4'i32, 501, + 0x001A6'i32, 718, + 0x001A7'i32, 501, + 0x001A9'i32, 718, + 0x001AC'i32, 501, + 0x001AE'i32, 718, + 0x001AF'i32, 501, + 0x001B3'i32, 501, + 0x001B5'i32, 501, + 0x001B7'i32, 719, + 0x001B8'i32, 501, + 0x001BC'i32, 501, + 0x001C4'i32, 502, + 0x001C5'i32, 501, + 0x001C7'i32, 502, + 0x001C8'i32, 501, + 0x001CA'i32, 502, + 0x001CB'i32, 501, + 0x001CD'i32, 501, + 0x001CF'i32, 501, + 0x001D1'i32, 501, + 0x001D3'i32, 501, + 0x001D5'i32, 501, + 0x001D7'i32, 501, + 0x001D9'i32, 501, + 0x001DB'i32, 501, + 0x001DE'i32, 501, + 0x001E0'i32, 501, + 0x001E2'i32, 501, + 0x001E4'i32, 501, + 0x001E6'i32, 501, + 0x001E8'i32, 501, + 0x001EA'i32, 501, + 0x001EC'i32, 501, + 0x001EE'i32, 501, + 0x001F1'i32, 502, + 0x001F2'i32, 501, + 0x001F4'i32, 501, + 0x001F6'i32, 403, + 0x001F7'i32, 444, + 0x001F8'i32, 501, + 0x001FA'i32, 501, + 0x001FC'i32, 501, + 0x001FE'i32, 501, + 0x00200'i32, 501, + 0x00202'i32, 501, + 0x00204'i32, 501, + 0x00206'i32, 501, + 0x00208'i32, 501, + 0x0020A'i32, 501, + 0x0020C'i32, 501, + 0x0020E'i32, 501, + 0x00210'i32, 501, + 0x00212'i32, 501, + 0x00214'i32, 501, + 0x00216'i32, 501, + 0x00218'i32, 501, + 0x0021A'i32, 501, + 0x0021C'i32, 501, + 0x0021E'i32, 501, + 0x00220'i32, 370, + 0x00222'i32, 501, + 0x00224'i32, 501, + 0x00226'i32, 501, + 0x00228'i32, 501, + 0x0022A'i32, 501, + 0x0022C'i32, 501, + 0x0022E'i32, 501, + 0x00230'i32, 501, + 0x00232'i32, 501, + 0x0023A'i32, 11295, + 0x0023B'i32, 501, + 0x0023D'i32, 337, + 0x0023E'i32, 11292, + 0x00241'i32, 501, + 0x00243'i32, 305, + 0x00244'i32, 569, + 0x00245'i32, 571, + 0x00246'i32, 501, + 0x00248'i32, 501, + 0x0024A'i32, 501, + 0x0024C'i32, 501, + 0x0024E'i32, 501, + 0x00370'i32, 501, + 0x00372'i32, 501, + 0x00376'i32, 501, + 0x0037F'i32, 616, + 0x00386'i32, 538, + 0x0038C'i32, 564, + 0x003CF'i32, 508, + 0x003D8'i32, 501, + 0x003DA'i32, 501, + 0x003DC'i32, 501, + 0x003DE'i32, 501, + 0x003E0'i32, 501, + 0x003E2'i32, 501, + 0x003E4'i32, 501, + 0x003E6'i32, 501, + 0x003E8'i32, 501, + 0x003EA'i32, 501, + 0x003EC'i32, 501, + 0x003EE'i32, 501, + 0x003F4'i32, 440, + 0x003F7'i32, 501, + 0x003F9'i32, 493, + 0x003FA'i32, 501, + 0x00460'i32, 501, + 0x00462'i32, 501, + 0x00464'i32, 501, + 0x00466'i32, 501, + 0x00468'i32, 501, + 0x0046A'i32, 501, + 0x0046C'i32, 501, + 0x0046E'i32, 501, + 0x00470'i32, 501, + 0x00472'i32, 501, + 0x00474'i32, 501, + 0x00476'i32, 501, + 0x00478'i32, 501, + 0x0047A'i32, 501, + 0x0047C'i32, 501, + 0x0047E'i32, 501, + 0x00480'i32, 501, + 0x0048A'i32, 501, + 0x0048C'i32, 501, + 0x0048E'i32, 501, + 0x00490'i32, 501, + 0x00492'i32, 501, + 0x00494'i32, 501, + 0x00496'i32, 501, + 0x00498'i32, 501, + 0x0049A'i32, 501, + 0x0049C'i32, 501, + 0x0049E'i32, 501, + 0x004A0'i32, 501, + 0x004A2'i32, 501, + 0x004A4'i32, 501, + 0x004A6'i32, 501, + 0x004A8'i32, 501, + 0x004AA'i32, 501, + 0x004AC'i32, 501, + 0x004AE'i32, 501, + 0x004B0'i32, 501, + 0x004B2'i32, 501, + 0x004B4'i32, 501, + 0x004B6'i32, 501, + 0x004B8'i32, 501, + 0x004BA'i32, 501, + 0x004BC'i32, 501, + 0x004BE'i32, 501, + 0x004C0'i32, 515, + 0x004C1'i32, 501, + 0x004C3'i32, 501, + 0x004C5'i32, 501, + 0x004C7'i32, 501, + 0x004C9'i32, 501, + 0x004CB'i32, 501, + 0x004CD'i32, 501, + 0x004D0'i32, 501, + 0x004D2'i32, 501, + 0x004D4'i32, 501, + 0x004D6'i32, 501, + 0x004D8'i32, 501, + 0x004DA'i32, 501, + 0x004DC'i32, 501, + 0x004DE'i32, 501, + 0x004E0'i32, 501, + 0x004E2'i32, 501, + 0x004E4'i32, 501, + 0x004E6'i32, 501, + 0x004E8'i32, 501, + 0x004EA'i32, 501, + 0x004EC'i32, 501, + 0x004EE'i32, 501, + 0x004F0'i32, 501, + 0x004F2'i32, 501, + 0x004F4'i32, 501, + 0x004F6'i32, 501, + 0x004F8'i32, 501, + 0x004FA'i32, 501, + 0x004FC'i32, 501, + 0x004FE'i32, 501, + 0x00500'i32, 501, + 0x00502'i32, 501, + 0x00504'i32, 501, + 0x00506'i32, 501, + 0x00508'i32, 501, + 0x0050A'i32, 501, + 0x0050C'i32, 501, + 0x0050E'i32, 501, + 0x00510'i32, 501, + 0x00512'i32, 501, + 0x00514'i32, 501, + 0x00516'i32, 501, + 0x00518'i32, 501, + 0x0051A'i32, 501, + 0x0051C'i32, 501, + 0x0051E'i32, 501, + 0x00520'i32, 501, + 0x00522'i32, 501, + 0x00524'i32, 501, + 0x00526'i32, 501, + 0x00528'i32, 501, + 0x0052A'i32, 501, + 0x0052C'i32, 501, + 0x0052E'i32, 501, + 0x010C7'i32, 7764, + 0x010CD'i32, 7764, + 0x01E00'i32, 501, + 0x01E02'i32, 501, + 0x01E04'i32, 501, + 0x01E06'i32, 501, + 0x01E08'i32, 501, + 0x01E0A'i32, 501, + 0x01E0C'i32, 501, + 0x01E0E'i32, 501, + 0x01E10'i32, 501, + 0x01E12'i32, 501, + 0x01E14'i32, 501, + 0x01E16'i32, 501, + 0x01E18'i32, 501, + 0x01E1A'i32, 501, + 0x01E1C'i32, 501, + 0x01E1E'i32, 501, + 0x01E20'i32, 501, + 0x01E22'i32, 501, + 0x01E24'i32, 501, + 0x01E26'i32, 501, + 0x01E28'i32, 501, + 0x01E2A'i32, 501, + 0x01E2C'i32, 501, + 0x01E2E'i32, 501, + 0x01E30'i32, 501, + 0x01E32'i32, 501, + 0x01E34'i32, 501, + 0x01E36'i32, 501, + 0x01E38'i32, 501, + 0x01E3A'i32, 501, + 0x01E3C'i32, 501, + 0x01E3E'i32, 501, + 0x01E40'i32, 501, + 0x01E42'i32, 501, + 0x01E44'i32, 501, + 0x01E46'i32, 501, + 0x01E48'i32, 501, + 0x01E4A'i32, 501, + 0x01E4C'i32, 501, + 0x01E4E'i32, 501, + 0x01E50'i32, 501, + 0x01E52'i32, 501, + 0x01E54'i32, 501, + 0x01E56'i32, 501, + 0x01E58'i32, 501, + 0x01E5A'i32, 501, + 0x01E5C'i32, 501, + 0x01E5E'i32, 501, + 0x01E60'i32, 501, + 0x01E62'i32, 501, + 0x01E64'i32, 501, + 0x01E66'i32, 501, + 0x01E68'i32, 501, + 0x01E6A'i32, 501, + 0x01E6C'i32, 501, + 0x01E6E'i32, 501, + 0x01E70'i32, 501, + 0x01E72'i32, 501, + 0x01E74'i32, 501, + 0x01E76'i32, 501, + 0x01E78'i32, 501, + 0x01E7A'i32, 501, + 0x01E7C'i32, 501, + 0x01E7E'i32, 501, + 0x01E80'i32, 501, + 0x01E82'i32, 501, + 0x01E84'i32, 501, + 0x01E86'i32, 501, + 0x01E88'i32, 501, + 0x01E8A'i32, 501, + 0x01E8C'i32, 501, + 0x01E8E'i32, 501, + 0x01E90'i32, 501, + 0x01E92'i32, 501, + 0x01E94'i32, 501, + 0x01E9E'i32, -7115, + 0x01EA0'i32, 501, + 0x01EA2'i32, 501, + 0x01EA4'i32, 501, + 0x01EA6'i32, 501, + 0x01EA8'i32, 501, + 0x01EAA'i32, 501, + 0x01EAC'i32, 501, + 0x01EAE'i32, 501, + 0x01EB0'i32, 501, + 0x01EB2'i32, 501, + 0x01EB4'i32, 501, + 0x01EB6'i32, 501, + 0x01EB8'i32, 501, + 0x01EBA'i32, 501, + 0x01EBC'i32, 501, + 0x01EBE'i32, 501, + 0x01EC0'i32, 501, + 0x01EC2'i32, 501, + 0x01EC4'i32, 501, + 0x01EC6'i32, 501, + 0x01EC8'i32, 501, + 0x01ECA'i32, 501, + 0x01ECC'i32, 501, + 0x01ECE'i32, 501, + 0x01ED0'i32, 501, + 0x01ED2'i32, 501, + 0x01ED4'i32, 501, + 0x01ED6'i32, 501, + 0x01ED8'i32, 501, + 0x01EDA'i32, 501, + 0x01EDC'i32, 501, + 0x01EDE'i32, 501, + 0x01EE0'i32, 501, + 0x01EE2'i32, 501, + 0x01EE4'i32, 501, + 0x01EE6'i32, 501, + 0x01EE8'i32, 501, + 0x01EEA'i32, 501, + 0x01EEC'i32, 501, + 0x01EEE'i32, 501, + 0x01EF0'i32, 501, + 0x01EF2'i32, 501, + 0x01EF4'i32, 501, + 0x01EF6'i32, 501, + 0x01EF8'i32, 501, + 0x01EFA'i32, 501, + 0x01EFC'i32, 501, + 0x01EFE'i32, 501, + 0x01F59'i32, 492, + 0x01F5B'i32, 492, + 0x01F5D'i32, 492, + 0x01F5F'i32, 492, + 0x01FBC'i32, 491, + 0x01FCC'i32, 491, + 0x01FEC'i32, 493, + 0x01FFC'i32, 491, + 0x02126'i32, -7017, + 0x0212A'i32, -7883, + 0x0212B'i32, -7762, + 0x02132'i32, 528, + 0x02183'i32, 501, + 0x02C60'i32, 501, + 0x02C62'i32, -10243, + 0x02C63'i32, -3314, + 0x02C64'i32, -10227, + 0x02C67'i32, 501, + 0x02C69'i32, 501, + 0x02C6B'i32, 501, + 0x02C6D'i32, -10280, + 0x02C6E'i32, -10249, + 0x02C6F'i32, -10283, + 0x02C70'i32, -10282, + 0x02C72'i32, 501, + 0x02C75'i32, 501, + 0x02C80'i32, 501, + 0x02C82'i32, 501, + 0x02C84'i32, 501, + 0x02C86'i32, 501, + 0x02C88'i32, 501, + 0x02C8A'i32, 501, + 0x02C8C'i32, 501, + 0x02C8E'i32, 501, + 0x02C90'i32, 501, + 0x02C92'i32, 501, + 0x02C94'i32, 501, + 0x02C96'i32, 501, + 0x02C98'i32, 501, + 0x02C9A'i32, 501, + 0x02C9C'i32, 501, + 0x02C9E'i32, 501, + 0x02CA0'i32, 501, + 0x02CA2'i32, 501, + 0x02CA4'i32, 501, + 0x02CA6'i32, 501, + 0x02CA8'i32, 501, + 0x02CAA'i32, 501, + 0x02CAC'i32, 501, + 0x02CAE'i32, 501, + 0x02CB0'i32, 501, + 0x02CB2'i32, 501, + 0x02CB4'i32, 501, + 0x02CB6'i32, 501, + 0x02CB8'i32, 501, + 0x02CBA'i32, 501, + 0x02CBC'i32, 501, + 0x02CBE'i32, 501, + 0x02CC0'i32, 501, + 0x02CC2'i32, 501, + 0x02CC4'i32, 501, + 0x02CC6'i32, 501, + 0x02CC8'i32, 501, + 0x02CCA'i32, 501, + 0x02CCC'i32, 501, + 0x02CCE'i32, 501, + 0x02CD0'i32, 501, + 0x02CD2'i32, 501, + 0x02CD4'i32, 501, + 0x02CD6'i32, 501, + 0x02CD8'i32, 501, + 0x02CDA'i32, 501, + 0x02CDC'i32, 501, + 0x02CDE'i32, 501, + 0x02CE0'i32, 501, + 0x02CE2'i32, 501, + 0x02CEB'i32, 501, + 0x02CED'i32, 501, + 0x02CF2'i32, 501, + 0x0A640'i32, 501, + 0x0A642'i32, 501, + 0x0A644'i32, 501, + 0x0A646'i32, 501, + 0x0A648'i32, 501, + 0x0A64A'i32, 501, + 0x0A64C'i32, 501, + 0x0A64E'i32, 501, + 0x0A650'i32, 501, + 0x0A652'i32, 501, + 0x0A654'i32, 501, + 0x0A656'i32, 501, + 0x0A658'i32, 501, + 0x0A65A'i32, 501, + 0x0A65C'i32, 501, + 0x0A65E'i32, 501, + 0x0A660'i32, 501, + 0x0A662'i32, 501, + 0x0A664'i32, 501, + 0x0A666'i32, 501, + 0x0A668'i32, 501, + 0x0A66A'i32, 501, + 0x0A66C'i32, 501, + 0x0A680'i32, 501, + 0x0A682'i32, 501, + 0x0A684'i32, 501, + 0x0A686'i32, 501, + 0x0A688'i32, 501, + 0x0A68A'i32, 501, + 0x0A68C'i32, 501, + 0x0A68E'i32, 501, + 0x0A690'i32, 501, + 0x0A692'i32, 501, + 0x0A694'i32, 501, + 0x0A696'i32, 501, + 0x0A698'i32, 501, + 0x0A69A'i32, 501, + 0x0A722'i32, 501, + 0x0A724'i32, 501, + 0x0A726'i32, 501, + 0x0A728'i32, 501, + 0x0A72A'i32, 501, + 0x0A72C'i32, 501, + 0x0A72E'i32, 501, + 0x0A732'i32, 501, + 0x0A734'i32, 501, + 0x0A736'i32, 501, + 0x0A738'i32, 501, + 0x0A73A'i32, 501, + 0x0A73C'i32, 501, + 0x0A73E'i32, 501, + 0x0A740'i32, 501, + 0x0A742'i32, 501, + 0x0A744'i32, 501, + 0x0A746'i32, 501, + 0x0A748'i32, 501, + 0x0A74A'i32, 501, + 0x0A74C'i32, 501, + 0x0A74E'i32, 501, + 0x0A750'i32, 501, + 0x0A752'i32, 501, + 0x0A754'i32, 501, + 0x0A756'i32, 501, + 0x0A758'i32, 501, + 0x0A75A'i32, 501, + 0x0A75C'i32, 501, + 0x0A75E'i32, 501, + 0x0A760'i32, 501, + 0x0A762'i32, 501, + 0x0A764'i32, 501, + 0x0A766'i32, 501, + 0x0A768'i32, 501, + 0x0A76A'i32, 501, + 0x0A76C'i32, 501, + 0x0A76E'i32, 501, + 0x0A779'i32, 501, + 0x0A77B'i32, 501, + 0x0A77D'i32, -34832, + 0x0A77E'i32, 501, + 0x0A780'i32, 501, + 0x0A782'i32, 501, + 0x0A784'i32, 501, + 0x0A786'i32, 501, + 0x0A78B'i32, 501, + 0x0A78D'i32, -41780, + 0x0A790'i32, 501, + 0x0A792'i32, 501, + 0x0A796'i32, 501, + 0x0A798'i32, 501, + 0x0A79A'i32, 501, + 0x0A79C'i32, 501, + 0x0A79E'i32, 501, + 0x0A7A0'i32, 501, + 0x0A7A2'i32, 501, + 0x0A7A4'i32, 501, + 0x0A7A6'i32, 501, + 0x0A7A8'i32, 501, + 0x0A7AA'i32, -41808, + 0x0A7AB'i32, -41819, + 0x0A7AC'i32, -41815, + 0x0A7AD'i32, -41805, + 0x0A7AE'i32, -41808, + 0x0A7B0'i32, -41758, + 0x0A7B1'i32, -41782, + 0x0A7B2'i32, -41761, + 0x0A7B3'i32, 1428, + 0x0A7B4'i32, 501, + 0x0A7B6'i32, 501, + 0x0A7B8'i32, 501, + 0x0A7BA'i32, 501, + 0x0A7BC'i32, 501, + 0x0A7BE'i32, 501, + 0x0A7C2'i32, 501, + 0x0A7C4'i32, 452, + 0x0A7C5'i32, -41807, + 0x0A7C6'i32, -34884, ] toUpperRanges = [ - 0x00061, 0x0007A, 468, - 0x000E0, 0x000F6, 468, - 0x000F8, 0x000FE, 468, - 0x0023F, 0x00240, 11315, - 0x00256, 0x00257, 295, - 0x0028A, 0x0028B, 283, - 0x0037B, 0x0037D, 630, - 0x003AD, 0x003AF, 463, - 0x003B1, 0x003C1, 468, - 0x003C3, 0x003CB, 468, - 0x003CD, 0x003CE, 437, - 0x00430, 0x0044F, 468, - 0x00450, 0x0045F, 420, - 0x00561, 0x00586, 452, - 0x010D0, 0x010FA, 3508, - 0x010FD, 0x010FF, 3508, - 0x013F8, 0x013FD, 492, - 0x01C83, 0x01C84, -5742, - 0x01F00, 0x01F07, 508, - 0x01F10, 0x01F15, 508, - 0x01F20, 0x01F27, 508, - 0x01F30, 0x01F37, 508, - 0x01F40, 0x01F45, 508, - 0x01F60, 0x01F67, 508, - 0x01F70, 0x01F71, 574, - 0x01F72, 0x01F75, 586, - 0x01F76, 0x01F77, 600, - 0x01F78, 0x01F79, 628, - 0x01F7A, 0x01F7B, 612, - 0x01F7C, 0x01F7D, 626, - 0x01F80, 0x01F87, 508, - 0x01F90, 0x01F97, 508, - 0x01FA0, 0x01FA7, 508, - 0x01FB0, 0x01FB1, 508, - 0x01FD0, 0x01FD1, 508, - 0x01FE0, 0x01FE1, 508, - 0x02C30, 0x02C5E, 452, - 0x02D00, 0x02D25, -6764, - 0x0AB70, 0x0ABBF, -38364, - 0x0FF41, 0x0FF5A, 468, - 0x10428, 0x1044F, 460, - 0x104D8, 0x104FB, 460, - 0x10CC0, 0x10CF2, 436, - 0x118C0, 0x118DF, 468, - 0x16E60, 0x16E7F, 468, - 0x1E922, 0x1E943, 466, + 0x00061'i32, 0x0007A'i32, 468, + 0x000E0'i32, 0x000F6'i32, 468, + 0x000F8'i32, 0x000FE'i32, 468, + 0x0023F'i32, 0x00240'i32, 11315, + 0x00256'i32, 0x00257'i32, 295, + 0x0028A'i32, 0x0028B'i32, 283, + 0x0037B'i32, 0x0037D'i32, 630, + 0x003AD'i32, 0x003AF'i32, 463, + 0x003B1'i32, 0x003C1'i32, 468, + 0x003C3'i32, 0x003CB'i32, 468, + 0x003CD'i32, 0x003CE'i32, 437, + 0x00430'i32, 0x0044F'i32, 468, + 0x00450'i32, 0x0045F'i32, 420, + 0x00561'i32, 0x00586'i32, 452, + 0x010D0'i32, 0x010FA'i32, 3508, + 0x010FD'i32, 0x010FF'i32, 3508, + 0x013F8'i32, 0x013FD'i32, 492, + 0x01C83'i32, 0x01C84'i32, -5742, + 0x01F00'i32, 0x01F07'i32, 508, + 0x01F10'i32, 0x01F15'i32, 508, + 0x01F20'i32, 0x01F27'i32, 508, + 0x01F30'i32, 0x01F37'i32, 508, + 0x01F40'i32, 0x01F45'i32, 508, + 0x01F60'i32, 0x01F67'i32, 508, + 0x01F70'i32, 0x01F71'i32, 574, + 0x01F72'i32, 0x01F75'i32, 586, + 0x01F76'i32, 0x01F77'i32, 600, + 0x01F78'i32, 0x01F79'i32, 628, + 0x01F7A'i32, 0x01F7B'i32, 612, + 0x01F7C'i32, 0x01F7D'i32, 626, + 0x01F80'i32, 0x01F87'i32, 508, + 0x01F90'i32, 0x01F97'i32, 508, + 0x01FA0'i32, 0x01FA7'i32, 508, + 0x01FB0'i32, 0x01FB1'i32, 508, + 0x01FD0'i32, 0x01FD1'i32, 508, + 0x01FE0'i32, 0x01FE1'i32, 508, + 0x02C30'i32, 0x02C5E'i32, 452, + 0x02D00'i32, 0x02D25'i32, -6764, + 0x0AB70'i32, 0x0ABBF'i32, -38364, + 0x0FF41'i32, 0x0FF5A'i32, 468, + 0x10428'i32, 0x1044F'i32, 460, + 0x104D8'i32, 0x104FB'i32, 460, + 0x10CC0'i32, 0x10CF2'i32, 436, + 0x118C0'i32, 0x118DF'i32, 468, + 0x16E60'i32, 0x16E7F'i32, 468, + 0x1E922'i32, 0x1E943'i32, 466, ] toUpperSinglets = [ - 0x000B5, 1243, - 0x000FF, 621, - 0x00101, 499, - 0x00103, 499, - 0x00105, 499, - 0x00107, 499, - 0x00109, 499, - 0x0010B, 499, - 0x0010D, 499, - 0x0010F, 499, - 0x00111, 499, - 0x00113, 499, - 0x00115, 499, - 0x00117, 499, - 0x00119, 499, - 0x0011B, 499, - 0x0011D, 499, - 0x0011F, 499, - 0x00121, 499, - 0x00123, 499, - 0x00125, 499, - 0x00127, 499, - 0x00129, 499, - 0x0012B, 499, - 0x0012D, 499, - 0x0012F, 499, - 0x00131, 268, - 0x00133, 499, - 0x00135, 499, - 0x00137, 499, - 0x0013A, 499, - 0x0013C, 499, - 0x0013E, 499, - 0x00140, 499, - 0x00142, 499, - 0x00144, 499, - 0x00146, 499, - 0x00148, 499, - 0x0014B, 499, - 0x0014D, 499, - 0x0014F, 499, - 0x00151, 499, - 0x00153, 499, - 0x00155, 499, - 0x00157, 499, - 0x00159, 499, - 0x0015B, 499, - 0x0015D, 499, - 0x0015F, 499, - 0x00161, 499, - 0x00163, 499, - 0x00165, 499, - 0x00167, 499, - 0x00169, 499, - 0x0016B, 499, - 0x0016D, 499, - 0x0016F, 499, - 0x00171, 499, - 0x00173, 499, - 0x00175, 499, - 0x00177, 499, - 0x0017A, 499, - 0x0017C, 499, - 0x0017E, 499, - 0x0017F, 200, - 0x00180, 695, - 0x00183, 499, - 0x00185, 499, - 0x00188, 499, - 0x0018C, 499, - 0x00192, 499, - 0x00195, 597, - 0x00199, 499, - 0x0019A, 663, - 0x0019E, 630, - 0x001A1, 499, - 0x001A3, 499, - 0x001A5, 499, - 0x001A8, 499, - 0x001AD, 499, - 0x001B0, 499, - 0x001B4, 499, - 0x001B6, 499, - 0x001B9, 499, - 0x001BD, 499, - 0x001BF, 556, - 0x001C5, 499, - 0x001C6, 498, - 0x001C8, 499, - 0x001C9, 498, - 0x001CB, 499, - 0x001CC, 498, - 0x001CE, 499, - 0x001D0, 499, - 0x001D2, 499, - 0x001D4, 499, - 0x001D6, 499, - 0x001D8, 499, - 0x001DA, 499, - 0x001DC, 499, - 0x001DD, 421, - 0x001DF, 499, - 0x001E1, 499, - 0x001E3, 499, - 0x001E5, 499, - 0x001E7, 499, - 0x001E9, 499, - 0x001EB, 499, - 0x001ED, 499, - 0x001EF, 499, - 0x001F2, 499, - 0x001F3, 498, - 0x001F5, 499, - 0x001F9, 499, - 0x001FB, 499, - 0x001FD, 499, - 0x001FF, 499, - 0x00201, 499, - 0x00203, 499, - 0x00205, 499, - 0x00207, 499, - 0x00209, 499, - 0x0020B, 499, - 0x0020D, 499, - 0x0020F, 499, - 0x00211, 499, - 0x00213, 499, - 0x00215, 499, - 0x00217, 499, - 0x00219, 499, - 0x0021B, 499, - 0x0021D, 499, - 0x0021F, 499, - 0x00223, 499, - 0x00225, 499, - 0x00227, 499, - 0x00229, 499, - 0x0022B, 499, - 0x0022D, 499, - 0x0022F, 499, - 0x00231, 499, - 0x00233, 499, - 0x0023C, 499, - 0x00242, 499, - 0x00247, 499, - 0x00249, 499, - 0x0024B, 499, - 0x0024D, 499, - 0x0024F, 499, - 0x00250, 11283, - 0x00251, 11280, - 0x00252, 11282, - 0x00253, 290, - 0x00254, 294, - 0x00259, 298, - 0x0025B, 297, - 0x0025C, 42819, - 0x00260, 295, - 0x00261, 42815, - 0x00263, 293, - 0x00265, 42780, - 0x00266, 42808, - 0x00268, 291, - 0x00269, 289, - 0x0026A, 42808, - 0x0026B, 11243, - 0x0026C, 42805, - 0x0026F, 289, - 0x00271, 11249, - 0x00272, 287, - 0x00275, 286, - 0x0027D, 11227, - 0x00280, 282, - 0x00282, 42807, - 0x00283, 282, - 0x00287, 42782, - 0x00288, 282, - 0x00289, 431, - 0x0028C, 429, - 0x00292, 281, - 0x0029D, 42761, - 0x0029E, 42758, - 0x00371, 499, - 0x00373, 499, - 0x00377, 499, - 0x003AC, 462, - 0x003C2, 469, - 0x003CC, 436, - 0x003D0, 438, - 0x003D1, 443, - 0x003D5, 453, - 0x003D6, 446, - 0x003D7, 492, - 0x003D9, 499, - 0x003DB, 499, - 0x003DD, 499, - 0x003DF, 499, - 0x003E1, 499, - 0x003E3, 499, - 0x003E5, 499, - 0x003E7, 499, - 0x003E9, 499, - 0x003EB, 499, - 0x003ED, 499, - 0x003EF, 499, - 0x003F0, 414, - 0x003F1, 420, - 0x003F2, 507, - 0x003F3, 384, - 0x003F5, 404, - 0x003F8, 499, - 0x003FB, 499, - 0x00461, 499, - 0x00463, 499, - 0x00465, 499, - 0x00467, 499, - 0x00469, 499, - 0x0046B, 499, - 0x0046D, 499, - 0x0046F, 499, - 0x00471, 499, - 0x00473, 499, - 0x00475, 499, - 0x00477, 499, - 0x00479, 499, - 0x0047B, 499, - 0x0047D, 499, - 0x0047F, 499, - 0x00481, 499, - 0x0048B, 499, - 0x0048D, 499, - 0x0048F, 499, - 0x00491, 499, - 0x00493, 499, - 0x00495, 499, - 0x00497, 499, - 0x00499, 499, - 0x0049B, 499, - 0x0049D, 499, - 0x0049F, 499, - 0x004A1, 499, - 0x004A3, 499, - 0x004A5, 499, - 0x004A7, 499, - 0x004A9, 499, - 0x004AB, 499, - 0x004AD, 499, - 0x004AF, 499, - 0x004B1, 499, - 0x004B3, 499, - 0x004B5, 499, - 0x004B7, 499, - 0x004B9, 499, - 0x004BB, 499, - 0x004BD, 499, - 0x004BF, 499, - 0x004C2, 499, - 0x004C4, 499, - 0x004C6, 499, - 0x004C8, 499, - 0x004CA, 499, - 0x004CC, 499, - 0x004CE, 499, - 0x004CF, 485, - 0x004D1, 499, - 0x004D3, 499, - 0x004D5, 499, - 0x004D7, 499, - 0x004D9, 499, - 0x004DB, 499, - 0x004DD, 499, - 0x004DF, 499, - 0x004E1, 499, - 0x004E3, 499, - 0x004E5, 499, - 0x004E7, 499, - 0x004E9, 499, - 0x004EB, 499, - 0x004ED, 499, - 0x004EF, 499, - 0x004F1, 499, - 0x004F3, 499, - 0x004F5, 499, - 0x004F7, 499, - 0x004F9, 499, - 0x004FB, 499, - 0x004FD, 499, - 0x004FF, 499, - 0x00501, 499, - 0x00503, 499, - 0x00505, 499, - 0x00507, 499, - 0x00509, 499, - 0x0050B, 499, - 0x0050D, 499, - 0x0050F, 499, - 0x00511, 499, - 0x00513, 499, - 0x00515, 499, - 0x00517, 499, - 0x00519, 499, - 0x0051B, 499, - 0x0051D, 499, - 0x0051F, 499, - 0x00521, 499, - 0x00523, 499, - 0x00525, 499, - 0x00527, 499, - 0x00529, 499, - 0x0052B, 499, - 0x0052D, 499, - 0x0052F, 499, - 0x01C80, -5754, - 0x01C81, -5753, - 0x01C82, -5744, - 0x01C85, -5743, - 0x01C86, -5736, - 0x01C87, -5681, - 0x01C88, 35766, - 0x01D79, 35832, - 0x01D7D, 4314, - 0x01D8E, 35884, - 0x01E01, 499, - 0x01E03, 499, - 0x01E05, 499, - 0x01E07, 499, - 0x01E09, 499, - 0x01E0B, 499, - 0x01E0D, 499, - 0x01E0F, 499, - 0x01E11, 499, - 0x01E13, 499, - 0x01E15, 499, - 0x01E17, 499, - 0x01E19, 499, - 0x01E1B, 499, - 0x01E1D, 499, - 0x01E1F, 499, - 0x01E21, 499, - 0x01E23, 499, - 0x01E25, 499, - 0x01E27, 499, - 0x01E29, 499, - 0x01E2B, 499, - 0x01E2D, 499, - 0x01E2F, 499, - 0x01E31, 499, - 0x01E33, 499, - 0x01E35, 499, - 0x01E37, 499, - 0x01E39, 499, - 0x01E3B, 499, - 0x01E3D, 499, - 0x01E3F, 499, - 0x01E41, 499, - 0x01E43, 499, - 0x01E45, 499, - 0x01E47, 499, - 0x01E49, 499, - 0x01E4B, 499, - 0x01E4D, 499, - 0x01E4F, 499, - 0x01E51, 499, - 0x01E53, 499, - 0x01E55, 499, - 0x01E57, 499, - 0x01E59, 499, - 0x01E5B, 499, - 0x01E5D, 499, - 0x01E5F, 499, - 0x01E61, 499, - 0x01E63, 499, - 0x01E65, 499, - 0x01E67, 499, - 0x01E69, 499, - 0x01E6B, 499, - 0x01E6D, 499, - 0x01E6F, 499, - 0x01E71, 499, - 0x01E73, 499, - 0x01E75, 499, - 0x01E77, 499, - 0x01E79, 499, - 0x01E7B, 499, - 0x01E7D, 499, - 0x01E7F, 499, - 0x01E81, 499, - 0x01E83, 499, - 0x01E85, 499, - 0x01E87, 499, - 0x01E89, 499, - 0x01E8B, 499, - 0x01E8D, 499, - 0x01E8F, 499, - 0x01E91, 499, - 0x01E93, 499, - 0x01E95, 499, - 0x01E9B, 441, - 0x01EA1, 499, - 0x01EA3, 499, - 0x01EA5, 499, - 0x01EA7, 499, - 0x01EA9, 499, - 0x01EAB, 499, - 0x01EAD, 499, - 0x01EAF, 499, - 0x01EB1, 499, - 0x01EB3, 499, - 0x01EB5, 499, - 0x01EB7, 499, - 0x01EB9, 499, - 0x01EBB, 499, - 0x01EBD, 499, - 0x01EBF, 499, - 0x01EC1, 499, - 0x01EC3, 499, - 0x01EC5, 499, - 0x01EC7, 499, - 0x01EC9, 499, - 0x01ECB, 499, - 0x01ECD, 499, - 0x01ECF, 499, - 0x01ED1, 499, - 0x01ED3, 499, - 0x01ED5, 499, - 0x01ED7, 499, - 0x01ED9, 499, - 0x01EDB, 499, - 0x01EDD, 499, - 0x01EDF, 499, - 0x01EE1, 499, - 0x01EE3, 499, - 0x01EE5, 499, - 0x01EE7, 499, - 0x01EE9, 499, - 0x01EEB, 499, - 0x01EED, 499, - 0x01EEF, 499, - 0x01EF1, 499, - 0x01EF3, 499, - 0x01EF5, 499, - 0x01EF7, 499, - 0x01EF9, 499, - 0x01EFB, 499, - 0x01EFD, 499, - 0x01EFF, 499, - 0x01F51, 508, - 0x01F53, 508, - 0x01F55, 508, - 0x01F57, 508, - 0x01FB3, 509, - 0x01FBE, -6705, - 0x01FC3, 509, - 0x01FE5, 507, - 0x01FF3, 509, - 0x0214E, 472, - 0x02184, 499, - 0x02C61, 499, - 0x02C65, -10295, - 0x02C66, -10292, - 0x02C68, 499, - 0x02C6A, 499, - 0x02C6C, 499, - 0x02C73, 499, - 0x02C76, 499, - 0x02C81, 499, - 0x02C83, 499, - 0x02C85, 499, - 0x02C87, 499, - 0x02C89, 499, - 0x02C8B, 499, - 0x02C8D, 499, - 0x02C8F, 499, - 0x02C91, 499, - 0x02C93, 499, - 0x02C95, 499, - 0x02C97, 499, - 0x02C99, 499, - 0x02C9B, 499, - 0x02C9D, 499, - 0x02C9F, 499, - 0x02CA1, 499, - 0x02CA3, 499, - 0x02CA5, 499, - 0x02CA7, 499, - 0x02CA9, 499, - 0x02CAB, 499, - 0x02CAD, 499, - 0x02CAF, 499, - 0x02CB1, 499, - 0x02CB3, 499, - 0x02CB5, 499, - 0x02CB7, 499, - 0x02CB9, 499, - 0x02CBB, 499, - 0x02CBD, 499, - 0x02CBF, 499, - 0x02CC1, 499, - 0x02CC3, 499, - 0x02CC5, 499, - 0x02CC7, 499, - 0x02CC9, 499, - 0x02CCB, 499, - 0x02CCD, 499, - 0x02CCF, 499, - 0x02CD1, 499, - 0x02CD3, 499, - 0x02CD5, 499, - 0x02CD7, 499, - 0x02CD9, 499, - 0x02CDB, 499, - 0x02CDD, 499, - 0x02CDF, 499, - 0x02CE1, 499, - 0x02CE3, 499, - 0x02CEC, 499, - 0x02CEE, 499, - 0x02CF3, 499, - 0x02D27, -6764, - 0x02D2D, -6764, - 0x0A641, 499, - 0x0A643, 499, - 0x0A645, 499, - 0x0A647, 499, - 0x0A649, 499, - 0x0A64B, 499, - 0x0A64D, 499, - 0x0A64F, 499, - 0x0A651, 499, - 0x0A653, 499, - 0x0A655, 499, - 0x0A657, 499, - 0x0A659, 499, - 0x0A65B, 499, - 0x0A65D, 499, - 0x0A65F, 499, - 0x0A661, 499, - 0x0A663, 499, - 0x0A665, 499, - 0x0A667, 499, - 0x0A669, 499, - 0x0A66B, 499, - 0x0A66D, 499, - 0x0A681, 499, - 0x0A683, 499, - 0x0A685, 499, - 0x0A687, 499, - 0x0A689, 499, - 0x0A68B, 499, - 0x0A68D, 499, - 0x0A68F, 499, - 0x0A691, 499, - 0x0A693, 499, - 0x0A695, 499, - 0x0A697, 499, - 0x0A699, 499, - 0x0A69B, 499, - 0x0A723, 499, - 0x0A725, 499, - 0x0A727, 499, - 0x0A729, 499, - 0x0A72B, 499, - 0x0A72D, 499, - 0x0A72F, 499, - 0x0A733, 499, - 0x0A735, 499, - 0x0A737, 499, - 0x0A739, 499, - 0x0A73B, 499, - 0x0A73D, 499, - 0x0A73F, 499, - 0x0A741, 499, - 0x0A743, 499, - 0x0A745, 499, - 0x0A747, 499, - 0x0A749, 499, - 0x0A74B, 499, - 0x0A74D, 499, - 0x0A74F, 499, - 0x0A751, 499, - 0x0A753, 499, - 0x0A755, 499, - 0x0A757, 499, - 0x0A759, 499, - 0x0A75B, 499, - 0x0A75D, 499, - 0x0A75F, 499, - 0x0A761, 499, - 0x0A763, 499, - 0x0A765, 499, - 0x0A767, 499, - 0x0A769, 499, - 0x0A76B, 499, - 0x0A76D, 499, - 0x0A76F, 499, - 0x0A77A, 499, - 0x0A77C, 499, - 0x0A77F, 499, - 0x0A781, 499, - 0x0A783, 499, - 0x0A785, 499, - 0x0A787, 499, - 0x0A78C, 499, - 0x0A791, 499, - 0x0A793, 499, - 0x0A794, 548, - 0x0A797, 499, - 0x0A799, 499, - 0x0A79B, 499, - 0x0A79D, 499, - 0x0A79F, 499, - 0x0A7A1, 499, - 0x0A7A3, 499, - 0x0A7A5, 499, - 0x0A7A7, 499, - 0x0A7A9, 499, - 0x0A7B5, 499, - 0x0A7B7, 499, - 0x0A7B9, 499, - 0x0A7BB, 499, - 0x0A7BD, 499, - 0x0A7BF, 499, - 0x0A7C3, 499, - 0x0AB53, -428, + 0x000B5'i32, 1243, + 0x000FF'i32, 621, + 0x00101'i32, 499, + 0x00103'i32, 499, + 0x00105'i32, 499, + 0x00107'i32, 499, + 0x00109'i32, 499, + 0x0010B'i32, 499, + 0x0010D'i32, 499, + 0x0010F'i32, 499, + 0x00111'i32, 499, + 0x00113'i32, 499, + 0x00115'i32, 499, + 0x00117'i32, 499, + 0x00119'i32, 499, + 0x0011B'i32, 499, + 0x0011D'i32, 499, + 0x0011F'i32, 499, + 0x00121'i32, 499, + 0x00123'i32, 499, + 0x00125'i32, 499, + 0x00127'i32, 499, + 0x00129'i32, 499, + 0x0012B'i32, 499, + 0x0012D'i32, 499, + 0x0012F'i32, 499, + 0x00131'i32, 268, + 0x00133'i32, 499, + 0x00135'i32, 499, + 0x00137'i32, 499, + 0x0013A'i32, 499, + 0x0013C'i32, 499, + 0x0013E'i32, 499, + 0x00140'i32, 499, + 0x00142'i32, 499, + 0x00144'i32, 499, + 0x00146'i32, 499, + 0x00148'i32, 499, + 0x0014B'i32, 499, + 0x0014D'i32, 499, + 0x0014F'i32, 499, + 0x00151'i32, 499, + 0x00153'i32, 499, + 0x00155'i32, 499, + 0x00157'i32, 499, + 0x00159'i32, 499, + 0x0015B'i32, 499, + 0x0015D'i32, 499, + 0x0015F'i32, 499, + 0x00161'i32, 499, + 0x00163'i32, 499, + 0x00165'i32, 499, + 0x00167'i32, 499, + 0x00169'i32, 499, + 0x0016B'i32, 499, + 0x0016D'i32, 499, + 0x0016F'i32, 499, + 0x00171'i32, 499, + 0x00173'i32, 499, + 0x00175'i32, 499, + 0x00177'i32, 499, + 0x0017A'i32, 499, + 0x0017C'i32, 499, + 0x0017E'i32, 499, + 0x0017F'i32, 200, + 0x00180'i32, 695, + 0x00183'i32, 499, + 0x00185'i32, 499, + 0x00188'i32, 499, + 0x0018C'i32, 499, + 0x00192'i32, 499, + 0x00195'i32, 597, + 0x00199'i32, 499, + 0x0019A'i32, 663, + 0x0019E'i32, 630, + 0x001A1'i32, 499, + 0x001A3'i32, 499, + 0x001A5'i32, 499, + 0x001A8'i32, 499, + 0x001AD'i32, 499, + 0x001B0'i32, 499, + 0x001B4'i32, 499, + 0x001B6'i32, 499, + 0x001B9'i32, 499, + 0x001BD'i32, 499, + 0x001BF'i32, 556, + 0x001C5'i32, 499, + 0x001C6'i32, 498, + 0x001C8'i32, 499, + 0x001C9'i32, 498, + 0x001CB'i32, 499, + 0x001CC'i32, 498, + 0x001CE'i32, 499, + 0x001D0'i32, 499, + 0x001D2'i32, 499, + 0x001D4'i32, 499, + 0x001D6'i32, 499, + 0x001D8'i32, 499, + 0x001DA'i32, 499, + 0x001DC'i32, 499, + 0x001DD'i32, 421, + 0x001DF'i32, 499, + 0x001E1'i32, 499, + 0x001E3'i32, 499, + 0x001E5'i32, 499, + 0x001E7'i32, 499, + 0x001E9'i32, 499, + 0x001EB'i32, 499, + 0x001ED'i32, 499, + 0x001EF'i32, 499, + 0x001F2'i32, 499, + 0x001F3'i32, 498, + 0x001F5'i32, 499, + 0x001F9'i32, 499, + 0x001FB'i32, 499, + 0x001FD'i32, 499, + 0x001FF'i32, 499, + 0x00201'i32, 499, + 0x00203'i32, 499, + 0x00205'i32, 499, + 0x00207'i32, 499, + 0x00209'i32, 499, + 0x0020B'i32, 499, + 0x0020D'i32, 499, + 0x0020F'i32, 499, + 0x00211'i32, 499, + 0x00213'i32, 499, + 0x00215'i32, 499, + 0x00217'i32, 499, + 0x00219'i32, 499, + 0x0021B'i32, 499, + 0x0021D'i32, 499, + 0x0021F'i32, 499, + 0x00223'i32, 499, + 0x00225'i32, 499, + 0x00227'i32, 499, + 0x00229'i32, 499, + 0x0022B'i32, 499, + 0x0022D'i32, 499, + 0x0022F'i32, 499, + 0x00231'i32, 499, + 0x00233'i32, 499, + 0x0023C'i32, 499, + 0x00242'i32, 499, + 0x00247'i32, 499, + 0x00249'i32, 499, + 0x0024B'i32, 499, + 0x0024D'i32, 499, + 0x0024F'i32, 499, + 0x00250'i32, 11283, + 0x00251'i32, 11280, + 0x00252'i32, 11282, + 0x00253'i32, 290, + 0x00254'i32, 294, + 0x00259'i32, 298, + 0x0025B'i32, 297, + 0x0025C'i32, 42819, + 0x00260'i32, 295, + 0x00261'i32, 42815, + 0x00263'i32, 293, + 0x00265'i32, 42780, + 0x00266'i32, 42808, + 0x00268'i32, 291, + 0x00269'i32, 289, + 0x0026A'i32, 42808, + 0x0026B'i32, 11243, + 0x0026C'i32, 42805, + 0x0026F'i32, 289, + 0x00271'i32, 11249, + 0x00272'i32, 287, + 0x00275'i32, 286, + 0x0027D'i32, 11227, + 0x00280'i32, 282, + 0x00282'i32, 42807, + 0x00283'i32, 282, + 0x00287'i32, 42782, + 0x00288'i32, 282, + 0x00289'i32, 431, + 0x0028C'i32, 429, + 0x00292'i32, 281, + 0x0029D'i32, 42761, + 0x0029E'i32, 42758, + 0x00371'i32, 499, + 0x00373'i32, 499, + 0x00377'i32, 499, + 0x003AC'i32, 462, + 0x003C2'i32, 469, + 0x003CC'i32, 436, + 0x003D0'i32, 438, + 0x003D1'i32, 443, + 0x003D5'i32, 453, + 0x003D6'i32, 446, + 0x003D7'i32, 492, + 0x003D9'i32, 499, + 0x003DB'i32, 499, + 0x003DD'i32, 499, + 0x003DF'i32, 499, + 0x003E1'i32, 499, + 0x003E3'i32, 499, + 0x003E5'i32, 499, + 0x003E7'i32, 499, + 0x003E9'i32, 499, + 0x003EB'i32, 499, + 0x003ED'i32, 499, + 0x003EF'i32, 499, + 0x003F0'i32, 414, + 0x003F1'i32, 420, + 0x003F2'i32, 507, + 0x003F3'i32, 384, + 0x003F5'i32, 404, + 0x003F8'i32, 499, + 0x003FB'i32, 499, + 0x00461'i32, 499, + 0x00463'i32, 499, + 0x00465'i32, 499, + 0x00467'i32, 499, + 0x00469'i32, 499, + 0x0046B'i32, 499, + 0x0046D'i32, 499, + 0x0046F'i32, 499, + 0x00471'i32, 499, + 0x00473'i32, 499, + 0x00475'i32, 499, + 0x00477'i32, 499, + 0x00479'i32, 499, + 0x0047B'i32, 499, + 0x0047D'i32, 499, + 0x0047F'i32, 499, + 0x00481'i32, 499, + 0x0048B'i32, 499, + 0x0048D'i32, 499, + 0x0048F'i32, 499, + 0x00491'i32, 499, + 0x00493'i32, 499, + 0x00495'i32, 499, + 0x00497'i32, 499, + 0x00499'i32, 499, + 0x0049B'i32, 499, + 0x0049D'i32, 499, + 0x0049F'i32, 499, + 0x004A1'i32, 499, + 0x004A3'i32, 499, + 0x004A5'i32, 499, + 0x004A7'i32, 499, + 0x004A9'i32, 499, + 0x004AB'i32, 499, + 0x004AD'i32, 499, + 0x004AF'i32, 499, + 0x004B1'i32, 499, + 0x004B3'i32, 499, + 0x004B5'i32, 499, + 0x004B7'i32, 499, + 0x004B9'i32, 499, + 0x004BB'i32, 499, + 0x004BD'i32, 499, + 0x004BF'i32, 499, + 0x004C2'i32, 499, + 0x004C4'i32, 499, + 0x004C6'i32, 499, + 0x004C8'i32, 499, + 0x004CA'i32, 499, + 0x004CC'i32, 499, + 0x004CE'i32, 499, + 0x004CF'i32, 485, + 0x004D1'i32, 499, + 0x004D3'i32, 499, + 0x004D5'i32, 499, + 0x004D7'i32, 499, + 0x004D9'i32, 499, + 0x004DB'i32, 499, + 0x004DD'i32, 499, + 0x004DF'i32, 499, + 0x004E1'i32, 499, + 0x004E3'i32, 499, + 0x004E5'i32, 499, + 0x004E7'i32, 499, + 0x004E9'i32, 499, + 0x004EB'i32, 499, + 0x004ED'i32, 499, + 0x004EF'i32, 499, + 0x004F1'i32, 499, + 0x004F3'i32, 499, + 0x004F5'i32, 499, + 0x004F7'i32, 499, + 0x004F9'i32, 499, + 0x004FB'i32, 499, + 0x004FD'i32, 499, + 0x004FF'i32, 499, + 0x00501'i32, 499, + 0x00503'i32, 499, + 0x00505'i32, 499, + 0x00507'i32, 499, + 0x00509'i32, 499, + 0x0050B'i32, 499, + 0x0050D'i32, 499, + 0x0050F'i32, 499, + 0x00511'i32, 499, + 0x00513'i32, 499, + 0x00515'i32, 499, + 0x00517'i32, 499, + 0x00519'i32, 499, + 0x0051B'i32, 499, + 0x0051D'i32, 499, + 0x0051F'i32, 499, + 0x00521'i32, 499, + 0x00523'i32, 499, + 0x00525'i32, 499, + 0x00527'i32, 499, + 0x00529'i32, 499, + 0x0052B'i32, 499, + 0x0052D'i32, 499, + 0x0052F'i32, 499, + 0x01C80'i32, -5754, + 0x01C81'i32, -5753, + 0x01C82'i32, -5744, + 0x01C85'i32, -5743, + 0x01C86'i32, -5736, + 0x01C87'i32, -5681, + 0x01C88'i32, 35766, + 0x01D79'i32, 35832, + 0x01D7D'i32, 4314, + 0x01D8E'i32, 35884, + 0x01E01'i32, 499, + 0x01E03'i32, 499, + 0x01E05'i32, 499, + 0x01E07'i32, 499, + 0x01E09'i32, 499, + 0x01E0B'i32, 499, + 0x01E0D'i32, 499, + 0x01E0F'i32, 499, + 0x01E11'i32, 499, + 0x01E13'i32, 499, + 0x01E15'i32, 499, + 0x01E17'i32, 499, + 0x01E19'i32, 499, + 0x01E1B'i32, 499, + 0x01E1D'i32, 499, + 0x01E1F'i32, 499, + 0x01E21'i32, 499, + 0x01E23'i32, 499, + 0x01E25'i32, 499, + 0x01E27'i32, 499, + 0x01E29'i32, 499, + 0x01E2B'i32, 499, + 0x01E2D'i32, 499, + 0x01E2F'i32, 499, + 0x01E31'i32, 499, + 0x01E33'i32, 499, + 0x01E35'i32, 499, + 0x01E37'i32, 499, + 0x01E39'i32, 499, + 0x01E3B'i32, 499, + 0x01E3D'i32, 499, + 0x01E3F'i32, 499, + 0x01E41'i32, 499, + 0x01E43'i32, 499, + 0x01E45'i32, 499, + 0x01E47'i32, 499, + 0x01E49'i32, 499, + 0x01E4B'i32, 499, + 0x01E4D'i32, 499, + 0x01E4F'i32, 499, + 0x01E51'i32, 499, + 0x01E53'i32, 499, + 0x01E55'i32, 499, + 0x01E57'i32, 499, + 0x01E59'i32, 499, + 0x01E5B'i32, 499, + 0x01E5D'i32, 499, + 0x01E5F'i32, 499, + 0x01E61'i32, 499, + 0x01E63'i32, 499, + 0x01E65'i32, 499, + 0x01E67'i32, 499, + 0x01E69'i32, 499, + 0x01E6B'i32, 499, + 0x01E6D'i32, 499, + 0x01E6F'i32, 499, + 0x01E71'i32, 499, + 0x01E73'i32, 499, + 0x01E75'i32, 499, + 0x01E77'i32, 499, + 0x01E79'i32, 499, + 0x01E7B'i32, 499, + 0x01E7D'i32, 499, + 0x01E7F'i32, 499, + 0x01E81'i32, 499, + 0x01E83'i32, 499, + 0x01E85'i32, 499, + 0x01E87'i32, 499, + 0x01E89'i32, 499, + 0x01E8B'i32, 499, + 0x01E8D'i32, 499, + 0x01E8F'i32, 499, + 0x01E91'i32, 499, + 0x01E93'i32, 499, + 0x01E95'i32, 499, + 0x01E9B'i32, 441, + 0x01EA1'i32, 499, + 0x01EA3'i32, 499, + 0x01EA5'i32, 499, + 0x01EA7'i32, 499, + 0x01EA9'i32, 499, + 0x01EAB'i32, 499, + 0x01EAD'i32, 499, + 0x01EAF'i32, 499, + 0x01EB1'i32, 499, + 0x01EB3'i32, 499, + 0x01EB5'i32, 499, + 0x01EB7'i32, 499, + 0x01EB9'i32, 499, + 0x01EBB'i32, 499, + 0x01EBD'i32, 499, + 0x01EBF'i32, 499, + 0x01EC1'i32, 499, + 0x01EC3'i32, 499, + 0x01EC5'i32, 499, + 0x01EC7'i32, 499, + 0x01EC9'i32, 499, + 0x01ECB'i32, 499, + 0x01ECD'i32, 499, + 0x01ECF'i32, 499, + 0x01ED1'i32, 499, + 0x01ED3'i32, 499, + 0x01ED5'i32, 499, + 0x01ED7'i32, 499, + 0x01ED9'i32, 499, + 0x01EDB'i32, 499, + 0x01EDD'i32, 499, + 0x01EDF'i32, 499, + 0x01EE1'i32, 499, + 0x01EE3'i32, 499, + 0x01EE5'i32, 499, + 0x01EE7'i32, 499, + 0x01EE9'i32, 499, + 0x01EEB'i32, 499, + 0x01EED'i32, 499, + 0x01EEF'i32, 499, + 0x01EF1'i32, 499, + 0x01EF3'i32, 499, + 0x01EF5'i32, 499, + 0x01EF7'i32, 499, + 0x01EF9'i32, 499, + 0x01EFB'i32, 499, + 0x01EFD'i32, 499, + 0x01EFF'i32, 499, + 0x01F51'i32, 508, + 0x01F53'i32, 508, + 0x01F55'i32, 508, + 0x01F57'i32, 508, + 0x01FB3'i32, 509, + 0x01FBE'i32, -6705, + 0x01FC3'i32, 509, + 0x01FE5'i32, 507, + 0x01FF3'i32, 509, + 0x0214E'i32, 472, + 0x02184'i32, 499, + 0x02C61'i32, 499, + 0x02C65'i32, -10295, + 0x02C66'i32, -10292, + 0x02C68'i32, 499, + 0x02C6A'i32, 499, + 0x02C6C'i32, 499, + 0x02C73'i32, 499, + 0x02C76'i32, 499, + 0x02C81'i32, 499, + 0x02C83'i32, 499, + 0x02C85'i32, 499, + 0x02C87'i32, 499, + 0x02C89'i32, 499, + 0x02C8B'i32, 499, + 0x02C8D'i32, 499, + 0x02C8F'i32, 499, + 0x02C91'i32, 499, + 0x02C93'i32, 499, + 0x02C95'i32, 499, + 0x02C97'i32, 499, + 0x02C99'i32, 499, + 0x02C9B'i32, 499, + 0x02C9D'i32, 499, + 0x02C9F'i32, 499, + 0x02CA1'i32, 499, + 0x02CA3'i32, 499, + 0x02CA5'i32, 499, + 0x02CA7'i32, 499, + 0x02CA9'i32, 499, + 0x02CAB'i32, 499, + 0x02CAD'i32, 499, + 0x02CAF'i32, 499, + 0x02CB1'i32, 499, + 0x02CB3'i32, 499, + 0x02CB5'i32, 499, + 0x02CB7'i32, 499, + 0x02CB9'i32, 499, + 0x02CBB'i32, 499, + 0x02CBD'i32, 499, + 0x02CBF'i32, 499, + 0x02CC1'i32, 499, + 0x02CC3'i32, 499, + 0x02CC5'i32, 499, + 0x02CC7'i32, 499, + 0x02CC9'i32, 499, + 0x02CCB'i32, 499, + 0x02CCD'i32, 499, + 0x02CCF'i32, 499, + 0x02CD1'i32, 499, + 0x02CD3'i32, 499, + 0x02CD5'i32, 499, + 0x02CD7'i32, 499, + 0x02CD9'i32, 499, + 0x02CDB'i32, 499, + 0x02CDD'i32, 499, + 0x02CDF'i32, 499, + 0x02CE1'i32, 499, + 0x02CE3'i32, 499, + 0x02CEC'i32, 499, + 0x02CEE'i32, 499, + 0x02CF3'i32, 499, + 0x02D27'i32, -6764, + 0x02D2D'i32, -6764, + 0x0A641'i32, 499, + 0x0A643'i32, 499, + 0x0A645'i32, 499, + 0x0A647'i32, 499, + 0x0A649'i32, 499, + 0x0A64B'i32, 499, + 0x0A64D'i32, 499, + 0x0A64F'i32, 499, + 0x0A651'i32, 499, + 0x0A653'i32, 499, + 0x0A655'i32, 499, + 0x0A657'i32, 499, + 0x0A659'i32, 499, + 0x0A65B'i32, 499, + 0x0A65D'i32, 499, + 0x0A65F'i32, 499, + 0x0A661'i32, 499, + 0x0A663'i32, 499, + 0x0A665'i32, 499, + 0x0A667'i32, 499, + 0x0A669'i32, 499, + 0x0A66B'i32, 499, + 0x0A66D'i32, 499, + 0x0A681'i32, 499, + 0x0A683'i32, 499, + 0x0A685'i32, 499, + 0x0A687'i32, 499, + 0x0A689'i32, 499, + 0x0A68B'i32, 499, + 0x0A68D'i32, 499, + 0x0A68F'i32, 499, + 0x0A691'i32, 499, + 0x0A693'i32, 499, + 0x0A695'i32, 499, + 0x0A697'i32, 499, + 0x0A699'i32, 499, + 0x0A69B'i32, 499, + 0x0A723'i32, 499, + 0x0A725'i32, 499, + 0x0A727'i32, 499, + 0x0A729'i32, 499, + 0x0A72B'i32, 499, + 0x0A72D'i32, 499, + 0x0A72F'i32, 499, + 0x0A733'i32, 499, + 0x0A735'i32, 499, + 0x0A737'i32, 499, + 0x0A739'i32, 499, + 0x0A73B'i32, 499, + 0x0A73D'i32, 499, + 0x0A73F'i32, 499, + 0x0A741'i32, 499, + 0x0A743'i32, 499, + 0x0A745'i32, 499, + 0x0A747'i32, 499, + 0x0A749'i32, 499, + 0x0A74B'i32, 499, + 0x0A74D'i32, 499, + 0x0A74F'i32, 499, + 0x0A751'i32, 499, + 0x0A753'i32, 499, + 0x0A755'i32, 499, + 0x0A757'i32, 499, + 0x0A759'i32, 499, + 0x0A75B'i32, 499, + 0x0A75D'i32, 499, + 0x0A75F'i32, 499, + 0x0A761'i32, 499, + 0x0A763'i32, 499, + 0x0A765'i32, 499, + 0x0A767'i32, 499, + 0x0A769'i32, 499, + 0x0A76B'i32, 499, + 0x0A76D'i32, 499, + 0x0A76F'i32, 499, + 0x0A77A'i32, 499, + 0x0A77C'i32, 499, + 0x0A77F'i32, 499, + 0x0A781'i32, 499, + 0x0A783'i32, 499, + 0x0A785'i32, 499, + 0x0A787'i32, 499, + 0x0A78C'i32, 499, + 0x0A791'i32, 499, + 0x0A793'i32, 499, + 0x0A794'i32, 548, + 0x0A797'i32, 499, + 0x0A799'i32, 499, + 0x0A79B'i32, 499, + 0x0A79D'i32, 499, + 0x0A79F'i32, 499, + 0x0A7A1'i32, 499, + 0x0A7A3'i32, 499, + 0x0A7A5'i32, 499, + 0x0A7A7'i32, 499, + 0x0A7A9'i32, 499, + 0x0A7B5'i32, 499, + 0x0A7B7'i32, 499, + 0x0A7B9'i32, 499, + 0x0A7BB'i32, 499, + 0x0A7BD'i32, 499, + 0x0A7BF'i32, 499, + 0x0A7C3'i32, 499, + 0x0AB53'i32, -428, ] toTitleSinglets = [ - 0x001C4, 501, - 0x001C6, 499, - 0x001C7, 501, - 0x001C9, 499, - 0x001CA, 501, - 0x001CC, 499, - 0x001F1, 501, - 0x001F3, 499, + 0x001C4'i32, 501, + 0x001C6'i32, 499, + 0x001C7'i32, 501, + 0x001C9'i32, 499, + 0x001CA'i32, 501, + 0x001CC'i32, 499, + 0x001F1'i32, 501, + 0x001F3'i32, 499, ] alphaRanges = [ - 0x00041, 0x0005A, - 0x00061, 0x0007A, - 0x000C0, 0x000D6, - 0x000D8, 0x000F6, - 0x000F8, 0x002C1, - 0x002C6, 0x002D1, - 0x002E0, 0x002E4, - 0x00370, 0x00374, - 0x00376, 0x00377, - 0x0037A, 0x0037D, - 0x00388, 0x0038A, - 0x0038E, 0x003A1, - 0x003A3, 0x003F5, - 0x003F7, 0x00481, - 0x0048A, 0x0052F, - 0x00531, 0x00556, - 0x00560, 0x00588, - 0x005D0, 0x005EA, - 0x005EF, 0x005F2, - 0x00620, 0x0064A, - 0x0066E, 0x0066F, - 0x00671, 0x006D3, - 0x006E5, 0x006E6, - 0x006EE, 0x006EF, - 0x006FA, 0x006FC, - 0x00712, 0x0072F, - 0x0074D, 0x007A5, - 0x007CA, 0x007EA, - 0x007F4, 0x007F5, - 0x00800, 0x00815, - 0x00840, 0x00858, - 0x00860, 0x0086A, - 0x008A0, 0x008B4, - 0x008B6, 0x008BD, - 0x00904, 0x00939, - 0x00958, 0x00961, - 0x00971, 0x00980, - 0x00985, 0x0098C, - 0x0098F, 0x00990, - 0x00993, 0x009A8, - 0x009AA, 0x009B0, - 0x009B6, 0x009B9, - 0x009DC, 0x009DD, - 0x009DF, 0x009E1, - 0x009F0, 0x009F1, - 0x00A05, 0x00A0A, - 0x00A0F, 0x00A10, - 0x00A13, 0x00A28, - 0x00A2A, 0x00A30, - 0x00A32, 0x00A33, - 0x00A35, 0x00A36, - 0x00A38, 0x00A39, - 0x00A59, 0x00A5C, - 0x00A72, 0x00A74, - 0x00A85, 0x00A8D, - 0x00A8F, 0x00A91, - 0x00A93, 0x00AA8, - 0x00AAA, 0x00AB0, - 0x00AB2, 0x00AB3, - 0x00AB5, 0x00AB9, - 0x00AE0, 0x00AE1, - 0x00B05, 0x00B0C, - 0x00B0F, 0x00B10, - 0x00B13, 0x00B28, - 0x00B2A, 0x00B30, - 0x00B32, 0x00B33, - 0x00B35, 0x00B39, - 0x00B5C, 0x00B5D, - 0x00B5F, 0x00B61, - 0x00B85, 0x00B8A, - 0x00B8E, 0x00B90, - 0x00B92, 0x00B95, - 0x00B99, 0x00B9A, - 0x00B9E, 0x00B9F, - 0x00BA3, 0x00BA4, - 0x00BA8, 0x00BAA, - 0x00BAE, 0x00BB9, - 0x00C05, 0x00C0C, - 0x00C0E, 0x00C10, - 0x00C12, 0x00C28, - 0x00C2A, 0x00C39, - 0x00C58, 0x00C5A, - 0x00C60, 0x00C61, - 0x00C85, 0x00C8C, - 0x00C8E, 0x00C90, - 0x00C92, 0x00CA8, - 0x00CAA, 0x00CB3, - 0x00CB5, 0x00CB9, - 0x00CE0, 0x00CE1, - 0x00CF1, 0x00CF2, - 0x00D05, 0x00D0C, - 0x00D0E, 0x00D10, - 0x00D12, 0x00D3A, - 0x00D54, 0x00D56, - 0x00D5F, 0x00D61, - 0x00D7A, 0x00D7F, - 0x00D85, 0x00D96, - 0x00D9A, 0x00DB1, - 0x00DB3, 0x00DBB, - 0x00DC0, 0x00DC6, - 0x00E01, 0x00E30, - 0x00E32, 0x00E33, - 0x00E40, 0x00E46, - 0x00E81, 0x00E82, - 0x00E86, 0x00E8A, - 0x00E8C, 0x00EA3, - 0x00EA7, 0x00EB0, - 0x00EB2, 0x00EB3, - 0x00EC0, 0x00EC4, - 0x00EDC, 0x00EDF, - 0x00F40, 0x00F47, - 0x00F49, 0x00F6C, - 0x00F88, 0x00F8C, - 0x01000, 0x0102A, - 0x01050, 0x01055, - 0x0105A, 0x0105D, - 0x01065, 0x01066, - 0x0106E, 0x01070, - 0x01075, 0x01081, - 0x010A0, 0x010C5, - 0x010D0, 0x010FA, - 0x010FC, 0x01248, - 0x0124A, 0x0124D, - 0x01250, 0x01256, - 0x0125A, 0x0125D, - 0x01260, 0x01288, - 0x0128A, 0x0128D, - 0x01290, 0x012B0, - 0x012B2, 0x012B5, - 0x012B8, 0x012BE, - 0x012C2, 0x012C5, - 0x012C8, 0x012D6, - 0x012D8, 0x01310, - 0x01312, 0x01315, - 0x01318, 0x0135A, - 0x01380, 0x0138F, - 0x013A0, 0x013F5, - 0x013F8, 0x013FD, - 0x01401, 0x0166C, - 0x0166F, 0x0167F, - 0x01681, 0x0169A, - 0x016A0, 0x016EA, - 0x016F1, 0x016F8, - 0x01700, 0x0170C, - 0x0170E, 0x01711, - 0x01720, 0x01731, - 0x01740, 0x01751, - 0x01760, 0x0176C, - 0x0176E, 0x01770, - 0x01780, 0x017B3, - 0x01820, 0x01878, - 0x01880, 0x01884, - 0x01887, 0x018A8, - 0x018B0, 0x018F5, - 0x01900, 0x0191E, - 0x01950, 0x0196D, - 0x01970, 0x01974, - 0x01980, 0x019AB, - 0x019B0, 0x019C9, - 0x01A00, 0x01A16, - 0x01A20, 0x01A54, - 0x01B05, 0x01B33, - 0x01B45, 0x01B4B, - 0x01B83, 0x01BA0, - 0x01BAE, 0x01BAF, - 0x01BBA, 0x01BE5, - 0x01C00, 0x01C23, - 0x01C4D, 0x01C4F, - 0x01C5A, 0x01C7D, - 0x01C80, 0x01C88, - 0x01C90, 0x01CBA, - 0x01CBD, 0x01CBF, - 0x01CE9, 0x01CEC, - 0x01CEE, 0x01CF3, - 0x01CF5, 0x01CF6, - 0x01D00, 0x01DBF, - 0x01E00, 0x01F15, - 0x01F18, 0x01F1D, - 0x01F20, 0x01F45, - 0x01F48, 0x01F4D, - 0x01F50, 0x01F57, - 0x01F5F, 0x01F7D, - 0x01F80, 0x01FB4, - 0x01FB6, 0x01FBC, - 0x01FC2, 0x01FC4, - 0x01FC6, 0x01FCC, - 0x01FD0, 0x01FD3, - 0x01FD6, 0x01FDB, - 0x01FE0, 0x01FEC, - 0x01FF2, 0x01FF4, - 0x01FF6, 0x01FFC, - 0x02090, 0x0209C, - 0x0210A, 0x02113, - 0x02119, 0x0211D, - 0x0212A, 0x0212D, - 0x0212F, 0x02139, - 0x0213C, 0x0213F, - 0x02145, 0x02149, - 0x02183, 0x02184, - 0x02C00, 0x02C2E, - 0x02C30, 0x02C5E, - 0x02C60, 0x02CE4, - 0x02CEB, 0x02CEE, - 0x02CF2, 0x02CF3, - 0x02D00, 0x02D25, - 0x02D30, 0x02D67, - 0x02D80, 0x02D96, - 0x02DA0, 0x02DA6, - 0x02DA8, 0x02DAE, - 0x02DB0, 0x02DB6, - 0x02DB8, 0x02DBE, - 0x02DC0, 0x02DC6, - 0x02DC8, 0x02DCE, - 0x02DD0, 0x02DD6, - 0x02DD8, 0x02DDE, - 0x03005, 0x03006, - 0x03031, 0x03035, - 0x0303B, 0x0303C, - 0x03041, 0x03096, - 0x0309D, 0x0309F, - 0x030A1, 0x030FA, - 0x030FC, 0x030FF, - 0x03105, 0x0312F, - 0x03131, 0x0318E, - 0x031A0, 0x031BA, - 0x031F0, 0x031FF, - 0x0A000, 0x0A48C, - 0x0A4D0, 0x0A4FD, - 0x0A500, 0x0A60C, - 0x0A610, 0x0A61F, - 0x0A62A, 0x0A62B, - 0x0A640, 0x0A66E, - 0x0A67F, 0x0A69D, - 0x0A6A0, 0x0A6E5, - 0x0A717, 0x0A71F, - 0x0A722, 0x0A788, - 0x0A78B, 0x0A7BF, - 0x0A7C2, 0x0A7C6, - 0x0A7F7, 0x0A801, - 0x0A803, 0x0A805, - 0x0A807, 0x0A80A, - 0x0A80C, 0x0A822, - 0x0A840, 0x0A873, - 0x0A882, 0x0A8B3, - 0x0A8F2, 0x0A8F7, - 0x0A8FD, 0x0A8FE, - 0x0A90A, 0x0A925, - 0x0A930, 0x0A946, - 0x0A960, 0x0A97C, - 0x0A984, 0x0A9B2, - 0x0A9E0, 0x0A9E4, - 0x0A9E6, 0x0A9EF, - 0x0A9FA, 0x0A9FE, - 0x0AA00, 0x0AA28, - 0x0AA40, 0x0AA42, - 0x0AA44, 0x0AA4B, - 0x0AA60, 0x0AA76, - 0x0AA7E, 0x0AAAF, - 0x0AAB5, 0x0AAB6, - 0x0AAB9, 0x0AABD, - 0x0AADB, 0x0AADD, - 0x0AAE0, 0x0AAEA, - 0x0AAF2, 0x0AAF4, - 0x0AB01, 0x0AB06, - 0x0AB09, 0x0AB0E, - 0x0AB11, 0x0AB16, - 0x0AB20, 0x0AB26, - 0x0AB28, 0x0AB2E, - 0x0AB30, 0x0AB5A, - 0x0AB5C, 0x0AB67, - 0x0AB70, 0x0ABE2, - 0x0D7B0, 0x0D7C6, - 0x0D7CB, 0x0D7FB, - 0x0F900, 0x0FA6D, - 0x0FA70, 0x0FAD9, - 0x0FB00, 0x0FB06, - 0x0FB13, 0x0FB17, - 0x0FB1F, 0x0FB28, - 0x0FB2A, 0x0FB36, - 0x0FB38, 0x0FB3C, - 0x0FB40, 0x0FB41, - 0x0FB43, 0x0FB44, - 0x0FB46, 0x0FBB1, - 0x0FBD3, 0x0FD3D, - 0x0FD50, 0x0FD8F, - 0x0FD92, 0x0FDC7, - 0x0FDF0, 0x0FDFB, - 0x0FE70, 0x0FE74, - 0x0FE76, 0x0FEFC, - 0x0FF21, 0x0FF3A, - 0x0FF41, 0x0FF5A, - 0x0FF66, 0x0FFBE, - 0x0FFC2, 0x0FFC7, - 0x0FFCA, 0x0FFCF, - 0x0FFD2, 0x0FFD7, - 0x0FFDA, 0x0FFDC, - 0x10000, 0x1000B, - 0x1000D, 0x10026, - 0x10028, 0x1003A, - 0x1003C, 0x1003D, - 0x1003F, 0x1004D, - 0x10050, 0x1005D, - 0x10080, 0x100FA, - 0x10280, 0x1029C, - 0x102A0, 0x102D0, - 0x10300, 0x1031F, - 0x1032D, 0x10340, - 0x10342, 0x10349, - 0x10350, 0x10375, - 0x10380, 0x1039D, - 0x103A0, 0x103C3, - 0x103C8, 0x103CF, - 0x10400, 0x1049D, - 0x104B0, 0x104D3, - 0x104D8, 0x104FB, - 0x10500, 0x10527, - 0x10530, 0x10563, - 0x10600, 0x10736, - 0x10740, 0x10755, - 0x10760, 0x10767, - 0x10800, 0x10805, - 0x1080A, 0x10835, - 0x10837, 0x10838, - 0x1083F, 0x10855, - 0x10860, 0x10876, - 0x10880, 0x1089E, - 0x108E0, 0x108F2, - 0x108F4, 0x108F5, - 0x10900, 0x10915, - 0x10920, 0x10939, - 0x10980, 0x109B7, - 0x109BE, 0x109BF, - 0x10A10, 0x10A13, - 0x10A15, 0x10A17, - 0x10A19, 0x10A35, - 0x10A60, 0x10A7C, - 0x10A80, 0x10A9C, - 0x10AC0, 0x10AC7, - 0x10AC9, 0x10AE4, - 0x10B00, 0x10B35, - 0x10B40, 0x10B55, - 0x10B60, 0x10B72, - 0x10B80, 0x10B91, - 0x10C00, 0x10C48, - 0x10C80, 0x10CB2, - 0x10CC0, 0x10CF2, - 0x10D00, 0x10D23, - 0x10F00, 0x10F1C, - 0x10F30, 0x10F45, - 0x10FE0, 0x10FF6, - 0x11003, 0x11037, - 0x11083, 0x110AF, - 0x110D0, 0x110E8, - 0x11103, 0x11126, - 0x11150, 0x11172, - 0x11183, 0x111B2, - 0x111C1, 0x111C4, - 0x11200, 0x11211, - 0x11213, 0x1122B, - 0x11280, 0x11286, - 0x1128A, 0x1128D, - 0x1128F, 0x1129D, - 0x1129F, 0x112A8, - 0x112B0, 0x112DE, - 0x11305, 0x1130C, - 0x1130F, 0x11310, - 0x11313, 0x11328, - 0x1132A, 0x11330, - 0x11332, 0x11333, - 0x11335, 0x11339, - 0x1135D, 0x11361, - 0x11400, 0x11434, - 0x11447, 0x1144A, - 0x11480, 0x114AF, - 0x114C4, 0x114C5, - 0x11580, 0x115AE, - 0x115D8, 0x115DB, - 0x11600, 0x1162F, - 0x11680, 0x116AA, - 0x11700, 0x1171A, - 0x11800, 0x1182B, - 0x118A0, 0x118DF, - 0x119A0, 0x119A7, - 0x119AA, 0x119D0, - 0x11A0B, 0x11A32, - 0x11A5C, 0x11A89, - 0x11AC0, 0x11AF8, - 0x11C00, 0x11C08, - 0x11C0A, 0x11C2E, - 0x11C72, 0x11C8F, - 0x11D00, 0x11D06, - 0x11D08, 0x11D09, - 0x11D0B, 0x11D30, - 0x11D60, 0x11D65, - 0x11D67, 0x11D68, - 0x11D6A, 0x11D89, - 0x11EE0, 0x11EF2, - 0x12000, 0x12399, - 0x12480, 0x12543, - 0x13000, 0x1342E, - 0x14400, 0x14646, - 0x16800, 0x16A38, - 0x16A40, 0x16A5E, - 0x16AD0, 0x16AED, - 0x16B00, 0x16B2F, - 0x16B40, 0x16B43, - 0x16B63, 0x16B77, - 0x16B7D, 0x16B8F, - 0x16E40, 0x16E7F, - 0x16F00, 0x16F4A, - 0x16F93, 0x16F9F, - 0x16FE0, 0x16FE1, - 0x18800, 0x18AF2, - 0x1B000, 0x1B11E, - 0x1B150, 0x1B152, - 0x1B164, 0x1B167, - 0x1B170, 0x1B2FB, - 0x1BC00, 0x1BC6A, - 0x1BC70, 0x1BC7C, - 0x1BC80, 0x1BC88, - 0x1BC90, 0x1BC99, - 0x1D400, 0x1D454, - 0x1D456, 0x1D49C, - 0x1D49E, 0x1D49F, - 0x1D4A5, 0x1D4A6, - 0x1D4A9, 0x1D4AC, - 0x1D4AE, 0x1D4B9, - 0x1D4BD, 0x1D4C3, - 0x1D4C5, 0x1D505, - 0x1D507, 0x1D50A, - 0x1D50D, 0x1D514, - 0x1D516, 0x1D51C, - 0x1D51E, 0x1D539, - 0x1D53B, 0x1D53E, - 0x1D540, 0x1D544, - 0x1D54A, 0x1D550, - 0x1D552, 0x1D6A5, - 0x1D6A8, 0x1D6C0, - 0x1D6C2, 0x1D6DA, - 0x1D6DC, 0x1D6FA, - 0x1D6FC, 0x1D714, - 0x1D716, 0x1D734, - 0x1D736, 0x1D74E, - 0x1D750, 0x1D76E, - 0x1D770, 0x1D788, - 0x1D78A, 0x1D7A8, - 0x1D7AA, 0x1D7C2, - 0x1D7C4, 0x1D7CB, - 0x1E100, 0x1E12C, - 0x1E137, 0x1E13D, - 0x1E2C0, 0x1E2EB, - 0x1E800, 0x1E8C4, - 0x1E900, 0x1E943, - 0x1EE00, 0x1EE03, - 0x1EE05, 0x1EE1F, - 0x1EE21, 0x1EE22, - 0x1EE29, 0x1EE32, - 0x1EE34, 0x1EE37, - 0x1EE4D, 0x1EE4F, - 0x1EE51, 0x1EE52, - 0x1EE61, 0x1EE62, - 0x1EE67, 0x1EE6A, - 0x1EE6C, 0x1EE72, - 0x1EE74, 0x1EE77, - 0x1EE79, 0x1EE7C, - 0x1EE80, 0x1EE89, - 0x1EE8B, 0x1EE9B, - 0x1EEA1, 0x1EEA3, - 0x1EEA5, 0x1EEA9, - 0x1EEAB, 0x1EEBB, - 0x2F800, 0x2FA1D, + 0x00041'i32, 0x0005A'i32, + 0x00061'i32, 0x0007A'i32, + 0x000C0'i32, 0x000D6'i32, + 0x000D8'i32, 0x000F6'i32, + 0x000F8'i32, 0x002C1'i32, + 0x002C6'i32, 0x002D1'i32, + 0x002E0'i32, 0x002E4'i32, + 0x00370'i32, 0x00374'i32, + 0x00376'i32, 0x00377'i32, + 0x0037A'i32, 0x0037D'i32, + 0x00388'i32, 0x0038A'i32, + 0x0038E'i32, 0x003A1'i32, + 0x003A3'i32, 0x003F5'i32, + 0x003F7'i32, 0x00481'i32, + 0x0048A'i32, 0x0052F'i32, + 0x00531'i32, 0x00556'i32, + 0x00560'i32, 0x00588'i32, + 0x005D0'i32, 0x005EA'i32, + 0x005EF'i32, 0x005F2'i32, + 0x00620'i32, 0x0064A'i32, + 0x0066E'i32, 0x0066F'i32, + 0x00671'i32, 0x006D3'i32, + 0x006E5'i32, 0x006E6'i32, + 0x006EE'i32, 0x006EF'i32, + 0x006FA'i32, 0x006FC'i32, + 0x00712'i32, 0x0072F'i32, + 0x0074D'i32, 0x007A5'i32, + 0x007CA'i32, 0x007EA'i32, + 0x007F4'i32, 0x007F5'i32, + 0x00800'i32, 0x00815'i32, + 0x00840'i32, 0x00858'i32, + 0x00860'i32, 0x0086A'i32, + 0x008A0'i32, 0x008B4'i32, + 0x008B6'i32, 0x008BD'i32, + 0x00904'i32, 0x00939'i32, + 0x00958'i32, 0x00961'i32, + 0x00971'i32, 0x00980'i32, + 0x00985'i32, 0x0098C'i32, + 0x0098F'i32, 0x00990'i32, + 0x00993'i32, 0x009A8'i32, + 0x009AA'i32, 0x009B0'i32, + 0x009B6'i32, 0x009B9'i32, + 0x009DC'i32, 0x009DD'i32, + 0x009DF'i32, 0x009E1'i32, + 0x009F0'i32, 0x009F1'i32, + 0x00A05'i32, 0x00A0A'i32, + 0x00A0F'i32, 0x00A10'i32, + 0x00A13'i32, 0x00A28'i32, + 0x00A2A'i32, 0x00A30'i32, + 0x00A32'i32, 0x00A33'i32, + 0x00A35'i32, 0x00A36'i32, + 0x00A38'i32, 0x00A39'i32, + 0x00A59'i32, 0x00A5C'i32, + 0x00A72'i32, 0x00A74'i32, + 0x00A85'i32, 0x00A8D'i32, + 0x00A8F'i32, 0x00A91'i32, + 0x00A93'i32, 0x00AA8'i32, + 0x00AAA'i32, 0x00AB0'i32, + 0x00AB2'i32, 0x00AB3'i32, + 0x00AB5'i32, 0x00AB9'i32, + 0x00AE0'i32, 0x00AE1'i32, + 0x00B05'i32, 0x00B0C'i32, + 0x00B0F'i32, 0x00B10'i32, + 0x00B13'i32, 0x00B28'i32, + 0x00B2A'i32, 0x00B30'i32, + 0x00B32'i32, 0x00B33'i32, + 0x00B35'i32, 0x00B39'i32, + 0x00B5C'i32, 0x00B5D'i32, + 0x00B5F'i32, 0x00B61'i32, + 0x00B85'i32, 0x00B8A'i32, + 0x00B8E'i32, 0x00B90'i32, + 0x00B92'i32, 0x00B95'i32, + 0x00B99'i32, 0x00B9A'i32, + 0x00B9E'i32, 0x00B9F'i32, + 0x00BA3'i32, 0x00BA4'i32, + 0x00BA8'i32, 0x00BAA'i32, + 0x00BAE'i32, 0x00BB9'i32, + 0x00C05'i32, 0x00C0C'i32, + 0x00C0E'i32, 0x00C10'i32, + 0x00C12'i32, 0x00C28'i32, + 0x00C2A'i32, 0x00C39'i32, + 0x00C58'i32, 0x00C5A'i32, + 0x00C60'i32, 0x00C61'i32, + 0x00C85'i32, 0x00C8C'i32, + 0x00C8E'i32, 0x00C90'i32, + 0x00C92'i32, 0x00CA8'i32, + 0x00CAA'i32, 0x00CB3'i32, + 0x00CB5'i32, 0x00CB9'i32, + 0x00CE0'i32, 0x00CE1'i32, + 0x00CF1'i32, 0x00CF2'i32, + 0x00D05'i32, 0x00D0C'i32, + 0x00D0E'i32, 0x00D10'i32, + 0x00D12'i32, 0x00D3A'i32, + 0x00D54'i32, 0x00D56'i32, + 0x00D5F'i32, 0x00D61'i32, + 0x00D7A'i32, 0x00D7F'i32, + 0x00D85'i32, 0x00D96'i32, + 0x00D9A'i32, 0x00DB1'i32, + 0x00DB3'i32, 0x00DBB'i32, + 0x00DC0'i32, 0x00DC6'i32, + 0x00E01'i32, 0x00E30'i32, + 0x00E32'i32, 0x00E33'i32, + 0x00E40'i32, 0x00E46'i32, + 0x00E81'i32, 0x00E82'i32, + 0x00E86'i32, 0x00E8A'i32, + 0x00E8C'i32, 0x00EA3'i32, + 0x00EA7'i32, 0x00EB0'i32, + 0x00EB2'i32, 0x00EB3'i32, + 0x00EC0'i32, 0x00EC4'i32, + 0x00EDC'i32, 0x00EDF'i32, + 0x00F40'i32, 0x00F47'i32, + 0x00F49'i32, 0x00F6C'i32, + 0x00F88'i32, 0x00F8C'i32, + 0x01000'i32, 0x0102A'i32, + 0x01050'i32, 0x01055'i32, + 0x0105A'i32, 0x0105D'i32, + 0x01065'i32, 0x01066'i32, + 0x0106E'i32, 0x01070'i32, + 0x01075'i32, 0x01081'i32, + 0x010A0'i32, 0x010C5'i32, + 0x010D0'i32, 0x010FA'i32, + 0x010FC'i32, 0x01248'i32, + 0x0124A'i32, 0x0124D'i32, + 0x01250'i32, 0x01256'i32, + 0x0125A'i32, 0x0125D'i32, + 0x01260'i32, 0x01288'i32, + 0x0128A'i32, 0x0128D'i32, + 0x01290'i32, 0x012B0'i32, + 0x012B2'i32, 0x012B5'i32, + 0x012B8'i32, 0x012BE'i32, + 0x012C2'i32, 0x012C5'i32, + 0x012C8'i32, 0x012D6'i32, + 0x012D8'i32, 0x01310'i32, + 0x01312'i32, 0x01315'i32, + 0x01318'i32, 0x0135A'i32, + 0x01380'i32, 0x0138F'i32, + 0x013A0'i32, 0x013F5'i32, + 0x013F8'i32, 0x013FD'i32, + 0x01401'i32, 0x0166C'i32, + 0x0166F'i32, 0x0167F'i32, + 0x01681'i32, 0x0169A'i32, + 0x016A0'i32, 0x016EA'i32, + 0x016F1'i32, 0x016F8'i32, + 0x01700'i32, 0x0170C'i32, + 0x0170E'i32, 0x01711'i32, + 0x01720'i32, 0x01731'i32, + 0x01740'i32, 0x01751'i32, + 0x01760'i32, 0x0176C'i32, + 0x0176E'i32, 0x01770'i32, + 0x01780'i32, 0x017B3'i32, + 0x01820'i32, 0x01878'i32, + 0x01880'i32, 0x01884'i32, + 0x01887'i32, 0x018A8'i32, + 0x018B0'i32, 0x018F5'i32, + 0x01900'i32, 0x0191E'i32, + 0x01950'i32, 0x0196D'i32, + 0x01970'i32, 0x01974'i32, + 0x01980'i32, 0x019AB'i32, + 0x019B0'i32, 0x019C9'i32, + 0x01A00'i32, 0x01A16'i32, + 0x01A20'i32, 0x01A54'i32, + 0x01B05'i32, 0x01B33'i32, + 0x01B45'i32, 0x01B4B'i32, + 0x01B83'i32, 0x01BA0'i32, + 0x01BAE'i32, 0x01BAF'i32, + 0x01BBA'i32, 0x01BE5'i32, + 0x01C00'i32, 0x01C23'i32, + 0x01C4D'i32, 0x01C4F'i32, + 0x01C5A'i32, 0x01C7D'i32, + 0x01C80'i32, 0x01C88'i32, + 0x01C90'i32, 0x01CBA'i32, + 0x01CBD'i32, 0x01CBF'i32, + 0x01CE9'i32, 0x01CEC'i32, + 0x01CEE'i32, 0x01CF3'i32, + 0x01CF5'i32, 0x01CF6'i32, + 0x01D00'i32, 0x01DBF'i32, + 0x01E00'i32, 0x01F15'i32, + 0x01F18'i32, 0x01F1D'i32, + 0x01F20'i32, 0x01F45'i32, + 0x01F48'i32, 0x01F4D'i32, + 0x01F50'i32, 0x01F57'i32, + 0x01F5F'i32, 0x01F7D'i32, + 0x01F80'i32, 0x01FB4'i32, + 0x01FB6'i32, 0x01FBC'i32, + 0x01FC2'i32, 0x01FC4'i32, + 0x01FC6'i32, 0x01FCC'i32, + 0x01FD0'i32, 0x01FD3'i32, + 0x01FD6'i32, 0x01FDB'i32, + 0x01FE0'i32, 0x01FEC'i32, + 0x01FF2'i32, 0x01FF4'i32, + 0x01FF6'i32, 0x01FFC'i32, + 0x02090'i32, 0x0209C'i32, + 0x0210A'i32, 0x02113'i32, + 0x02119'i32, 0x0211D'i32, + 0x0212A'i32, 0x0212D'i32, + 0x0212F'i32, 0x02139'i32, + 0x0213C'i32, 0x0213F'i32, + 0x02145'i32, 0x02149'i32, + 0x02183'i32, 0x02184'i32, + 0x02C00'i32, 0x02C2E'i32, + 0x02C30'i32, 0x02C5E'i32, + 0x02C60'i32, 0x02CE4'i32, + 0x02CEB'i32, 0x02CEE'i32, + 0x02CF2'i32, 0x02CF3'i32, + 0x02D00'i32, 0x02D25'i32, + 0x02D30'i32, 0x02D67'i32, + 0x02D80'i32, 0x02D96'i32, + 0x02DA0'i32, 0x02DA6'i32, + 0x02DA8'i32, 0x02DAE'i32, + 0x02DB0'i32, 0x02DB6'i32, + 0x02DB8'i32, 0x02DBE'i32, + 0x02DC0'i32, 0x02DC6'i32, + 0x02DC8'i32, 0x02DCE'i32, + 0x02DD0'i32, 0x02DD6'i32, + 0x02DD8'i32, 0x02DDE'i32, + 0x03005'i32, 0x03006'i32, + 0x03031'i32, 0x03035'i32, + 0x0303B'i32, 0x0303C'i32, + 0x03041'i32, 0x03096'i32, + 0x0309D'i32, 0x0309F'i32, + 0x030A1'i32, 0x030FA'i32, + 0x030FC'i32, 0x030FF'i32, + 0x03105'i32, 0x0312F'i32, + 0x03131'i32, 0x0318E'i32, + 0x031A0'i32, 0x031BA'i32, + 0x031F0'i32, 0x031FF'i32, + 0x03400'i32, 0x04DB5'i32, + 0x04E00'i32, 0x09FEF'i32, + 0x0A000'i32, 0x0A48C'i32, + 0x0A4D0'i32, 0x0A4FD'i32, + 0x0A500'i32, 0x0A60C'i32, + 0x0A610'i32, 0x0A61F'i32, + 0x0A62A'i32, 0x0A62B'i32, + 0x0A640'i32, 0x0A66E'i32, + 0x0A67F'i32, 0x0A69D'i32, + 0x0A6A0'i32, 0x0A6E5'i32, + 0x0A717'i32, 0x0A71F'i32, + 0x0A722'i32, 0x0A788'i32, + 0x0A78B'i32, 0x0A7BF'i32, + 0x0A7C2'i32, 0x0A7C6'i32, + 0x0A7F7'i32, 0x0A801'i32, + 0x0A803'i32, 0x0A805'i32, + 0x0A807'i32, 0x0A80A'i32, + 0x0A80C'i32, 0x0A822'i32, + 0x0A840'i32, 0x0A873'i32, + 0x0A882'i32, 0x0A8B3'i32, + 0x0A8F2'i32, 0x0A8F7'i32, + 0x0A8FD'i32, 0x0A8FE'i32, + 0x0A90A'i32, 0x0A925'i32, + 0x0A930'i32, 0x0A946'i32, + 0x0A960'i32, 0x0A97C'i32, + 0x0A984'i32, 0x0A9B2'i32, + 0x0A9E0'i32, 0x0A9E4'i32, + 0x0A9E6'i32, 0x0A9EF'i32, + 0x0A9FA'i32, 0x0A9FE'i32, + 0x0AA00'i32, 0x0AA28'i32, + 0x0AA40'i32, 0x0AA42'i32, + 0x0AA44'i32, 0x0AA4B'i32, + 0x0AA60'i32, 0x0AA76'i32, + 0x0AA7E'i32, 0x0AAAF'i32, + 0x0AAB5'i32, 0x0AAB6'i32, + 0x0AAB9'i32, 0x0AABD'i32, + 0x0AADB'i32, 0x0AADD'i32, + 0x0AAE0'i32, 0x0AAEA'i32, + 0x0AAF2'i32, 0x0AAF4'i32, + 0x0AB01'i32, 0x0AB06'i32, + 0x0AB09'i32, 0x0AB0E'i32, + 0x0AB11'i32, 0x0AB16'i32, + 0x0AB20'i32, 0x0AB26'i32, + 0x0AB28'i32, 0x0AB2E'i32, + 0x0AB30'i32, 0x0AB5A'i32, + 0x0AB5C'i32, 0x0AB67'i32, + 0x0AB70'i32, 0x0ABE2'i32, + 0x0AC00'i32, 0x0D7A3'i32, + 0x0D7B0'i32, 0x0D7C6'i32, + 0x0D7CB'i32, 0x0D7FB'i32, + 0x0F900'i32, 0x0FA6D'i32, + 0x0FA70'i32, 0x0FAD9'i32, + 0x0FB00'i32, 0x0FB06'i32, + 0x0FB13'i32, 0x0FB17'i32, + 0x0FB1F'i32, 0x0FB28'i32, + 0x0FB2A'i32, 0x0FB36'i32, + 0x0FB38'i32, 0x0FB3C'i32, + 0x0FB40'i32, 0x0FB41'i32, + 0x0FB43'i32, 0x0FB44'i32, + 0x0FB46'i32, 0x0FBB1'i32, + 0x0FBD3'i32, 0x0FD3D'i32, + 0x0FD50'i32, 0x0FD8F'i32, + 0x0FD92'i32, 0x0FDC7'i32, + 0x0FDF0'i32, 0x0FDFB'i32, + 0x0FE70'i32, 0x0FE74'i32, + 0x0FE76'i32, 0x0FEFC'i32, + 0x0FF21'i32, 0x0FF3A'i32, + 0x0FF41'i32, 0x0FF5A'i32, + 0x0FF66'i32, 0x0FFBE'i32, + 0x0FFC2'i32, 0x0FFC7'i32, + 0x0FFCA'i32, 0x0FFCF'i32, + 0x0FFD2'i32, 0x0FFD7'i32, + 0x0FFDA'i32, 0x0FFDC'i32, + 0x10000'i32, 0x1000B'i32, + 0x1000D'i32, 0x10026'i32, + 0x10028'i32, 0x1003A'i32, + 0x1003C'i32, 0x1003D'i32, + 0x1003F'i32, 0x1004D'i32, + 0x10050'i32, 0x1005D'i32, + 0x10080'i32, 0x100FA'i32, + 0x10280'i32, 0x1029C'i32, + 0x102A0'i32, 0x102D0'i32, + 0x10300'i32, 0x1031F'i32, + 0x1032D'i32, 0x10340'i32, + 0x10342'i32, 0x10349'i32, + 0x10350'i32, 0x10375'i32, + 0x10380'i32, 0x1039D'i32, + 0x103A0'i32, 0x103C3'i32, + 0x103C8'i32, 0x103CF'i32, + 0x10400'i32, 0x1049D'i32, + 0x104B0'i32, 0x104D3'i32, + 0x104D8'i32, 0x104FB'i32, + 0x10500'i32, 0x10527'i32, + 0x10530'i32, 0x10563'i32, + 0x10600'i32, 0x10736'i32, + 0x10740'i32, 0x10755'i32, + 0x10760'i32, 0x10767'i32, + 0x10800'i32, 0x10805'i32, + 0x1080A'i32, 0x10835'i32, + 0x10837'i32, 0x10838'i32, + 0x1083F'i32, 0x10855'i32, + 0x10860'i32, 0x10876'i32, + 0x10880'i32, 0x1089E'i32, + 0x108E0'i32, 0x108F2'i32, + 0x108F4'i32, 0x108F5'i32, + 0x10900'i32, 0x10915'i32, + 0x10920'i32, 0x10939'i32, + 0x10980'i32, 0x109B7'i32, + 0x109BE'i32, 0x109BF'i32, + 0x10A10'i32, 0x10A13'i32, + 0x10A15'i32, 0x10A17'i32, + 0x10A19'i32, 0x10A35'i32, + 0x10A60'i32, 0x10A7C'i32, + 0x10A80'i32, 0x10A9C'i32, + 0x10AC0'i32, 0x10AC7'i32, + 0x10AC9'i32, 0x10AE4'i32, + 0x10B00'i32, 0x10B35'i32, + 0x10B40'i32, 0x10B55'i32, + 0x10B60'i32, 0x10B72'i32, + 0x10B80'i32, 0x10B91'i32, + 0x10C00'i32, 0x10C48'i32, + 0x10C80'i32, 0x10CB2'i32, + 0x10CC0'i32, 0x10CF2'i32, + 0x10D00'i32, 0x10D23'i32, + 0x10F00'i32, 0x10F1C'i32, + 0x10F30'i32, 0x10F45'i32, + 0x10FE0'i32, 0x10FF6'i32, + 0x11003'i32, 0x11037'i32, + 0x11083'i32, 0x110AF'i32, + 0x110D0'i32, 0x110E8'i32, + 0x11103'i32, 0x11126'i32, + 0x11150'i32, 0x11172'i32, + 0x11183'i32, 0x111B2'i32, + 0x111C1'i32, 0x111C4'i32, + 0x11200'i32, 0x11211'i32, + 0x11213'i32, 0x1122B'i32, + 0x11280'i32, 0x11286'i32, + 0x1128A'i32, 0x1128D'i32, + 0x1128F'i32, 0x1129D'i32, + 0x1129F'i32, 0x112A8'i32, + 0x112B0'i32, 0x112DE'i32, + 0x11305'i32, 0x1130C'i32, + 0x1130F'i32, 0x11310'i32, + 0x11313'i32, 0x11328'i32, + 0x1132A'i32, 0x11330'i32, + 0x11332'i32, 0x11333'i32, + 0x11335'i32, 0x11339'i32, + 0x1135D'i32, 0x11361'i32, + 0x11400'i32, 0x11434'i32, + 0x11447'i32, 0x1144A'i32, + 0x11480'i32, 0x114AF'i32, + 0x114C4'i32, 0x114C5'i32, + 0x11580'i32, 0x115AE'i32, + 0x115D8'i32, 0x115DB'i32, + 0x11600'i32, 0x1162F'i32, + 0x11680'i32, 0x116AA'i32, + 0x11700'i32, 0x1171A'i32, + 0x11800'i32, 0x1182B'i32, + 0x118A0'i32, 0x118DF'i32, + 0x119A0'i32, 0x119A7'i32, + 0x119AA'i32, 0x119D0'i32, + 0x11A0B'i32, 0x11A32'i32, + 0x11A5C'i32, 0x11A89'i32, + 0x11AC0'i32, 0x11AF8'i32, + 0x11C00'i32, 0x11C08'i32, + 0x11C0A'i32, 0x11C2E'i32, + 0x11C72'i32, 0x11C8F'i32, + 0x11D00'i32, 0x11D06'i32, + 0x11D08'i32, 0x11D09'i32, + 0x11D0B'i32, 0x11D30'i32, + 0x11D60'i32, 0x11D65'i32, + 0x11D67'i32, 0x11D68'i32, + 0x11D6A'i32, 0x11D89'i32, + 0x11EE0'i32, 0x11EF2'i32, + 0x12000'i32, 0x12399'i32, + 0x12480'i32, 0x12543'i32, + 0x13000'i32, 0x1342E'i32, + 0x14400'i32, 0x14646'i32, + 0x16800'i32, 0x16A38'i32, + 0x16A40'i32, 0x16A5E'i32, + 0x16AD0'i32, 0x16AED'i32, + 0x16B00'i32, 0x16B2F'i32, + 0x16B40'i32, 0x16B43'i32, + 0x16B63'i32, 0x16B77'i32, + 0x16B7D'i32, 0x16B8F'i32, + 0x16E40'i32, 0x16E7F'i32, + 0x16F00'i32, 0x16F4A'i32, + 0x16F93'i32, 0x16F9F'i32, + 0x16FE0'i32, 0x16FE1'i32, + 0x17000'i32, 0x187F7'i32, + 0x18800'i32, 0x18AF2'i32, + 0x1B000'i32, 0x1B11E'i32, + 0x1B150'i32, 0x1B152'i32, + 0x1B164'i32, 0x1B167'i32, + 0x1B170'i32, 0x1B2FB'i32, + 0x1BC00'i32, 0x1BC6A'i32, + 0x1BC70'i32, 0x1BC7C'i32, + 0x1BC80'i32, 0x1BC88'i32, + 0x1BC90'i32, 0x1BC99'i32, + 0x1D400'i32, 0x1D454'i32, + 0x1D456'i32, 0x1D49C'i32, + 0x1D49E'i32, 0x1D49F'i32, + 0x1D4A5'i32, 0x1D4A6'i32, + 0x1D4A9'i32, 0x1D4AC'i32, + 0x1D4AE'i32, 0x1D4B9'i32, + 0x1D4BD'i32, 0x1D4C3'i32, + 0x1D4C5'i32, 0x1D505'i32, + 0x1D507'i32, 0x1D50A'i32, + 0x1D50D'i32, 0x1D514'i32, + 0x1D516'i32, 0x1D51C'i32, + 0x1D51E'i32, 0x1D539'i32, + 0x1D53B'i32, 0x1D53E'i32, + 0x1D540'i32, 0x1D544'i32, + 0x1D54A'i32, 0x1D550'i32, + 0x1D552'i32, 0x1D6A5'i32, + 0x1D6A8'i32, 0x1D6C0'i32, + 0x1D6C2'i32, 0x1D6DA'i32, + 0x1D6DC'i32, 0x1D6FA'i32, + 0x1D6FC'i32, 0x1D714'i32, + 0x1D716'i32, 0x1D734'i32, + 0x1D736'i32, 0x1D74E'i32, + 0x1D750'i32, 0x1D76E'i32, + 0x1D770'i32, 0x1D788'i32, + 0x1D78A'i32, 0x1D7A8'i32, + 0x1D7AA'i32, 0x1D7C2'i32, + 0x1D7C4'i32, 0x1D7CB'i32, + 0x1E100'i32, 0x1E12C'i32, + 0x1E137'i32, 0x1E13D'i32, + 0x1E2C0'i32, 0x1E2EB'i32, + 0x1E800'i32, 0x1E8C4'i32, + 0x1E900'i32, 0x1E943'i32, + 0x1EE00'i32, 0x1EE03'i32, + 0x1EE05'i32, 0x1EE1F'i32, + 0x1EE21'i32, 0x1EE22'i32, + 0x1EE29'i32, 0x1EE32'i32, + 0x1EE34'i32, 0x1EE37'i32, + 0x1EE4D'i32, 0x1EE4F'i32, + 0x1EE51'i32, 0x1EE52'i32, + 0x1EE61'i32, 0x1EE62'i32, + 0x1EE67'i32, 0x1EE6A'i32, + 0x1EE6C'i32, 0x1EE72'i32, + 0x1EE74'i32, 0x1EE77'i32, + 0x1EE79'i32, 0x1EE7C'i32, + 0x1EE80'i32, 0x1EE89'i32, + 0x1EE8B'i32, 0x1EE9B'i32, + 0x1EEA1'i32, 0x1EEA3'i32, + 0x1EEA5'i32, 0x1EEA9'i32, + 0x1EEAB'i32, 0x1EEBB'i32, + 0x20000'i32, 0x2A6D6'i32, + 0x2A700'i32, 0x2B734'i32, + 0x2B740'i32, 0x2B81D'i32, + 0x2B820'i32, 0x2CEA1'i32, + 0x2CEB0'i32, 0x2EBE0'i32, + 0x2F800'i32, 0x2FA1D'i32, ] alphaSinglets = [ - 0x000AA, - 0x000B5, - 0x000BA, - 0x002EC, - 0x002EE, - 0x0037F, - 0x00386, - 0x0038C, - 0x00559, - 0x006D5, - 0x006FF, - 0x00710, - 0x007B1, - 0x007FA, - 0x0081A, - 0x00824, - 0x00828, - 0x0093D, - 0x00950, - 0x009B2, - 0x009BD, - 0x009CE, - 0x009FC, - 0x00A5E, - 0x00ABD, - 0x00AD0, - 0x00AF9, - 0x00B3D, - 0x00B71, - 0x00B83, - 0x00B9C, - 0x00BD0, - 0x00C3D, - 0x00C80, - 0x00CBD, - 0x00CDE, - 0x00D3D, - 0x00D4E, - 0x00DBD, - 0x00E84, - 0x00EA5, - 0x00EBD, - 0x00EC6, - 0x00F00, - 0x0103F, - 0x01061, - 0x0108E, - 0x010C7, - 0x010CD, - 0x01258, - 0x012C0, - 0x017D7, - 0x017DC, - 0x018AA, - 0x01AA7, - 0x01CFA, - 0x01F59, - 0x01F5B, - 0x01F5D, - 0x01FBE, - 0x02071, - 0x0207F, - 0x02102, - 0x02107, - 0x02115, - 0x02124, - 0x02126, - 0x02128, - 0x0214E, - 0x02D27, - 0x02D2D, - 0x02D6F, - 0x02E2F, - 0x03400, - 0x04DB5, - 0x04E00, - 0x09FEF, - 0x0A8FB, - 0x0A9CF, - 0x0AA7A, - 0x0AAB1, - 0x0AAC0, - 0x0AAC2, - 0x0AC00, - 0x0D7A3, - 0x0FB1D, - 0x0FB3E, - 0x10808, - 0x1083C, - 0x10A00, - 0x10F27, - 0x11144, - 0x11176, - 0x111DA, - 0x111DC, - 0x11288, - 0x1133D, - 0x11350, - 0x1145F, - 0x114C7, - 0x11644, - 0x116B8, - 0x118FF, - 0x119E1, - 0x119E3, - 0x11A00, - 0x11A3A, - 0x11A50, - 0x11A9D, - 0x11C40, - 0x11D46, - 0x11D98, - 0x16F50, - 0x16FE3, - 0x17000, - 0x187F7, - 0x1D4A2, - 0x1D4BB, - 0x1D546, - 0x1E14E, - 0x1E94B, - 0x1EE24, - 0x1EE27, - 0x1EE39, - 0x1EE3B, - 0x1EE42, - 0x1EE47, - 0x1EE49, - 0x1EE4B, - 0x1EE54, - 0x1EE57, - 0x1EE59, - 0x1EE5B, - 0x1EE5D, - 0x1EE5F, - 0x1EE64, - 0x1EE7E, - 0x20000, - 0x2A6D6, - 0x2A700, - 0x2B734, - 0x2B740, - 0x2B81D, - 0x2B820, - 0x2CEA1, - 0x2CEB0, - 0x2EBE0, + 0x000AA'i32, + 0x000B5'i32, + 0x000BA'i32, + 0x002EC'i32, + 0x002EE'i32, + 0x0037F'i32, + 0x00386'i32, + 0x0038C'i32, + 0x00559'i32, + 0x006D5'i32, + 0x006FF'i32, + 0x00710'i32, + 0x007B1'i32, + 0x007FA'i32, + 0x0081A'i32, + 0x00824'i32, + 0x00828'i32, + 0x0093D'i32, + 0x00950'i32, + 0x009B2'i32, + 0x009BD'i32, + 0x009CE'i32, + 0x009FC'i32, + 0x00A5E'i32, + 0x00ABD'i32, + 0x00AD0'i32, + 0x00AF9'i32, + 0x00B3D'i32, + 0x00B71'i32, + 0x00B83'i32, + 0x00B9C'i32, + 0x00BD0'i32, + 0x00C3D'i32, + 0x00C80'i32, + 0x00CBD'i32, + 0x00CDE'i32, + 0x00D3D'i32, + 0x00D4E'i32, + 0x00DBD'i32, + 0x00E84'i32, + 0x00EA5'i32, + 0x00EBD'i32, + 0x00EC6'i32, + 0x00F00'i32, + 0x0103F'i32, + 0x01061'i32, + 0x0108E'i32, + 0x010C7'i32, + 0x010CD'i32, + 0x01258'i32, + 0x012C0'i32, + 0x017D7'i32, + 0x017DC'i32, + 0x018AA'i32, + 0x01AA7'i32, + 0x01CFA'i32, + 0x01F59'i32, + 0x01F5B'i32, + 0x01F5D'i32, + 0x01FBE'i32, + 0x02071'i32, + 0x0207F'i32, + 0x02102'i32, + 0x02107'i32, + 0x02115'i32, + 0x02124'i32, + 0x02126'i32, + 0x02128'i32, + 0x0214E'i32, + 0x02D27'i32, + 0x02D2D'i32, + 0x02D6F'i32, + 0x02E2F'i32, + 0x0A8FB'i32, + 0x0A9CF'i32, + 0x0AA7A'i32, + 0x0AAB1'i32, + 0x0AAC0'i32, + 0x0AAC2'i32, + 0x0FB1D'i32, + 0x0FB3E'i32, + 0x10808'i32, + 0x1083C'i32, + 0x10A00'i32, + 0x10F27'i32, + 0x11144'i32, + 0x11176'i32, + 0x111DA'i32, + 0x111DC'i32, + 0x11288'i32, + 0x1133D'i32, + 0x11350'i32, + 0x1145F'i32, + 0x114C7'i32, + 0x11644'i32, + 0x116B8'i32, + 0x118FF'i32, + 0x119E1'i32, + 0x119E3'i32, + 0x11A00'i32, + 0x11A3A'i32, + 0x11A50'i32, + 0x11A9D'i32, + 0x11C40'i32, + 0x11D46'i32, + 0x11D98'i32, + 0x16F50'i32, + 0x16FE3'i32, + 0x1D4A2'i32, + 0x1D4BB'i32, + 0x1D546'i32, + 0x1E14E'i32, + 0x1E94B'i32, + 0x1EE24'i32, + 0x1EE27'i32, + 0x1EE39'i32, + 0x1EE3B'i32, + 0x1EE42'i32, + 0x1EE47'i32, + 0x1EE49'i32, + 0x1EE4B'i32, + 0x1EE54'i32, + 0x1EE57'i32, + 0x1EE59'i32, + 0x1EE5B'i32, + 0x1EE5D'i32, + 0x1EE5F'i32, + 0x1EE64'i32, + 0x1EE7E'i32, ] spaceRanges = [ - 0x00009, 0x0000D, - 0x00020, 0x00020, - 0x00085, 0x00085, - 0x000A0, 0x000A0, - 0x01680, 0x01680, - 0x02000, 0x0200A, - 0x02028, 0x02029, - 0x0202F, 0x0202F, - 0x0205F, 0x0205F, - 0x03000, 0x03000, + 0x00009'i32, 0x0000D'i32, + 0x00020'i32, 0x00020'i32, + 0x00085'i32, 0x00085'i32, + 0x000A0'i32, 0x000A0'i32, + 0x01680'i32, 0x01680'i32, + 0x02000'i32, 0x0200A'i32, + 0x02028'i32, 0x02029'i32, + 0x0202F'i32, 0x0202F'i32, + 0x0205F'i32, 0x0205F'i32, + 0x03000'i32, 0x03000'i32, ] unicodeSpaces = [ diff --git a/lib/pure/ioselects/ioselectors_epoll.nim b/lib/pure/ioselects/ioselectors_epoll.nim index 8526eb8a3..10658b78e 100644 --- a/lib/pure/ioselects/ioselectors_epoll.nim +++ b/lib/pure/ioselects/ioselectors_epoll.nim @@ -9,7 +9,7 @@ # This module implements Linux epoll(). -import posix, times, epoll +import std/[posix, times, epoll] # Maximum number of events that can be returned const MAX_EPOLL_EVENTS = 64 @@ -72,11 +72,16 @@ type SelectEvent* = ptr SelectEventImpl proc newSelector*[T](): Selector[T] = + proc initialNumFD(): int {.inline.} = + when defined(nuttx): + result = NEPOLL_MAX + else: + result = 1024 # Retrieve the maximum fd count (for current OS) via getrlimit() var maxFD = maxDescriptors() doAssert(maxFD > 0) # Start with a reasonable size, checkFd() will grow this on demand - const numFD = 1024 + let numFD = initialNumFD() var epollFD = epoll_create1(O_CLOEXEC) if epollFD < 0: @@ -133,7 +138,7 @@ template checkFd(s, f) = var numFD = s.numFD while numFD <= f: numFD *= 2 when hasThreadSupport: - s.fds = reallocSharedArray(s.fds, numFD) + s.fds = reallocSharedArray(s.fds, s.numFD, numFD) else: s.fds.setLen(numFD) for i in s.numFD ..< numFD: diff --git a/lib/pure/ioselects/ioselectors_kqueue.nim b/lib/pure/ioselects/ioselectors_kqueue.nim index 68be969c7..513578eda 100644 --- a/lib/pure/ioselects/ioselectors_kqueue.nim +++ b/lib/pure/ioselects/ioselectors_kqueue.nim @@ -9,7 +9,7 @@ # This module implements BSD kqueue(). -import posix, times, kqueue, nativesockets +import std/[posix, times, kqueue, nativesockets] const # Maximum number of events that can be returned. @@ -194,7 +194,9 @@ when hasThreadSupport: if s.changesLength > 0: if kevent(s.kqFD, addr(s.changes[0]), cint(s.changesLength), nil, 0, nil) == -1: - raiseIOSelectorsError(osLastError()) + let res = osLastError() + if cint(res) != ENOENT: # ignore pipes whose read end is closed + raiseIOSelectorsError(res) s.changesLength = 0 else: template modifyKQueue[T](s: Selector[T], nident: uint, nfilter: cshort, @@ -211,7 +213,9 @@ else: if length > 0: if kevent(s.kqFD, addr(s.changes[0]), length, nil, 0, nil) == -1: - raiseIOSelectorsError(osLastError()) + let res = osLastError() + if cint(res) != ENOENT: # ignore pipes whose read end is closed + raiseIOSelectorsError(res) s.changes.setLen(0) proc registerHandle*[T](s: Selector[T], fd: int | SocketHandle, diff --git a/lib/pure/ioselects/ioselectors_poll.nim b/lib/pure/ioselects/ioselectors_poll.nim index 12812ac80..7c5347156 100644 --- a/lib/pure/ioselects/ioselectors_poll.nim +++ b/lib/pure/ioselects/ioselectors_poll.nim @@ -9,7 +9,7 @@ # This module implements Posix poll(). -import posix, times +import std/[posix, times] # Maximum number of events that can be returned const MAX_POLL_EVENTS = 64 diff --git a/lib/pure/ioselects/ioselectors_select.nim b/lib/pure/ioselects/ioselectors_select.nim index 34c88d85e..6c516395b 100644 --- a/lib/pure/ioselects/ioselectors_select.nim +++ b/lib/pure/ioselects/ioselectors_select.nim @@ -9,10 +9,10 @@ # This module implements Posix and Windows select(). -import times, nativesockets +import std/[times, nativesockets] when defined(windows): - import winlean + import std/winlean when defined(gcc): {.passl: "-lws2_32".} elif defined(vcc): @@ -313,8 +313,8 @@ proc selectInto*[T](s: Selector[T], timeout: int, verifySelectParams(timeout) if timeout != -1: - when defined(genode) or defined(freertos) or defined(zephyr): - tv.tv_sec = Time(timeout div 1_000) + when defined(genode) or defined(freertos) or defined(zephyr) or defined(nuttx): + tv.tv_sec = posix.Time(timeout div 1_000) else: tv.tv_sec = timeout.int32 div 1_000 tv.tv_usec = (timeout.int32 %% 1_000) * 1_000 diff --git a/lib/pure/json.nim b/lib/pure/json.nim index d0b1a4051..53fa7553a 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -41,13 +41,14 @@ ## For a `JsonNode` who's kind is `JObject`, you can access its fields using ## the `[]` operator. The following example shows how to do this: ## -## .. code-block:: Nim +## ```Nim ## import std/json ## ## let jsonNode = parseJson("""{"key": 3.14}""") ## ## doAssert jsonNode.kind == JObject ## doAssert jsonNode["key"].kind == JFloat +## ``` ## ## Reading values ## -------------- @@ -62,12 +63,13 @@ ## ## To retrieve the value of `"key"` you can do the following: ## -## .. code-block:: Nim +## ```Nim ## import std/json ## ## let jsonNode = parseJson("""{"key": 3.14}""") ## ## doAssert jsonNode["key"].getFloat() == 3.14 +## ``` ## ## **Important:** The `[]` operator will raise an exception when the ## specified field does not exist. @@ -79,7 +81,7 @@ ## when the field is not found. The `get`-family of procedures will return a ## type's default value when called on `nil`. ## -## .. code-block:: Nim +## ```Nim ## import std/json ## ## let jsonNode = parseJson("{}") @@ -88,6 +90,7 @@ ## doAssert jsonNode{"nope"}.getFloat() == 0 ## doAssert jsonNode{"nope"}.getStr() == "" ## doAssert jsonNode{"nope"}.getBool() == false +## ``` ## ## Using default values ## -------------------- @@ -95,7 +98,7 @@ ## The `get`-family helpers also accept an additional parameter which allow ## you to fallback to a default value should the key's values be `null`: ## -## .. code-block:: Nim +## ```Nim ## import std/json ## ## let jsonNode = parseJson("""{"key": 3.14, "key2": null}""") @@ -103,6 +106,7 @@ ## doAssert jsonNode["key"].getFloat(6.28) == 3.14 ## doAssert jsonNode["key2"].getFloat(3.14) == 3.14 ## doAssert jsonNode{"nope"}.getFloat(3.14) == 3.14 # note the {} +## ``` ## ## Unmarshalling ## ------------- @@ -113,7 +117,7 @@ ## Note: Use `Option <options.html#Option>`_ for keys sometimes missing in json ## responses, and backticks around keys with a reserved keyword as name. ## -## .. code-block:: Nim +## ```Nim ## import std/json ## import std/options ## @@ -127,6 +131,7 @@ ## let user = to(userJson, User) ## if user.`type`.isSome(): ## assert user.`type`.get() != "robot" +## ``` ## ## Creating JSON ## ============= @@ -134,7 +139,7 @@ ## This module can also be used to comfortably create JSON using the `%*` ## operator: ## -## .. code-block:: nim +## ```nim ## import std/json ## ## var hisName = "John" @@ -148,6 +153,7 @@ ## var j2 = %* {"name": "Isaac", "books": ["Robot Dreams"]} ## j2["details"] = %* {"age":35, "pi":3.1415} ## echo j2 +## ``` ## ## See also: std/jsonutils for hookable json serialization/deserialization ## of arbitrary types. @@ -159,9 +165,9 @@ runnableExamples: a1, a2, a0, a3, a4: int doAssert $(%* Foo()) == """{"a1":0,"a2":0,"a0":0,"a3":0,"a4":0}""" -import hashes, tables, strutils, lexbase, streams, macros, parsejson +import std/[hashes, tables, strutils, lexbase, streams, macros, parsejson] -import options # xxx remove this dependency using same approach as https://github.com/nim-lang/Nim/pull/14563 +import std/options # xxx remove this dependency using same approach as https://github.com/nim-lang/Nim/pull/14563 import std/private/since when defined(nimPreviewSlimSystem): @@ -438,7 +444,7 @@ macro `%*`*(x: untyped): untyped = ## `%` for every element. result = toJsonImpl(x) -proc `==`*(a, b: JsonNode): bool {.noSideEffect.} = +proc `==`*(a, b: JsonNode): bool {.noSideEffect, raises: [].} = ## Check two nodes for equality if a.isNil: if b.isNil: return true @@ -458,18 +464,20 @@ proc `==`*(a, b: JsonNode): bool {.noSideEffect.} = of JNull: result = true of JArray: - result = a.elems == b.elems + {.cast(raises: []).}: # bug #19303 + result = a.elems == b.elems of JObject: # we cannot use OrderedTable's equality here as # the order does not matter for equality here. if a.fields.len != b.fields.len: return false for key, val in a.fields: if not b.fields.hasKey(key): return false - when defined(nimHasEffectsOf): - {.noSideEffect.}: + {.cast(raises: []).}: + when defined(nimHasEffectsOf): + {.noSideEffect.}: + if b.fields[key] != val: return false + else: if b.fields[key] != val: return false - else: - if b.fields[key] != val: return false result = true proc hash*(n: OrderedTable[string, JsonNode]): Hash {.noSideEffect.} @@ -541,7 +549,7 @@ proc `[]`*[U, V](a: JsonNode, x: HSlice[U, V]): JsonNode = ## ## Returns the inclusive range `[a[x.a], a[x.b]]`: runnableExamples: - import json + import std/json let arr = %[0,1,2,3,4,5] doAssert arr[2..4] == %[2,3,4] doAssert arr[2..^2] == %[2,3,4] @@ -626,7 +634,7 @@ proc delete*(obj: JsonNode, key: string) = obj.fields.del(key) proc copy*(p: JsonNode): JsonNode = - ## Performs a deep copy of `a`. + ## Performs a deep copy of `p`. case p.kind of JString: result = newJString(p.str) @@ -855,7 +863,7 @@ proc parseJson(p: var JsonParser; rawIntegers, rawFloats: bool, depth = 0): Json case p.tok of tkString: # we capture 'p.a' here, so we need to give it a fresh buffer afterwards: - when defined(gcArc) or defined(gcOrc): + when defined(gcArc) or defined(gcOrc) or defined(gcAtomicArc): result = JsonNode(kind: JString, str: move p.a) else: result = JsonNode(kind: JString) @@ -957,7 +965,7 @@ proc parseJson*(s: Stream, filename: string = ""; rawIntegers = false, rawFloats p.close() when defined(js): - from math import `mod` + from std/math import `mod` from std/jsffi import JsObject, `[]`, to from std/private/jsutils import getProtoName, isInteger, isSafeInteger @@ -983,9 +991,9 @@ when defined(js): else: assert false proc len(x: JsObject): int = - asm """ + {.emit: """ `result` = `x`.length; - """ + """.} proc convertObject(x: JsObject): JsonNode = var isRawNumber = false @@ -996,14 +1004,15 @@ when defined(js): result.add(x[i].convertObject()) of JObject: result = newJObject() - asm """for (var property in `x`) { + {.emit: """for (var property in `x`) { if (`x`.hasOwnProperty(property)) { - """ + """.} + var nimProperty: cstring var nimValue: JsObject - asm "`nimProperty` = property; `nimValue` = `x`[property];" + {.emit: "`nimProperty` = property; `nimValue` = `x`[property];".} result[$nimProperty] = nimValue.convertObject() - asm "}}" + {.emit: "}}".} of JInt: result = newJInt(x.to(int)) of JFloat: @@ -1059,313 +1068,306 @@ template verifyJsonKind(node: JsonNode, kinds: set[JsonNodeKind], ] raise newException(JsonKindError, msg) -when defined(nimFixedForwardGeneric): - - macro isRefSkipDistinct*(arg: typed): untyped = - ## internal only, do not use - var impl = getTypeImpl(arg) - if impl.kind == nnkBracketExpr and impl[0].eqIdent("typeDesc"): - impl = getTypeImpl(impl[1]) - while impl.kind == nnkDistinctTy: - impl = getTypeImpl(impl[0]) - result = newLit(impl.kind == nnkRefTy) - - # The following forward declarations don't work in older versions of Nim - - # forward declare all initFromJson - - proc initFromJson(dst: var string; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson(dst: var bool; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson(dst: var JsonNode; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[T: SomeInteger](dst: var T; jsonNode: JsonNode, jsonPath: var string) - proc initFromJson[T: SomeFloat](dst: var T; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[T: enum](dst: var T; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[T](dst: var seq[T]; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[S, T](dst: var array[S, T]; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[T](dst: var Table[string, T]; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[T](dst: var OrderedTable[string, T]; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[T](dst: var ref T; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[T](dst: var Option[T]; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[T: distinct](dst: var T; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[T: object|tuple](dst: var T; jsonNode: JsonNode; jsonPath: var string) - - # initFromJson definitions - - proc initFromJson(dst: var string; jsonNode: JsonNode; jsonPath: var string) = - verifyJsonKind(jsonNode, {JString, JNull}, jsonPath) - # since strings don't have a nil state anymore, this mapping of - # JNull to the default string is questionable. `none(string)` and - # `some("")` have the same potentional json value `JNull`. - if jsonNode.kind == JNull: - dst = "" - else: - dst = jsonNode.str - - proc initFromJson(dst: var bool; jsonNode: JsonNode; jsonPath: var string) = - verifyJsonKind(jsonNode, {JBool}, jsonPath) - dst = jsonNode.bval - - proc initFromJson(dst: var JsonNode; jsonNode: JsonNode; jsonPath: var string) = - if jsonNode == nil: - raise newException(KeyError, "key not found: " & jsonPath) - dst = jsonNode.copy - - proc initFromJson[T: SomeInteger](dst: var T; jsonNode: JsonNode, jsonPath: var string) = - when T is uint|uint64 or (not defined(js) and int.sizeof == 4): - verifyJsonKind(jsonNode, {JInt, JString}, jsonPath) - case jsonNode.kind - of JString: - let x = parseBiggestUInt(jsonNode.str) - dst = cast[T](x) - else: - dst = T(jsonNode.num) - else: - verifyJsonKind(jsonNode, {JInt}, jsonPath) - dst = cast[T](jsonNode.num) - - proc initFromJson[T: SomeFloat](dst: var T; jsonNode: JsonNode; jsonPath: var string) = - if jsonNode.kind == JString: - case jsonNode.str - of "nan": - let b = NaN - dst = T(b) - # dst = NaN # would fail some tests because range conversions would cause CT error - # in some cases; but this is not a hot-spot inside this branch and backend can optimize this. - of "inf": - let b = Inf - dst = T(b) - of "-inf": - let b = -Inf - dst = T(b) - else: raise newException(JsonKindError, "expected 'nan|inf|-inf', got " & jsonNode.str) - else: - verifyJsonKind(jsonNode, {JInt, JFloat}, jsonPath) - if jsonNode.kind == JFloat: - dst = T(jsonNode.fnum) - else: - dst = T(jsonNode.num) - - proc initFromJson[T: enum](dst: var T; jsonNode: JsonNode; jsonPath: var string) = - verifyJsonKind(jsonNode, {JString}, jsonPath) - dst = parseEnum[T](jsonNode.getStr) - - proc initFromJson[T](dst: var seq[T]; jsonNode: JsonNode; jsonPath: var string) = - verifyJsonKind(jsonNode, {JArray}, jsonPath) - dst.setLen jsonNode.len - let orignalJsonPathLen = jsonPath.len - for i in 0 ..< jsonNode.len: - jsonPath.add '[' - jsonPath.addInt i - jsonPath.add ']' - initFromJson(dst[i], jsonNode[i], jsonPath) - jsonPath.setLen orignalJsonPathLen - - proc initFromJson[S,T](dst: var array[S,T]; jsonNode: JsonNode; jsonPath: var string) = - verifyJsonKind(jsonNode, {JArray}, jsonPath) - let originalJsonPathLen = jsonPath.len - for i in 0 ..< jsonNode.len: - jsonPath.add '[' - jsonPath.addInt i - jsonPath.add ']' - initFromJson(dst[i.S], jsonNode[i], jsonPath) # `.S` for enum indexed arrays - jsonPath.setLen originalJsonPathLen - - proc initFromJson[T](dst: var Table[string,T]; jsonNode: JsonNode; jsonPath: var string) = - dst = initTable[string, T]() - verifyJsonKind(jsonNode, {JObject}, jsonPath) - let originalJsonPathLen = jsonPath.len - for key in keys(jsonNode.fields): - jsonPath.add '.' - jsonPath.add key - initFromJson(mgetOrPut(dst, key, default(T)), jsonNode[key], jsonPath) - jsonPath.setLen originalJsonPathLen - - proc initFromJson[T](dst: var OrderedTable[string,T]; jsonNode: JsonNode; jsonPath: var string) = - dst = initOrderedTable[string,T]() - verifyJsonKind(jsonNode, {JObject}, jsonPath) - let originalJsonPathLen = jsonPath.len - for key in keys(jsonNode.fields): - jsonPath.add '.' - jsonPath.add key - initFromJson(mgetOrPut(dst, key, default(T)), jsonNode[key], jsonPath) - jsonPath.setLen originalJsonPathLen - - proc initFromJson[T](dst: var ref T; jsonNode: JsonNode; jsonPath: var string) = - verifyJsonKind(jsonNode, {JObject, JNull}, jsonPath) - if jsonNode.kind == JNull: - dst = nil - else: - dst = new(T) - initFromJson(dst[], jsonNode, jsonPath) +macro isRefSkipDistinct*(arg: typed): untyped = + ## internal only, do not use + var impl = getTypeImpl(arg) + if impl.kind == nnkBracketExpr and impl[0].eqIdent("typeDesc"): + impl = getTypeImpl(impl[1]) + while impl.kind == nnkDistinctTy: + impl = getTypeImpl(impl[0]) + result = newLit(impl.kind == nnkRefTy) + +# The following forward declarations don't work in older versions of Nim + +# forward declare all initFromJson + +proc initFromJson(dst: var string; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson(dst: var bool; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson(dst: var JsonNode; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T: SomeInteger](dst: var T; jsonNode: JsonNode, jsonPath: var string) +proc initFromJson[T: SomeFloat](dst: var T; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T: enum](dst: var T; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T](dst: var seq[T]; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[S, T](dst: var array[S, T]; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T](dst: var Table[string, T]; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T](dst: var OrderedTable[string, T]; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T](dst: var ref T; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T](dst: var Option[T]; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T: distinct](dst: var T; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T: object|tuple](dst: var T; jsonNode: JsonNode; jsonPath: var string) + +# initFromJson definitions + +proc initFromJson(dst: var string; jsonNode: JsonNode; jsonPath: var string) = + verifyJsonKind(jsonNode, {JString, JNull}, jsonPath) + # since strings don't have a nil state anymore, this mapping of + # JNull to the default string is questionable. `none(string)` and + # `some("")` have the same potentional json value `JNull`. + if jsonNode.kind == JNull: + dst = "" + else: + dst = jsonNode.str - proc initFromJson[T](dst: var Option[T]; jsonNode: JsonNode; jsonPath: var string) = - if jsonNode != nil and jsonNode.kind != JNull: - when T is ref: - dst = some(new(T)) - else: - dst = some(default(T)) - initFromJson(dst.get, jsonNode, jsonPath) +proc initFromJson(dst: var bool; jsonNode: JsonNode; jsonPath: var string) = + verifyJsonKind(jsonNode, {JBool}, jsonPath) + dst = jsonNode.bval + +proc initFromJson(dst: var JsonNode; jsonNode: JsonNode; jsonPath: var string) = + if jsonNode == nil: + raise newException(KeyError, "key not found: " & jsonPath) + dst = jsonNode.copy - macro assignDistinctImpl[T: distinct](dst: var T;jsonNode: JsonNode; jsonPath: var string) = - let typInst = getTypeInst(dst) - let typImpl = getTypeImpl(dst) - let baseTyp = typImpl[0] +proc initFromJson[T: SomeInteger](dst: var T; jsonNode: JsonNode, jsonPath: var string) = + when T is uint|uint64 or int.sizeof == 4: + verifyJsonKind(jsonNode, {JInt, JString}, jsonPath) + case jsonNode.kind + of JString: + let x = parseBiggestUInt(jsonNode.str) + dst = cast[T](x) + else: + dst = T(jsonNode.num) + else: + verifyJsonKind(jsonNode, {JInt}, jsonPath) + dst = cast[T](jsonNode.num) + +proc initFromJson[T: SomeFloat](dst: var T; jsonNode: JsonNode; jsonPath: var string) = + verifyJsonKind(jsonNode, {JInt, JFloat, JString}, jsonPath) + if jsonNode.kind == JString: + case jsonNode.str + of "nan": + let b = NaN + dst = T(b) + # dst = NaN # would fail some tests because range conversions would cause CT error + # in some cases; but this is not a hot-spot inside this branch and backend can optimize this. + of "inf": + let b = Inf + dst = T(b) + of "-inf": + let b = -Inf + dst = T(b) + else: raise newException(JsonKindError, "expected 'nan|inf|-inf', got " & jsonNode.str) + else: + if jsonNode.kind == JFloat: + dst = T(jsonNode.fnum) + else: + dst = T(jsonNode.num) + +proc initFromJson[T: enum](dst: var T; jsonNode: JsonNode; jsonPath: var string) = + verifyJsonKind(jsonNode, {JString}, jsonPath) + dst = parseEnum[T](jsonNode.getStr) + +proc initFromJson[T](dst: var seq[T]; jsonNode: JsonNode; jsonPath: var string) = + verifyJsonKind(jsonNode, {JArray}, jsonPath) + dst.setLen jsonNode.len + let orignalJsonPathLen = jsonPath.len + for i in 0 ..< jsonNode.len: + jsonPath.add '[' + jsonPath.addInt i + jsonPath.add ']' + initFromJson(dst[i], jsonNode[i], jsonPath) + jsonPath.setLen orignalJsonPathLen + +proc initFromJson[S,T](dst: var array[S,T]; jsonNode: JsonNode; jsonPath: var string) = + verifyJsonKind(jsonNode, {JArray}, jsonPath) + let originalJsonPathLen = jsonPath.len + for i in 0 ..< jsonNode.len: + jsonPath.add '[' + jsonPath.addInt i + jsonPath.add ']' + initFromJson(dst[i.S], jsonNode[i], jsonPath) # `.S` for enum indexed arrays + jsonPath.setLen originalJsonPathLen + +proc initFromJson[T](dst: var Table[string,T]; jsonNode: JsonNode; jsonPath: var string) = + dst = initTable[string, T]() + verifyJsonKind(jsonNode, {JObject}, jsonPath) + let originalJsonPathLen = jsonPath.len + for key in keys(jsonNode.fields): + jsonPath.add '.' + jsonPath.add key + initFromJson(mgetOrPut(dst, key, default(T)), jsonNode[key], jsonPath) + jsonPath.setLen originalJsonPathLen + +proc initFromJson[T](dst: var OrderedTable[string,T]; jsonNode: JsonNode; jsonPath: var string) = + dst = initOrderedTable[string,T]() + verifyJsonKind(jsonNode, {JObject}, jsonPath) + let originalJsonPathLen = jsonPath.len + for key in keys(jsonNode.fields): + jsonPath.add '.' + jsonPath.add key + initFromJson(mgetOrPut(dst, key, default(T)), jsonNode[key], jsonPath) + jsonPath.setLen originalJsonPathLen + +proc initFromJson[T](dst: var ref T; jsonNode: JsonNode; jsonPath: var string) = + verifyJsonKind(jsonNode, {JObject, JNull}, jsonPath) + if jsonNode.kind == JNull: + dst = nil + else: + dst = new(T) + initFromJson(dst[], jsonNode, jsonPath) - result = quote do: +proc initFromJson[T](dst: var Option[T]; jsonNode: JsonNode; jsonPath: var string) = + if jsonNode != nil and jsonNode.kind != JNull: + when T is ref: + dst = some(new(T)) + else: + dst = some(default(T)) + initFromJson(dst.get, jsonNode, jsonPath) + +macro assignDistinctImpl[T: distinct](dst: var T;jsonNode: JsonNode; jsonPath: var string) = + let typInst = getTypeInst(dst) + let typImpl = getTypeImpl(dst) + let baseTyp = typImpl[0] + + result = quote do: + initFromJson(`baseTyp`(`dst`), `jsonNode`, `jsonPath`) + +proc initFromJson[T: distinct](dst: var T; jsonNode: JsonNode; jsonPath: var string) = + assignDistinctImpl(dst, jsonNode, jsonPath) + +proc detectIncompatibleType(typeExpr, lineinfoNode: NimNode) = + if typeExpr.kind == nnkTupleConstr: + error("Use a named tuple instead of: " & typeExpr.repr, lineinfoNode) + +proc foldObjectBody(dst, typeNode, tmpSym, jsonNode, jsonPath, originalJsonPathLen: NimNode) = + case typeNode.kind + of nnkEmpty: + discard + of nnkRecList, nnkTupleTy: + for it in typeNode: + foldObjectBody(dst, it, tmpSym, jsonNode, jsonPath, originalJsonPathLen) + + of nnkIdentDefs: + typeNode.expectLen 3 + let fieldSym = typeNode[0] + let fieldNameLit = newLit(fieldSym.strVal) + let fieldPathLit = newLit("." & fieldSym.strVal) + let fieldType = typeNode[1] + + # Detecting incompatiple tuple types in `assignObjectImpl` only + # would be much cleaner, but the ast for tuple types does not + # contain usable type information. + detectIncompatibleType(fieldType, fieldSym) + + dst.add quote do: + jsonPath.add `fieldPathLit` when nimvm: - # workaround #12282 - var tmp: `baseTyp` - initFromJson( tmp, `jsonNode`, `jsonPath`) - `dst` = `typInst`(tmp) - else: - initFromJson( `baseTyp`(`dst`), `jsonNode`, `jsonPath`) - - proc initFromJson[T: distinct](dst: var T; jsonNode: JsonNode; jsonPath: var string) = - assignDistinctImpl(dst, jsonNode, jsonPath) - - proc detectIncompatibleType(typeExpr, lineinfoNode: NimNode) = - if typeExpr.kind == nnkTupleConstr: - error("Use a named tuple instead of: " & typeExpr.repr, lineinfoNode) - - proc foldObjectBody(dst, typeNode, tmpSym, jsonNode, jsonPath, originalJsonPathLen: NimNode) = - case typeNode.kind - of nnkEmpty: - discard - of nnkRecList, nnkTupleTy: - for it in typeNode: - foldObjectBody(dst, it, tmpSym, jsonNode, jsonPath, originalJsonPathLen) - - of nnkIdentDefs: - typeNode.expectLen 3 - let fieldSym = typeNode[0] - let fieldNameLit = newLit(fieldSym.strVal) - let fieldPathLit = newLit("." & fieldSym.strVal) - let fieldType = typeNode[1] - - # Detecting incompatiple tuple types in `assignObjectImpl` only - # would be much cleaner, but the ast for tuple types does not - # contain usable type information. - detectIncompatibleType(fieldType, fieldSym) - - dst.add quote do: - jsonPath.add `fieldPathLit` - when nimvm: - when isRefSkipDistinct(`tmpSym`.`fieldSym`): - # workaround #12489 - var tmp: `fieldType` - initFromJson(tmp, getOrDefault(`jsonNode`,`fieldNameLit`), `jsonPath`) - `tmpSym`.`fieldSym` = tmp - else: - initFromJson(`tmpSym`.`fieldSym`, getOrDefault(`jsonNode`,`fieldNameLit`), `jsonPath`) + when isRefSkipDistinct(`tmpSym`.`fieldSym`): + # workaround #12489 + var tmp: `fieldType` + initFromJson(tmp, getOrDefault(`jsonNode`,`fieldNameLit`), `jsonPath`) + `tmpSym`.`fieldSym` = tmp else: initFromJson(`tmpSym`.`fieldSym`, getOrDefault(`jsonNode`,`fieldNameLit`), `jsonPath`) - jsonPath.setLen `originalJsonPathLen` - - of nnkRecCase: - let kindSym = typeNode[0][0] - let kindNameLit = newLit(kindSym.strVal) - let kindPathLit = newLit("." & kindSym.strVal) - let kindType = typeNode[0][1] - let kindOffsetLit = newLit(uint(getOffset(kindSym))) - dst.add quote do: - var kindTmp: `kindType` - jsonPath.add `kindPathLit` - initFromJson(kindTmp, `jsonNode`[`kindNameLit`], `jsonPath`) - jsonPath.setLen `originalJsonPathLen` - when defined js: + else: + initFromJson(`tmpSym`.`fieldSym`, getOrDefault(`jsonNode`,`fieldNameLit`), `jsonPath`) + jsonPath.setLen `originalJsonPathLen` + + of nnkRecCase: + let kindSym = typeNode[0][0] + let kindNameLit = newLit(kindSym.strVal) + let kindPathLit = newLit("." & kindSym.strVal) + let kindType = typeNode[0][1] + let kindOffsetLit = newLit(uint(getOffset(kindSym))) + dst.add quote do: + var kindTmp: `kindType` + jsonPath.add `kindPathLit` + initFromJson(kindTmp, `jsonNode`[`kindNameLit`], `jsonPath`) + jsonPath.setLen `originalJsonPathLen` + when defined js: + `tmpSym`.`kindSym` = kindTmp + else: + when nimvm: `tmpSym`.`kindSym` = kindTmp else: - when nimvm: - `tmpSym`.`kindSym` = kindTmp - else: - # fuck it, assign kind field anyway - ((cast[ptr `kindType`](cast[uint](`tmpSym`.addr) + `kindOffsetLit`))[]) = kindTmp - dst.add nnkCaseStmt.newTree(nnkDotExpr.newTree(tmpSym, kindSym)) - for i in 1 ..< typeNode.len: - foldObjectBody(dst, typeNode[i], tmpSym, jsonNode, jsonPath, originalJsonPathLen) - - of nnkOfBranch, nnkElse: - let ofBranch = newNimNode(typeNode.kind) - for i in 0 ..< typeNode.len-1: - ofBranch.add copyNimTree(typeNode[i]) - let dstInner = newNimNode(nnkStmtListExpr) - foldObjectBody(dstInner, typeNode[^1], tmpSym, jsonNode, jsonPath, originalJsonPathLen) - # resOuter now contains the inner stmtList - ofBranch.add dstInner - dst[^1].expectKind nnkCaseStmt - dst[^1].add ofBranch - - of nnkObjectTy: - typeNode[0].expectKind nnkEmpty - typeNode[1].expectKind {nnkEmpty, nnkOfInherit} - if typeNode[1].kind == nnkOfInherit: - let base = typeNode[1][0] - var impl = getTypeImpl(base) - while impl.kind in {nnkRefTy, nnkPtrTy}: - impl = getTypeImpl(impl[0]) - foldObjectBody(dst, impl, tmpSym, jsonNode, jsonPath, originalJsonPathLen) - let body = typeNode[2] - foldObjectBody(dst, body, tmpSym, jsonNode, jsonPath, originalJsonPathLen) + # fuck it, assign kind field anyway + ((cast[ptr `kindType`](cast[uint](`tmpSym`.addr) + `kindOffsetLit`))[]) = kindTmp + dst.add nnkCaseStmt.newTree(nnkDotExpr.newTree(tmpSym, kindSym)) + for i in 1 ..< typeNode.len: + foldObjectBody(dst, typeNode[i], tmpSym, jsonNode, jsonPath, originalJsonPathLen) + + of nnkOfBranch, nnkElse: + let ofBranch = newNimNode(typeNode.kind) + for i in 0 ..< typeNode.len-1: + ofBranch.add copyNimTree(typeNode[i]) + let dstInner = newNimNode(nnkStmtListExpr) + foldObjectBody(dstInner, typeNode[^1], tmpSym, jsonNode, jsonPath, originalJsonPathLen) + # resOuter now contains the inner stmtList + ofBranch.add dstInner + dst[^1].expectKind nnkCaseStmt + dst[^1].add ofBranch + + of nnkObjectTy: + typeNode[0].expectKind nnkEmpty + typeNode[1].expectKind {nnkEmpty, nnkOfInherit} + if typeNode[1].kind == nnkOfInherit: + let base = typeNode[1][0] + var impl = getTypeImpl(base) + while impl.kind in {nnkRefTy, nnkPtrTy}: + impl = getTypeImpl(impl[0]) + foldObjectBody(dst, impl, tmpSym, jsonNode, jsonPath, originalJsonPathLen) + let body = typeNode[2] + foldObjectBody(dst, body, tmpSym, jsonNode, jsonPath, originalJsonPathLen) - else: - error("unhandled kind: " & $typeNode.kind, typeNode) - - macro assignObjectImpl[T](dst: var T; jsonNode: JsonNode; jsonPath: var string) = - let typeSym = getTypeInst(dst) - let originalJsonPathLen = genSym(nskLet, "originalJsonPathLen") - result = newStmtList() - result.add quote do: - let `originalJsonPathLen` = len(`jsonPath`) - if typeSym.kind in {nnkTupleTy, nnkTupleConstr}: - # both, `dst` and `typeSym` don't have good lineinfo. But nothing - # else is available here. - detectIncompatibleType(typeSym, dst) - foldObjectBody(result, typeSym, dst, jsonNode, jsonPath, originalJsonPathLen) - else: - foldObjectBody(result, typeSym.getTypeImpl, dst, jsonNode, jsonPath, originalJsonPathLen) - - proc initFromJson[T: object|tuple](dst: var T; jsonNode: JsonNode; jsonPath: var string) = - assignObjectImpl(dst, jsonNode, jsonPath) - - proc to*[T](node: JsonNode, t: typedesc[T]): T = - ## `Unmarshals`:idx: the specified node into the object type specified. - ## - ## Known limitations: - ## - ## * Heterogeneous arrays are not supported. - ## * Sets in object variants are not supported. - ## * Not nil annotations are not supported. - ## - runnableExamples: - let jsonNode = parseJson(""" - { - "person": { - "name": "Nimmer", - "age": 21 - }, - "list": [1, 2, 3, 4] - } - """) - - type - Person = object - name: string - age: int - - Data = object - person: Person - list: seq[int] - - var data = to(jsonNode, Data) - doAssert data.person.name == "Nimmer" - doAssert data.person.age == 21 - doAssert data.list == @[1, 2, 3, 4] - - var jsonPath = "" - initFromJson(result, node, jsonPath) + else: + error("unhandled kind: " & $typeNode.kind, typeNode) + +macro assignObjectImpl[T](dst: var T; jsonNode: JsonNode; jsonPath: var string) = + let typeSym = getTypeInst(dst) + let originalJsonPathLen = genSym(nskLet, "originalJsonPathLen") + result = newStmtList() + result.add quote do: + let `originalJsonPathLen` = len(`jsonPath`) + if typeSym.kind in {nnkTupleTy, nnkTupleConstr}: + # both, `dst` and `typeSym` don't have good lineinfo. But nothing + # else is available here. + detectIncompatibleType(typeSym, dst) + foldObjectBody(result, typeSym, dst, jsonNode, jsonPath, originalJsonPathLen) + else: + foldObjectBody(result, typeSym.getTypeImpl, dst, jsonNode, jsonPath, originalJsonPathLen) + +proc initFromJson[T: object|tuple](dst: var T; jsonNode: JsonNode; jsonPath: var string) = + assignObjectImpl(dst, jsonNode, jsonPath) + +proc to*[T](node: JsonNode, t: typedesc[T]): T = + ## `Unmarshals`:idx: the specified node into the object type specified. + ## + ## Known limitations: + ## + ## * Heterogeneous arrays are not supported. + ## * Sets in object variants are not supported. + ## * Not nil annotations are not supported. + ## + runnableExamples: + let jsonNode = parseJson(""" + { + "person": { + "name": "Nimmer", + "age": 21 + }, + "list": [1, 2, 3, 4] + } + """) + + type + Person = object + name: string + age: int + + Data = object + person: Person + list: seq[int] + + var data = to(jsonNode, Data) + doAssert data.person.name == "Nimmer" + doAssert data.person.age == 21 + doAssert data.list == @[1, 2, 3, 4] + + var jsonPath = "" + result = default(T) + initFromJson(result, node, jsonPath) when false: - import os + import std/os var s = newFileStream(paramStr(1), fmRead) if s == nil: quit("cannot open the file" & paramStr(1)) var x: JsonParser diff --git a/lib/pure/lexbase.nim b/lib/pure/lexbase.nim index 336a57ec1..1efd97b24 100644 --- a/lib/pure/lexbase.nim +++ b/lib/pure/lexbase.nim @@ -12,7 +12,7 @@ ## needs refilling. import - strutils, streams + std/[strutils, streams] when defined(nimPreviewSlimSystem): import std/assertions @@ -104,9 +104,9 @@ proc fillBaseLexer(L: var BaseLexer, pos: int): int = result = 0 proc handleCR*(L: var BaseLexer, pos: int): int = - ## Call this if you scanned over '\c' in the buffer; it returns the + ## Call this if you scanned over `'\c'` in the buffer; it returns the ## position to continue the scanning from. `pos` must be the position - ## of the '\c'. + ## of the `'\c'`. assert(L.buf[pos] == '\c') inc(L.lineNumber) result = fillBaseLexer(L, pos) @@ -115,9 +115,9 @@ proc handleCR*(L: var BaseLexer, pos: int): int = L.lineStart = result proc handleLF*(L: var BaseLexer, pos: int): int = - ## Call this if you scanned over '\L' in the buffer; it returns the + ## Call this if you scanned over `'\L'` in the buffer; it returns the ## position to continue the scanning from. `pos` must be the position - ## of the '\L'. + ## of the `'\L'`. assert(L.buf[pos] == '\L') inc(L.lineNumber) result = fillBaseLexer(L, pos) #L.lastNL := result-1; // BUGFIX: was: result; diff --git a/lib/pure/logging.nim b/lib/pure/logging.nim index cbe8a827a..c30f68af8 100644 --- a/lib/pure/logging.nim +++ b/lib/pure/logging.nim @@ -17,10 +17,11 @@ ## ## To get started, first create a logger: ## -## .. code-block:: +## ```Nim ## import std/logging ## ## var logger = newConsoleLogger() +## ``` ## ## The logger that was created above logs to the console, but this module ## also provides loggers that log to files, such as the @@ -30,9 +31,10 @@ ## Once a logger has been created, call its `log proc ## <#log.e,ConsoleLogger,Level,varargs[string,]>`_ to log a message: ## -## .. code-block:: +## ```Nim ## logger.log(lvlInfo, "a log message") ## # Output: INFO a log message +## ``` ## ## The ``INFO`` within the output is the result of a format string being ## prepended to the message, and it will differ depending on the message's @@ -47,9 +49,8 @@ ## ## .. warning:: ## For loggers that log to a console or to files, only error and fatal -## messages will cause their output buffers to be flushed immediately. -## Use the `flushFile proc <syncio.html#flushFile,File>`_ to flush the buffer -## manually if needed. +## messages will cause their output buffers to be flushed immediately by default. +## set ``flushThreshold`` when creating the logger to change this. ## ## Handlers ## -------- @@ -59,7 +60,7 @@ ## used with the `addHandler proc<#addHandler,Logger>`_, which is demonstrated ## in the following example: ## -## .. code-block:: +## ```Nim ## import std/logging ## ## var consoleLog = newConsoleLogger() @@ -69,17 +70,19 @@ ## addHandler(consoleLog) ## addHandler(fileLog) ## addHandler(rollingLog) +## ``` ## ## After doing this, use either the `log template ## <#log.t,Level,varargs[string,]>`_ or one of the level-specific templates, ## such as the `error template<#error.t,varargs[string,]>`_, to log messages ## to all registered handlers at once. ## -## .. code-block:: +## ```Nim ## # This example uses the loggers created above ## log(lvlError, "an error occurred") ## error("an error occurred") # Equivalent to the above line ## info("something normal happened") # Will not be written to errors.log +## ``` ## ## Note that a message's level is still checked against each handler's ## ``levelThreshold`` and the global log filter. @@ -117,12 +120,13 @@ ## ## The following example illustrates how to use format strings: ## -## .. code-block:: +## ```Nim ## import std/logging ## ## var logger = newConsoleLogger(fmtStr="[$time] - $levelname: ") ## logger.log(lvlInfo, "this is a message") ## # Output: [19:50:13] - INFO: this is a message +## ``` ## ## Notes when using multiple threads ## --------------------------------- @@ -142,9 +146,9 @@ ## * `strscans module<strscans.html>`_ for ``scanf`` and ``scanp`` macros, which ## offer easier substring extraction than regular expressions -import strutils, times +import std/[strutils, times] when not defined(js): - import os + import std/os when defined(nimPreviewSlimSystem): import std/syncio @@ -200,6 +204,15 @@ const ## If a different format string is preferred, refer to the ## `documentation about format strings<#basic-usage-format-strings>`_ ## for more information, including a list of available variables. + defaultFlushThreshold = when NimMajor >= 2: + when defined(nimV1LogFlushBehavior): lvlError else: lvlAll + else: + when defined(nimFlushAllLogs): lvlAll else: lvlError + ## The threshold above which log messages to file-like loggers + ## are automatically flushed. + ## + ## By default, only error and fatal messages are logged, + ## but defining ``-d:nimFlushAllLogs`` will make all levels be flushed type Logger* = ref object of RootObj @@ -228,6 +241,8 @@ type ## * `FileLogger<#FileLogger>`_ ## * `RollingFileLogger<#RollingFileLogger>`_ useStderr*: bool ## If true, writes to stderr; otherwise, writes to stdout + flushThreshold*: Level ## Only messages that are at or above this + ## threshold will be flushed immediately when not defined(js): type @@ -243,6 +258,8 @@ when not defined(js): ## * `ConsoleLogger<#ConsoleLogger>`_ ## * `RollingFileLogger<#RollingFileLogger>`_ file*: File ## The wrapped file + flushThreshold*: Level ## Only messages that are at or above this + ## threshold will be flushed immediately RollingFileLogger* = ref object of FileLogger ## A logger that writes log messages to a file while performing log @@ -348,8 +365,8 @@ method log*(logger: ConsoleLogger, level: Level, args: varargs[string, `$`]) = ## `setLogFilter proc<#setLogFilter,Level>`_. ## ## **Note:** Only error and fatal messages will cause the output buffer - ## to be flushed immediately. Use the `flushFile proc - ## <syncio.html#flushFile,File>`_ to flush the buffer manually if needed. + ## to be flushed immediately by default. Set ``flushThreshold`` when creating + ## the logger to change this. ## ## See also: ## * `log method<#log.e,FileLogger,Level,varargs[string,]>`_ @@ -360,10 +377,11 @@ method log*(logger: ConsoleLogger, level: Level, args: varargs[string, `$`]) = ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## var consoleLog = newConsoleLogger() ## consoleLog.log(lvlInfo, "this is a message") ## consoleLog.log(lvlError, "error code is: ", 404) + ## ``` if level >= logging.level and level >= logger.levelThreshold: let ln = substituteLog(logger.fmtStr, level, args) when defined(js): @@ -380,12 +398,12 @@ method log*(logger: ConsoleLogger, level: Level, args: varargs[string, `$`]) = if logger.useStderr: handle = stderr writeLine(handle, ln) - if level in {lvlError, lvlFatal}: flushFile(handle) + if level >= logger.flushThreshold: flushFile(handle) except IOError: discard proc newConsoleLogger*(levelThreshold = lvlAll, fmtStr = defaultFmtStr, - useStderr = false): ConsoleLogger = + useStderr = false, flushThreshold = defaultFlushThreshold): ConsoleLogger = ## Creates a new `ConsoleLogger<#ConsoleLogger>`_. ## ## By default, log messages are written to ``stdout``. If ``useStderr`` is @@ -402,13 +420,15 @@ proc newConsoleLogger*(levelThreshold = lvlAll, fmtStr = defaultFmtStr, ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## var normalLog = newConsoleLogger() ## var formatLog = newConsoleLogger(fmtStr=verboseFmtStr) ## var errorLog = newConsoleLogger(levelThreshold=lvlError, useStderr=true) + ## ``` new result result.fmtStr = fmtStr result.levelThreshold = levelThreshold + result.flushThreshold = flushThreshold result.useStderr = useStderr when not defined(js): @@ -424,8 +444,8 @@ when not defined(js): ## ## **Notes:** ## * Only error and fatal messages will cause the output buffer - ## to be flushed immediately. Use the `flushFile proc - ## <syncio.html#flushFile,File>`_ to flush the buffer manually if needed. + ## to be flushed immediately by default. Set ``flushThreshold`` when creating + ## the logger to change this. ## * This method is not available for the JavaScript backend. ## ## See also: @@ -437,13 +457,14 @@ when not defined(js): ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## var fileLog = newFileLogger("messages.log") ## fileLog.log(lvlInfo, "this is a message") ## fileLog.log(lvlError, "error code is: ", 404) + ## ``` if level >= logging.level and level >= logger.levelThreshold: writeLine(logger.file, substituteLog(logger.fmtStr, level, args)) - if level in {lvlError, lvlFatal}: flushFile(logger.file) + if level >= logger.flushThreshold: flushFile(logger.file) proc defaultFilename*(): string = ## Returns the filename that is used by default when naming log files. @@ -454,7 +475,8 @@ when not defined(js): proc newFileLogger*(file: File, levelThreshold = lvlAll, - fmtStr = defaultFmtStr): FileLogger = + fmtStr = defaultFmtStr, + flushThreshold = defaultFlushThreshold): FileLogger = ## Creates a new `FileLogger<#FileLogger>`_ that uses the given file handle. ## ## **Note:** This proc is not available for the JavaScript backend. @@ -467,7 +489,7 @@ when not defined(js): ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## var messages = open("messages.log", fmWrite) ## var formatted = open("formatted.log", fmWrite) ## var errors = open("errors.log", fmWrite) @@ -475,16 +497,19 @@ when not defined(js): ## var normalLog = newFileLogger(messages) ## var formatLog = newFileLogger(formatted, fmtStr=verboseFmtStr) ## var errorLog = newFileLogger(errors, levelThreshold=lvlError) + ## ``` new(result) result.file = file result.levelThreshold = levelThreshold + result.flushThreshold = flushThreshold result.fmtStr = fmtStr proc newFileLogger*(filename = defaultFilename(), mode: FileMode = fmAppend, levelThreshold = lvlAll, fmtStr = defaultFmtStr, - bufSize: int = -1): FileLogger = + bufSize: int = -1, + flushThreshold = defaultFlushThreshold): FileLogger = ## Creates a new `FileLogger<#FileLogger>`_ that logs to a file with the ## given filename. ## @@ -503,12 +528,13 @@ when not defined(js): ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## var normalLog = newFileLogger("messages.log") ## var formatLog = newFileLogger("formatted.log", fmtStr=verboseFmtStr) ## var errorLog = newFileLogger("errors.log", levelThreshold=lvlError) + ## ``` let file = open(filename, mode, bufSize = bufSize) - newFileLogger(file, levelThreshold, fmtStr) + newFileLogger(file, levelThreshold, fmtStr, flushThreshold) # ------ @@ -540,7 +566,8 @@ when not defined(js): levelThreshold = lvlAll, fmtStr = defaultFmtStr, maxLines: Positive = 1000, - bufSize: int = -1): RollingFileLogger = + bufSize: int = -1, + flushThreshold = defaultFlushThreshold): RollingFileLogger = ## Creates a new `RollingFileLogger<#RollingFileLogger>`_. ## ## Once the current log file being written to contains ``maxLines`` lines, @@ -562,11 +589,12 @@ when not defined(js): ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## var normalLog = newRollingFileLogger("messages.log") ## var formatLog = newRollingFileLogger("formatted.log", fmtStr=verboseFmtStr) ## var shortLog = newRollingFileLogger("short.log", maxLines=200) ## var errorLog = newRollingFileLogger("errors.log", levelThreshold=lvlError) + ## ``` new(result) result.levelThreshold = levelThreshold result.fmtStr = fmtStr @@ -576,6 +604,7 @@ when not defined(js): result.curLine = 0 result.baseName = filename result.baseMode = mode + result.flushThreshold = flushThreshold result.logFiles = countFiles(filename) @@ -602,8 +631,8 @@ when not defined(js): ## ## **Notes:** ## * Only error and fatal messages will cause the output buffer - ## to be flushed immediately. Use the `flushFile proc - ## <syncio.html#flushFile,File>`_ to flush the buffer manually if needed. + ## to be flushed immediately by default. Set ``flushThreshold`` when creating + ## the logger to change this. ## * This method is not available for the JavaScript backend. ## ## See also: @@ -615,10 +644,11 @@ when not defined(js): ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## var rollingLog = newRollingFileLogger("messages.log") ## rollingLog.log(lvlInfo, "this is a message") ## rollingLog.log(lvlError, "error code is: ", 404) + ## ``` if level >= logging.level and level >= logger.levelThreshold: if logger.curLine >= logger.maxLines: logger.file.close() @@ -629,7 +659,7 @@ when not defined(js): bufSize = logger.bufSize) writeLine(logger.file, substituteLog(logger.fmtStr, level, args)) - if level in {lvlError, lvlFatal}: flushFile(logger.file) + if level >= logger.flushThreshold: flushFile(logger.file) logger.curLine.inc # -------- @@ -648,11 +678,12 @@ template log*(level: Level, args: varargs[string, `$`]) = ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## var logger = newConsoleLogger() ## addHandler(logger) ## ## log(lvlInfo, "This is an example.") + ## ``` ## ## See also: ## * `debug template<#debug.t,varargs[string,]>`_ @@ -677,11 +708,12 @@ template debug*(args: varargs[string, `$`]) = ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## var logger = newConsoleLogger() ## addHandler(logger) ## ## debug("myProc called with arguments: foo, 5") + ## ``` ## ## See also: ## * `log template<#log.t,Level,varargs[string,]>`_ @@ -698,11 +730,12 @@ template info*(args: varargs[string, `$`]) = ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## var logger = newConsoleLogger() ## addHandler(logger) ## ## info("Application started successfully.") + ## ``` ## ## See also: ## * `log template<#log.t,Level,varargs[string,]>`_ @@ -719,11 +752,12 @@ template notice*(args: varargs[string, `$`]) = ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## var logger = newConsoleLogger() ## addHandler(logger) ## ## notice("An important operation has completed.") + ## ``` ## ## See also: ## * `log template<#log.t,Level,varargs[string,]>`_ @@ -739,11 +773,12 @@ template warn*(args: varargs[string, `$`]) = ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## var logger = newConsoleLogger() ## addHandler(logger) ## ## warn("The previous operation took too long to process.") + ## ``` ## ## See also: ## * `log template<#log.t,Level,varargs[string,]>`_ @@ -761,11 +796,12 @@ template error*(args: varargs[string, `$`]) = ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## var logger = newConsoleLogger() ## addHandler(logger) ## ## error("An exception occurred while processing the form.") + ## ``` ## ## See also: ## * `log template<#log.t,Level,varargs[string,]>`_ @@ -782,11 +818,12 @@ template fatal*(args: varargs[string, `$`]) = ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## var logger = newConsoleLogger() ## addHandler(logger) ## ## fatal("Can't open database -- exiting.") + ## ``` ## ## See also: ## * `log template<#log.t,Level,varargs[string,]>`_ @@ -802,6 +839,7 @@ proc addHandler*(handler: Logger) = ## each of those threads. ## ## See also: + ## * `removeHandler proc`_ ## * `getHandlers proc<#getHandlers>`_ runnableExamples: var logger = newConsoleLogger() @@ -809,6 +847,16 @@ proc addHandler*(handler: Logger) = doAssert logger in getHandlers() handlers.add(handler) +proc removeHandler*(handler: Logger) = + ## Removes a logger from the list of registered handlers. + ## + ## Note that for n times a logger is registered, n calls to this proc + ## are required to remove that logger. + for i, hnd in handlers: + if hnd == handler: + handlers.delete(i) + return + proc getHandlers*(): seq[Logger] = ## Returns a list of all the registered handlers. ## diff --git a/lib/pure/marshal.nim b/lib/pure/marshal.nim index 848d7e3fb..f9b3d3e4c 100644 --- a/lib/pure/marshal.nim +++ b/lib/pure/marshal.nim @@ -54,7 +54,7 @@ Please use alternative packages for serialization. It is possible to reimplement this module using generics and type traits. Please contribute a new implementation.""".} -import streams, typeinfo, json, intsets, tables, unicode +import std/[streams, typeinfo, json, intsets, tables, unicode] when defined(nimPreviewSlimSystem): import std/[assertions, formatfloat] @@ -210,7 +210,8 @@ proc loadAny(p: var JsonParser, a: Any, t: var Table[BiggestInt, pointer]) = setPointer(a, nil) next(p) of jsonInt: - setPointer(a, t.getOrDefault(p.getInt)) + var raw = t.getOrDefault(p.getInt) + setPointer(a, addr raw) next(p) of jsonArrayStart: next(p) @@ -304,7 +305,7 @@ proc store*[T](s: Stream, data: sink T) = var stored = initIntSet() var d: T - when defined(gcArc) or defined(gcOrc): + when defined(gcArc) or defined(gcOrc)or defined(gcAtomicArc): d = data else: shallowCopy(d, data) @@ -333,7 +334,7 @@ proc `$$`*[T](x: sink T): string = else: var stored = initIntSet() var d: T - when defined(gcArc) or defined(gcOrc): + when defined(gcArc) or defined(gcOrc) or defined(gcAtomicArc): d = x else: shallowCopy(d, x) diff --git a/lib/pure/math.nim b/lib/pure/math.nim index 15324f882..ed7d2382f 100644 --- a/lib/pure/math.nim +++ b/lib/pure/math.nim @@ -47,7 +47,6 @@ runnableExamples: ## * `fenv module <fenv.html>`_ for handling of floating-point rounding ## and exceptions (overflow, zero-divide, etc.) ## * `random module <random.html>`_ for a fast and tiny random number generator -## * `mersenne module <mersenne.html>`_ for the Mersenne Twister random number generator ## * `stats module <stats.html>`_ for statistical analysis ## * `strformat module <strformat.html>`_ for formatting floats for printing ## * `system module <system.html>`_ for some very basic and trivial math operators @@ -58,13 +57,14 @@ import std/private/since {.push debugger: off.} # the user does not want to trace a part # of the standard library! -import bitops, fenv +import std/[bitops, fenv] +import system/countbits_impl when defined(nimPreviewSlimSystem): import std/assertions -when defined(c) or defined(cpp): +when not defined(js) and not defined(nimscript): # C proc c_isnan(x: float): bool {.importc: "isnan", header: "<math.h>".} # a generic like `x: SomeFloat` might work too if this is implemented via a C macro. @@ -78,6 +78,41 @@ when defined(c) or defined(cpp): importc: "frexpf", header: "<math.h>".} func c_frexp2(x: cdouble, exponent: var cint): cdouble {. importc: "frexp", header: "<math.h>".} + + type + div_t {.importc, header: "<stdlib.h>".} = object + quot: cint + rem: cint + ldiv_t {.importc, header: "<stdlib.h>".} = object + quot: clong + rem: clong + lldiv_t {.importc, header: "<stdlib.h>".} = object + quot: clonglong + rem: clonglong + + when cint isnot clong: + func divmod_c(x, y: cint): div_t {.importc: "div", header: "<stdlib.h>".} + when clong isnot clonglong: + func divmod_c(x, y: clonglong): lldiv_t {.importc: "lldiv", header: "<stdlib.h>".} + func divmod_c(x, y: clong): ldiv_t {.importc: "ldiv", header: "<stdlib.h>".} + func divmod*[T: SomeInteger](x, y: T): (T, T) {.inline.} = + ## Specialized instructions for computing both division and modulus. + ## Return structure is: (quotient, remainder) + runnableExamples: + doAssert divmod(5, 2) == (2, 1) + doAssert divmod(5, -3) == (-1, 2) + when T is cint | clong | clonglong: + when compileOption("overflowChecks"): + if y == 0: + raise new(DivByZeroDefect) + elif (x == T.low and y == -1.T): + raise new(OverflowDefect) + let res = divmod_c(x, y) + result[0] = res.quot + result[1] = res.rem + else: + result[0] = x div y + result[1] = x mod y func binom*(n, k: int): int = ## Computes the [binomial coefficient](https://en.wikipedia.org/wiki/Binomial_coefficient). @@ -121,7 +156,7 @@ func fac*(n: int): int = {.push checks: off, line_dir: off, stack_trace: off.} -when defined(posix) and not defined(genode): +when defined(posix) and not defined(genode) and not defined(macosx): {.passl: "-lm".} const @@ -166,7 +201,7 @@ func isNaN*(x: SomeFloat): bool {.inline, since: (1,5,1).} = template fn: untyped = result = x != x when nimvm: fn() else: - when defined(js): fn() + when defined(js) or defined(nimscript): fn() else: result = c_isnan(x) when defined(js): @@ -185,13 +220,13 @@ when defined(js): let a = newFloat64Array(buffer) let b = newUint32Array(buffer) a[0] = x - asm """ + {.emit: """ function updateBit(num, bitPos, bitVal) { return (num & ~(1 << bitPos)) | (bitVal << bitPos); } `b`[1] = updateBit(`b`[1], 31, `sgn`); - `result` = `a`[0] - """ + `result` = `a`[0]; + """.} proc signbit*(x: SomeFloat): bool {.inline, since: (1, 5, 1).} = ## Returns true if `x` is negative, false otherwise. @@ -236,7 +271,6 @@ func classify*(x: float): FloatClass = ## Classifies a floating point value. ## ## Returns `x`'s class as specified by the `FloatClass enum<#FloatClass>`_. - ## Doesn't work with `--passc:-ffast-math`. runnableExamples: doAssert classify(0.3) == fcNormal doAssert classify(0.0) == fcZero @@ -245,6 +279,7 @@ func classify*(x: float): FloatClass = doAssert classify(5.0e-324) == fcSubnormal # JavaScript and most C compilers have no classify: + if isNan(x): return fcNan if x == 0.0: if 1.0 / x == Inf: return fcZero @@ -253,7 +288,6 @@ func classify*(x: float): FloatClass = if x * 0.5 == x: if x > 0.0: return fcInf else: return fcNegInf - if x != x: return fcNan if abs(x) < MinFloatNormal: return fcSubnormal return fcNormal @@ -327,68 +361,8 @@ func nextPowerOfTwo*(x: int): int = result = result or (result shr 1) result += 1 + ord(x <= 0) -func sum*[T](x: openArray[T]): T = - ## Computes the sum of the elements in `x`. - ## - ## If `x` is empty, 0 is returned. - ## - ## **See also:** - ## * `prod func <#prod,openArray[T]>`_ - ## * `cumsum func <#cumsum,openArray[T]>`_ - ## * `cumsummed func <#cumsummed,openArray[T]>`_ - runnableExamples: - doAssert sum([1, 2, 3, 4]) == 10 - doAssert sum([-4, 3, 5]) == 4 - - for i in items(x): result = result + i - -func prod*[T](x: openArray[T]): T = - ## Computes the product of the elements in `x`. - ## - ## If `x` is empty, 1 is returned. - ## - ## **See also:** - ## * `sum func <#sum,openArray[T]>`_ - ## * `fac func <#fac,int>`_ - runnableExamples: - doAssert prod([1, 2, 3, 4]) == 24 - doAssert prod([-4, 3, 5]) == -60 - - result = T(1) - for i in items(x): result = result * i - -func cumsummed*[T](x: openArray[T]): seq[T] = - ## Returns the cumulative (aka prefix) summation of `x`. - ## - ## If `x` is empty, `@[]` is returned. - ## - ## **See also:** - ## * `sum func <#sum,openArray[T]>`_ - ## * `cumsum func <#cumsum,openArray[T]>`_ for the in-place version - runnableExamples: - doAssert cumsummed([1, 2, 3, 4]) == @[1, 3, 6, 10] - - let xLen = x.len - if xLen == 0: - return @[] - result.setLen(xLen) - result[0] = x[0] - for i in 1 ..< xLen: result[i] = result[i - 1] + x[i] -func cumsum*[T](x: var openArray[T]) = - ## Transforms `x` in-place (must be declared as `var`) into its - ## cumulative (aka prefix) summation. - ## - ## **See also:** - ## * `sum func <#sum,openArray[T]>`_ - ## * `cumsummed func <#cumsummed,openArray[T]>`_ for a version which - ## returns a cumsummed sequence - runnableExamples: - var a = [1, 2, 3, 4] - cumsum(a) - doAssert a == @[1, 3, 6, 10] - for i in 1 ..< x.len: x[i] = x[i - 1] + x[i] when not defined(js): # C func sqrt*(x: float32): float32 {.importc: "sqrtf", header: "<math.h>".} @@ -854,6 +828,14 @@ else: # JS doAssert -6.5 mod 2.5 == -1.5 doAssert 6.5 mod -2.5 == 1.5 doAssert -6.5 mod -2.5 == -1.5 + + func divmod*[T:SomeInteger](num, denom: T): (T, T) = + runnableExamples: + doAssert divmod(5, 2) == (2, 1) + doAssert divmod(5, -3) == (-1, 2) + result[0] = num div denom + result[1] = num mod denom + func round*[T: float32|float64](x: T, places: int): T = ## Decimal rounding on a binary floating point number. @@ -1134,6 +1116,69 @@ func sgn*[T: SomeNumber](x: T): int {.inline.} = {.pop.} {.pop.} +func sum*[T](x: openArray[T]): T = + ## Computes the sum of the elements in `x`. + ## + ## If `x` is empty, 0 is returned. + ## + ## **See also:** + ## * `prod func <#prod,openArray[T]>`_ + ## * `cumsum func <#cumsum,openArray[T]>`_ + ## * `cumsummed func <#cumsummed,openArray[T]>`_ + runnableExamples: + doAssert sum([1, 2, 3, 4]) == 10 + doAssert sum([-4, 3, 5]) == 4 + + for i in items(x): result = result + i + +func prod*[T](x: openArray[T]): T = + ## Computes the product of the elements in `x`. + ## + ## If `x` is empty, 1 is returned. + ## + ## **See also:** + ## * `sum func <#sum,openArray[T]>`_ + ## * `fac func <#fac,int>`_ + runnableExamples: + doAssert prod([1, 2, 3, 4]) == 24 + doAssert prod([-4, 3, 5]) == -60 + + result = T(1) + for i in items(x): result = result * i + +func cumsummed*[T](x: openArray[T]): seq[T] = + ## Returns the cumulative (aka prefix) summation of `x`. + ## + ## If `x` is empty, `@[]` is returned. + ## + ## **See also:** + ## * `sum func <#sum,openArray[T]>`_ + ## * `cumsum func <#cumsum,openArray[T]>`_ for the in-place version + runnableExamples: + doAssert cumsummed([1, 2, 3, 4]) == @[1, 3, 6, 10] + + let xLen = x.len + if xLen == 0: + return @[] + result.setLen(xLen) + result[0] = x[0] + for i in 1 ..< xLen: result[i] = result[i - 1] + x[i] + +func cumsum*[T](x: var openArray[T]) = + ## Transforms `x` in-place (must be declared as `var`) into its + ## cumulative (aka prefix) summation. + ## + ## **See also:** + ## * `sum func <#sum,openArray[T]>`_ + ## * `cumsummed func <#cumsummed,openArray[T]>`_ for a version which + ## returns a cumsummed sequence + runnableExamples: + var a = [1, 2, 3, 4] + cumsum(a) + doAssert a == @[1, 3, 6, 10] + + for i in 1 ..< x.len: x[i] = x[i - 1] + x[i] + func `^`*[T: SomeNumber](x: T, y: Natural): T = ## Computes `x` to the power of `y`. ## @@ -1185,40 +1230,42 @@ func gcd*[T](x, y: T): T = swap x, y abs x -func gcd*(x, y: SomeInteger): SomeInteger = - ## Computes the greatest common (positive) divisor of `x` and `y`, - ## using the binary GCD (aka Stein's) algorithm. - ## - ## **See also:** - ## * `gcd func <#gcd,T,T>`_ for a float version - ## * `lcm func <#lcm,T,T>`_ - runnableExamples: - doAssert gcd(12, 8) == 4 - doAssert gcd(17, 63) == 1 - - when x is SomeSignedInt: - var x = abs(x) - else: - var x = x - when y is SomeSignedInt: - var y = abs(y) - else: - var y = y - - if x == 0: - return y - if y == 0: - return x - - let shift = countTrailingZeroBits(x or y) - y = y shr countTrailingZeroBits(y) - while x != 0: - x = x shr countTrailingZeroBits(x) - if y > x: - swap y, x - x -= y - y shl shift - +when useBuiltins: + ## this func uses bitwise comparisons from C compilers, which are not always available. + func gcd*(x, y: SomeInteger): SomeInteger = + ## Computes the greatest common (positive) divisor of `x` and `y`, + ## using the binary GCD (aka Stein's) algorithm. + ## + ## **See also:** + ## * `gcd func <#gcd,T,T>`_ for a float version + ## * `lcm func <#lcm,T,T>`_ + runnableExamples: + doAssert gcd(12, 8) == 4 + doAssert gcd(17, 63) == 1 + + when x is SomeSignedInt: + var x = abs(x) + else: + var x = x + when y is SomeSignedInt: + var y = abs(y) + else: + var y = y + + if x == 0: + return y + if y == 0: + return x + + let shift = countTrailingZeroBits(x or y) + y = y shr countTrailingZeroBits(y) + while x != 0: + x = x shr countTrailingZeroBits(x) + if y > x: + swap y, x + x -= y + y shl shift + func gcd*[T](x: openArray[T]): T {.since: (1, 1).} = ## Computes the greatest common (positive) divisor of the elements of `x`. ## diff --git a/lib/pure/md5.nim b/lib/pure/md5.nim index cd4d1e6b8..9c3f6d51b 100644 --- a/lib/pure/md5.nim +++ b/lib/pure/md5.nim @@ -14,10 +14,12 @@ ## See also ## ======== ## * `base64 module<base64.html>`_ for a Base64 encoder and decoder -## * `std/sha1 module <sha1.html>`_ for the SHA-1 checksum algorithm +## * `sha1 module <sha1.html>`_ for the SHA-1 checksum algorithm ## * `hashes module<hashes.html>`_ for efficient computations of hash values ## for diverse Nim types +{.deprecated: "use command `nimble install checksums` and import `checksums/md5` instead".} + when defined(nimHasStyleChecks): {.push styleChecks: off.} @@ -98,32 +100,19 @@ proc decode(dest: var openArray[uint8], src: openArray[uint32]) = dest[i+3] = uint8(src[j] shr 24 and 0xff'u32) inc(i, 4) -template slice(s: string, a, b): openArray[uint8] = - when nimvm: - # toOpenArray is not implemented in VM - var s2 = newSeq[uint8](s.len) - for i in 0 ..< s2.len: - s2[i] = uint8(s[i]) - s2 - else: - s.toOpenArrayByte(a, b) - template slice(s: cstring, a, b): openArray[uint8] = when nimvm: # toOpenArray is not implemented in VM - slice($s, a, b) + toOpenArrayByte($s, a, b) else: when defined(js): # toOpenArrayByte for cstring is not implemented in JS - slice($s, a, b) + toOpenArrayByte($s, a, b) else: s.toOpenArrayByte(a, b) template slice(s: openArray[uint8], a, b): openArray[uint8] = - when nimvm: - s[a .. b] - else: - s.toOpenArray(a, b) + s.toOpenArray(a, b) const useMem = declared(copyMem) @@ -343,4 +332,4 @@ proc md5Final*(c: var MD5Context, digest: var MD5Digest) = when defined(nimHasStyleChecks): - {.pop.} #{.push styleChecks: off.} + {.pop.} #{.push styleChecks: off.} \ No newline at end of file diff --git a/lib/pure/memfiles.nim b/lib/pure/memfiles.nim index 28563b6fe..8eec551c4 100644 --- a/lib/pure/memfiles.nim +++ b/lib/pure/memfiles.nim @@ -16,13 +16,16 @@ ## other "line-like", variable length, delimited records). when defined(windows): - import winlean + import std/winlean + when defined(nimPreviewSlimSystem): + import std/widestrs elif defined(posix): - import posix + import std/posix else: {.error: "the memfiles module is not supported on your operating system!".} -import os, streams +import std/streams +import std/oserrors when defined(nimPreviewSlimSystem): import std/[syncio, assertions] @@ -32,6 +35,35 @@ proc newEIO(msg: string): ref IOError = new(result) result.msg = msg +proc setFileSize(fh: FileHandle, newFileSize = -1, oldSize = -1): OSErrorCode = + ## Set the size of open file pointed to by `fh` to `newFileSize` if != -1, + ## allocating | freeing space from the file system. This routine returns the + ## last OSErrorCode found rather than raising to support old rollback/clean-up + ## code style. [ Should maybe move to std/osfiles. ] + if newFileSize < 0 or newFileSize == oldSize: + return + when defined(windows): + var sizeHigh = int32(newFileSize shr 32) + let sizeLow = int32(newFileSize and 0xffffffff) + let status = setFilePointer(fh, sizeLow, addr(sizeHigh), FILE_BEGIN) + let lastErr = osLastError() + if (status == INVALID_SET_FILE_POINTER and lastErr.int32 != NO_ERROR) or + setEndOfFile(fh) == 0: + result = lastErr + else: + if newFileSize > oldSize: # grow the file + var e: cint # posix_fallocate truncates up when needed. + when declared(posix_fallocate): + while (e = posix_fallocate(fh, 0, newFileSize); e == EINTR): + discard + if e in [EINVAL, EOPNOTSUPP] and ftruncate(fh, newFileSize) == -1: + result = osLastError() # fallback arguable; Most portable BUT allows SEGV + elif e != 0: + result = osLastError() + else: # shrink the file + if ftruncate(fh.cint, newFileSize) == -1: + result = osLastError() + type MemFile* = object ## represents a memory mapped file mem*: pointer ## a pointer to the memory mapped file. The pointer @@ -122,7 +154,7 @@ proc open*(filename: string, mode: FileMode = fmRead, ## ## Example: ## - ## .. code-block:: nim + ## ```nim ## var ## mm, mm_full, mm_half: MemFile ## @@ -134,6 +166,7 @@ proc open*(filename: string, mode: FileMode = fmRead, ## ## # Read the first 512 bytes ## mm_half = memfiles.open("/tmp/test.mmap", mode = fmReadWrite, mappedSize = 512) + ## ``` # The file can be resized only when write mode is used: if mode == fmAppend: @@ -171,25 +204,13 @@ proc open*(filename: string, mode: FileMode = fmRead, else: FILE_ATTRIBUTE_NORMAL or flags, 0) - when useWinUnicode: - result.fHandle = callCreateFile(createFileW, newWideCString(filename)) - else: - result.fHandle = callCreateFile(createFileA, filename) + result.fHandle = callCreateFile(createFileW, newWideCString(filename)) if result.fHandle == INVALID_HANDLE_VALUE: fail(osLastError(), "error opening file") - if newFileSize != -1: - var - sizeHigh = int32(newFileSize shr 32) - sizeLow = int32(newFileSize and 0xffffffff) - - var status = setFilePointer(result.fHandle, sizeLow, addr(sizeHigh), - FILE_BEGIN) - let lastErr = osLastError() - if (status == INVALID_SET_FILE_POINTER and lastErr.int32 != NO_ERROR) or - (setEndOfFile(result.fHandle) == 0): - fail(lastErr, "error setting file size") + if (let e = setFileSize(result.fHandle.FileHandle, newFileSize); + e != 0.OSErrorCode): fail(e, "error setting file size") # since the strings are always 'nil', we simply always call # CreateFileMappingW which should be slightly faster anyway: @@ -223,7 +244,7 @@ proc open*(filename: string, mode: FileMode = fmRead, result.wasOpened = true if not allowRemap and result.fHandle != INVALID_HANDLE_VALUE: - if closeHandle(result.fHandle) == 0: + if closeHandle(result.fHandle) != 0: result.fHandle = INVALID_HANDLE_VALUE else: @@ -238,42 +259,31 @@ proc open*(filename: string, mode: FileMode = fmRead, flags = flags or O_CREAT or O_TRUNC var permissionsMode = S_IRUSR or S_IWUSR result.handle = open(filename, flags, permissionsMode) + if result.handle != -1: + if (let e = setFileSize(result.handle.FileHandle, newFileSize); + e != 0.OSErrorCode): fail(e, "error setting file size") else: result.handle = open(filename, flags) if result.handle == -1: - # XXX: errno is supposed to be set here - # Is there an exception that wraps it? fail(osLastError(), "error opening file") - if newFileSize != -1: - if ftruncate(result.handle, newFileSize) == -1: - fail(osLastError(), "error setting file size") - - if mappedSize != -1: - result.size = mappedSize - else: - var stat: Stat + if mappedSize != -1: #XXX Logic here differs from `when windows` branch .. + result.size = mappedSize #.. which always fstats&Uses min(mappedSize, st). + else: # if newFileSize!=-1: result.size=newFileSize # if trust setFileSize + var stat: Stat #^^.. BUT some FSes (eg. Linux HugeTLBfs) round to 2MiB. if fstat(result.handle, stat) != -1: - # XXX: Hmm, this could be unsafe - # Why is mmap taking int anyway? - result.size = int(stat.st_size) + result.size = stat.st_size.int # int may be 32-bit-unsafe for 2..<4 GiB else: fail(osLastError(), "error getting file size") result.flags = if mapFlags == cint(-1): MAP_SHARED else: mapFlags - #Ensure exactly one of MAP_PRIVATE cr MAP_SHARED is set + # Ensure exactly one of MAP_PRIVATE cr MAP_SHARED is set if int(result.flags and MAP_PRIVATE) == 0: result.flags = result.flags or MAP_SHARED - result.mem = mmap( - nil, - result.size, - if readonly: PROT_READ else: PROT_READ or PROT_WRITE, - result.flags, - result.handle, - offset) - + let pr = if readonly: PROT_READ else: PROT_READ or PROT_WRITE + result.mem = mmap(nil, result.size, pr, result.flags, result.handle, offset) if result.mem == cast[pointer](MAP_FAILED): fail(osLastError(), "file mapping failed") @@ -303,34 +313,58 @@ proc flush*(f: var MemFile; attempts: Natural = 3) = if lastErr != EBUSY.OSErrorCode: raiseOSError(lastErr, "error flushing mapping") -when defined(posix) or defined(nimdoc): - proc resize*(f: var MemFile, newFileSize: int) {.raises: [IOError, OSError].} = - ## resize and re-map the file underlying an `allowRemap MemFile`. - ## **Note**: this assumes the entire file is mapped read-write at offset zero. - ## Also, the value of `.mem` will probably change. - ## **Note**: This is not (yet) available on Windows. - when defined(posix): - if f.handle == -1: - raise newException(IOError, - "Cannot resize MemFile opened with allowRemap=false") - if ftruncate(f.handle, newFileSize) == -1: - raiseOSError(osLastError()) - when defined(linux): #Maybe NetBSD, too? - #On Linux this can be over 100 times faster than a munmap,mmap cycle. - proc mremap(old: pointer; oldSize, newSize: csize; flags: cint): - pointer {.importc: "mremap", header: "<sys/mman.h>".} - let newAddr = mremap(f.mem, csize(f.size), csize(newFileSize), cint(1)) - if newAddr == cast[pointer](MAP_FAILED): - raiseOSError(osLastError()) - else: - if munmap(f.mem, f.size) != 0: - raiseOSError(osLastError()) - let newAddr = mmap(nil, newFileSize, PROT_READ or PROT_WRITE, - f.flags, f.handle, 0) - if newAddr == cast[pointer](MAP_FAILED): - raiseOSError(osLastError()) - f.mem = newAddr +proc resize*(f: var MemFile, newFileSize: int) {.raises: [IOError, OSError].} = + ## Resize & re-map the file underlying an `allowRemap MemFile`. If the OS/FS + ## supports it, file space is reserved to ensure room for new virtual pages. + ## Caller should wait often enough for `flush` to finish to limit use of + ## system RAM for write buffering, perhaps just prior to this call. + ## **Note**: this assumes the entire file is mapped read-write at offset 0. + ## Also, the value of `.mem` will probably change. + if newFileSize < 1: # Q: include system/bitmasks & use PageSize ? + raise newException(IOError, "Cannot resize MemFile to < 1 byte") + when defined(windows): + if not f.wasOpened: + raise newException(IOError, "Cannot resize unopened MemFile") + if f.fHandle == INVALID_HANDLE_VALUE: + raise newException(IOError, + "Cannot resize MemFile opened with allowRemap=false") + if unmapViewOfFile(f.mem) == 0 or closeHandle(f.mapHandle) == 0: # Un-do map + raiseOSError(osLastError()) + if newFileSize != f.size: # Seek to size & `setEndOfFile` => allocated. + if (let e = setFileSize(f.fHandle.FileHandle, newFileSize); + e != 0.OSErrorCode): raiseOSError(e) + f.mapHandle = createFileMappingW(f.fHandle, nil, PAGE_READWRITE, 0,0,nil) + if f.mapHandle == 0: # Re-do map + raiseOSError(osLastError()) + if (let m = mapViewOfFileEx(f.mapHandle, FILE_MAP_READ or FILE_MAP_WRITE, + 0, 0, WinSizeT(newFileSize), nil); m != nil): + f.mem = m f.size = newFileSize + else: + raiseOSError(osLastError()) + elif defined(posix): + if f.handle == -1: + raise newException(IOError, + "Cannot resize MemFile opened with allowRemap=false") + if newFileSize != f.size: + if (let e = setFileSize(f.handle.FileHandle, newFileSize, f.size); + e != 0.OSErrorCode): raiseOSError(e) + when defined(linux): #Maybe NetBSD, too? + # On Linux this can be over 100 times faster than a munmap,mmap cycle. + proc mremap(old: pointer; oldSize, newSize: csize_t; flags: cint): + pointer {.importc: "mremap", header: "<sys/mman.h>".} + let newAddr = mremap(f.mem, csize_t(f.size), csize_t(newFileSize), 1.cint) + if newAddr == cast[pointer](MAP_FAILED): + raiseOSError(osLastError()) + else: + if munmap(f.mem, f.size) != 0: + raiseOSError(osLastError()) + let newAddr = mmap(nil, newFileSize, PROT_READ or PROT_WRITE, + f.flags, f.handle, 0) + if newAddr == cast[pointer](MAP_FAILED): + raiseOSError(osLastError()) + f.mem = newAddr + f.size = newFileSize proc close*(f: var MemFile) = ## closes the memory mapped file `f`. All changes are written back to the @@ -378,7 +412,7 @@ proc `==`*(x, y: MemSlice): bool = proc `$`*(ms: MemSlice): string {.inline.} = ## Return a Nim string built from a MemSlice. result.setLen(ms.size) - copyMem(addr(result[0]), ms.data, ms.size) + copyMem(result.cstring, ms.data, ms.size) iterator memSlices*(mfile: MemFile, delim = '\l', eat = '\r'): MemSlice {.inline.} = ## Iterates over \[optional `eat`] `delim`-delimited slices in MemFile `mfile`. @@ -404,15 +438,15 @@ iterator memSlices*(mfile: MemFile, delim = '\l', eat = '\r'): MemSlice {.inline ## functions, not str* functions). ## ## Example: - ## - ## .. code-block:: nim + ## ```nim ## var count = 0 ## for slice in memSlices(memfiles.open("foo")): ## if slice.size > 0 and cast[cstring](slice.data)[0] != '#': ## inc(count) ## echo count + ## ``` - proc c_memchr(cstr: pointer, c: char, n: csize): pointer {. + proc c_memchr(cstr: pointer, c: char, n: csize_t): pointer {. importc: "memchr", header: "<string.h>".} proc `-!`(p, q: pointer): int {.inline.} = return cast[int](p) -% cast[int](q) var ms: MemSlice @@ -420,7 +454,7 @@ iterator memSlices*(mfile: MemFile, delim = '\l', eat = '\r'): MemSlice {.inline ms.data = mfile.mem var remaining = mfile.size while remaining > 0: - ending = c_memchr(ms.data, delim, remaining) + ending = c_memchr(ms.data, delim, csize_t(remaining)) if ending == nil: # unterminated final slice ms.size = remaining # Weird case..check eat? yield ms @@ -440,11 +474,11 @@ iterator lines*(mfile: MemFile, buf: var string, delim = '\l', ## <#memSlices.i,MemFile,char,char>`_, but Nim strings are returned. ## ## Example: - ## - ## .. code-block:: nim + ## ```nim ## var buffer: string = "" ## for line in lines(memfiles.open("foo"), buffer): ## echo line + ## ``` for ms in memSlices(mfile, delim, eat): setLen(buf, ms.size) @@ -459,10 +493,10 @@ iterator lines*(mfile: MemFile, delim = '\l', eat = '\r'): string {.inline.} = ## <#memSlices.i,MemFile,char,char>`_, but Nim strings are returned. ## ## Example: - ## - ## .. code-block:: nim + ## ```nim ## for line in lines(memfiles.open("foo")): ## echo line + ## ``` var buf = newStringOfCap(80) for line in lines(mfile, buf, delim, eat): @@ -492,8 +526,8 @@ proc mmsSetPosition(s: Stream, pos: int) = proc mmsGetPosition(s: Stream): int = MemMapFileStream(s).pos proc mmsPeekData(s: Stream, buffer: pointer, bufLen: int): int = - let startAddress = cast[ByteAddress](MemMapFileStream(s).mf.mem) - let p = cast[ByteAddress](MemMapFileStream(s).pos) + let startAddress = cast[int](MemMapFileStream(s).mf.mem) + let p = cast[int](MemMapFileStream(s).pos) let l = min(bufLen, MemMapFileStream(s).mf.size - p) moveMem(buffer, cast[pointer](startAddress + p), l) result = l @@ -508,8 +542,8 @@ proc mmsWriteData(s: Stream, buffer: pointer, bufLen: int) = let size = MemMapFileStream(s).mf.size if MemMapFileStream(s).pos + bufLen > size: raise newEIO("cannot write to stream") - let p = cast[ByteAddress](MemMapFileStream(s).mf.mem) + - cast[ByteAddress](MemMapFileStream(s).pos) + let p = cast[int](MemMapFileStream(s).mf.mem) + + cast[int](MemMapFileStream(s).pos) moveMem(cast[pointer](p), buffer, bufLen) inc(MemMapFileStream(s).pos, bufLen) diff --git a/lib/pure/mersenne.nim b/lib/pure/mersenne.nim deleted file mode 100644 index 37c5085b1..000000000 --- a/lib/pure/mersenne.nim +++ /dev/null @@ -1,51 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Nim Contributors -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -## The [Mersenne Twister](https://en.wikipedia.org/wiki/Mersenne_Twister) -## random number generator. -## .. note:: The procs in this module work at compile-time. - -{.deprecated: "use `std/random` instead".} - -runnableExamples: - var rand = newMersenneTwister(uint32.high) ## must be "var" - doAssert rand.getNum() != rand.getNum() ## pseudorandom number -## See also -## ======== -## * `random module<random.html>`_ for Nim's standard random number generator -type - MersenneTwister* = object - ## The Mersenne Twister. - mt: array[0..623, uint32] - index: int - -proc newMersenneTwister*(seed: uint32): MersenneTwister = - ## Creates a new `MersenneTwister` with seed `seed`. - result.index = 0 - result.mt[0] = seed - for i in 1'u32 .. 623'u32: - result.mt[i] = (0x6c078965'u32 * (result.mt[i-1] xor - (result.mt[i-1] shr 30'u32)) + i) - -proc generateNumbers(m: var MersenneTwister) = - - for i in 0..623: - var y = (m.mt[i] and 0x80000000'u32) + - (m.mt[(i+1) mod 624] and 0x7fffffff'u32) - m.mt[i] = m.mt[(i+397) mod 624] xor uint32(y shr 1'u32) - if (y mod 2'u32) != 0: - m.mt[i] = m.mt[i] xor 0x9908b0df'u32 - -proc getNum*(m: var MersenneTwister): uint32 = - ## Returns the next pseudorandom `uint32`. - if m.index == 0: - generateNumbers(m) - result = m.mt[m.index] - m.index = (m.index + 1) mod m.mt.len - result = result xor (result shr 11'u32) - result = result xor ((result shl 7'u32) and 0x9d2c5680'u32) - result = result xor ((result shl 15'u32) and 0xefc60000'u32) - result = result xor (result shr 18'u32) diff --git a/lib/pure/mimetypes.nim b/lib/pure/mimetypes.nim index 346fb39ee..ff639e8e5 100644 --- a/lib/pure/mimetypes.nim +++ b/lib/pure/mimetypes.nim @@ -26,8 +26,8 @@ runnableExamples: doAssert m.getMimetype("fakext") == "text/fakelang" doAssert m.getMimetype("FaKeXT") == "text/fakelang" -import tables -from strutils import startsWith, toLowerAscii, strip +import std/tables +from std/strutils import startsWith, toLowerAscii, strip when defined(nimPreviewSlimSystem): import std/assertions @@ -38,1848 +38,704 @@ type mimes: OrderedTable[string, string] const mimes* = { - "123": "application/vnd.lotus-1-2-3", - "1km": "application/vnd.1000minds.decision-model+xml", - "323": "text/h323", - "3dm": "text/vnd.in3d.3dml", - "3dmf": "x-world/x-3dmf", - "3dml": "text/vnd.in3d.3dml", - "3ds": "image/x-3ds", - "3g2": "video/3gpp2", - "3gp": "video/3gpp", - "3gpp": "audio/3gpp", - "3gpp2": "video/3gpp2", - "3mf": "application/vnd.ms-3mfdocument", - "669": "audio/x-mod", - "726": "audio/32kadpcm", - "7z": "application/x-7z-compressed", - "a": "text/plain", - "a2l": "application/a2l", - "aa3": "audio/atrac3", - "aab": "application/x-authorware-bin", - "aac": "audio/x-aac", - "aal": "audio/atrac-advanced-lossless", - "aam": "application/x-authorware-map", - "aas": "application/x-authorware-seg", - "abc": "text/vnd.abc", - "abw": "application/x-abiword", + "ez": "application/andrew-inset", + "aw": "application/applixware", + "atom": "application/atom+xml", + "atomcat": "application/atomcat+xml", + "atomsvc": "application/atomsvc+xml", + "ccxml": "application/ccxml+xml", + "cdmia": "application/cdmi-capability", + "cdmic": "application/cdmi-container", + "cdmid": "application/cdmi-domain", + "cdmio": "application/cdmi-object", + "cdmiq": "application/cdmi-queue", + "cu": "application/cu-seeme", + "davmount": "application/davmount+xml", + "dbk": "application/docbook+xml", + "dssc": "application/dssc+der", + "xdssc": "application/dssc+xml", + "ecma": "application/ecmascript", + "emma": "application/emma+xml", + "epub": "application/epub+zip", + "exi": "application/exi", + "pfr": "application/font-tdpfr", + "gml": "application/gml+xml", + "gpx": "application/gpx+xml", + "gxf": "application/gxf", + "stk": "application/hyperstudio", + "ink": "application/inkml+xml", + "inkml": "application/inkml+xml", + "ipfix": "application/ipfix", + "jar": "application/java-archive", + "ser": "application/java-serialized-object", + "class": "application/java-vm", + "json": "application/json", + "jsonml": "application/jsonml+json", + "lostxml": "application/lost+xml", + "hqx": "application/mac-binhex40", + "cpt": "application/mac-compactpro", + "mads": "application/mads+xml", + "mrc": "application/marc", + "mrcx": "application/marcxml+xml", + "ma": "application/mathematica", + "nb": "application/mathematica", + "mb": "application/mathematica", + "mathml": "application/mathml+xml", + "mbox": "application/mbox", + "mscml": "application/mediaservercontrol+xml", + "metalink": "application/metalink+xml", + "meta4": "application/metalink4+xml", + "mets": "application/mets+xml", + "mods": "application/mods+xml", + "m21": "application/mp21", + "mp21": "application/mp21", + "mp4s": "application/mp4", + "doc": "application/msword", + "dot": "application/msword", + "mxf": "application/mxf", + "bin": "application/octet-stream", + "dms": "application/octet-stream", + "lrf": "application/octet-stream", + "mar": "application/octet-stream", + "so": "application/octet-stream", + "dist": "application/octet-stream", + "distz": "application/octet-stream", + "pkg": "application/octet-stream", + "bpk": "application/octet-stream", + "dump": "application/octet-stream", + "elc": "application/octet-stream", + "deploy": "application/octet-stream", + "oda": "application/oda", + "opf": "application/oebps-package+xml", + "ogx": "application/ogg", + "omdoc": "application/omdoc+xml", + "onetoc": "application/onenote", + "onetoc2": "application/onenote", + "onetmp": "application/onenote", + "onepkg": "application/onenote", + "oxps": "application/oxps", + "xer": "application/patch-ops-error+xml", + "pdf": "application/pdf", + "pgp": "application/pgp-encrypted", + "asc": "application/pgp-signature", + "sig": "application/pgp-signature", + "prf": "application/pics-rules", + "p10": "application/pkcs10", + "p7m": "application/pkcs7-mime", + "p7c": "application/pkcs7-mime", + "p7s": "application/pkcs7-signature", + "p8": "application/pkcs8", "ac": "application/pkix-attr-cert", - "ac3": "audio/ac3", - "acc": "application/vnd.americandynamics.acc", - "ace": "application/x-ace-compressed", - "acn": "audio/asc", - "acu": "application/vnd.acucobol", - "acutc": "application/vnd.acucorp", - "acx": "application/internet-property-stream", - "adp": "audio/adpcm", - "aep": "application/vnd.audiograph", - "afl": "video/animaflex", - "afm": "application/x-font-type1", - "afp": "application/vnd.ibm.modcap", - "ahead": "application/vnd.ahead.space", + "cer": "application/pkix-cert", + "crl": "application/pkix-crl", + "pkipath": "application/pkix-pkipath", + "pki": "application/pkixcmp", + "pls": "application/pls+xml", "ai": "application/postscript", - "aif": "audio/x-aiff", - "aifc": "audio/x-aiff", - "aiff": "audio/x-aiff", - "aim": "application/x-aim", - "aip": "text/x-audiosoft-intra", - "air": "application/vnd.adobe.air-application-installer-package+zip", - "ait": "application/vnd.dvb.ait", - "alc": "chemical/x-alchemy", - "ami": "application/vnd.amiga.ami", - "aml": "application/aml", - "amr": "audio/amr", - "ani": "application/x-navi-animation", - "anx": "application/x-annodex", - "aos": "application/x-nokia-9000-communicator-add-on-software", - "apinotes": "text/apinotes", - "apk": "application/vnd.android.package-archive", - "apkg": "application/vnd.anki", - "apng": "image/apng", - "appcache": "text/cache-manifest", - "appimage": "application/appimage", - "application": "application/x-ms-application", - "apr": "application/vnd.lotus-approach", - "aps": "application/mime", - "apxml": "application/auth-policy+xml", - "arc": "application/x-freearc", - "arj": "application/x-arj", - "art": "message/rfc822", - "asar": "binary/asar", - "asc": "text/plain", - "ascii": "text/vnd.ascii-art", - "asf": "application/vnd.ms-asf", - "asice": "application/vnd.etsi.asic-e+zip", - "asics": "application/vnd.etsi.asic-s+zip", - "asm": "text/x-asm", - "asn": "chemical/x-ncbi-asn1-spec", + "eps": "application/postscript", + "ps": "application/postscript", + "cww": "application/prs.cww", + "pskcxml": "application/pskc+xml", + "rdf": "application/rdf+xml", + "rif": "application/reginfo+xml", + "rnc": "application/relax-ng-compact-syntax", + "rl": "application/resource-lists+xml", + "rld": "application/resource-lists-diff+xml", + "rs": "application/rls-services+xml", + "gbr": "application/rpki-ghostbusters", + "mft": "application/rpki-manifest", + "roa": "application/rpki-roa", + "rsd": "application/rsd+xml", + "rss": "application/rss+xml", + "rtf": "application/rtf", + "sbml": "application/sbml+xml", + "scq": "application/scvp-cv-request", + "scs": "application/scvp-cv-response", + "spq": "application/scvp-vp-request", + "spp": "application/scvp-vp-response", + "sdp": "application/sdp", + "setpay": "application/set-payment-initiation", + "setreg": "application/set-registration-initiation", + "shf": "application/shf+xml", + "smi": "application/smil+xml", + "smil": "application/smil+xml", + "rq": "application/sparql-query", + "srx": "application/sparql-results+xml", + "gram": "application/srgs", + "grxml": "application/srgs+xml", + "sru": "application/sru+xml", + "ssdl": "application/ssdl+xml", + "ssml": "application/ssml+xml", + "tei": "application/tei+xml", + "teicorpus": "application/tei+xml", + "tfi": "application/thraud+xml", + "tsd": "application/timestamped-data", + "plb": "application/vnd.3gpp.pic-bw-large", + "psb": "application/vnd.3gpp.pic-bw-small", + "pvb": "application/vnd.3gpp.pic-bw-var", + "tcap": "application/vnd.3gpp2.tcap", + "pwn": "application/vnd.3m.post-it-notes", "aso": "application/vnd.accpac.simply.aso", - "asp": "text/asp", - "asr": "video/x-ms-asf", - "asx": "video/x-ms-asf", - "at3": "audio/atrac3", + "imp": "application/vnd.accpac.simply.imp", + "acu": "application/vnd.acucobol", "atc": "application/vnd.acucorp", - "atf": "application/atf", - "atfx": "application/atfx", - "atom": "application/atom+xml", - "atomcat": "application/atomcat+xml", - "atomdeleted": "application/atomdeleted+xml", - "atomsrv": "application/atomserv+xml", - "atomsvc": "application/atomsvc+xml", - "atx": "application/vnd.antix.game-component", - "atxml": "application/atxml", - "au": "audio/basic", - "auc": "application/tamp-apex-update-confirm", - "avi": "video/x-msvideo", - "avs": "video/avs-video", - "aw": "application/applixware", - "awb": "audio/amr-wb", - "axa": "audio/x-annodex", - "axs": "application/olescript", - "axv": "video/x-annodex", + "acutc": "application/vnd.acucorp", + "air": "application/vnd.adobe.air-application-installer-package+zip", + "fcdt": "application/vnd.adobe.formscentral.fcdt", + "fxp": "application/vnd.adobe.fxp", + "fxpl": "application/vnd.adobe.fxp", + "xdp": "application/vnd.adobe.xdp+xml", + "xfdf": "application/vnd.adobe.xfdf", + "ahead": "application/vnd.ahead.space", "azf": "application/vnd.airzip.filesecure.azf", "azs": "application/vnd.airzip.filesecure.azs", - "azv": "image/vnd.airzip.accelerator.azv", "azw": "application/vnd.amazon.ebook", - "azw3": "application/vnd.amazon.mobi8-ebook", - "b": "chemical/x-molconn-Z", - "bak": "application/x-trash", - "bar": "application/vnd.qualcomm.brew-app-res", - "bas": "text/plain", - "bash": "text/shell", - "bat": "application/x-msdos-program", - "bcpio": "application/x-bcpio", - "bdf": "application/x-font-bdf", - "bdm": "application/vnd.syncml.dm+wbxml", - "bdoc": "application/bdoc", - "bed": "application/vnd.realvnc.bed", - "bh2": "application/vnd.fujitsu.oasysprs", - "bib": "text/x-bibtex", - "bik": "video/vnd.radgamettools.bink", - "bin": "application/octet-stream", - "bk2": "video/vnd.radgamettools.bink", - "bkm": "application/vnd.nervana", - "blb": "application/x-blorb", - "blend": "binary/blender", - "blorb": "application/x-blorb", - "bm": "image/bmp", - "bmed": "multipart/vnd.bint.med-plus", + "acc": "application/vnd.americandynamics.acc", + "ami": "application/vnd.amiga.ami", + "apk": "application/vnd.android.package-archive", + "cii": "application/vnd.anser-web-certificate-issue-initiation", + "fti": "application/vnd.anser-web-funds-transfer-initiation", + "atx": "application/vnd.antix.game-component", + "mpkg": "application/vnd.apple.installer+xml", + "m3u8": "application/vnd.apple.mpegurl", + "swi": "application/vnd.aristanetworks.swi", + "iota": "application/vnd.astraea-software.iota", + "aep": "application/vnd.audiograph", + "mpm": "application/vnd.blueice.multipass", "bmi": "application/vnd.bmi", - "bmml": "application/vnd.balsamiq.bmml+xml", - "bmp": "image/bmp", - "bmpr": "application/vnd.balsamiq.bmpr", - "boo": "application/book", - "book": "application/book", - "box": "application/vnd.previewsystems.box", - "boz": "application/x-bzip2", - "bpd": "application/vnd.hbci", - "bpk": "application/octet-stream", - "brf": "text/plain", - "bsd": "chemical/x-crossfire", - "bsh": "application/x-bsh", - "bsp": "model/vnd.valve.source.compiled-map", - "btf": "image/prs.btif", - "btif": "image/prs.btif", - "bz": "application/x-bzip", - "bz2": "application/x-bzip2", - "c": "text/x-csrc", - "c++": "text/x-c++src", - "c11amc": "application/vnd.cluetrust.cartomobile-config", - "c11amz": "application/vnd.cluetrust.cartomobile-config-pkg", - "c3d": "chemical/x-chem3d", - "c3ex": "application/cccex", + "rep": "application/vnd.businessobjects", + "cdxml": "application/vnd.chemdraw+xml", + "mmd": "application/vnd.chipnuts.karaoke-mmd", + "cdy": "application/vnd.cinderella", + "cla": "application/vnd.claymore", + "rp9": "application/vnd.cloanto.rp9", + "c4g": "application/vnd.clonk.c4group", "c4d": "application/vnd.clonk.c4group", "c4f": "application/vnd.clonk.c4group", - "c4g": "application/vnd.clonk.c4group", "c4p": "application/vnd.clonk.c4group", "c4u": "application/vnd.clonk.c4group", - "cab": "application/vnd.ms-cab-compressed", - "cac": "chemical/x-cache", - "cache": "application/x-cache", - "caf": "audio/x-caf", - "cap": "application/vnd.tcpdump.pcap", - "car": "application/vnd.curl.car", - "cascii": "chemical/x-cactvs-binary", - "cat": "application/vnd.ms-pki.seccat", - "cb7": "application/x-cbr", - "cba": "application/x-cbr", - "cbin": "chemical/x-cactvs-binary", - "cbor": "application/cbor", - "cbr": "application/x-cbr", - "cbt": "application/x-cbr", - "cbz": "application/vnd.comicbook+zip", - "cc": "text/plain", - "ccad": "application/clariscad", - "ccc": "text/vnd.net2phone.commcenter.command", - "ccmp": "application/ccmp+xml", - "cco": "application/x-cocoa", - "cct": "application/x-director", - "ccxml": "application/ccxml+xml", - "cda": "application/x-cdf", + "c11amc": "application/vnd.cluetrust.cartomobile-config", + "c11amz": "application/vnd.cluetrust.cartomobile-config-pkg", + "csp": "application/vnd.commonspace", "cdbcmsg": "application/vnd.contact.cmsg", - "cdf": "application/x-netcdf", - "cdfx": "application/cdfx+xml", - "cdkey": "application/vnd.mediastation.cdkey", - "cdmia": "application/cdmi-capability", - "cdmic": "application/cdmi-container", - "cdmid": "application/cdmi-domain", - "cdmio": "application/cdmi-object", - "cdmiq": "application/cdmi-queue", - "cdr": "image/x-coreldraw", - "cdt": "image/x-coreldrawtemplate", - "cdx": "chemical/x-cdx", - "cdxml": "application/vnd.chemdraw+xml", - "cdy": "application/vnd.cinderella", - "cea": "application/cea", - "cef": "chemical/x-cxf", - "cellml": "application/cellml+xml", - "cer": "application/pkix-cert", - "cfg": "text/cfg", - "cfs": "application/x-cfs-compressed", - "cgm": "image/cgm", - "cha": "application/x-chat", - "chat": "application/x-chat", - "chm": "application/vnd.ms-htmlhelp", - "chrt": "application/vnd.kde.kchart", - "cif": "chemical/x-cif", - "cii": "application/vnd.anser-web-certificate-issue-initiation", - "cil": "application/vnd.ms-artgalry", - "cl": "application/simple-filter+xml", - "cla": "application/vnd.claymore", - "class": "application/java-vm", + "cmc": "application/vnd.cosmocaller", + "clkx": "application/vnd.crick.clicker", "clkk": "application/vnd.crick.clicker.keyboard", "clkp": "application/vnd.crick.clicker.palette", "clkt": "application/vnd.crick.clicker.template", "clkw": "application/vnd.crick.clicker.wordbank", - "clkx": "application/vnd.crick.clicker", - "clp": "application/x-msclip", - "cls": "text/x-tex", - "clue": "application/clue_info+xml", - "cmake": "text/cmake", - "cmc": "application/vnd.cosmocaller", - "cmdf": "chemical/x-cmdf", - "cml": "chemical/x-cml", - "cmp": "application/vnd.yellowriver-custom-menu", - "cmsc": "application/cms", - "cmx": "image/x-cmx", - "cnd": "text/jcr-cnd", - "cnf": "text/cnf", - "cod": "application/vnd.rim.cod", - "coffee": "application/vnd.coffeescript", - "com": "application/x-msdos-program", - "conf": "text/plain", - "copyright": "text/vnd.debian.copyright", - "cpa": "chemical/x-compass", - "cpio": "application/x-cpio", - "cpkg": "application/vnd.xmpie.cpkg", - "cpl": "application/cpl+xml", - "cpp": "text/x-c++src", - "cpt": "application/mac-compactpro", - "cr2": "image/x-canon-cr2", - "crd": "application/x-mscardfile", - "crl": "application/pkix-crl", - "crt": "application/x-x509-ca-cert", - "crtr": "application/vnd.multiad.creator", - "crw": "image/x-canon-crw", - "crx": "application/x-chrome-extension", - "cryptonote": "application/vnd.rig.cryptonote", - "cs": "text/c#", - "csf": "chemical/x-cache-csf", - "csh": "application/x-csh", - "csl": "application/vnd.citationstyles.style+xml", - "csm": "chemical/x-csml", - "csml": "chemical/x-csml", - "cson": "text/cson", - "csp": "application/vnd.commonspace", - "csrattrs": "application/csrattrs", - "css": "text/css", - "cst": "application/vnd.commonspace", - "csv": "text/csv", - "csvs": "text/csv-schema", - "ctab": "chemical/x-cactvs-binary", - "ctx": "chemical/x-ctx", - "cu": "application/cu-seeme", - "cub": "chemical/x-gaussian-cube", - "cuc": "application/tamp-community-update-confirm", - "curl": "text/vnd.curl", - "cw": "application/prs.cww", - "cww": "application/prs.cww", - "cxf": "chemical/x-cxf", - "cxt": "application/x-director", - "cxx": "text/plain", - "d": "text/x-dsrc", - "dae": "model/vnd.collada+xml", - "daf": "application/vnd.mobius.daf", + "wbs": "application/vnd.criticaltools.wbs+xml", + "pml": "application/vnd.ctc-posml", + "ppd": "application/vnd.cups-ppd", + "car": "application/vnd.curl.car", + "pcurl": "application/vnd.curl.pcurl", "dart": "application/vnd.dart", - "dat": "application/x-ns-proxy-autoconfig", - "dataless": "application/vnd.fdsn.seed", - "davmount": "application/davmount+xml", - "dbk": "application/docbook+xml", - "dcd": "application/dcd", - "dcf": "application/vnd.oma.drm.content", - "dcm": "application/dicom", - "dcr": "application/x-director", - "dcurl": "text/vnd.curl.dcurl", - "dd": "application/vnd.oma.dd+xml", - "dd2": "application/vnd.oma.dd2+xml", - "ddd": "application/vnd.fujixerox.ddd", - "ddf": "application/vnd.syncml.dmddf+xml", - "deb": "application/vnd.debian.binary-package", - "deepv": "application/x-deepv", - "def": "text/plain", - "deploy": "application/octet-stream", - "der": "application/x-x509-ca-cert", - "dfac": "application/vnd.dreamfactory", - "dgc": "application/x-dgc-compressed", - "dib": "image/bmp", - "dic": "text/x-c", - "dif": "video/x-dv", - "diff": "text/x-diff", - "dii": "application/dii", - "dim": "application/vnd.fastcopy-disk-image", - "dir": "application/x-director", - "dis": "application/vnd.mobius.dis", - "disposition-notification": "message/disposition-notification", - "dist": "application/vnd.apple.installer+xml", - "distz": "application/vnd.apple.installer+xml", - "dit": "application/dit", - "djv": "image/vnd.djvu", - "djvu": "image/vnd.djvu", - "dl": "video/dl", - "dll": "application/x-msdos-program", - "dls": "audio/dls", - "dm": "application/vnd.oma.drm.message", - "dmg": "application/x-apple-diskimage", - "dmp": "application/vnd.tcpdump.pcap", - "dms": "text/vnd.dmclientscript", + "rdz": "application/vnd.data-vision.rdz", + "uvf": "application/vnd.dece.data", + "uvvf": "application/vnd.dece.data", + "uvd": "application/vnd.dece.data", + "uvvd": "application/vnd.dece.data", + "uvt": "application/vnd.dece.ttml+xml", + "uvvt": "application/vnd.dece.ttml+xml", + "uvx": "application/vnd.dece.unspecified", + "uvvx": "application/vnd.dece.unspecified", + "uvz": "application/vnd.dece.zip", + "uvvz": "application/vnd.dece.zip", + "fe_launch": "application/vnd.denovo.fcselayout-link", "dna": "application/vnd.dna", - "doc": "application/msword", - "docjson": "application/vnd.document+json", - "docm": "application/vnd.ms-word.document.macroenabled.12", - "docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", - "dor": "model/vnd.gdl", - "dot": "text/vnd.graphviz", - "dotm": "application/vnd.ms-word.template.macroenabled.12", - "dotx": "application/vnd.openxmlformats-officedocument.wordprocessingml.template", - "dp": "application/vnd.osgi.dp", + "mlp": "application/vnd.dolby.mlp", "dpg": "application/vnd.dpgraph", - "dpgraph": "application/vnd.dpgraph", - "dpkg": "application/vnd.xmpie.dpkg", - "dr": "application/vnd.oma.drm.rights+xml", - "dra": "audio/vnd.dra", - "drc": "application/vnd.oma.drm.rights+wbxml", - "drle": "image/dicom-rle", - "drw": "application/drafting", - "dsc": "text/prs.lines.tag", - "dsm": "application/vnd.desmume.movie", - "dssc": "application/dssc+der", - "dtb": "application/x-dtbook+xml", - "dtd": "application/xml-dtd", - "dts": "audio/vnd.dts", - "dtshd": "audio/vnd.dts.hd", - "dump": "application/octet-stream", - "dv": "video/x-dv", - "dvb": "video/vnd.dvb.file", - "dvc": "application/dvcs", - "dvi": "application/x-dvi", - "dwf": "model/vnd.dwf", - "dwg": "image/vnd.dwg", - "dx": "chemical/x-jcamp-dx", - "dxf": "image/vnd.dxf", - "dxp": "application/vnd.spotfire.dxp", - "dxr": "application/x-director", - "dzr": "application/vnd.dzr", - "ear": "binary/zip", - "ecelp4800": "audio/vnd.nuera.ecelp4800", - "ecelp7470": "audio/vnd.nuera.ecelp7470", - "ecelp9600": "audio/vnd.nuera.ecelp9600", - "ecig": "application/vnd.evolv.ecig.settings", - "ecigprofile": "application/vnd.evolv.ecig.profile", - "ecigtheme": "application/vnd.evolv.ecig.theme", - "ecma": "application/ecmascript", - "edm": "application/vnd.novadigm.edm", - "edx": "application/vnd.novadigm.edx", - "efi": "application/efi", - "efif": "application/vnd.picsel", - "ei6": "application/vnd.pg.osasli", - "ejs": "text/ejs", - "el": "text/plain", - "elc": "application/x-bytecode.elisp", - "emb": "chemical/x-embl-dl-nucleotide", - "embl": "chemical/x-embl-dl-nucleotide", - "emf": "image/emf", - "eml": "message/rfc822", - "emm": "application/vnd.ibm.electronic-media", - "emma": "application/emma+xml", - "emotionml": "application/emotionml+xml", - "emz": "application/x-msmetafile", - "ent": "text/xml-external-parsed-entity", - "entity": "application/vnd.nervana", - "env": "application/x-envoy", - "enw": "audio/evrcnw", - "eol": "audio/vnd.digital-winds", - "eot": "application/vnd.ms-fontobject", - "ep": "application/vnd.bluetooth.ep.oob", - "eps": "application/postscript", - "eps2": "application/postscript", - "eps3": "application/postscript", - "epsf": "application/postscript", - "epsi": "application/postscript", - "epub": "application/epub+zip", - "erb": "text/erb", - "erf": "image/x-epson-erf", - "es": "application/ecmascript", - "es3": "application/vnd.eszigno3+xml", - "esa": "application/vnd.osgi.subsystem", - "escn": "text/godot", + "dfac": "application/vnd.dreamfactory", + "kpxx": "application/vnd.ds-keypoint", + "ait": "application/vnd.dvb.ait", + "svc": "application/vnd.dvb.service", + "geo": "application/vnd.dynageo", + "mag": "application/vnd.ecowin.chart", + "nml": "application/vnd.enliven", "esf": "application/vnd.epson.esf", - "espass": "application/vnd.espass-espass+zip", + "msf": "application/vnd.epson.msf", + "qam": "application/vnd.epson.quickanime", + "slt": "application/vnd.epson.salt", + "ssf": "application/vnd.epson.ssf", + "es3": "application/vnd.eszigno3+xml", "et3": "application/vnd.eszigno3+xml", - "etx": "text/x-setext", - "eva": "application/x-eva", - "evb": "audio/evrcb", - "evc": "audio/evrc", - "evw": "audio/evrcwb", - "evy": "application/x-envoy", - "exe": "application/x-msdos-program", - "exi": "application/exi", - "exr": "image/aces", - "ext": "application/vnd.novadigm.ext", - "eyaml": "text/yaml", - "ez": "application/andrew-inset", "ez2": "application/vnd.ezpix-album", "ez3": "application/vnd.ezpix-package", - "f": "text/x-fortran", - "f4v": "video/x-f4v", - "f77": "text/x-fortran", - "f90": "text/plain", - "fb": "application/x-maker", - "fbdoc": "application/x-maker", - "fbs": "image/vnd.fastbidsheet", - "fbx": "model/filmbox", - "fcdt": "application/vnd.adobe.formscentral.fcdt", - "fch": "chemical/x-gaussian-checkpoint", - "fchk": "chemical/x-gaussian-checkpoint", - "fcs": "application/vnd.isac.fcs", "fdf": "application/vnd.fdf", - "fdt": "application/fdt+xml", - "fe_launch": "application/vnd.denovo.fcselayout-link", - "feature": "text/gherkin", - "fg5": "application/vnd.fujitsu.oasysgp", - "fgd": "application/x-director", - "fh": "image/x-freehand", - "fh4": "image/x-freehand", - "fh5": "image/x-freehand", - "fh7": "image/x-freehand", - "fhc": "image/x-freehand", - "fif": "image/fif", - "fig": "application/x-xfig", - "finf": "application/fastinfoset", - "fish": "text/fish", - "fit": "image/fits", - "fits": "image/fits", - "fla": "application/vnd.dtg.local.flash", - "flac": "audio/x-flac", - "fli": "video/x-fli", - "flo": "application/vnd.micrografx.flo", - "flr": "x-world/x-vrml", - "flv": "video/x-flv", - "flw": "application/vnd.kde.kivio", - "flx": "text/vnd.fmi.flexstor", - "fly": "text/vnd.fly", + "mseed": "application/vnd.fdsn.mseed", + "seed": "application/vnd.fdsn.seed", + "dataless": "application/vnd.fdsn.seed", + "gph": "application/vnd.flographit", + "ftc": "application/vnd.fluxtime.clip", "fm": "application/vnd.framemaker", - "fmf": "video/x-atomic3d-feature", - "fnc": "application/vnd.frogans.fnc", - "fo": "application/vnd.software602.filler.form+xml", - "for": "text/x-fortran", - "fpx": "image/vnd.fpx", "frame": "application/vnd.framemaker", - "frl": "application/freeloader", - "frm": "application/vnd.ufdl", + "maker": "application/vnd.framemaker", + "book": "application/vnd.framemaker", + "fnc": "application/vnd.frogans.fnc", + "ltf": "application/vnd.frogans.ltf", "fsc": "application/vnd.fsc.weblaunch", - "fst": "image/vnd.fst", - "ftc": "application/vnd.fluxtime.clip", - "fti": "application/vnd.anser-web-funds-transfer-initiation", - "fts": "image/fits", - "funk": "audio/make", - "fvt": "video/vnd.fvt", - "fxm": "video/x-javafx", - "fxp": "application/vnd.adobe.fxp", - "fxpl": "application/vnd.adobe.fxp", + "oas": "application/vnd.fujitsu.oasys", + "oa2": "application/vnd.fujitsu.oasys2", + "oa3": "application/vnd.fujitsu.oasys3", + "fg5": "application/vnd.fujitsu.oasysgp", + "bh2": "application/vnd.fujitsu.oasysprs", + "ddd": "application/vnd.fujixerox.ddd", + "xdw": "application/vnd.fujixerox.docuworks", + "xbd": "application/vnd.fujixerox.docuworks.binder", "fzs": "application/vnd.fuzzysheet", - "g": "text/plain", - "g2w": "application/vnd.geoplan", - "g3": "image/g3fax", - "g3w": "application/vnd.geospace", - "gac": "application/vnd.groove-account", - "gal": "chemical/x-gaussian-log", - "gam": "application/x-tads", - "gamin": "chemical/x-gamess-input", - "gau": "chemical/x-gaussian-input", - "gbr": "application/rpki-ghostbusters", - "gca": "application/x-gca-compressed", - "gcd": "text/x-pcs-gcd", - "gcf": "application/x-graphing-calculator", - "gcg": "chemical/x-gcg8-sequence", - "gdl": "model/vnd.gdl", - "gdoc": "application/vnd.google-apps.document", - "gemspec": "text/ruby", - "gen": "chemical/x-genbank", - "geo": "application/vnd.dynageo", - "geojson": "application/geo+json", - "gex": "application/vnd.geometry-explorer", - "gf": "application/x-tex-gf", + "txd": "application/vnd.genomatix.tuxedo", "ggb": "application/vnd.geogebra.file", + "ggs": "application/vnd.geogebra.slides", "ggt": "application/vnd.geogebra.tool", - "ghf": "application/vnd.groove-help", - "gif": "image/gif", - "gim": "application/vnd.groove-identity-message", - "gjc": "chemical/x-gaussian-input", - "gjf": "chemical/x-gaussian-input", - "gl": "video/gl", - "glb": "model/gltf-binary", - "gltf": "model/gltf+json", - "gml": "application/gml+xml", + "gex": "application/vnd.geometry-explorer", + "gre": "application/vnd.geometry-explorer", + "gxt": "application/vnd.geonext", + "g2w": "application/vnd.geoplan", + "g3w": "application/vnd.geospace", "gmx": "application/vnd.gmx", - "gnumeric": "application/x-gnumeric", - "go": "text/go", - "gotmpl": "text/gotmpl", - "gph": "application/vnd.flographit", - "gpt": "chemical/x-mopac-graph", - "gpx": "application/gpx+xml", + "kml": "application/vnd.google-earth.kml+xml", + "kmz": "application/vnd.google-earth.kmz", "gqf": "application/vnd.grafeq", "gqs": "application/vnd.grafeq", - "gradle": "text/groovy", - "gram": "application/srgs", - "gramps": "application/x-gramps-xml", - "gre": "application/vnd.geometry-explorer", - "groovy": "text/groovy", + "gac": "application/vnd.groove-account", + "ghf": "application/vnd.groove-help", + "gim": "application/vnd.groove-identity-message", "grv": "application/vnd.groove-injector", - "grxml": "application/srgs+xml", - "gsd": "audio/x-gsm", - "gsf": "application/x-font-ghostscript", - "gsheet": "application/vnd.google-apps.spreadsheet", - "gslides": "application/vnd.google-apps.presentation", - "gsm": "model/vnd.gdl", - "gsp": "application/x-gsp", - "gss": "application/x-gss", - "gtar": "application/x-gtar", "gtm": "application/vnd.groove-tool-message", - "gtw": "model/vnd.gtw", - "gv": "text/vnd.graphviz", - "gxf": "application/gxf", - "gxt": "application/vnd.geonext", - "gyb": "text/gyb", - "gyp": "text/gyp", - "gypi": "text/gyp", - "gz": "application/gzip", - "h": "text/x-chdr", - "h++": "text/x-c++hdr", - "h261": "video/h261", - "h263": "video/h263", - "h264": "video/h264", + "tpl": "application/vnd.groove-tool-template", + "vcg": "application/vnd.groove-vcard", "hal": "application/vnd.hal+xml", - "hbc": "application/vnd.hbci", + "zmm": "application/vnd.handheld-entertainment+xml", "hbci": "application/vnd.hbci", - "hbs": "text/x-handlebars-template", - "hdd": "application/x-virtualbox-hdd", - "hdf": "application/x-hdf", - "hdr": "image/vnd.radiance", - "hdt": "application/vnd.hdt", - "heic": "image/heic", - "heics": "image/heic-sequence", - "heif": "image/heif", - "heifs": "image/heif-sequence", - "help": "application/x-helpfile", - "hgl": "application/vnd.hp-hpgl", - "hh": "text/plain", - "hin": "chemical/x-hin", - "hjson": "application/hjson", - "hlb": "text/x-script", - "hlp": "application/winhlp", - "hpg": "application/vnd.hp-hpgl", + "les": "application/vnd.hhe.lesson-player", "hpgl": "application/vnd.hp-hpgl", - "hpi": "application/vnd.hp-hpid", "hpid": "application/vnd.hp-hpid", - "hpp": "text/x-c++hdr", "hps": "application/vnd.hp-hps", - "hpub": "application/prs.hpub+zip", - "hqx": "application/mac-binhex40", - "hs": "text/x-haskell", - "hta": "application/hta", - "htc": "text/x-component", - "htke": "application/vnd.kenameaapp", - "html": "text/html", - "htt": "text/webviewhtml", - "hvd": "application/vnd.yamaha.hv-dic", - "hvp": "application/vnd.yamaha.hv-voice", - "hvs": "application/vnd.yamaha.hv-script", - "hx": "text/haxe", - "hxml": "text/haxe", - "hxx": "text/plain", - "i2g": "application/vnd.intergeo", - "ic0": "application/vnd.commerce-battelle", - "ic1": "application/vnd.commerce-battelle", - "ic2": "application/vnd.commerce-battelle", - "ic3": "application/vnd.commerce-battelle", - "ic4": "application/vnd.commerce-battelle", - "ic5": "application/vnd.commerce-battelle", - "ic6": "application/vnd.commerce-battelle", - "ic7": "application/vnd.commerce-battelle", - "ic8": "application/vnd.commerce-battelle", - "ica": "application/vnd.commerce-battelle", + "jlt": "application/vnd.hp-jlyt", + "pcl": "application/vnd.hp-pcl", + "pclxl": "application/vnd.hp-pclxl", + "sfd-hdstx": "application/vnd.hydrostatix.sof-data", + "mpy": "application/vnd.ibm.minipay", + "afp": "application/vnd.ibm.modcap", + "listafp": "application/vnd.ibm.modcap", + "list3820": "application/vnd.ibm.modcap", + "irm": "application/vnd.ibm.rights-management", + "sc": "application/vnd.ibm.secure-container", "icc": "application/vnd.iccprofile", - "icd": "application/vnd.commerce-battelle", - "ice": "x-conference/x-cooltalk", - "icf": "application/vnd.commerce-battelle", "icm": "application/vnd.iccprofile", - "icns": "binary/icns", - "ico": "image/x-icon", - "ics": "text/calendar", - "icz": "text/calendar", - "idc": "text/plain", - "idl": "text/idl", - "ief": "image/ief", - "iefs": "image/ief", - "ifb": "text/calendar", - "ifm": "application/vnd.shana.informed.formdata", - "iges": "model/iges", "igl": "application/vnd.igloader", - "igm": "application/vnd.insors.igm", - "ign": "application/vnd.coreos.ignition+json", - "ignition": "application/vnd.coreos.ignition+json", - "igs": "model/iges", - "igx": "application/vnd.micrografx.igx", - "iif": "application/vnd.shana.informed.interchange", - "iii": "application/x-iphone", - "ima": "application/x-ima", - "imap": "application/x-httpd-imap", - "imf": "application/vnd.imagemeter.folder+zip", - "img": "application/octet-stream", - "imgcal": "application/vnd.3lightssoftware.imagescal", - "imi": "application/vnd.imagemeter.image+zip", - "imp": "application/vnd.accpac.simply.imp", - "ims": "application/vnd.ms-ims", - "imscc": "application/vnd.ims.imsccv1p1", - "in": "text/plain", - "inc": "text/inc", - "inf": "application/inf", - "info": "application/x-info", - "ini": "text/ini", - "ink": "application/inkml+xml", - "inkml": "application/inkml+xml", - "inp": "chemical/x-gamess-input", - "ins": "application/x-internet-signup", - "install": "application/x-install-instructions", - "iota": "application/vnd.astraea-software.iota", - "ip": "application/x-ip2", - "ipfix": "application/ipfix", - "ipk": "application/vnd.shana.informed.package", - "irm": "application/vnd.ibm.rights-management", - "irp": "application/vnd.irepository.package+xml", - "ism": "model/vnd.gdl", - "iso": "application/x-iso9660-image", - "isp": "application/x-internet-signup", - "ist": "chemical/x-isostar", - "istr": "chemical/x-isostar", - "isu": "video/x-isvideo", - "it": "audio/it", - "itp": "application/vnd.shana.informed.formtemplate", - "its": "application/its+xml", - "iv": "application/x-inventor", "ivp": "application/vnd.immervision-ivp", - "ivr": "i-world/i-vrml", "ivu": "application/vnd.immervision-ivu", - "ivy": "application/x-livescreen", - "j2": "text/jinja", - "jad": "text/vnd.sun.j2me.app-descriptor", - "jade": "text/jade", + "igm": "application/vnd.insors.igm", + "xpw": "application/vnd.intercon.formnet", + "xpx": "application/vnd.intercon.formnet", + "i2g": "application/vnd.intergeo", + "qbo": "application/vnd.intu.qbo", + "qfx": "application/vnd.intu.qfx", + "rcprofile": "application/vnd.ipunplugged.rcprofile", + "irp": "application/vnd.irepository.package+xml", + "xpr": "application/vnd.is-xpr", + "fcs": "application/vnd.isac.fcs", "jam": "application/vnd.jam", - "jar": "application/x-java-archive", - "jardiff": "application/x-java-archive-diff", - "java": "text/x-java-source", - "jcm": "application/x-java-commerce", - "jdx": "chemical/x-jcamp-dx", - "jenkinsfile": "text/groovy", - "jfif": "image/jpeg", - "jinja": "text/jinja", - "jinja2": "text/jinja", + "rms": "application/vnd.jcp.javame.midlet-rms", "jisp": "application/vnd.jisp", - "jls": "image/jls", - "jlt": "application/vnd.hp-jlyt", - "jl": "text/julia", - "jmz": "application/x-jmol", - "jng": "image/x-jng", - "jnlp": "application/x-java-jnlp-file", "joda": "application/vnd.joost.joda-archive", - "jp2": "image/jp2", - "jpe": "image/jpeg", - "jpeg": "image/jpeg", - "jpf": "image/jpx", - "jpg": "image/jpeg", - "jpg2": "image/jp2", - "jpgm": "image/jpm", - "jpgv": "video/jpeg", - "jpm": "image/jpm", - "jps": "image/x-jps", - "jpx": "image/jpx", - "jrd": "application/jrd+json", - "js": "application/javascript", - "json": "application/json", - "json-patch": "application/json-patch+json", - "json5": "application/json5", - "jsonld": "application/ld+json", - "jsonml": "application/jsonml+json", - "jsx": "text/jsx", - "jtd": "text/vnd.esmertec.theme-descriptor", - "jut": "image/jutvision", - "kar": "audio/midi", + "ktz": "application/vnd.kahootz", + "ktr": "application/vnd.kahootz", "karbon": "application/vnd.kde.karbon", - "kcm": "application/vnd.nervana", - "key": "application/pgp-keys", - "keynote": "application/vnd.apple.keynote", + "chrt": "application/vnd.kde.kchart", "kfo": "application/vnd.kde.kformula", - "kia": "application/vnd.kidspiration", - "kil": "application/x-killustrator", - "kin": "chemical/x-kinemage", - "kml": "application/vnd.google-earth.kml+xml", - "kmz": "application/vnd.google-earth.kmz", - "kne": "application/vnd.kinar", - "knp": "application/vnd.kinar", - "kom": "application/vnd.hbci", + "flw": "application/vnd.kde.kivio", "kon": "application/vnd.kde.kontour", - "koz": "audio/vnd.audikoz", "kpr": "application/vnd.kde.kpresenter", "kpt": "application/vnd.kde.kpresenter", - "kpxx": "application/vnd.ds-keypoint", - "ksh": "application/x-ksh", "ksp": "application/vnd.kde.kspread", - "kt": "text/kotlin", - "ktr": "application/vnd.kahootz", - "ktx": "image/ktx", - "ktz": "application/vnd.kahootz", "kwd": "application/vnd.kde.kword", "kwt": "application/vnd.kde.kword", - "l16": "audio/l16", - "la": "audio/nspaudio", - "lam": "audio/x-liveaudio", - "lasjson": "application/vnd.las.las+json", + "htke": "application/vnd.kenameaapp", + "kia": "application/vnd.kidspiration", + "kne": "application/vnd.kinar", + "knp": "application/vnd.kinar", + "skp": "application/vnd.koan", + "skd": "application/vnd.koan", + "skt": "application/vnd.koan", + "skm": "application/vnd.koan", + "sse": "application/vnd.kodak-descriptor", "lasxml": "application/vnd.las.las+xml", - "latex": "application/x-latex", - "lbc": "audio/ilbc", "lbd": "application/vnd.llamagraphics.life-balance.desktop", "lbe": "application/vnd.llamagraphics.life-balance.exchange+xml", - "le": "application/vnd.bluetooth.le.oob", - "les": "application/vnd.hhe.lesson-player", - "less": "text/less", - "lgr": "application/lgr+xml", - "lha": "application/octet-stream", - "lhs": "text/x-literate-haskell", - "lhx": "application/octet-stream", - "lin": "application/bbolin", - "link66": "application/vnd.route66.link66+xml", - "list": "text/plain", - "list3820": "application/vnd.ibm.modcap", - "listafp": "application/vnd.ibm.modcap", - "lmp": "model/vnd.gdl", - "lnk": "application/x-ms-shortcut", - "log": "text/plain", - "lostsyncxml": "application/lostsync+xml", - "lostxml": "application/lost+xml", - "lrf": "application/octet-stream", - "lrm": "application/vnd.ms-lrm", - "lsf": "video/x-la-asf", - "lsp": "text/x-script.lisp", - "lst": "text/plain", - "lsx": "video/x-la-asf", - "ltf": "application/vnd.frogans.ltf", - "ltx": "application/x-latex", - "lua": "text/x-lua", - "luac": "application/x-lua-bytecode", - "lvp": "audio/vnd.lucent.voice", + "123": "application/vnd.lotus-1-2-3", + "apr": "application/vnd.lotus-approach", + "pre": "application/vnd.lotus-freelance", + "nsf": "application/vnd.lotus-notes", + "org": "application/vnd.lotus-organizer", + "scm": "application/vnd.lotus-screencam", "lwp": "application/vnd.lotus-wordpro", - "lxf": "application/lxf", - "lyx": "application/x-lyx", - "lzh": "application/octet-stream", - "lzx": "application/x-lzx", - "m": "application/vnd.wolfram.mathematica.package", - "m13": "application/x-msmediaview", - "m14": "application/x-msmediaview", - "m15": "audio/x-mod", - "m1v": "video/mpeg", - "m21": "application/mp21", - "m2a": "audio/mpeg", - "m2v": "video/mpeg", - "m3a": "audio/mpeg", - "m3g": "application/m3g", - "m3u": "audio/x-mpegurl", - "m3u8": "application/vnd.apple.mpegurl", - "m4a": "audio/x-m4a", - "m4s": "video/iso.segment", - "m4u": "video/vnd.mpegurl", - "m4v": "video/x-m4v", - "ma": "application/mathematica", - "mads": "application/mads+xml", - "mag": "application/vnd.ecowin.chart", - "mail": "message/rfc822", - "maker": "application/vnd.framemaker", - "man": "application/x-troff-man", - "manifest": "text/cache-manifest", - "map": "application/x-navimap", - "mar": "text/plain", - "markdown": "text/markdown", - "mathml": "application/mathml+xml", - "mb": "application/mathematica", - "mbd": "application/mbedlet", - "mbk": "application/vnd.mobius.mbk", - "mbox": "application/mbox", - "mc$": "application/x-magic-cap-package-1.0", - "mc1": "application/vnd.medcalcdata", + "portpkg": "application/vnd.macports.portpkg", "mcd": "application/vnd.mcd", - "mcf": "image/vasa", - "mcif": "chemical/x-mmcif", - "mcm": "chemical/x-macmolecule", - "mcp": "application/netmc", - "mcurl": "text/vnd.curl.mcurl", - "md": "text/markdown", - "mdb": "application/x-msaccess", - "mdc": "application/vnd.marlin.drm.mdcf", - "mdi": "image/vnd.ms-modi", - "me": "application/x-troff-me", - "med": "audio/x-mod", - "mesh": "model/mesh", - "meta4": "application/metalink4+xml", - "metalink": "application/metalink+xml", - "mets": "application/mets+xml", - "mf4": "application/mf4", + "mc1": "application/vnd.medcalcdata", + "cdkey": "application/vnd.mediastation.cdkey", + "mwf": "application/vnd.mfer", "mfm": "application/vnd.mfmp", - "mft": "application/rpki-manifest", - "mgp": "application/vnd.osgeo.mapguide.package", - "mgz": "application/vnd.proteus.magazine", - "mht": "message/rfc822", - "mhtml": "message/rfc822", - "mib": "text/mib", - "mid": "audio/midi", - "midi": "audio/midi", - "mie": "application/x-mie", - "mif": "application/x-mif", - "mime": "message/rfc822", - "miz": "text/mizar", - "mj2": "video/mj2", - "mjf": "audio/x-vnd.audioexplosion.mjuicemediafile", - "mjp2": "video/mj2", - "mjpg": "video/x-motion-jpeg", - "mjs": "application/javascript", - "mk": "text/makefile", - "mk3d": "video/x-matroska-3d", - "mka": "audio/x-matroska", - "mkd": "text/x-markdown", - "mks": "video/x-matroska", - "mkv": "video/x-matroska", - "mlp": "application/vnd.dolby.mlp", - "mm": "application/x-freemind", - "mmd": "application/vnd.chipnuts.karaoke-mmd", - "mmdb": "application/vnd.maxmind.maxmind-db", - "mme": "application/base64", - "mmf": "application/vnd.smaf", - "mml": "text/mathml", - "mmod": "chemical/x-macromodel-input", - "mmr": "image/vnd.fujixerox.edmics-mmr", - "mms": "application/vnd.wap.mms-message", - "mng": "video/x-mng", - "mny": "application/x-msmoney", - "mobi": "application/x-mobipocket-ebook", - "moc": "text/x-moc", - "mod": "audio/x-mod", - "model-inter": "application/vnd.vd-study", - "mods": "application/mods+xml", - "modulemap": "text/modulemap", - "mol": "chemical/x-mdl-molfile", - "mol2": "chemical/x-mol2", - "moml": "model/vnd.moml+xml", - "moo": "chemical/x-mopac-out", - "moov": "video/quicktime", - "mop": "chemical/x-mopac-input", - "mopcrt": "chemical/x-mopac-input", - "mov": "video/quicktime", - "movie": "video/x-sgi-movie", - "mp1": "audio/mpeg", - "mp2": "audio/mpeg", - "mp21": "application/mp21", - "mp2a": "audio/mpeg", - "mp3": "audio/mp3", - "mp4": "video/mp4", - "mp4a": "audio/mp4", - "mp4s": "application/mp4", - "mp4v": "video/mp4", - "mpa": "video/mpeg", - "mpc": "application/vnd.mophun.certificate", - "mpd": "application/dash+xml", - "mpdd": "application/dashdelta", - "mpe": "video/mpeg", - "mpeg": "video/mpeg", - "mpega": "audio/mpeg", - "mpf": "text/vnd.ms-mediapackage", - "mpg": "video/mpeg", - "mpg4": "video/mp4", - "mpga": "audio/mpeg", - "mpkg": "application/vnd.apple.installer+xml", - "mpm": "application/vnd.blueice.multipass", + "flo": "application/vnd.micrografx.flo", + "igx": "application/vnd.micrografx.igx", + "mif": "application/vnd.mif", + "daf": "application/vnd.mobius.daf", + "dis": "application/vnd.mobius.dis", + "mbk": "application/vnd.mobius.mbk", + "mqy": "application/vnd.mobius.mqy", + "msl": "application/vnd.mobius.msl", + "plc": "application/vnd.mobius.plc", + "txf": "application/vnd.mobius.txf", "mpn": "application/vnd.mophun.application", + "mpc": "application/vnd.mophun.certificate", + "xul": "application/vnd.mozilla.xul+xml", + "cil": "application/vnd.ms-artgalry", + "cab": "application/vnd.ms-cab-compressed", + "xls": "application/vnd.ms-excel", + "xlm": "application/vnd.ms-excel", + "xla": "application/vnd.ms-excel", + "xlc": "application/vnd.ms-excel", + "xlt": "application/vnd.ms-excel", + "xlw": "application/vnd.ms-excel", + "xlam": "application/vnd.ms-excel.addin.macroenabled.12", + "xlsb": "application/vnd.ms-excel.sheet.binary.macroenabled.12", + "xlsm": "application/vnd.ms-excel.sheet.macroenabled.12", + "xltm": "application/vnd.ms-excel.template.macroenabled.12", + "eot": "application/vnd.ms-fontobject", + "chm": "application/vnd.ms-htmlhelp", + "ims": "application/vnd.ms-ims", + "lrm": "application/vnd.ms-lrm", + "thmx": "application/vnd.ms-officetheme", + "cat": "application/vnd.ms-pki.seccat", + "stl": "application/vnd.ms-pki.stl", + "ppt": "application/vnd.ms-powerpoint", + "pps": "application/vnd.ms-powerpoint", + "pot": "application/vnd.ms-powerpoint", + "ppam": "application/vnd.ms-powerpoint.addin.macroenabled.12", + "pptm": "application/vnd.ms-powerpoint.presentation.macroenabled.12", + "sldm": "application/vnd.ms-powerpoint.slide.macroenabled.12", + "ppsm": "application/vnd.ms-powerpoint.slideshow.macroenabled.12", + "potm": "application/vnd.ms-powerpoint.template.macroenabled.12", "mpp": "application/vnd.ms-project", "mpt": "application/vnd.ms-project", - "mpv": "application/x-project", - "mpv2": "video/mpeg", - "mpx": "application/x-project", - "mpy": "application/vnd.ibm.minipay", - "mqy": "application/vnd.mobius.mqy", - "mrc": "application/marc", - "mrcx": "application/marcxml+xml", - "ms": "application/x-troff-ms", - "msa": "application/vnd.msa-disk-image", - "mscml": "application/mediaservercontrol+xml", - "msd": "application/vnd.fdsn.mseed", - "mseed": "application/vnd.fdsn.mseed", + "docm": "application/vnd.ms-word.document.macroenabled.12", + "dotm": "application/vnd.ms-word.template.macroenabled.12", + "wps": "application/vnd.ms-works", + "wks": "application/vnd.ms-works", + "wcm": "application/vnd.ms-works", + "wdb": "application/vnd.ms-works", + "wpl": "application/vnd.ms-wpl", + "xps": "application/vnd.ms-xpsdocument", "mseq": "application/vnd.mseq", - "msf": "application/vnd.epson.msf", - "msg": "application/vnd.ms-outlook", - "msh": "model/mesh", - "msi": "application/x-msi", - "msl": "application/vnd.mobius.msl", - "msm": "model/vnd.gdl", - "msty": "application/vnd.muvee.style", - "mtm": "audio/x-mod", - "mts": "model/vnd.mts", - "multitrack": "audio/vnd.presonus.multitrack", "mus": "application/vnd.musician", - "musd": "application/mmt-usd+xml", - "musicxml": "application/vnd.recordare.musicxml+xml", - "mv": "video/x-sgi-movie", - "mvb": "application/x-msmediaview", - "mvt": "application/vnd.mapbox-vector-tile", - "mwc": "application/vnd.dpgraph", - "mwf": "application/vnd.mfer", - "mxf": "application/mxf", - "mxi": "application/vnd.vd-study", - "mxl": "application/vnd.recordare.musicxml", - "mxmf": "audio/mobile-xmf", - "mxml": "application/xv+xml", - "mxs": "application/vnd.triscape.mxs", - "mxu": "video/vnd.mpegurl", - "my": "audio/make", - "mzz": "application/x-vnd.audioexplosion.mzz", - "n-gage": "application/vnd.nokia.n-gage.symbian.install", - "n3": "text/n3", - "nap": "image/naplps", - "naplps": "image/naplps", - "nb": "application/mathematica", - "nbp": "application/vnd.wolfram.player", - "nc": "application/x-netcdf", - "ncm": "application/vnd.nokia.configuration-message", - "ncx": "application/x-dtbncx+xml", - "ndc": "application/vnd.osa.netdeploy", - "ndjson": "application/json", - "ndl": "application/vnd.lotus-notes", - "nds": "application/vnd.nintendo.nitro.rom", - "nef": "image/x-nikon-nef", - "nfo": "text/x-nfo", - "ngdat": "application/vnd.nokia.n-gage.data", - "ngdoc": "text/ngdoc", - "nif": "image/x-niff", - "niff": "image/x-niff", + "msty": "application/vnd.muvee.style", + "taglet": "application/vnd.mynfc", + "nlu": "application/vnd.neurolanguage.nlu", "nim": "text/nim", "nimble": "text/nimble", "nimf": "text/nim", "nims": "text/nim", + "ntf": "application/vnd.nitf", "nitf": "application/vnd.nitf", - "nix": "application/x-mix-transfer", - "nlu": "application/vnd.neurolanguage.nlu", - "nml": "application/vnd.enliven", "nnd": "application/vnd.noblenet-directory", "nns": "application/vnd.noblenet-sealer", "nnw": "application/vnd.noblenet-web", - "notebook": "application/vnd.smart.notebook", - "npx": "image/vnd.net-fpx", - "nq": "application/n-quads", - "ns2": "application/vnd.lotus-notes", - "ns3": "application/vnd.lotus-notes", - "ns4": "application/vnd.lotus-notes", - "nsc": "application/x-conference", - "nsf": "application/vnd.lotus-notes", - "nsg": "application/vnd.lotus-notes", - "nsh": "application/vnd.lotus-notes", - "nt": "application/n-triples", - "ntf": "application/vnd.lotus-notes", - "numbers": "application/vnd.apple.numbers", - "nvd": "application/x-navidoc", - "nwc": "application/x-nwc", - "nws": "message/rfc822", - "nzb": "application/x-nzb", - "o": "application/x-object", - "o4a": "application/vnd.oma.drm.dcf", - "o4v": "application/vnd.oma.drm.dcf", - "oa2": "application/vnd.fujitsu.oasys2", - "oa3": "application/vnd.fujitsu.oasys3", - "oas": "application/vnd.fujitsu.oasys", - "obd": "application/x-msbinder", - "obg": "application/vnd.openblox.game-binary", - "obgx": "application/vnd.openblox.game+xml", - "obj": "application/x-tgif", - "oda": "application/oda", - "odb": "application/vnd.oasis.opendocument.database", + "ngdat": "application/vnd.nokia.n-gage.data", + "n-gage": "application/vnd.nokia.n-gage.symbian.install", + "rpst": "application/vnd.nokia.radio-preset", + "rpss": "application/vnd.nokia.radio-presets", + "edm": "application/vnd.novadigm.edm", + "edx": "application/vnd.novadigm.edx", + "ext": "application/vnd.novadigm.ext", "odc": "application/vnd.oasis.opendocument.chart", - "odd": "application/tei+xml", + "otc": "application/vnd.oasis.opendocument.chart-template", + "odb": "application/vnd.oasis.opendocument.database", "odf": "application/vnd.oasis.opendocument.formula", "odft": "application/vnd.oasis.opendocument.formula-template", "odg": "application/vnd.oasis.opendocument.graphics", - "odi": "application/vnd.oasis.opendocument.image", - "odm": "application/vnd.oasis.opendocument.text-master", - "odp": "application/vnd.oasis.opendocument.presentation", - "ods": "application/vnd.oasis.opendocument.spreadsheet", - "odt": "application/vnd.oasis.opendocument.text", - "odx": "application/odx", - "oeb": "application/vnd.openeye.oeb", - "oga": "audio/ogg", - "ogex": "model/vnd.opengex", - "ogg": "audio/ogg", - "ogv": "video/ogg", - "ogx": "application/ogg", - "old": "application/x-trash", - "omc": "application/x-omc", - "omcd": "application/x-omcdatamaker", - "omcr": "application/x-omcregerator", - "omdoc": "application/omdoc+xml", - "omg": "audio/atrac3", - "onepkg": "application/onenote", - "onetmp": "application/onenote", - "onetoc": "application/onenote", - "onetoc2": "application/onenote", - "opf": "application/oebps-package+xml", - "opml": "text/x-opml", - "oprc": "application/vnd.palm", - "opus": "audio/ogg", - "or2": "application/vnd.lotus-organizer", - "or3": "application/vnd.lotus-organizer", - "orf": "image/x-olympus-orf", - "org": "text/x-org", - "orq": "application/ocsp-request", - "ors": "application/ocsp-response", - "osf": "application/vnd.yamaha.openscoreformat", - "osfpvg": "application/vnd.yamaha.openscoreformat.osfpvg+xml", - "osm": "application/vnd.openstreetmap.data+xml", - "otc": "application/vnd.oasis.opendocument.chart-template", - "otf": "font/otf", "otg": "application/vnd.oasis.opendocument.graphics-template", - "oth": "application/vnd.oasis.opendocument.text-web", + "odi": "application/vnd.oasis.opendocument.image", "oti": "application/vnd.oasis.opendocument.image-template", + "odp": "application/vnd.oasis.opendocument.presentation", "otp": "application/vnd.oasis.opendocument.presentation-template", + "ods": "application/vnd.oasis.opendocument.spreadsheet", "ots": "application/vnd.oasis.opendocument.spreadsheet-template", + "odt": "application/vnd.oasis.opendocument.text", + "odm": "application/vnd.oasis.opendocument.text-master", "ott": "application/vnd.oasis.opendocument.text-template", - "ova": "application/x-virtualbox-ova", - "ovf": "application/x-virtualbox-ovf", - "owx": "application/owl+xml", - "oxlicg": "application/vnd.oxli.countgraph", - "oxps": "application/oxps", + "oth": "application/vnd.oasis.opendocument.text-web", + "xo": "application/vnd.olpc-sugar", + "dd2": "application/vnd.oma.dd2+xml", "oxt": "application/vnd.openofficeorg.extension", - "oza": "application/x-oz-application", - "p": "text/x-pascal", - "p10": "application/pkcs10", - "p12": "application/pkcs12", - "p2p": "application/vnd.wfa.p2p", - "p7a": "application/x-pkcs7-signature", - "p7b": "application/x-pkcs7-certificates", - "p7c": "application/pkcs7-mime", - "p7m": "application/pkcs7-mime", - "p7r": "application/x-pkcs7-certreqresp", - "p7s": "application/pkcs7-signature", - "p8": "application/pkcs8", - "pac": "application/x-ns-proxy-autoconfig", - "pack": "application/x-java-pack200", - "package": "application/vnd.autopackage", - "pages": "application/vnd.apple.pages", - "par": "text/plain-bas", - "part": "application/pro_eng", - "pas": "text/pascal", - "pat": "image/x-coreldrawpattern", - "patch": "text/x-diff", - "paw": "application/vnd.pawaafile", - "pbd": "application/vnd.powerbuilder6", - "pbm": "image/x-portable-bitmap", - "pcap": "application/vnd.tcpdump.pcap", - "pcf": "application/x-font-pcf", - "pcl": "application/vnd.hp-pcl", - "pclxl": "application/vnd.hp-pclxl", - "pct": "image/x-pict", - "pcurl": "application/vnd.curl.pcurl", - "pcx": "image/x-pcx", - "pdb": "application/vnd.palm", - "pde": "text/x-processing", - "pdf": "application/pdf", - "pdx": "application/pdx", - "pem": "text/pem", - "pfa": "application/x-font-type1", - "pfb": "application/x-font-type1", - "pfm": "application/x-font-type1", - "pfr": "application/font-tdpfr", - "pfunk": "audio/make", - "pfx": "application/pkcs12", - "pgb": "image/vnd.globalgraphics.pgb", - "pgm": "image/x-portable-graymap", - "pgn": "application/x-chess-pgn", - "pgp": "application/pgp-encrypted", - "php": "application/x-httpd-php", - "php3": "application/x-httpd-php3", - "php3p": "application/x-httpd-php3-preprocessed", - "php4": "application/x-httpd-php4", - "php5": "application/x-httpd-php5", - "phps": "application/x-httpd-php-source", - "pht": "application/x-httpd-php", - "phtml": "application/x-httpd-php", - "pic": "image/pict", - "pict": "image/pict", - "pil": "application/vnd.piaccess.application-license", - "pk": "application/x-tex-pk", - "pkd": "application/vnd.hbci", - "pkg": "application/vnd.apple.installer+xml", - "pki": "application/pkixcmp", - "pkipath": "application/pkix-pkipath", - "pko": "application/ynd.ms-pkipko", - "pkpass": "application/vnd.apple.pkpass", - "pl": "application/x-perl", - "plantuml": "text/plantuml", - "plb": "application/vnd.3gpp.pic-bw-large", - "plc": "application/vnd.mobius.plc", - "plf": "application/vnd.pocketlearn", - "plj": "audio/vnd.everad.plj", - "plp": "application/vnd.panoply", - "pls": "application/pls+xml", - "plx": "application/x-pixclscript", - "ply": "model/stanford", - "pm": "text/plain", - "pm4": "application/x-pagemaker", - "pm5": "application/x-pagemaker", - "pma": "application/x-perfmon", - "pmc": "application/x-perfmon", - "pml": "application/vnd.ctc-posml", - "pmr": "application/x-perfmon", - "pmw": "application/x-perfmon", - "png": "image/png", - "pnm": "image/x-portable-anymap", - "po": "text/pofile", - "pod": "text/x-pod", - "portpkg": "application/vnd.macports.portpkg", - "pot": "application/vnd.ms-powerpoint", - "potm": "application/vnd.ms-powerpoint.template.macroenabled.12", - "potx": "application/vnd.openxmlformats-officedocument.presentationml.template", - "pov": "model/x-pov", - "pp": "text/puppet", - "ppa": "application/vnd.ms-powerpoint", - "ppam": "application/vnd.ms-powerpoint.addin.macroenabled.12", - "ppd": "application/vnd.cups-ppd", - "ppkg": "application/vnd.xmpie.ppkg", - "ppm": "image/x-portable-pixmap", - "pps": "application/vnd.ms-powerpoint", - "ppsm": "application/vnd.ms-powerpoint.slideshow.macroenabled.12", - "ppsx": "application/vnd.openxmlformats-officedocument.presentationml.slideshow", - "ppt": "application/vnd.ms-powerpoint", - "pptm": "application/vnd.ms-powerpoint.presentation.macroenabled.12", "pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation", - "ppz": "application/mspowerpoint", + "sldx": "application/vnd.openxmlformats-officedocument.presentationml.slide", + "ppsx": "application/vnd.openxmlformats-officedocument.presentationml.slideshow", + "potx": "application/vnd.openxmlformats-officedocument.presentationml.template", + "xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "xltx": "application/vnd.openxmlformats-officedocument.spreadsheetml.template", + "docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "dotx": "application/vnd.openxmlformats-officedocument.wordprocessingml.template", + "mgp": "application/vnd.osgeo.mapguide.package", + "dp": "application/vnd.osgi.dp", + "esa": "application/vnd.osgi.subsystem", + "pdb": "application/vnd.palm", "pqa": "application/vnd.palm", - "prc": "application/vnd.palm", - "pre": "application/vnd.lotus-freelance", - "preminet": "application/vnd.preminet", - "prf": "application/pics-rules", - "proto": "text/proto", - "provn": "text/provenance-notation", - "provx": "application/provenance+xml", - "prt": "application/pro_eng", - "prz": "application/vnd.lotus-freelance", - "ps": "application/postscript", - "psb": "application/vnd.3gpp.pic-bw-small", - "psd": "image/vnd.adobe.photoshop", - "pseg3820": "application/vnd.ibm.modcap", - "psf": "application/x-font-linux-psf", - "psid": "audio/prs.sid", - "pskcxml": "application/pskc+xml", - "pti": "image/prs.pti", - "ptid": "application/vnd.pvi.ptid1", - "pub": "application/x-mspublisher", - "purs": "text/purescript", - "pvb": "application/vnd.3gpp.pic-bw-var", - "pvu": "paleovu/x-pv", - "pwn": "application/vnd.3m.post-it-notes", - "pwz": "application/vnd.ms-powerpoint", - "pxd": "text/cython", - "pxi": "text/cython", - "py": "text/x-script.phyton", - "pya": "audio/vnd.ms-playready.media.pya", - "pyc": "application/x-python-code", - "pyi": "text/pyi", - "pyo": "application/x-python-code", - "pyv": "video/vnd.ms-playready.media.pyv", - "pyx": "text/cython", - "qam": "application/vnd.epson.quickanime", - "qbo": "application/vnd.intu.qbo", - "qca": "application/vnd.ericsson.quickcall", - "qcall": "application/vnd.ericsson.quickcall", - "qcp": "audio/qcelp", - "qd3": "x-world/x-3dmf", - "qd3d": "x-world/x-3dmf", - "qfx": "application/vnd.intu.qfx", - "qgs": "application/x-qgis", - "qif": "image/x-quicktime", + "oprc": "application/vnd.palm", + "paw": "application/vnd.pawaafile", + "str": "application/vnd.pg.format", + "ei6": "application/vnd.pg.osasli", + "efif": "application/vnd.picsel", + "wg": "application/vnd.pmi.widget", + "plf": "application/vnd.pocketlearn", + "pbd": "application/vnd.powerbuilder6", + "box": "application/vnd.previewsystems.box", + "mgz": "application/vnd.proteus.magazine", "qps": "application/vnd.publishare-delta-tree", - "qt": "video/quicktime", - "qtc": "video/x-qtc", - "qti": "image/x-quicktime", - "qtif": "image/x-quicktime", - "qtl": "application/x-quicktimeplayer", - "quiz": "application/vnd.quobject-quoxdocument", - "quox": "application/vnd.quobject-quoxdocument", - "qvd": "application/vnd.theqvd", + "ptid": "application/vnd.pvi.ptid1", + "qxd": "application/vnd.quark.quarkxpress", + "qxt": "application/vnd.quark.quarkxpress", "qwd": "application/vnd.quark.quarkxpress", "qwt": "application/vnd.quark.quarkxpress", - "qxb": "application/vnd.quark.quarkxpress", - "qxd": "application/vnd.quark.quarkxpress", "qxl": "application/vnd.quark.quarkxpress", - "qxt": "application/vnd.quark.quarkxpress", - "r": "text/r", - "ra": "audio/x-realaudio", - "ram": "audio/x-pn-realaudio", - "raml": "application/raml+yaml", - "rapd": "application/route-apd+xml", - "rar": "application/x-rar-compressed", - "ras": "image/x-cmu-raster", - "rast": "image/cmu-raster", - "rb": "application/x-ruby", - "rcprofile": "application/vnd.ipunplugged.rcprofile", - "rct": "application/prs.nprend", - "rd": "chemical/x-mdl-rdfile", - "rda": "text/r", - "rdata": "text/r", - "rds": "text/r", - "rdf": "application/rdf+xml", - "rdf-crypt": "application/prs.rdf-xml-crypt", - "rdz": "application/vnd.data-vision.rdz", - "relo": "application/p2p-overlay+xml", - "rep": "application/vnd.businessobjects", - "request": "application/vnd.nervana", - "res": "application/x-dtbresource+xml", - "rexx": "text/x-script.rexx", - "rf": "image/vnd.rn-realflash", - "rfcxml": "application/rfc+xml", - "rgb": "image/x-rgb", - "rgbe": "image/vnd.radiance", - "rhtml": "application/x-httpd-eruby", - "rif": "application/reginfo+xml", - "rip": "audio/vnd.rip", - "ris": "application/x-research-info-systems", - "rl": "application/resource-lists+xml", - "rlc": "image/vnd.fujixerox.edmics-rlc", - "rld": "application/resource-lists-diff+xml", - "rlib": "text/rust", + "qxb": "application/vnd.quark.quarkxpress", + "bed": "application/vnd.realvnc.bed", + "mxl": "application/vnd.recordare.musicxml", + "musicxml": "application/vnd.recordare.musicxml+xml", + "cryptonote": "application/vnd.rig.cryptonote", + "cod": "application/vnd.rim.cod", "rm": "application/vnd.rn-realmedia", - "rmi": "audio/mid", - "rmm": "audio/x-pn-realaudio", - "rmp": "audio/x-pn-realaudio-plugin", - "rms": "application/vnd.jcp.javame.midlet-rms", "rmvb": "application/vnd.rn-realmedia-vbr", - "rnc": "application/relax-ng-compact-syntax", - "rnd": "application/prs.nprend", - "rng": "text/xml", - "rnx": "application/vnd.rn-realplayer", - "roa": "application/rpki-roa", - "roff": "text/troff", - "ros": "chemical/x-rosdal", - "rp": "image/vnd.rn-realpix", - "rp9": "application/vnd.cloanto.rp9", - "rpm": "application/x-redhat-package-manager", - "rpss": "application/vnd.nokia.radio-presets", - "rpst": "application/vnd.nokia.radio-preset", - "rq": "application/sparql-query", - "rs": "application/rls-services+xml", - "rsd": "application/rsd+xml", - "rsheet": "application/urc-ressheet+xml", - "rsm": "model/vnd.gdl", - "rss": "application/rss+xml", - "rst": "text/prs.fallenstein.rst", - "rt": "text/richtext", - "rtf": "text/rtf", - "rtx": "text/richtext", - "run": "application/x-makeself", - "rusd": "application/route-usd+xml", - "rv": "video/vnd.rn-realvideo", - "rxn": "chemical/x-mdl-rxnfile", - "s": "text/x-asm", - "s11": "video/vnd.sealed.mpeg1", - "s14": "video/vnd.sealed.mpeg4", - "s1a": "application/vnd.sealedmedia.softseal.pdf", - "s1e": "application/vnd.sealed.xls", - "s1g": "image/vnd.sealedmedia.softseal.gif", - "s1h": "application/vnd.sealedmedia.softseal.html", - "s1j": "image/vnd.sealedmedia.softseal.jpg", - "s1m": "audio/vnd.sealedmedia.softseal.mpeg", - "s1n": "image/vnd.sealed.png", - "s1p": "application/vnd.sealed.ppt", - "s1q": "video/vnd.sealedmedia.softseal.mov", - "s1w": "application/vnd.sealed.doc", - "s3df": "application/vnd.sealed.3df", - "s3m": "audio/s3m", - "sac": "application/tamp-sequence-adjust-confirm", - "saf": "application/vnd.yamaha.smaf-audio", - "sam": "application/vnd.lotus-wordpro", - "sandboxed": "text/html-sandboxed", - "sass": "text/x-sass", - "saveme": "application/octet-stream", - "sbk": "application/x-tbook", - "sbml": "application/sbml+xml", - "sc": "application/vnd.ibm.secure-container", - "scala": "text/x-scala", - "scd": "application/x-msschedule", - "sce": "application/vnd.etsi.asic-e+zip", - "scim": "application/scim+json", - "scld": "application/vnd.doremir.scorecloud-binary-document", - "scm": "application/vnd.lotus-screencam", - "scq": "application/scvp-cv-request", - "scr": "application/x-silverlight", - "scs": "application/scvp-cv-response", - "scsf": "application/vnd.sealed.csf", - "scss": "text/x-scss", - "sct": "text/scriptlet", - "scurl": "text/vnd.curl.scurl", - "sd": "chemical/x-mdl-sdfile", - "sd2": "audio/x-sd2", - "sda": "application/vnd.stardivision.draw", - "sdc": "application/vnd.stardivision.calc", - "sdd": "application/vnd.stardivision.impress", - "sdf": "application/vnd.kinar", - "sdkd": "application/vnd.solent.sdkm+xml", - "sdkm": "application/vnd.solent.sdkm+xml", - "sdml": "text/plain", - "sdo": "application/vnd.sealed.doc", - "sdoc": "application/vnd.sealed.doc", - "sdp": "application/sdp", - "sdr": "application/sounder", - "sdw": "application/vnd.stardivision.writer", - "sea": "application/x-sea", + "link66": "application/vnd.route66.link66+xml", + "st": "application/vnd.sailingtracker.track", "see": "application/vnd.seemail", - "seed": "application/vnd.fdsn.seed", - "sem": "application/vnd.sealed.eml", "sema": "application/vnd.sema", "semd": "application/vnd.semd", "semf": "application/vnd.semf", - "seml": "application/vnd.sealed.eml", - "ser": "application/java-serialized-object", - "set": "application/set", - "setpay": "application/set-payment-initiation", - "setreg": "application/set-registration-initiation", - "sfc": "application/vnd.nintendo.snes.rom", - "sfd": "application/vnd.font-fontforge-sfd", - "sfd-hdstx": "application/vnd.hydrostatix.sof-data", + "ifm": "application/vnd.shana.informed.formdata", + "itp": "application/vnd.shana.informed.formtemplate", + "iif": "application/vnd.shana.informed.interchange", + "ipk": "application/vnd.shana.informed.package", + "twd": "application/vnd.simtech-mindmapper", + "twds": "application/vnd.simtech-mindmapper", + "mmf": "application/vnd.smaf", + "teacher": "application/vnd.smart.teacher", + "sdkm": "application/vnd.solent.sdkm+xml", + "sdkd": "application/vnd.solent.sdkm+xml", + "dxp": "application/vnd.spotfire.dxp", "sfs": "application/vnd.spotfire.sfs", - "sfv": "text/x-sfv", - "sgf": "application/x-go-sgf", - "sgi": "image/sgi", - "sgif": "image/vnd.sealedmedia.softseal.gif", - "sgl": "application/vnd.stardivision.writer-global", - "sgm": "text/sgml", - "sgml": "text/sgml", - "sh": "application/x-sh", - "shar": "application/x-shar", - "shex": "text/shex", - "shf": "application/shf+xml", - "shp": "application/x-qgis", - "shx": "application/x-qgis", - "si": "text/vnd.wap.si", - "sic": "application/vnd.wap.sic", - "sid": "image/x-mrsid-image", - "sieve": "application/sieve", - "sig": "application/pgp-signature", - "sik": "application/x-trash", - "sil": "audio/silk", - "silo": "model/mesh", - "sis": "application/vnd.symbian.install", - "sisx": "x-epoc/x-sisx-app", - "sit": "application/x-stuffit", - "sitx": "application/x-stuffitx", - "siv": "application/sieve", - "sjp": "image/vnd.sealedmedia.softseal.jpg", - "sjpg": "image/vnd.sealedmedia.softseal.jpg", - "skd": "application/vnd.koan", - "skm": "application/vnd.koan", - "skp": "application/vnd.koan", - "skt": "application/vnd.koan", - "sl": "text/vnd.wap.sl", - "sla": "application/vnd.scribus", - "slaz": "application/vnd.scribus", - "slc": "application/vnd.wap.slc", - "sldm": "application/vnd.ms-powerpoint.slide.macroenabled.12", - "sldx": "application/vnd.openxmlformats-officedocument.presentationml.slide", - "sls": "application/route-s-tsid+xml", - "slt": "application/vnd.epson.salt", - "sm": "application/vnd.stepmania.stepchart", - "smc": "application/vnd.nintendo.snes.rom", + "sdc": "application/vnd.stardivision.calc", + "sda": "application/vnd.stardivision.draw", + "sdd": "application/vnd.stardivision.impress", "smf": "application/vnd.stardivision.math", - "smh": "application/vnd.sealed.mht", - "smht": "application/vnd.sealed.mht", - "smi": "application/smil+xml", - "smil": "application/smil+xml", - "smk": "video/vnd.radgamettools.smacker", - "sml": "application/smil+xml", - "smo": "video/vnd.sealedmedia.softseal.mov", - "smov": "video/vnd.sealedmedia.softseal.mov", - "smp": "audio/vnd.sealedmedia.softseal.mpeg", - "smp3": "audio/vnd.sealedmedia.softseal.mpeg", - "smpg": "video/vnd.sealed.mpeg1", - "sms": "application/vnd.3gpp2.sms", - "smv": "video/x-smv", + "sdw": "application/vnd.stardivision.writer", + "vor": "application/vnd.stardivision.writer", + "sgl": "application/vnd.stardivision.writer-global", "smzip": "application/vnd.stepmania.package", - "snd": "audio/basic", - "snf": "application/x-font-snf", - "so": "application/octet-stream", - "soa": "text/dns", - "soc": "application/sgml-open-catalog", - "sol": "application/solids", - "spc": "text/x-speech", - "spd": "application/vnd.sealedmedia.softseal.pdf", - "spdf": "application/vnd.sealedmedia.softseal.pdf", - "spec": "text/spec", - "spf": "application/vnd.yamaha.smaf-phrase", - "spl": "application/x-futuresplash", - "spn": "image/vnd.sealed.png", - "spng": "image/vnd.sealed.png", - "spo": "text/vnd.in3d.spot", - "spot": "text/vnd.in3d.spot", - "spp": "application/scvp-vp-response", - "sppt": "application/vnd.sealed.ppt", - "spq": "application/scvp-vp-request", - "spr": "application/x-sprite", - "sprite": "application/x-sprite", - "spx": "audio/ogg", - "sql": "application/x-sql", - "sr": "application/vnd.sigrok.session", - "src": "application/x-wais-source", - "srt": "application/x-subrip", - "sru": "application/sru+xml", - "srx": "application/sparql-results+xml", - "ssdl": "application/ssdl+xml", - "sse": "application/vnd.kodak-descriptor", - "ssf": "application/vnd.epson.ssf", - "ssi": "text/x-server-parsed-html", - "ssm": "application/streamingmedia", - "ssml": "application/ssml+xml", - "sst": "application/vnd.ms-pki.certstore", - "ssw": "video/vnd.sealed.swf", - "sswf": "video/vnd.sealed.swf", - "st": "application/vnd.sailingtracker.track", + "sm": "application/vnd.stepmania.stepchart", + "sxc": "application/vnd.sun.xml.calc", "stc": "application/vnd.sun.xml.calc.template", + "sxd": "application/vnd.sun.xml.draw", "std": "application/vnd.sun.xml.draw.template", - "step": "application/step", - "stf": "application/vnd.wt.stf", + "sxi": "application/vnd.sun.xml.impress", "sti": "application/vnd.sun.xml.impress.template", - "stif": "application/vnd.sealed.tiff", - "stk": "application/hyperstudio", - "stl": "application/vnd.ms-pki.stl", - "stm": "audio/x-stm", - "stml": "application/vnd.sealedmedia.softseal.html", - "stp": "application/step", - "str": "application/vnd.pg.format", - "study-inter": "application/vnd.vd-study", + "sxm": "application/vnd.sun.xml.math", + "sxw": "application/vnd.sun.xml.writer", + "sxg": "application/vnd.sun.xml.writer.global", "stw": "application/vnd.sun.xml.writer.template", - "sty": "text/x-tex", - "styl": "text/stylus", - "sub": "text/vnd.dvb.subtitle", "sus": "application/vnd.sus-calendar", "susp": "application/vnd.sus-calendar", - "sv4cpio": "application/x-sv4cpio", - "sv4crc": "application/x-sv4crc", - "svc": "application/vnd.dvb.service", "svd": "application/vnd.svd", - "svf": "image/x-dwg", - "svg": "image/svg+xml", - "svgz": "image/svg+xml", - "sw": "chemical/x-swissprot", - "swa": "application/x-director", - "swf": "application/x-shockwave-flash", - "swfl": "application/x-shockwave-flash", - "swi": "application/vnd.aristanetworks.swi", - "swift": "text/swift", - "swiftdeps": "text/swiftdeps", - "sxc": "application/vnd.sun.xml.calc", - "sxd": "application/vnd.sun.xml.draw", - "sxg": "application/vnd.sun.xml.writer.global", - "sxi": "application/vnd.sun.xml.impress", - "sxl": "application/vnd.sealed.xls", - "sxls": "application/vnd.sealed.xls", - "sxm": "application/vnd.sun.xml.math", - "sxw": "application/vnd.sun.xml.writer", - "t": "text/troff", - "t3": "application/x-t3vm-image", - "t38": "image/t38", - "tac": "text/twisted", - "tag": "text/prs.lines.tag", - "taglet": "application/vnd.mynfc", - "talk": "text/x-speech", - "tam": "application/vnd.onepager", - "tamp": "application/vnd.onepagertamp", - "tamx": "application/vnd.onepagertamx", + "sis": "application/vnd.symbian.install", + "sisx": "application/vnd.symbian.install", + "xsm": "application/vnd.syncml+xml", + "bdm": "application/vnd.syncml.dm+wbxml", + "xdm": "application/vnd.syncml.dm+xml", "tao": "application/vnd.tao.intent-module-archive", - "tap": "image/vnd.tencent.tap", - "tar": "application/x-tar", - "tat": "application/vnd.onepagertat", - "tatp": "application/vnd.onepagertatp", - "tatx": "application/vnd.onepagertatx", - "tau": "application/tamp-apex-update", - "taz": "application/x-gtar", - "tbk": "application/toolbook", - "tcap": "application/vnd.3gpp2.tcap", - "tcl": "application/x-tcl", - "tcsh": "text/x-script.tcsh", - "tcu": "application/tamp-community-update", - "td": "application/urc-targetdesc+xml", - "teacher": "application/vnd.smart.teacher", - "tei": "application/tei+xml", - "teicorpus": "application/tei+xml", - "ter": "application/tamp-error", - "tex": "application/x-tex", - "texi": "application/x-texinfo", - "texinfo": "application/x-texinfo", - "text": "text/plain", - "tf": "text/terraform", - "tfi": "application/thraud+xml", - "tfm": "application/x-tex-tfm", - "tfx": "image/tiff-fx", - "tga": "image/x-tga", - "tgf": "chemical/x-mdl-tgf", - "tgz": "application/gzip", - "thmx": "application/vnd.ms-officetheme", - "thrift": "text/thrift", - "tif": "image/tiff", - "tiff": "image/tiff", - "tk": "text/x-tcl", - "tlclient": "application/vnd.cendio.thinlinc.clientconf", - "tm": "text/texmacs", + "pcap": "application/vnd.tcpdump.pcap", + "cap": "application/vnd.tcpdump.pcap", + "dmp": "application/vnd.tcpdump.pcap", "tmo": "application/vnd.tmobile-livetv", - "tnef": "application/vnd.ms-tnef", - "tnf": "application/vnd.ms-tnef", - "toml": "text/toml", - "torrent": "application/x-bittorrent", - "tpl": "application/vnd.groove-tool-template", "tpt": "application/vnd.trid.tpt", - "tr": "text/troff", + "mxs": "application/vnd.triscape.mxs", "tra": "application/vnd.trueapp", - "tree": "application/vnd.rainstor.data", - "trig": "application/trig", - "trm": "application/x-msterminal", - "ts": "video/mp2t", - "tsa": "application/tamp-sequence-adjust", - "tscn": "text/godot", - "tsd": "application/timestamped-data", - "tsi": "audio/tsp-audio", - "tsp": "audio/tsplayer", - "tsq": "application/timestamp-query", - "tsr": "application/timestamp-reply", - "tst": "application/vnd.etsi.timestamp-token", - "tsv": "text/tab-separated-values", - "tsx": "text/tsx", - "ttc": "font/collection", - "ttf": "font/ttf", - "ttl": "text/turtle", - "ttml": "application/ttml+xml", - "tuc": "application/tamp-update-confirm", - "tur": "application/tamp-update", - "turbot": "image/florian", - "twd": "application/vnd.simtech-mindmapper", - "twds": "application/vnd.simtech-mindmapper", - "txd": "application/vnd.genomatix.tuxedo", - "txf": "application/vnd.mobius.txf", - "txt": "text/plain", - "u32": "application/x-authorware-bin", - "u8dsn": "message/global-delivery-status", - "u8hdr": "message/global-headers", - "u8mdn": "message/global-disposition-notification", - "u8msg": "message/global", - "udeb": "application/vnd.debian.binary-package", "ufd": "application/vnd.ufdl", "ufdl": "application/vnd.ufdl", - "uil": "text/x-uil", - "uis": "application/urc-uisocketdesc+xml", - "uls": "text/iuls", - "ult": "audio/x-mod", - "ulx": "application/x-glulx", + "utz": "application/vnd.uiq.theme", "umj": "application/vnd.umajin", - "uni": "audio/x-mod", - "unis": "text/uri-list", "unityweb": "application/vnd.unity", - "unv": "application/i-deas", - "uo": "application/vnd.uoml+xml", "uoml": "application/vnd.uoml+xml", - "upa": "application/vnd.hbci", - "uri": "text/uri-list", - "uric": "text/vnd.si.uricatalogue", - "urim": "application/vnd.uri-map", - "urimap": "application/vnd.uri-map", - "uris": "text/uri-list", - "urls": "text/uri-list", - "ustar": "application/x-ustar", - "utz": "application/vnd.uiq.theme", - "uu": "text/x-uuencode", - "uue": "text/x-uuencode", - "uva": "audio/vnd.dece.audio", - "uvd": "application/vnd.dece.data", - "uvf": "application/vnd.dece.data", - "uvg": "image/vnd.dece.graphic", - "uvh": "video/vnd.dece.hd", - "uvi": "image/vnd.dece.graphic", - "uvm": "video/vnd.dece.mobile", - "uvp": "video/vnd.dece.pd", - "uvs": "video/vnd.dece.sd", - "uvt": "application/vnd.dece.ttml+xml", - "uvu": "video/vnd.dece.mp4", - "uvv": "video/vnd.dece.video", - "uvva": "audio/vnd.dece.audio", - "uvvd": "application/vnd.dece.data", - "uvvf": "application/vnd.dece.data", - "uvvg": "image/vnd.dece.graphic", - "uvvh": "video/vnd.dece.hd", - "uvvi": "image/vnd.dece.graphic", - "uvvm": "video/vnd.dece.mobile", - "uvvp": "video/vnd.dece.pd", - "uvvs": "video/vnd.dece.sd", - "uvvt": "application/vnd.dece.ttml+xml", - "uvvu": "video/vnd.dece.mp4", - "uvvv": "video/vnd.dece.video", - "uvvx": "application/vnd.dece.unspecified", - "uvvz": "application/vnd.dece.zip", - "uvx": "application/vnd.dece.unspecified", - "uvz": "application/vnd.dece.zip", - "val": "chemical/x-ncbi-asn1-binary", - "vbk": "audio/vnd.nortel.vbk", - "vbox": "application/x-virtualbox-vbox", - "vbox-extpack": "application/x-virtualbox-vbox-extpack", - "vcard": "text/vcard", - "vcd": "application/x-cdlink", - "vcf": "text/x-vcard", - "vcg": "application/vnd.groove-vcard", - "vcs": "text/x-vcalendar", "vcx": "application/vnd.vcx", - "vda": "application/vda", - "vdi": "application/x-virtualbox-vdi", - "vdo": "video/vdo", - "vdx": "text/vdx", - "vew": "application/vnd.lotus-approach", - "vfr": "application/vnd.tml", - "vhd": "application/x-virtualbox-vhd", - "viaframe": "application/vnd.tml", - "vim": "text/vim", - "vis": "application/vnd.visionary", - "viv": "video/vnd.vivo", - "vivo": "video/vivo", - "vmd": "application/vocaltec-media-desc", - "vmdk": "application/x-virtualbox-vmdk", - "vmf": "application/vocaltec-media-file", - "vms": "chemical/x-vamas-iso14976", - "vmt": "application/vnd.valve.source.material", - "vob": "video/x-ms-vob", - "voc": "audio/voc", - "vor": "application/vnd.stardivision.writer", - "vos": "video/vosaic", - "vox": "audio/voxware", - "vpm": "multipart/voice-message", - "vqe": "audio/x-twinvq-plugin", - "vqf": "audio/x-twinvq", - "vql": "audio/x-twinvq-plugin", - "vrm": "x-world/x-vrml", - "vrml": "model/vrml", - "vrt": "x-world/x-vrt", - "vsc": "application/vnd.vidsoft.vidconference", "vsd": "application/vnd.visio", - "vsf": "application/vnd.vsf", - "vss": "application/vnd.visio", "vst": "application/vnd.visio", + "vss": "application/vnd.visio", "vsw": "application/vnd.visio", - "vtf": "image/vnd.valve.source.texture", - "vtt": "text/vtt", - "vtu": "model/vnd.vtu", - "vue": "text/vue", - "vwx": "application/vnd.vectorworks", - "vxml": "application/voicexml+xml", - "w3d": "application/x-director", - "w60": "application/wordperfect6.0", - "w61": "application/wordperfect6.1", - "w6w": "application/msword", - "wad": "application/x-doom", - "wadl": "application/vnd.sun.wadl+xml", - "war": "binary/zip", - "wasm": "application/wasm", - "wav": "audio/wave", - "wax": "audio/x-ms-wax", - "wb1": "application/x-qpro", - "wbmp": "image/vnd.wap.wbmp", - "wbs": "application/vnd.criticaltools.wbs+xml", + "vis": "application/vnd.visionary", + "vsf": "application/vnd.vsf", "wbxml": "application/vnd.wap.wbxml", - "wcm": "application/vnd.ms-works", - "wdb": "application/vnd.ms-works", - "wdp": "image/vnd.ms-photo", - "web": "application/vnd.xara", - "weba": "audio/webm", - "webapp": "application/x-web-app-manifest+json", - "webm": "video/webm", - "webmanifest": "application/manifest+json", - "webp": "image/webp", - "wg": "application/vnd.pmi.widget", - "wgt": "application/widget", - "whl": "binary/wheel", - "wif": "application/watcherinfo+xml", - "win": "model/vnd.gdl", - "wiz": "application/msword", - "wk": "application/x-123", - "wk1": "application/vnd.lotus-1-2-3", - "wk3": "application/vnd.lotus-1-2-3", - "wk4": "application/vnd.lotus-1-2-3", - "wks": "application/vnd.ms-works", - "wkt": "text/wkt", - "wlnk": "application/link-format", - "wm": "video/x-ms-wm", - "wma": "audio/x-ms-wma", - "wmc": "application/vnd.wmc", - "wmd": "application/x-ms-wmd", - "wmf": "image/wmf", - "wml": "text/vnd.wap.wml", "wmlc": "application/vnd.wap.wmlc", - "wmls": "text/vnd.wap.wmlscript", "wmlsc": "application/vnd.wap.wmlscriptc", - "wmv": "video/x-ms-wmv", - "wmx": "video/x-ms-wmx", - "wmz": "application/x-ms-wmz", - "woff": "font/woff", - "woff2": "font/woff2", - "word": "application/msword", - "wp": "application/wordperfect", - "wp5": "application/wordperfect", - "wp6": "application/wordperfect", + "wtb": "application/vnd.webturbo", + "nbp": "application/vnd.wolfram.player", "wpd": "application/vnd.wordperfect", - "wpl": "application/vnd.ms-wpl", - "wps": "application/vnd.ms-works", - "wq1": "application/x-lotus", "wqd": "application/vnd.wqd", - "wri": "application/x-mswrite", - "wrl": "model/vrml", - "wrz": "model/vrml", - "wsc": "message/vnd.wfa.wsc", + "stf": "application/vnd.wt.stf", + "xar": "application/vnd.xara", + "xfdl": "application/vnd.xfdl", + "hvd": "application/vnd.yamaha.hv-dic", + "hvs": "application/vnd.yamaha.hv-script", + "hvp": "application/vnd.yamaha.hv-voice", + "osf": "application/vnd.yamaha.openscoreformat", + "osfpvg": "application/vnd.yamaha.openscoreformat.osfpvg+xml", + "saf": "application/vnd.yamaha.smaf-audio", + "spf": "application/vnd.yamaha.smaf-phrase", + "cmp": "application/vnd.yellowriver-custom-menu", + "zir": "application/vnd.zul", + "zirz": "application/vnd.zul", + "zaz": "application/vnd.zzazz.deck+xml", + "vxml": "application/voicexml+xml", + "wasm": "application/wasm", + "wgt": "application/widget", + "hlp": "application/winhlp", "wsdl": "application/wsdl+xml", - "wsgi": "text/wsgi", "wspolicy": "application/wspolicy+xml", - "wsrc": "application/x-wais-source", - "wtb": "application/vnd.webturbo", - "wtk": "application/x-wintalk", - "wv": "application/vnd.wv.csp+wbxml", - "wvx": "video/x-ms-wvx", - "wz": "application/x-wingz", - "x-png": "image/png", + "7z": "application/x-7z-compressed", + "abw": "application/x-abiword", + "ace": "application/x-ace-compressed", + "dmg": "application/x-apple-diskimage", + "aab": "application/x-authorware-bin", "x32": "application/x-authorware-bin", - "x3d": "application/vnd.hzn-3d-crossword", - "x3db": "model/x3d+xml", - "x3dbz": "model/x3d+binary", - "x3dv": "model/x3d-vrml", - "x3dvz": "model/x3d-vrml", - "x3dz": "model/x3d+xml", - "x_b": "model/vnd.parasolid.transmit.binary", - "x_t": "model/vnd.parasolid.transmit.text", - "xaf": "x-world/x-vrml", - "xaml": "application/xaml+xml", - "xap": "application/x-silverlight-app", - "xar": "application/vnd.xara", - "xav": "application/xcap-att+xml", + "u32": "application/x-authorware-bin", + "vox": "application/x-authorware-bin", + "aam": "application/x-authorware-map", + "aas": "application/x-authorware-seg", + "bcpio": "application/x-bcpio", + "torrent": "application/x-bittorrent", + "blb": "application/x-blorb", + "blorb": "application/x-blorb", + "bz": "application/x-bzip", + "bz2": "application/x-bzip2", + "boz": "application/x-bzip2", + "cbr": "application/x-cbr", + "cba": "application/x-cbr", + "cbt": "application/x-cbr", + "cbz": "application/x-cbr", + "cb7": "application/x-cbr", + "vcd": "application/x-cdlink", + "cfs": "application/x-cfs-compressed", + "chat": "application/x-chat", + "pgn": "application/x-chess-pgn", + "nsc": "application/x-conference", + "cpio": "application/x-cpio", + "csh": "application/x-csh", + "deb": "application/x-debian-package", + "udeb": "application/x-debian-package", + "dgc": "application/x-dgc-compressed", + "dir": "application/x-director", + "dcr": "application/x-director", + "dxr": "application/x-director", + "cst": "application/x-director", + "cct": "application/x-director", + "cxt": "application/x-director", + "w3d": "application/x-director", + "fgd": "application/x-director", + "swa": "application/x-director", + "wad": "application/x-doom", + "ncx": "application/x-dtbncx+xml", + "dtb": "application/x-dtbook+xml", + "res": "application/x-dtbresource+xml", + "dvi": "application/x-dvi", + "evy": "application/x-envoy", + "eva": "application/x-eva", + "bdf": "application/x-font-bdf", + "gsf": "application/x-font-ghostscript", + "psf": "application/x-font-linux-psf", + "pcf": "application/x-font-pcf", + "snf": "application/x-font-snf", + "pfa": "application/x-font-type1", + "pfb": "application/x-font-type1", + "pfm": "application/x-font-type1", + "afm": "application/x-font-type1", + "arc": "application/x-freearc", + "spl": "application/x-futuresplash", + "gca": "application/x-gca-compressed", + "ulx": "application/x-glulx", + "gnumeric": "application/x-gnumeric", + "gramps": "application/x-gramps-xml", + "gtar": "application/x-gtar", + "hdf": "application/x-hdf", + "install": "application/x-install-instructions", + "iso": "application/x-iso9660-image", + "jnlp": "application/x-java-jnlp-file", + "latex": "application/x-latex", + "lzh": "application/x-lzh-compressed", + "lha": "application/x-lzh-compressed", + "mie": "application/x-mie", + "prc": "application/x-mobipocket-ebook", + "mobi": "application/x-mobipocket-ebook", + "application": "application/x-ms-application", + "lnk": "application/x-ms-shortcut", + "wmd": "application/x-ms-wmd", + "wmz": "application/x-ms-wmz", "xbap": "application/x-ms-xbap", - "xbd": "application/vnd.fujixerox.docuworks.binder", - "xbm": "image/x-xbitmap", - "xca": "application/xcap-caps+xml", - "xcf": "application/x-xcf", - "xcs": "application/calendar+xml", - "xct": "application/vnd.fujixerox.docuworks.container", - "xdd": "application/bacnet-xdd+zip", - "xdf": "application/xcap-diff+xml", - "xdm": "application/vnd.syncml.dm+xml", - "xdp": "application/vnd.adobe.xdp+xml", - "xdr": "video/x-amt-demorun", - "xdssc": "application/dssc+xml", - "xdw": "application/vnd.fujixerox.docuworks", - "xel": "application/xcap-el+xml", - "xenc": "application/xenc+xml", - "xer": "application/patch-ops-error+xml", - "xfd": "application/vnd.xfdl", - "xfdf": "application/vnd.adobe.xfdf", - "xfdl": "application/vnd.xfdl", - "xgz": "xgl/drawing", - "xht": "application/xhtml+xml", - "xhtm": "application/xhtml+xml", - "xhtml": "application/xhtml+xml", - "xhvml": "application/xv+xml", - "xif": "image/vnd.xiff", - "xl": "application/excel", - "xla": "application/vnd.ms-excel", - "xlam": "application/vnd.ms-excel.addin.macroenabled.12", - "xlb": "application/vndms-excel", - "xlc": "application/vnd.ms-excel", + "mdb": "application/x-msaccess", + "obd": "application/x-msbinder", + "crd": "application/x-mscardfile", + "clp": "application/x-msclip", + "exe": "application/x-msdownload", + "dll": "application/x-msdownload", + "com": "application/x-msdownload", + "bat": "application/x-msdownload", + "msi": "application/x-msdownload", + "mvb": "application/x-msmediaview", + "m13": "application/x-msmediaview", + "m14": "application/x-msmediaview", + "wmf": "application/x-msmetafile", + "wmz": "application/x-msmetafile", + "emf": "application/x-msmetafile", + "emz": "application/x-msmetafile", + "mny": "application/x-msmoney", + "pub": "application/x-mspublisher", + "scd": "application/x-msschedule", + "trm": "application/x-msterminal", + "wri": "application/x-mswrite", + "nc": "application/x-netcdf", + "cdf": "application/x-netcdf", + "nzb": "application/x-nzb", + "p12": "application/x-pkcs12", + "pfx": "application/x-pkcs12", + "p7b": "application/x-pkcs7-certificates", + "spc": "application/x-pkcs7-certificates", + "p7r": "application/x-pkcs7-certreqresp", + "rar": "application/x-rar-compressed", + "ris": "application/x-research-info-systems", + "sh": "application/x-sh", + "shar": "application/x-shar", + "swf": "application/x-shockwave-flash", + "xap": "application/x-silverlight-app", + "sql": "application/x-sql", + "sit": "application/x-stuffit", + "sitx": "application/x-stuffitx", + "srt": "application/x-subrip", + "sv4cpio": "application/x-sv4cpio", + "sv4crc": "application/x-sv4crc", + "t3": "application/x-t3vm-image", + "gam": "application/x-tads", + "tar": "application/x-tar", + "tcl": "application/x-tcl", + "tex": "application/x-tex", + "tfm": "application/x-tex-tfm", + "texinfo": "application/x-texinfo", + "texi": "application/x-texinfo", + "obj": "application/x-tgif", + "ustar": "application/x-ustar", + "src": "application/x-wais-source", + "der": "application/x-x509-ca-cert", + "crt": "application/x-x509-ca-cert", + "fig": "application/x-xfig", "xlf": "application/x-xliff+xml", - "xlim": "application/vnd.xmpie.xlim", - "xlm": "application/vnd.ms-excel", - "xls": "application/vnd.ms-excel", - "xlsb": "application/vnd.ms-excel.sheet.binary.macroenabled.12", - "xlsm": "application/vnd.ms-excel.sheet.macroenabled.12", - "xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", - "xlt": "application/vnd.ms-excel", - "xltm": "application/vnd.ms-excel.template.macroenabled.12", - "xltx": "application/vnd.openxmlformats-officedocument.spreadsheetml.template", - "xlw": "application/vnd.ms-excel", - "xm": "audio/xm", - "xml": "text/xml", - "xmls": "application/dskpp+xml", - "xmt_bin": "model/vnd.parasolid.transmit.binary", - "xmt_txt": "model/vnd.parasolid.transmit.text", - "xmz": "xgl/movie", - "xns": "application/xcap-ns+xml", - "xo": "application/vnd.olpc-sugar", - "xof": "x-world/x-vrml", - "xop": "application/xop+xml", - "xpdl": "application/xml", "xpi": "application/x-xpinstall", - "xpix": "application/x-vnd.ls-xpix", - "xpl": "application/xproc+xml", - "xpm": "image/x-xpixmap", - "xpr": "application/vnd.is-xpr", - "xps": "application/vnd.ms-xpsdocument", - "xpw": "application/vnd.intercon.formnet", - "xpx": "application/vnd.intercon.formnet", - "xq": "text/xquery", - "xql": "text/xquery", - "xqm": "text/xquery", - "xqu": "text/xquery", - "xquery": "text/xquery", - "xqy": "text/xquery", - "xsd": "text/xml", - "xsf": "application/prs.xsf+xml", - "xsl": "application/xslt+xml", - "xslt": "application/xslt+xml", - "xsm": "application/vnd.syncml+xml", - "xspf": "application/xspf+xml", - "xsr": "video/x-amt-showrun", - "xtel": "chemical/x-xtel", - "xul": "application/vnd.mozilla.xul+xml", - "xvm": "application/xv+xml", - "xvml": "application/xv+xml", - "xwd": "image/x-xwindowdump", - "xyz": "chemical/x-xyz", - "xyze": "image/vnd.radiance", "xz": "application/x-xz", - "yaml": "text/yaml", - "yang": "application/yang", - "yin": "application/yin+xml", - "yme": "application/vnd.yaoweme", - "yml": "text/yaml", - "ymp": "text/x-suse-ymp", "z1": "application/x-zmachine", "z2": "application/x-zmachine", "z3": "application/x-zmachine", @@ -1888,19 +744,293 @@ const mimes* = { "z6": "application/x-zmachine", "z7": "application/x-zmachine", "z8": "application/x-zmachine", - "zaz": "application/vnd.zzazz.deck+xml", - "zfc": "application/vnd.filmit.zfc", - "zfo": "application/vnd.software602.filler.form-xml-zip", - "zig": "text/zig", + "xaml": "application/xaml+xml", + "xdf": "application/xcap-diff+xml", + "xenc": "application/xenc+xml", + "xhtml": "application/xhtml+xml", + "xht": "application/xhtml+xml", + "xml": "application/xml", + "xsl": "application/xml", + "dtd": "application/xml-dtd", + "xop": "application/xop+xml", + "xpl": "application/xproc+xml", + "xslt": "application/xslt+xml", + "xspf": "application/xspf+xml", + "mxml": "application/xv+xml", + "xhvml": "application/xv+xml", + "xvml": "application/xv+xml", + "xvm": "application/xv+xml", + "yang": "application/yang", + "yin": "application/yin+xml", "zip": "application/zip", - "zir": "application/vnd.zul", - "zirz": "application/vnd.zul", - "zmm": "application/vnd.handheld-entertainment+xml", - "zmt": "chemical/x-mopac-input", - "zone": "text/dns", - "zoo": "application/octet-stream", - "zsh": "text/x-script.zsh", - "~": "application/x-trash" + "adp": "audio/adpcm", + "au": "audio/basic", + "snd": "audio/basic", + "mid": "audio/midi", + "midi": "audio/midi", + "kar": "audio/midi", + "rmi": "audio/midi", + "m4a": "audio/mp4", + "mp4a": "audio/mp4", + "mpga": "audio/mpeg", + "mp2": "audio/mpeg", + "mp2a": "audio/mpeg", + "mp3": "audio/mpeg", + "m2a": "audio/mpeg", + "m3a": "audio/mpeg", + "oga": "audio/ogg", + "ogg": "audio/ogg", + "spx": "audio/ogg", + "opus": "audio/ogg", + "s3m": "audio/s3m", + "sil": "audio/silk", + "uva": "audio/vnd.dece.audio", + "uvva": "audio/vnd.dece.audio", + "eol": "audio/vnd.digital-winds", + "dra": "audio/vnd.dra", + "dts": "audio/vnd.dts", + "dtshd": "audio/vnd.dts.hd", + "lvp": "audio/vnd.lucent.voice", + "pya": "audio/vnd.ms-playready.media.pya", + "ecelp4800": "audio/vnd.nuera.ecelp4800", + "ecelp7470": "audio/vnd.nuera.ecelp7470", + "ecelp9600": "audio/vnd.nuera.ecelp9600", + "rip": "audio/vnd.rip", + "weba": "audio/webm", + "aac": "audio/x-aac", + "aif": "audio/x-aiff", + "aiff": "audio/x-aiff", + "aifc": "audio/x-aiff", + "caf": "audio/x-caf", + "flac": "audio/x-flac", + "mka": "audio/x-matroska", + "m3u": "audio/x-mpegurl", + "wax": "audio/x-ms-wax", + "wma": "audio/x-ms-wma", + "ram": "audio/x-pn-realaudio", + "ra": "audio/x-pn-realaudio", + "rmp": "audio/x-pn-realaudio-plugin", + "wav": "audio/x-wav", + "xm": "audio/xm", + "cdx": "chemical/x-cdx", + "cif": "chemical/x-cif", + "cmdf": "chemical/x-cmdf", + "cml": "chemical/x-cml", + "csml": "chemical/x-csml", + "xyz": "chemical/x-xyz", + "ttc": "font/collection", + "otf": "font/otf", + "ttf": "font/ttf", + "woff": "font/woff", + "woff2": "font/woff2", + "bmp": "image/bmp", + "cgm": "image/cgm", + "g3": "image/g3fax", + "gif": "image/gif", + "ief": "image/ief", + "jpeg": "image/jpeg", + "jpg": "image/jpeg", + "jpe": "image/jpeg", + "ktx": "image/ktx", + "png": "image/png", + "btif": "image/prs.btif", + "sgi": "image/sgi", + "svg": "image/svg+xml", + "svgz": "image/svg+xml", + "tiff": "image/tiff", + "tif": "image/tiff", + "psd": "image/vnd.adobe.photoshop", + "uvi": "image/vnd.dece.graphic", + "uvvi": "image/vnd.dece.graphic", + "uvg": "image/vnd.dece.graphic", + "uvvg": "image/vnd.dece.graphic", + "djvu": "image/vnd.djvu", + "djv": "image/vnd.djvu", + "sub": "image/vnd.dvb.subtitle", + "dwg": "image/vnd.dwg", + "dxf": "image/vnd.dxf", + "fbs": "image/vnd.fastbidsheet", + "fpx": "image/vnd.fpx", + "fst": "image/vnd.fst", + "mmr": "image/vnd.fujixerox.edmics-mmr", + "rlc": "image/vnd.fujixerox.edmics-rlc", + "mdi": "image/vnd.ms-modi", + "wdp": "image/vnd.ms-photo", + "npx": "image/vnd.net-fpx", + "wbmp": "image/vnd.wap.wbmp", + "xif": "image/vnd.xiff", + "webp": "image/webp", + "3ds": "image/x-3ds", + "ras": "image/x-cmu-raster", + "cmx": "image/x-cmx", + "fh": "image/x-freehand", + "fhc": "image/x-freehand", + "fh4": "image/x-freehand", + "fh5": "image/x-freehand", + "fh7": "image/x-freehand", + "ico": "image/x-icon", + "sid": "image/x-mrsid-image", + "pcx": "image/x-pcx", + "pic": "image/x-pict", + "pct": "image/x-pict", + "pnm": "image/x-portable-anymap", + "pbm": "image/x-portable-bitmap", + "pgm": "image/x-portable-graymap", + "ppm": "image/x-portable-pixmap", + "rgb": "image/x-rgb", + "tga": "image/x-tga", + "xbm": "image/x-xbitmap", + "xpm": "image/x-xpixmap", + "xwd": "image/x-xwindowdump", + "eml": "message/rfc822", + "mime": "message/rfc822", + "igs": "model/iges", + "iges": "model/iges", + "msh": "model/mesh", + "mesh": "model/mesh", + "silo": "model/mesh", + "dae": "model/vnd.collada+xml", + "dwf": "model/vnd.dwf", + "gdl": "model/vnd.gdl", + "gtw": "model/vnd.gtw", + "mts": "model/vnd.mts", + "vtu": "model/vnd.vtu", + "wrl": "model/vrml", + "vrml": "model/vrml", + "x3db": "model/x3d+binary", + "x3dbz": "model/x3d+binary", + "x3dv": "model/x3d+vrml", + "x3dvz": "model/x3d+vrml", + "x3d": "model/x3d+xml", + "x3dz": "model/x3d+xml", + "appcache": "text/cache-manifest", + "ics": "text/calendar", + "ifb": "text/calendar", + "css": "text/css", + "csv": "text/csv", + "html": "text/html", + "htm": "text/html", + "js": "text/javascript", + "mjs": "text/javascript", + "n3": "text/n3", + "txt": "text/plain", + "text": "text/plain", + "conf": "text/plain", + "def": "text/plain", + "list": "text/plain", + "log": "text/plain", + "in": "text/plain", + "dsc": "text/prs.lines.tag", + "rtx": "text/richtext", + "sgml": "text/sgml", + "sgm": "text/sgml", + "tsv": "text/tab-separated-values", + "t": "text/troff", + "tr": "text/troff", + "roff": "text/troff", + "man": "text/troff", + "me": "text/troff", + "ms": "text/troff", + "ttl": "text/turtle", + "uri": "text/uri-list", + "uris": "text/uri-list", + "urls": "text/uri-list", + "vcard": "text/vcard", + "curl": "text/vnd.curl", + "dcurl": "text/vnd.curl.dcurl", + "mcurl": "text/vnd.curl.mcurl", + "scurl": "text/vnd.curl.scurl", + "sub": "text/vnd.dvb.subtitle", + "fly": "text/vnd.fly", + "flx": "text/vnd.fmi.flexstor", + "gv": "text/vnd.graphviz", + "3dml": "text/vnd.in3d.3dml", + "spot": "text/vnd.in3d.spot", + "jad": "text/vnd.sun.j2me.app-descriptor", + "wml": "text/vnd.wap.wml", + "wmls": "text/vnd.wap.wmlscript", + "s": "text/x-asm", + "asm": "text/x-asm", + "c": "text/x-c", + "cc": "text/x-c", + "cxx": "text/x-c", + "cpp": "text/x-c", + "h": "text/x-c", + "hh": "text/x-c", + "dic": "text/x-c", + "f": "text/x-fortran", + "for": "text/x-fortran", + "f77": "text/x-fortran", + "f90": "text/x-fortran", + "java": "text/x-java-source", + "nfo": "text/x-nfo", + "opml": "text/x-opml", + "p": "text/x-pascal", + "pas": "text/x-pascal", + "etx": "text/x-setext", + "sfv": "text/x-sfv", + "uu": "text/x-uuencode", + "vcs": "text/x-vcalendar", + "vcf": "text/x-vcard", + "3gp": "video/3gpp", + "3g2": "video/3gpp2", + "h261": "video/h261", + "h263": "video/h263", + "h264": "video/h264", + "jpgv": "video/jpeg", + "jpm": "video/jpm", + "jpgm": "video/jpm", + "mj2": "video/mj2", + "mjp2": "video/mj2", + "mp4": "video/mp4", + "mp4v": "video/mp4", + "mpg4": "video/mp4", + "mpeg": "video/mpeg", + "mpg": "video/mpeg", + "mpe": "video/mpeg", + "m1v": "video/mpeg", + "m2v": "video/mpeg", + "ogv": "video/ogg", + "qt": "video/quicktime", + "mov": "video/quicktime", + "uvh": "video/vnd.dece.hd", + "uvvh": "video/vnd.dece.hd", + "uvm": "video/vnd.dece.mobile", + "uvvm": "video/vnd.dece.mobile", + "uvp": "video/vnd.dece.pd", + "uvvp": "video/vnd.dece.pd", + "uvs": "video/vnd.dece.sd", + "uvvs": "video/vnd.dece.sd", + "uvv": "video/vnd.dece.video", + "uvvv": "video/vnd.dece.video", + "dvb": "video/vnd.dvb.file", + "fvt": "video/vnd.fvt", + "mxu": "video/vnd.mpegurl", + "m4u": "video/vnd.mpegurl", + "pyv": "video/vnd.ms-playready.media.pyv", + "uvu": "video/vnd.uvvu.mp4", + "uvvu": "video/vnd.uvvu.mp4", + "viv": "video/vnd.vivo", + "webm": "video/webm", + "f4v": "video/x-f4v", + "fli": "video/x-fli", + "flv": "video/x-flv", + "m4v": "video/x-m4v", + "mkv": "video/x-matroska", + "mk3d": "video/x-matroska", + "mks": "video/x-matroska", + "mng": "video/x-mng", + "asf": "video/x-ms-asf", + "asx": "video/x-ms-asf", + "vob": "video/x-ms-vob", + "wm": "video/x-ms-wm", + "wmv": "video/x-ms-wmv", + "wmx": "video/x-ms-wmx", + "wvx": "video/x-ms-wvx", + "avi": "video/x-msvideo", + "movie": "video/x-sgi-movie", + "smv": "video/x-smv", + "ice": "x-conference/x-cooltalk", } diff --git a/lib/pure/nativesockets.nim b/lib/pure/nativesockets.nim index dcba996a4..656c98a20 100644 --- a/lib/pure/nativesockets.nim +++ b/lib/pure/nativesockets.nim @@ -12,7 +12,7 @@ # TODO: Clean up the exports a bit and everything else in general. -import os, options +import std/[os, options] import std/private/since import std/strbasics @@ -23,15 +23,16 @@ when hostOS == "solaris": {.passl: "-lsocket -lnsl".} const useWinVersion = defined(windows) or defined(nimdoc) -const useNimNetLite = defined(nimNetLite) or defined(freertos) or defined(zephyr) +const useNimNetLite = defined(nimNetLite) or defined(freertos) or defined(zephyr) or + defined(nuttx) when useWinVersion: - import winlean + import std/winlean export WSAEWOULDBLOCK, WSAECONNRESET, WSAECONNABORTED, WSAENETRESET, WSANOTINITIALISED, WSAENOTSOCK, WSAEINPROGRESS, WSAEINTR, WSAEDISCON, ERROR_NETNAME_DELETED else: - import posix + import std/posix export fcntl, F_GETFL, O_NONBLOCK, F_SETFL, EAGAIN, EWOULDBLOCK, MSG_NOSIGNAL, EINTR, EINPROGRESS, ECONNRESET, EPIPE, ENETRESET, EBADF export Sockaddr_storage, Sockaddr_un, Sockaddr_un_path_length @@ -66,7 +67,7 @@ type ## some procedures, such as getaddrinfo) AF_UNIX = 1, ## for local socket (using a file). Unsupported on Windows. AF_INET = 2, ## for network protocol IPv4 or - AF_INET6 = when defined(macosx): 30 else: 23 ## for network protocol IPv6. + AF_INET6 = when defined(macosx): 30 elif defined(windows): 23 else: 10 ## for network protocol IPv6. SockType* = enum ## second argument to `socket` proc SOCK_STREAM = 1, ## reliable stream-oriented service or Stream Sockets @@ -96,6 +97,8 @@ type length*: int addrList*: seq[string] +const IPPROTO_NONE* = IPPROTO_IP ## Use this if your socket type requires a protocol value of zero (e.g. Unix sockets). + when useWinVersion: let osInvalidSocket* = winlean.INVALID_SOCKET @@ -169,7 +172,7 @@ when not useWinVersion: else: proc toInt(domain: Domain): cint = - result = toU32(ord(domain)).cint + result = cast[cint](uint32(ord(domain))) proc toKnownDomain*(family: cint): Option[Domain] = ## Converts the platform-dependent `cint` to the Domain or none(), @@ -304,7 +307,7 @@ proc getAddrInfo*(address: string, port: Port, domain: Domain = AF_INET, let socketPort = if sockType == SOCK_RAW: "" else: $port var gaiResult = getaddrinfo(address, socketPort.cstring, addr(hints), result) if gaiResult != 0'i32: - when useWinVersion or defined(freertos): + when useWinVersion or defined(freertos) or defined(nuttx): raiseOSError(osLastError()) else: raiseOSError(osLastError(), $gai_strerror(gaiResult)) @@ -351,7 +354,7 @@ proc getSockDomain*(socket: SocketHandle): Domain = else: raise newException(IOError, "Unknown socket family in getSockDomain") -when not useNimNetLite: +when not useNimNetLite: proc getServByName*(name, proto: string): Servent {.tags: [ReadIOEffect].} = ## Searches the database from the beginning and finds the first entry for ## which the service name specified by `name` matches the s_name member @@ -375,9 +378,9 @@ when not useNimNetLite: ## ## On posix this will search through the `/etc/services` file. when useWinVersion: - var s = winlean.getservbyport(ze(int16(port)).cint, proto) + var s = winlean.getservbyport(uint16(port).cint, proto) else: - var s = posix.getservbyport(ze(int16(port)).cint, proto) + var s = posix.getservbyport(uint16(port).cint, proto) if s == nil: raiseOSError(osLastError(), "Service not found.") result.name = $s.s_name result.aliases = cstringArrayToSeq(s.s_aliases) @@ -386,21 +389,37 @@ when not useNimNetLite: proc getHostByAddr*(ip: string): Hostent {.tags: [ReadIOEffect].} = ## This function will lookup the hostname of an IP Address. - var myaddr: InAddr - myaddr.s_addr = inet_addr(ip) + var + addrInfo = getAddrInfo(ip, Port(0), AF_UNSPEC) + myAddr: pointer + addrLen = 0 + family = 0 + + defer: freeAddrInfo(addrInfo) + + if addrInfo.ai_addr.sa_family.cint == nativeAfInet: + family = nativeAfInet + myAddr = addr cast[ptr Sockaddr_in](addrInfo.ai_addr).sin_addr + addrLen = 4 + elif addrInfo.ai_addr.sa_family.cint == nativeAfInet6: + family = nativeAfInet6 + myAddr = addr cast[ptr Sockaddr_in6](addrInfo.ai_addr).sin6_addr + addrLen = 16 + else: + raise newException(IOError, "Unknown socket family in `getHostByAddr()`") when useWinVersion: - var s = winlean.gethostbyaddr(addr(myaddr), sizeof(myaddr).cuint, - cint(AF_INET)) + var s = winlean.gethostbyaddr(cast[ptr InAddr](myAddr), addrLen.cuint, + cint(family)) if s == nil: raiseOSError(osLastError()) else: var s = when defined(android4): - posix.gethostbyaddr(cast[cstring](addr(myaddr)), sizeof(myaddr).cint, - cint(posix.AF_INET)) + posix.gethostbyaddr(cast[cstring](myAddr), addrLen.cint, + cint(family)) else: - posix.gethostbyaddr(addr(myaddr), sizeof(myaddr).SockLen, - cint(posix.AF_INET)) + posix.gethostbyaddr(myAddr, addrLen.SockLen, + cint(family)) if s == nil: raiseOSError(osLastError(), $hstrerror(h_errno)) @@ -423,7 +442,20 @@ when not useNimNetLite: result.addrList.add($inet_ntoa(inaddrPtr[])) inc(i) else: - result.addrList = cstringArrayToSeq(s.h_addr_list) + let strAddrLen = when not useWinVersion: posix.INET6_ADDRSTRLEN.int + else: 46 + var i = 0 + while not isNil(s.h_addr_list[i]): + var ipStr = newString(strAddrLen) + if inet_ntop(nativeAfInet6, cast[pointer](s.h_addr_list[i]), + cstring(ipStr), len(ipStr).int32) == nil: + raiseOSError(osLastError()) + when not useWinVersion: + if posix.IN6_IS_ADDR_V4MAPPED(cast[ptr In6Addr](s.h_addr_list[i])) != 0: + ipStr.setSlice("::ffff:".len..<strAddrLen) + setLen(ipStr, len(cstring(ipStr))) + result.addrList.add(ipStr) + inc(i) result.length = int(s.h_length) proc getHostByName*(name: string): Hostent {.tags: [ReadIOEffect].} = @@ -481,13 +513,13 @@ when not useNimNetLite: result = newString(addrLen) let addr6 = addr cast[ptr Sockaddr_in6](sockAddr).sin6_addr when not useWinVersion: - if posix.inet_ntop(posix.AF_INET6, addr6, addr result[0], + if posix.inet_ntop(posix.AF_INET6, addr6, cast[cstring](addr result[0]), result.len.int32) == nil: raiseOSError(osLastError()) if posix.IN6_IS_ADDR_V4MAPPED(addr6) != 0: result.setSlice("::ffff:".len..<addrLen) else: - if winlean.inet_ntop(winlean.AF_INET6, addr6, addr result[0], + if winlean.inet_ntop(winlean.AF_INET6, addr6, cast[cstring](addr result[0]), result.len.int32) == nil: raiseOSError(osLastError()) setLen(result, len(cstring(result))) @@ -509,23 +541,23 @@ when not useNimNetLite: if sockAddr.sa_family.cint == nativeAfInet: let addr4 = addr cast[ptr Sockaddr_in](sockAddr).sin_addr when not useWinVersion: - if posix.inet_ntop(posix.AF_INET, addr4, addr strAddress[0], + if posix.inet_ntop(posix.AF_INET, addr4, cast[cstring](addr strAddress[0]), strAddress.len.int32) == nil: raiseOSError(osLastError()) else: - if winlean.inet_ntop(winlean.AF_INET, addr4, addr strAddress[0], + if winlean.inet_ntop(winlean.AF_INET, addr4, cast[cstring](addr strAddress[0]), strAddress.len.int32) == nil: raiseOSError(osLastError()) elif sockAddr.sa_family.cint == nativeAfInet6: let addr6 = addr cast[ptr Sockaddr_in6](sockAddr).sin6_addr when not useWinVersion: - if posix.inet_ntop(posix.AF_INET6, addr6, addr strAddress[0], + if posix.inet_ntop(posix.AF_INET6, addr6, cast[cstring](addr strAddress[0]), strAddress.len.int32) == nil: raiseOSError(osLastError()) if posix.IN6_IS_ADDR_V4MAPPED(addr6) != 0: strAddress.setSlice("::ffff:".len..<length) else: - if winlean.inet_ntop(winlean.AF_INET6, addr6, addr strAddress[0], + if winlean.inet_ntop(winlean.AF_INET6, addr6, cast[cstring](addr strAddress[0]), strAddress.len.int32) == nil: raiseOSError(osLastError()) else: @@ -584,7 +616,7 @@ when not useNimNetLite: # Cannot use INET6_ADDRSTRLEN here, because it's a C define. result[0] = newString(64) if inet_ntop(name.sin6_family.cint, - addr name.sin6_addr, addr result[0][0], (result[0].len+1).int32).isNil: + addr name.sin6_addr, cast[cstring](addr result[0][0]), (result[0].len+1).int32).isNil: raiseOSError(osLastError()) setLen(result[0], result[0].cstring.len) result[1] = Port(nativesockets.ntohs(name.sin6_port)) @@ -621,7 +653,7 @@ when not useNimNetLite: # Cannot use INET6_ADDRSTRLEN here, because it's a C define. result[0] = newString(64) if inet_ntop(name.sin6_family.cint, - addr name.sin6_addr, addr result[0][0], (result[0].len+1).int32).isNil: + addr name.sin6_addr, cast[cstring](addr result[0][0]), (result[0].len+1).int32).isNil: raiseOSError(osLastError()) setLen(result[0], result[0].cstring.len) result[1] = Port(nativesockets.ntohs(name.sin6_port)) diff --git a/lib/pure/net.nim b/lib/pure/net.nim index 16390d9af..24c94b651 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -93,23 +93,24 @@ import std/private/since when defined(nimPreviewSlimSystem): import std/assertions -import nativesockets -import os, strutils, times, sets, options, std/monotimes -import ssl_config +import std/nativesockets +import std/[os, strutils, times, sets, options, monotimes] +import std/ssl_config export nativesockets.Port, nativesockets.`$`, nativesockets.`==` -export Domain, SockType, Protocol +export Domain, SockType, Protocol, IPPROTO_NONE const useWinVersion = defined(windows) or defined(nimdoc) -const useNimNetLite = defined(nimNetLite) or defined(freertos) or defined(zephyr) +const useNimNetLite = defined(nimNetLite) or defined(freertos) or defined(zephyr) or + defined(nuttx) const defineSsl = defined(ssl) or defined(nimdoc) when useWinVersion: - from winlean import WSAESHUTDOWN + from std/winlean import WSAESHUTDOWN when defineSsl: - import openssl + import std/openssl when not defined(nimDisableCertificateValidation): - from ssl_certs import scanSSLCertificates + from std/ssl_certs import scanSSLCertificates # Note: The enumerations are mapped to Window's constants. @@ -208,7 +209,7 @@ when defined(nimHasStyleChecks): when defined(posix) and not defined(lwip): - from posix import TPollfd, POLLIN, POLLPRI, POLLOUT, POLLWRBAND, Tnfds + from std/posix import TPollfd, POLLIN, POLLPRI, POLLOUT, POLLWRBAND, Tnfds template monitorPollEvent(x: var SocketHandle, y: cint, timeout: int): int = var tpollfd: TPollfd @@ -480,8 +481,8 @@ proc parseIPv6Address(addressStr: string): IpAddress = proc parseIpAddress*(addressStr: string): IpAddress = ## Parses an IP address ## - ## Raises ValueError on error. - ## + ## Raises ValueError on error. + ## ## For IPv4 addresses, only the strict form as ## defined in RFC 6943 is considered valid, see ## https://datatracker.ietf.org/doc/html/rfc6943#section-3.1.1. @@ -621,10 +622,11 @@ when defineSsl: proc newContext*(protVersion = protSSLv23, verifyMode = CVerifyPeer, certFile = "", keyFile = "", cipherList = CiphersIntermediate, - caDir = "", caFile = ""): SslContext = + caDir = "", caFile = "", ciphersuites = CiphersModern): SslContext = ## Creates an SSL context. ## - ## protVersion is currently unsed. + ## Protocol version is currently ignored by default and TLS is used. + ## With `-d:openssl10`, only SSLv23 and TLSv1 may be used. ## ## There are three options for verify mode: ## `CVerifyNone`: certificates are not verified; @@ -651,7 +653,19 @@ when defineSsl: ## or using ECDSA: ## - `openssl ecparam -out mykey.pem -name secp256k1 -genkey` ## - `openssl req -new -key mykey.pem -x509 -nodes -days 365 -out mycert.pem` - let mtd = TLS_method() + var mtd: PSSL_METHOD + when defined(openssl10): + case protVersion + of protSSLv23: + mtd = SSLv23_method() + of protSSLv2: + raiseSSLError("SSLv2 is no longer secure and has been deprecated, use protSSLv23") + of protSSLv3: + raiseSSLError("SSLv3 is no longer secure and has been deprecated, use protSSLv23") + of protTLSv1: + mtd = TLSv1_method() + else: + mtd = TLS_method() if mtd == nil: raiseSSLError("Failed to create TLS context") var newCTX = SSL_CTX_new(mtd) @@ -662,16 +676,16 @@ when defineSsl: raiseSSLError() when not defined(openssl10) and not defined(libressl): let sslVersion = getOpenSSLVersion() - if sslVersion >= 0x010101000 and not sslVersion == 0x020000000: + if sslVersion >= 0x010101000 and sslVersion != 0x020000000: # In OpenSSL >= 1.1.1, TLSv1.3 cipher suites can only be configured via # this API. - if newCTX.SSL_CTX_set_ciphersuites(cipherList) != 1: + if newCTX.SSL_CTX_set_ciphersuites(ciphersuites) != 1: raiseSSLError() # Automatically the best ECDH curve for client exchange. Without this, ECDH # ciphers will be ignored by the server. # # From OpenSSL >= 1.1.0, this setting is set by default and can't be - # overriden. + # overridden. if newCTX.SSL_CTX_set_ecdh_auto(1) != 1: raiseSSLError() @@ -1025,7 +1039,7 @@ proc bindAddr*(socket: Socket, port = Port(0), address = "") {. proc acceptAddr*(server: Socket, client: var owned(Socket), address: var string, flags = {SocketFlag.SafeDisconn}, inheritable = defined(nimInheritHandles)) {. - tags: [ReadIOEffect], gcsafe, locks: 0.} = + tags: [ReadIOEffect], gcsafe.} = ## Blocks until a connection is being made from a client. When a connection ## is made sets `client` to the client socket and `address` to the address ## of the connecting client. @@ -1140,7 +1154,7 @@ proc accept*(server: Socket, client: var owned(Socket), acceptAddr(server, client, addrDummy, flags) when defined(posix) and not defined(lwip): - from posix import Sigset, sigwait, sigismember, sigemptyset, sigaddset, + from std/posix import Sigset, sigwait, sigismember, sigemptyset, sigaddset, sigprocmask, pthread_sigmask, SIGPIPE, SIG_BLOCK, SIG_UNBLOCK template blockSigpipe(body: untyped): untyped = @@ -1254,9 +1268,9 @@ proc close*(socket: Socket, flags = {SocketFlag.SafeDisconn}) = socket.fd = osInvalidSocket when defined(posix): - from posix import TCP_NODELAY + from std/posix import TCP_NODELAY else: - from winlean import TCP_NODELAY + from std/winlean import TCP_NODELAY proc toCInt*(opt: SOBool): cint = ## Converts a `SOBool` into its Socket Option cint representation. @@ -1307,7 +1321,7 @@ when defined(nimdoc) or (defined(posix) and not useNimNetLite): when not defined(nimdoc): var socketAddr = makeUnixAddr(path) if socket.fd.connect(cast[ptr SockAddr](addr socketAddr), - (sizeof(socketAddr.sun_family) + path.len).SockLen) != 0'i32: + (offsetOf(socketAddr, sun_path) + path.len + 1).SockLen) != 0'i32: raiseOSError(osLastError()) proc bindUnix*(socket: Socket, path: string) = @@ -1316,7 +1330,7 @@ when defined(nimdoc) or (defined(posix) and not useNimNetLite): when not defined(nimdoc): var socketAddr = makeUnixAddr(path) if socket.fd.bindAddr(cast[ptr SockAddr](addr socketAddr), - (sizeof(socketAddr.sun_family) + path.len).SockLen) != 0'i32: + (offsetOf(socketAddr, sun_path) + path.len + 1).SockLen) != 0'i32: raiseOSError(osLastError()) when defineSsl: @@ -1722,15 +1736,36 @@ proc send*(socket: Socket, data: pointer, size: int): int {. result = send(socket.fd, data, size, int32(MSG_NOSIGNAL)) proc send*(socket: Socket, data: string, - flags = {SocketFlag.SafeDisconn}) {.tags: [WriteIOEffect].} = - ## sends data to a socket. - let sent = send(socket, cstring(data), data.len) - if sent < 0: - let lastError = osLastError() - socketError(socket, lastError = lastError, flags = flags) + flags = {SocketFlag.SafeDisconn}, maxRetries = 100) {.tags: [WriteIOEffect].} = + ## Sends data to a socket. Will try to send all the data by handling interrupts + ## and incomplete writes up to `maxRetries`. + var written = 0 + var attempts = 0 + while data.len - written > 0: + let sent = send(socket, cstring(data), data.len) + + if sent < 0: + let lastError = osLastError() + let isBlockingErr = + when defined(nimdoc): + false + elif useWinVersion: + lastError.int32 == WSAEINTR or + lastError.int32 == WSAEWOULDBLOCK + else: + lastError.int32 == EINTR or + lastError.int32 == EWOULDBLOCK or + lastError.int32 == EAGAIN - if sent != data.len: - raiseOSError(osLastError(), "Could not send all data.") + if not isBlockingErr: + let lastError = osLastError() + socketError(socket, lastError = lastError, flags = flags) + else: + attempts.inc() + if attempts > maxRetries: + raiseOSError(osLastError(), "Could not send all data.") + else: + written.inc(sent) template `&=`*(socket: Socket; data: typed) = ## an alias for 'send'. @@ -1781,7 +1816,7 @@ proc sendTo*(socket: Socket, address: string, port: Port, ## This proc sends `data` to the specified `address`, ## which may be an IP address or a hostname, if a hostname is specified ## this function will try each IP of that hostname. - ## + ## ## Generally for use with connection-less (UDP) sockets. ## ## If an error occurs an OSError exception will be raised. @@ -1793,9 +1828,9 @@ proc sendTo*(socket: Socket, address: IpAddress, port: Port, data: string, flags = 0'i32): int {. discardable, tags: [WriteIOEffect].} = ## This proc sends `data` to the specified `IpAddress` and returns - ## the number of bytes written. + ## the number of bytes written. ## - ## Generally for use with connection-less (UDP) sockets. + ## Generally for use with connection-less (UDP) sockets. ## ## If an error occurs an OSError exception will be raised. ## @@ -1885,14 +1920,18 @@ proc `==`*(lhs, rhs: IpAddress): bool = proc `$`*(address: IpAddress): string = ## Converts an IpAddress into the textual representation - result = "" case address.family of IpAddressFamily.IPv4: - for i in 0 .. 3: - if i != 0: - result.add('.') - result.add($address.address_v4[i]) + result = newStringOfCap(15) + result.addInt address.address_v4[0] + result.add '.' + result.addInt address.address_v4[1] + result.add '.' + result.addInt address.address_v4[2] + result.add '.' + result.addInt address.address_v4[3] of IpAddressFamily.IPv6: + result = newStringOfCap(39) var currentZeroStart = -1 currentZeroCount = 0 @@ -2001,8 +2040,10 @@ proc dial*(address: string, port: Port, if success: result = newSocket(lastFd, domain, sockType, protocol, buffered) elif lastError != 0.OSErrorCode: + lastFd.close() raiseOSError(lastError) else: + lastFd.close() raise newException(IOError, "Couldn't resolve address: " & address) proc connect*(socket: Socket, address: string, diff --git a/lib/pure/nimprof.nim b/lib/pure/nimprof.nim index 3b1f703e3..bf8367d1d 100644 --- a/lib/pure/nimprof.nim +++ b/lib/pure/nimprof.nim @@ -15,13 +15,15 @@ when not defined(profiler) and not defined(memProfiler): {.error: "Profiling support is turned off! Enable profiling by passing `--profiler:on --stackTrace:on` to the compiler (see the Nim Compiler User Guide for more options).".} -when defined(nimHasUsed): - {.used.} +{.used.} # We don't want to profile the profiling code ... {.push profiler: off.} -import hashes, algorithm, strutils, tables, sets +import std/[hashes, algorithm, strutils, tables, sets] + +when defined(nimPreviewSlimSystem): + import std/[syncio, sysatomics] when not defined(memProfiler): include "system/timers" @@ -67,7 +69,7 @@ when not defined(memProfiler): else: interval = intervalInUs * 1000 - tickCountCorrection when withThreads: - import locks + import std/locks var profilingLock: Lock @@ -122,13 +124,13 @@ when defined(memProfiler): var gTicker {.threadvar.}: int - proc requestedHook(): bool {.nimcall, locks: 0.} = + proc requestedHook(): bool {.nimcall.} = if gTicker == 0: gTicker = SamplingInterval result = true dec gTicker - proc hook(st: StackTrace, size: int) {.nimcall, locks: 0.} = + proc hook(st: StackTrace, size: int) {.nimcall.} = when defined(ignoreAllocationSize): hookAux(st, 1) else: @@ -140,7 +142,7 @@ else: gTicker: int # we use an additional counter to # avoid calling 'getTicks' too frequently - proc requestedHook(): bool {.nimcall, locks: 0.} = + proc requestedHook(): bool {.nimcall.} = if interval == 0: result = true elif gTicker == 0: gTicker = 500 @@ -149,7 +151,7 @@ else: else: dec gTicker - proc hook(st: StackTrace) {.nimcall, locks: 0.} = + proc hook(st: StackTrace) {.nimcall.} = #echo "profiling! ", interval if interval == 0: hookAux(st, 1) diff --git a/lib/pure/nimtracker.nim b/lib/pure/nimtracker.nim deleted file mode 100644 index a66dfc2ea..000000000 --- a/lib/pure/nimtracker.nim +++ /dev/null @@ -1,88 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2016 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## Memory tracking support for Nim. - -when not defined(memTracker) and not isMainModule: - {.error: "Memory tracking support is turned off!".} - -{.push memtracker: off.} -# we import the low level wrapper and are careful not to use Nim's -# memory manager for anything here. -import sqlite3 - -var - dbHandle: PSqlite3 - insertStmt {.threadvar.}: Pstmt - -const insertQuery = "INSERT INTO tracking(op, address, size, file, line) values (?, ?, ?, ?, ?)" - -template sbind(x: int; value) = - when value is cstring: - let ret = insertStmt.bindText(x, value, value.len.int32, SQLITE_TRANSIENT) - if ret != SQLITE_OK: - quit "could not bind value" - else: - let ret = insertStmt.bindInt64(x, value) - if ret != SQLITE_OK: - quit "could not bind value" - -when defined(memTracker): - proc logEntries(log: TrackLog) {.nimcall, locks: 0, tags: [], gcsafe.} = - if insertStmt.isNil: - if prepare_v2(dbHandle, insertQuery, - insertQuery.len, insertStmt, nil) != SQLITE_OK: - quit "could not bind query to insertStmt " & $sqlite3.errmsg(dbHandle) - for i in 0..log.count-1: - var success = false - let e = log.data[i] - discard sqlite3.reset(insertStmt) - discard clearBindings(insertStmt) - sbind 1, e.op - sbind(2, cast[int](e.address)) - sbind 3, e.size - sbind 4, e.file - sbind 5, e.line - if step(insertStmt) == SQLITE_DONE: - success = true - if not success: - quit "could not write to database! " & $sqlite3.errmsg(dbHandle) - -proc execQuery(q: string) = - var s: Pstmt - if prepare_v2(dbHandle, q, q.len.int32, s, nil) == SQLITE_OK: - discard step(s) - if finalize(s) != SQLITE_OK: - quit "could not finalize " & $sqlite3.errmsg(dbHandle) - else: - quit "could not prepare statement " & $sqlite3.errmsg(dbHandle) - -proc setupDb() = - execQuery """create table if not exists tracking( - id integer primary key, - op varchar not null, - address integer not null, - size integer not null, - file varchar not null, - line integer not null - )""" - execQuery "delete from tracking" - -if sqlite3.open("memtrack.db", dbHandle) == SQLITE_OK: - setupDb() - const query = "INSERT INTO tracking(op, address, size, file, line) values (?, ?, ?, ?, ?)" - if prepare_v2(dbHandle, insertQuery, - insertQuery.len, insertStmt, nil) == SQLITE_OK: - when defined(memTracker): - setTrackLogger logEntries - else: - quit "could not prepare statement B " & $sqlite3.errmsg(dbHandle) -else: - quit "could not setup sqlite " & $sqlite3.errmsg(dbHandle) -{.pop.} diff --git a/lib/pure/oids.nim b/lib/pure/oids.nim index 43eadad27..4d6ceefd7 100644 --- a/lib/pure/oids.nim +++ b/lib/pure/oids.nim @@ -14,9 +14,12 @@ ## This implementation calls `initRand()` for the first call of ## `genOid`. -import hashes, times, endians, random +import std/[hashes, times, endians, random] from std/private/decode_helpers import handleHexChar +when defined(nimPreviewSlimSystem): + import std/sysatomics + type Oid* = object ## An OID. time: int64 @@ -41,9 +44,9 @@ proc hexbyte*(hex: char): int {.inline.} = proc parseOid*(str: cstring): Oid = ## Parses an OID. - var bytes = cast[cstring](addr(result.time)) + var bytes = cast[cstring](cast[pointer](cast[int](addr(result.time)) + 4)) var i = 0 - while i < 16: + while i < 12: bytes[i] = chr((hexbyte(str[2 * i]) shl 4) or hexbyte(str[2 * i + 1])) inc(i) @@ -51,12 +54,12 @@ proc `$`*(oid: Oid): string = ## Converts an OID to a string. const hex = "0123456789abcdef" - result.setLen 32 + result.setLen 24 var o = oid - var bytes = cast[cstring](addr(o)) + var bytes = cast[cstring](cast[pointer](cast[int](addr(o)) + 4)) var i = 0 - while i < 16: + while i < 12: let b = bytes[i].ord result[2 * i] = hex[(b and 0xF0) shr 4] result[2 * i + 1] = hex[b and 0xF] @@ -83,9 +86,9 @@ template genOid(result: var Oid, incr: var int, fuzz: int32) = proc genOid*(): Oid = ## Generates a new OID. runnableExamples: - doAssert ($genOid()).len == 32 + doAssert ($genOid()).len == 24 runnableExamples("-r:off"): - echo $genOid() # for example, "00000000632c452db08c3d19ee9073e5" + echo $genOid() # for example, "5fc7f546ddbbc84800006aaf" genOid(result, incr, fuzz) proc generatedTime*(oid: Oid): Time = diff --git a/lib/pure/options.nim b/lib/pure/options.nim index 9dc4e096b..b34ff72c0 100644 --- a/lib/pure/options.nim +++ b/lib/pure/options.nim @@ -53,7 +53,7 @@ Pattern matching supports pattern matching on `Option`s, with the `Some(<pattern>)` and `None()` patterns. -.. code-block:: nim + ```nim {.experimental: "caseStmtMacros".} import fusion/matching @@ -65,6 +65,7 @@ supports pattern matching on `Option`s, with the `Some(<pattern>)` and assert false assertMatch(some(some(none(int))), Some(Some(None()))) + ``` ]## # xxx pending https://github.com/timotheecour/Nim/issues/376 use `runnableExamples` and `whichModule` @@ -73,7 +74,7 @@ when defined(nimHasEffectsOf): else: {.pragma: effectsOf.} -import typetraits +import std/typetraits when defined(nimPreviewSlimSystem): import std/assertions @@ -81,7 +82,7 @@ when defined(nimPreviewSlimSystem): when (NimMajor, NimMinor) >= (1, 1): type - SomePointer = ref | ptr | pointer | proc + SomePointer = ref | ptr | pointer | proc | iterator {.closure.} else: type SomePointer = ref | ptr | pointer @@ -89,7 +90,7 @@ else: type Option*[T] = object ## An optional type that may or may not contain a value of type `T`. - ## When `T` is a a pointer type (`ptr`, `pointer`, `ref` or `proc`), + ## When `T` is a a pointer type (`ptr`, `pointer`, `ref`, `proc` or `iterator {.closure.}`), ## `none(T)` is represented as `nil`. when T is SomePointer: val: T @@ -116,9 +117,10 @@ proc option*[T](val: sink T): Option[T] {.inline.} = assert option[Foo](nil).isNone assert option(42).isSome - result.val = val - when T isnot SomePointer: - result.has = true + when T is SomePointer: + result = Option[T](val: val) + else: + result = Option[T](has: true, val: val) proc some*[T](val: sink T): Option[T] {.inline.} = ## Returns an `Option` that has the value `val`. @@ -135,10 +137,9 @@ proc some*[T](val: sink T): Option[T] {.inline.} = when T is SomePointer: assert not val.isNil - result.val = val + result = Option[T](val: val) else: - result.has = true - result.val = val + result = Option[T](has: true, val: val) proc none*(T: typedesc): Option[T] {.inline.} = ## Returns an `Option` for this type that has no value. @@ -151,7 +152,7 @@ proc none*(T: typedesc): Option[T] {.inline.} = assert none(int).isNone # the default is the none type - discard + result = Option[T]() proc none*[T]: Option[T] {.inline.} = ## Alias for `none(T) <#none,typedesc>`_. diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 5c387ec03..78ebb1c88 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -8,8 +8,10 @@ # ## This module contains basic operating system facilities like -## retrieving environment variables, reading command line arguments, -## working with directories, running shell commands, etc. +## retrieving environment variables, working with directories, +## running shell commands, etc. + +## .. importdoc:: symlinks.nim, appdirs.nim, dirs.nim, ospaths2.nim runnableExamples: let myFile = "/path/to/my/file.nim" @@ -20,22 +22,40 @@ runnableExamples: assert myFile.changeFileExt("c") == "/path/to/my/file.c" ## **See also:** +## * `paths <paths.html>`_ and `files <files.html>`_ modules for high-level file manipulation ## * `osproc module <osproc.html>`_ for process communication beyond ## `execShellCmd proc`_ -## * `parseopt module <parseopt.html>`_ for command-line parser beyond -## `parseCmdLine proc`_ ## * `uri module <uri.html>`_ ## * `distros module <distros.html>`_ ## * `dynlib module <dynlib.html>`_ ## * `streams module <streams.html>`_ +import std/private/ospaths2 +export ospaths2 + +import std/private/osfiles +export osfiles + +import std/private/osdirs +export osdirs + +import std/private/ossymlinks +export ossymlinks + +import std/private/osappdirs +export osappdirs + +import std/private/oscommon include system/inclrtl import std/private/since -import strutils, pathnorm +import std/cmdline +export cmdline + +import std/[strutils, pathnorm] when defined(nimPreviewSlimSystem): - import std/[syncio, assertions] + import std/[syncio, assertions, widestrs] const weirdTarget = defined(nimscript) or defined(js) @@ -56,9 +76,9 @@ since (1, 1): when weirdTarget: discard elif defined(windows): - import winlean, times + import std/[winlean, times] elif defined(posix): - import posix, times + import std/[posix, times] proc toTime(ts: Timespec): times.Time {.inline.} = result = initTime(ts.tv_sec.int64, ts.tv_nsec.int) @@ -78,955 +98,17 @@ elif defined(js): else: {.pragma: noNimJs.} -proc normalizePathAux(path: var string){.inline, raises: [], noSideEffect.} - -type - ReadEnvEffect* = object of ReadIOEffect ## Effect that denotes a read - ## from an environment variable. - WriteEnvEffect* = object of WriteIOEffect ## Effect that denotes a write - ## to an environment variable. - - ReadDirEffect* = object of ReadIOEffect ## Effect that denotes a read - ## operation from the directory - ## structure. - WriteDirEffect* = object of WriteIOEffect ## Effect that denotes a write - ## operation to - ## the directory structure. - - OSErrorCode* = distinct int32 ## Specifies an OS Error Code. - -include "includes/osseps" - -proc absolutePathInternal(path: string): string {.gcsafe.} - -proc normalizePathEnd(path: var string, trailingSep = false) = - ## Ensures ``path`` has exactly 0 or 1 trailing `DirSep`, depending on - ## ``trailingSep``, and taking care of edge cases: it preservers whether - ## a path is absolute or relative, and makes sure trailing sep is `DirSep`, - ## not `AltSep`. Trailing `/.` are compressed, see examples. - if path.len == 0: return - var i = path.len - while i >= 1: - if path[i-1] in {DirSep, AltSep}: dec(i) - elif path[i-1] == '.' and i >= 2 and path[i-2] in {DirSep, AltSep}: dec(i) - else: break - if trailingSep: - # foo// => foo - path.setLen(i) - # foo => foo/ - path.add DirSep - elif i > 0: - # foo// => foo - path.setLen(i) - else: - # // => / (empty case was already taken care of) - path = $DirSep - -proc normalizePathEnd(path: string, trailingSep = false): string = - ## outplace overload - runnableExamples: - when defined(posix): - assert normalizePathEnd("/lib//.//", trailingSep = true) == "/lib/" - assert normalizePathEnd("lib/./.", trailingSep = false) == "lib" - assert normalizePathEnd(".//./.", trailingSep = false) == "." - assert normalizePathEnd("", trailingSep = true) == "" # not / ! - assert normalizePathEnd("/", trailingSep = false) == "/" # not "" ! - result = path - result.normalizePathEnd(trailingSep) - -since((1, 1)): - export normalizePathEnd - -template endsWith(a: string, b: set[char]): bool = - a.len > 0 and a[^1] in b - -proc joinPathImpl(result: var string, state: var int, tail: string) = - let trailingSep = tail.endsWith({DirSep, AltSep}) or tail.len == 0 and result.endsWith({DirSep, AltSep}) - normalizePathEnd(result, trailingSep=false) - addNormalizePath(tail, result, state, DirSep) - normalizePathEnd(result, trailingSep=trailingSep) - -proc joinPath*(head, tail: string): string {. - noSideEffect, rtl, extern: "nos$1".} = - ## Joins two directory names to one. - ## - ## returns normalized path concatenation of `head` and `tail`, preserving - ## whether or not `tail` has a trailing slash (or, if tail if empty, whether - ## head has one). - ## - ## See also: - ## * `joinPath(parts: varargs[string]) proc`_ - ## * `/ proc`_ - ## * `splitPath proc`_ - ## * `uri.combine proc <uri.html#combine,Uri,Uri>`_ - ## * `uri./ proc <uri.html#/,Uri,string>`_ - runnableExamples: - when defined(posix): - assert joinPath("usr", "lib") == "usr/lib" - assert joinPath("usr", "lib/") == "usr/lib/" - assert joinPath("usr", "") == "usr" - assert joinPath("usr/", "") == "usr/" - assert joinPath("", "") == "" - assert joinPath("", "lib") == "lib" - assert joinPath("", "/lib") == "/lib" - assert joinPath("usr/", "/lib") == "usr/lib" - assert joinPath("usr/lib", "../bin") == "usr/bin" - - result = newStringOfCap(head.len + tail.len) - var state = 0 - joinPathImpl(result, state, head) - joinPathImpl(result, state, tail) - when false: - if len(head) == 0: - result = tail - elif head[len(head)-1] in {DirSep, AltSep}: - if tail.len > 0 and tail[0] in {DirSep, AltSep}: - result = head & substr(tail, 1) - else: - result = head & tail - else: - if tail.len > 0 and tail[0] in {DirSep, AltSep}: - result = head & tail - else: - result = head & DirSep & tail - -proc joinPath*(parts: varargs[string]): string {.noSideEffect, - rtl, extern: "nos$1OpenArray".} = - ## The same as `joinPath(head, tail) proc`_, - ## but works with any number of directory parts. - ## - ## You need to pass at least one element or the proc - ## will assert in debug builds and crash on release builds. - ## - ## See also: - ## * `joinPath(head, tail) proc`_ - ## * `/ proc`_ - ## * `/../ proc`_ - ## * `splitPath proc`_ - runnableExamples: - when defined(posix): - assert joinPath("a") == "a" - assert joinPath("a", "b", "c") == "a/b/c" - assert joinPath("usr/lib", "../../var", "log") == "var/log" - - var estimatedLen = 0 - for p in parts: estimatedLen += p.len - result = newStringOfCap(estimatedLen) - var state = 0 - for i in 0..high(parts): - joinPathImpl(result, state, parts[i]) - -proc `/`*(head, tail: string): string {.noSideEffect, inline.} = - ## The same as `joinPath(head, tail) proc`_. - ## - ## See also: - ## * `/../ proc`_ - ## * `joinPath(head, tail) proc`_ - ## * `joinPath(parts: varargs[string]) proc`_ - ## * `splitPath proc`_ - ## * `uri.combine proc <uri.html#combine,Uri,Uri>`_ - ## * `uri./ proc <uri.html#/,Uri,string>`_ - runnableExamples: - when defined(posix): - assert "usr" / "" == "usr" - assert "" / "lib" == "lib" - assert "" / "/lib" == "/lib" - assert "usr/" / "/lib/" == "usr/lib/" - assert "usr" / "lib" / "../bin" == "usr/bin" - - result = joinPath(head, tail) - -when doslikeFileSystem: - import std/private/ntpath - -proc splitPath*(path: string): tuple[head, tail: string] {. - noSideEffect, rtl, extern: "nos$1".} = - ## Splits a directory into `(head, tail)` tuple, so that - ## ``head / tail == path`` (except for edge cases like "/usr"). - ## - ## See also: - ## * `joinPath(head, tail) proc`_ - ## * `joinPath(parts: varargs[string]) proc`_ - ## * `/ proc`_ - ## * `/../ proc`_ - ## * `relativePath proc`_ - runnableExamples: - assert splitPath("usr/local/bin") == ("usr/local", "bin") - assert splitPath("usr/local/bin/") == ("usr/local/bin", "") - assert splitPath("/bin/") == ("/bin", "") - when (NimMajor, NimMinor) <= (1, 0): - assert splitPath("/bin") == ("", "bin") - else: - assert splitPath("/bin") == ("/", "bin") - assert splitPath("bin") == ("", "bin") - assert splitPath("") == ("", "") - - when doslikeFileSystem: - let (drive, splitpath) = splitDrive(path) - let stop = drive.len - else: - const stop = 0 - - var sepPos = -1 - for i in countdown(len(path)-1, stop): - if path[i] in {DirSep, AltSep}: - sepPos = i - break - if sepPos >= 0: - result.head = substr(path, 0, - when (NimMajor, NimMinor) <= (1, 0): - sepPos-1 - else: - if likely(sepPos >= 1): sepPos-1 else: 0 - ) - result.tail = substr(path, sepPos+1) - else: - when doslikeFileSystem: - result.head = drive - result.tail = splitpath - else: - result.head = "" - result.tail = path - -proc isAbsolute*(path: string): bool {.rtl, noSideEffect, extern: "nos$1", raises: [].} = - ## Checks whether a given `path` is absolute. - ## - ## On Windows, network paths are considered absolute too. - runnableExamples: - assert not "".isAbsolute - assert not ".".isAbsolute - when defined(posix): - assert "/".isAbsolute - assert not "a/".isAbsolute - assert "/a/".isAbsolute - - if len(path) == 0: return false - - when doslikeFileSystem: - var len = len(path) - result = (path[0] in {'/', '\\'}) or - (len > 1 and path[0] in {'a'..'z', 'A'..'Z'} and path[1] == ':') - elif defined(macos): - # according to https://perldoc.perl.org/File/Spec/Mac.html `:a` is a relative path - result = path[0] != ':' - elif defined(RISCOS): - result = path[0] == '$' - elif defined(posix) or defined(js): - # `or defined(js)` wouldn't be needed pending https://github.com/nim-lang/Nim/issues/13469 - # This works around the problem for posix, but Windows is still broken with nim js -d:nodejs - result = path[0] == '/' - else: - doAssert false # if ever hits here, adapt as needed - -when FileSystemCaseSensitive: - template `!=?`(a, b: char): bool = a != b -else: - template `!=?`(a, b: char): bool = toLowerAscii(a) != toLowerAscii(b) - -when doslikeFileSystem: - proc isAbsFromCurrentDrive(path: string): bool {.noSideEffect, raises: [].} = - ## An absolute path from the root of the current drive (e.g. "\foo") - path.len > 0 and - (path[0] == AltSep or - (path[0] == DirSep and - (path.len == 1 or path[1] notin {DirSep, AltSep, ':'}))) - - proc sameRoot(path1, path2: string): bool {.noSideEffect, raises: [].} = - ## Return true if path1 and path2 have a same root. - ## - ## Detail of Windows path formats: - ## https://docs.microsoft.com/en-us/dotnet/standard/io/file-path-formats - - assert(isAbsolute(path1)) - assert(isAbsolute(path2)) - - if isAbsFromCurrentDrive(path1) and isAbsFromCurrentDrive(path2): - result = true - elif cmpIgnoreCase(splitDrive(path1).drive, splitDrive(path2).drive) == 0: - result = true - else: - result = false - -proc relativePath*(path, base: string, sep = DirSep): string {. - rtl, extern: "nos$1".} = - ## Converts `path` to a path relative to `base`. - ## - ## The `sep` (default: DirSep_) is used for the path normalizations, - ## this can be useful to ensure the relative path only contains `'/'` - ## so that it can be used for URL constructions. - ## - ## On Windows, if a root of `path` and a root of `base` are different, - ## returns `path` as is because it is impossible to make a relative path. - ## That means an absolute path can be returned. - ## - ## See also: - ## * `splitPath proc`_ - ## * `parentDir proc`_ - ## * `tailDir proc`_ - runnableExamples: - assert relativePath("/Users/me/bar/z.nim", "/Users/other/bad", '/') == "../../me/bar/z.nim" - assert relativePath("/Users/me/bar/z.nim", "/Users/other", '/') == "../me/bar/z.nim" - when not doslikeFileSystem: # On Windows, UNC-paths start with `//` - assert relativePath("/Users///me/bar//z.nim", "//Users/", '/') == "me/bar/z.nim" - assert relativePath("/Users/me/bar/z.nim", "/Users/me", '/') == "bar/z.nim" - assert relativePath("", "/users/moo", '/') == "" - assert relativePath("foo", ".", '/') == "foo" - assert relativePath("foo", "foo", '/') == "." - - if path.len == 0: return "" - var base = if base == ".": "" else: base - var path = path - path.normalizePathAux - base.normalizePathAux - let a1 = isAbsolute(path) - let a2 = isAbsolute(base) - if a1 and not a2: - base = absolutePathInternal(base) - elif a2 and not a1: - path = absolutePathInternal(path) - - when doslikeFileSystem: - if isAbsolute(path) and isAbsolute(base): - if not sameRoot(path, base): - return path - - var f = default PathIter - var b = default PathIter - var ff = (0, -1) - var bb = (0, -1) # (int, int) - result = newStringOfCap(path.len) - # skip the common prefix: - while f.hasNext(path) and b.hasNext(base): - ff = next(f, path) - bb = next(b, base) - let diff = ff[1] - ff[0] - if diff != bb[1] - bb[0]: break - var same = true - for i in 0..diff: - if path[i + ff[0]] !=? base[i + bb[0]]: - same = false - break - if not same: break - ff = (0, -1) - bb = (0, -1) - # for i in 0..diff: - # result.add base[i + bb[0]] - - # /foo/bar/xxx/ -- base - # /foo/bar/baz -- path path - # ../baz - # every directory that is in 'base', needs to add '..' - while true: - if bb[1] >= bb[0]: - if result.len > 0 and result[^1] != sep: - result.add sep - result.add ".." - if not b.hasNext(base): break - bb = b.next(base) - - # add the rest of 'path': - while true: - if ff[1] >= ff[0]: - if result.len > 0 and result[^1] != sep: - result.add sep - for i in 0..ff[1] - ff[0]: - result.add path[i + ff[0]] - if not f.hasNext(path): break - ff = f.next(path) - - when not defined(nimOldRelativePathBehavior): - if result.len == 0: result.add "." - -proc isRelativeTo*(path: string, base: string): bool {.since: (1, 1).} = - ## Returns true if `path` is relative to `base`. - runnableExamples: - doAssert isRelativeTo("./foo//bar", "foo") - doAssert isRelativeTo("foo/bar", ".") - doAssert isRelativeTo("/foo/bar.nim", "/foo/bar.nim") - doAssert not isRelativeTo("foo/bar.nims", "foo/bar.nim") - let path = path.normalizePath - let base = base.normalizePath - let ret = relativePath(path, base) - result = path.len > 0 and not ret.startsWith ".." - -proc parentDirPos(path: string): int = - var q = 1 - if len(path) >= 1 and path[len(path)-1] in {DirSep, AltSep}: q = 2 - for i in countdown(len(path)-q, 0): - if path[i] in {DirSep, AltSep}: return i - result = -1 - -proc parentDir*(path: string): string {. - noSideEffect, rtl, extern: "nos$1".} = - ## Returns the parent directory of `path`. - ## - ## This is similar to ``splitPath(path).head`` when ``path`` doesn't end - ## in a dir separator, but also takes care of path normalizations. - ## The remainder can be obtained with `lastPathPart(path) proc`_. - ## - ## See also: - ## * `relativePath proc`_ - ## * `splitPath proc`_ - ## * `tailDir proc`_ - ## * `parentDirs iterator`_ - runnableExamples: - assert parentDir("") == "" - when defined(posix): - assert parentDir("/usr/local/bin") == "/usr/local" - assert parentDir("foo/bar//") == "foo" - assert parentDir("//foo//bar//.") == "/foo" - assert parentDir("./foo") == "." - assert parentDir("/./foo//./") == "/" - assert parentDir("a//./") == "." - assert parentDir("a/b/c/..") == "a" - result = pathnorm.normalizePath(path) - when doslikeFileSystem: - let (drive, splitpath) = splitDrive(result) - result = splitpath - var sepPos = parentDirPos(result) - if sepPos >= 0: - result = substr(result, 0, sepPos) - normalizePathEnd(result) - elif result == ".." or result == "." or result.len == 0 or result[^1] in {DirSep, AltSep}: - # `.` => `..` and .. => `../..`(etc) would be a sensible alternative - # `/` => `/` (as done with splitFile) would be a sensible alternative - result = "" - else: - result = "." - when doslikeFileSystem: - if result.len == 0: - discard - elif drive.len > 0 and result.len == 1 and result[0] in {DirSep, AltSep}: - result = drive - else: - result = drive & result - -proc tailDir*(path: string): string {. - noSideEffect, rtl, extern: "nos$1".} = - ## Returns the tail part of `path`. - ## - ## See also: - ## * `relativePath proc`_ - ## * `splitPath proc`_ - ## * `parentDir proc`_ - runnableExamples: - assert tailDir("/bin") == "bin" - assert tailDir("bin") == "" - assert tailDir("bin/") == "" - assert tailDir("/usr/local/bin") == "usr/local/bin" - assert tailDir("//usr//local//bin//") == "usr//local//bin//" - assert tailDir("./usr/local/bin") == "usr/local/bin" - assert tailDir("usr/local/bin") == "local/bin" - - var i = 0 - when doslikeFileSystem: - let (drive, splitpath) = path.splitDrive - if drive != "": - return splitpath.strip(chars = {DirSep, AltSep}, trailing = false) - while i < len(path): - if path[i] in {DirSep, AltSep}: - while i < len(path) and path[i] in {DirSep, AltSep}: inc i - return substr(path, i) - inc i - result = "" - -proc isRootDir*(path: string): bool {. - noSideEffect, rtl, extern: "nos$1".} = - ## Checks whether a given `path` is a root directory. - runnableExamples: - assert isRootDir("") - assert isRootDir(".") - assert isRootDir("/") - assert isRootDir("a") - assert not isRootDir("/a") - assert not isRootDir("a/b/c") - - when doslikeFileSystem: - if splitDrive(path).path == "": - return true - result = parentDirPos(path) < 0 - -iterator parentDirs*(path: string, fromRoot=false, inclusive=true): string = - ## Walks over all parent directories of a given `path`. - ## - ## If `fromRoot` is true (default: false), the traversal will start from - ## the file system root directory. - ## If `inclusive` is true (default), the original argument will be included - ## in the traversal. - ## - ## Relative paths won't be expanded by this iterator. Instead, it will traverse - ## only the directories appearing in the relative path. - ## - ## See also: - ## * `parentDir proc`_ - ## - runnableExamples: - let g = "a/b/c" - - for p in g.parentDirs: - echo p - # a/b/c - # a/b - # a - - for p in g.parentDirs(fromRoot=true): - echo p - # a/ - # a/b/ - # a/b/c - - for p in g.parentDirs(inclusive=false): - echo p - # a/b - # a - - if not fromRoot: - var current = path - if inclusive: yield path - while true: - if current.isRootDir: break - current = current.parentDir - yield current - else: - when doslikeFileSystem: - let start = path.splitDrive.drive.len - else: - const start = 0 - for i in countup(start, path.len - 2): # ignore the last / - # deal with non-normalized paths such as /foo//bar//baz - if path[i] in {DirSep, AltSep} and - (i == 0 or path[i-1] notin {DirSep, AltSep}): - yield path.substr(0, i) - - if inclusive: yield path - -proc `/../`*(head, tail: string): string {.noSideEffect.} = - ## The same as ``parentDir(head) / tail``, unless there is no parent - ## directory. Then ``head / tail`` is performed instead. - ## - ## See also: - ## * `/ proc`_ - ## * `parentDir proc`_ - runnableExamples: - when defined(posix): - assert "a/b/c" /../ "d/e" == "a/b/d/e" - assert "a" /../ "d/e" == "a/d/e" - - when doslikeFileSystem: - let (drive, head) = splitDrive(head) - let sepPos = parentDirPos(head) - if sepPos >= 0: - result = substr(head, 0, sepPos-1) / tail - else: - result = head / tail - when doslikeFileSystem: - result = drive / result - -proc normExt(ext: string): string = - if ext == "" or ext[0] == ExtSep: result = ext # no copy needed here - else: result = ExtSep & ext - -proc searchExtPos*(path: string): int = - ## Returns index of the `'.'` char in `path` if it signifies the beginning - ## of extension. Returns -1 otherwise. - ## - ## See also: - ## * `splitFile proc`_ - ## * `extractFilename proc`_ - ## * `lastPathPart proc`_ - ## * `changeFileExt proc`_ - ## * `addFileExt proc`_ - runnableExamples: - assert searchExtPos("a/b/c") == -1 - assert searchExtPos("c.nim") == 1 - assert searchExtPos("a/b/c.nim") == 5 - assert searchExtPos("a.b.c.nim") == 5 - - # BUGFIX: do not search until 0! .DS_Store is no file extension! - result = -1 - for i in countdown(len(path)-1, 1): - if path[i] == ExtSep: - result = i - break - elif path[i] in {DirSep, AltSep}: - break # do not skip over path - -proc splitFile*(path: string): tuple[dir, name, ext: string] {. - noSideEffect, rtl, extern: "nos$1".} = - ## Splits a filename into `(dir, name, extension)` tuple. - ## - ## `dir` does not end in DirSep_ unless it's `/`. - ## `extension` includes the leading dot. - ## - ## If `path` has no extension, `ext` is the empty string. - ## If `path` has no directory component, `dir` is the empty string. - ## If `path` has no filename component, `name` and `ext` are empty strings. - ## - ## See also: - ## * `searchExtPos proc`_ - ## * `extractFilename proc`_ - ## * `lastPathPart proc`_ - ## * `changeFileExt proc`_ - ## * `addFileExt proc`_ - runnableExamples: - var (dir, name, ext) = splitFile("usr/local/nimc.html") - assert dir == "usr/local" - assert name == "nimc" - assert ext == ".html" - (dir, name, ext) = splitFile("/usr/local/os") - assert dir == "/usr/local" - assert name == "os" - assert ext == "" - (dir, name, ext) = splitFile("/usr/local/") - assert dir == "/usr/local" - assert name == "" - assert ext == "" - (dir, name, ext) = splitFile("/tmp.txt") - assert dir == "/" - assert name == "tmp" - assert ext == ".txt" - - var namePos = 0 - var dotPos = 0 - when doslikeFileSystem: - let (drive, _) = splitDrive(path) - let stop = len(drive) - result.dir = drive - else: - const stop = 0 - for i in countdown(len(path) - 1, stop): - if path[i] in {DirSep, AltSep} or i == 0: - if path[i] in {DirSep, AltSep}: - result.dir = substr(path, 0, if likely(i >= 1): i - 1 else: 0) - namePos = i + 1 - if dotPos > i: - result.name = substr(path, namePos, dotPos - 1) - result.ext = substr(path, dotPos) - else: - result.name = substr(path, namePos) - break - elif path[i] == ExtSep and i > 0 and i < len(path) - 1 and - path[i - 1] notin {DirSep, AltSep} and - path[i + 1] != ExtSep and dotPos == 0: - dotPos = i - -proc extractFilename*(path: string): string {. - noSideEffect, rtl, extern: "nos$1".} = - ## Extracts the filename of a given `path`. - ## - ## This is the same as ``name & ext`` from `splitFile(path) proc`_. - ## - ## See also: - ## * `searchExtPos proc`_ - ## * `splitFile proc`_ - ## * `lastPathPart proc`_ - ## * `changeFileExt proc`_ - ## * `addFileExt proc`_ - runnableExamples: - assert extractFilename("foo/bar/") == "" - assert extractFilename("foo/bar") == "bar" - assert extractFilename("foo/bar.baz") == "bar.baz" - - if path.len == 0 or path[path.len-1] in {DirSep, AltSep}: - result = "" - else: - result = splitPath(path).tail - -proc lastPathPart*(path: string): string {. - noSideEffect, rtl, extern: "nos$1".} = - ## Like `extractFilename proc`_, but ignores - ## trailing dir separator; aka: `baseName`:idx: in some other languages. - ## - ## See also: - ## * `searchExtPos proc`_ - ## * `splitFile proc`_ - ## * `extractFilename proc`_ - ## * `changeFileExt proc`_ - ## * `addFileExt proc`_ - runnableExamples: - assert lastPathPart("foo/bar/") == "bar" - assert lastPathPart("foo/bar") == "bar" - - let path = path.normalizePathEnd(trailingSep = false) - result = extractFilename(path) - -proc changeFileExt*(filename, ext: string): string {. - noSideEffect, rtl, extern: "nos$1".} = - ## Changes the file extension to `ext`. - ## - ## If the `filename` has no extension, `ext` will be added. - ## If `ext` == "" then any extension is removed. - ## - ## `Ext` should be given without the leading `'.'`, because some - ## filesystems may use a different character. (Although I know - ## of none such beast.) - ## - ## See also: - ## * `searchExtPos proc`_ - ## * `splitFile proc`_ - ## * `extractFilename proc`_ - ## * `lastPathPart proc`_ - ## * `addFileExt proc`_ - runnableExamples: - assert changeFileExt("foo.bar", "baz") == "foo.baz" - assert changeFileExt("foo.bar", "") == "foo" - assert changeFileExt("foo", "baz") == "foo.baz" - - var extPos = searchExtPos(filename) - if extPos < 0: result = filename & normExt(ext) - else: result = substr(filename, 0, extPos-1) & normExt(ext) - -proc addFileExt*(filename, ext: string): string {. - noSideEffect, rtl, extern: "nos$1".} = - ## Adds the file extension `ext` to `filename`, unless - ## `filename` already has an extension. - ## - ## `Ext` should be given without the leading `'.'`, because some - ## filesystems may use a different character. - ## (Although I know of none such beast.) - ## - ## See also: - ## * `searchExtPos proc`_ - ## * `splitFile proc`_ - ## * `extractFilename proc`_ - ## * `lastPathPart proc`_ - ## * `changeFileExt proc`_ - runnableExamples: - assert addFileExt("foo.bar", "baz") == "foo.bar" - assert addFileExt("foo.bar", "") == "foo.bar" - assert addFileExt("foo", "baz") == "foo.baz" - - var extPos = searchExtPos(filename) - if extPos < 0: result = filename & normExt(ext) - else: result = filename - -proc cmpPaths*(pathA, pathB: string): int {. - noSideEffect, rtl, extern: "nos$1".} = - ## Compares two paths. - ## - ## On a case-sensitive filesystem this is done - ## case-sensitively otherwise case-insensitively. Returns: - ## - ## | 0 if pathA == pathB - ## | < 0 if pathA < pathB - ## | > 0 if pathA > pathB - runnableExamples: - when defined(macosx): - assert cmpPaths("foo", "Foo") == 0 - elif defined(posix): - assert cmpPaths("foo", "Foo") > 0 - - let a = normalizePath(pathA) - let b = normalizePath(pathB) - if FileSystemCaseSensitive: - result = cmp(a, b) - else: - when defined(nimscript): - result = cmpic(a, b) - elif defined(nimdoc): discard - else: - result = cmpIgnoreCase(a, b) - -proc unixToNativePath*(path: string, drive=""): string {. - noSideEffect, rtl, extern: "nos$1".} = - ## Converts an UNIX-like path to a native one. - ## - ## On an UNIX system this does nothing. Else it converts - ## `'/'`, `'.'`, `'..'` to the appropriate things. - ## - ## On systems with a concept of "drives", `drive` is used to determine - ## which drive label to use during absolute path conversion. - ## `drive` defaults to the drive of the current working directory, and is - ## ignored on systems that do not have a concept of "drives". - when defined(unix): - result = path - else: - if path.len == 0: return "" - - var start: int - if path[0] == '/': - # an absolute path - when doslikeFileSystem: - if drive != "": - result = drive & ":" & DirSep - else: - result = $DirSep - elif defined(macos): - result = "" # must not start with ':' - else: - result = $DirSep - start = 1 - elif path[0] == '.' and (path.len == 1 or path[1] == '/'): - # current directory - result = $CurDir - start = when doslikeFileSystem: 1 else: 2 - else: - result = "" - start = 0 - - var i = start - while i < len(path): # ../../../ --> :::: - if i+2 < path.len and path[i] == '.' and path[i+1] == '.' and path[i+2] == '/': - # parent directory - when defined(macos): - if result[high(result)] == ':': - add result, ':' - else: - add result, ParDir - else: - add result, ParDir & DirSep - inc(i, 3) - elif path[i] == '/': - add result, DirSep - inc(i) - else: - add result, path[i] - inc(i) - -include "includes/oserr" -include "includes/osenv" -proc getHomeDir*(): string {.rtl, extern: "nos$1", - tags: [ReadEnvEffect, ReadIOEffect].} = - ## Returns the home directory of the current user. - ## - ## This proc is wrapped by the `expandTilde proc`_ - ## for the convenience of processing paths coming from user configuration files. - ## - ## See also: - ## * `getConfigDir proc`_ - ## * `getTempDir proc`_ - ## * `expandTilde proc`_ - ## * `getCurrentDir proc`_ - ## * `setCurrentDir proc`_ - runnableExamples: - assert getHomeDir() == expandTilde("~") +import std/oserrors +export oserrors +import std/envvars +export envvars - when defined(windows): return getEnv("USERPROFILE") & "\\" - else: return getEnv("HOME") & "/" +import std/private/osseps +export osseps -proc getConfigDir*(): string {.rtl, extern: "nos$1", - tags: [ReadEnvEffect, ReadIOEffect].} = - ## Returns the config directory of the current user for applications. - ## - ## On non-Windows OSs, this proc conforms to the XDG Base Directory - ## spec. Thus, this proc returns the value of the `XDG_CONFIG_HOME` environment - ## variable if it is set, otherwise it returns the default configuration - ## directory ("~/.config/"). - ## - ## An OS-dependent trailing slash is always present at the end of the - ## returned string: `\\` on Windows and `/` on all other OSs. - ## - ## See also: - ## * `getHomeDir proc`_ - ## * `getTempDir proc`_ - ## * `expandTilde proc`_ - ## * `getCurrentDir proc`_ - ## * `setCurrentDir proc`_ - when defined(windows): - result = getEnv("APPDATA") - else: - result = getEnv("XDG_CONFIG_HOME", getEnv("HOME") / ".config") - result.normalizePathEnd(trailingSep = true) -proc getCacheDir*(): string = - ## Returns the cache directory of the current user for applications. - ## - ## This makes use of the following environment variables: - ## - ## * On Windows: `getEnv("LOCALAPPDATA")` - ## - ## * On macOS: `getEnv("XDG_CACHE_HOME", getEnv("HOME") / "Library/Caches")` - ## - ## * On other platforms: `getEnv("XDG_CACHE_HOME", getEnv("HOME") / ".cache")` - ## - ## **See also:** - ## * `getHomeDir proc`_ - ## * `getTempDir proc`_ - ## * `getConfigDir proc`_ - # follows https://crates.io/crates/platform-dirs - when defined(windows): - result = getEnv("LOCALAPPDATA") - elif defined(osx): - result = getEnv("XDG_CACHE_HOME", getEnv("HOME") / "Library/Caches") - else: - result = getEnv("XDG_CACHE_HOME", getEnv("HOME") / ".cache") - result.normalizePathEnd(false) - -proc getCacheDir*(app: string): string = - ## Returns the cache directory for an application `app`. - ## - ## * On Windows, this uses: `getCacheDir() / app / "cache"` - ## * On other platforms, this uses: `getCacheDir() / app` - when defined(windows): - getCacheDir() / app / "cache" - else: - getCacheDir() / app - - -when defined(windows): - type DWORD = uint32 - - proc getTempPath( - nBufferLength: DWORD, lpBuffer: WideCString - ): DWORD {.stdcall, dynlib: "kernel32.dll", importc: "GetTempPathW".} = - ## Retrieves the path of the directory designated for temporary files. - -template getEnvImpl(result: var string, tempDirList: openArray[string]) = - for dir in tempDirList: - if existsEnv(dir): - result = getEnv(dir) - break - -template getTempDirImpl(result: var string) = - when defined(windows): - getEnvImpl(result, ["TMP", "TEMP", "USERPROFILE"]) - else: - getEnvImpl(result, ["TMPDIR", "TEMP", "TMP", "TEMPDIR"]) - -proc getTempDir*(): string {.rtl, extern: "nos$1", - tags: [ReadEnvEffect, ReadIOEffect].} = - ## Returns the temporary directory of the current user for applications to - ## save temporary files in. - ## - ## On Windows, it calls [GetTempPath](https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-gettemppathw). - ## On Posix based platforms, it will check `TMPDIR`, `TEMP`, `TMP` and `TEMPDIR` environment variables in order. - ## On all platforms, `/tmp` will be returned if the procs fails. - ## - ## You can override this implementation - ## by adding `-d:tempDir=mytempname` to your compiler invocation. - ## - ## **Note:** This proc does not check whether the returned path exists. - ## - ## See also: - ## * `getHomeDir proc`_ - ## * `getConfigDir proc`_ - ## * `expandTilde proc`_ - ## * `getCurrentDir proc`_ - ## * `setCurrentDir proc`_ - const tempDirDefault = "/tmp" - when defined(tempDir): - const tempDir {.strdefine.}: string = tempDirDefault - result = tempDir - else: - when nimvm: - getTempDirImpl(result) - else: - when defined(windows): - let size = getTempPath(0, nil) - # If the function fails, the return value is zero. - if size > 0: - let buffer = newWideCString(size.int) - if getTempPath(size, buffer) > 0: - result = $buffer - elif defined(android): result = "/data/local/tmp" - else: - getTempDirImpl(result) - if result.len == 0: - result = tempDirDefault - normalizePathEnd(result, trailingSep=true) - proc expandTilde*(path: string): string {. tags: [ReadEnvEffect, ReadIOEffect].} = ## Expands ``~`` or a path starting with ``~/`` to a full path, replacing @@ -1130,106 +212,10 @@ when not weirdTarget: importc: "system", header: "<stdlib.h>".} when not defined(windows): - proc c_rename(oldname, newname: cstring): cint {. - importc: "rename", header: "<stdio.h>".} - proc c_strlen(a: cstring): cint {. - importc: "strlen", header: "<string.h>", noSideEffect.} proc c_free(p: pointer) {. importc: "free", header: "<stdlib.h>".} -when defined(windows) and not weirdTarget: - when useWinUnicode: - template wrapUnary(varname, winApiProc, arg: untyped) = - var varname = winApiProc(newWideCString(arg)) - - template wrapBinary(varname, winApiProc, arg, arg2: untyped) = - var varname = winApiProc(newWideCString(arg), arg2) - proc findFirstFile(a: string, b: var WIN32_FIND_DATA): Handle = - result = findFirstFileW(newWideCString(a), b) - template findNextFile(a, b: untyped): untyped = findNextFileW(a, b) - template getCommandLine(): untyped = getCommandLineW() - - template getFilename(f: untyped): untyped = - $cast[WideCString](addr(f.cFileName[0])) - else: - template findFirstFile(a, b: untyped): untyped = findFirstFileA(a, b) - template findNextFile(a, b: untyped): untyped = findNextFileA(a, b) - template getCommandLine(): untyped = getCommandLineA() - - template getFilename(f: untyped): untyped = $cstring(addr f.cFileName) - - proc skipFindData(f: WIN32_FIND_DATA): bool {.inline.} = - # Note - takes advantage of null delimiter in the cstring - const dot = ord('.') - result = f.cFileName[0].int == dot and (f.cFileName[1].int == 0 or - f.cFileName[1].int == dot and f.cFileName[2].int == 0) - -proc fileExists*(filename: string): bool {.rtl, extern: "nos$1", - tags: [ReadDirEffect], noNimJs.} = - ## Returns true if `filename` exists and is a regular file or symlink. - ## - ## Directories, device files, named pipes and sockets return false. - ## - ## See also: - ## * `dirExists proc`_ - ## * `symlinkExists proc`_ - when defined(windows): - when useWinUnicode: - wrapUnary(a, getFileAttributesW, filename) - else: - var a = getFileAttributesA(filename) - if a != -1'i32: - result = (a and FILE_ATTRIBUTE_DIRECTORY) == 0'i32 - else: - var res: Stat - return stat(filename, res) >= 0'i32 and S_ISREG(res.st_mode) - -proc dirExists*(dir: string): bool {.rtl, extern: "nos$1", tags: [ReadDirEffect], - noNimJs.} = - ## Returns true if the directory `dir` exists. If `dir` is a file, false - ## is returned. Follows symlinks. - ## - ## See also: - ## * `fileExists proc`_ - ## * `symlinkExists proc`_ - when defined(windows): - when useWinUnicode: - wrapUnary(a, getFileAttributesW, dir) - else: - var a = getFileAttributesA(dir) - if a != -1'i32: - result = (a and FILE_ATTRIBUTE_DIRECTORY) != 0'i32 - else: - var res: Stat - result = stat(dir, res) >= 0'i32 and S_ISDIR(res.st_mode) - -proc symlinkExists*(link: string): bool {.rtl, extern: "nos$1", - tags: [ReadDirEffect], - noWeirdTarget.} = - ## Returns true if the symlink `link` exists. Will return true - ## regardless of whether the link points to a directory or file. - ## - ## See also: - ## * `fileExists proc`_ - ## * `dirExists proc`_ - when defined(windows): - when useWinUnicode: - wrapUnary(a, getFileAttributesW, link) - else: - var a = getFileAttributesA(link) - if a != -1'i32: - # xxx see: bug #16784 (bug9); checking `IO_REPARSE_TAG_SYMLINK` - # may also be needed. - result = (a and FILE_ATTRIBUTE_REPARSE_POINT) != 0'i32 - else: - var res: Stat - result = lstat(link, res) >= 0'i32 and S_ISLNK(res.st_mode) - - -when not defined(windows): - const maxSymlinkLen = 1024 - const ExeExts* = ## Platform specific file extension for executables. ## On Windows ``["exe", "cmd", "bat"]``, on Posix ``[""]``. @@ -1269,7 +255,7 @@ proc findExe*(exe: string, followSymlinks: bool = true; for ext in extensions: var x = addFileExt(x, ext) if fileExists(x): - when not defined(windows): + when not (defined(windows) or defined(nintendoswitch)): while followSymlinks: # doubles as if here if x.symlinkExists: var r = newString(maxSymlinkLen) @@ -1368,356 +354,6 @@ proc fileNewer*(a, b: string): bool {.rtl, extern: "nos$1", noWeirdTarget.} = else: result = getLastModificationTime(a) > getLastModificationTime(b) -when not defined(nimscript): - proc getCurrentDir*(): string {.rtl, extern: "nos$1", tags: [].} = - ## Returns the `current working directory`:idx: i.e. where the built - ## binary is run. - ## - ## So the path returned by this proc is determined at run time. - ## - ## See also: - ## * `getHomeDir proc`_ - ## * `getConfigDir proc`_ - ## * `getTempDir proc`_ - ## * `setCurrentDir proc`_ - ## * `currentSourcePath template <system.html#currentSourcePath.t>`_ - ## * `getProjectPath proc <macros.html#getProjectPath>`_ - when defined(nodejs): - var ret: cstring - {.emit: "`ret` = process.cwd();".} - return $ret - elif defined(js): - doAssert false, "use -d:nodejs to have `getCurrentDir` defined" - elif defined(windows): - var bufsize = MAX_PATH.int32 - when useWinUnicode: - var res = newWideCString("", bufsize) - while true: - var L = getCurrentDirectoryW(bufsize, res) - if L == 0'i32: - raiseOSError(osLastError()) - elif L > bufsize: - res = newWideCString("", L) - bufsize = L - else: - result = res$L - break - else: - result = newString(bufsize) - while true: - var L = getCurrentDirectoryA(bufsize, result) - if L == 0'i32: - raiseOSError(osLastError()) - elif L > bufsize: - result = newString(L) - bufsize = L - else: - setLen(result, L) - break - else: - var bufsize = 1024 # should be enough - result = newString(bufsize) - while true: - if getcwd(result.cstring, bufsize) != nil: - setLen(result, c_strlen(result.cstring)) - break - else: - let err = osLastError() - if err.int32 == ERANGE: - bufsize = bufsize shl 1 - doAssert(bufsize >= 0) - result = newString(bufsize) - else: - raiseOSError(osLastError()) - -proc setCurrentDir*(newDir: string) {.inline, tags: [], noWeirdTarget.} = - ## Sets the `current working directory`:idx:; `OSError` - ## is raised if `newDir` cannot been set. - ## - ## See also: - ## * `getHomeDir proc`_ - ## * `getConfigDir proc`_ - ## * `getTempDir proc`_ - ## * `getCurrentDir proc`_ - when defined(windows): - when useWinUnicode: - if setCurrentDirectoryW(newWideCString(newDir)) == 0'i32: - raiseOSError(osLastError(), newDir) - else: - if setCurrentDirectoryA(newDir) == 0'i32: raiseOSError(osLastError(), newDir) - else: - if chdir(newDir) != 0'i32: raiseOSError(osLastError(), newDir) - - -proc absolutePath*(path: string, root = getCurrentDir()): string = - ## Returns the absolute path of `path`, rooted at `root` (which must be absolute; - ## default: current directory). - ## If `path` is absolute, return it, ignoring `root`. - ## - ## See also: - ## * `normalizedPath proc`_ - ## * `normalizePath proc`_ - runnableExamples: - assert absolutePath("a") == getCurrentDir() / "a" - - if isAbsolute(path): path - else: - if not root.isAbsolute: - raise newException(ValueError, "The specified root is not absolute: " & root) - joinPath(root, path) - -proc absolutePathInternal(path: string): string = - absolutePath(path, getCurrentDir()) - -proc normalizeExe*(file: var string) {.since: (1, 3, 5).} = - ## on posix, prepends `./` if `file` doesn't contain `/` and is not `"", ".", ".."`. - runnableExamples: - import std/sugar - when defined(posix): - doAssert "foo".dup(normalizeExe) == "./foo" - doAssert "foo/../bar".dup(normalizeExe) == "foo/../bar" - doAssert "".dup(normalizeExe) == "" - when defined(posix): - if file.len > 0 and DirSep notin file and file != "." and file != "..": - file = "./" & file - -proc normalizePath*(path: var string) {.rtl, extern: "nos$1", tags: [].} = - ## Normalize a path. - ## - ## Consecutive directory separators are collapsed, including an initial double slash. - ## - ## On relative paths, double dot (`..`) sequences are collapsed if possible. - ## On absolute paths they are always collapsed. - ## - ## .. warning:: URL-encoded and Unicode attempts at directory traversal are not detected. - ## Triple dot is not handled. - ## - ## See also: - ## * `absolutePath proc`_ - ## * `normalizedPath proc`_ for outplace version - ## * `normalizeExe proc`_ - runnableExamples: - when defined(posix): - var a = "a///b//..//c///d" - a.normalizePath() - assert a == "a/c/d" - - path = pathnorm.normalizePath(path) - when false: - let isAbs = isAbsolute(path) - var stack: seq[string] = @[] - for p in split(path, {DirSep}): - case p - of "", ".": - continue - of "..": - if stack.len == 0: - if isAbs: - discard # collapse all double dots on absoluta paths - else: - stack.add(p) - elif stack[^1] == "..": - stack.add(p) - else: - discard stack.pop() - else: - stack.add(p) - - if isAbs: - path = DirSep & join(stack, $DirSep) - elif stack.len > 0: - path = join(stack, $DirSep) - else: - path = "." - -proc normalizePathAux(path: var string) = normalizePath(path) - -proc normalizedPath*(path: string): string {.rtl, extern: "nos$1", tags: [].} = - ## Returns a normalized path for the current OS. - ## - ## See also: - ## * `absolutePath proc`_ - ## * `normalizePath proc`_ for the in-place version - runnableExamples: - when defined(posix): - assert normalizedPath("a///b//..//c///d") == "a/c/d" - result = pathnorm.normalizePath(path) - -when defined(windows) and not weirdTarget: - proc openHandle(path: string, followSymlink=true, writeAccess=false): Handle = - var flags = FILE_FLAG_BACKUP_SEMANTICS or FILE_ATTRIBUTE_NORMAL - if not followSymlink: - flags = flags or FILE_FLAG_OPEN_REPARSE_POINT - let access = if writeAccess: GENERIC_WRITE else: 0'i32 - - when useWinUnicode: - result = createFileW( - newWideCString(path), access, - FILE_SHARE_DELETE or FILE_SHARE_READ or FILE_SHARE_WRITE, - nil, OPEN_EXISTING, flags, 0 - ) - else: - result = createFileA( - path, access, - FILE_SHARE_DELETE or FILE_SHARE_READ or FILE_SHARE_WRITE, - nil, OPEN_EXISTING, flags, 0 - ) - -proc sameFile*(path1, path2: string): bool {.rtl, extern: "nos$1", - tags: [ReadDirEffect], noWeirdTarget.} = - ## Returns true if both pathname arguments refer to the same physical - ## file or directory. - ## - ## Raises `OSError` if any of the files does not - ## exist or information about it can not be obtained. - ## - ## This proc will return true if given two alternative hard-linked or - ## sym-linked paths to the same file or directory. - ## - ## See also: - ## * `sameFileContent proc`_ - when defined(windows): - var success = true - var f1 = openHandle(path1) - var f2 = openHandle(path2) - - var lastErr: OSErrorCode - if f1 != INVALID_HANDLE_VALUE and f2 != INVALID_HANDLE_VALUE: - var fi1, fi2: BY_HANDLE_FILE_INFORMATION - - if getFileInformationByHandle(f1, addr(fi1)) != 0 and - getFileInformationByHandle(f2, addr(fi2)) != 0: - result = fi1.dwVolumeSerialNumber == fi2.dwVolumeSerialNumber and - fi1.nFileIndexHigh == fi2.nFileIndexHigh and - fi1.nFileIndexLow == fi2.nFileIndexLow - else: - lastErr = osLastError() - success = false - else: - lastErr = osLastError() - success = false - - discard closeHandle(f1) - discard closeHandle(f2) - - if not success: raiseOSError(lastErr, $(path1, path2)) - else: - var a, b: Stat - if stat(path1, a) < 0'i32 or stat(path2, b) < 0'i32: - raiseOSError(osLastError(), $(path1, path2)) - else: - result = a.st_dev == b.st_dev and a.st_ino == b.st_ino - -type - FilePermission* = enum ## File access permission, modelled after UNIX. - ## - ## See also: - ## * `getFilePermissions`_ - ## * `setFilePermissions`_ - ## * `FileInfo object`_ - fpUserExec, ## execute access for the file owner - fpUserWrite, ## write access for the file owner - fpUserRead, ## read access for the file owner - fpGroupExec, ## execute access for the group - fpGroupWrite, ## write access for the group - fpGroupRead, ## read access for the group - fpOthersExec, ## execute access for others - fpOthersWrite, ## write access for others - fpOthersRead ## read access for others - -proc getFilePermissions*(filename: string): set[FilePermission] {. - rtl, extern: "nos$1", tags: [ReadDirEffect], noWeirdTarget.} = - ## Retrieves file permissions for `filename`. - ## - ## `OSError` is raised in case of an error. - ## On Windows, only the ``readonly`` flag is checked, every other - ## permission is available in any case. - ## - ## See also: - ## * `setFilePermissions proc`_ - ## * `FilePermission enum`_ - when defined(posix): - var a: Stat - if stat(filename, a) < 0'i32: raiseOSError(osLastError(), filename) - result = {} - if (a.st_mode and S_IRUSR.Mode) != 0.Mode: result.incl(fpUserRead) - if (a.st_mode and S_IWUSR.Mode) != 0.Mode: result.incl(fpUserWrite) - if (a.st_mode and S_IXUSR.Mode) != 0.Mode: result.incl(fpUserExec) - - if (a.st_mode and S_IRGRP.Mode) != 0.Mode: result.incl(fpGroupRead) - if (a.st_mode and S_IWGRP.Mode) != 0.Mode: result.incl(fpGroupWrite) - if (a.st_mode and S_IXGRP.Mode) != 0.Mode: result.incl(fpGroupExec) - - if (a.st_mode and S_IROTH.Mode) != 0.Mode: result.incl(fpOthersRead) - if (a.st_mode and S_IWOTH.Mode) != 0.Mode: result.incl(fpOthersWrite) - if (a.st_mode and S_IXOTH.Mode) != 0.Mode: result.incl(fpOthersExec) - else: - when useWinUnicode: - wrapUnary(res, getFileAttributesW, filename) - else: - var res = getFileAttributesA(filename) - if res == -1'i32: raiseOSError(osLastError(), filename) - if (res and FILE_ATTRIBUTE_READONLY) != 0'i32: - result = {fpUserExec, fpUserRead, fpGroupExec, fpGroupRead, - fpOthersExec, fpOthersRead} - else: - result = {fpUserExec..fpOthersRead} - -proc setFilePermissions*(filename: string, permissions: set[FilePermission], - followSymlinks = true) - {.rtl, extern: "nos$1", tags: [ReadDirEffect, WriteDirEffect], - noWeirdTarget.} = - ## Sets the file permissions for `filename`. - ## - ## If `followSymlinks` set to true (default) and ``filename`` points to a - ## symlink, permissions are set to the file symlink points to. - ## `followSymlinks` set to false is a noop on Windows and some POSIX - ## systems (including Linux) on which `lchmod` is either unavailable or always - ## fails, given that symlinks permissions there are not observed. - ## - ## `OSError` is raised in case of an error. - ## On Windows, only the ``readonly`` flag is changed, depending on - ## ``fpUserWrite`` permission. - ## - ## See also: - ## * `getFilePermissions proc`_ - ## * `FilePermission enum`_ - when defined(posix): - var p = 0.Mode - if fpUserRead in permissions: p = p or S_IRUSR.Mode - if fpUserWrite in permissions: p = p or S_IWUSR.Mode - if fpUserExec in permissions: p = p or S_IXUSR.Mode - - if fpGroupRead in permissions: p = p or S_IRGRP.Mode - if fpGroupWrite in permissions: p = p or S_IWGRP.Mode - if fpGroupExec in permissions: p = p or S_IXGRP.Mode - - if fpOthersRead in permissions: p = p or S_IROTH.Mode - if fpOthersWrite in permissions: p = p or S_IWOTH.Mode - if fpOthersExec in permissions: p = p or S_IXOTH.Mode - - if not followSymlinks and filename.symlinkExists: - when declared(lchmod): - if lchmod(filename, cast[Mode](p)) != 0: - raiseOSError(osLastError(), $(filename, permissions)) - else: - if chmod(filename, cast[Mode](p)) != 0: - raiseOSError(osLastError(), $(filename, permissions)) - else: - when useWinUnicode: - wrapUnary(res, getFileAttributesW, filename) - else: - var res = getFileAttributesA(filename) - if res == -1'i32: raiseOSError(osLastError(), filename) - if fpUserWrite in permissions: - res = res and not FILE_ATTRIBUTE_READONLY - else: - res = res or FILE_ATTRIBUTE_READONLY - when useWinUnicode: - wrapBinary(res2, setFileAttributesW, filename, res) - else: - var res2 = setFileAttributesA(filename, res) - if res2 == - 1'i32: raiseOSError(osLastError(), $(filename, permissions)) proc isAdmin*: bool {.noWeirdTarget.} = ## Returns whether the caller's process is a member of the Administrators local @@ -1750,310 +386,6 @@ proc isAdmin*: bool {.noWeirdTarget.} = else: result = geteuid() == 0 -proc createSymlink*(src, dest: string) {.noWeirdTarget.} = - ## Create a symbolic link at `dest` which points to the item specified - ## by `src`. On most operating systems, will fail if a link already exists. - ## - ## .. warning:: Some OS's (such as Microsoft Windows) restrict the creation - ## of symlinks to root users (administrators) or users with developer mode enabled. - ## - ## See also: - ## * `createHardlink proc`_ - ## * `expandSymlink proc`_ - - when defined(windows): - const SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE = 2 - # allows anyone with developer mode on to create a link - let flag = dirExists(src).int32 or SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE - when useWinUnicode: - var wSrc = newWideCString(src) - var wDst = newWideCString(dest) - if createSymbolicLinkW(wDst, wSrc, flag) == 0 or getLastError() != 0: - raiseOSError(osLastError(), $(src, dest)) - else: - if createSymbolicLinkA(dest, src, flag) == 0 or getLastError() != 0: - raiseOSError(osLastError(), $(src, dest)) - else: - if symlink(src, dest) != 0: - raiseOSError(osLastError(), $(src, dest)) - -proc expandSymlink*(symlinkPath: string): string {.noWeirdTarget.} = - ## Returns a string representing the path to which the symbolic link points. - ## - ## On Windows this is a noop, `symlinkPath` is simply returned. - ## - ## See also: - ## * `createSymlink proc`_ - when defined(windows): - result = symlinkPath - else: - result = newString(maxSymlinkLen) - var len = readlink(symlinkPath, result.cstring, maxSymlinkLen) - if len < 0: - raiseOSError(osLastError(), symlinkPath) - if len > maxSymlinkLen: - result = newString(len+1) - len = readlink(symlinkPath, result.cstring, len) - setLen(result, len) - -const hasCCopyfile = defined(osx) and not defined(nimLegacyCopyFile) - # xxx instead of `nimLegacyCopyFile`, support something like: `when osxVersion >= (10, 5)` - -when hasCCopyfile: - # `copyfile` API available since osx 10.5. - {.push nodecl, header: "<copyfile.h>".} - type - copyfile_state_t {.nodecl.} = pointer - copyfile_flags_t = cint - proc copyfile_state_alloc(): copyfile_state_t - proc copyfile_state_free(state: copyfile_state_t): cint - proc c_copyfile(src, dst: cstring, state: copyfile_state_t, flags: copyfile_flags_t): cint {.importc: "copyfile".} - # replace with `let` pending bootstrap >= 1.4.0 - var - COPYFILE_DATA {.nodecl.}: copyfile_flags_t - COPYFILE_XATTR {.nodecl.}: copyfile_flags_t - {.pop.} - -type - CopyFlag* = enum ## Copy options. - cfSymlinkAsIs, ## Copy symlinks as symlinks - cfSymlinkFollow, ## Copy the files symlinks point to - cfSymlinkIgnore ## Ignore symlinks - -const copyFlagSymlink = {cfSymlinkAsIs, cfSymlinkFollow, cfSymlinkIgnore} - -proc copyFile*(source, dest: string, options = {cfSymlinkFollow}) {.rtl, - extern: "nos$1", tags: [ReadDirEffect, ReadIOEffect, WriteIOEffect], - noWeirdTarget.} = - ## Copies a file from `source` to `dest`, where `dest.parentDir` must exist. - ## - ## On non-Windows OSes, `options` specify the way file is copied; by default, - ## if `source` is a symlink, copies the file symlink points to. `options` is - ## ignored on Windows: symlinks are skipped. - ## - ## If this fails, `OSError` is raised. - ## - ## On the Windows platform this proc will - ## copy the source file's attributes into dest. - ## - ## On other platforms you need - ## to use `getFilePermissions`_ and - ## `setFilePermissions`_ - ## procs - ## to copy them by hand (or use the convenience `copyFileWithPermissions - ## proc`_), - ## otherwise `dest` will inherit the default permissions of a newly - ## created file for the user. - ## - ## If `dest` already exists, the file attributes - ## will be preserved and the content overwritten. - ## - ## On OSX, `copyfile` C api will be used (available since OSX 10.5) unless - ## `-d:nimLegacyCopyFile` is used. - ## - ## See also: - ## * `CopyFlag enum`_ - ## * `copyDir proc`_ - ## * `copyFileWithPermissions proc`_ - ## * `tryRemoveFile proc`_ - ## * `removeFile proc`_ - ## * `moveFile proc`_ - - doAssert card(copyFlagSymlink * options) == 1, "There should be exactly " & - "one cfSymlink* in options" - let isSymlink = source.symlinkExists - if isSymlink and (cfSymlinkIgnore in options or defined(windows)): - return - when defined(windows): - when useWinUnicode: - let s = newWideCString(source) - let d = newWideCString(dest) - if copyFileW(s, d, 0'i32) == 0'i32: - raiseOSError(osLastError(), $(source, dest)) - else: - if copyFileA(source, dest, 0'i32) == 0'i32: - raiseOSError(osLastError(), $(source, dest)) - else: - if isSymlink and cfSymlinkAsIs in options: - createSymlink(expandSymlink(source), dest) - else: - when hasCCopyfile: - let state = copyfile_state_alloc() - # xxx `COPYFILE_STAT` could be used for one-shot - # `copyFileWithPermissions`. - let status = c_copyfile(source.cstring, dest.cstring, state, - COPYFILE_DATA) - if status != 0: - let err = osLastError() - discard copyfile_state_free(state) - raiseOSError(err, $(source, dest)) - let status2 = copyfile_state_free(state) - if status2 != 0: raiseOSError(osLastError(), $(source, dest)) - else: - # generic version of copyFile which works for any platform: - const bufSize = 8000 # better for memory manager - var d, s: File - if not open(s, source):raiseOSError(osLastError(), source) - if not open(d, dest, fmWrite): - close(s) - raiseOSError(osLastError(), dest) - var buf = alloc(bufSize) - while true: - var bytesread = readBuffer(s, buf, bufSize) - if bytesread > 0: - var byteswritten = writeBuffer(d, buf, bytesread) - if bytesread != byteswritten: - dealloc(buf) - close(s) - close(d) - raiseOSError(osLastError(), dest) - if bytesread != bufSize: break - dealloc(buf) - close(s) - flushFile(d) - close(d) - -proc copyFileToDir*(source, dir: string, options = {cfSymlinkFollow}) - {.noWeirdTarget, since: (1,3,7).} = - ## Copies a file `source` into directory `dir`, which must exist. - ## - ## On non-Windows OSes, `options` specify the way file is copied; by default, - ## if `source` is a symlink, copies the file symlink points to. `options` is - ## ignored on Windows: symlinks are skipped. - ## - ## See also: - ## * `CopyFlag enum`_ - ## * `copyFile proc`_ - if dir.len == 0: # treating "" as "." is error prone - raise newException(ValueError, "dest is empty") - copyFile(source, dir / source.lastPathPart, options) - -when not declared(ENOENT) and not defined(windows): - when defined(nimscript): - when not defined(haiku): - const ENOENT = cint(2) # 2 on most systems including Solaris - else: - const ENOENT = cint(-2147459069) - else: - var ENOENT {.importc, header: "<errno.h>".}: cint - -when defined(windows) and not weirdTarget: - when useWinUnicode: - template deleteFile(file: untyped): untyped = deleteFileW(file) - template setFileAttributes(file, attrs: untyped): untyped = - setFileAttributesW(file, attrs) - else: - template deleteFile(file: untyped): untyped = deleteFileA(file) - template setFileAttributes(file, attrs: untyped): untyped = - setFileAttributesA(file, attrs) - -proc tryRemoveFile*(file: string): bool {.rtl, extern: "nos$1", tags: [WriteDirEffect], noWeirdTarget.} = - ## Removes the `file`. - ## - ## If this fails, returns `false`. This does not fail - ## if the file never existed in the first place. - ## - ## On Windows, ignores the read-only attribute. - ## - ## See also: - ## * `copyFile proc`_ - ## * `copyFileWithPermissions proc`_ - ## * `removeFile proc`_ - ## * `moveFile proc`_ - result = true - when defined(windows): - when useWinUnicode: - let f = newWideCString(file) - else: - let f = file - if deleteFile(f) == 0: - result = false - let err = getLastError() - if err == ERROR_FILE_NOT_FOUND or err == ERROR_PATH_NOT_FOUND: - result = true - elif err == ERROR_ACCESS_DENIED and - setFileAttributes(f, FILE_ATTRIBUTE_NORMAL) != 0 and - deleteFile(f) != 0: - result = true - else: - if unlink(file) != 0'i32 and errno != ENOENT: - result = false - -proc removeFile*(file: string) {.rtl, extern: "nos$1", tags: [WriteDirEffect], noWeirdTarget.} = - ## Removes the `file`. - ## - ## If this fails, `OSError` is raised. This does not fail - ## if the file never existed in the first place. - ## - ## On Windows, ignores the read-only attribute. - ## - ## See also: - ## * `removeDir proc`_ - ## * `copyFile proc`_ - ## * `copyFileWithPermissions proc`_ - ## * `tryRemoveFile proc`_ - ## * `moveFile proc`_ - if not tryRemoveFile(file): - raiseOSError(osLastError(), file) - -proc tryMoveFSObject(source, dest: string, isDir: bool): bool {.noWeirdTarget.} = - ## Moves a file (or directory if `isDir` is true) from `source` to `dest`. - ## - ## Returns false in case of `EXDEV` error or `AccessDeniedError` on Windows (if `isDir` is true). - ## In case of other errors `OSError` is raised. - ## Returns true in case of success. - when defined(windows): - when useWinUnicode: - let s = newWideCString(source) - let d = newWideCString(dest) - result = moveFileExW(s, d, MOVEFILE_COPY_ALLOWED or MOVEFILE_REPLACE_EXISTING) != 0'i32 - else: - result = moveFileExA(source, dest, MOVEFILE_COPY_ALLOWED or MOVEFILE_REPLACE_EXISTING) != 0'i32 - else: - result = c_rename(source, dest) == 0'i32 - - if not result: - let err = osLastError() - let isAccessDeniedError = - when defined(windows): - const AccessDeniedError = OSErrorCode(5) - isDir and err == AccessDeniedError - else: - err == EXDEV.OSErrorCode - if not isAccessDeniedError: - raiseOSError(err, $(source, dest)) - -proc moveFile*(source, dest: string) {.rtl, extern: "nos$1", - tags: [ReadDirEffect, ReadIOEffect, WriteIOEffect], noWeirdTarget.} = - ## Moves a file from `source` to `dest`. - ## - ## Symlinks are not followed: if `source` is a symlink, it is itself moved, - ## not its target. - ## - ## If this fails, `OSError` is raised. - ## If `dest` already exists, it will be overwritten. - ## - ## Can be used to `rename files`:idx:. - ## - ## See also: - ## * `moveDir proc`_ - ## * `copyFile proc`_ - ## * `copyFileWithPermissions proc`_ - ## * `removeFile proc`_ - ## * `tryRemoveFile proc`_ - - if not tryMoveFSObject(source, dest, isDir = false): - when defined(windows): - doAssert false - else: - # Fallback to copy & del - copyFile(source, dest, {cfSymlinkAsIs}) - try: - removeFile(source) - except: - discard tryRemoveFile(dest) - raise - proc exitStatusLikeShell*(status: cint): cint = ## Converts exit code from `c_system` into a shell exit code. @@ -2079,122 +411,11 @@ proc execShellCmd*(command: string): int {.rtl, extern: "nos$1", ## <osproc.html#execProcess,string,string,openArray[string],StringTableRef,set[ProcessOption]>`_. ## ## **Examples:** - ## - ## .. code-block:: + ## ```Nim ## discard execShellCmd("ls -la") + ## ``` result = exitStatusLikeShell(c_system(command)) -# Templates for filtering directories and files -when defined(windows) and not weirdTarget: - template isDir(f: WIN32_FIND_DATA): bool = - (f.dwFileAttributes and FILE_ATTRIBUTE_DIRECTORY) != 0'i32 - template isFile(f: WIN32_FIND_DATA): bool = - not isDir(f) -else: - template isDir(f: string): bool {.dirty.} = - dirExists(f) - template isFile(f: string): bool {.dirty.} = - fileExists(f) - -template defaultWalkFilter(item): bool = - ## Walk filter used to return true on both - ## files and directories - true - -template walkCommon(pattern: string, filter) = - ## Common code for getting the files and directories with the - ## specified `pattern` - when defined(windows): - var - f: WIN32_FIND_DATA - res: int - res = findFirstFile(pattern, f) - if res != -1: - defer: findClose(res) - let dotPos = searchExtPos(pattern) - while true: - if not skipFindData(f) and filter(f): - # Windows bug/gotcha: 't*.nim' matches 'tfoo.nims' -.- so we check - # that the file extensions have the same length ... - let ff = getFilename(f) - let idx = ff.len - pattern.len + dotPos - if dotPos < 0 or idx >= ff.len or (idx >= 0 and ff[idx] == '.') or - (dotPos >= 0 and dotPos+1 < pattern.len and pattern[dotPos+1] == '*'): - yield splitFile(pattern).dir / extractFilename(ff) - if findNextFile(res, f) == 0'i32: - let errCode = getLastError() - if errCode == ERROR_NO_MORE_FILES: break - else: raiseOSError(errCode.OSErrorCode) - else: # here we use glob - var - f: Glob - res: int - f.gl_offs = 0 - f.gl_pathc = 0 - f.gl_pathv = nil - res = glob(pattern, 0, nil, addr(f)) - defer: globfree(addr(f)) - if res == 0: - for i in 0.. f.gl_pathc - 1: - assert(f.gl_pathv[i] != nil) - let path = $f.gl_pathv[i] - if filter(path): - yield path - -iterator walkPattern*(pattern: string): string {.tags: [ReadDirEffect], noWeirdTarget.} = - ## Iterate over all the files and directories that match the `pattern`. - ## - ## On POSIX this uses the `glob`:idx: call. - ## `pattern` is OS dependent, but at least the `"*.ext"` - ## notation is supported. - ## - ## See also: - ## * `walkFiles iterator`_ - ## * `walkDirs iterator`_ - ## * `walkDir iterator`_ - ## * `walkDirRec iterator`_ - runnableExamples: - import std/sequtils - let paths = toSeq(walkPattern("lib/pure/*")) # works on Windows too - assert "lib/pure/concurrency".unixToNativePath in paths - assert "lib/pure/os.nim".unixToNativePath in paths - walkCommon(pattern, defaultWalkFilter) - -iterator walkFiles*(pattern: string): string {.tags: [ReadDirEffect], noWeirdTarget.} = - ## Iterate over all the files that match the `pattern`. - ## - ## On POSIX this uses the `glob`:idx: call. - ## `pattern` is OS dependent, but at least the `"*.ext"` - ## notation is supported. - ## - ## See also: - ## * `walkPattern iterator`_ - ## * `walkDirs iterator`_ - ## * `walkDir iterator`_ - ## * `walkDirRec iterator`_ - runnableExamples: - import std/sequtils - assert "lib/pure/os.nim".unixToNativePath in toSeq(walkFiles("lib/pure/*.nim")) # works on Windows too - walkCommon(pattern, isFile) - -iterator walkDirs*(pattern: string): string {.tags: [ReadDirEffect], noWeirdTarget.} = - ## Iterate over all the directories that match the `pattern`. - ## - ## On POSIX this uses the `glob`:idx: call. - ## `pattern` is OS dependent, but at least the `"*.ext"` - ## notation is supported. - ## - ## See also: - ## * `walkPattern iterator`_ - ## * `walkFiles iterator`_ - ## * `walkDir iterator`_ - ## * `walkDirRec iterator`_ - runnableExamples: - import std/sequtils - let paths = toSeq(walkDirs("lib/pure/*")) # works on Windows too - assert "lib/pure/concurrency".unixToNativePath in paths - walkCommon(pattern, isDir) - proc expandFilename*(filename: string): string {.rtl, extern: "nos$1", tags: [ReadDirEffect], noWeirdTarget.} = ## Returns the full (`absolute`:idx:) path of an existing file `filename`. @@ -2202,32 +423,18 @@ proc expandFilename*(filename: string): string {.rtl, extern: "nos$1", ## Raises `OSError` in case of an error. Follows symlinks. when defined(windows): var bufsize = MAX_PATH.int32 - when useWinUnicode: - var unused: WideCString = nil - var res = newWideCString("", bufsize) - while true: - var L = getFullPathNameW(newWideCString(filename), bufsize, res, unused) - if L == 0'i32: - raiseOSError(osLastError(), filename) - elif L > bufsize: - res = newWideCString("", L) - bufsize = L - else: - result = res$L - break - else: - var unused: cstring = nil - result = newString(bufsize) - while true: - var L = getFullPathNameA(filename, bufsize, result, unused) - if L == 0'i32: - raiseOSError(osLastError(), filename) - elif L > bufsize: - result = newString(L) - bufsize = L - else: - setLen(result, L) - break + var unused: WideCString = nil + var res = newWideCString(bufsize) + while true: + var L = getFullPathNameW(newWideCString(filename), bufsize, res, unused) + if L == 0'i32: + raiseOSError(osLastError(), filename) + elif L > bufsize: + res = newWideCString(L) + bufsize = L + else: + result = res$L + break # getFullPathName doesn't do case corrections, so we have to use this convoluted # way of retrieving the true filename for x in walkFiles(result): @@ -2245,376 +452,14 @@ proc expandFilename*(filename: string): string {.rtl, extern: "nos$1", result = $r c_free(cast[pointer](r)) -type - PathComponent* = enum ## Enumeration specifying a path component. - ## - ## See also: - ## * `walkDirRec iterator`_ - ## * `FileInfo object`_ - pcFile, ## path refers to a file - pcLinkToFile, ## path refers to a symbolic link to a file - pcDir, ## path refers to a directory - pcLinkToDir ## path refers to a symbolic link to a directory - proc getCurrentCompilerExe*(): string {.compileTime.} = discard - ## This is `getAppFilename()`_ at compile time. + ## Returns the path of the currently running Nim compiler or nimble executable. ## ## Can be used to retrieve the currently executing ## Nim compiler from a Nim or nimscript program, or the nimble binary ## inside a nimble program (likewise with other binaries built from ## compiler API). -when defined(posix) and not weirdTarget: - proc getSymlinkFileKind(path: string): PathComponent = - # Helper function. - var s: Stat - assert(path != "") - if stat(path, s) == 0'i32 and S_ISDIR(s.st_mode): - result = pcLinkToDir - else: - result = pcLinkToFile - -proc staticWalkDir(dir: string; relative: bool): seq[ - tuple[kind: PathComponent, path: string]] = - discard - -iterator walkDir*(dir: string; relative = false, checkDir = false): - tuple[kind: PathComponent, path: string] {.tags: [ReadDirEffect].} = - ## Walks over the directory `dir` and yields for each directory or file in - ## `dir`. The component type and full path for each item are returned. - ## - ## Walking is not recursive. If ``relative`` is true (default: false) - ## the resulting path is shortened to be relative to ``dir``. - ## - ## If `checkDir` is true, `OSError` is raised when `dir` - ## doesn't exist. - ## - ## **Example:** - ## - ## This directory structure:: - ## dirA / dirB / fileB1.txt - ## / dirC - ## / fileA1.txt - ## / fileA2.txt - ## - ## and this code: - runnableExamples("-r:off"): - import std/[strutils, sugar] - # note: order is not guaranteed - # this also works at compile time - assert collect(for k in walkDir("dirA"): k.path).join(" ") == - "dirA/dirB dirA/dirC dirA/fileA2.txt dirA/fileA1.txt" - ## See also: - ## * `walkPattern iterator`_ - ## * `walkFiles iterator`_ - ## * `walkDirs iterator`_ - ## * `walkDirRec iterator`_ - - when nimvm: - for k, v in items(staticWalkDir(dir, relative)): - yield (k, v) - else: - when weirdTarget: - for k, v in items(staticWalkDir(dir, relative)): - yield (k, v) - elif defined(windows): - var f: WIN32_FIND_DATA - var h = findFirstFile(dir / "*", f) - if h == -1: - if checkDir: - raiseOSError(osLastError(), dir) - else: - defer: findClose(h) - while true: - var k = pcFile - if not skipFindData(f): - if (f.dwFileAttributes and FILE_ATTRIBUTE_DIRECTORY) != 0'i32: - k = pcDir - if (f.dwFileAttributes and FILE_ATTRIBUTE_REPARSE_POINT) != 0'i32: - k = succ(k) - let xx = if relative: extractFilename(getFilename(f)) - else: dir / extractFilename(getFilename(f)) - yield (k, xx) - if findNextFile(h, f) == 0'i32: - let errCode = getLastError() - if errCode == ERROR_NO_MORE_FILES: break - else: raiseOSError(errCode.OSErrorCode) - else: - var d = opendir(dir) - if d == nil: - if checkDir: - raiseOSError(osLastError(), dir) - else: - defer: discard closedir(d) - while true: - var x = readdir(d) - if x == nil: break - var y = $cstring(addr x.d_name) - if y != "." and y != "..": - var s: Stat - let path = dir / y - if not relative: - y = path - var k = pcFile - - template kSetGeneric() = # pure Posix component `k` resolution - if lstat(path.cstring, s) < 0'i32: continue # don't yield - elif S_ISDIR(s.st_mode): - k = pcDir - elif S_ISLNK(s.st_mode): - k = getSymlinkFileKind(path) - - when defined(linux) or defined(macosx) or - defined(bsd) or defined(genode) or defined(nintendoswitch): - case x.d_type - of DT_DIR: k = pcDir - of DT_LNK: - if dirExists(path): k = pcLinkToDir - else: k = pcLinkToFile - of DT_UNKNOWN: - kSetGeneric() - else: # e.g. DT_REG etc - discard # leave it as pcFile - else: # assuming that field `d_type` is not present - kSetGeneric() - - yield (k, y) - -iterator walkDirRec*(dir: string, - yieldFilter = {pcFile}, followFilter = {pcDir}, - relative = false, checkDir = false): string {.tags: [ReadDirEffect].} = - ## Recursively walks over the directory `dir` and yields for each file - ## or directory in `dir`. - ## - ## If ``relative`` is true (default: false) the resulting path is - ## shortened to be relative to ``dir``, otherwise the full path is returned. - ## - ## If `checkDir` is true, `OSError` is raised when `dir` - ## doesn't exist. - ## - ## .. warning:: Modifying the directory structure while the iterator - ## is traversing may result in undefined behavior! - ## - ## Walking is recursive. `followFilter` controls the behaviour of the iterator: - ## - ## ===================== ============================================= - ## yieldFilter meaning - ## ===================== ============================================= - ## ``pcFile`` yield real files (default) - ## ``pcLinkToFile`` yield symbolic links to files - ## ``pcDir`` yield real directories - ## ``pcLinkToDir`` yield symbolic links to directories - ## ===================== ============================================= - ## - ## ===================== ============================================= - ## followFilter meaning - ## ===================== ============================================= - ## ``pcDir`` follow real directories (default) - ## ``pcLinkToDir`` follow symbolic links to directories - ## ===================== ============================================= - ## - ## - ## See also: - ## * `walkPattern iterator`_ - ## * `walkFiles iterator`_ - ## * `walkDirs iterator`_ - ## * `walkDir iterator`_ - - var stack = @[""] - var checkDir = checkDir - while stack.len > 0: - let d = stack.pop() - for k, p in walkDir(dir / d, relative = true, checkDir = checkDir): - let rel = d / p - if k in {pcDir, pcLinkToDir} and k in followFilter: - stack.add rel - if k in yieldFilter: - yield if relative: rel else: dir / rel - checkDir = false - # We only check top-level dir, otherwise if a subdir is invalid (eg. wrong - # permissions), it'll abort iteration and there would be no way to - # continue iteration. - # Future work can provide a way to customize this and do error reporting. - -proc rawRemoveDir(dir: string) {.noWeirdTarget.} = - when defined(windows): - when useWinUnicode: - wrapUnary(res, removeDirectoryW, dir) - else: - var res = removeDirectoryA(dir) - let lastError = osLastError() - if res == 0'i32 and lastError.int32 != 3'i32 and - lastError.int32 != 18'i32 and lastError.int32 != 2'i32: - raiseOSError(lastError, dir) - else: - if rmdir(dir) != 0'i32 and errno != ENOENT: raiseOSError(osLastError(), dir) - -proc removeDir*(dir: string, checkDir = false) {.rtl, extern: "nos$1", tags: [ - WriteDirEffect, ReadDirEffect], benign, noWeirdTarget.} = - ## Removes the directory `dir` including all subdirectories and files - ## in `dir` (recursively). - ## - ## If this fails, `OSError` is raised. This does not fail if the directory never - ## existed in the first place, unless `checkDir` = true. - ## - ## See also: - ## * `tryRemoveFile proc`_ - ## * `removeFile proc`_ - ## * `existsOrCreateDir proc`_ - ## * `createDir proc`_ - ## * `copyDir proc`_ - ## * `copyDirWithPermissions proc`_ - ## * `moveDir proc`_ - for kind, path in walkDir(dir, checkDir = checkDir): - case kind - of pcFile, pcLinkToFile, pcLinkToDir: removeFile(path) - of pcDir: removeDir(path, true) - # for subdirectories there is no benefit in `checkDir = false` - # (unless perhaps for edge case of concurrent processes also deleting - # the same files) - rawRemoveDir(dir) - -proc rawCreateDir(dir: string): bool {.noWeirdTarget.} = - # Try to create one directory (not the whole path). - # returns `true` for success, `false` if the path has previously existed - # - # This is a thin wrapper over mkDir (or alternatives on other systems), - # so in case of a pre-existing path we don't check that it is a directory. - when defined(solaris): - let res = mkdir(dir, 0o777) - if res == 0'i32: - result = true - elif errno in {EEXIST, ENOSYS}: - result = false - else: - raiseOSError(osLastError(), dir) - elif defined(haiku): - let res = mkdir(dir, 0o777) - if res == 0'i32: - result = true - elif errno == EEXIST or errno == EROFS: - result = false - else: - raiseOSError(osLastError(), dir) - elif defined(posix): - let res = mkdir(dir, 0o777) - if res == 0'i32: - result = true - elif errno == EEXIST: - result = false - else: - #echo res - raiseOSError(osLastError(), dir) - else: - when useWinUnicode: - wrapUnary(res, createDirectoryW, dir) - else: - let res = createDirectoryA(dir) - - if res != 0'i32: - result = true - elif getLastError() == 183'i32: - result = false - else: - raiseOSError(osLastError(), dir) - -proc existsOrCreateDir*(dir: string): bool {.rtl, extern: "nos$1", - tags: [WriteDirEffect, ReadDirEffect], noWeirdTarget.} = - ## Checks if a `directory`:idx: `dir` exists, and creates it otherwise. - ## - ## Does not create parent directories (raises `OSError` if parent directories do not exist). - ## Returns `true` if the directory already exists, and `false` otherwise. - ## - ## See also: - ## * `removeDir proc`_ - ## * `createDir proc`_ - ## * `copyDir proc`_ - ## * `copyDirWithPermissions proc`_ - ## * `moveDir proc`_ - result = not rawCreateDir(dir) - if result: - # path already exists - need to check that it is indeed a directory - if not dirExists(dir): - raise newException(IOError, "Failed to create '" & dir & "'") - -proc createDir*(dir: string) {.rtl, extern: "nos$1", - tags: [WriteDirEffect, ReadDirEffect], noWeirdTarget.} = - ## Creates the `directory`:idx: `dir`. - ## - ## The directory may contain several subdirectories that do not exist yet. - ## The full path is created. If this fails, `OSError` is raised. - ## - ## It does **not** fail if the directory already exists because for - ## most usages this does not indicate an error. - ## - ## See also: - ## * `removeDir proc`_ - ## * `existsOrCreateDir proc`_ - ## * `copyDir proc`_ - ## * `copyDirWithPermissions proc`_ - ## * `moveDir proc`_ - if dir == "": - return - var omitNext = isAbsolute(dir) - for p in parentDirs(dir, fromRoot=true): - if omitNext: - omitNext = false - else: - discard existsOrCreateDir(p) - -proc copyDir*(source, dest: string) {.rtl, extern: "nos$1", - tags: [ReadDirEffect, WriteIOEffect, ReadIOEffect], benign, noWeirdTarget.} = - ## Copies a directory from `source` to `dest`. - ## - ## On non-Windows OSes, symlinks are copied as symlinks. On Windows, symlinks - ## are skipped. - ## - ## If this fails, `OSError` is raised. - ## - ## On the Windows platform this proc will copy the attributes from - ## `source` into `dest`. - ## - ## On other platforms created files and directories will inherit the - ## default permissions of a newly created file/directory for the user. - ## Use `copyDirWithPermissions proc`_ - ## to preserve attributes recursively on these platforms. - ## - ## See also: - ## * `copyDirWithPermissions proc`_ - ## * `copyFile proc`_ - ## * `copyFileWithPermissions proc`_ - ## * `removeDir proc`_ - ## * `existsOrCreateDir proc`_ - ## * `createDir proc`_ - ## * `moveDir proc`_ - createDir(dest) - for kind, path in walkDir(source): - var noSource = splitPath(path).tail - if kind == pcDir: - copyDir(path, dest / noSource) - else: - copyFile(path, dest / noSource, {cfSymlinkAsIs}) - -proc moveDir*(source, dest: string) {.tags: [ReadIOEffect, WriteIOEffect], noWeirdTarget.} = - ## Moves a directory from `source` to `dest`. - ## - ## Symlinks are not followed: if `source` contains symlinks, they themself are - ## moved, not their target. - ## - ## If this fails, `OSError` is raised. - ## - ## See also: - ## * `moveFile proc`_ - ## * `copyDir proc`_ - ## * `copyDirWithPermissions proc`_ - ## * `removeDir proc`_ - ## * `existsOrCreateDir proc`_ - ## * `createDir proc`_ - if not tryMoveFSObject(source, dest, isDir = true): - # Fallback to copy & del - copyDir(source, dest) - removeDir(source) - proc createHardlink*(src, dest: string) {.noWeirdTarget.} = ## Create a hard link at `dest` which points to the item specified ## by `src`. @@ -2625,386 +470,32 @@ proc createHardlink*(src, dest: string) {.noWeirdTarget.} = ## See also: ## * `createSymlink proc`_ when defined(windows): - when useWinUnicode: - var wSrc = newWideCString(src) - var wDst = newWideCString(dest) - if createHardLinkW(wDst, wSrc, nil) == 0: - raiseOSError(osLastError(), $(src, dest)) - else: - if createHardLinkA(dest, src, nil) == 0: - raiseOSError(osLastError(), $(src, dest)) + var wSrc = newWideCString(src) + var wDst = newWideCString(dest) + if createHardLinkW(wDst, wSrc, nil) == 0: + raiseOSError(osLastError(), $(src, dest)) else: if link(src, dest) != 0: raiseOSError(osLastError(), $(src, dest)) -proc copyFileWithPermissions*(source, dest: string, - ignorePermissionErrors = true, - options = {cfSymlinkFollow}) {.noWeirdTarget.} = - ## Copies a file from `source` to `dest` preserving file permissions. - ## - ## On non-Windows OSes, `options` specify the way file is copied; by default, - ## if `source` is a symlink, copies the file symlink points to. `options` is - ## ignored on Windows: symlinks are skipped. - ## - ## This is a wrapper proc around `copyFile`_, - ## `getFilePermissions`_ and `setFilePermissions`_ - ## procs on non-Windows platforms. - ## - ## On Windows this proc is just a wrapper for `copyFile proc`_ since - ## that proc already copies attributes. - ## - ## On non-Windows systems permissions are copied after the file itself has - ## been copied, which won't happen atomically and could lead to a race - ## condition. If `ignorePermissionErrors` is true (default), errors while - ## reading/setting file attributes will be ignored, otherwise will raise - ## `OSError`. - ## - ## See also: - ## * `CopyFlag enum`_ - ## * `copyFile proc`_ - ## * `copyDir proc`_ - ## * `tryRemoveFile proc`_ - ## * `removeFile proc`_ - ## * `moveFile proc`_ - ## * `copyDirWithPermissions proc`_ - copyFile(source, dest, options) - when not defined(windows): - try: - setFilePermissions(dest, getFilePermissions(source), followSymlinks = - (cfSymlinkFollow in options)) - except: - if not ignorePermissionErrors: - raise - -proc copyDirWithPermissions*(source, dest: string, - ignorePermissionErrors = true) - {.rtl, extern: "nos$1", tags: [ReadDirEffect, WriteIOEffect, ReadIOEffect], - benign, noWeirdTarget.} = - ## Copies a directory from `source` to `dest` preserving file permissions. - ## - ## On non-Windows OSes, symlinks are copied as symlinks. On Windows, symlinks - ## are skipped. - ## - ## If this fails, `OSError` is raised. This is a wrapper proc around - ## `copyDir`_ and `copyFileWithPermissions`_ procs - ## on non-Windows platforms. - ## - ## On Windows this proc is just a wrapper for `copyDir proc`_ since - ## that proc already copies attributes. - ## - ## On non-Windows systems permissions are copied after the file or directory - ## itself has been copied, which won't happen atomically and could lead to a - ## race condition. If `ignorePermissionErrors` is true (default), errors while - ## reading/setting file attributes will be ignored, otherwise will raise - ## `OSError`. - ## - ## See also: - ## * `copyDir proc`_ - ## * `copyFile proc`_ - ## * `copyFileWithPermissions proc`_ - ## * `removeDir proc`_ - ## * `moveDir proc`_ - ## * `existsOrCreateDir proc`_ - ## * `createDir proc`_ - createDir(dest) - when not defined(windows): - try: - setFilePermissions(dest, getFilePermissions(source), followSymlinks = - false) - except: - if not ignorePermissionErrors: - raise - for kind, path in walkDir(source): - var noSource = splitPath(path).tail - if kind == pcDir: - copyDirWithPermissions(path, dest / noSource, ignorePermissionErrors) - else: - copyFileWithPermissions(path, dest / noSource, ignorePermissionErrors, {cfSymlinkAsIs}) - proc inclFilePermissions*(filename: string, permissions: set[FilePermission]) {. rtl, extern: "nos$1", tags: [ReadDirEffect, WriteDirEffect], noWeirdTarget.} = ## A convenience proc for: - ## - ## .. code-block:: nim + ## ```nim ## setFilePermissions(filename, getFilePermissions(filename)+permissions) + ## ``` setFilePermissions(filename, getFilePermissions(filename)+permissions) proc exclFilePermissions*(filename: string, permissions: set[FilePermission]) {. rtl, extern: "nos$1", tags: [ReadDirEffect, WriteDirEffect], noWeirdTarget.} = ## A convenience proc for: - ## - ## .. code-block:: nim + ## ```nim ## setFilePermissions(filename, getFilePermissions(filename)-permissions) + ## ``` setFilePermissions(filename, getFilePermissions(filename)-permissions) -proc parseCmdLine*(c: string): seq[string] {. - noSideEffect, rtl, extern: "nos$1".} = - ## Splits a `command line`:idx: into several components. - ## - ## **Note**: This proc is only occasionally useful, better use the - ## `parseopt module <parseopt.html>`_. - ## - ## On Windows, it uses the `following parsing rules - ## <http://msdn.microsoft.com/en-us/library/17w5ykft.aspx>`_: - ## - ## * Arguments are delimited by white space, which is either a space or a tab. - ## * The caret character (^) is not recognized as an escape character or - ## delimiter. The character is handled completely by the command-line parser - ## in the operating system before being passed to the argv array in the - ## program. - ## * A string surrounded by double quotation marks ("string") is interpreted - ## as a single argument, regardless of white space contained within. A - ## quoted string can be embedded in an argument. - ## * A double quotation mark preceded by a backslash (\") is interpreted as a - ## literal double quotation mark character ("). - ## * Backslashes are interpreted literally, unless they immediately precede - ## a double quotation mark. - ## * If an even number of backslashes is followed by a double quotation mark, - ## one backslash is placed in the argv array for every pair of backslashes, - ## and the double quotation mark is interpreted as a string delimiter. - ## * If an odd number of backslashes is followed by a double quotation mark, - ## one backslash is placed in the argv array for every pair of backslashes, - ## and the double quotation mark is "escaped" by the remaining backslash, - ## causing a literal double quotation mark (") to be placed in argv. - ## - ## On Posix systems, it uses the following parsing rules: - ## Components are separated by whitespace unless the whitespace - ## occurs within ``"`` or ``'`` quotes. - ## - ## See also: - ## * `parseopt module <parseopt.html>`_ - ## * `paramCount proc`_ - ## * `paramStr proc`_ - ## * `commandLineParams proc`_ - - result = @[] - var i = 0 - var a = "" - while true: - setLen(a, 0) - # eat all delimiting whitespace - while i < c.len and c[i] in {' ', '\t', '\l', '\r'}: inc(i) - if i >= c.len: break - when defined(windows): - # parse a single argument according to the above rules: - var inQuote = false - while i < c.len: - case c[i] - of '\\': - var j = i - while j < c.len and c[j] == '\\': inc(j) - if j < c.len and c[j] == '"': - for k in 1..(j-i) div 2: a.add('\\') - if (j-i) mod 2 == 0: - i = j - else: - a.add('"') - i = j+1 - else: - a.add(c[i]) - inc(i) - of '"': - inc(i) - if not inQuote: inQuote = true - elif i < c.len and c[i] == '"': - a.add(c[i]) - inc(i) - else: - inQuote = false - break - of ' ', '\t': - if not inQuote: break - a.add(c[i]) - inc(i) - else: - a.add(c[i]) - inc(i) - else: - case c[i] - of '\'', '\"': - var delim = c[i] - inc(i) # skip ' or " - while i < c.len and c[i] != delim: - add a, c[i] - inc(i) - if i < c.len: inc(i) - else: - while i < c.len and c[i] > ' ': - add(a, c[i]) - inc(i) - add(result, a) - -when defined(nimdoc): - # Common forward declaration docstring block for parameter retrieval procs. - proc paramCount*(): int {.tags: [ReadIOEffect].} = - ## Returns the number of `command line arguments`:idx: given to the - ## application. - ## - ## Unlike `argc`:idx: in C, if your binary was called without parameters this - ## will return zero. - ## You can query each individual parameter with `paramStr proc`_ - ## or retrieve all of them in one go with `commandLineParams proc`_. - ## - ## **Availability**: When generating a dynamic library (see `--app:lib`) on - ## Posix this proc is not defined. - ## Test for availability using `declared() <system.html#declared,untyped>`_. - ## - ## See also: - ## * `parseopt module <parseopt.html>`_ - ## * `parseCmdLine proc`_ - ## * `paramStr proc`_ - ## * `commandLineParams proc`_ - ## - ## **Examples:** - ## - ## .. code-block:: nim - ## when declared(paramCount): - ## # Use paramCount() here - ## else: - ## # Do something else! - - proc paramStr*(i: int): string {.tags: [ReadIOEffect].} = - ## Returns the `i`-th `command line argument`:idx: given to the application. - ## - ## `i` should be in the range `1..paramCount()`, the `IndexDefect` - ## exception will be raised for invalid values. Instead of iterating - ## over `paramCount()`_ with this proc you can - ## call the convenience `commandLineParams()`_. - ## - ## Similarly to `argv`:idx: in C, - ## it is possible to call `paramStr(0)` but this will return OS specific - ## contents (usually the name of the invoked executable). You should avoid - ## this and call `getAppFilename()`_ instead. - ## - ## **Availability**: When generating a dynamic library (see `--app:lib`) on - ## Posix this proc is not defined. - ## Test for availability using `declared() <system.html#declared,untyped>`_. - ## - ## See also: - ## * `parseopt module <parseopt.html>`_ - ## * `parseCmdLine proc`_ - ## * `paramCount proc`_ - ## * `commandLineParams proc`_ - ## * `getAppFilename proc`_ - ## - ## **Examples:** - ## - ## .. code-block:: nim - ## when declared(paramStr): - ## # Use paramStr() here - ## else: - ## # Do something else! - -elif defined(nimscript): discard -elif defined(nodejs): - type Argv = object of JsRoot - let argv {.importjs: "process.argv".} : Argv - proc len(argv: Argv): int {.importjs: "#.length".} - proc `[]`(argv: Argv, i: int): cstring {.importjs: "#[#]".} - - proc paramCount*(): int {.tags: [ReadDirEffect].} = - result = argv.len - 2 - - proc paramStr*(i: int): string {.tags: [ReadIOEffect].} = - let i = i + 1 - if i < argv.len and i >= 0: - result = $argv[i] - else: - raise newException(IndexDefect, formatErrorIndexBound(i - 1, argv.len - 2)) -elif defined(windows): - # Since we support GUI applications with Nim, we sometimes generate - # a WinMain entry proc. But a WinMain proc has no access to the parsed - # command line arguments. The way to get them differs. Thus we parse them - # ourselves. This has the additional benefit that the program's behaviour - # is always the same -- independent of the used C compiler. - var - ownArgv {.threadvar.}: seq[string] - ownParsedArgv {.threadvar.}: bool - - proc paramCount*(): int {.rtl, extern: "nos$1", tags: [ReadIOEffect].} = - # Docstring in nimdoc block. - if not ownParsedArgv: - ownArgv = parseCmdLine($getCommandLine()) - ownParsedArgv = true - result = ownArgv.len-1 - - proc paramStr*(i: int): string {.rtl, extern: "nos$1", - tags: [ReadIOEffect].} = - # Docstring in nimdoc block. - if not ownParsedArgv: - ownArgv = parseCmdLine($getCommandLine()) - ownParsedArgv = true - if i < ownArgv.len and i >= 0: - result = ownArgv[i] - else: - raise newException(IndexDefect, formatErrorIndexBound(i, ownArgv.len-1)) - -elif defined(genode): - proc paramStr*(i: int): string = - raise newException(OSError, "paramStr is not implemented on Genode") - - proc paramCount*(): int = - raise newException(OSError, "paramCount is not implemented on Genode") -elif weirdTarget or (defined(posix) and appType == "lib"): - proc paramStr*(i: int): string {.tags: [ReadIOEffect].} = - raise newException(OSError, "paramStr is not implemented on current platform") - - proc paramCount*(): int {.tags: [ReadIOEffect].} = - raise newException(OSError, "paramCount is not implemented on current platform") -elif not defined(createNimRtl) and - not(defined(posix) and appType == "lib"): - # On Posix, there is no portable way to get the command line from a DLL. - var - cmdCount {.importc: "cmdCount".}: cint - cmdLine {.importc: "cmdLine".}: cstringArray - - proc paramStr*(i: int): string {.tags: [ReadIOEffect].} = - # Docstring in nimdoc block. - if i < cmdCount and i >= 0: - result = $cmdLine[i] - else: - raise newException(IndexDefect, formatErrorIndexBound(i, cmdCount-1)) - - proc paramCount*(): int {.tags: [ReadIOEffect].} = - # Docstring in nimdoc block. - result = cmdCount-1 - -when declared(paramCount) or defined(nimdoc): - proc commandLineParams*(): seq[string] = - ## Convenience proc which returns the command line parameters. - ## - ## This returns **only** the parameters. If you want to get the application - ## executable filename, call `getAppFilename()`_. - ## - ## **Availability**: On Posix there is no portable way to get the command - ## line from a DLL and thus the proc isn't defined in this environment. You - ## can test for its availability with `declared() - ## <system.html#declared,untyped>`_. - ## - ## See also: - ## * `parseopt module <parseopt.html>`_ - ## * `parseCmdLine proc`_ - ## * `paramCount proc`_ - ## * `paramStr proc`_ - ## * `getAppFilename proc`_ - ## - ## **Examples:** - ## - ## .. code-block:: nim - ## when declared(commandLineParams): - ## # Use commandLineParams() here - ## else: - ## # Do something else! - result = @[] - for i in 1..paramCount(): - result.add(paramStr(i)) -else: - proc commandLineParams*(): seq[string] {.error: - "commandLineParams() unsupported by dynamic libraries".} = - discard - when not weirdTarget and (defined(freebsd) or defined(dragonfly) or defined(netbsd)): proc sysctl(name: ptr cint, namelen: cuint, oldp: pointer, oldplen: ptr csize_t, newp: pointer, newplen: csize_t): cint @@ -3121,7 +612,7 @@ when defined(haiku): B_FIND_PATH_IMAGE_PATH = 1000 proc find_path(codePointer: pointer, baseDirectory: cint, subPath: cstring, - pathBuffer: cstring, bufferSize: csize): int32 + pathBuffer: cstring, bufferSize: csize_t): int32 {.importc, header: "<FindDirectory.h>".} proc getApplHaiku(): string = @@ -3133,10 +624,12 @@ when defined(haiku): else: result = "" -proc getAppFilename*(): string {.rtl, extern: "nos$1", tags: [ReadIOEffect], noWeirdTarget.} = +proc getAppFilename*(): string {.rtl, extern: "nos$1", tags: [ReadIOEffect], noWeirdTarget, raises: [].} = ## Returns the filename of the application's executable. ## This proc will resolve symlinks. ## + ## Returns empty string when name is unavailable + ## ## See also: ## * `getAppDir proc`_ ## * `getCurrentCompilerExe proc`_ @@ -3147,32 +640,18 @@ proc getAppFilename*(): string {.rtl, extern: "nos$1", tags: [ReadIOEffect], noW # /proc/<pid>/path/a.out (complete pathname) when defined(windows): var bufsize = int32(MAX_PATH) - when useWinUnicode: - var buf = newWideCString("", bufsize) - while true: - var L = getModuleFileNameW(0, buf, bufsize) - if L == 0'i32: - result = "" # error! - break - elif L > bufsize: - buf = newWideCString("", L) - bufsize = L - else: - result = buf$L - break - else: - result = newString(bufsize) - while true: - var L = getModuleFileNameA(0, result, bufsize) - if L == 0'i32: - result = "" # error! - break - elif L > bufsize: - result = newString(L) - bufsize = L - else: - setLen(result, L) - break + var buf = newWideCString(bufsize) + while true: + var L = getModuleFileNameW(0, buf, bufsize) + if L == 0'i32: + result = "" # error! + break + elif L > bufsize: + buf = newWideCString(L) + bufsize = L + else: + result = buf$L + break elif defined(macosx): var size = cuint32(0) getExecPath1(nil, size) @@ -3180,24 +659,29 @@ proc getAppFilename*(): string {.rtl, extern: "nos$1", tags: [ReadIOEffect], noW if getExecPath2(result.cstring, size): result = "" # error! if result.len > 0: - result = result.expandFilename + try: + result = result.expandFilename + except OSError: + result = "" else: when defined(linux) or defined(aix): result = getApplAux("/proc/self/exe") elif defined(solaris): result = getApplAux("/proc/" & $getpid() & "/path/a.out") elif defined(genode): - raiseOSError(OSErrorCode(-1), "POSIX command line not supported") + result = "" # Not supported elif defined(freebsd) or defined(dragonfly) or defined(netbsd): result = getApplFreebsd() elif defined(haiku): result = getApplHaiku() elif defined(openbsd): - result = getApplOpenBsd() + result = try: getApplOpenBsd() except OSError: "" + elif defined(nintendoswitch): + result = "" # little heuristic that may work on other POSIX-like systems: if result.len == 0: - result = getApplHeuristic() + result = try: getApplHeuristic() except OSError: "" proc getAppDir*(): string {.rtl, extern: "nos$1", tags: [ReadIOEffect], noWeirdTarget.} = ## Returns the directory of the application's executable. @@ -3208,7 +692,10 @@ proc getAppDir*(): string {.rtl, extern: "nos$1", tags: [ReadIOEffect], noWeirdT proc sleep*(milsecs: int) {.rtl, extern: "nos$1", tags: [TimeEffect], noWeirdTarget.} = ## Sleeps `milsecs` milliseconds. + ## A negative `milsecs` causes sleep to return immediately. when defined(windows): + if milsecs < 0: + return # fixes #23732 winlean.sleep(int32(milsecs)) else: var a, b: Timespec @@ -3259,20 +746,25 @@ type creationTime*: times.Time ## Time file was created. Not supported on all systems! blockSize*: int ## Preferred I/O block size for this object. ## In some filesystems, this may vary from file to file. + isSpecial*: bool ## Is file special? (on Unix some "files" + ## can be special=non-regular like FIFOs, + ## devices); for directories `isSpecial` + ## is always `false`, for symlinks it is + ## the same as for the link's target. template rawToFormalFileInfo(rawInfo, path, formalInfo): untyped = ## Transforms the native file info structure into the one nim uses. ## 'rawInfo' is either a 'BY_HANDLE_FILE_INFORMATION' structure on Windows, ## or a 'Stat' structure on posix when defined(windows): - template merge(a, b): untyped = - int64( + template merge[T](a, b): untyped = + cast[T]( (uint64(cast[uint32](a))) or (uint64(cast[uint32](b)) shl 32) ) formalInfo.id.device = rawInfo.dwVolumeSerialNumber - formalInfo.id.file = merge(rawInfo.nFileIndexLow, rawInfo.nFileIndexHigh) - formalInfo.size = merge(rawInfo.nFileSizeLow, rawInfo.nFileSizeHigh) + formalInfo.id.file = merge[FileId](rawInfo.nFileIndexLow, rawInfo.nFileIndexHigh) + formalInfo.size = merge[BiggestInt](rawInfo.nFileSizeLow, rawInfo.nFileSizeHigh) formalInfo.linkCount = rawInfo.nNumberOfLinks formalInfo.lastAccessTime = fromWinTime(rdFileTime(rawInfo.ftLastAccessTime)) formalInfo.lastWriteTime = fromWinTime(rdFileTime(rawInfo.ftLastWriteTime)) @@ -3319,14 +811,14 @@ template rawToFormalFileInfo(rawInfo, path, formalInfo): untyped = checkAndIncludeMode(S_IWOTH, fpOthersWrite) checkAndIncludeMode(S_IXOTH, fpOthersExec) - formalInfo.kind = + (formalInfo.kind, formalInfo.isSpecial) = if S_ISDIR(rawInfo.st_mode): - pcDir + (pcDir, false) elif S_ISLNK(rawInfo.st_mode): assert(path != "") # symlinks can't occur for file handles getSymlinkFileKind(path) else: - pcFile + (pcFile, not S_ISREG(rawInfo.st_mode)) when defined(js): when not declared(FileHandle): @@ -3379,7 +871,8 @@ proc getFileInfo*(path: string, followSymlink = true): FileInfo {.noWeirdTarget. ## ## When `followSymlink` is true (default), symlinks are followed and the ## information retrieved is information related to the symlink's target. - ## Otherwise, information on the symlink itself is retrieved. + ## Otherwise, information on the symlink itself is retrieved (however, + ## field `isSpecial` is still determined from the target on Unix). ## ## If the information cannot be retrieved, such as when the path doesn't ## exist, or when permission restrictions prevent the program from retrieving @@ -3461,10 +954,7 @@ proc isHidden*(path: string): bool {.noWeirdTarget.} = assert ".foo/".isHidden when defined(windows): - when useWinUnicode: - wrapUnary(attributes, getFileAttributesW, path) - else: - var attributes = getFileAttributesA(path) + wrapUnary(attributes, getFileAttributesW, path) if attributes != -1'i32: result = (attributes and FILE_ATTRIBUTE_HIDDEN) != 0'i32 else: @@ -3500,14 +990,21 @@ proc setLastModificationTime*(file: string, t: times.Time) {.noWeirdTarget.} = discard h.closeHandle if res == 0'i32: raiseOSError(osLastError(), file) -func isValidFilename*(filename: string, maxLen = 259.Positive): bool {.since: (1, 1), deprecated: "Deprecated since v1.5.1".} = - ## Returns true if ``filename`` is valid for crossplatform use. + +func isValidFilename*(filename: string, maxLen = 259.Positive): bool {.since: (1, 1).} = + ## Returns `true` if `filename` is valid for crossplatform use. ## ## This is useful if you want to copy or save files across Windows, Linux, Mac, etc. - ## You can pass full paths as argument too, but func only checks filenames. - ## ## It uses `invalidFilenameChars`, `invalidFilenames` and `maxLen` to verify the specified `filename`. ## + ## See also: + ## + ## * https://docs.microsoft.com/en-us/dotnet/api/system.io.pathtoolongexception + ## * https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file + ## * https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247%28v=vs.85%29.aspx + ## + ## .. warning:: This only checks filenames, not whole paths + ## (because basically you can mount anything as a path on Linux). runnableExamples: assert not isValidFilename(" foo") # Leading white space assert not isValidFilename("foo ") # Trailing white space @@ -3518,9 +1015,6 @@ func isValidFilename*(filename: string, maxLen = 259.Positive): bool {.since: (1 assert not isValidFilename("") # Empty string assert not isValidFilename("foo/") # Filename is empty - # https://docs.microsoft.com/en-us/dotnet/api/system.io.pathtoolongexception - # https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file - # https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247%28v=vs.85%29.aspx result = true let f = filename.splitFile() if unlikely(f.name.len + f.ext.len > maxLen or f.name.len == 0 or @@ -3529,11 +1023,10 @@ func isValidFilename*(filename: string, maxLen = 259.Positive): bool {.since: (1 for invalid in invalidFilenames: if cmpIgnoreCase(f.name, invalid) == 0: return false + # deprecated declarations -when not defined(nimscript): - when not defined(js): # `noNimJs` doesn't work with templates, this should improve. - template existsFile*(args: varargs[untyped]): untyped {.deprecated: "use fileExists".} = - fileExists(args) - template existsDir*(args: varargs[untyped]): untyped {.deprecated: "use dirExists".} = - dirExists(args) - # {.deprecated: [existsFile: fileExists].} # pending bug #14819; this would avoid above mentioned issue +when not weirdTarget: + template existsFile*(args: varargs[untyped]): untyped {.deprecated: "use fileExists".} = + fileExists(args) + template existsDir*(args: varargs[untyped]): untyped {.deprecated: "use dirExists".} = + dirExists(args) diff --git a/lib/pure/osproc.nim b/lib/pure/osproc.nim index a0079cf95..c304ecca6 100644 --- a/lib/pure/osproc.nim +++ b/lib/pure/osproc.nim @@ -18,21 +18,23 @@ include "system/inclrtl" import - strutils, os, strtabs, streams, cpuinfo, streamwrapper, - std/private/since + std/[strutils, os, strtabs, streams, cpuinfo, streamwrapper, + private/since] export quoteShell, quoteShellWindows, quoteShellPosix when defined(windows): - import winlean + import std/winlean else: - import posix + import std/posix when defined(linux) and defined(useClone): - import linux + import std/linux when defined(nimPreviewSlimSystem): import std/[syncio, assertions] + when defined(windows): + import std/widestrs type @@ -74,7 +76,7 @@ type proc execProcess*(command: string, workingDir: string = "", args: openArray[string] = [], env: StringTableRef = nil, options: set[ProcessOption] = {poStdErrToStdOut, poUsePath, poEvalCommand}): - string {.rtl, extern: "nosp$1", + string {.rtl, extern: "nosp$1", raises: [OSError, IOError], tags: [ExecIOEffect, ReadIOEffect, RootEffect].} ## A convenience procedure that executes ``command`` with ``startProcess`` ## and returns its output as a string. @@ -89,12 +91,12 @@ proc execProcess*(command: string, workingDir: string = "", ## * `execCmd proc <#execCmd,string>`_ ## ## Example: - ## - ## .. code-block:: Nim - ## let outp = execProcess("nim", args=["c", "-r", "mytestfile.nim"], options={poUsePath}) - ## let outp_shell = execProcess("nim c -r mytestfile.nim") - ## # Note: outp may have an interleave of text from the nim compile - ## # and any output from mytestfile when it runs + ## ```Nim + ## let outp = execProcess("nim", args=["c", "-r", "mytestfile.nim"], options={poUsePath}) + ## let outp_shell = execProcess("nim c -r mytestfile.nim") + ## # Note: outp may have an interleave of text from the nim compile + ## # and any output from mytestfile when it runs + ## ``` proc execCmd*(command: string): int {.rtl, extern: "nosp$1", tags: [ExecIOEffect, ReadIOEffect, RootEffect].} @@ -111,14 +113,14 @@ proc execCmd*(command: string): int {.rtl, extern: "nosp$1", ## <#execProcess,string,string,openArray[string],StringTableRef,set[ProcessOption]>`_ ## ## Example: - ## - ## .. code-block:: Nim - ## let errC = execCmd("nim c -r mytestfile.nim") + ## ```Nim + ## let errC = execCmd("nim c -r mytestfile.nim") + ## ``` proc startProcess*(command: string, workingDir: string = "", args: openArray[string] = [], env: StringTableRef = nil, options: set[ProcessOption] = {poStdErrToStdOut}): - owned(Process) {.rtl, extern: "nosp$1", + owned(Process) {.rtl, extern: "nosp$1", raises: [OSError, IOError], tags: [ExecIOEffect, ReadEnvEffect, RootEffect].} ## Starts a process. `Command` is the executable file, `workingDir` is the ## process's working directory. If ``workingDir == ""`` the current directory @@ -150,7 +152,7 @@ proc startProcess*(command: string, workingDir: string = "", ## <#execProcess,string,string,openArray[string],StringTableRef,set[ProcessOption]>`_ ## * `execCmd proc <#execCmd,string>`_ -proc close*(p: Process) {.rtl, extern: "nosp$1", tags: [WriteIOEffect].} +proc close*(p: Process) {.rtl, extern: "nosp$1", raises: [IOError, OSError], tags: [WriteIOEffect].} ## When the process has finished executing, cleanup related handles. ## ## .. warning:: If the process has not finished executing, this will forcibly @@ -199,7 +201,7 @@ proc kill*(p: Process) {.rtl, extern: "nosp$1", tags: [].} ## * `terminate proc <#terminate,Process>`_ ## * `posix_utils.sendSignal(pid: Pid, signal: int) <posix_utils.html#sendSignal,Pid,int>`_ -proc running*(p: Process): bool {.rtl, extern: "nosp$1", tags: [].} +proc running*(p: Process): bool {.rtl, extern: "nosp$1", raises: [OSError], tags: [].} ## Returns true if the process `p` is still running. Returns immediately. proc processID*(p: Process): int {.rtl, extern: "nosp$1".} = @@ -210,7 +212,7 @@ proc processID*(p: Process): int {.rtl, extern: "nosp$1".} = return p.id proc waitForExit*(p: Process, timeout: int = -1): int {.rtl, - extern: "nosp$1", tags: [].} + extern: "nosp$1", raises: [OSError, ValueError], tags: [TimeEffect].} ## Waits for the process to finish and returns `p`'s error code. ## ## .. warning:: Be careful when using `waitForExit` for processes created without @@ -218,8 +220,12 @@ proc waitForExit*(p: Process, timeout: int = -1): int {.rtl, ## ## On posix, if the process has exited because of a signal, 128 + signal ## number will be returned. + ## + ## .. warning:: When working with `timeout` parameters, remember that the value is + ## typically expressed in milliseconds, and ensure that the correct unit of time + ## is used to avoid unexpected behavior. -proc peekExitCode*(p: Process): int {.rtl, extern: "nosp$1", tags: [].} +proc peekExitCode*(p: Process): int {.rtl, extern: "nosp$1", raises: [OSError], tags: [].} ## Return `-1` if the process is still running. Otherwise the process' exit code. ## ## On posix, if the process has exited because of a signal, 128 + signal @@ -235,7 +241,7 @@ proc inputStream*(p: Process): Stream {.rtl, extern: "nosp$1", tags: [].} ## * `outputStream proc <#outputStream,Process>`_ ## * `errorStream proc <#errorStream,Process>`_ -proc outputStream*(p: Process): Stream {.rtl, extern: "nosp$1", tags: [].} +proc outputStream*(p: Process): Stream {.rtl, extern: "nosp$1", raises: [IOError, OSError], tags: [].} ## Returns ``p``'s output stream for reading from. ## ## You cannot perform peek/write/setOption operations to this stream. @@ -287,7 +293,7 @@ proc peekableErrorStream*(p: Process): Stream {.rtl, extern: "nosp$1", tags: [], ## * `errorStream proc <#errorStream,Process>`_ ## * `peekableOutputStream proc <#peekableOutputStream,Process>`_ -proc inputHandle*(p: Process): FileHandle {.rtl, extern: "nosp$1", +proc inputHandle*(p: Process): FileHandle {.rtl, raises: [], extern: "nosp$1", tags: [].} = ## Returns ``p``'s input file handle for writing to. ## @@ -300,7 +306,7 @@ proc inputHandle*(p: Process): FileHandle {.rtl, extern: "nosp$1", result = p.inHandle proc outputHandle*(p: Process): FileHandle {.rtl, extern: "nosp$1", - tags: [].} = + raises: [], tags: [].} = ## Returns ``p``'s output file handle for reading from. ## ## .. warning:: The returned `FileHandle` should not be closed manually as @@ -312,7 +318,7 @@ proc outputHandle*(p: Process): FileHandle {.rtl, extern: "nosp$1", result = p.outHandle proc errorHandle*(p: Process): FileHandle {.rtl, extern: "nosp$1", - tags: [].} = + raises: [], tags: [].} = ## Returns ``p``'s error file handle for reading from. ## ## .. warning:: The returned `FileHandle` should not be closed manually as @@ -323,7 +329,7 @@ proc errorHandle*(p: Process): FileHandle {.rtl, extern: "nosp$1", ## * `outputHandle proc <#outputHandle,Process>`_ result = p.errHandle -proc countProcessors*(): int {.rtl, extern: "nosp$1".} = +proc countProcessors*(): int {.rtl, extern: "nosp$1", raises: [].} = ## Returns the number of the processors/cores the machine has. ## Returns 0 if it cannot be detected. ## It is implemented just calling `cpuinfo.countProcessors`. @@ -337,6 +343,7 @@ proc execProcesses*(cmds: openArray[string], beforeRunEvent: proc(idx: int) = nil, afterRunEvent: proc(idx: int, p: Process) = nil): int {.rtl, extern: "nosp$1", + raises: [ValueError, OSError, IOError], tags: [ExecIOEffect, TimeEffect, ReadEnvEffect, RootEffect], effectsOf: [beforeRunEvent, afterRunEvent].} = ## Executes the commands `cmds` in parallel. @@ -450,7 +457,7 @@ proc execProcesses*(cmds: openArray[string], if afterRunEvent != nil: afterRunEvent(i, p) close(p) -iterator lines*(p: Process, keepNewLines = false): string {.since: (1, 3), tags: [ReadIOEffect].} = +iterator lines*(p: Process, keepNewLines = false): string {.since: (1, 3), raises: [OSError, IOError, ValueError], tags: [ReadIOEffect, TimeEffect].} = ## Convenience iterator for working with `startProcess` to read data from a ## background process. ## @@ -458,8 +465,7 @@ iterator lines*(p: Process, keepNewLines = false): string {.since: (1, 3), tags: ## * `readLines proc <#readLines,Process>`_ ## ## Example: - ## - ## .. code-block:: Nim + ## ```Nim ## const opts = {poUsePath, poDaemon, poStdErrToStdOut} ## var ps: seq[Process] ## for prog in ["a", "b"]: # run 2 progs in parallel @@ -471,6 +477,7 @@ iterator lines*(p: Process, keepNewLines = false): string {.since: (1, 3), tags: ## i.inc ## if i > 100: break ## p.close + ## ``` var outp = p.outputStream var line = newStringOfCap(120) while outp.readLine(line): @@ -479,7 +486,8 @@ iterator lines*(p: Process, keepNewLines = false): string {.since: (1, 3), tags: yield line discard waitForExit(p) -proc readLines*(p: Process): (seq[string], int) {.since: (1, 3).} = +proc readLines*(p: Process): (seq[string], int) {.since: (1, 3), + raises: [OSError, IOError, ValueError], tags: [ReadIOEffect, TimeEffect].} = ## Convenience function for working with `startProcess` to read data from a ## background process. ## @@ -487,8 +495,7 @@ proc readLines*(p: Process): (seq[string], int) {.since: (1, 3).} = ## * `lines iterator <#lines.i,Process>`_ ## ## Example: - ## - ## .. code-block:: Nim + ## ```Nim ## const opts = {poUsePath, poDaemon, poStdErrToStdOut} ## var ps: seq[Process] ## for prog in ["a", "b"]: # run 2 progs in parallel @@ -498,6 +505,7 @@ proc readLines*(p: Process): (seq[string], int) {.since: (1, 3).} = ## if exCode != 0: ## for line in lines: echo line ## p.close + ## ``` for line in p.lines: result[0].add(line) result[1] = p.peekExitCode @@ -566,12 +574,8 @@ when defined(windows) and not defined(useNimRtl): if a == 0: raiseOSError(osLastError()) proc newFileHandleStream(handle: Handle): owned FileHandleStream = - new(result) - result.handle = handle - result.closeImpl = hsClose - result.atEndImpl = hsAtEnd - result.readDataImpl = hsReadData - result.writeDataImpl = hsWriteData + result = FileHandleStream(handle: handle, closeImpl: hsClose, atEndImpl: hsAtEnd, + readDataImpl: hsReadData, writeDataImpl: hsWriteData) proc buildCommandLine(a: string, args: openArray[string]): string = result = quoteShell(a) @@ -714,22 +718,15 @@ when defined(windows) and not defined(useNimRtl): if len(workingDir) > 0: wd = workingDir if env != nil: e = buildEnv(env) if poEchoCmd in options: echo($cmdl) - when useWinUnicode: - var tmp = newWideCString(cmdl) - var ee = - if e.str.isNil: newWideCString(cstring(nil)) - else: newWideCString(e.str, e.len) - var wwd = newWideCString(wd) - var flags = NORMAL_PRIORITY_CLASS or CREATE_UNICODE_ENVIRONMENT - if poDaemon in options: flags = flags or CREATE_NO_WINDOW - success = winlean.createProcessW(nil, tmp, nil, nil, 1, flags, - ee, wwd, si, procInfo) - else: - var ee = - if e.str.isNil: cstring(nil) - else: cstring(e.str) - success = winlean.createProcessA(nil, - cmdl, nil, nil, 1, NORMAL_PRIORITY_CLASS, ee, wd, si, procInfo) + var tmp = newWideCString(cmdl) + var ee = + if e.str.isNil: newWideCString(cstring(nil)) + else: newWideCString(e.str, e.len) + var wwd = newWideCString(wd) + var flags = NORMAL_PRIORITY_CLASS or CREATE_UNICODE_ENVIRONMENT + if poDaemon in options: flags = flags or CREATE_NO_WINDOW + success = winlean.createProcessW(nil, tmp, nil, nil, 1, flags, + ee, wwd, si, procInfo) let lastError = osLastError() if poParentStreams notin options: @@ -745,7 +742,7 @@ when defined(windows) and not defined(useNimRtl): const errFileNotFound = 2.int if lastError.int in {errInvalidParameter, errFileNotFound}: raiseOSError(lastError, - "Requested command not found: '$1'. OS error:" % command) + "Requested command not found: '" & command & "'. OS error:") else: raiseOSError(lastError, command) result.fProcessHandle = procInfo.hProcess @@ -870,13 +867,9 @@ when defined(windows) and not defined(useNimRtl): si.hStdError = getStdHandle(STD_ERROR_HANDLE) si.hStdInput = getStdHandle(STD_INPUT_HANDLE) si.hStdOutput = getStdHandle(STD_OUTPUT_HANDLE) - when useWinUnicode: - var c = newWideCString(command) - var res = winlean.createProcessW(nil, c, nil, nil, 0, - NORMAL_PRIORITY_CLASS, nil, nil, si, procInfo) - else: - var res = winlean.createProcessA(nil, command, nil, nil, 0, - NORMAL_PRIORITY_CLASS, nil, nil, si, procInfo) + var c = newWideCString(command) + var res = winlean.createProcessW(nil, c, nil, nil, 0, + NORMAL_PRIORITY_CLASS, nil, nil, si, procInfo) if res == 0: raiseOSError(osLastError()) else: @@ -953,13 +946,13 @@ elif not defined(useNimRtl): not defined(useClone) and not defined(linux) when useProcessAuxSpawn: proc startProcessAuxSpawn(data: StartProcessData): Pid {. - tags: [ExecIOEffect, ReadEnvEffect, ReadDirEffect, RootEffect], gcsafe.} + raises: [OSError], tags: [ExecIOEffect, ReadEnvEffect, ReadDirEffect, RootEffect], gcsafe.} else: proc startProcessAuxFork(data: StartProcessData): Pid {. - tags: [ExecIOEffect, ReadEnvEffect, ReadDirEffect, RootEffect], gcsafe.} + raises: [OSError], tags: [ExecIOEffect, ReadEnvEffect, ReadDirEffect, RootEffect], gcsafe.} {.push stacktrace: off, profiler: off.} proc startProcessAfterFork(data: ptr StartProcessData) {. - tags: [ExecIOEffect, ReadEnvEffect, ReadDirEffect, RootEffect], cdecl, gcsafe.} + raises: [OSError], tags: [ExecIOEffect, ReadEnvEffect, ReadDirEffect, RootEffect], cdecl, gcsafe.} {.pop.} proc startProcess(command: string, workingDir: string = "", @@ -1060,13 +1053,15 @@ elif not defined(useNimRtl): var mask: Sigset chck sigemptyset(mask) chck posix_spawnattr_setsigmask(attr, mask) - if poDaemon in data.options: - chck posix_spawnattr_setpgroup(attr, 0'i32) + when not defined(nuttx): + if poDaemon in data.options: + chck posix_spawnattr_setpgroup(attr, 0'i32) var flags = POSIX_SPAWN_USEVFORK or POSIX_SPAWN_SETSIGMASK - if poDaemon in data.options: - flags = flags or POSIX_SPAWN_SETPGROUP + when not defined(nuttx): + if poDaemon in data.options: + flags = flags or POSIX_SPAWN_SETPGROUP chck posix_spawnattr_setflags(attr, flags) if not (poParentStreams in data.options): @@ -1128,15 +1123,13 @@ elif not defined(useNimRtl): var error: cint let sizeRead = read(data.pErrorPipe[readIdx], addr error, sizeof(error)) if sizeRead == sizeof(error): - raiseOSError(osLastError(), - "Could not find command: '$1'. OS error: $2" % - [$data.sysCommand, $strerror(error)]) + raiseOSError(OSErrorCode(error), + "Could not find command: '" & $data.sysCommand & "'. OS error: " & $strerror(error)) return pid {.push stacktrace: off, profiler: off.} - proc startProcessFail(data: ptr StartProcessData) = - var error: cint = errno + proc startProcessFail(data: ptr StartProcessData, error: cint = errno) = discard write(data.pErrorPipe[writeIdx], addr error, sizeof(error)) exitnow(1) @@ -1173,7 +1166,11 @@ elif not defined(useNimRtl): if (poUsePath in data.options): when defined(uClibc) or defined(linux) or defined(haiku): # uClibc environment (OpenWrt included) doesn't have the full execvpe - let exe = findExe(data.sysCommand) + var exe: string + try: + exe = findExe(data.sysCommand) + except OSError as e: + startProcessFail(data, e.errorCode) discard execve(exe.cstring, data.sysArgs, data.sysEnv) else: # MacOSX doesn't have execvpe, so we need workaround. @@ -1237,7 +1234,7 @@ elif not defined(useNimRtl): when defined(macosx) or defined(freebsd) or defined(netbsd) or defined(openbsd) or defined(dragonfly): - import kqueue + import std/kqueue proc waitForExit(p: Process, timeout: int = -1): int = if p.exitFlag: @@ -1354,119 +1351,68 @@ elif not defined(useNimRtl): p.exitStatus = status break else: - doAssert false, "unreachable!" + raiseAssert "unreachable!" result = exitStatusLikeShell(p.exitStatus) else: - import times - - const - hasThreadSupport = compileOption("threads") and not defined(nimscript) + import std/times except getTime + import std/monotimes proc waitForExit(p: Process, timeout: int = -1): int = - template adjustTimeout(t, s, e: Timespec) = - var diff: int - var b: Timespec - b.tv_sec = e.tv_sec - b.tv_nsec = e.tv_nsec - e.tv_sec = e.tv_sec - s.tv_sec - if e.tv_nsec >= s.tv_nsec: - e.tv_nsec -= s.tv_nsec - else: - if e.tv_sec == posix.Time(0): - raise newException(ValueError, "System time was modified") - else: - diff = s.tv_nsec - e.tv_nsec - e.tv_nsec = 1_000_000_000 - diff - t.tv_sec = t.tv_sec - e.tv_sec - if t.tv_nsec >= e.tv_nsec: - t.tv_nsec -= e.tv_nsec - else: - t.tv_sec = t.tv_sec - posix.Time(1) - diff = e.tv_nsec - t.tv_nsec - t.tv_nsec = 1_000_000_000 - diff - s.tv_sec = b.tv_sec - s.tv_nsec = b.tv_nsec - if p.exitFlag: return exitStatusLikeShell(p.exitStatus) - if timeout == -1: - var status: cint = 1 + if timeout < 0: + # Backwards compatibility with previous verison to + # handle cases where timeout == -1, but extend + # to handle cases where timeout < 0 + var status: cint if waitpid(p.id, status, 0) < 0: raiseOSError(osLastError()) p.exitFlag = true p.exitStatus = status else: - var nmask, omask: Sigset - var sinfo: SigInfo - var stspec, enspec, tmspec: Timespec - - discard sigemptyset(nmask) - discard sigemptyset(omask) - discard sigaddset(nmask, SIGCHLD) - - when hasThreadSupport: - if pthread_sigmask(SIG_BLOCK, nmask, omask) == -1: - raiseOSError(osLastError()) - else: - if sigprocmask(SIG_BLOCK, nmask, omask) == -1: - raiseOSError(osLastError()) - - if timeout >= 1000: - tmspec.tv_sec = posix.Time(timeout div 1_000) - tmspec.tv_nsec = (timeout %% 1_000) * 1_000_000 - else: - tmspec.tv_sec = posix.Time(0) - tmspec.tv_nsec = (timeout * 1_000_000) - - try: - if clock_gettime(CLOCK_REALTIME, stspec) == -1: - raiseOSError(osLastError()) - while true: - let res = sigtimedwait(nmask, sinfo, tmspec) - if res == SIGCHLD: - if sinfo.si_pid == p.id: - var status: cint = 1 - if waitpid(p.id, status, 0) < 0: - raiseOSError(osLastError()) - p.exitFlag = true - p.exitStatus = status - break - else: - # we have SIGCHLD, but not for process we are waiting, - # so we need to adjust timeout value and continue - if clock_gettime(CLOCK_REALTIME, enspec) == -1: - raiseOSError(osLastError()) - adjustTimeout(tmspec, stspec, enspec) - elif res < 0: - let err = osLastError() - if err.cint == EINTR: - # we have received another signal, so we need to - # adjust timeout and continue - if clock_gettime(CLOCK_REALTIME, enspec) == -1: - raiseOSError(osLastError()) - adjustTimeout(tmspec, stspec, enspec) - elif err.cint == EAGAIN: - # timeout expired, so we trying to kill process - if posix.kill(p.id, SIGKILL) == -1: - raiseOSError(osLastError()) - var status: cint = 1 - if waitpid(p.id, status, 0) < 0: - raiseOSError(osLastError()) - p.exitFlag = true - p.exitStatus = status - break - else: - raiseOSError(err) - finally: - when hasThreadSupport: - if pthread_sigmask(SIG_UNBLOCK, nmask, omask) == -1: - raiseOSError(osLastError()) + # Max 50ms delay + const maxWait = initDuration(milliseconds = 50) + let wait = initDuration(milliseconds = timeout) + let deadline = getMonoTime() + wait + # starting 50μs delay + var delay = initDuration(microseconds = 50) + + while true: + var status: cint + let pid = waitpid(p.id, status, WNOHANG) + if p.id == pid : + p.exitFlag = true + p.exitStatus = status + break + elif pid.int == -1: + raiseOsError(osLastError()) else: - if sigprocmask(SIG_UNBLOCK, nmask, omask) == -1: - raiseOSError(osLastError()) + # Continue waiting if needed + if getMonoTime() >= deadline: + # Previous version of `waitForExit` + # foricibly killed the process. + # We keep this so we don't break programs + # that depend on this behavior + if posix.kill(p.id, SIGKILL) < 0: + raiseOSError(osLastError()) + else: + const max = 1_000_000_000 + let + newWait = getMonoTime() + delay + ticks = newWait.ticks() + ns = ticks mod max + secs = ticks div max + var + waitSpec: TimeSpec + unused: Timespec + waitSpec.tv_sec = posix.Time(secs) + waitSpec.tv_nsec = clong ns + discard posix.clock_nanosleep(CLOCK_MONOTONIC, TIMER_ABSTIME, waitSpec, unused) + let remaining = deadline - getMonoTime() + delay = min([delay * 2, remaining, maxWait]) result = exitStatusLikeShell(p.exitStatus) @@ -1576,7 +1522,7 @@ proc execCmdEx*(command: string, options: set[ProcessOption] = { poStdErrToStdOut, poUsePath}, env: StringTableRef = nil, workingDir = "", input = ""): tuple[ output: string, - exitCode: int] {.tags: + exitCode: int] {.raises: [OSError, IOError], tags: [ExecIOEffect, ReadIOEffect, RootEffect], gcsafe.} = ## A convenience proc that runs the `command`, and returns its `output` and ## `exitCode`. `env` and `workingDir` params behave as for `startProcess`. @@ -1593,8 +1539,7 @@ proc execCmdEx*(command: string, options: set[ProcessOption] = { ## <#execProcess,string,string,openArray[string],StringTableRef,set[ProcessOption]>`_ ## ## Example: - ## - ## .. code-block:: Nim + ## ```Nim ## var result = execCmdEx("nim r --hints:off -", options = {}, input = "echo 3*4") ## import std/[strutils, strtabs] ## stripLineEnd(result[0]) ## portable way to remove trailing newline, if any @@ -1603,6 +1548,7 @@ proc execCmdEx*(command: string, options: set[ProcessOption] = { ## when defined(posix): ## assert execCmdEx("echo $FO", env = newStringTable({"FO": "B"})) == ("B\n", 0) ## assert execCmdEx("echo $PWD", workingDir = "/") == ("/\n", 0) + ## ``` when (NimMajor, NimMinor, NimPatch) < (1, 3, 5): doAssert input.len == 0 diff --git a/lib/pure/oswalkdir.nim b/lib/pure/oswalkdir.nim deleted file mode 100644 index 866f9ed70..000000000 --- a/lib/pure/oswalkdir.nim +++ /dev/null @@ -1,13 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## This module is deprecated, `import os` instead. -{.deprecated: "import os.nim instead".} -import os -export PathComponent, walkDir, walkDirRec diff --git a/lib/pure/parsecfg.nim b/lib/pure/parsecfg.nim index 54584a253..8a43daf54 100644 --- a/lib/pure/parsecfg.nim +++ b/lib/pure/parsecfg.nim @@ -45,9 +45,7 @@ runnableExamples("-r:off"): ## Configuration file example ]## -## -## .. code-block:: nim -## +## ```none ## charset = "utf-8" ## [Package] ## name = "hello" @@ -55,6 +53,7 @@ runnableExamples("-r:off"): ## [Author] ## name = "nim-lang" ## website = "nim-lang.org" +## ``` ##[ ## Creating a configuration file @@ -171,7 +170,7 @@ runnableExamples: assert dict.getSectionValue(section4, "does_that_mean_anything_special") == "False" assert dict.getSectionValue(section4, "purpose") == "formatting for readability" -import strutils, lexbase, streams, tables +import std/[strutils, lexbase, streams, tables] import std/private/decode_helpers import std/private/since @@ -453,7 +452,7 @@ proc getKeyValPair(c: var CfgParser, kind: CfgEventKind): CfgEvent = if c.tok.kind == tkSymbol: case kind of cfgOption, cfgKeyValuePair: - result = CfgEvent(kind: kind, key: c.tok.literal, value: "") + result = CfgEvent(kind: kind, key: c.tok.literal.move, value: "") else: discard rawGetTok(c, c.tok) if c.tok.kind in {tkEquals, tkColon}: @@ -482,7 +481,7 @@ proc next*(c: var CfgParser): CfgEvent {.rtl, extern: "npc$1".} = of tkBracketLe: rawGetTok(c, c.tok) if c.tok.kind == tkSymbol: - result = CfgEvent(kind: cfgSectionStart, section: c.tok.literal) + result = CfgEvent(kind: cfgSectionStart, section: c.tok.literal.move) else: result = CfgEvent(kind: cfgError, msg: errorStr(c, "symbol expected, but found: " & c.tok.literal)) diff --git a/lib/pure/parsecsv.nim b/lib/pure/parsecsv.nim index a8d1cfaab..c7bf0c9c1 100644 --- a/lib/pure/parsecsv.nim +++ b/lib/pure/parsecsv.nim @@ -13,7 +13,7 @@ ## Basic usage ## =========== ## -## .. code-block:: nim +## ```nim ## import std/parsecsv ## from std/os import paramStr ## from std/streams import newFileStream @@ -29,11 +29,12 @@ ## for val in items(x.row): ## echo "##", val, "##" ## close(x) +## ``` ## ## For CSV files with a header row, the header can be read and then used as a ## reference for item access with `rowEntry <#rowEntry,CsvParser,string>`_: ## -## .. code-block:: nim +## ```nim ## import std/parsecsv ## ## # Prepare a file @@ -52,6 +53,7 @@ ## for col in items(p.headers): ## echo "##", col, ":", p.rowEntry(col), "##" ## p.close() +## ``` ## ## See also ## ======== @@ -65,7 +67,7 @@ ## * `parsesql module <parsesql.html>`_ for a SQL parser ## * `other parsers <lib.html#pure-libraries-parsers>`_ for other parsers -import lexbase, streams +import std/[lexbase, streams] when defined(nimPreviewSlimSystem): import std/syncio @@ -347,7 +349,7 @@ proc rowEntry*(self: var CsvParser, entry: string): var string = raise newException(KeyError, "Entry `" & entry & "` doesn't exist") when not defined(testing) and isMainModule: - import os + import std/os var s = newFileStream(paramStr(1), fmRead) if s == nil: quit("cannot open the file" & paramStr(1)) var x: CsvParser diff --git a/lib/pure/parsejson.nim b/lib/pure/parsejson.nim index fcbcf8e36..9292a8596 100644 --- a/lib/pure/parsejson.nim +++ b/lib/pure/parsejson.nim @@ -11,7 +11,7 @@ ## and exported by the `json` standard library ## module, but can also be used in its own right. -import strutils, lexbase, streams, unicode +import std/[strutils, lexbase, streams, unicode] import std/private/decode_helpers when defined(nimPreviewSlimSystem): diff --git a/lib/pure/parseopt.nim b/lib/pure/parseopt.nim index f8d03e092..03f151b66 100644 --- a/lib/pure/parseopt.nim +++ b/lib/pure/parseopt.nim @@ -48,7 +48,7 @@ ## ## Here is an example: ## -## .. code-block:: +## ```Nim ## import std/parseopt ## ## var p = initOptParser("-ab -e:5 --foo --bar=20 file.txt") @@ -71,10 +71,34 @@ ## # Option: foo ## # Option and value: bar, 20 ## # Argument: file.txt +## ``` ## ## The `getopt iterator<#getopt.i,OptParser>`_, which is provided for ## convenience, can be used to iterate through all command line options as well. ## +## To set a default value for a variable assigned through `getopt` and accept arguments from the cmd line. +## Assign the default value to a variable before parsing. +## Then set the variable to the new value while parsing. +## +## Here is an example: +## +## ```Nim +## import std/parseopt +## +## var varName: string = "defaultValue" +## +## for kind, key, val in getopt(): +## case kind +## of cmdArgument: +## discard +## of cmdLongOption, cmdShortOption: +## case key: +## of "varName": # --varName:<value> in the console when executing +## varName = val # do input sanitization in production systems +## of cmdEnd: +## discard +## ``` +## ## `shortNoVal` and `longNoVal` ## ============================ ## @@ -98,7 +122,7 @@ ## `shortNoVal` and `longNoVal`, which is the default, and providing ## arguments for those two parameters: ## -## .. code-block:: +## ```Nim ## import std/parseopt ## ## proc printToken(kind: CmdLineKind, key: string, val: string) = @@ -132,6 +156,7 @@ ## # Output: ## # Option and value: j, 4 ## # Option and value: first, bar +## ``` ## ## See also ## ======== @@ -151,7 +176,8 @@ include "system/inclrtl" -import os +import std/strutils +import std/os type CmdLineKind* = enum ## The detected command line token. @@ -194,34 +220,6 @@ proc parseWord(s: string, i: int, w: var string, proc initOptParser*(cmdline: seq[string], shortNoVal: set[char] = {}, longNoVal: seq[string] = @[]; - allowWhitespaceAfterColon = true): OptParser - -proc initOptParser*(cmdline = "", shortNoVal: set[char] = {}, - longNoVal: seq[string] = @[]; - allowWhitespaceAfterColon = true): OptParser = - ## Initializes the command line parser. - ## - ## If `cmdline == ""`, the real command line as provided by the - ## `os` module is retrieved instead if it is available. If the - ## command line is not available, a `ValueError` will be raised. - ## - ## `shortNoVal` and `longNoVal` are used to specify which options - ## do not take values. See the `documentation about these - ## parameters<#nimshortnoval-and-nimlongnoval>`_ for more information on - ## how this affects parsing. - ## - ## See also: - ## * `getopt iterator<#getopt.i,OptParser>`_ - runnableExamples: - var p = initOptParser() - p = initOptParser("--left --debug:3 -l -r:2") - p = initOptParser("--left --debug:3 -l -r:2", - shortNoVal = {'l'}, longNoVal = @["left"]) - - initOptParser(parseCmdLine(cmdline), shortNoVal, longNoVal, allowWhitespaceAfterColon) - -proc initOptParser*(cmdline: seq[string], shortNoVal: set[char] = {}, - longNoVal: seq[string] = @[]; allowWhitespaceAfterColon = true): OptParser = ## Initializes the command line parser. ## @@ -252,18 +250,55 @@ proc initOptParser*(cmdline: seq[string], shortNoVal: set[char] = {}, result.cmds[i] = cmdline[i] else: when declared(paramCount): - result.cmds = newSeq[string](paramCount()) - for i in countup(1, paramCount()): - result.cmds[i-1] = paramStr(i) + when defined(nimscript): + var ctr = 0 + var firstNimsFound = false + for i in countup(0, paramCount()): + if firstNimsFound: + result.cmds[ctr] = paramStr(i) + inc ctr, 1 + if paramStr(i).endsWith(".nims") and not firstNimsFound: + firstNimsFound = true + result.cmds = newSeq[string](paramCount()-i) + else: + result.cmds = newSeq[string](paramCount()) + for i in countup(1, paramCount()): + result.cmds[i-1] = paramStr(i) else: # we cannot provide this for NimRtl creation on Posix, because we can't # access the command line arguments then! - doAssert false, "empty command line given but" & + raiseAssert "empty command line given but" & " real command line is not accessible" result.kind = cmdEnd result.key = "" result.val = "" +proc initOptParser*(cmdline = "", shortNoVal: set[char] = {}, + longNoVal: seq[string] = @[]; + allowWhitespaceAfterColon = true): OptParser = + ## Initializes the command line parser. + ## + ## If `cmdline == ""`, the real command line as provided by the + ## `os` module is retrieved instead if it is available. If the + ## command line is not available, a `ValueError` will be raised. + ## + ## `shortNoVal` and `longNoVal` are used to specify which options + ## do not take values. See the `documentation about these + ## parameters<#nimshortnoval-and-nimlongnoval>`_ for more information on + ## how this affects parsing. + ## + ## This does not provide a way of passing default values to arguments. + ## + ## See also: + ## * `getopt iterator<#getopt.i,OptParser>`_ + runnableExamples: + var p = initOptParser() + p = initOptParser("--left --debug:3 -l -r:2") + p = initOptParser("--left --debug:3 -l -r:2", + shortNoVal = {'l'}, longNoVal = @["left"]) + + initOptParser(parseCmdLine(cmdline), shortNoVal, longNoVal, allowWhitespaceAfterColon) + proc handleShortOption(p: var OptParser; cmd: string) = var i = p.pos p.kind = cmdShortOption @@ -356,7 +391,7 @@ proc next*(p: var OptParser) {.rtl, extern: "npo$1".} = handleShortOption(p, p.cmds[p.idx]) else: p.kind = cmdArgument - p.key = p.cmds[p.idx] + p.key = p.cmds[p.idx] inc p.idx p.pos = 0 @@ -368,15 +403,14 @@ when declared(quoteShellCommand): ## * `remainingArgs proc<#remainingArgs,OptParser>`_ ## ## **Examples:** - ## - ## .. code-block:: + ## ```Nim ## var p = initOptParser("--left -r:2 -- foo.txt bar.txt") ## while true: ## p.next() ## if p.kind == cmdLongOption and p.key == "": # Look for "--" ## break - ## else: continue ## doAssert p.cmdLineRest == "foo.txt bar.txt" + ## ``` result = p.cmds[p.idx .. ^1].quoteShellCommand proc remainingArgs*(p: OptParser): seq[string] {.rtl, extern: "npo$1".} = @@ -386,15 +420,14 @@ proc remainingArgs*(p: OptParser): seq[string] {.rtl, extern: "npo$1".} = ## * `cmdLineRest proc<#cmdLineRest,OptParser>`_ ## ## **Examples:** - ## - ## .. code-block:: + ## ```Nim ## var p = initOptParser("--left -r:2 -- foo.txt bar.txt") ## while true: ## p.next() ## if p.kind == cmdLongOption and p.key == "": # Look for "--" ## break - ## else: continue ## doAssert p.remainingArgs == @["foo.txt", "bar.txt"] + ## ``` result = @[] for i in p.idx..<p.cmds.len: result.add p.cmds[i] @@ -403,14 +436,15 @@ iterator getopt*(p: var OptParser): tuple[kind: CmdLineKind, key, ## Convenience iterator for iterating over the given ## `OptParser<#OptParser>`_. ## - ## There is no need to check for `cmdEnd` while iterating. + ## There is no need to check for `cmdEnd` while iterating. If using `getopt` + ## with case switching, checking for `cmdEnd` is required. ## ## See also: ## * `initOptParser proc<#initOptParser,string,set[char],seq[string]>`_ ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## # these are placeholders, of course ## proc writeHelp() = discard ## proc writeVersion() = discard @@ -430,6 +464,7 @@ iterator getopt*(p: var OptParser): tuple[kind: CmdLineKind, key, ## if filename == "": ## # no filename has been given, so we show the help ## writeHelp() + ## ``` p.pos = 0 p.idx = 0 while true: @@ -451,15 +486,15 @@ iterator getopt*(cmdline: seq[string] = @[], ## parameters<#nimshortnoval-and-nimlongnoval>`_ for more information on ## how this affects parsing. ## - ## There is no need to check for `cmdEnd` while iterating. + ## There is no need to check for `cmdEnd` while iterating. If using `getopt` + ## with case switching, checking for `cmdEnd` is required. ## ## See also: ## * `initOptParser proc<#initOptParser,seq[string],set[char],seq[string]>`_ ## ## **Examples:** ## - ## .. code-block:: - ## + ## ```Nim ## # these are placeholders, of course ## proc writeHelp() = discard ## proc writeVersion() = discard @@ -479,6 +514,7 @@ iterator getopt*(cmdline: seq[string] = @[], ## if filename == "": ## # no filename has been written, so we show the help ## writeHelp() + ## ``` var p = initOptParser(cmdline, shortNoVal = shortNoVal, longNoVal = longNoVal) while true: diff --git a/lib/pure/parsesql.nim b/lib/pure/parsesql.nim index 3af840e29..a7c938d01 100644 --- a/lib/pure/parsesql.nim +++ b/lib/pure/parsesql.nim @@ -12,7 +12,7 @@ ## ## Unstable API. -import strutils, lexbase +import std/[strutils, lexbase] import std/private/decode_helpers when defined(nimPreviewSlimSystem): @@ -60,7 +60,7 @@ const reservedKeywords = @[ # statements - "select", "from", "where", "group", "limit", "having", + "select", "from", "where", "group", "limit", "offset", "having", # functions "count", ] @@ -509,6 +509,7 @@ type nkFromItemPair, nkGroup, nkLimit, + nkOffset, nkHaving, nkOrder, nkJoin, @@ -1126,6 +1127,11 @@ proc parseSelect(p: var SqlParser): SqlNode = var l = newNode(nkLimit) l.add(parseExpr(p)) result.add(l) + if isKeyw(p, "offset"): + getTok(p) + var o = newNode(nkOffset) + o.add(parseExpr(p)) + result.add(o) proc parseStmt(p: var SqlParser; parent: SqlNode) = if isKeyw(p, "create"): @@ -1388,6 +1394,9 @@ proc ra(n: SqlNode, s: var SqlWriter) = of nkLimit: s.addKeyw("limit") s.addMulti(n) + of nkOffset: + s.addKeyw("offset") + s.addMulti(n) of nkHaving: s.addKeyw("having") s.addMulti(n) @@ -1482,7 +1491,7 @@ proc treeRepr*(s: SqlNode): string = result = newStringOfCap(128) treeReprAux(s, 0, result) -import streams +import std/streams proc open(L: var SqlLexer, input: Stream, filename: string) = lexbase.open(L, input) diff --git a/lib/pure/parseutils.nim b/lib/pure/parseutils.nim index 64949428f..2ca255fa0 100644 --- a/lib/pure/parseutils.nim +++ b/lib/pure/parseutils.nim @@ -12,29 +12,28 @@ ## ## To unpack raw bytes look at the `streams <streams.html>`_ module. ## -## .. code-block:: nim -## :test: +## ```nim test +## let logs = @["2019-01-10: OK_", "2019-01-11: FAIL_", "2019-01: aaaa"] +## var outp: seq[string] ## -## let logs = @["2019-01-10: OK_", "2019-01-11: FAIL_", "2019-01: aaaa"] -## var outp: seq[string] +## for log in logs: +## var res: string +## if parseUntil(log, res, ':') == 10: # YYYY-MM-DD == 10 +## outp.add(res & " - " & captureBetween(log, ' ', '_')) +## doAssert outp == @["2019-01-10 - OK", "2019-01-11 - FAIL"] +## ``` ## -## for log in logs: -## var res: string -## if parseUntil(log, res, ':') == 10: # YYYY-MM-DD == 10 -## outp.add(res & " - " & captureBetween(log, ' ', '_')) -## doAssert outp == @["2019-01-10 - OK", "2019-01-11 - FAIL"] +## ```nim test +## from std/strutils import Digits, parseInt ## -## .. code-block:: nim -## :test: -## from std/strutils import Digits, parseInt -## -## let -## input1 = "2019 school start" -## input2 = "3 years back" -## startYear = input1[0 .. skipWhile(input1, Digits)-1] # 2019 -## yearsBack = input2[0 .. skipWhile(input2, Digits)-1] # 3 -## examYear = parseInt(startYear) + parseInt(yearsBack) -## doAssert "Examination is in " & $examYear == "Examination is in 2022" +## let +## input1 = "2019 school start" +## input2 = "3 years back" +## startYear = input1[0 .. skipWhile(input1, Digits)-1] # 2019 +## yearsBack = input2[0 .. skipWhile(input2, Digits)-1] # 3 +## examYear = parseInt(startYear) + parseInt(yearsBack) +## doAssert "Examination is in " & $examYear == "Examination is in 2022" +## ``` ## ## **See also:** ## * `strutils module<strutils.html>`_ for combined and identical parsing proc's @@ -50,6 +49,8 @@ include "system/inclrtl" +template toOa(s: string): openArray[char] = openArray[char](s) + const Whitespace = {' ', '\t', '\v', '\r', '\l', '\f'} IdentChars = {'a'..'z', 'A'..'Z', '0'..'9', '_'} @@ -59,8 +60,7 @@ const proc toLower(c: char): char {.inline.} = result = if c in {'A'..'Z'}: chr(ord(c)-ord('A')+ord('a')) else: c -proc parseBin*[T: SomeInteger](s: string, number: var T, start = 0, - maxLen = 0): int {.noSideEffect.} = +proc parseBin*[T: SomeInteger](s: openArray[char], number: var T, maxLen = 0): int {.noSideEffect.} = ## Parses a binary number and stores its value in ``number``. ## ## Returns the number of the parsed characters or 0 in case of an error. @@ -89,7 +89,7 @@ proc parseBin*[T: SomeInteger](s: string, number: var T, start = 0, var num64: int64 doAssert parseBin("0100111001101001111011010100111001101001", num64) == 40 doAssert num64 == 336784608873 - var i = start + var i = 0 var output = T(0) var foundDigit = false let last = min(s.len, if maxLen == 0: s.len else: i + maxLen) @@ -104,10 +104,9 @@ proc parseBin*[T: SomeInteger](s: string, number: var T, start = 0, inc(i) if foundDigit: number = output - result = i - start + result = i -proc parseOct*[T: SomeInteger](s: string, number: var T, start = 0, - maxLen = 0): int {.noSideEffect.} = +proc parseOct*[T: SomeInteger](s: openArray[char], number: var T, maxLen = 0): int {.noSideEffect.} = ## Parses an octal number and stores its value in ``number``. ## ## Returns the number of the parsed characters or 0 in case of an error. @@ -136,7 +135,7 @@ proc parseOct*[T: SomeInteger](s: string, number: var T, start = 0, var num64: int64 doAssert parseOct("2346475523464755", num64) == 16 doAssert num64 == 86216859871725 - var i = start + var i = 0 var output = T(0) var foundDigit = false let last = min(s.len, if maxLen == 0: s.len else: i + maxLen) @@ -151,10 +150,9 @@ proc parseOct*[T: SomeInteger](s: string, number: var T, start = 0, inc(i) if foundDigit: number = output - result = i - start + result = i -proc parseHex*[T: SomeInteger](s: string, number: var T, start = 0, - maxLen = 0): int {.noSideEffect.} = +proc parseHex*[T: SomeInteger](s: openArray[char], number: var T, maxLen = 0): int {.noSideEffect.} = ## Parses a hexadecimal number and stores its value in ``number``. ## ## Returns the number of the parsed characters or 0 in case of an error. @@ -184,7 +182,7 @@ proc parseHex*[T: SomeInteger](s: string, number: var T, start = 0, var num64: int64 doAssert parseHex("4E69ED4E69ED", num64) == 12 doAssert num64 == 86216859871725 - var i = start + var i = 0 var output = T(0) var foundDigit = false let last = min(s.len, if maxLen == 0: s.len else: i + maxLen) @@ -206,9 +204,9 @@ proc parseHex*[T: SomeInteger](s: string, number: var T, start = 0, inc(i) if foundDigit: number = output - result = i - start + result = i -proc parseIdent*(s: string, ident: var string, start = 0): int = +proc parseIdent*(s: openArray[char], ident: var string): int = ## Parses an identifier and stores it in ``ident``. Returns ## the number of the parsed characters or 0 in case of an error. ## If error, the value of `ident` is not changed. @@ -220,14 +218,14 @@ proc parseIdent*(s: string, ident: var string, start = 0): int = doAssert res == "ello" doAssert parseIdent("Hello World", res, 6) == 5 doAssert res == "World" - var i = start + var i = 0 if i < s.len and s[i] in IdentStartChars: inc(i) while i < s.len and s[i] in IdentChars: inc(i) - ident = substr(s, start, i-1) - result = i-start + ident = substr(s.toOpenArray(0, i-1)) + result = i -proc parseIdent*(s: string, start = 0): string = +proc parseIdent*(s: openArray[char]): string = ## Parses an identifier and returns it or an empty string in ## case of an error. runnableExamples: @@ -236,13 +234,13 @@ proc parseIdent*(s: string, start = 0): string = doAssert parseIdent("Hello World", 5) == "" doAssert parseIdent("Hello World", 6) == "World" result = "" - var i = start + var i = 0 if i < s.len and s[i] in IdentStartChars: inc(i) while i < s.len and s[i] in IdentChars: inc(i) - result = substr(s, start, i-1) + result = substr(s.toOpenArray(0, i - 1)) -proc parseChar*(s: string, c: var char, start = 0): int = +proc parseChar*(s: openArray[char], c: var char): int = ## Parses a single character, stores it in `c` and returns 1. ## In case of error (if start >= s.len) it returns 0 ## and the value of `c` is unchanged. @@ -252,11 +250,11 @@ proc parseChar*(s: string, c: var char, start = 0): int = doAssert c == '\0' doAssert "nim".parseChar(c, 0) == 1 doAssert c == 'n' - if start < s.len: - c = s[start] + if s.len > 0: + c = s[0] result = 1 -proc skipWhitespace*(s: string, start = 0): int {.inline.} = +proc skipWhitespace*(s: openArray[char]): int {.inline.} = ## Skips the whitespace starting at ``s[start]``. Returns the number of ## skipped characters. runnableExamples: @@ -265,9 +263,9 @@ proc skipWhitespace*(s: string, start = 0): int {.inline.} = doAssert skipWhitespace("Hello World", 5) == 1 doAssert skipWhitespace("Hello World", 5) == 2 result = 0 - while start+result < s.len and s[start+result] in Whitespace: inc(result) + while result < s.len and s[result] in Whitespace: inc(result) -proc skip*(s, token: string, start = 0): int {.inline.} = +proc skip*(s, token: openArray[char]): int {.inline.} = ## Skips the `token` starting at ``s[start]``. Returns the length of `token` ## or 0 if there was no `token` at ``s[start]``. runnableExamples: @@ -277,22 +275,22 @@ proc skip*(s, token: string, start = 0): int {.inline.} = doAssert skip("CAPlow", "CAP", 0) == 3 doAssert skip("CAPlow", "cap", 0) == 0 result = 0 - while start+result < s.len and result < token.len and - s[result+start] == token[result]: + while result < s.len and result < token.len and + s[result] == token[result]: inc(result) if result != token.len: result = 0 -proc skipIgnoreCase*(s, token: string, start = 0): int = +proc skipIgnoreCase*(s, token: openArray[char]): int = ## Same as `skip` but case is ignored for token matching. runnableExamples: doAssert skipIgnoreCase("CAPlow", "CAP", 0) == 3 doAssert skipIgnoreCase("CAPlow", "cap", 0) == 3 result = 0 - while start+result < s.len and result < token.len and - toLower(s[result+start]) == toLower(token[result]): inc(result) + while result < s.len and result < token.len and + toLower(s[result]) == toLower(token[result]): inc(result) if result != token.len: result = 0 -proc skipUntil*(s: string, until: set[char], start = 0): int {.inline.} = +proc skipUntil*(s: openArray[char], until: set[char]): int {.inline.} = ## Skips all characters until one char from the set `until` is found ## or the end is reached. ## Returns number of characters skipped. @@ -301,9 +299,9 @@ proc skipUntil*(s: string, until: set[char], start = 0): int {.inline.} = doAssert skipUntil("Hello World", {'W'}, 0) == 6 doAssert skipUntil("Hello World", {'W', 'd'}, 0) == 6 result = 0 - while start+result < s.len and s[result+start] notin until: inc(result) + while result < s.len and s[result] notin until: inc(result) -proc skipUntil*(s: string, until: char, start = 0): int {.inline.} = +proc skipUntil*(s: openArray[char], until: char): int {.inline.} = ## Skips all characters until the char `until` is found ## or the end is reached. ## Returns number of characters skipped. @@ -313,24 +311,23 @@ proc skipUntil*(s: string, until: char, start = 0): int {.inline.} = doAssert skipUntil("Hello World", 'W', 0) == 6 doAssert skipUntil("Hello World", 'w', 0) == 11 result = 0 - while start+result < s.len and s[result+start] != until: inc(result) + while result < s.len and s[result] != until: inc(result) -proc skipWhile*(s: string, toSkip: set[char], start = 0): int {.inline.} = - ## Skips all characters while one char from the set `token` is found. +proc skipWhile*(s: openArray[char], toSkip: set[char]): int {.inline.} = + ## Skips all characters while one char from the set `toSkip` is found. ## Returns number of characters skipped. runnableExamples: doAssert skipWhile("Hello World", {'H', 'e'}) == 2 doAssert skipWhile("Hello World", {'e'}) == 0 doAssert skipWhile("Hello World", {'W', 'o', 'r'}, 6) == 3 result = 0 - while start+result < s.len and s[result+start] in toSkip: inc(result) + while result < s.len and s[result] in toSkip: inc(result) -proc fastSubstr(s: string; token: var string; start, length: int) = +proc fastSubstr(s: openArray[char]; token: var string; length: int) = token.setLen length - for i in 0 ..< length: token[i] = s[i+start] + for i in 0 ..< length: token[i] = s[i] -proc parseUntil*(s: string, token: var string, until: set[char], - start = 0): int {.inline.} = +proc parseUntil*(s: openArray[char], token: var string, until: set[char]): int {.inline.} = ## Parses a token and stores it in ``token``. Returns ## the number of the parsed characters or 0 in case of an error. A token ## consists of the characters notin `until`. @@ -342,14 +339,13 @@ proc parseUntil*(s: string, token: var string, until: set[char], doAssert myToken == "Hello " doAssert parseUntil("Hello World", myToken, {'W', 'r'}, 3) == 3 doAssert myToken == "lo " - var i = start + var i = 0 while i < s.len and s[i] notin until: inc(i) - result = i-start - fastSubstr(s, token, start, result) + result = i + fastSubstr(s, token, result) #token = substr(s, start, i-1) -proc parseUntil*(s: string, token: var string, until: char, - start = 0): int {.inline.} = +proc parseUntil*(s: openArray[char], token: var string, until: char): int {.inline.} = ## Parses a token and stores it in ``token``. Returns ## the number of the parsed characters or 0 in case of an error. A token ## consists of any character that is not the `until` character. @@ -361,14 +357,13 @@ proc parseUntil*(s: string, token: var string, until: char, doAssert myToken == "Hell" doAssert parseUntil("Hello World", myToken, 'o', 2) == 2 doAssert myToken == "ll" - var i = start + var i = 0 while i < s.len and s[i] != until: inc(i) - result = i-start - fastSubstr(s, token, start, result) + result = i + fastSubstr(s, token, result) #token = substr(s, start, i-1) -proc parseUntil*(s: string, token: var string, until: string, - start = 0): int {.inline.} = +proc parseUntil*(s: openArray[char], token: var string, until: string): int {.inline.} = ## Parses a token and stores it in ``token``. Returns ## the number of the parsed characters or 0 in case of an error. A token ## consists of any character that comes before the `until` token. @@ -382,7 +377,7 @@ proc parseUntil*(s: string, token: var string, until: string, if until.len == 0: token.setLen(0) return 0 - var i = start + var i = 0 while i < s.len: if until.len > 0 and s[i] == until[0]: var u = 1 @@ -390,12 +385,11 @@ proc parseUntil*(s: string, token: var string, until: string, inc u if u >= until.len: break inc(i) - result = i-start - fastSubstr(s, token, start, result) + result = i + fastSubstr(s, token, result) #token = substr(s, start, i-1) -proc parseWhile*(s: string, token: var string, validChars: set[char], - start = 0): int {.inline.} = +proc parseWhile*(s: openArray[char], token: var string, validChars: set[char]): int {.inline.} = ## Parses a token and stores it in ``token``. Returns ## the number of the parsed characters or 0 in case of an error. A token ## consists of the characters in `validChars`. @@ -405,22 +399,22 @@ proc parseWhile*(s: string, token: var string, validChars: set[char], doAssert myToken.len() == 0 doAssert parseWhile("Hello World", myToken, {'W', 'o', 'r'}, 6) == 3 doAssert myToken == "Wor" - var i = start + var i = 0 while i < s.len and s[i] in validChars: inc(i) - result = i-start - fastSubstr(s, token, start, result) + result = i + fastSubstr(s, token, result) #token = substr(s, start, i-1) -proc captureBetween*(s: string, first: char, second = '\0', start = 0): string = +proc captureBetween*(s: openArray[char], first: char, second = '\0'): string = ## Finds the first occurrence of ``first``, then returns everything from there ## up to ``second`` (if ``second`` is '\0', then ``first`` is used). runnableExamples: doAssert captureBetween("Hello World", 'e') == "llo World" doAssert captureBetween("Hello World", 'e', 'r') == "llo Wo" - doAssert captureBetween("Hello World", 'l', start = 6) == "d" - var i = skipUntil(s, first, start)+1+start + doAssert captureBetween("Hello World".toOpenArray(6, "Hello World".high), 'l') == "d" + var i = skipUntil(s, first) + 1 result = "" - discard s.parseUntil(result, if second == '\0': first else: second, i) + discard parseUntil(s.toOpenArray(i, s.high), result, if second == '\0': first else: second) proc integerOutOfRangeError() {.noinline.} = raise newException(ValueError, "Parsed integer outside of valid range") @@ -429,10 +423,10 @@ proc integerOutOfRangeError() {.noinline.} = when defined(js): {.push overflowChecks: off.} -proc rawParseInt(s: string, b: var BiggestInt, start = 0): int = +proc rawParseInt(s: openArray[char], b: var BiggestInt): int = var sign: BiggestInt = -1 - i = start + i = 0 if i < s.len: if s[i] == '+': inc(i) elif s[i] == '-': @@ -452,47 +446,47 @@ proc rawParseInt(s: string, b: var BiggestInt, start = 0): int = integerOutOfRangeError() else: b = b * sign - result = i - start + result = i when defined(js): {.pop.} # overflowChecks: off -proc parseBiggestInt*(s: string, number: var BiggestInt, start = 0): int {. +proc parseBiggestInt*(s: openArray[char], number: var BiggestInt): int {. rtl, extern: "npuParseBiggestInt", noSideEffect, raises: [ValueError].} = - ## Parses an integer starting at `start` and stores the value into `number`. + ## Parses an integer and stores the value into `number`. ## Result is the number of processed chars or 0 if there is no integer. ## `ValueError` is raised if the parsed integer is out of the valid range. runnableExamples: var res: BiggestInt - doAssert parseBiggestInt("9223372036854775807", res, 0) == 19 + doAssert parseBiggestInt("9223372036854775807", res) == 19 doAssert res == 9223372036854775807 + doAssert parseBiggestInt("-2024_05_09", res) == 11 + doAssert res == -20240509 var res = BiggestInt(0) # use 'res' for exception safety (don't write to 'number' in case of an # overflow exception): - result = rawParseInt(s, res, start) + result = rawParseInt(s, res) if result != 0: number = res -proc parseInt*(s: string, number: var int, start = 0): int {. +proc parseInt*(s: openArray[char], number: var int): int {. rtl, extern: "npuParseInt", noSideEffect, raises: [ValueError].} = - ## Parses an integer starting at `start` and stores the value into `number`. + ## Parses an integer and stores the value into `number`. ## Result is the number of processed chars or 0 if there is no integer. ## `ValueError` is raised if the parsed integer is out of the valid range. runnableExamples: var res: int - doAssert parseInt("2019", res, 0) == 4 - doAssert res == 2019 - doAssert parseInt("2019", res, 2) == 2 - doAssert res == 19 + doAssert parseInt("-2024_05_02", res) == 11 + doAssert res == -20240502 var res = BiggestInt(0) - result = parseBiggestInt(s, res, start) + result = parseBiggestInt(s, res) when sizeof(int) <= 4: if res < low(int) or res > high(int): integerOutOfRangeError() if result != 0: number = int(res) -proc parseSaturatedNatural*(s: string, b: var int, start = 0): int {. +proc parseSaturatedNatural*(s: openArray[char], b: var int): int {. raises: [].} = ## Parses a natural number into ``b``. This cannot raise an overflow ## error. ``high(int)`` is returned for an overflow. @@ -502,7 +496,7 @@ proc parseSaturatedNatural*(s: string, b: var int, start = 0): int {. var res = 0 discard parseSaturatedNatural("848", res) doAssert res == 848 - var i = start + var i = 0 if i < s.len and s[i] == '+': inc(i) if i < s.len and s[i] in {'0'..'9'}: b = 0 @@ -514,13 +508,13 @@ proc parseSaturatedNatural*(s: string, b: var int, start = 0): int {. b = high(int) inc(i) while i < s.len and s[i] == '_': inc(i) # underscores are allowed and ignored - result = i - start + result = i -proc rawParseUInt(s: string, b: var BiggestUInt, start = 0): int = +proc rawParseUInt(s: openArray[char], b: var BiggestUInt): int = var res = 0.BiggestUInt prev = 0.BiggestUInt - i = start + i = 0 if i < s.len - 1 and s[i] == '-' and s[i + 1] in {'0'..'9'}: integerOutOfRangeError() if i < s.len and s[i] == '+': inc(i) # Allow @@ -534,11 +528,11 @@ proc rawParseUInt(s: string, b: var BiggestUInt, start = 0): int = inc(i) while i < s.len and s[i] == '_': inc(i) # underscores are allowed and ignored b = res - result = i - start + result = i -proc parseBiggestUInt*(s: string, number: var BiggestUInt, start = 0): int {. +proc parseBiggestUInt*(s: openArray[char], number: var BiggestUInt): int {. rtl, extern: "npuParseBiggestUInt", noSideEffect, raises: [ValueError].} = - ## Parses an unsigned integer starting at `start` and stores the value + ## Parses an unsigned integer and stores the value ## into `number`. ## `ValueError` is raised if the parsed integer is out of the valid range. runnableExamples: @@ -550,13 +544,13 @@ proc parseBiggestUInt*(s: string, number: var BiggestUInt, start = 0): int {. var res = BiggestUInt(0) # use 'res' for exception safety (don't write to 'number' in case of an # overflow exception): - result = rawParseUInt(s, res, start) + result = rawParseUInt(s, res) if result != 0: number = res -proc parseUInt*(s: string, number: var uint, start = 0): int {. +proc parseUInt*(s: openArray[char], number: var uint): int {. rtl, extern: "npuParseUInt", noSideEffect, raises: [ValueError].} = - ## Parses an unsigned integer starting at `start` and stores the value + ## Parses an unsigned integer and stores the value ## into `number`. ## `ValueError` is raised if the parsed integer is out of the valid range. runnableExamples: @@ -566,22 +560,22 @@ proc parseUInt*(s: string, number: var uint, start = 0): int {. doAssert parseUInt("3450", res, 2) == 2 doAssert res == 50 var res = BiggestUInt(0) - result = parseBiggestUInt(s, res, start) + result = parseBiggestUInt(s, res) when sizeof(BiggestUInt) > sizeof(uint) and sizeof(uint) <= 4: if res > 0xFFFF_FFFF'u64: integerOutOfRangeError() if result != 0: number = uint(res) -proc parseBiggestFloat*(s: string, number: var BiggestFloat, start = 0): int {. +proc parseBiggestFloat*(s: openArray[char], number: var BiggestFloat): int {. magic: "ParseBiggestFloat", importc: "nimParseBiggestFloat", noSideEffect.} - ## Parses a float starting at `start` and stores the value into `number`. + ## Parses a float and stores the value into `number`. ## Result is the number of processed chars or 0 if a parsing error ## occurred. -proc parseFloat*(s: string, number: var float, start = 0): int {. +proc parseFloat*(s: openArray[char], number: var float): int {. rtl, extern: "npuParseFloat", noSideEffect.} = - ## Parses a float starting at `start` and stores the value into `number`. + ## Parses a float and stores the value into `number`. ## Result is the number of processed chars or 0 if there occurred a parsing ## error. runnableExamples: @@ -593,10 +587,75 @@ proc parseFloat*(s: string, number: var float, start = 0): int {. doAssert parseFloat("32.57", res, 3) == 2 doAssert res == 57.00 var bf = BiggestFloat(0.0) - result = parseBiggestFloat(s, bf, start) + result = parseBiggestFloat(s, bf) if result != 0: number = bf +func toLowerAscii(c: char): char = + if c in {'A'..'Z'}: char(uint8(c) xor 0b0010_0000'u8) else: c + +func parseSize*(s: openArray[char], size: var int64, alwaysBin=false): int = + ## Parse a size qualified by binary or metric units into `size`. This format + ## is often called "human readable". Result is the number of processed chars + ## or 0 on parse errors and size is rounded to the nearest integer. Trailing + ## garbage like "/s" in "1k/s" is allowed and detected by `result < s.len`. + ## + ## To simplify use, following non-rare wild conventions, and since fractional + ## data like milli-bytes is so rare, unit matching is case-insensitive but for + ## the 'i' distinguishing binary-metric from metric (which cannot be 'I'). + ## + ## An optional trailing 'B|b' is ignored but processed. I.e., you must still + ## know if units are bytes | bits or infer this fact via the case of s[^1] (if + ## users can even be relied upon to use 'B' for byte and 'b' for bit or have + ## that be s[^1]). + ## + ## If `alwaysBin==true` then scales are always binary-metric, but e.g. "KiB" + ## is still accepted for clarity. If the value would exceed the range of + ## `int64`, `size` saturates to `int64.high`. Supported metric prefix chars + ## include k, m, g, t, p, e, z, y (but z & y saturate unless the number is a + ## small fraction). + ## + ## **See also:** + ## * https://en.wikipedia.org/wiki/Binary_prefix + ## * `formatSize module<strutils.html>`_ for formatting + runnableExamples: + var res: int64 # caller must still know if 'b' refers to bytes|bits + doAssert parseSize("10.5 MB", res) == 7 + doAssert res == 10_500_000 # decimal metric Mega prefix + doAssert parseSize("64 mib", res) == 6 + doAssert res == 67108864 # 64 shl 20 + doAssert parseSize("1G/h", res, true) == 2 # '/' stops parse + doAssert res == 1073741824 # 1 shl 30, forced binary metric + const prefix = "b" & "kmgtpezy" # byte|bit & lowCase metric-ish prefixes + const scaleM = [1.0, 1e3, 1e6, 1e9, 1e12, 1e15, 1e18, 1e21, 1e24] # 10^(3*idx) + const scaleB = [1.0, 1024, 1048576, 1073741824, 1099511627776.0, # 2^(10*idx) + 1125899906842624.0, 1152921504606846976.0, # ldexp? + 1.180591620717411303424e21, 1.208925819614629174706176e24] + var number: float + var scale = 1.0 + result = parseFloat(s, number) + if number < 0: # While parseFloat accepts negatives .. + result = 0 #.. we do not since sizes cannot be < 0 + if result > 0: + let start = result # Save spot to maybe unwind white to EOS + while result < s.len and s[result] in Whitespace: + inc result + if result < s.len: # Illegal starting char => unity + if (let si = prefix.find(s[result].toLowerAscii); si >= 0): + inc result # Now parse the scale + scale = if alwaysBin: scaleB[si] else: scaleM[si] + if result < s.len and s[result] == 'i': + scale = scaleB[si] # Switch from default to binary-metric + inc result + if result < s.len and s[result].toLowerAscii == 'b': + inc result # Skip optional '[bB]' + else: # Unwind result advancement when there.. + result = start #..is no unit to the end of `s`. + var sizeF = number * scale + 0.5 # Saturate to int64.high when too big + size = if sizeF > 9223372036854774784.0: int64.high else: sizeF.int64 +# Above constant=2^63-1024 avoids C UB; github.com/nim-lang/Nim/issues/20102 or +# stackoverflow.com/questions/20923556/math-pow2-63-1-math-pow2-63-512-is-true + type InterpolatedKind* = enum ## Describes for `interpolatedFragments` ## which part of the interpolated string is @@ -606,7 +665,7 @@ type ikVar, ## ``var`` part of the interpolated string ikExpr ## ``expr`` part of the interpolated string -iterator interpolatedFragments*(s: string): tuple[kind: InterpolatedKind, +iterator interpolatedFragments*(s: openArray[char]): tuple[kind: InterpolatedKind, value: string] = ## Tokenizes the string `s` into substrings for interpolation purposes. ## @@ -641,7 +700,7 @@ iterator interpolatedFragments*(s: string): tuple[kind: InterpolatedKind, else: discard inc j raise newException(ValueError, - "Expected closing '}': " & substr(s, i, s.high)) + "Expected closing '}': " & substr(s.toOpenArray(i, s.high))) inc i, 2 # skip ${ kind = ikExpr elif j+1 < s.len and s[j+1] in IdentStartChars: @@ -655,15 +714,374 @@ iterator interpolatedFragments*(s: string): tuple[kind: InterpolatedKind, kind = ikDollar else: raise newException(ValueError, - "Unable to parse a variable name at " & substr(s, i, s.high)) + "Unable to parse a variable name at " & substr(s.toOpenArray(i, s.high))) else: while j < s.len and s[j] != '$': inc j kind = ikStr if j > i: # do not copy the trailing } for ikExpr: - yield (kind, substr(s, i, j-1-ord(kind == ikExpr))) + yield (kind, substr(s.toOpenArray(i, j-1-ord(kind == ikExpr)))) else: break i = j {.pop.} + + +proc parseBin*[T: SomeInteger](s: string, number: var T, start = 0, + maxLen = 0): int {.noSideEffect.} = + ## Parses a binary number and stores its value in ``number``. + ## + ## Returns the number of the parsed characters or 0 in case of an error. + ## If error, the value of ``number`` is not changed. + ## + ## If ``maxLen == 0``, the parsing continues until the first non-bin character + ## or to the end of the string. Otherwise, no more than ``maxLen`` characters + ## are parsed starting from the ``start`` position. + ## + ## It does not check for overflow. If the value represented by the string is + ## too big to fit into ``number``, only the value of last fitting characters + ## will be stored in ``number`` without producing an error. + runnableExamples: + var num: int + doAssert parseBin("0100_1110_0110_1001_1110_1101", num) == 29 + doAssert num == 5138925 + doAssert parseBin("3", num) == 0 + var num8: int8 + doAssert parseBin("0b_0100_1110_0110_1001_1110_1101", num8) == 32 + doAssert num8 == 0b1110_1101'i8 + doAssert parseBin("0b_0100_1110_0110_1001_1110_1101", num8, 3, 9) == 9 + doAssert num8 == 0b0100_1110'i8 + var num8u: uint8 + doAssert parseBin("0b_0100_1110_0110_1001_1110_1101", num8u) == 32 + doAssert num8u == 237 + var num64: int64 + doAssert parseBin("0100111001101001111011010100111001101001", num64) == 40 + doAssert num64 == 336784608873 + parseBin(s.toOpenArray(start, s.high), number, maxLen) + +proc parseOct*[T: SomeInteger](s: string, number: var T, start = 0, + maxLen = 0): int {.noSideEffect.} = + ## Parses an octal number and stores its value in ``number``. + ## + ## Returns the number of the parsed characters or 0 in case of an error. + ## If error, the value of ``number`` is not changed. + ## + ## If ``maxLen == 0``, the parsing continues until the first non-oct character + ## or to the end of the string. Otherwise, no more than ``maxLen`` characters + ## are parsed starting from the ``start`` position. + ## + ## It does not check for overflow. If the value represented by the string is + ## too big to fit into ``number``, only the value of last fitting characters + ## will be stored in ``number`` without producing an error. + runnableExamples: + var num: int + doAssert parseOct("0o23464755", num) == 10 + doAssert num == 5138925 + doAssert parseOct("8", num) == 0 + var num8: int8 + doAssert parseOct("0o_1464_755", num8) == 11 + doAssert num8 == -19 + doAssert parseOct("0o_1464_755", num8, 3, 3) == 3 + doAssert num8 == 102 + var num8u: uint8 + doAssert parseOct("1464755", num8u) == 7 + doAssert num8u == 237 + var num64: int64 + doAssert parseOct("2346475523464755", num64) == 16 + doAssert num64 == 86216859871725 + parseOct(s.toOpenArray(start, s.high), number, maxLen) + +proc parseHex*[T: SomeInteger](s: string, number: var T, start = 0, + maxLen = 0): int {.noSideEffect.} = + ## Parses a hexadecimal number and stores its value in ``number``. + ## + ## Returns the number of the parsed characters or 0 in case of an error. + ## If error, the value of ``number`` is not changed. + ## + ## If ``maxLen == 0``, the parsing continues until the first non-hex character + ## or to the end of the string. Otherwise, no more than ``maxLen`` characters + ## are parsed starting from the ``start`` position. + ## + ## It does not check for overflow. If the value represented by the string is + ## too big to fit into ``number``, only the value of last fitting characters + ## will be stored in ``number`` without producing an error. + runnableExamples: + var num: int + doAssert parseHex("4E_69_ED", num) == 8 + doAssert num == 5138925 + doAssert parseHex("X", num) == 0 + doAssert parseHex("#ABC", num) == 4 + var num8: int8 + doAssert parseHex("0x_4E_69_ED", num8) == 11 + doAssert num8 == 0xED'i8 + doAssert parseHex("0x_4E_69_ED", num8, 3, 2) == 2 + doAssert num8 == 0x4E'i8 + var num8u: uint8 + doAssert parseHex("0x_4E_69_ED", num8u) == 11 + doAssert num8u == 237 + var num64: int64 + doAssert parseHex("4E69ED4E69ED", num64) == 12 + doAssert num64 == 86216859871725 + parseHex(s.toOpenArray(start, s.high), number, maxLen) + +proc parseIdent*(s: string, ident: var string, start = 0): int = + ## Parses an identifier and stores it in ``ident``. Returns + ## the number of the parsed characters or 0 in case of an error. + ## If error, the value of `ident` is not changed. + runnableExamples: + var res: string + doAssert parseIdent("Hello World", res, 0) == 5 + doAssert res == "Hello" + doAssert parseIdent("Hello World", res, 1) == 4 + doAssert res == "ello" + doAssert parseIdent("Hello World", res, 6) == 5 + doAssert res == "World" + parseIdent(s.toOpenArray(start, s.high), ident) + +proc parseIdent*(s: string, start = 0): string = + ## Parses an identifier and returns it or an empty string in + ## case of an error. + runnableExamples: + doAssert parseIdent("Hello World", 0) == "Hello" + doAssert parseIdent("Hello World", 1) == "ello" + doAssert parseIdent("Hello World", 5) == "" + doAssert parseIdent("Hello World", 6) == "World" + parseIdent(s.toOpenArray(start, s.high)) + +proc parseChar*(s: string, c: var char, start = 0): int = + ## Parses a single character, stores it in `c` and returns 1. + ## In case of error (if start >= s.len) it returns 0 + ## and the value of `c` is unchanged. + runnableExamples: + var c: char + doAssert "nim".parseChar(c, 3) == 0 + doAssert c == '\0' + doAssert "nim".parseChar(c, 0) == 1 + doAssert c == 'n' + parseChar(s.toOpenArray(start, s.high), c) + +proc skipWhitespace*(s: string, start = 0): int {.inline.} = + ## Skips the whitespace starting at ``s[start]``. Returns the number of + ## skipped characters. + runnableExamples: + doAssert skipWhitespace("Hello World", 0) == 0 + doAssert skipWhitespace(" Hello World", 0) == 1 + doAssert skipWhitespace("Hello World", 5) == 1 + doAssert skipWhitespace("Hello World", 5) == 2 + skipWhitespace(s.toOpenArray(start, s.high)) + +proc skip*(s, token: string, start = 0): int {.inline.} = + ## Skips the `token` starting at ``s[start]``. Returns the length of `token` + ## or 0 if there was no `token` at ``s[start]``. + runnableExamples: + doAssert skip("2019-01-22", "2019", 0) == 4 + doAssert skip("2019-01-22", "19", 0) == 0 + doAssert skip("2019-01-22", "19", 2) == 2 + doAssert skip("CAPlow", "CAP", 0) == 3 + doAssert skip("CAPlow", "cap", 0) == 0 + skip(s.toOpenArray(start, s.high), token) + +proc skipIgnoreCase*(s, token: string, start = 0): int = + ## Same as `skip` but case is ignored for token matching. + runnableExamples: + doAssert skipIgnoreCase("CAPlow", "CAP", 0) == 3 + doAssert skipIgnoreCase("CAPlow", "cap", 0) == 3 + skipIgnoreCase(s.toOpenArray(start, s.high), token) + +proc skipUntil*(s: string, until: set[char], start = 0): int {.inline.} = + ## Skips all characters until one char from the set `until` is found + ## or the end is reached. + ## Returns number of characters skipped. + runnableExamples: + doAssert skipUntil("Hello World", {'W', 'e'}, 0) == 1 + doAssert skipUntil("Hello World", {'W'}, 0) == 6 + doAssert skipUntil("Hello World", {'W', 'd'}, 0) == 6 + skipUntil(s.toOpenArray(start, s.high), until) + +proc skipUntil*(s: string, until: char, start = 0): int {.inline.} = + ## Skips all characters until the char `until` is found + ## or the end is reached. + ## Returns number of characters skipped. + runnableExamples: + doAssert skipUntil("Hello World", 'o', 0) == 4 + doAssert skipUntil("Hello World", 'o', 4) == 0 + doAssert skipUntil("Hello World", 'W', 0) == 6 + doAssert skipUntil("Hello World", 'w', 0) == 11 + skipUntil(s.toOpenArray(start, s.high), until) + +proc skipWhile*(s: string, toSkip: set[char], start = 0): int {.inline.} = + ## Skips all characters while one char from the set `toSkip` is found. + ## Returns number of characters skipped. + runnableExamples: + doAssert skipWhile("Hello World", {'H', 'e'}) == 2 + doAssert skipWhile("Hello World", {'e'}) == 0 + doAssert skipWhile("Hello World", {'W', 'o', 'r'}, 6) == 3 + skipWhile(s.toOpenArray(start, s.high), toSkip) + +proc parseUntil*(s: string, token: var string, until: set[char], + start = 0): int {.inline.} = + ## Parses a token and stores it in ``token``. Returns + ## the number of the parsed characters or 0 in case of an error. A token + ## consists of the characters notin `until`. + runnableExamples: + var myToken: string + doAssert parseUntil("Hello World", myToken, {'W', 'o', 'r'}) == 4 + doAssert myToken == "Hell" + doAssert parseUntil("Hello World", myToken, {'W', 'r'}) == 6 + doAssert myToken == "Hello " + doAssert parseUntil("Hello World", myToken, {'W', 'r'}, 3) == 3 + doAssert myToken == "lo " + parseUntil(s.toOpenArray(start, s.high), token, until) + +proc parseUntil*(s: string, token: var string, until: char, + start = 0): int {.inline.} = + ## Parses a token and stores it in ``token``. Returns + ## the number of the parsed characters or 0 in case of an error. A token + ## consists of any character that is not the `until` character. + runnableExamples: + var myToken: string + doAssert parseUntil("Hello World", myToken, 'W') == 6 + doAssert myToken == "Hello " + doAssert parseUntil("Hello World", myToken, 'o') == 4 + doAssert myToken == "Hell" + doAssert parseUntil("Hello World", myToken, 'o', 2) == 2 + doAssert myToken == "ll" + parseUntil(s.toOpenArray(start, s.high), token, until) + +proc parseUntil*(s: string, token: var string, until: string, + start = 0): int {.inline.} = + ## Parses a token and stores it in ``token``. Returns + ## the number of the parsed characters or 0 in case of an error. A token + ## consists of any character that comes before the `until` token. + runnableExamples: + var myToken: string + doAssert parseUntil("Hello World", myToken, "Wor") == 6 + doAssert myToken == "Hello " + doAssert parseUntil("Hello World", myToken, "Wor", 2) == 4 + doAssert myToken == "llo " + parseUntil(s.toOpenArray(start, s.high), token, until) + +proc parseWhile*(s: string, token: var string, validChars: set[char], + start = 0): int {.inline.} = + ## Parses a token and stores it in ``token``. Returns + ## the number of the parsed characters or 0 in case of an error. A token + ## consists of the characters in `validChars`. + runnableExamples: + var myToken: string + doAssert parseWhile("Hello World", myToken, {'W', 'o', 'r'}, 0) == 0 + doAssert myToken.len() == 0 + doAssert parseWhile("Hello World", myToken, {'W', 'o', 'r'}, 6) == 3 + doAssert myToken == "Wor" + parseWhile(s.toOpenArray(start, s.high), token, validChars) + +proc captureBetween*(s: string, first: char, second = '\0', start = 0): string = + ## Finds the first occurrence of ``first``, then returns everything from there + ## up to ``second`` (if ``second`` is '\0', then ``first`` is used). + runnableExamples: + doAssert captureBetween("Hello World", 'e') == "llo World" + doAssert captureBetween("Hello World", 'e', 'r') == "llo Wo" + doAssert captureBetween("Hello World", 'l', start = 6) == "d" + captureBetween(s.toOpenArray(start, s.high), first, second) + +proc parseBiggestInt*(s: string, number: var BiggestInt, start = 0): int {.noSideEffect, raises: [ValueError].} = + ## Parses an integer starting at `start` and stores the value into `number`. + ## Result is the number of processed chars or 0 if there is no integer. + ## `ValueError` is raised if the parsed integer is out of the valid range. + runnableExamples: + var res: BiggestInt + doAssert parseBiggestInt("9223372036854775807", res, 0) == 19 + doAssert res == 9223372036854775807 + doAssert parseBiggestInt("-2024_05_09", res) == 11 + doAssert res == -20240509 + doAssert parseBiggestInt("-2024_05_02", res, 7) == 4 + doAssert res == 502 + parseBiggestInt(s.toOpenArray(start, s.high), number) + +proc parseInt*(s: string, number: var int, start = 0): int {.noSideEffect, raises: [ValueError].} = + ## Parses an integer starting at `start` and stores the value into `number`. + ## Result is the number of processed chars or 0 if there is no integer. + ## `ValueError` is raised if the parsed integer is out of the valid range. + runnableExamples: + var res: int + doAssert parseInt("-2024_05_02", res) == 11 + doAssert res == -20240502 + doAssert parseInt("-2024_05_02", res, 7) == 4 + doAssert res == 502 + parseInt(s.toOpenArray(start, s.high), number) + + +proc parseSaturatedNatural*(s: string, b: var int, start = 0): int {. + raises: [].} = + ## Parses a natural number into ``b``. This cannot raise an overflow + ## error. ``high(int)`` is returned for an overflow. + ## The number of processed character is returned. + ## This is usually what you really want to use instead of `parseInt`:idx:. + runnableExamples: + var res = 0 + discard parseSaturatedNatural("848", res) + doAssert res == 848 + parseSaturatedNatural(s.toOpenArray(start, s.high), b) + + +proc parseBiggestUInt*(s: string, number: var BiggestUInt, start = 0): int {.noSideEffect, raises: [ValueError].} = + ## Parses an unsigned integer starting at `start` and stores the value + ## into `number`. + ## `ValueError` is raised if the parsed integer is out of the valid range. + runnableExamples: + var res: BiggestUInt + doAssert parseBiggestUInt("12", res, 0) == 2 + doAssert res == 12 + doAssert parseBiggestUInt("1111111111111111111", res, 0) == 19 + doAssert res == 1111111111111111111'u64 + parseBiggestUInt(s.toOpenArray(start, s.high), number) + +proc parseUInt*(s: string, number: var uint, start = 0): int {.noSideEffect, raises: [ValueError].} = + ## Parses an unsigned integer starting at `start` and stores the value + ## into `number`. + ## `ValueError` is raised if the parsed integer is out of the valid range. + runnableExamples: + var res: uint + doAssert parseUInt("3450", res) == 4 + doAssert res == 3450 + doAssert parseUInt("3450", res, 2) == 2 + doAssert res == 50 + parseUInt(s.toOpenArray(start, s.high), number) + +proc parseBiggestFloat*(s: string, number: var BiggestFloat, start = 0): int {.noSideEffect.} = + ## Parses a float starting at `start` and stores the value into `number`. + ## Result is the number of processed chars or 0 if a parsing error + ## occurred. + parseFloat(s.toOpenArray(start, s.high), number) + +proc parseFloat*(s: string, number: var float, start = 0): int {.noSideEffect.} = + ## Parses a float starting at `start` and stores the value into `number`. + ## Result is the number of processed chars or 0 if there occurred a parsing + ## error. + runnableExamples: + var res: float + doAssert parseFloat("32", res, 0) == 2 + doAssert res == 32.0 + doAssert parseFloat("32.57", res, 0) == 5 + doAssert res == 32.57 + doAssert parseFloat("32.57", res, 3) == 2 + doAssert res == 57.00 + parseFloat(s.toOpenArray(start, s.high), number) + +iterator interpolatedFragments*(s: string): tuple[kind: InterpolatedKind, + value: string] = + ## Tokenizes the string `s` into substrings for interpolation purposes. + ## + runnableExamples: + var outp: seq[tuple[kind: InterpolatedKind, value: string]] + for k, v in interpolatedFragments(" $this is ${an example} $$"): + outp.add (k, v) + doAssert outp == @[(ikStr, " "), + (ikVar, "this"), + (ikStr, " is "), + (ikExpr, "an example"), + (ikStr, " "), + (ikDollar, "$")] + for x in s.toOa.interpolatedFragments: + yield x + diff --git a/lib/pure/parsexml.nim b/lib/pure/parsexml.nim index 3ba5a19d5..c760799a2 100644 --- a/lib/pure/parsexml.nim +++ b/lib/pure/parsexml.nim @@ -36,43 +36,43 @@ The file ``examples/htmltitle.nim`` demonstrates how to use the XML parser to accomplish a simple task: To determine the title of an HTML document. -.. code-block:: nim + ```nim + # Example program to show the parsexml module + # This program reads an HTML file and writes its title to stdout. + # Errors and whitespace are ignored. - # Example program to show the parsexml module - # This program reads an HTML file and writes its title to stdout. - # Errors and whitespace are ignored. + import std/[os, streams, parsexml, strutils] - import os, streams, parsexml, strutils + if paramCount() < 1: + quit("Usage: htmltitle filename[.html]") - if paramCount() < 1: - quit("Usage: htmltitle filename[.html]") - - var filename = addFileExt(paramStr(1), "html") - var s = newFileStream(filename, fmRead) - if s == nil: quit("cannot open the file " & filename) - var x: XmlParser - open(x, s, filename) - while true: - x.next() - case x.kind - of xmlElementStart: - if cmpIgnoreCase(x.elementName, "title") == 0: - var title = "" - x.next() # skip "<title>" - while x.kind == xmlCharData: - title.add(x.charData) - x.next() - if x.kind == xmlElementEnd and cmpIgnoreCase(x.elementName, "title") == 0: - echo("Title: " & title) - quit(0) # Success! - else: - echo(x.errorMsgExpected("/title")) + var filename = addFileExt(paramStr(1), "html") + var s = newFileStream(filename, fmRead) + if s == nil: quit("cannot open the file " & filename) + var x: XmlParser + open(x, s, filename) + while true: + x.next() + case x.kind + of xmlElementStart: + if cmpIgnoreCase(x.elementName, "title") == 0: + var title = "" + x.next() # skip "<title>" + while x.kind == xmlCharData: + title.add(x.charData) + x.next() + if x.kind == xmlElementEnd and cmpIgnoreCase(x.elementName, "title") == 0: + echo("Title: " & title) + quit(0) # Success! + else: + echo(x.errorMsgExpected("/title")) - of xmlEof: break # end of file reached - else: discard # ignore other events + of xmlEof: break # end of file reached + else: discard # ignore other events - x.close() - quit("Could not determine title!") + x.close() + quit("Could not determine title!") + ``` ]## @@ -85,72 +85,72 @@ The file ``examples/htmlrefs.nim`` demonstrates how to use the XML parser to accomplish another simple task: To determine all the links an HTML document contains. -.. code-block:: nim + ```nim + # Example program to show the new parsexml module + # This program reads an HTML file and writes all its used links to stdout. + # Errors and whitespace are ignored. - # Example program to show the new parsexml module - # This program reads an HTML file and writes all its used links to stdout. - # Errors and whitespace are ignored. + import std/[os, streams, parsexml, strutils] - import os, streams, parsexml, strutils + proc `=?=` (a, b: string): bool = + # little trick: define our own comparator that ignores case + return cmpIgnoreCase(a, b) == 0 - proc `=?=` (a, b: string): bool = - # little trick: define our own comparator that ignores case - return cmpIgnoreCase(a, b) == 0 + if paramCount() < 1: + quit("Usage: htmlrefs filename[.html]") - if paramCount() < 1: - quit("Usage: htmlrefs filename[.html]") - - var links = 0 # count the number of links - var filename = addFileExt(paramStr(1), "html") - var s = newFileStream(filename, fmRead) - if s == nil: quit("cannot open the file " & filename) - var x: XmlParser - open(x, s, filename) - next(x) # get first event - block mainLoop: - while true: - case x.kind - of xmlElementOpen: - # the <a href = "xyz"> tag we are interested in always has an attribute, - # thus we search for ``xmlElementOpen`` and not for ``xmlElementStart`` - if x.elementName =?= "a": - x.next() - if x.kind == xmlAttribute: - if x.attrKey =?= "href": - var link = x.attrValue - inc(links) - # skip until we have an ``xmlElementClose`` event - while true: - x.next() - case x.kind - of xmlEof: break mainLoop - of xmlElementClose: break - else: discard - x.next() # skip ``xmlElementClose`` - # now we have the description for the ``a`` element - var desc = "" - while x.kind == xmlCharData: - desc.add(x.charData) - x.next() - echo(desc & ": " & link) - else: - x.next() - of xmlEof: break # end of file reached - of xmlError: - echo(errorMsg(x)) + var links = 0 # count the number of links + var filename = addFileExt(paramStr(1), "html") + var s = newFileStream(filename, fmRead) + if s == nil: quit("cannot open the file " & filename) + var x: XmlParser + open(x, s, filename) + next(x) # get first event + block mainLoop: + while true: + case x.kind + of xmlElementOpen: + # the <a href = "xyz"> tag we are interested in always has an attribute, + # thus we search for ``xmlElementOpen`` and not for ``xmlElementStart`` + if x.elementName =?= "a": + x.next() + if x.kind == xmlAttribute: + if x.attrKey =?= "href": + var link = x.attrValue + inc(links) + # skip until we have an ``xmlElementClose`` event + while true: + x.next() + case x.kind + of xmlEof: break mainLoop + of xmlElementClose: break + else: discard + x.next() # skip ``xmlElementClose`` + # now we have the description for the ``a`` element + var desc = "" + while x.kind == xmlCharData: + desc.add(x.charData) + x.next() + echo(desc & ": " & link) + else: x.next() - else: x.next() # skip other events + of xmlEof: break # end of file reached + of xmlError: + echo(errorMsg(x)) + x.next() + else: x.next() # skip other events - echo($links & " link(s) found!") - x.close() + echo($links & " link(s) found!") + x.close() + ``` ]## import - strutils, lexbase, streams, unicode + std/[strutils, lexbase, streams, unicode] when defined(nimPreviewSlimSystem): - import std/assertions + import std/[assertions, syncio] # the parser treats ``<br />`` as ``<br></br>`` @@ -792,7 +792,7 @@ proc next*(my: var XmlParser) = my.state = stateNormal when not defined(testing) and isMainModule: - import os + import std/os var s = newFileStream(paramStr(1), fmRead) if s == nil: quit("cannot open the file" & paramStr(1)) var x: XmlParser diff --git a/lib/pure/pathnorm.nim b/lib/pure/pathnorm.nim index a71ae0762..4cdc02303 100644 --- a/lib/pure/pathnorm.nim +++ b/lib/pure/pathnorm.nim @@ -14,7 +14,7 @@ # Yes, this uses import here, not include so that # we don't end up exporting these symbols from pathnorm and os: -import includes/osseps +import std/private/osseps type PathIter* = object diff --git a/lib/pure/pegs.nim b/lib/pure/pegs.nim index a95700825..2969fd6d7 100644 --- a/lib/pure/pegs.nim +++ b/lib/pure/pegs.nim @@ -22,11 +22,11 @@ when defined(nimPreviewSlimSystem): const useUnicode = true ## change this to deactivate proper UTF-8 support -import strutils, macros +import std/[strutils, macros] import std/private/decode_helpers when useUnicode: - import unicode + import std/unicode export unicode.`==` const @@ -168,8 +168,9 @@ func charSet*(s: set[char]): Peg {.rtl, extern: "npegs$1".} = ## constructs a PEG from a character set `s` assert '\0' notin s result = Peg(kind: pkCharChoice) - new(result.charChoice) - result.charChoice[] = s + {.cast(noSideEffect).}: + new(result.charChoice) + result.charChoice[] = s func len(a: Peg): int {.inline.} = return a.sons.len func add(d: var Peg, s: Peg) {.inline.} = add(d.sons, s) @@ -313,21 +314,21 @@ func capture*(a: Peg = Peg(kind: pkEmpty)): Peg {.rtl, extern: "npegsCapture".} func backref*(index: range[1..MaxSubpatterns], reverse: bool = false): Peg {. rtl, extern: "npegs$1".} = ## constructs a back reference of the given `index`. `index` starts counting - ## from 1. `reverse` specifies wether indexing starts from the end of the + ## from 1. `reverse` specifies whether indexing starts from the end of the ## capture list. result = Peg(kind: pkBackRef, index: (if reverse: -index else: index - 1)) func backrefIgnoreCase*(index: range[1..MaxSubpatterns], reverse: bool = false): Peg {. rtl, extern: "npegs$1".} = ## constructs a back reference of the given `index`. `index` starts counting - ## from 1. `reverse` specifies wether indexing starts from the end of the + ## from 1. `reverse` specifies whether indexing starts from the end of the ## capture list. Ignores case for matching. result = Peg(kind: pkBackRefIgnoreCase, index: (if reverse: -index else: index - 1)) func backrefIgnoreStyle*(index: range[1..MaxSubpatterns], reverse: bool = false): Peg {. rtl, extern: "npegs$1".} = ## constructs a back reference of the given `index`. `index` starts counting - ## from 1. `reverse` specifies wether indexing starts from the end of the + ## from 1. `reverse` specifies whether indexing starts from the end of the ## capture list. Ignores style for matching. result = Peg(kind: pkBackRefIgnoreStyle, index: (if reverse: -index else: index - 1)) @@ -561,7 +562,7 @@ template matchOrParse(mopProc: untyped) = # procs. For the former, *enter* and *leave* event handler code generators # are provided which just return *discard*. - proc mopProc(s: string, p: Peg, start: int, c: var Captures): int = + proc mopProc(s: string, p: Peg, start: int, c: var Captures): int {.gcsafe, raises: [].} = proc matchBackRef(s: string, p: Peg, start: int, c: var Captures): int = # Parse handler code must run in an *of* clause of its own for each # *PegKind*, so we encapsulate the identical clause body for @@ -888,7 +889,7 @@ macro mkHandlerTplts(handlers: untyped): untyped = # Transforms the handler spec in *handlers* into handler templates. # The AST structure of *handlers[0]*: # - # .. code-block:: + # ``` # StmtList # Call # Ident "pkNonTerminal" @@ -909,6 +910,7 @@ macro mkHandlerTplts(handlers: untyped): untyped = # StmtList # <handler code block> # ... + # ``` func mkEnter(hdName, body: NimNode): NimNode = template helper(hdName, body) {.dirty.} = template hdName(s, p, start) = @@ -958,60 +960,61 @@ template eventParser*(pegAst, handlers: untyped): (proc(s: string): int) = ## match, else the length of the total match. The following example code ## evaluates an arithmetic expression defined by a simple PEG: ## - ## .. code-block:: nim - ## import std/[strutils, pegs] + ## ```nim + ## import std/[strutils, pegs] ## - ## let - ## pegAst = """ - ## Expr <- Sum - ## Sum <- Product (('+' / '-')Product)* - ## Product <- Value (('*' / '/')Value)* - ## Value <- [0-9]+ / '(' Expr ')' - ## """.peg - ## txt = "(5+3)/2-7*22" + ## let + ## pegAst = """ + ## Expr <- Sum + ## Sum <- Product (('+' / '-')Product)* + ## Product <- Value (('*' / '/')Value)* + ## Value <- [0-9]+ / '(' Expr ')' + ## """.peg + ## txt = "(5+3)/2-7*22" ## - ## var - ## pStack: seq[string] = @[] - ## valStack: seq[float] = @[] - ## opStack = "" - ## let - ## parseArithExpr = pegAst.eventParser: - ## pkNonTerminal: - ## enter: - ## pStack.add p.nt.name - ## leave: - ## pStack.setLen pStack.high - ## if length > 0: - ## let matchStr = s.substr(start, start+length-1) - ## case p.nt.name - ## of "Value": - ## try: - ## valStack.add matchStr.parseFloat - ## echo valStack - ## except ValueError: - ## discard - ## of "Sum", "Product": - ## try: - ## let val = matchStr.parseFloat - ## except ValueError: - ## if valStack.len > 1 and opStack.len > 0: - ## valStack[^2] = case opStack[^1] - ## of '+': valStack[^2] + valStack[^1] - ## of '-': valStack[^2] - valStack[^1] - ## of '*': valStack[^2] * valStack[^1] - ## else: valStack[^2] / valStack[^1] - ## valStack.setLen valStack.high - ## echo valStack - ## opStack.setLen opStack.high - ## echo opStack - ## pkChar: - ## leave: - ## if length == 1 and "Value" != pStack[^1]: - ## let matchChar = s[start] - ## opStack.add matchChar - ## echo opStack + ## var + ## pStack: seq[string] = @[] + ## valStack: seq[float] = @[] + ## opStack = "" + ## let + ## parseArithExpr = pegAst.eventParser: + ## pkNonTerminal: + ## enter: + ## pStack.add p.nt.name + ## leave: + ## pStack.setLen pStack.high + ## if length > 0: + ## let matchStr = s.substr(start, start+length-1) + ## case p.nt.name + ## of "Value": + ## try: + ## valStack.add matchStr.parseFloat + ## echo valStack + ## except ValueError: + ## discard + ## of "Sum", "Product": + ## try: + ## let val = matchStr.parseFloat + ## except ValueError: + ## if valStack.len > 1 and opStack.len > 0: + ## valStack[^2] = case opStack[^1] + ## of '+': valStack[^2] + valStack[^1] + ## of '-': valStack[^2] - valStack[^1] + ## of '*': valStack[^2] * valStack[^1] + ## else: valStack[^2] / valStack[^1] + ## valStack.setLen valStack.high + ## echo valStack + ## opStack.setLen opStack.high + ## echo opStack + ## pkChar: + ## leave: + ## if length == 1 and "Value" != pStack[^1]: + ## let matchChar = s[start] + ## opStack.add matchChar + ## echo opStack ## - ## let pLen = parseArithExpr(txt) + ## let pLen = parseArithExpr(txt) + ## ``` ## ## The *handlers* parameter consists of code blocks for *PegKinds*, ## which define the grammar elements of interest. Each block can contain @@ -1041,10 +1044,10 @@ template eventParser*(pegAst, handlers: untyped): (proc(s: string): int) = # by *mkHandlerTplts*. template mkDoEnter(hdPostf, s, pegNode, start) = when declared(`enter hdPostf`): - `enter hdPostf`(s, pegNode, start): + `enter hdPostf`(s, pegNode, start) else: discard - let hdPostf = ident(substr(strVal(pegKind), 2)) + let hdPostf = ident(substr($pegKind, 2)) getAst(mkDoEnter(hdPostf, s, pegNode, start)) macro leave(pegKind, s, pegNode, start, length: untyped): untyped = @@ -1052,10 +1055,10 @@ template eventParser*(pegAst, handlers: untyped): (proc(s: string): int) = # a grammar element of kind *pegKind*. template mkDoLeave(hdPostf, s, pegNode, start, length) = when declared(`leave hdPostf`): - `leave hdPostf`(s, pegNode, start, length): + `leave hdPostf`(s, pegNode, start, length) else: discard - let hdPostf = ident(substr(strVal(pegKind), 2)) + let hdPostf = ident(substr($pegKind, 2)) getAst(mkDoLeave(hdPostf, s, pegNode, start, length)) matchOrParse(parseIt) @@ -1172,7 +1175,7 @@ iterator findAll*(s: string, pattern: Peg, start = 0): string = func findAll*(s: string, pattern: Peg, start = 0): seq[string] {. rtl, extern: "npegs$1".} = ## returns all matching *substrings* of `s` that match `pattern`. - ## If it does not match, @[] is returned. + ## If it does not match, `@[]` is returned. result = @[] for it in findAll(s, pattern, start): result.add it @@ -1180,8 +1183,7 @@ template `=~`*(s: string, pattern: Peg): bool = ## This calls ``match`` with an implicit declared ``matches`` array that ## can be used in the scope of the ``=~`` call: ## - ## .. code-block:: nim - ## + ## ```nim ## if line =~ peg"\s* {\w+} \s* '=' \s* {\w+}": ## # matches a key=value pair: ## echo("Key: ", matches[0]) @@ -1193,10 +1195,10 @@ template `=~`*(s: string, pattern: Peg): bool = ## echo("comment: ", matches[0]) ## else: ## echo("syntax error") - ## + ## ``` bind MaxSubpatterns when not declaredInScope(matches): - var matches {.inject.}: array[0..MaxSubpatterns-1, string] + var matches {.inject.} = default(array[0..MaxSubpatterns-1, string]) match(s, pattern, matches) # ------------------------- more string handling ------------------------------ @@ -1229,14 +1231,15 @@ func replacef*(s: string, sub: Peg, by: string): string {. ## Replaces `sub` in `s` by the string `by`. Captures can be accessed in `by` ## with the notation ``$i`` and ``$#`` (see strutils.`%`). Examples: ## - ## .. code-block:: nim + ## ```nim ## "var1=key; var2=key2".replacef(peg"{\ident}'='{\ident}", "$1<-$2$2") + ## ``` ## ## Results in: ## - ## .. code-block:: nim - ## + ## ```nim ## "var1<-keykey; val2<-key2key2" + ## ``` result = "" var i = 0 var caps: array[0..MaxSubpatterns-1, string] @@ -1304,8 +1307,7 @@ func replace*(s: string, sub: Peg, cb: proc( ## The callback proc receives the index of the current match (starting with 0), ## the count of captures and an open array with the captures of each match. Examples: ## - ## .. code-block:: nim - ## + ## ```nim ## func handleMatches*(m: int, n: int, c: openArray[string]): string = ## result = "" ## if m > 0: @@ -1317,12 +1319,13 @@ func replace*(s: string, sub: Peg, cb: proc( ## ## let s = "Var1=key1;var2=Key2; VAR3" ## echo s.replace(peg"{\ident}('='{\ident})* ';'* \s*", handleMatches) + ## ``` ## ## Results in: ## - ## .. code-block:: nim - ## + ## ```nim ## "var1: 'key1', var2: 'Key2', var3: ''" + ## ``` result = "" var i = 0 var caps: array[0..MaxSubpatterns-1, string] @@ -1360,18 +1363,19 @@ iterator split*(s: string, sep: Peg): string = ## Substrings are separated by the PEG `sep`. ## Examples: ## - ## .. code-block:: nim + ## ```nim ## for word in split("00232this02939is39an22example111", peg"\d+"): ## writeLine(stdout, word) + ## ``` ## ## Results in: ## - ## .. code-block:: nim + ## ```nim ## "this" ## "is" ## "an" ## "example" - ## + ## ``` var c: Captures var first = 0 @@ -1823,9 +1827,7 @@ type skip: Peg func pegError(p: PegParser, msg: string, line = -1, col = -1) = - var e: ref EInvalidPeg - new(e) - e.msg = errorStr(p, msg, line, col) + var e = (ref EInvalidPeg)(msg: errorStr(p, msg, line, col)) raise e func getTok(p: var PegParser) = @@ -1909,7 +1911,8 @@ func primary(p: var PegParser): Peg = getTok(p) elif not arrowIsNextTok(p): var nt = getNonTerminal(p, p.tok.literal) - incl(nt.flags, ntUsed) + {.cast(noSideEffect).}: + incl(nt.flags, ntUsed) result = nonterminal(nt).token(p) getTok(p) else: @@ -2002,12 +2005,14 @@ func parseRule(p: var PegParser): NonTerminal = result = getNonTerminal(p, p.tok.literal) if ntDeclared in result.flags: pegError(p, "attempt to redefine: " & result.name) - result.line = getLine(p) - result.col = getColumn(p) + {.cast(noSideEffect).}: + result.line = getLine(p) + result.col = getColumn(p) getTok(p) eat(p, tkArrow) - result.rule = parseExpr(p) - incl(result.flags, ntDeclared) # NOW inlining may be attempted + {.cast(noSideEffect).}: + result.rule = parseExpr(p) + incl(result.flags, ntDeclared) # NOW inlining may be attempted else: pegError(p, "rule expected, but found: " & p.tok.literal) @@ -2058,9 +2063,9 @@ func parsePeg*(pattern: string, filename = "pattern", line = 1, col = 0): Peg = func peg*(pattern: string): Peg = ## constructs a Peg object from the `pattern`. The short name has been - ## chosen to encourage its use as a raw string modifier:: + ## chosen to encourage its use as a raw string modifier: ## - ## peg"{\ident} \s* '=' \s* {.*}" + ## peg"{\ident} \s* '=' \s* {.*}" result = parsePeg(pattern, "pattern") func escapePeg*(s: string): string = diff --git a/lib/pure/prelude.nim b/lib/pure/prelude.nim index 3c98d9d57..9428f29eb 100644 --- a/lib/pure/prelude.nim +++ b/lib/pure/prelude.nim @@ -14,7 +14,7 @@ when defined(nimdoc) and isMainModule: runnableExamples: include std/prelude # same as: - # import std/[os, strutils, times, parseutils, hashes, tables, sets, sequtils, parseopt] + # import std/[os, strutils, times, parseutils, hashes, tables, sets, sequtils, parseopt, strformat] let x = 1 assert "foo $# $#" % [$x, "bar"] == "foo 1 bar" assert toSeq(1..3) == @[1, 2, 3] diff --git a/lib/pure/punycode.nim b/lib/pure/punycode.nim deleted file mode 100644 index 49e46496e..000000000 --- a/lib/pure/punycode.nim +++ /dev/null @@ -1,209 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2016 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## Implements a representation of Unicode with the limited -## ASCII character subset. - -import strutils -import unicode - -# issue #3045 - -const - Base = 36 - TMin = 1 - TMax = 26 - Skew = 38 - Damp = 700 - InitialBias = 72 - InitialN = 128 - Delimiter = '-' - -type - PunyError* = object of ValueError - -func decodeDigit(x: char): int {.raises: [PunyError].} = - if '0' <= x and x <= '9': - result = ord(x) - (ord('0') - 26) - elif 'A' <= x and x <= 'Z': - result = ord(x) - ord('A') - elif 'a' <= x and x <= 'z': - result = ord(x) - ord('a') - else: - raise newException(PunyError, "Bad input") - -func encodeDigit(digit: int): Rune {.raises: [PunyError].} = - if 0 <= digit and digit < 26: - result = Rune(digit + ord('a')) - elif 26 <= digit and digit < 36: - result = Rune(digit + (ord('0') - 26)) - else: - raise newException(PunyError, "internal error in punycode encoding") - -func isBasic(c: char): bool = ord(c) < 0x80 -func isBasic(r: Rune): bool = int(r) < 0x80 - -func adapt(delta, numPoints: int, first: bool): int = - var d = if first: delta div Damp else: delta div 2 - d += d div numPoints - var k = 0 - while d > ((Base-TMin)*TMax) div 2: - d = d div (Base - TMin) - k += Base - result = k + (Base - TMin + 1) * d div (d + Skew) - -func encode*(prefix, s: string): string {.raises: [PunyError].} = - ## Encode a string that may contain Unicode. - ## Prepend `prefix` to the result - result = prefix - var (d, n, bias) = (0, InitialN, InitialBias) - var (b, remaining) = (0, 0) - for r in s.runes: - if r.isBasic: - # basic Ascii character - inc b - result.add($r) - else: - # special character - inc remaining - - var h = b - if b > 0: - result.add(Delimiter) # we have some Ascii chars - while remaining != 0: - var m: int = high(int32) - for r in s.runes: - if m > int(r) and int(r) >= n: - m = int(r) - d += (m - n) * (h + 1) - if d < 0: - raise newException(PunyError, "invalid label " & s) - n = m - for r in s.runes: - if int(r) < n: - inc d - if d < 0: - raise newException(PunyError, "invalid label " & s) - continue - if int(r) > n: - continue - var q = d - var k = Base - while true: - var t = k - bias - if t < TMin: - t = TMin - elif t > TMax: - t = TMax - if q < t: - break - result.add($encodeDigit(t + (q - t) mod (Base - t))) - q = (q - t) div (Base - t) - k += Base - result.add($encodeDigit(q)) - bias = adapt(d, h + 1, h == b) - d = 0 - inc h - dec remaining - inc d - inc n - -func encode*(s: string): string {.raises: [PunyError].} = - ## Encode a string that may contain Unicode. Prefix is empty. - result = encode("", s) - -func decode*(encoded: string): string {.raises: [PunyError].} = - ## Decode a Punycode-encoded string - var - n = InitialN - i = 0 - bias = InitialBias - var d = rfind(encoded, Delimiter) - var output: seq[Rune] - - if d > 0: - # found Delimiter - for j in 0..<d: - var c = encoded[j] # char - if not c.isBasic: - raise newException(PunyError, "Encoded contains a non-basic char") - output.add(Rune(c)) # add the character - inc d - else: - d = 0 # set to first index - - while (d < len(encoded)): - var oldi = i - var w = 1 - var k = Base - while true: - if d == len(encoded): - raise newException(PunyError, "Bad input: " & encoded) - var c = encoded[d]; inc d - var digit = int(decodeDigit(c)) - if digit > (high(int32) - i) div w: - raise newException(PunyError, "Too large a value: " & $digit) - i += digit * w - var t: int - if k <= bias: - t = TMin - elif k >= bias + TMax: - t = TMax - else: - t = k - bias - if digit < t: - break - w *= Base - t - k += Base - bias = adapt(i - oldi, len(output) + 1, oldi == 0) - - if i div (len(output) + 1) > high(int32) - n: - raise newException(PunyError, "Value too large") - - n += i div (len(output) + 1) - i = i mod (len(output) + 1) - insert(output, Rune(n), i) - inc i - - result = $output - -runnableExamples: - static: - block: - doAssert encode("") == "" - doAssert encode("a") == "a-" - doAssert encode("A") == "A-" - doAssert encode("3") == "3-" - doAssert encode("-") == "--" - doAssert encode("--") == "---" - doAssert encode("abc") == "abc-" - doAssert encode("London") == "London-" - doAssert encode("Lloyd-Atkinson") == "Lloyd-Atkinson-" - doAssert encode("This has spaces") == "This has spaces-" - doAssert encode("ü") == "tda" - doAssert encode("München") == "Mnchen-3ya" - doAssert encode("Mnchen-3ya") == "Mnchen-3ya-" - doAssert encode("München-Ost") == "Mnchen-Ost-9db" - doAssert encode("Bahnhof München-Ost") == "Bahnhof Mnchen-Ost-u6b" - block: - doAssert decode("") == "" - doAssert decode("a-") == "a" - doAssert decode("A-") == "A" - doAssert decode("3-") == "3" - doAssert decode("--") == "-" - doAssert decode("---") == "--" - doAssert decode("abc-") == "abc" - doAssert decode("London-") == "London" - doAssert decode("Lloyd-Atkinson-") == "Lloyd-Atkinson" - doAssert decode("This has spaces-") == "This has spaces" - doAssert decode("tda") == "ü" - doAssert decode("Mnchen-3ya") == "München" - doAssert decode("Mnchen-3ya-") == "Mnchen-3ya" - doAssert decode("Mnchen-Ost-9db") == "München-Ost" - doAssert decode("Bahnhof Mnchen-Ost-u6b") == "Bahnhof München-Ost" diff --git a/lib/pure/random.nim b/lib/pure/random.nim index de93f468f..3ec77d37e 100644 --- a/lib/pure/random.nim +++ b/lib/pure/random.nim @@ -66,29 +66,41 @@ runnableExamples: ## See also ## ======== ## * `std/sysrand module <sysrand.html>`_ for a cryptographically secure pseudorandom number generator -## * `mersenne module <mersenne.html>`_ for the Mersenne Twister random number generator ## * `math module <math.html>`_ for basic math routines ## * `stats module <stats.html>`_ for statistical analysis ## * `list of cryptographic and hashing modules <lib.html#pure-libraries-hashing>`_ ## in the standard library -import algorithm, math -import std/private/since +import std/[algorithm, math] +import std/private/[since, jsutils] when defined(nimPreviewSlimSystem): - import std/assertions + import std/[assertions] include system/inclrtl {.push debugger: off.} +template whenHasBigInt64(yes64, no64): untyped = + when defined(js): + when compiles(compileOption("jsbigint64")): + when compileOption("jsbigint64"): + yes64 + else: + no64 + else: + no64 + else: + yes64 -when defined(js): - type Ui = uint32 - const randMax = 4_294_967_295u32 -else: +whenHasBigInt64: type Ui = uint64 const randMax = 18_446_744_073_709_551_615u64 +do: + type Ui = uint32 + + const randMax = 4_294_967_295u32 + type Rand* = object ## State of a random number generator. @@ -106,17 +118,17 @@ type ## generator are **not** thread-safe! a0, a1: Ui -when defined(js): - var state = Rand( - a0: 0x69B4C98Cu32, - a1: 0xFED1DD30u32) # global for backwards compatibility -else: +whenHasBigInt64: const DefaultRandSeed = Rand( a0: 0x69B4C98CB8530805u64, a1: 0xFED1DD3004688D67CAu64) # racy for multi-threading but good enough for now: var state = DefaultRandSeed # global for backwards compatibility +do: + var state = Rand( + a0: 0x69B4C98Cu32, + a1: 0xFED1DD30u32) # global for backwards compatibility func isValid(r: Rand): bool {.inline.} = ## Check whether state of `r` is valid. @@ -209,10 +221,10 @@ proc skipRandomNumbers*(s: var Rand) = doAssert vals == [501737, 497901, 500683, 500157] - when defined(js): - const helper = [0xbeac0467u32, 0xd86b048bu32] - else: + whenHasBigInt64: const helper = [0xbeac0467eba5facbu64, 0xd86b048b86aa9922u64] + do: + const helper = [0xbeac0467u32, 0xd86b048bu32] var s0 = Ui 0 s1 = Ui 0 @@ -232,11 +244,14 @@ proc rand[T: uint | uint64](r: var Rand; max: T): T = let max = uint64(max) when T.high.uint64 == uint64.high: if max == uint64.high: return T(next(r)) + var iters = 0 while true: let x = next(r) # avoid `mod` bias - if x <= randMax - (randMax mod max): + if x <= randMax - (randMax mod max) or iters > 20: return T(x mod (max + 1)) + else: + inc iters proc rand*(r: var Rand; max: Natural): int {.benign.} = ## Returns a random integer in the range `0..max` using the given state. @@ -292,7 +307,13 @@ proc rand*(r: var Rand; max: range[0.0 .. high(float)]): float {.benign.} = let x = next(r) when defined(js): - result = (float(x) / float(high(uint32))) * max + when compiles(compileOption("jsbigint64")): + when compileOption("jsbigint64"): + result = (float(x) / float(high(uint64))) * max + else: + result = (float(x) / float(high(uint32))) * max + else: + result = (float(x) / float(high(uint32))) * max else: let u = (0x3FFu64 shl 52u64) or (x shr 12u64) result = (cast[float](u) - 1.0) * max @@ -338,9 +359,9 @@ proc rand*[T: Ordinal or SomeFloat](r: var Rand; x: HSlice[T, T]): T = when T is SomeFloat: result = rand(r, x.b - x.a) + x.a else: # Integers and Enum types - when defined(js): + whenJsNoBigInt64: result = cast[T](rand(r, cast[uint](x.b) - cast[uint](x.a)) + cast[uint](x.a)) - else: + do: result = cast[T](rand(r, cast[uint64](x.b) - cast[uint64](x.a)) + cast[uint64](x.a)) proc rand*[T: Ordinal or SomeFloat](x: HSlice[T, T]): T = @@ -379,14 +400,11 @@ proc rand*[T: Ordinal](r: var Rand; t: typedesc[T]): T {.since: (1, 7, 1).} = when T is range or T is enum: result = rand(r, low(T)..high(T)) elif T is bool: - when defined(js): - result = (r.next or 0) < 0 - else: - result = cast[int64](r.next) < 0 + result = r.next < randMax div 2 else: - when defined(js): + whenJsNoBigInt64: result = cast[T](r.next shr (sizeof(uint)*8 - sizeof(T)*8)) - else: + do: result = cast[T](r.next shr (sizeof(uint64)*8 - sizeof(T)*8)) proc rand*[T: Ordinal](t: typedesc[T]): T = @@ -671,7 +689,7 @@ when not defined(standalone): import std/[hashes, os, sysrand, monotimes] when compileOption("threads"): - import locks + import std/locks var baseSeedLock: Lock baseSeedLock.initLock diff --git a/lib/pure/rationals.nim b/lib/pure/rationals.nim index aba0f4ce4..5f806bd70 100644 --- a/lib/pure/rationals.nim +++ b/lib/pure/rationals.nim @@ -21,7 +21,7 @@ runnableExamples: doAssert r1 * r2 == -3 // 8 doAssert r1 / r2 == -2 // 3 -import math, hashes +import std/[math, hashes] when defined(nimPreviewSlimSystem): import std/assertions @@ -40,16 +40,16 @@ func reduce*[T: SomeInteger](x: var Rational[T]) = reduce(r) doAssert r.num == 1 doAssert r.den == 2 - + if x.den == 0: + raise newException(DivByZeroDefect, "division by zero") let common = gcd(x.num, x.den) if x.den > 0: x.num = x.num div common x.den = x.den div common - elif x.den < 0: - x.num = -x.num div common - x.den = -x.den div common - else: - raise newException(DivByZeroDefect, "division by zero") + when T isnot SomeUnsignedInt: + if x.den < 0: + x.num = -x.num div common + x.den = -x.den div common func initRational*[T: SomeInteger](num, den: T): Rational[T] = ## Creates a new rational number with numerator `num` and denominator `den`. @@ -318,3 +318,23 @@ func hash*[T](x: Rational[T]): Hash = h = h !& hash(copy.num) h = h !& hash(copy.den) result = !$h + +func `^`*[T: SomeInteger](x: Rational[T], y: T): Rational[T] = + ## Computes `x` to the power of `y`. + ## + ## The exponent `y` must be an integer. Negative exponents are supported + ## but floating point exponents are not. + runnableExamples: + doAssert (-3 // 5) ^ 0 == (1 // 1) + doAssert (-3 // 5) ^ 1 == (-3 // 5) + doAssert (-3 // 5) ^ 2 == (9 // 25) + doAssert (-3 // 5) ^ -2 == (25 // 9) + + if y >= 0: + result.num = x.num ^ y + result.den = x.den ^ y + else: + result.num = x.den ^ -y + result.den = x.num ^ -y + # Note that all powers of reduced rationals are already reduced, + # so we don't need to call reduce() here diff --git a/lib/pure/reservedmem.nim b/lib/pure/reservedmem.nim index 99fbe1429..ffa0128dc 100644 --- a/lib/pure/reservedmem.nim +++ b/lib/pure/reservedmem.nim @@ -9,7 +9,7 @@ ## :Authors: Zahary Karadjov ## -## This module provides utilities for reserving a portions of the +## This module provides utilities for reserving portions of the ## address space of a program without consuming physical memory. ## It can be used to implement a dynamically resizable buffer that ## is guaranteed to remain in the same memory location. The buffer @@ -18,7 +18,10 @@ ## ## Unstable API. -from os import raiseOSError, osLastError +from std/oserrors import raiseOSError, osLastError + +when defined(nimPreviewSlimSystem): + import std/assertions template distance*(lhs, rhs: pointer): int = cast[int](rhs) - cast[int](lhs) @@ -41,7 +44,7 @@ type mem: ReservedMem when defined(windows): - import winlean + import std/winlean import std/private/win_getsysteminfo proc getAllocationGranularity: uint = @@ -65,7 +68,7 @@ when defined(windows): raiseOSError(osLastError()) else: - import posix + import std/posix let allocationGranularity = sysconf(SC_PAGESIZE) diff --git a/lib/pure/ropes.nim b/lib/pure/ropes.nim index b973fd222..8750aca87 100644 --- a/lib/pure/ropes.nim +++ b/lib/pure/ropes.nim @@ -17,7 +17,7 @@ ## runtime efficiency. include system/inclrtl -import streams +import std/streams when defined(nimPreviewSlimSystem): import std/[syncio, formatfloat, assertions] diff --git a/lib/pure/segfaults.nim b/lib/pure/segfaults.nim index b0eac2299..65b059e86 100644 --- a/lib/pure/segfaults.nim +++ b/lib/pure/segfaults.nim @@ -26,7 +26,7 @@ se.msg = "Could not access value because it is nil." when defined(windows): include "../system/ansi_c" - import winlean + import std/winlean const EXCEPTION_ACCESS_VIOLATION = DWORD(0xc0000005'i32) @@ -65,7 +65,7 @@ when defined(windows): c_signal(SIGSEGV, segfaultHandler) else: - import posix + import std/posix var sa: Sigaction diff --git a/lib/pure/selectors.nim b/lib/pure/selectors.nim index ab7e104fc..ac180e2bd 100644 --- a/lib/pure/selectors.nim +++ b/lib/pure/selectors.nim @@ -27,7 +27,8 @@ ## ## TODO: `/dev/poll`, `event ports` and filesystem events. -import os, nativesockets +import std/nativesockets +import std/oserrors when defined(nimPreviewSlimSystem): import std/assertions @@ -36,7 +37,7 @@ const hasThreadSupport = compileOption("threads") and defined(threadsafe) const ioselSupportedPlatform* = defined(macosx) or defined(freebsd) or defined(netbsd) or defined(openbsd) or - defined(dragonfly) or + defined(dragonfly) or defined(nuttx) or (defined(linux) and not defined(android) and not defined(emscripten)) ## This constant is used to determine whether the destination platform is ## fully supported by `ioselectors` module. @@ -204,12 +205,11 @@ when defined(nimdoc): ## to `value`. This `value` can be modified in the scope of ## the `withData` call. ## - ## .. code-block:: nim - ## + ## ```nim ## s.withData(fd, value) do: ## # block is executed only if `fd` registered in selector `s` ## value.uid = 1000 - ## + ## ``` template withData*[T](s: Selector[T], fd: SocketHandle|int, value, body1, body2: untyped) = @@ -217,15 +217,14 @@ when defined(nimdoc): ## to `value`. This `value` can be modified in the scope of ## the `withData` call. ## - ## .. code-block:: nim - ## + ## ```nim ## s.withData(fd, value) do: ## # block is executed only if `fd` registered in selector `s`. ## value.uid = 1000 ## do: ## # block is executed if `fd` not registered in selector `s`. ## raise - ## + ## ``` proc contains*[T](s: Selector[T], fd: SocketHandle|int): bool {.inline.} = ## Determines whether selector contains a file descriptor. @@ -236,9 +235,9 @@ when defined(nimdoc): ## For *poll* and *select* selectors `-1` is returned. else: - import strutils + import std/strutils when hasThreadSupport: - import locks + import std/locks type SharedArray[T] = UncheckedArray[T] @@ -246,8 +245,8 @@ else: proc allocSharedArray[T](nsize: int): ptr SharedArray[T] = result = cast[ptr SharedArray[T]](allocShared0(sizeof(T) * nsize)) - proc reallocSharedArray[T](sa: ptr SharedArray[T], nsize: int): ptr SharedArray[T] = - result = cast[ptr SharedArray[T]](reallocShared(sa, sizeof(T) * nsize)) + proc reallocSharedArray[T](sa: ptr SharedArray[T], oldsize, nsize: int): ptr SharedArray[T] = + result = cast[ptr SharedArray[T]](reallocShared0(sa, oldsize * sizeof(T), sizeof(T) * nsize)) proc deallocSharedArray[T](sa: ptr SharedArray[T]) = deallocShared(cast[pointer](sa)) @@ -289,7 +288,7 @@ else: setBlocking(fd.SocketHandle, false) when not defined(windows): - import posix + import std/posix template setKey(s, pident, pevents, pparam, pdata: untyped) = var skey = addr(s.fds[pident]) @@ -327,7 +326,7 @@ else: doAssert(timeout >= -1, "Cannot select with a negative value, got: " & $timeout) when defined(linux) or defined(windows) or defined(macosx) or defined(bsd) or - defined(solaris) or defined(zephyr) or defined(freertos): + defined(solaris) or defined(zephyr) or defined(freertos) or defined(nuttx) or defined(haiku): template maxDescriptors*(): int = ## Returns the maximum number of active file descriptors for the current ## process. This involves a system call. For now `maxDescriptors` is @@ -343,7 +342,18 @@ else: res = int(fdLim.rlim_cur) - 1 res - when defined(linux) and not defined(emscripten): + when defined(nimIoselector): + when nimIoselector == "epoll": + include ioselects/ioselectors_epoll + elif nimIoselector == "kqueue": + include ioselects/ioselectors_kqueue + elif nimIoselector == "poll": + include ioselects/ioselectors_poll + elif nimIoselector == "select": + include ioselects/ioselectors_select + else: + {.fatal: "Unknown nimIoselector specified by define.".} + elif defined(linux) and not defined(emscripten): include ioselects/ioselectors_epoll elif bsdPlatform: include ioselects/ioselectors_kqueue @@ -359,5 +369,7 @@ else: include ioselects/ioselectors_select elif defined(zephyr): include ioselects/ioselectors_poll + elif defined(nuttx): + include ioselects/ioselectors_epoll else: include ioselects/ioselectors_poll diff --git a/lib/pure/smtp.nim b/lib/pure/smtp.nim deleted file mode 100644 index f5196ce1e..000000000 --- a/lib/pure/smtp.nim +++ /dev/null @@ -1,358 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2012 Dominik Picheta -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## This module implements the SMTP client protocol as specified by RFC 5321, -## this can be used to send mail to any SMTP Server. -## -## This module also implements the protocol used to format messages, -## as specified by RFC 2822. -## -## Example gmail use: -## -## -## .. code-block:: Nim -## var msg = createMessage("Hello from Nim's SMTP", -## "Hello!.\n Is this awesome or what?", -## @["foo@gmail.com"]) -## let smtpConn = newSmtp(useSsl = true, debug=true) -## smtpConn.connect("smtp.gmail.com", Port 465) -## smtpConn.auth("username", "password") -## smtpConn.sendmail("username@gmail.com", @["foo@gmail.com"], $msg) -## -## -## Example for startTls use: -## -## -## .. code-block:: Nim -## var msg = createMessage("Hello from Nim's SMTP", -## "Hello!.\n Is this awesome or what?", -## @["foo@gmail.com"]) -## let smtpConn = newSmtp(debug=true) -## smtpConn.connect("smtp.mailtrap.io", Port 2525) -## smtpConn.startTls() -## smtpConn.auth("username", "password") -## smtpConn.sendmail("username@gmail.com", @["foo@gmail.com"], $msg) -## -## -## For SSL support this module relies on OpenSSL. If you want to -## enable SSL, compile with `-d:ssl`. - -import net, strutils, strtabs, base64, os, strutils -import asyncnet, asyncdispatch - -export Port - -type - Message* = object - msgTo: seq[string] - msgCc: seq[string] - msgSubject: string - msgOtherHeaders: StringTableRef - msgBody: string - - ReplyError* = object of IOError - - SmtpBase[SocketType] = ref object - sock: SocketType - address: string - debug: bool - - Smtp* = SmtpBase[Socket] - AsyncSmtp* = SmtpBase[AsyncSocket] - -proc containsNewline(xs: seq[string]): bool = - for x in xs: - if x.contains({'\c', '\L'}): - return true - -proc debugSend*(smtp: Smtp | AsyncSmtp, cmd: string) {.multisync.} = - ## Sends `cmd` on the socket connected to the SMTP server. - ## - ## If the `smtp` object was created with `debug` enabled, - ## debugSend will invoke `echo("C:" & cmd)` before sending. - ## - ## This is a lower level proc and not something that you typically - ## would need to call when using this module. One exception to - ## this is if you are implementing any - ## `SMTP extensions<https://en.wikipedia.org/wiki/Extended_SMTP>`_. - - if smtp.debug: - echo("C:" & cmd) - await smtp.sock.send(cmd) - -proc debugRecv*(smtp: Smtp | AsyncSmtp): Future[string] {.multisync.} = - ## Receives a line of data from the socket connected to the - ## SMTP server. - ## - ## If the `smtp` object was created with `debug` enabled, - ## debugRecv will invoke `echo("S:" & result.string)` after - ## the data is received. - ## - ## This is a lower level proc and not something that you typically - ## would need to call when using this module. One exception to - ## this is if you are implementing any - ## `SMTP extensions<https://en.wikipedia.org/wiki/Extended_SMTP>`_. - ## - ## See `checkReply(reply)<#checkReply,AsyncSmtp,string>`_. - result = await smtp.sock.recvLine() - if smtp.debug: - echo("S:" & result) - -proc quitExcpt(smtp: Smtp, msg: string) = - smtp.debugSend("QUIT") - raise newException(ReplyError, msg) - -const compiledWithSsl = defined(ssl) - -when not defined(ssl): - let defaultSSLContext: SslContext = nil -else: - var defaultSSLContext {.threadvar.}: SslContext - - proc getSSLContext(): SslContext = - if defaultSSLContext == nil: - defaultSSLContext = newContext(verifyMode = CVerifyNone) - result = defaultSSLContext - -proc createMessage*(mSubject, mBody: string, mTo, mCc: seq[string], - otherHeaders: openArray[tuple[name, value: string]]): Message = - ## Creates a new MIME compliant message. - ## - ## You need to make sure that `mSubject`, `mTo` and `mCc` don't contain - ## any newline characters. Failing to do so will raise `AssertionDefect`. - doAssert(not mSubject.contains({'\c', '\L'}), - "'mSubject' shouldn't contain any newline characters") - doAssert(not (mTo.containsNewline() or mCc.containsNewline()), - "'mTo' and 'mCc' shouldn't contain any newline characters") - - result.msgTo = mTo - result.msgCc = mCc - result.msgSubject = mSubject - result.msgBody = mBody - result.msgOtherHeaders = newStringTable() - for n, v in items(otherHeaders): - result.msgOtherHeaders[n] = v - -proc createMessage*(mSubject, mBody: string, mTo, - mCc: seq[string] = @[]): Message = - ## Alternate version of the above. - ## - ## You need to make sure that `mSubject`, `mTo` and `mCc` don't contain - ## any newline characters. Failing to do so will raise `AssertionDefect`. - doAssert(not mSubject.contains({'\c', '\L'}), - "'mSubject' shouldn't contain any newline characters") - doAssert(not (mTo.containsNewline() or mCc.containsNewline()), - "'mTo' and 'mCc' shouldn't contain any newline characters") - result.msgTo = mTo - result.msgCc = mCc - result.msgSubject = mSubject - result.msgBody = mBody - result.msgOtherHeaders = newStringTable() - -proc `$`*(msg: Message): string = - ## stringify for `Message`. - result = "" - if msg.msgTo.len() > 0: - result = "TO: " & msg.msgTo.join(", ") & "\c\L" - if msg.msgCc.len() > 0: - result.add("CC: " & msg.msgCc.join(", ") & "\c\L") - # TODO: Folding? i.e when a line is too long, shorten it... - result.add("Subject: " & msg.msgSubject & "\c\L") - for key, value in pairs(msg.msgOtherHeaders): - result.add(key & ": " & value & "\c\L") - - result.add("\c\L") - result.add(msg.msgBody) - -proc newSmtp*(useSsl = false, debug = false, sslContext: SslContext = nil): Smtp = - ## Creates a new `Smtp` instance. - new result - result.debug = debug - result.sock = newSocket() - if useSsl: - when compiledWithSsl: - if sslContext == nil: - getSSLContext().wrapSocket(result.sock) - else: - sslContext.wrapSocket(result.sock) - else: - {.error: "SMTP module compiled without SSL support".} - -proc newAsyncSmtp*(useSsl = false, debug = false, sslContext: SslContext = nil): AsyncSmtp = - ## Creates a new `AsyncSmtp` instance. - new result - result.debug = debug - result.sock = newAsyncSocket() - if useSsl: - when compiledWithSsl: - if sslContext == nil: - getSSLContext().wrapSocket(result.sock) - else: - sslContext.wrapSocket(result.sock) - else: - {.error: "SMTP module compiled without SSL support".} - -proc quitExcpt(smtp: AsyncSmtp, msg: string): Future[void] = - var retFuture = newFuture[void]() - var sendFut = smtp.debugSend("QUIT") - sendFut.callback = - proc () = - retFuture.fail(newException(ReplyError, msg)) - return retFuture - -proc checkReply*(smtp: Smtp | AsyncSmtp, reply: string) {.multisync.} = - ## Calls `debugRecv<#debugRecv,AsyncSmtp>`_ and checks that the received - ## data starts with `reply`. If the received data does not start - ## with `reply`, then a `QUIT` command will be sent to the SMTP - ## server and a `ReplyError` exception will be raised. - ## - ## This is a lower level proc and not something that you typically - ## would need to call when using this module. One exception to - ## this is if you are implementing any - ## `SMTP extensions<https://en.wikipedia.org/wiki/Extended_SMTP>`_. - var line = await smtp.debugRecv() - if not line.startsWith(reply): - await quitExcpt(smtp, "Expected " & reply & " reply, got: " & line) - -proc helo*(smtp: Smtp | AsyncSmtp) {.multisync.} = - # Sends the HELO request - await smtp.debugSend("HELO " & smtp.address & "\c\L") - await smtp.checkReply("250") - -proc recvEhlo(smtp: Smtp | AsyncSmtp): Future[bool] {.multisync.} = - ## Skips "250-" lines, read until "250 " found. - ## Return `true` if server supports `EHLO`, false otherwise. - while true: - var line = await smtp.sock.recvLine() - if smtp.debug: - echo("S:" & line) - if line.startsWith("250-"): continue - elif line.startsWith("250 "): return true # last line - else: return false - -proc ehlo*(smtp: Smtp | AsyncSmtp): Future[bool] {.multisync.} = - ## Sends EHLO request. - await smtp.debugSend("EHLO " & smtp.address & "\c\L") - return await smtp.recvEhlo() - -proc connect*(smtp: Smtp | AsyncSmtp, - address: string, port: Port) {.multisync.} = - ## Establishes a connection with a SMTP server. - ## May fail with ReplyError or with a socket error. - smtp.address = address - await smtp.sock.connect(address, port) - await smtp.checkReply("220") - let speaksEsmtp = await smtp.ehlo() - if not speaksEsmtp: - await smtp.helo() - -proc startTls*(smtp: Smtp | AsyncSmtp, sslContext: SslContext = nil) {.multisync.} = - ## Put the SMTP connection in TLS (Transport Layer Security) mode. - ## May fail with ReplyError - await smtp.debugSend("STARTTLS\c\L") - await smtp.checkReply("220") - when compiledWithSsl: - if sslContext == nil: - getSSLContext().wrapConnectedSocket(smtp.sock, handshakeAsClient) - else: - sslContext.wrapConnectedSocket(smtp.sock, handshakeAsClient) - let speaksEsmtp = await smtp.ehlo() - if not speaksEsmtp: - await smtp.helo() - else: - {.error: "SMTP module compiled without SSL support".} - -proc auth*(smtp: Smtp | AsyncSmtp, username, password: string) {.multisync.} = - ## Sends an AUTH command to the server to login as the `username` - ## using `password`. - ## May fail with ReplyError. - - await smtp.debugSend("AUTH LOGIN\c\L") - await smtp.checkReply("334") # TODO: Check whether it's asking for the "Username:" - # i.e "334 VXNlcm5hbWU6" - await smtp.debugSend(encode(username) & "\c\L") - await smtp.checkReply("334") # TODO: Same as above, only "Password:" (I think?) - - await smtp.debugSend(encode(password) & "\c\L") - await smtp.checkReply("235") # Check whether the authentication was successful. - -proc sendMail*(smtp: Smtp | AsyncSmtp, fromAddr: string, - toAddrs: seq[string], msg: string) {.multisync.} = - ## Sends `msg` from `fromAddr` to the addresses specified in `toAddrs`. - ## Messages may be formed using `createMessage` by converting the - ## Message into a string. - ## - ## You need to make sure that `fromAddr` and `toAddrs` don't contain - ## any newline characters. Failing to do so will raise `AssertionDefect`. - doAssert(not (toAddrs.containsNewline() or fromAddr.contains({'\c', '\L'})), - "'toAddrs' and 'fromAddr' shouldn't contain any newline characters") - - await smtp.debugSend("MAIL FROM:<" & fromAddr & ">\c\L") - await smtp.checkReply("250") - for address in items(toAddrs): - await smtp.debugSend("RCPT TO:<" & address & ">\c\L") - await smtp.checkReply("250") - - # Send the message - await smtp.debugSend("DATA " & "\c\L") - await smtp.checkReply("354") - await smtp.sock.send(msg & "\c\L") - await smtp.debugSend(".\c\L") - await smtp.checkReply("250") - -proc close*(smtp: Smtp | AsyncSmtp) {.multisync.} = - ## Disconnects from the SMTP server and closes the socket. - await smtp.debugSend("QUIT\c\L") - smtp.sock.close() - -when not defined(testing) and isMainModule: - # To test with a real SMTP service, create a smtp.ini file, e.g.: - # username = "" - # password = "" - # smtphost = "smtp.gmail.com" - # port = 465 - # use_tls = true - # sender = "" - # recipient = "" - - import parsecfg - - proc `[]`(c: Config, key: string): string = c.getSectionValue("", key) - - let - conf = loadConfig("smtp.ini") - msg = createMessage("Hello from Nim's SMTP!", - "Hello!\n Is this awesome or what?", @[conf["recipient"]]) - - assert conf["smtphost"] != "" - - proc async_test() {.async.} = - let client = newAsyncSmtp( - conf["use_tls"].parseBool, - debug = true - ) - await client.connect(conf["smtphost"], conf["port"].parseInt.Port) - await client.auth(conf["username"], conf["password"]) - await client.sendMail(conf["sender"], @[conf["recipient"]], $msg) - await client.close() - echo "async email sent" - - proc sync_test() = - var smtpConn = newSmtp( - conf["use_tls"].parseBool, - debug = true - ) - smtpConn.connect(conf["smtphost"], conf["port"].parseInt.Port) - smtpConn.auth(conf["username"], conf["password"]) - smtpConn.sendMail(conf["sender"], @[conf["recipient"]], $msg) - smtpConn.close() - echo "sync email sent" - - waitFor async_test() - sync_test() diff --git a/lib/pure/smtp.nim.cfg b/lib/pure/smtp.nim.cfg deleted file mode 100644 index 521e21de4..000000000 --- a/lib/pure/smtp.nim.cfg +++ /dev/null @@ -1 +0,0 @@ --d:ssl diff --git a/lib/pure/ssl_certs.nim b/lib/pure/ssl_certs.nim index c40eadf04..d60cd22eb 100644 --- a/lib/pure/ssl_certs.nim +++ b/lib/pure/ssl_certs.nim @@ -10,7 +10,7 @@ ## The default locations can be overridden using the SSL_CERT_FILE and ## SSL_CERT_DIR environment variables. -import os, strutils +import std/[os, strutils] # FWIW look for files before scanning entire dirs. @@ -88,7 +88,7 @@ when defined(haiku): proc find_paths_etc(architecture: cstring, baseDirectory: cint, subPath: cstring, flags: uint32, paths: var ptr UncheckedArray[cstring], - pathCount: var csize): int32 + pathCount: var csize_t): int32 {.importc, header: "<FindDirectory.h>".} proc free(p: pointer) {.importc, header: "<stdlib.h>".} @@ -137,7 +137,7 @@ iterator scanSSLCertificates*(useEnvVars = false): string = else: var paths: ptr UncheckedArray[cstring] - size: csize + size: csize_t let err = find_paths_etc( nil, B_FIND_PATH_DATA_DIRECTORY, "ssl/CARootCertificates.pem", B_FIND_PATH_EXISTING_ONLY, paths, size @@ -150,7 +150,7 @@ iterator scanSSLCertificates*(useEnvVars = false): string = # Certificates management on windows # when defined(windows) or defined(nimdoc): # -# import openssl +# import std/openssl # # type # PCCertContext {.final, pure.} = pointer diff --git a/lib/pure/stats.nim b/lib/pure/stats.nim index 7f797529d..6a4fd8f01 100644 --- a/lib/pure/stats.nim +++ b/lib/pure/stats.nim @@ -53,7 +53,7 @@ runnableExamples: doAssert statistics.kurtosis() ~= -1.0 doAssert statistics.kurtosisS() ~= -0.7000000000000008 -from math import FloatClass, sqrt, pow, round +from std/math import FloatClass, sqrt, pow, round when defined(nimPreviewSlimSystem): import std/[assertions, formatfloat] diff --git a/lib/pure/streams.nim b/lib/pure/streams.nim index 50b3085d2..56f49d7b1 100644 --- a/lib/pure/streams.nim +++ b/lib/pure/streams.nim @@ -15,6 +15,11 @@ ## Other modules may provide other implementations for this standard ## stream interface. ## +## .. warning:: Due to the use of `pointer`, the `readData`, `peekData` and +## `writeData` interfaces are not available on the compile-time VM, and must +## be cast from a `ptr string` on the JS backend. However, `readDataStr` is +## available generally in place of `readData`. +## ## Basic usage ## =========== ## @@ -27,67 +32,67 @@ ## StringStream example ## -------------------- ## -## .. code-block:: Nim -## -## import std/streams +## ```Nim +## import std/streams ## -## var strm = newStringStream("""The first line -## the second line -## the third line""") +## var strm = newStringStream("""The first line +## the second line +## the third line""") ## -## var line = "" +## var line = "" ## -## while strm.readLine(line): -## echo line +## while strm.readLine(line): +## echo line ## -## # Output: -## # The first line -## # the second line -## # the third line +## # Output: +## # The first line +## # the second line +## # the third line ## -## strm.close() +## strm.close() +## ``` ## ## FileStream example ## ------------------ ## ## Write file stream example: ## -## .. code-block:: Nim +## ```Nim +## import std/streams ## -## import std/streams +## var strm = newFileStream("somefile.txt", fmWrite) +## var line = "" ## -## var strm = newFileStream("somefile.txt", fmWrite) -## var line = "" +## if not isNil(strm): +## strm.writeLine("The first line") +## strm.writeLine("the second line") +## strm.writeLine("the third line") +## strm.close() ## -## if not isNil(strm): -## strm.writeLine("The first line") -## strm.writeLine("the second line") -## strm.writeLine("the third line") -## strm.close() -## -## # Output (somefile.txt): -## # The first line -## # the second line -## # the third line +## # Output (somefile.txt): +## # The first line +## # the second line +## # the third line +## ``` ## ## Read file stream example: ## -## .. code-block:: Nim -## -## import std/streams +## ```Nim +## import std/streams ## -## var strm = newFileStream("somefile.txt", fmRead) -## var line = "" +## var strm = newFileStream("somefile.txt", fmRead) +## var line = "" ## -## if not isNil(strm): -## while strm.readLine(line): -## echo line -## strm.close() +## if not isNil(strm): +## while strm.readLine(line): +## echo line +## strm.close() ## -## # Output: -## # The first line -## # the second line -## # the third line +## # Output: +## # The first line +## # the second line +## # the third line +## ``` ## ## See also ## ======== @@ -98,6 +103,7 @@ import std/private/since when defined(nimPreviewSlimSystem): import std/syncio + export FileMode proc newEIO(msg: string): owned(ref IOError) = new(result) @@ -114,7 +120,7 @@ type ## * That these fields here shouldn't be used directly. ## They are accessible so that a stream implementation can override them. closeImpl*: proc (s: Stream) - {.nimcall, raises: [Exception, IOError, OSError], tags: [WriteIOEffect], gcsafe.} + {.nimcall, raises: [IOError, OSError], tags: [WriteIOEffect], gcsafe.} atEndImpl*: proc (s: Stream): bool {.nimcall, raises: [Defect, IOError, OSError], tags: [], gcsafe.} setPositionImpl*: proc (s: Stream, pos: int) @@ -173,10 +179,20 @@ proc close*(s: Stream) = ## See also: ## * `flush proc <#flush,Stream>`_ runnableExamples: - var strm = newStringStream("The first line\nthe second line\nthe third line") - ## do something... - strm.close() - if not isNil(s.closeImpl): s.closeImpl(s) + block: + let strm = newStringStream("The first line\nthe second line\nthe third line") + ## do something... + strm.close() + + block: + let strm = newFileStream("amissingfile.txt") + # deferring works even if newFileStream fails + defer: strm.close() + if not isNil(strm): + ## do something... + + if not isNil(s) and not isNil(s.closeImpl): + s.closeImpl(s) proc atEnd*(s: Stream): bool = ## Checks if more data can be read from `s`. Returns ``true`` if all data has @@ -337,9 +353,9 @@ proc write*[T](s: Stream, x: T) = ## **Note:** Not available for JS backend. Use `write(Stream, string) ## <#write,Stream,string>`_ for now. ## - ## .. code-block:: Nim - ## - ## s.writeData(s, unsafeAddr(x), sizeof(x)) + ## ```Nim + ## s.writeData(s, unsafeAddr(x), sizeof(x)) + ## ``` runnableExamples: var strm = newStringStream("") strm.write("abcde") @@ -927,10 +943,14 @@ proc peekFloat64*(s: Stream): float64 = proc readStrPrivate(s: Stream, length: int, str: var string) = if length > len(str): setLen(str, length) - when defined(js): - let L = readData(s, addr(str), length) + var L: int + when nimvm: + L = readDataStr(s, str, 0..length-1) else: - let L = readData(s, cstring(str), length) + when defined(js): + L = readData(s, addr(str), length) + else: + L = readData(s, cstring(str), length) if L != len(str): setLen(str, L) proc readStr*(s: Stream, length: int, str: var string) {.since: (1, 3).} = @@ -1471,7 +1491,7 @@ when false: # do not import windows as this increases compile times: discard else: - import posix + import std/posix proc hsSetPosition(s: FileHandleStream, pos: int) = discard lseek(s.handle, pos, SEEK_SET) @@ -1519,7 +1539,7 @@ when false: of fmReadWrite: flags = O_RDWR or int(O_CREAT) of fmReadWriteExisting: flags = O_RDWR of fmAppend: flags = O_WRONLY or int(O_CREAT) or O_APPEND - static: doAssert false # handle bug #17888 + static: raiseAssert "unreachable" # handle bug #17888 var handle = open(filename, flags) if handle < 0: raise newEOS("posix.open() call failed") result = newFileHandleStream(handle) diff --git a/lib/pure/streamwrapper.nim b/lib/pure/streamwrapper.nim index a6c1901d2..99752a9ab 100644 --- a/lib/pure/streamwrapper.nim +++ b/lib/pure/streamwrapper.nim @@ -11,7 +11,7 @@ ## ## **Since** version 1.2. -import deques, streams +import std/[deques, streams] when defined(nimPreviewSlimSystem): import std/assertions @@ -91,14 +91,14 @@ proc newPipeOutStream*[T](s: sink (ref T)): owned PipeOutStream[T] = ## when setPosition/getPosition is called or write operation is performed. ## ## Example: - ## - ## .. code-block:: Nim + ## ```Nim ## import std/[osproc, streamwrapper] ## var ## p = startProcess(exePath) ## outStream = p.outputStream().newPipeOutStream() ## echo outStream.peekChar ## p.close() + ## ``` assert s.readDataImpl != nil diff --git a/lib/pure/strformat.nim b/lib/pure/strformat.nim index fe3cfdab0..7d093ebb3 100644 --- a/lib/pure/strformat.nim +++ b/lib/pure/strformat.nim @@ -133,13 +133,14 @@ runnableExamples: An expression like `&"{key} is {value:arg} {{z}}"` is transformed into: -.. code-block:: nim + ```nim var temp = newStringOfCap(educatedCapGuess) temp.formatValue(key, "") temp.add(" is ") temp.formatValue(value, arg) temp.add(" {z}") temp + ``` Parts of the string that are enclosed in the curly braces are interpreted as Nim code. To escape a `{` or `}`, double it. @@ -171,9 +172,9 @@ For strings and numeric types the optional argument is a so-called # Standard format specifiers for strings, integers and floats -The general form of a standard format specifier is:: +The general form of a standard format specifier is: - [[fill]align][sign][#][0][minimumwidth][.precision][type] + [[fill]align][sign][#][0][minimumwidth][.precision][type] The square brackets `[]` indicate an optional element. @@ -263,6 +264,12 @@ The available floating point presentation types are: exponent notation. `G` General format. Same as `g` except it switches to `E` if the number gets to large. +`i` Complex General format. This is only supported for + complex numbers, which it prints using the mathematical + (RE+IMj) format. The real and imaginary parts are printed + using the general format `g` by default, but it is + possible to combine this format with one of the other + formats (e.g `jf`). (None) Similar to `g`, except that it prints at least one digit after the decimal point. ================= ==================================================== @@ -272,13 +279,14 @@ The available floating point presentation types are: Because of the well defined order how templates and macros are expanded, strformat cannot expand template arguments: -.. code-block:: nim + ```nim template myTemplate(arg: untyped): untyped = echo "arg is: ", arg echo &"--- {arg} ---" let x = "abc" myTemplate(x) + ``` First the template `myTemplate` is expanded, where every identifier `arg` is substituted with its argument. The `arg` inside the @@ -289,12 +297,13 @@ identifier that cannot be resolved anymore. The workaround for this is to bind the template argument to a new local variable. -.. code-block:: nim + ```nim template myTemplate(arg: untyped): untyped = block: let arg1 {.inject.} = arg echo "arg is: ", arg1 echo &"--- {arg1} ---" + ``` The use of `{.inject.}` here is necessary again because of template expansion order and hygienic templates. But since we generally want to @@ -313,8 +322,8 @@ help with readability, since there is only so much you can cram into single letter DSLs. ]## -import macros, parseutils, unicode -import strutils except format +import std/[macros, parseutils, unicode] +import std/strutils except format when defined(nimPreviewSlimSystem): import std/assertions @@ -423,9 +432,9 @@ proc formatInt(n: SomeNumber; radix: int; spec: StandardFormatSpecifier): string proc parseStandardFormatSpecifier*(s: string; start = 0; ignoreUnknownSuffix = false): StandardFormatSpecifier = ## An exported helper proc that parses the "standard format specifiers", - ## as specified by the grammar:: + ## as specified by the grammar: ## - ## [[fill]align][sign][#][0][minimumwidth][.precision][type] + ## [[fill]align][sign][#][0][minimumwidth][.precision][type] ## ## This is only of interest if you want to write a custom `format` proc that ## should support the standard format specifiers. If `ignoreUnknownSuffix` is true, @@ -472,50 +481,48 @@ proc parseStandardFormatSpecifier*(s: string; start = 0; raise newException(ValueError, "invalid format string, cannot parse: " & s[i..^1]) +proc toRadix(typ: char): int = + case typ + of 'x', 'X': 16 + of 'd', '\0': 10 + of 'o': 8 + of 'b': 2 + else: + raise newException(ValueError, + "invalid type in format string for number, expected one " & + " of 'x', 'X', 'b', 'd', 'o' but got: " & typ) + proc formatValue*[T: SomeInteger](result: var string; value: T; - specifier: string) = + specifier: static string) = ## Standard format implementation for `SomeInteger`. It makes little ## sense to call this directly, but it is required to exist ## by the `&` macro. - if specifier.len == 0: + when specifier.len == 0: result.add $value - return - let spec = parseStandardFormatSpecifier(specifier) - var radix = 10 - case spec.typ - of 'x', 'X': radix = 16 - of 'd', '\0': discard - of 'b': radix = 2 - of 'o': radix = 8 else: - raise newException(ValueError, - "invalid type in format string for number, expected one " & - " of 'x', 'X', 'b', 'd', 'o' but got: " & spec.typ) - result.add formatInt(value, radix, spec) + const + spec = parseStandardFormatSpecifier(specifier) + radix = toRadix(spec.typ) -proc formatValue*(result: var string; value: SomeFloat; specifier: string) = - ## Standard format implementation for `SomeFloat`. It makes little + result.add formatInt(value, radix, spec) + +proc formatValue*[T: SomeInteger](result: var string; value: T; + specifier: string) = + ## Standard format implementation for `SomeInteger`. It makes little ## sense to call this directly, but it is required to exist ## by the `&` macro. if specifier.len == 0: result.add $value - return - let spec = parseStandardFormatSpecifier(specifier) - - var fmode = ffDefault - case spec.typ - of 'e', 'E': - fmode = ffScientific - of 'f', 'F': - fmode = ffDecimal - of 'g', 'G': - fmode = ffDefault - of '\0': discard else: - raise newException(ValueError, - "invalid type in format string for number, expected one " & - " of 'e', 'E', 'f', 'F', 'g', 'G' but got: " & spec.typ) + let + spec = parseStandardFormatSpecifier(specifier) + radix = toRadix(spec.typ) + result.add formatInt(value, radix, spec) + +proc formatFloat( + result: var string, value: SomeFloat, fmode: FloatFormatMode, + spec: StandardFormatSpecifier) = var f = formatBiggestFloat(value, fmode, spec.precision) var sign = false if value >= 0.0: @@ -550,23 +557,83 @@ proc formatValue*(result: var string; value: SomeFloat; specifier: string) = else: result.add res +proc toFloatFormatMode(typ: char): FloatFormatMode = + case typ + of 'e', 'E': ffScientific + of 'f', 'F': ffDecimal + of 'g', 'G': ffDefault + of '\0': ffDefault + else: + raise newException(ValueError, + "invalid type in format string for number, expected one " & + " of 'e', 'E', 'f', 'F', 'g', 'G' but got: " & typ) + +proc formatValue*(result: var string; value: SomeFloat; specifier: static string) = + ## Standard format implementation for `SomeFloat`. It makes little + ## sense to call this directly, but it is required to exist + ## by the `&` macro. + when specifier.len == 0: + result.add $value + else: + const + spec = parseStandardFormatSpecifier(specifier) + fmode = toFloatFormatMode(spec.typ) + + formatFloat(result, value, fmode, spec) + +proc formatValue*(result: var string; value: SomeFloat; specifier: string) = + ## Standard format implementation for `SomeFloat`. It makes little + ## sense to call this directly, but it is required to exist + ## by the `&` macro. + if specifier.len == 0: + result.add $value + else: + let + spec = parseStandardFormatSpecifier(specifier) + fmode = toFloatFormatMode(spec.typ) + + formatFloat(result, value, fmode, spec) + +proc formatValue*(result: var string; value: string; specifier: static string) = + ## Standard format implementation for `string`. It makes little + ## sense to call this directly, but it is required to exist + ## by the `&` macro. + const spec = parseStandardFormatSpecifier(specifier) + var value = + when spec.typ in {'s', '\0'}: value + else: static: + raise newException(ValueError, + "invalid type in format string for string, expected 's', but got " & + spec.typ) + when spec.precision != -1: + if spec.precision < runeLen(value): + const precision = cast[Natural](spec.precision) + setLen(value, Natural(runeOffset(value, precision))) + + result.add alignString(value, spec.minimumWidth, spec.align, spec.fill) + proc formatValue*(result: var string; value: string; specifier: string) = ## Standard format implementation for `string`. It makes little ## sense to call this directly, but it is required to exist ## by the `&` macro. let spec = parseStandardFormatSpecifier(specifier) - var value = value - case spec.typ - of 's', '\0': discard - else: - raise newException(ValueError, - "invalid type in format string for string, expected 's', but got " & - spec.typ) + var value = + if spec.typ in {'s', '\0'}: value + else: + raise newException(ValueError, + "invalid type in format string for string, expected 's', but got " & + spec.typ) if spec.precision != -1: if spec.precision < runeLen(value): - setLen(value, runeOffset(value, spec.precision)) + let precision = cast[Natural](spec.precision) + setLen(value, Natural(runeOffset(value, precision))) + result.add alignString(value, spec.minimumWidth, spec.align, spec.fill) +proc formatValue[T: not SomeInteger](result: var string; value: T; specifier: static string) = + mixin `$` + formatValue(result, $value, specifier) + proc formatValue[T: not SomeInteger](result: var string; value: T; specifier: string) = mixin `$` formatValue(result, $value, specifier) @@ -577,7 +644,8 @@ template formatValue(result: var string; value: char; specifier: string) = template formatValue(result: var string; value: cstring; specifier: string) = result.add value -proc strformatImpl(f: string; openChar, closeChar: char): NimNode = +proc strformatImpl(f: string; openChar, closeChar: char, + lineInfoNode: NimNode = nil): NimNode = template missingCloseChar = error("invalid format string: missing closing character '" & closeChar & "'") @@ -585,7 +653,7 @@ proc strformatImpl(f: string; openChar, closeChar: char): NimNode = error "openChar and closeChar must not be ':'" var i = 0 let res = genSym(nskVar, "fmtRes") - result = newNimNode(nnkStmtListExpr) + result = newNimNode(nnkStmtListExpr, lineInfoNode) # XXX: https://github.com/nim-lang/Nim/issues/8405 # When compiling with -d:useNimRtl, certain procs such as `count` from the strutils # module are not accessible at compile-time: @@ -644,6 +712,7 @@ proc strformatImpl(f: string; openChar, closeChar: char): NimNode = x = parseExpr(subexpr) except ValueError as e: error("could not parse `$#` in `$#`.\n$#" % [subexpr, f, e.msg]) + x.copyLineInfo(lineInfoNode) let formatSym = bindSym("formatValue", brOpen) var options = "" if f[i] == ':': @@ -661,18 +730,29 @@ proc strformatImpl(f: string; openChar, closeChar: char): NimNode = strlit.add closeChar inc i, 2 else: - doAssert false, "invalid format string: '$1' instead of '$1$1'" % $closeChar - inc i + raiseAssert "invalid format string: '$1' instead of '$1$1'" % $closeChar else: strlit.add f[i] inc i if strlit.len > 0: result.add newCall(bindSym"add", res, newLit(strlit)) result.add res + # workaround for #20381 + var blockExpr = newNimNode(nnkBlockExpr, lineInfoNode) + blockExpr.add(newEmptyNode()) + blockExpr.add(result) + result = blockExpr when defined(debugFmtDsl): echo repr result -macro fmt*(pattern: static string; openChar: static char, closeChar: static char): string = +macro fmt(pattern: static string; openChar: static char, closeChar: static char, lineInfoNode: untyped): string = + ## version of `fmt` with dummy untyped param for line info + strformatImpl(pattern, openChar, closeChar, lineInfoNode) + +when not defined(nimHasCallsitePragma): + {.pragma: callsite.} + +template fmt*(pattern: static string; openChar: static char, closeChar: static char): string {.callsite.} = ## Interpolates `pattern` using symbols in scope. runnableExamples: let x = 7 @@ -689,13 +769,13 @@ macro fmt*(pattern: static string; openChar: static char, closeChar: static char assert "<x>".fmt('<', '>') == "7" assert "<<<x>>>".fmt('<', '>') == "<7>" assert "`x`".fmt('`', '`') == "7" - strformatImpl(pattern, openChar, closeChar) + fmt(pattern, openChar, closeChar, dummyForLineInfo) -template fmt*(pattern: static string): untyped = +template fmt*(pattern: static string): untyped {.callsite.} = ## Alias for `fmt(pattern, '{', '}')`. - fmt(pattern, '{', '}') + fmt(pattern, '{', '}', dummyForLineInfo) -macro `&`*(pattern: string{lit}): string = +template `&`*(pattern: string{lit}): string {.callsite.} = ## `&pattern` is the same as `pattern.fmt`. ## For a specification of the `&` macro, see the module level documentation. # pending bug #18275, bug #18278, use `pattern: static string` @@ -707,4 +787,4 @@ macro `&`*(pattern: string{lit}): string = assert &"{x}\n" == "7\n" # regular string literal assert &"{x}\n" == "{x}\n".fmt # `fmt` can be used instead assert &"{x}\n" != fmt"{x}\n" # see `fmt` docs, this would use a raw string literal - strformatImpl(pattern.strVal, '{', '}') + fmt(pattern, '{', '}', dummyForLineInfo) diff --git a/lib/pure/strmisc.nim b/lib/pure/strmisc.nim index c8cd839be..a3e539e7e 100644 --- a/lib/pure/strmisc.nim +++ b/lib/pure/strmisc.nim @@ -27,20 +27,17 @@ func expandTabs*(s: string, tabSize: int = 8): string = doAssert expandTabs("a\tb\n\txy\t", 3) == "a b\n xy " result = newStringOfCap(s.len + s.len shr 2) - var pos = 0 template addSpaces(n) = - for j in 0 ..< n: + for _ in 1..n: result.add(' ') - pos += 1 + pos += n - for i in 0 ..< len(s): - let c = s[i] + var pos = 0 + let denominator = if tabSize > 0: tabSize else: 1 + for c in s: if c == '\t': - let - denominator = if tabSize > 0: tabSize else: 1 - numSpaces = tabSize - pos mod denominator - + let numSpaces = tabSize - pos mod denominator addSpaces(numSpaces) else: result.add(c) diff --git a/lib/pure/strscans.nim b/lib/pure/strscans.nim index 8a1ea125f..16ef9e642 100644 --- a/lib/pure/strscans.nim +++ b/lib/pure/strscans.nim @@ -12,7 +12,7 @@ This module contains a `scanf`:idx: macro that can be used for extracting substrings from an input string. This is often easier than regular expressions. Some examples as an appetizer: -.. code-block:: nim + ```nim # check if input string matches a triple of integers: const input = "(1,2,4)" var x, y, z: int @@ -26,6 +26,7 @@ Some examples as an appetizer: var myfloat: float if scanf(input, "$i-$i-$i $w$s$f", year, month, day, identifier, myfloat): echo "yes, we have a match!" + ``` As can be seen from the examples, strings are matched verbatim except for substrings starting with ``$``. These constructions are available: @@ -83,8 +84,7 @@ matches optional tokens without any result binding. In this example, we define a helper proc ``someSep`` that skips some separators which we then use in our scanf pattern to help us in the matching process: -.. code-block:: nim - + ```nim proc someSep(input: string; start: int; seps: set[char] = {':','-','.'}): int = # Note: The parameters and return value must match to what ``scanf`` requires result = 0 @@ -92,11 +92,11 @@ which we then use in our scanf pattern to help us in the matching process: if scanf(input, "$w$[someSep]$w", key, value): ... + ``` -It also possible to pass arguments to a user definable matcher: - -.. code-block:: nim +It is also possible to pass arguments to a user definable matcher: + ```nim proc ndigits(input: string; intVal: var int; start: int; n: int): int = # matches exactly ``n`` digits. Matchers need to return 0 if nothing # matched or otherwise the number of processed chars. @@ -115,6 +115,7 @@ It also possible to pass arguments to a user definable matcher: var year, month, day: int if scanf("2013-01-03", "${ndigits(4)}-${ndigits(2)}-${ndigits(2)}$.", year, month, day): ... + ``` The scanp macro @@ -145,8 +146,7 @@ not implemented. Simple example that parses the ``/etc/passwd`` file line by line: -.. code-block:: nim - + ```nim const etc_passwd = """root:x:0:0:root:/root:/bin/bash daemon:x:1:1:daemon:/usr/sbin:/bin/sh @@ -165,16 +165,16 @@ Simple example that parses the ``/etc/passwd`` file line by line: result.add entry else: break + ``` The ``scanp`` maps the grammar code into Nim code that performs the parsing. The parsing is performed with the help of 3 helper templates that that can be implemented for a custom type. These templates need to be named ``atom`` and ``nxt``. ``atom`` should be -overloaded to handle both single characters and sets of character. - -.. code-block:: nim +overloaded to handle both `char` and `set[char]`. + ```nim import std/streams template atom(input: Stream; idx: int; c: char): bool = @@ -190,11 +190,11 @@ overloaded to handle both single characters and sets of character. if scanp(content, idx, +( ~{'\L', '\0'} -> entry.add(peekChar($input))), '\L'): result.add entry + ``` Calling ordinary Nim procs inside the macro is possible: -.. code-block:: nim - + ```nim proc digits(s: string; intVal: var int; start: int): int = var x = 0 while result+start < s.len and s[result+start] in {'0'..'9'} and s[result+start] != ':': @@ -220,12 +220,12 @@ Calling ordinary Nim procs inside the macro is possible: result.add login & " " & homedir else: break + ``` When used for matching, keep in mind that likewise scanf, no backtracking is performed. -.. code-block:: nim - + ```nim proc skipUntil(s: string; until: string; unless = '\0'; start: int): int = # Skips all characters until the string `until` is found. Returns 0 # if the char `unless` is found first or the end is reached. @@ -256,12 +256,12 @@ is performed. for r in collectLinks(body): echo r + ``` In this example both macros are combined seamlessly in order to maximise efficiency and perform different checks. -.. code-block:: nim - + ```nim iterator parseIps*(soup: string): string = ## ipv4 only! const digits = {'0'..'9'} @@ -279,11 +279,11 @@ efficiency and perform different checks. yield buf buf.setLen(0) # need to clear `buf` each time, cause it might contain garbage idx.inc - + ``` ]## -import macros, parseutils +import std/[macros, parseutils] import std/private/since when defined(nimPreviewSlimSystem): @@ -472,7 +472,7 @@ macro scanf*(input: string; pattern: static[string]; results: varargs[typed]): b macro scanTuple*(input: untyped; pattern: static[string]; matcherTypes: varargs[untyped]): untyped {.since: (1, 5).}= ## Works identically as scanf, but instead of predeclaring variables it returns a tuple. - ## Tuple is started with a bool which indicates if the scan was successful + ## Tuple is started with a bool which indicates if the scan was successful ## followed by the requested data. ## If using a user defined matcher, provide the types in order they appear after pattern: ## `line.scanTuple("${yourMatcher()}", int)` diff --git a/lib/pure/strtabs.nim b/lib/pure/strtabs.nim index aa2886cfa..4b07aca5a 100644 --- a/lib/pure/strtabs.nim +++ b/lib/pure/strtabs.nim @@ -51,7 +51,7 @@ runnableExamples: import std/private/since import - hashes, strutils + std/[hashes, strutils] when defined(nimPreviewSlimSystem): import std/assertions @@ -61,7 +61,7 @@ when defined(js) or defined(nimscript) or defined(Standalone): {.pragma: rtlFunc.} else: {.pragma: rtlFunc, rtl.} - import os + import std/envvars include "system/inclrtl" @@ -261,10 +261,7 @@ proc newStringTable*(mode: StringTableMode): owned(StringTableRef) {. ## See also: ## * `newStringTable(keyValuePairs) proc ## <#newStringTable,varargs[tuple[string,string]],StringTableMode>`_ - new(result) - result.mode = mode - result.counter = 0 - newSeq(result.data, startSize) + result = StringTableRef(mode: mode, counter: 0, data: newSeq[KeyValuePair](startSize)) proc newStringTable*(keyValuePairs: varargs[string], mode: StringTableMode): owned(StringTableRef) {. diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index 3ae953a55..81be7db17 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -70,16 +70,16 @@ runnableExamples: ## easier substring extraction than regular expressions -import parseutils -from math import pow, floor, log10 -from algorithm import fill, reverse +import std/parseutils +from std/math import pow, floor, log10 +from std/algorithm import fill, reverse import std/enumutils -from unicode import toLower, toUpper +from std/unicode import toLower, toUpper export toLower, toUpper include "system/inclrtl" -import std/private/since +import std/private/[since, jsutils] from std/private/strimpl import cmpIgnoreStyleImpl, cmpIgnoreCaseImpl, startsWithImpl, endsWithImpl @@ -129,11 +129,11 @@ const ## Not very useful by its own, you can use it to create *inverted* sets to ## make the `find func<#find,string,set[char],Natural,int>`_ ## find **invalid** characters in strings. Example: - ## - ## .. code-block:: nim + ## ```nim ## let invalid = AllChars - Digits ## doAssert "01234".find(invalid) == -1 ## doAssert "01A34".find(invalid) == 2 + ## ``` func isAlphaAscii*(c: char): bool {.rtl, extern: "nsuIsAlphaAsciiChar".} = ## Checks whether or not character `c` is alphabetical. @@ -334,9 +334,9 @@ func normalize*(s: string): string {.rtl, extern: "nsuNormalize".} = func cmpIgnoreCase*(a, b: string): int {.rtl, extern: "nsuCmpIgnoreCase".} = ## Compares two strings in a case insensitive manner. Returns: ## - ## | 0 if a == b - ## | < 0 if a < b - ## | > 0 if a > b + ## | `0` if a == b + ## | `< 0` if a < b + ## | `> 0` if a > b runnableExamples: doAssert cmpIgnoreCase("FooBar", "foobar") == 0 doAssert cmpIgnoreCase("bar", "Foo") < 0 @@ -354,9 +354,9 @@ func cmpIgnoreStyle*(a, b: string): int {.rtl, extern: "nsuCmpIgnoreStyle".} = ## ## Returns: ## - ## | 0 if a == b - ## | < 0 if a < b - ## | > 0 if a > b + ## | `0` if a == b + ## | `< 0` if a < b + ## | `> 0` if a > b runnableExamples: doAssert cmpIgnoreStyle("foo_bar", "FooBar") == 0 doAssert cmpIgnoreStyle("foo_bar_5", "FooBar4") > 0 @@ -366,11 +366,14 @@ func cmpIgnoreStyle*(a, b: string): int {.rtl, extern: "nsuCmpIgnoreStyle".} = # --------- Private templates for different split separators ----------- func substrEq(s: string, pos: int, substr: string): bool = - var i = 0 + # Always returns false for empty `substr` var length = substr.len - while i < length and pos+i < s.len and s[pos+i] == substr[i]: - inc i - return i == length + if length > 0: + var i = 0 + while i < length and pos+i < s.len and s[pos+i] == substr[i]: + inc i + i == length + else: false template stringHasSep(s: string, index: int, seps: set[char]): bool = s[index] in seps @@ -420,14 +423,12 @@ iterator split*(s: string, sep: char, maxsplit: int = -1): string = ## ## Substrings are separated by the character `sep`. ## The code: - ## - ## .. code-block:: nim + ## ```nim ## for word in split(";;this;is;an;;example;;;", ';'): ## writeLine(stdout, word) - ## + ## ``` ## Results in: - ## - ## .. code-block:: + ## ``` ## "" ## "" ## "this" @@ -438,6 +439,7 @@ iterator split*(s: string, sep: char, maxsplit: int = -1): string = ## "" ## "" ## "" + ## ``` ## ## See also: ## * `rsplit iterator<#rsplit.i,string,char,int>`_ @@ -452,41 +454,49 @@ iterator split*(s: string, seps: set[char] = Whitespace, ## ## Substrings are separated by a substring containing only `seps`. ## - ## .. code-block:: nim + ## ```nim ## for word in split("this\lis an\texample"): ## writeLine(stdout, word) + ## ``` ## ## ...generates this output: ## - ## .. code-block:: + ## ``` ## "this" ## "is" ## "an" ## "example" + ## ``` ## ## And the following code: ## - ## .. code-block:: nim + ## ```nim ## for word in split("this:is;an$example", {';', ':', '$'}): ## writeLine(stdout, word) + ## ``` ## ## ...produces the same output as the first example. The code: ## - ## .. code-block:: nim + ## ```nim ## let date = "2012-11-20T22:08:08.398990" ## let separators = {' ', '-', ':', 'T'} ## for number in split(date, separators): ## writeLine(stdout, number) + ## ``` ## ## ...results in: ## - ## .. code-block:: + ## ``` ## "2012" ## "11" ## "20" ## "22" ## "08" ## "08.398990" + ## ``` + ## + ## .. note:: Empty separator set results in returning an original string, + ## following the interpretation "split by no element". ## ## See also: ## * `rsplit iterator<#rsplit.i,string,set[char],int>`_ @@ -501,23 +511,30 @@ iterator split*(s: string, sep: string, maxsplit: int = -1): string = ## Substrings are separated by the string `sep`. ## The code: ## - ## .. code-block:: nim + ## ```nim ## for word in split("thisDATAisDATAcorrupted", "DATA"): ## writeLine(stdout, word) + ## ``` ## ## Results in: ## - ## .. code-block:: + ## ``` ## "this" ## "is" ## "corrupted" + ## ``` + ## + ## .. note:: Empty separator string results in returning an original string, + ## following the interpretation "split by no element". ## ## See also: ## * `rsplit iterator<#rsplit.i,string,string,int,bool>`_ ## * `splitLines iterator<#splitLines.i,string>`_ ## * `splitWhitespace iterator<#splitWhitespace.i,string,int>`_ ## * `split func<#split,string,string,int>`_ - splitCommon(s, sep, maxsplit, sep.len) + let sepLen = if sep.len == 0: 1 # prevents infinite loop + else: sep.len + splitCommon(s, sep, maxsplit, sepLen) template rsplitCommon(s, sep, maxsplit, sepLen) = @@ -548,17 +565,19 @@ iterator rsplit*(s: string, sep: char, maxsplit: int = -1): string = ## Splits the string `s` into substrings from the right using a ## string separator. Works exactly the same as `split iterator - ## <#split.i,string,char,int>`_ except in reverse order. + ## <#split.i,string,char,int>`_ except in **reverse** order. ## - ## .. code-block:: nim + ## ```nim ## for piece in "foo:bar".rsplit(':'): ## echo piece + ## ``` ## ## Results in: ## - ## .. code-block:: nim + ## ``` ## "bar" ## "foo" + ## ``` ## ## Substrings are separated from the right by the char `sep`. ## @@ -573,20 +592,25 @@ iterator rsplit*(s: string, seps: set[char] = Whitespace, maxsplit: int = -1): string = ## Splits the string `s` into substrings from the right using a ## string separator. Works exactly the same as `split iterator - ## <#split.i,string,char,int>`_ except in reverse order. + ## <#split.i,string,char,int>`_ except in **reverse** order. ## - ## .. code-block:: nim + ## ```nim ## for piece in "foo bar".rsplit(WhiteSpace): ## echo piece + ## ``` ## ## Results in: ## - ## .. code-block:: nim + ## ``` ## "bar" ## "foo" + ## ``` ## ## Substrings are separated from the right by the set of chars `seps` ## + ## .. note:: Empty separator set results in returning an original string, + ## following the interpretation "split by no element". + ## ## See also: ## * `split iterator<#split.i,string,set[char],int>`_ ## * `splitLines iterator<#splitLines.i,string>`_ @@ -598,44 +622,52 @@ iterator rsplit*(s: string, sep: string, maxsplit: int = -1, keepSeparators: bool = false): string = ## Splits the string `s` into substrings from the right using a ## string separator. Works exactly the same as `split iterator - ## <#split.i,string,string,int>`_ except in reverse order. + ## <#split.i,string,string,int>`_ except in **reverse** order. ## - ## .. code-block:: nim + ## ```nim ## for piece in "foothebar".rsplit("the"): ## echo piece + ## ``` ## ## Results in: ## - ## .. code-block:: nim + ## ``` ## "bar" ## "foo" + ## ``` ## ## Substrings are separated from the right by the string `sep` ## + ## .. note:: Empty separator string results in returning an original string, + ## following the interpretation "split by no element". + ## ## See also: ## * `split iterator<#split.i,string,string,int>`_ ## * `splitLines iterator<#splitLines.i,string>`_ ## * `splitWhitespace iterator<#splitWhitespace.i,string,int>`_ ## * `rsplit func<#rsplit,string,string,int>`_ - rsplitCommon(s, sep, maxsplit, sep.len) + let sepLen = if sep.len == 0: 1 # prevents infinite loop + else: sep.len + rsplitCommon(s, sep, maxsplit, sepLen) iterator splitLines*(s: string, keepEol = false): string = ## Splits the string `s` into its containing lines. ## ## Every `character literal <manual.html#lexical-analysis-character-literals>`_ ## newline combination (CR, LF, CR-LF) is supported. The result strings - ## contain no trailing end of line characters unless parameter `keepEol` + ## contain no trailing end of line characters unless the parameter `keepEol` ## is set to `true`. ## ## Example: ## - ## .. code-block:: nim + ## ```nim ## for line in splitLines("\nthis\nis\nan\n\nexample\n"): ## writeLine(stdout, line) + ## ``` ## ## Results in: ## - ## .. code-block:: nim + ## ```nim ## "" ## "this" ## "is" @@ -643,6 +675,7 @@ iterator splitLines*(s: string, keepEol = false): string = ## "" ## "example" ## "" + ## ``` ## ## See also: ## * `splitWhitespace iterator<#splitWhitespace.i,string,int>`_ @@ -675,16 +708,17 @@ iterator splitWhitespace*(s: string, maxsplit: int = -1): string = ## ## The following code: ## - ## .. code-block:: nim + ## ```nim ## let s = " foo \t bar baz " ## for ms in [-1, 1, 2, 3]: ## echo "------ maxsplit = ", ms, ":" ## for item in s.splitWhitespace(maxsplit=ms): ## echo '"', item, '"' + ## ``` ## ## ...results in: ## - ## .. code-block:: + ## ``` ## ------ maxsplit = -1: ## "foo" ## "bar" @@ -700,6 +734,7 @@ iterator splitWhitespace*(s: string, maxsplit: int = -1): string = ## "foo" ## "bar" ## "baz" + ## ``` ## ## See also: ## * `splitLines iterator<#splitLines.i,string>`_ @@ -728,6 +763,9 @@ func split*(s: string, seps: set[char] = Whitespace, maxsplit: int = -1): seq[ ## The same as the `split iterator <#split.i,string,set[char],int>`_ (see its ## documentation), but is a func that returns a sequence of substrings. ## + ## .. note:: Empty separator set results in returning an original string, + ## following the interpretation "split by no element". + ## ## See also: ## * `split iterator <#split.i,string,set[char],int>`_ ## * `rsplit func<#rsplit,string,set[char],int>`_ @@ -736,6 +774,7 @@ func split*(s: string, seps: set[char] = Whitespace, maxsplit: int = -1): seq[ runnableExamples: doAssert "a,b;c".split({',', ';'}) == @["a", "b", "c"] doAssert "".split({' '}) == @[""] + doAssert "empty seps return unsplit s".split({}) == @["empty seps return unsplit s"] accResult(split(s, seps, maxsplit)) func split*(s: string, sep: string, maxsplit: int = -1): seq[string] {.rtl, @@ -745,6 +784,9 @@ func split*(s: string, sep: string, maxsplit: int = -1): seq[string] {.rtl, ## Substrings are separated by the string `sep`. This is a wrapper around the ## `split iterator <#split.i,string,string,int>`_. ## + ## .. note:: Empty separator string results in returning an original string, + ## following the interpretation "split by no element". + ## ## See also: ## * `split iterator <#split.i,string,string,int>`_ ## * `rsplit func<#rsplit,string,string,int>`_ @@ -757,14 +799,13 @@ func split*(s: string, sep: string, maxsplit: int = -1): seq[string] {.rtl, doAssert "a largely spaced sentence".split(" ") == @["a", "", "largely", "", "", "", "spaced", "sentence"] doAssert "a largely spaced sentence".split(" ", maxsplit = 1) == @["a", " largely spaced sentence"] - doAssert(sep.len > 0) - + doAssert "empty sep returns unsplit s".split("") == @["empty sep returns unsplit s"] accResult(split(s, sep, maxsplit)) func rsplit*(s: string, sep: char, maxsplit: int = -1): seq[string] {.rtl, extern: "nsuRSplitChar".} = ## The same as the `rsplit iterator <#rsplit.i,string,char,int>`_, but is a func - ## that returns a sequence of substrings. + ## that returns a sequence of substrings in original order. ## ## A possible common use case for `rsplit` is path manipulation, ## particularly on systems that don't use a common delimiter. @@ -772,13 +813,15 @@ func rsplit*(s: string, sep: char, maxsplit: int = -1): seq[string] {.rtl, ## For example, if a system had `#` as a delimiter, you could ## do the following to get the tail of the path: ## - ## .. code-block:: nim + ## ```nim ## var tailSplit = rsplit("Root#Object#Method#Index", '#', maxsplit=1) + ## ``` ## ## Results in `tailSplit` containing: ## - ## .. code-block:: nim + ## ```nim ## @["Root#Object#Method", "Index"] + ## ``` ## ## See also: ## * `rsplit iterator <#rsplit.i,string,char,int>`_ @@ -792,7 +835,7 @@ func rsplit*(s: string, seps: set[char] = Whitespace, maxsplit: int = -1): seq[string] {.rtl, extern: "nsuRSplitCharSet".} = ## The same as the `rsplit iterator <#rsplit.i,string,set[char],int>`_, but is a - ## func that returns a sequence of substrings. + ## func that returns a sequence of substrings in original order. ## ## A possible common use case for `rsplit` is path manipulation, ## particularly on systems that don't use a common delimiter. @@ -800,13 +843,18 @@ func rsplit*(s: string, seps: set[char] = Whitespace, ## For example, if a system had `#` as a delimiter, you could ## do the following to get the tail of the path: ## - ## .. code-block:: nim + ## ```nim ## var tailSplit = rsplit("Root#Object#Method#Index", {'#'}, maxsplit=1) + ## ``` ## ## Results in `tailSplit` containing: ## - ## .. code-block:: nim + ## ```nim ## @["Root#Object#Method", "Index"] + ## ``` + ## + ## .. note:: Empty separator set results in returning an original string, + ## following the interpretation "split by no element". ## ## See also: ## * `rsplit iterator <#rsplit.i,string,set[char],int>`_ @@ -819,7 +867,7 @@ func rsplit*(s: string, seps: set[char] = Whitespace, func rsplit*(s: string, sep: string, maxsplit: int = -1): seq[string] {.rtl, extern: "nsuRSplitString".} = ## The same as the `rsplit iterator <#rsplit.i,string,string,int,bool>`_, but is a func - ## that returns a sequence of substrings. + ## that returns a sequence of substrings in original order. ## ## A possible common use case for `rsplit` is path manipulation, ## particularly on systems that don't use a common delimiter. @@ -827,13 +875,18 @@ func rsplit*(s: string, sep: string, maxsplit: int = -1): seq[string] {.rtl, ## For example, if a system had `#` as a delimiter, you could ## do the following to get the tail of the path: ## - ## .. code-block:: nim + ## ```nim ## var tailSplit = rsplit("Root#Object#Method#Index", "#", maxsplit=1) + ## ``` ## ## Results in `tailSplit` containing: ## - ## .. code-block:: nim + ## ```nim ## @["Root#Object#Method", "Index"] + ## ``` + ## + ## .. note:: Empty separator string results in returning an original string, + ## following the interpretation "split by no element". ## ## See also: ## * `rsplit iterator <#rsplit.i,string,string,int,bool>`_ @@ -849,6 +902,7 @@ func rsplit*(s: string, sep: string, maxsplit: int = -1): seq[string] {.rtl, doAssert "".rsplit("Elon Musk") == @[""] doAssert "a largely spaced sentence".rsplit(" ") == @["a", "", "largely", "", "", "", "spaced", "sentence"] + doAssert "empty sep returns unsplit s".rsplit("") == @["empty sep returns unsplit s"] accResult(rsplit(s, sep, maxsplit)) result.reverse() @@ -944,14 +998,26 @@ func toHex*[T: SomeInteger](x: T, len: Positive): string = doAssert b.toHex(4) == "1001" doAssert toHex(62, 3) == "03E" doAssert toHex(-8, 6) == "FFFFF8" - toHexImpl(cast[BiggestUInt](x), len, x < 0) + whenJsNoBigInt64: + toHexImpl(cast[BiggestUInt](x), len, x < 0) + do: + when T is SomeSignedInt: + toHexImpl(cast[BiggestUInt](BiggestInt(x)), len, x < 0) + else: + toHexImpl(BiggestUInt(x), len, x < 0) func toHex*[T: SomeInteger](x: T): string = ## Shortcut for `toHex(x, T.sizeof * 2)` runnableExamples: doAssert toHex(1984'i64) == "00000000000007C0" doAssert toHex(1984'i16) == "07C0" - toHexImpl(cast[BiggestUInt](x), 2*sizeof(T), x < 0) + whenJsNoBigInt64: + toHexImpl(cast[BiggestUInt](x), 2*sizeof(T), x < 0) + do: + when T is SomeSignedInt: + toHexImpl(cast[BiggestUInt](BiggestInt(x)), 2*sizeof(T), x < 0) + else: + toHexImpl(BiggestUInt(x), 2*sizeof(T), x < 0) func toHex*(s: string): string {.rtl.} = ## Converts a bytes string to its hexadecimal representation. @@ -1241,7 +1307,7 @@ func parseEnum*[T: enum](s: string): T = ## type contains multiple fields with the same string value. ## ## Raises `ValueError` for an invalid value in `s`. The comparison is - ## done in a style insensitive way. + ## done in a style insensitive way (first letter is still case-sensitive). runnableExamples: type MyEnum = enum @@ -1261,7 +1327,7 @@ func parseEnum*[T: enum](s: string, default: T): T = ## type contains multiple fields with the same string value. ## ## Uses `default` for an invalid value in `s`. The comparison is done in a - ## style insensitive way. + ## style insensitive way (first letter is still case-sensitive). runnableExamples: type MyEnum = enum @@ -1648,7 +1714,7 @@ func removePrefix*(s: var string, chars: set[char] = Newlines) {.rtl, var start = 0 while start < s.len and s[start] in chars: start += 1 - if start > 0: s.delete(0, start - 1) + if start > 0: s.delete(0..start - 1) func removePrefix*(s: var string, c: char) {.rtl, extern: "nsuRemovePrefixChar".} = @@ -1675,8 +1741,8 @@ func removePrefix*(s: var string, prefix: string) {.rtl, var answers = "yesyes" answers.removePrefix("yes") doAssert answers == "yes" - if s.startsWith(prefix): - s.delete(0, prefix.len - 1) + if s.startsWith(prefix) and prefix.len > 0: + s.delete(0..prefix.len - 1) func removeSuffix*(s: var string, chars: set[char] = Newlines) {.rtl, extern: "nsuRemoveSuffixCharSet".} = @@ -1742,8 +1808,9 @@ func addSep*(dest: var string, sep = ", ", startLen: Natural = 0) {.inline.} = ## ## A shorthand for: ## - ## .. code-block:: nim + ## ```nim ## if dest.len > startLen: add(dest, sep) + ## ``` ## ## This is often useful for generating some code where the items need to ## be *separated* by `sep`. `sep` is only added if `dest` is longer than @@ -1808,7 +1875,7 @@ func join*(a: openArray[string], sep: string = ""): string {.rtl, else: result = "" -func join*[T: not string](a: openArray[T], sep: string = ""): string = +proc join*[T: not string](a: openArray[T], sep: string = ""): string = ## Converts all elements in the container `a` to strings using `$`, ## and concatenates them with `sep`. runnableExamples: @@ -1882,9 +1949,6 @@ func find*(a: SkipTable, s, sub: string, start: Natural = 0, last = -1): int {. when not (defined(js) or defined(nimdoc) or defined(nimscript)): func c_memchr(cstr: pointer, c: char, n: csize_t): pointer {. importc: "memchr", header: "<string.h>".} - func c_strstr(haystack, needle: cstring): cstring {. - importc: "strstr", header: "<string.h>".} - const hasCStringBuiltin = true else: const hasCStringBuiltin = false @@ -1917,7 +1981,7 @@ func find*(s: string, sub: char, start: Natural = 0, last = -1): int {.rtl, if length > 0: let found = c_memchr(s[start].unsafeAddr, sub, cast[csize_t](length)) if not found.isNil: - return cast[ByteAddress](found) -% cast[ByteAddress](s.cstring) + return cast[int](found) -% cast[int](s.cstring) else: findImpl() @@ -1939,6 +2003,14 @@ func find*(s: string, chars: set[char], start: Natural = 0, last = -1): int {. if s[i] in chars: return i +when defined(linux): + proc memmem(haystack: pointer, haystacklen: csize_t, + needle: pointer, needlelen: csize_t): pointer {.importc, header: """#define _GNU_SOURCE +#include <string.h>""".} +elif defined(bsd) or (defined(macosx) and not defined(ios)): + proc memmem(haystack: pointer, haystacklen: csize_t, + needle: pointer, needlelen: csize_t): pointer {.importc, header: "#include <string.h>".} + func find*(s, sub: string, start: Natural = 0, last = -1): int {.rtl, extern: "nsuFindStr".} = ## Searches for `sub` in `s` inside range `start..last` (both ends included). @@ -1960,11 +2032,12 @@ func find*(s, sub: string, start: Natural = 0, last = -1): int {.rtl, when nimvm: useSkipTable() else: - when hasCStringBuiltin: - if last < 0 and start < s.len: - let found = c_strstr(s[start].unsafeAddr, sub) + when declared(memmem): + let subLen = sub.len + if last < 0 and start < s.len and subLen != 0: + let found = memmem(s[start].unsafeAddr, csize_t(s.len - start), sub.cstring, csize_t(subLen)) result = if not found.isNil: - cast[ByteAddress](found) -% cast[ByteAddress](s.cstring) + cast[int](found) -% cast[int](s.cstring) else: -1 else: @@ -2208,17 +2281,31 @@ func replaceWord*(s, sub: string, by = ""): string {.rtl, add result, substr(s, i) func multiReplace*(s: string, replacements: varargs[(string, string)]): string = - ## Same as replace, but specialized for doing multiple replacements in a single - ## pass through the input string. + ## Same as `replace<#replace,string,string,string>`_, but specialized for + ## doing multiple replacements in a single pass through the input string. ## - ## `multiReplace` performs all replacements in a single pass, this means it - ## can be used to swap the occurrences of "a" and "b", for instance. + ## `multiReplace` scans the input string from left to right and replaces the + ## matching substrings in the same order as passed in the argument list. + ## + ## The implications of the order of scanning the string and matching the + ## replacements: + ## - In case of multiple matches at a given position, the earliest + ## replacement is applied. + ## - Overlaps are not handled. After performing a replacement, the scan + ## continues from the character after the matched substring. If the + ## resulting string then contains a possible match starting in a newly + ## placed substring, the additional replacement is not performed. ## ## If the resulting string is not longer than the original input string, ## only a single memory allocation is required. ## - ## The order of the replacements does matter. Earlier replacements are - ## preferred over later replacements in the argument list. + runnableExamples: + # Swapping occurrences of 'a' and 'b': + doAssert multireplace("abba", [("a", "b"), ("b", "a")]) == "baab" + + # The second replacement ("ab") is matched and performed first, the scan then + # continues from 'c', so the "bc" replacement is never matched and thus skipped. + doAssert multireplace("abc", [("bc", "x"), ("ab", "_b")]) == "_bc" result = newStringOfCap(s.len) var i = 0 var fastChk: set[char] = {} @@ -2367,8 +2454,8 @@ func validIdentifier*(s: string): bool {.rtl, extern: "nsuValidIdentifier".} = # floating point formatting: when not defined(js): - func c_sprintf(buf, frmt: cstring): cint {.header: "<stdio.h>", - importc: "sprintf", varargs.} + func c_snprintf(buf: cstring, n: csize_t, frmt: cstring): cint {.header: "<stdio.h>", + importc: "snprintf", varargs.} type FloatFormatMode* = enum @@ -2395,60 +2482,63 @@ func formatBiggestFloat*(f: BiggestFloat, format: FloatFormatMode = ffDefault, doAssert x.formatBiggestFloat() == "123.4560000000000" doAssert x.formatBiggestFloat(ffDecimal, 4) == "123.4560" doAssert x.formatBiggestFloat(ffScientific, 2) == "1.23e+02" - when defined(js): - var precision = precision - if precision == -1: - # use the same default precision as c_sprintf - precision = 6 - var res: cstring - case format - of ffDefault: - {.emit: "`res` = `f`.toString();".} - of ffDecimal: - {.emit: "`res` = `f`.toFixed(`precision`);".} - of ffScientific: - {.emit: "`res` = `f`.toExponential(`precision`);".} - result = $res - if 1.0 / f == -Inf: - # JavaScript removes the "-" from negative Zero, add it back here - result = "-" & $res - for i in 0 ..< result.len: - # Depending on the locale either dot or comma is produced, - # but nothing else is possible: - if result[i] in {'.', ','}: result[i] = decimalSep + when nimvm: + discard "implemented in the vmops" else: - const floatFormatToChar: array[FloatFormatMode, char] = ['g', 'f', 'e'] - var - frmtstr {.noinit.}: array[0..5, char] - buf {.noinit.}: array[0..2500, char] - L: cint - frmtstr[0] = '%' - if precision >= 0: - frmtstr[1] = '#' - frmtstr[2] = '.' - frmtstr[3] = '*' - frmtstr[4] = floatFormatToChar[format] - frmtstr[5] = '\0' - L = c_sprintf(addr buf, addr frmtstr, precision, f) + when defined(js): + var precision = precision + if precision == -1: + # use the same default precision as c_snprintf + precision = 6 + var res: cstring + case format + of ffDefault: + {.emit: "`res` = `f`.toString();".} + of ffDecimal: + {.emit: "`res` = `f`.toFixed(`precision`);".} + of ffScientific: + {.emit: "`res` = `f`.toExponential(`precision`);".} + result = $res + if 1.0 / f == -Inf: + # JavaScript removes the "-" from negative Zero, add it back here + result = "-" & $res + for i in 0 ..< result.len: + # Depending on the locale either dot or comma is produced, + # but nothing else is possible: + if result[i] in {'.', ','}: result[i] = decimalSep else: - frmtstr[1] = floatFormatToChar[format] - frmtstr[2] = '\0' - L = c_sprintf(addr buf, addr frmtstr, f) - result = newString(L) - for i in 0 ..< L: - # Depending on the locale either dot or comma is produced, - # but nothing else is possible: - if buf[i] in {'.', ','}: result[i] = decimalSep - else: result[i] = buf[i] - when defined(windows): - # VS pre 2015 violates the C standard: "The exponent always contains at - # least two digits, and only as many more digits as necessary to - # represent the exponent." [C11 §7.21.6.1] - # The following post-processing fixes this behavior. - if result.len > 4 and result[^4] == '+' and result[^3] == '0': - result[^3] = result[^2] - result[^2] = result[^1] - result.setLen(result.len - 1) + const floatFormatToChar: array[FloatFormatMode, char] = ['g', 'f', 'e'] + var + frmtstr {.noinit.}: array[0..5, char] + buf {.noinit.}: array[0..2500, char] + L: cint + frmtstr[0] = '%' + if precision >= 0: + frmtstr[1] = '#' + frmtstr[2] = '.' + frmtstr[3] = '*' + frmtstr[4] = floatFormatToChar[format] + frmtstr[5] = '\0' + L = c_snprintf(cast[cstring](addr buf), csize_t(2501), cast[cstring](addr frmtstr), precision, f) + else: + frmtstr[1] = floatFormatToChar[format] + frmtstr[2] = '\0' + L = c_snprintf(cast[cstring](addr buf), csize_t(2501), cast[cstring](addr frmtstr), f) + result = newString(L) + for i in 0 ..< L: + # Depending on the locale either dot or comma is produced, + # but nothing else is possible: + if buf[i] in {'.', ','}: result[i] = decimalSep + else: result[i] = buf[i] + when defined(windows): + # VS pre 2015 violates the C standard: "The exponent always contains at + # least two digits, and only as many more digits as necessary to + # represent the exponent." [C11 §7.21.6.1] + # The following post-processing fixes this behavior. + if result.len > 4 and result[^4] == '+' and result[^3] == '0': + result[^3] = result[^2] + result[^2] = result[^1] + result.setLen(result.len - 1) func formatFloat*(f: float, format: FloatFormatMode = ffDefault, precision: range[-1..32] = 16; decimalSep = '.'): string {. @@ -2488,7 +2578,8 @@ func trimZeros*(x: var string; decimalSep = '.') = var pos = last while pos >= 0 and x[pos] == '0': dec(pos) if pos > sPos: inc(pos) - x.delete(pos, last) + if last >= pos: + x.delete(pos..last) type BinaryPrefixMode* = enum ## The different names for binary prefixes. @@ -2580,13 +2671,13 @@ func formatEng*(f: BiggestFloat, ## decimal point or (if `trim` is true) the maximum number of digits to be ## shown. ## - ## .. code-block:: nim - ## + ## ```nim ## formatEng(0, 2, trim=false) == "0.00" ## formatEng(0, 2) == "0" ## formatEng(0.053, 0) == "53e-3" ## formatEng(52731234, 2) == "52.73e6" ## formatEng(-52731234, 2) == "-52.73e6" + ## ``` ## ## If `siPrefix` is set to true, the number will be displayed with the SI ## prefix corresponding to the exponent. For example 4100 will be displayed @@ -2601,8 +2692,7 @@ func formatEng*(f: BiggestFloat, ## different to appending the unit to the result as the location of the space ## is altered depending on whether there is an exponent. ## - ## .. code-block:: nim - ## + ## ```nim ## formatEng(4100, siPrefix=true, unit="V") == "4.1 kV" ## formatEng(4.1, siPrefix=true, unit="V") == "4.1 V" ## formatEng(4.1, siPrefix=true) == "4.1" # Note lack of space @@ -2612,6 +2702,7 @@ func formatEng*(f: BiggestFloat, ## formatEng(4100) == "4.1e3" ## formatEng(4100, unit="V") == "4.1e3 V" ## formatEng(4100, unit="", useUnitSpace=true) == "4.1e3 " # Space with useUnitSpace=true + ## ``` ## ## `decimalSep` is used as the decimal separator. ## @@ -2775,13 +2866,15 @@ func `%`*(formatstr: string, a: openArray[string]): string {.rtl, ## ## This is best explained by an example: ## - ## .. code-block:: nim + ## ```nim ## "$1 eats $2." % ["The cat", "fish"] + ## ``` ## ## Results in: ## - ## .. code-block:: nim + ## ```nim ## "The cat eats fish." + ## ``` ## ## The substitution variables (the thing after the `$`) are enumerated ## from 1 to `a.len`. @@ -2789,21 +2882,24 @@ func `%`*(formatstr: string, a: openArray[string]): string {.rtl, ## The notation `$#` can be used to refer to the next substitution ## variable: ## - ## .. code-block:: nim + ## ```nim ## "$# eats $#." % ["The cat", "fish"] + ## ``` ## ## Substitution variables can also be words (that is ## `[A-Za-z_]+[A-Za-z0-9_]*`) in which case the arguments in `a` with even ## indices are keys and with odd indices are the corresponding values. ## An example: ## - ## .. code-block:: nim + ## ```nim ## "$animal eats $food." % ["animal", "The cat", "food", "fish"] + ## ``` ## ## Results in: ## - ## .. code-block:: nim + ## ```nim ## "The cat eats fish." + ## ``` ## ## The variables are compared with `cmpIgnoreStyle`. `ValueError` is ## raised if an ill-formed format string has been passed to the `%` operator. @@ -2901,13 +2997,14 @@ iterator tokenize*(s: string, seps: set[char] = Whitespace): tuple[ ## Substrings are separated by a substring containing only `seps`. ## Example: ## - ## .. code-block:: nim + ## ```nim ## for word in tokenize(" this is an example "): ## writeLine(stdout, word) + ## ``` ## ## Results in: ## - ## .. code-block:: nim + ## ```nim ## (" ", true) ## ("this", false) ## (" ", true) @@ -2917,6 +3014,7 @@ iterator tokenize*(s: string, seps: set[char] = Whitespace): tuple[ ## (" ", true) ## ("example", false) ## (" ", true) + ## ``` var i = 0 while true: var j = i diff --git a/lib/pure/sugar.nim b/lib/pure/sugar.nim index ff2a3bb58..90ba20c13 100644 --- a/lib/pure/sugar.nim +++ b/lib/pure/sugar.nim @@ -11,7 +11,7 @@ ## macro system. import std/private/since -import macros +import std/macros proc checkPragma(ex, prag: var NimNode) = since (1, 3): @@ -203,7 +203,7 @@ proc freshIdentNodes(ast: NimNode): NimNode = # see also https://github.com/nim-lang/Nim/pull/8531#issuecomment-410436458 proc inspect(node: NimNode): NimNode = case node.kind: - of nnkIdent, nnkSym: + of nnkIdent, nnkSym, nnkOpenSymChoice, nnkClosedSymChoice, nnkOpenSym: result = ident($node) of nnkEmpty, nnkLiterals: result = node @@ -231,9 +231,19 @@ macro capture*(locals: varargs[typed], body: untyped): untyped {.since: (1, 1).} let locals = if locals.len == 1 and locals[0].kind == nnkBracket: locals[0] else: locals for arg in locals: - if arg.strVal == "result": - error("The variable name cannot be `result`!", arg) - params.add(newIdentDefs(ident(arg.strVal), freshIdentNodes getTypeInst arg)) + proc getIdent(n: NimNode): NimNode = + case n.kind + of nnkIdent, nnkSym: + let nStr = n.strVal + if nStr == "result": + error("The variable name cannot be `result`!", n) + result = ident(nStr) + of nnkHiddenDeref: result = n[0].getIdent() + else: + error("The argument to be captured `" & n.repr & "` is not a pure identifier. " & + "It is an unsupported `" & $n.kind & "` node.", n) + let argName = getIdent(arg) + params.add(newIdentDefs(argName, freshIdentNodes getTypeInst arg)) result = newNimNode(nnkCall) result.add(newProc(newEmptyNode(), params, body, nnkLambda)) for arg in locals: result.add(arg) @@ -335,9 +345,10 @@ proc collectImpl(init, body: NimNode): NimNode {.since: (1, 1).} = let res = genSym(nskVar, "collectResult") var bracketExpr: NimNode if init != nil: - expectKind init, {nnkCall, nnkIdent, nnkSym} + expectKind init, {nnkCall, nnkIdent, nnkSym, nnkClosedSymChoice, nnkOpenSymChoice, nnkOpenSym} bracketExpr = newTree(nnkBracketExpr, - if init.kind == nnkCall: freshIdentNodes(init[0]) else: freshIdentNodes(init)) + if init.kind in {nnkCall, nnkClosedSymChoice, nnkOpenSymChoice, nnkOpenSym}: + freshIdentNodes(init[0]) else: freshIdentNodes(init)) else: bracketExpr = newTree(nnkBracketExpr) let (resBody, keyType, valueType) = trans(body, res, bracketExpr) diff --git a/lib/pure/terminal.nim b/lib/pure/terminal.nim index 571c9b13c..53b3d61da 100644 --- a/lib/pure/terminal.nim +++ b/lib/pure/terminal.nim @@ -58,13 +58,13 @@ runnableExamples("-r:off"): stdout.styledWriteLine(fgRed, "red text ", styleBright, "bold red", fgDefault, " bold text") -import macros -import strformat -from strutils import toLowerAscii, `%` -import colors +import std/macros +import std/strformat +from std/strutils import toLowerAscii, `%`, parseInt +import std/colors when defined(windows): - import winlean + import std/winlean when defined(nimPreviewSlimSystem): import std/[syncio, assertions] @@ -96,10 +96,11 @@ const fgPrefix = "\e[38;2;" bgPrefix = "\e[48;2;" ansiResetCode* = "\e[0m" + getPos = "\e[6n" stylePrefix = "\e[" when defined(windows): - import winlean, os + import std/[winlean, os] const DUPLICATE_SAME_ACCESS = 2 @@ -220,6 +221,9 @@ when defined(windows): raiseOSError(osLastError()) return (int(c.dwCursorPosition.x), int(c.dwCursorPosition.y)) + proc getCursorPos*(): tuple [x, y: int] {.raises: [ValueError, IOError, OSError].} = + return getCursorPos(getStdHandle(STD_OUTPUT_HANDLE)) + proc setCursorPos(h: Handle, x, y: int) = var c: COORD c.x = int16(x) @@ -253,7 +257,7 @@ when defined(windows): if f == stderr: term.hStderr else: term.hStdout else: - import termios, posix, os, parseutils + import std/[termios, posix, os, parseutils] proc setRaw(fd: FileHandle, time: cint = TCSAFLUSH) = var mode: Termios @@ -267,6 +271,48 @@ else: mode.c_cc[VTIME] = 0.cuchar discard fd.tcSetAttr(time, addr mode) + proc getCursorPos*(): tuple [x, y: int] {.raises: [ValueError, IOError].} = + ## Returns cursor position (x, y) + ## writes to stdout and expects the terminal to respond via stdin + var + xStr = "" + yStr = "" + ch: char + ct: int + readX = false + + # use raw mode to ask terminal for cursor position + let fd = getFileHandle(stdin) + var oldMode: Termios + discard fd.tcGetAttr(addr oldMode) + fd.setRaw() + stdout.write(getPos) + flushFile(stdout) + + try: + # parse response format: [yyy;xxxR + while true: + let n = readBuffer(stdin, addr ch, 1) + if n == 0 or ch == 'R': + if xStr == "" or yStr == "": + raise newException(ValueError, "Got character position message that was missing data") + break + ct += 1 + if ct > 16: + raise newException(ValueError, "Got unterminated character position message from terminal") + if ch == ';': + readX = true + elif ch in {'0'..'9'}: + if readX: + xStr.add(ch) + else: + yStr.add(ch) + finally: + # restore previous terminal mode + discard fd.tcSetAttr(TCSADRAIN, addr oldMode) + + return (parseInt(xStr), parseInt(yStr)) + proc terminalWidthIoctl*(fds: openArray[int]): int = ## Returns terminal width from first fd that supports the ioctl. @@ -291,25 +337,57 @@ else: ## Returns some reasonable terminal width from either standard file ## descriptors, controlling terminal, environment variables or tradition. - var w = terminalWidthIoctl([0, 1, 2]) #Try standard file descriptors + # POSIX environment variable takes precendence. + # _COLUMNS_: This variable shall represent a decimal integer >0 used + # to indicate the user's preferred width in column positions for + # the terminal screen or window. If this variable is unset or null, + # the implementation determines the number of columns, appropriate + # for the terminal or window, in an unspecified manner. + # When COLUMNS is set, any terminal-width information implied by TERM + # is overridden. Users and conforming applications should not set COLUMNS + # unless they wish to override the system selection and produce output + # unrelated to the terminal characteristics. + # See POSIX Base Definitions Section 8.1 Environment Variable Definition + + var w: int + var s = getEnv("COLUMNS") # Try standard env var + if len(s) > 0 and parseSaturatedNatural(s, w) > 0 and w > 0: + return w + w = terminalWidthIoctl([0, 1, 2]) # Try standard file descriptors if w > 0: return w - var cterm = newString(L_ctermid) #Try controlling tty + var cterm = newString(L_ctermid) # Try controlling tty var fd = open(ctermid(cstring(cterm)), O_RDONLY) if fd != -1: w = terminalWidthIoctl([int(fd)]) discard close(fd) if w > 0: return w - var s = getEnv("COLUMNS") #Try standard env var - if len(s) > 0 and parseInt(s, w) > 0 and w > 0: - return w - return 80 #Finally default to venerable value + return 80 # Finally default to venerable value proc terminalHeight*(): int = ## Returns some reasonable terminal height from either standard file ## descriptors, controlling terminal, environment variables or tradition. ## Zero is returned if the height could not be determined. - var h = terminalHeightIoctl([0, 1, 2]) # Try standard file descriptors + # POSIX environment variable takes precendence. + # _LINES_: This variable shall represent a decimal integer >0 used + # to indicate the user's preferred number of lines on a page or + # the vertical screen or window size in lines. A line in this case + # is a vertical measure large enough to hold the tallest character + # in the character set being displayed. If this variable is unset or null, + # the implementation determines the number of lines, appropriate + # for the terminal or window (size, terminal baud rate, and so on), + # in an unspecified manner. + # When LINES is set, any terminal-height information implied by TERM + # is overridden. Users and conforming applications should not set LINES + # unless they wish to override the system selection and produce output + # unrelated to the terminal characteristics. + # See POSIX Base Definitions Section 8.1 Environment Variable Definition + + var h: int + var s = getEnv("LINES") # Try standard env var + if len(s) > 0 and parseSaturatedNatural(s, h) > 0 and h > 0: + return h + h = terminalHeightIoctl([0, 1, 2]) # Try standard file descriptors if h > 0: return h var cterm = newString(L_ctermid) # Try controlling tty var fd = open(ctermid(cstring(cterm)), O_RDONLY) @@ -317,9 +395,6 @@ else: h = terminalHeightIoctl([int(fd)]) discard close(fd) if h > 0: return h - var s = getEnv("LINES") # Try standard env var - if len(s) > 0 and parseInt(s, h) > 0 and h > 0: - return h return 0 # Could not determine height proc terminalSize*(): tuple[w, h: int] = @@ -644,9 +719,9 @@ proc setForegroundColor*(f: File, fg: ForegroundColor, bright = false) = 0, # fg8Bit not supported, see `enableTrueColors` instead. 0] # unused if fg == fgDefault: - discard setConsoleTextAttribute(h, toU16(old or defaultForegroundColor)) + discard setConsoleTextAttribute(h, cast[int16](cast[uint16](old) or cast[uint16](defaultForegroundColor))) else: - discard setConsoleTextAttribute(h, toU16(old or lookup[fg])) + discard setConsoleTextAttribute(h, cast[int16](cast[uint16](old) or cast[uint16](lookup[fg]))) else: gFG = ord(fg) if bright: inc(gFG, 60) @@ -673,9 +748,9 @@ proc setBackgroundColor*(f: File, bg: BackgroundColor, bright = false) = 0, # bg8Bit not supported, see `enableTrueColors` instead. 0] # unused if bg == bgDefault: - discard setConsoleTextAttribute(h, toU16(old or defaultBackgroundColor)) + discard setConsoleTextAttribute(h, cast[int16](cast[uint16](old) or cast[uint16](defaultBackgroundColor))) else: - discard setConsoleTextAttribute(h, toU16(old or lookup[bg])) + discard setConsoleTextAttribute(h, cast[int16](cast[uint16](old) or cast[uint16](lookup[bg]))) else: gBG = ord(bg) if bright: inc(gBG, 60) @@ -847,7 +922,7 @@ when defined(windows): stdout.write "\n" else: - import termios + import std/termios proc readPasswordFromStdin*(prompt: string, password: var string): bool {.tags: [ReadIOEffect, WriteIOEffect].} = @@ -903,7 +978,7 @@ proc isTrueColorSupported*(): bool = return getTerminal().trueColorIsSupported when defined(windows): - import os + import std/os proc enableTrueColors*() = ## Enables true color. diff --git a/lib/pure/times.nim b/lib/pure/times.nim index 80be55884..e59153455 100644 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -20,7 +20,7 @@ Examples ======== - .. code-block:: nim + ```nim import std/[times, os] # Simple benchmarking let time = cpuTime() @@ -37,6 +37,7 @@ # Arithmetic using TimeInterval echo "One year from now : ", now() + 1.years echo "One month from now : ", now() + 1.months + ``` Parsing and Formatting Dates ============================ @@ -44,10 +45,10 @@ The `DateTime` type can be parsed and formatted using the different `parse` and `format` procedures. - .. code-block:: nim - + ```nim let dt = parse("2000-01-01", "yyyy-MM-dd") echo dt.format("yyyy-MM-dd") + ``` The different format patterns that are supported are documented below. @@ -62,6 +63,8 @@ | `Monday -> Mon` `dddd` Full string for the day of the week. | `Saturday -> Saturday` | `Monday -> Monday` + `GG` The last two digits of the Iso Week-Year | `30/12/2012 -> 13` + `GGGG` The Iso week-calendar year padded to four digits | `30/12/2012 -> 2013` `h` The hours in one digit if possible. Ranging from 1-12. | `5pm -> 5` | `2am -> 2` `hh` The hours in two digits always. If the hour is one digit, 0 is prepended. | `5pm -> 05` @@ -104,6 +107,10 @@ | `24 AD -> 24` | `24 BC -> -23` | `12345 AD -> 12345` + `V` The Iso Week-Number as one or two digits | `3/2/2012 -> 5` + | `1/4/2012 -> 13` + `VV` The Iso Week-Number as two digits always. 0 is prepended if one digit. | `3/2/2012 -> 05` + | `1/4/2012 -> 13` `z` Displays the timezone offset from UTC. | `UTC+7 -> +7` | `UTC-5 -> -5` `zz` Same as above but with leading 0. | `UTC+7 -> +07` @@ -124,9 +131,10 @@ =========== ================================================================================= ============================================== Other strings can be inserted by putting them in `''`. For example - `hh'->'mm` will give `01->56`. The following characters can be - inserted without quoting them: `:` `-` `(` `)` `/` `[` `]` - `,`. A literal `'` can be specified with `''`. + `hh'->'mm` will give `01->56`. In addition to spaces, + the following characters can be inserted without quoting them: + `:` `-` `,` `.` `(` `)` `/` `[` `]`. + A literal `'` can be specified with `''`. However you don't need to necessarily separate format patterns, as an unambiguous format string like `yyyyMMddhhmmss` is also valid (although @@ -196,7 +204,7 @@ * `monotimes module <monotimes.html>`_ ]## -import strutils, math, options +import std/[strutils, math, options] import std/private/since include "system/inclrtl" @@ -206,7 +214,7 @@ when defined(nimPreviewSlimSystem): when defined(js): - import jscore + import std/jscore # This is really bad, but overflow checks are broken badly for # ints on the JS backend. See #6752. @@ -230,7 +238,7 @@ when defined(js): {.pop.} elif defined(posix): - import posix + import std/posix type CTime = posix.Time @@ -239,7 +247,7 @@ elif defined(posix): {.importc: "gettimeofday", header: "<sys/time.h>", sideEffect.} elif defined(windows): - import winlean, std/time_t + import std/winlean, std/time_t type CTime = time_t.Time @@ -408,6 +416,16 @@ const unitWeights: array[FixedTimeUnit, int64] = [ 7 * secondsInDay * 1e9.int64, ] +when (NimMajor, NimMinor) >= (1, 4): + # Newer versions of Nim don't track defects + {.pragma: parseFormatRaises, raises: [TimeParseError, TimeFormatParseError].} + {.pragma: parseRaises, raises: [TimeParseError].} +else: + # Still track when using older versions + {.pragma: parseFormatRaises, raises: [TimeParseError, TimeFormatParseError, Defect].} + {.pragma: parseRaises, raises: [TimeParseError, Defect].} + + # # Helper procs # @@ -527,7 +545,7 @@ proc getDayOfWeek*(monthday: MonthdayRange, month: Month, year: int): WeekDay assertValidDate monthday, month, year # 1970-01-01 is a Thursday, we adjust to the previous Monday let days = toEpochDay(monthday, month, year) - 3 - let weeks = floorDiv(days, 7) + let weeks = floorDiv(days, 7'i64) let wd = days - weeks * 7 # The value of d is 0 for a Sunday, 1 for a Monday, 2 for a Tuesday, etc. # so we must correct for the WeekDay type. @@ -592,8 +610,8 @@ proc stringifyUnit(value: int | int64, unit: TimeUnit): string = ## Stringify time unit with it's name, lowercased let strUnit = $unit result = "" - result.add($value) - result.add(" ") + result.addInt value + result.add ' ' if abs(value) != 1: result.add(strUnit.toLowerAscii()) else: @@ -636,11 +654,10 @@ template eqImpl(a: Duration|Time, b: Duration|Time): bool = const DurationZero* = Duration() ## \ ## Zero value for durations. Useful for comparisons. - ## - ## .. code-block:: nim - ## + ## ```nim ## doAssert initDuration(seconds = 1) > DurationZero ## doAssert initDuration(seconds = 0) == DurationZero + ## ``` proc initDuration*(nanoseconds, microseconds, milliseconds, seconds, minutes, hours, days, weeks: int64 = 0): Duration = @@ -953,27 +970,33 @@ proc toWinTime*(t: Time): int64 = ## since `1601-01-01T00:00:00Z`). result = t.seconds * rateDiff + epochDiff + t.nanosecond div 100 +proc getTimeImpl(typ: typedesc[Time]): Time = + discard "implemented in the vm" + proc getTime*(): Time {.tags: [TimeEffect], benign.} = ## Gets the current time as a `Time` with up to nanosecond resolution. - when defined(js): - let millis = newDate().getTime() - let seconds = convert(Milliseconds, Seconds, millis) - let nanos = convert(Milliseconds, Nanoseconds, - millis mod convert(Seconds, Milliseconds, 1).int) - result = initTime(seconds, nanos) - elif defined(macosx): - var a {.noinit.}: Timeval - gettimeofday(a) - result = initTime(a.tv_sec.int64, - convert(Microseconds, Nanoseconds, a.tv_usec.int)) - elif defined(posix): - var ts {.noinit.}: Timespec - discard clock_gettime(CLOCK_REALTIME, ts) - result = initTime(ts.tv_sec.int64, ts.tv_nsec.int) - elif defined(windows): - var f {.noinit.}: FILETIME - getSystemTimeAsFileTime(f) - result = fromWinTime(rdFileTime(f)) + when nimvm: + result = getTimeImpl(Time) + else: + when defined(js): + let millis = newDate().getTime() + let seconds = convert(Milliseconds, Seconds, millis) + let nanos = convert(Milliseconds, Nanoseconds, + millis mod convert(Seconds, Milliseconds, 1).int) + result = initTime(seconds, nanos) + elif defined(macosx): + var a {.noinit.}: Timeval + gettimeofday(a) + result = initTime(a.tv_sec.int64, + convert(Microseconds, Nanoseconds, a.tv_usec.int)) + elif defined(posix): + var ts {.noinit.}: Timespec + discard clock_gettime(CLOCK_REALTIME, ts) + result = initTime(ts.tv_sec.int64, ts.tv_nsec.int) + elif defined(windows): + var f {.noinit.}: FILETIME + getSystemTimeAsFileTime(f) + result = fromWinTime(rdFileTime(f)) proc `-`*(a, b: Time): Duration {.operator, extern: "ntDiffTime".} = ## Computes the duration between two points in time. @@ -1018,7 +1041,7 @@ proc high*(typ: typedesc[Time]): Time = initTime(high(int64), high(NanosecondRange)) proc low*(typ: typedesc[Time]): Time = - initTime(low(int64), 0) + initTime(0, 0) # # DateTime & Timezone @@ -1476,20 +1499,42 @@ proc `-=`*(a: var DateTime, b: Duration) = a = a - b proc getDateStr*(dt = now()): string {.rtl, extern: "nt$1", tags: [TimeEffect].} = - ## Gets the current local date as a string of the format `YYYY-MM-DD`. + ## Gets the current local date as a string of the format `YYYY-MM-dd`. runnableExamples: echo getDateStr(now() - 1.months) assertDateTimeInitialized dt - result = $dt.year & '-' & intToStr(dt.monthZero, 2) & - '-' & intToStr(dt.monthday, 2) + result = newStringOfCap(10) # len("YYYY-MM-dd") == 10 + result.addInt dt.year + result.add '-' + result.add intToStr(dt.monthZero, 2) + result.add '-' + result.add intToStr(dt.monthday, 2) proc getClockStr*(dt = now()): string {.rtl, extern: "nt$1", tags: [TimeEffect].} = ## Gets the current local clock time as a string of the format `HH:mm:ss`. runnableExamples: echo getClockStr(now() - 1.hours) assertDateTimeInitialized dt - result = intToStr(dt.hour, 2) & ':' & intToStr(dt.minute, 2) & - ':' & intToStr(dt.second, 2) + result = newStringOfCap(8) # len("HH:mm:ss") == 8 + result.add intToStr(dt.hour, 2) + result.add ':' + result.add intToStr(dt.minute, 2) + result.add ':' + result.add intToStr(dt.second, 2) + + +# +# Iso week forward declarations +# + +proc initDateTime*(weekday: WeekDay, isoweek: IsoWeekRange, isoyear: IsoYear, + hour: HourRange, minute: MinuteRange, second: SecondRange, + nanosecond: NanosecondRange, + zone: Timezone = local()): DateTime {.gcsafe, raises: [], tags: [], since: (1, 5).} + +proc initDateTime*(weekday: WeekDay, isoweek: IsoWeekRange, isoyear: IsoYear, + hour: HourRange, minute: MinuteRange, second: SecondRange, + zone: Timezone = local()): DateTime {.gcsafe, raises: [], tags: [], since: (1, 5).} # # TimeFormat @@ -1521,6 +1566,9 @@ type year: Option[int] month: Option[int] monthday: Option[int] + isoyear: Option[int] + yearweek: Option[int] + weekday: Option[WeekDay] utcOffset: Option[int] # '0' as default for these work fine @@ -1535,6 +1583,7 @@ type FormatPattern {.pure.} = enum d, dd, ddd, dddd + GG, GGGG h, hh, H, HH m, mm, M, MM, MMM, MMMM s, ss @@ -1544,6 +1593,7 @@ type YYYY uuuu UUUU + V, VV z, zz, zzz, zzzz ZZZ, ZZZZ g @@ -1582,7 +1632,7 @@ const "Sunday"], ) - FormatLiterals = {' ', '-', '/', ':', '(', ')', '[', ']', ','} + FormatLiterals = {' ', '-', '/', ':', '(', ')', '[', ']', ',', '.'} proc `$`*(f: TimeFormat): string = ## Returns the format string that was used to construct `f`. @@ -1672,6 +1722,8 @@ proc stringToPattern(str: string): FormatPattern = of "dd": result = dd of "ddd": result = ddd of "dddd": result = dddd + of "GG": result = GG + of "GGGG": result = GGGG of "h": result = h of "hh": result = hh of "H": result = H @@ -1694,6 +1746,8 @@ proc stringToPattern(str: string): FormatPattern = of "YYYY": result = YYYY of "uuuu": result = uuuu of "UUUU": result = UUUU + of "V": result = V + of "VV": result = VV of "z": result = z of "zz": result = zz of "zzz": result = zzz @@ -1743,6 +1797,10 @@ proc formatPattern(dt: DateTime, pattern: FormatPattern, result: var string, result.add loc.ddd[dt.weekday] of dddd: result.add loc.dddd[dt.weekday] + of GG: + result.add (dt.getIsoWeekAndYear.isoyear.int mod 100).intToStr(2) + of GGGG: + result.add $dt.getIsoWeekAndYear.isoyear of h: result.add( if dt.hour == 0: "12" @@ -1806,6 +1864,10 @@ proc formatPattern(dt: DateTime, pattern: FormatPattern, result: var string, result.add '+' & $year of UUUU: result.add $dt.year + of V: + result.add $dt.getIsoWeekAndYear.isoweek + of VV: + result.add dt.getIsoWeekAndYear.isoweek.intToStr(2) of z, zz, zzz, zzzz, ZZZ, ZZZZ: if dt.timezone != nil and dt.timezone.name == "Etc/UTC": result.add 'Z' @@ -1860,18 +1922,30 @@ proc parsePattern(input: string, pattern: FormatPattern, i: var int, result = monthday in MonthdayRange of ddd: result = false - for v in loc.ddd: + for d, v in loc.ddd: if input.substr(i, i+v.len-1).cmpIgnoreCase(v) == 0: + parsed.weekday = some(d.WeekDay) result = true i.inc v.len break of dddd: result = false - for v in loc.dddd: + for d, v in loc.dddd: if input.substr(i, i+v.len-1).cmpIgnoreCase(v) == 0: + parsed.weekday = some(d.WeekDay) result = true i.inc v.len break + of GG: + # Assumes current century + var isoyear = takeInt(2..2) + var thisCen = now().year div 100 + parsed.isoyear = some(thisCen*100 + isoyear) + result = isoyear > 0 + of GGGG: + let isoyear = takeInt(1..high(int)) + parsed.isoyear = some(isoyear) + result = isoyear > 0 of h, H: parsed.hour = takeInt(1..2) result = parsed.hour in HourRange @@ -1962,6 +2036,14 @@ proc parsePattern(input: string, pattern: FormatPattern, i: var int, parsed.year = some(year) of UUUU: parsed.year = some(takeInt(1..high(int), allowSign = true)) + of V: + let yearweek = takeInt(1..2) + parsed.yearweek = some(yearweek) + result = yearweek in IsoWeekRange + of VV: + let yearweek = takeInt(2..2) + parsed.yearweek = some(yearweek) + result = yearweek in IsoWeekRange of z, zz, zzz, zzzz, ZZZ, ZZZZ: case input[i] of '+', '-': @@ -2008,7 +2090,7 @@ proc parsePattern(input: string, pattern: FormatPattern, i: var int, i.inc 2 else: result = false - of Lit: doAssert false, "Can't happen" + of Lit: raiseAssert "Can't happen" proc toDateTime(p: ParsedTime, zone: Timezone, f: TimeFormat, input: string): DateTime = @@ -2063,6 +2145,38 @@ proc toDateTime(p: ParsedTime, zone: Timezone, f: TimeFormat, result = (dateTime(year, month, monthday, hour, minute, second, nanosecond, utc()).toTime + initDuration(seconds = p.utcOffset.get())).inZone(zone) +proc toDateTimeByWeek(p: ParsedTime, zone: Timezone, f: TimeFormat, + input: string): DateTime = + var isoyear = p.isoyear.get(0) + var yearweek = p.yearweek.get(1) + var weekday = p.weekday.get(dMon) + + if p.amPm != apUnknown: + raiseParseException(f, input, "Parsing iso weekyear dates does not support am/pm") + + if p.year.isSome: + raiseParseException(f, input, "Use iso-year GG or GGGG as year with iso week number") + + if p.month.isSome: + raiseParseException(f, input, "Use either iso week number V or VV or month") + + if p.monthday.isSome: + raiseParseException(f, input, "Use weekday ddd or dddd as day with with iso week number") + + if p.isoyear.isNone: + raiseParseException(f, input, "Need iso-year with week number") + + let hour = p.hour + let minute = p.minute + let second = p.second + let nanosecond = p.nanosecond + + if p.utcOffset.isNone: + result = initDateTime(weekday, yearweek.IsoWeekRange, isoyear.IsoYear, hour, minute, second, nanosecond, zone) + else: + result = (initDateTime(weekday, yearweek.IsoWeekRange, isoyear.IsoYear, hour, minute, second, nanosecond, zone).toTime + + initDuration(seconds = p.utcOffset.get())).inZone(zone) + proc format*(dt: DateTime, f: TimeFormat, loc: DateTimeLocale = DefaultLocale): string {.raises: [].} = ## Format `dt` using the format specified by `f`. @@ -2103,7 +2217,7 @@ proc format*(dt: DateTime, f: static[string]): string {.raises: [].} = const f2 = initTimeFormat(f) result = dt.format(f2) -proc formatValue*(result: var string; value: DateTime, specifier: string) = +proc formatValue*(result: var string; value: DateTime | Time, specifier: string) = ## adapter for strformat. Not intended to be called directly. result.add format(value, if specifier.len == 0: "yyyy-MM-dd'T'HH:mm:sszzz" else: specifier) @@ -2127,13 +2241,8 @@ proc format*(time: Time, f: static[string], zone: Timezone = local()): string const f2 = initTimeFormat(f) result = time.inZone(zone).format(f2) -template formatValue*(result: var string; value: Time, specifier: string) = - ## adapter for `strformat`. Not intended to be called directly. - result.add format(value, specifier) - proc parse*(input: string, f: TimeFormat, zone: Timezone = local(), - loc: DateTimeLocale = DefaultLocale): DateTime - {.raises: [TimeParseError, Defect].} = + loc: DateTimeLocale = DefaultLocale): DateTime {.parseRaises.} = ## Parses `input` as a `DateTime` using the format specified by `f`. ## If no UTC offset was parsed, then `input` is assumed to be specified in ## the `zone` timezone. If a UTC offset was parsed, the result will be @@ -2173,11 +2282,15 @@ proc parse*(input: string, f: TimeFormat, zone: Timezone = local(), raiseParseException(f, input, "Parsing ended but there was still patterns remaining") - result = toDateTime(parsed, zone, f, input) + if parsed.yearweek.isSome: + result = toDateTimeByWeek(parsed, zone, f, input) + elif parsed.isoyear.isSome: + raiseParseException(f, input, "Iso year GG or GGGG require iso week V or VV") + else: + result = toDateTime(parsed, zone, f, input) proc parse*(input, f: string, tz: Timezone = local(), - loc: DateTimeLocale = DefaultLocale): DateTime - {.raises: [TimeParseError, TimeFormatParseError, Defect].} = + loc: DateTimeLocale = DefaultLocale): DateTime {.parseFormatRaises.} = ## Shorthand for constructing a `TimeFormat` and using it to parse ## `input` as a `DateTime`. ## @@ -2190,14 +2303,12 @@ proc parse*(input, f: string, tz: Timezone = local(), result = input.parse(dtFormat, tz, loc = loc) proc parse*(input: string, f: static[string], zone: Timezone = local(), - loc: DateTimeLocale = DefaultLocale): - DateTime {.raises: [TimeParseError, Defect].} = + loc: DateTimeLocale = DefaultLocale): DateTime {.parseRaises.} = ## Overload that validates `f` at compile time. const f2 = initTimeFormat(f) result = input.parse(f2, zone, loc = loc) -proc parseTime*(input, f: string, zone: Timezone): Time - {.raises: [TimeParseError, TimeFormatParseError, Defect].} = +proc parseTime*(input, f: string, zone: Timezone): Time {.parseFormatRaises.} = ## Shorthand for constructing a `TimeFormat` and using it to parse ## `input` as a `DateTime`, then converting it a `Time`. ## @@ -2209,7 +2320,7 @@ proc parseTime*(input, f: string, zone: Timezone): Time parse(input, f, zone).toTime() proc parseTime*(input: string, f: static[string], zone: Timezone): Time - {.raises: [TimeParseError, Defect].} = + {.parseRaises.} = ## Overload that validates `format` at compile time. const f2 = initTimeFormat(f) result = input.parse(f2, zone).toTime() @@ -2638,13 +2749,13 @@ proc `-=`*(t: var Time, b: TimeInterval) = t = t - b # -# Day of year +# Iso week # proc initDateTime*(weekday: WeekDay, isoweek: IsoWeekRange, isoyear: IsoYear, hour: HourRange, minute: MinuteRange, second: SecondRange, nanosecond: NanosecondRange, - zone: Timezone = local()): DateTime {.since: (1, 5).} = + zone: Timezone = local()): DateTime {.raises: [], tags: [], since: (1, 5).} = ## Create a new `DateTime <#DateTime>`_ from a weekday and an ISO 8601 week number and year ## in the specified timezone. ## @@ -2656,12 +2767,12 @@ proc initDateTime*(weekday: WeekDay, isoweek: IsoWeekRange, isoyear: IsoYear, assert initDateTime(2, mJan, 2021, 00, 00, 00) == initDateTime(dSat, 53, 2020.IsoYear, 00, 00, 00) # source https://webspace.science.uu.nl/~gent0113/calendar/isocalendar.htm - let d = isoweek * 7 + weekday.int - initDateTime(4, mJan, isoyear.int, 00, 00, 00).weekday.int - 4 + let d = isoweek * 7 + weekday.int - initDateTime(4, mJan, isoyear.int, 00, 00, 00, zone).weekday.int - 4 initDateTime(1, mJan, isoyear.int, hour, minute, second, nanosecond, zone) + initTimeInterval(days=d) proc initDateTime*(weekday: WeekDay, isoweek: IsoWeekRange, isoyear: IsoYear, hour: HourRange, minute: MinuteRange, second: SecondRange, - zone: Timezone = local()): DateTime {.since: (1, 5).} = + zone: Timezone = local()): DateTime {.raises: [], tags: [], since: (1, 5).} = initDateTime(weekday, isoweek, isoyear, hour, minute, second, 0, zone) # @@ -2677,7 +2788,9 @@ proc epochTime*(): float {.tags: [TimeEffect].} = ## ## .. warning:: Unsuitable for benchmarking (but still better than `now`), ## use `monotimes.getMonoTime` or `cpuTime` instead, depending on the use case. - when defined(macosx): + when defined(js): + result = newDate().getTime() / 1000 + elif defined(macosx): var a {.noinit.}: Timeval gettimeofday(a) result = toBiggestFloat(a.tv_sec.int64) + toBiggestFloat( @@ -2694,8 +2807,6 @@ proc epochTime*(): float {.tags: [TimeEffect].} = var secs = i64 div rateDiff var subsecs = i64 mod rateDiff result = toFloat(int(secs)) + toFloat(int(subsecs)) * 0.0000001 - elif defined(js): - result = newDate().getTime() / 1000 else: {.error: "unknown OS".} diff --git a/lib/pure/typetraits.nim b/lib/pure/typetraits.nim index c20f9e645..78af84fdd 100644 --- a/lib/pure/typetraits.nim +++ b/lib/pure/typetraits.nim @@ -96,6 +96,23 @@ proc supportsCopyMem*(t: typedesc): bool {.magic: "TypeTrait".} ## ## Other languages name a type like these `blob`:idx:. +proc hasDefaultValue*(t: typedesc): bool {.magic: "TypeTrait".} = + ## Returns true if `t` has a valid default value. + runnableExamples: + {.experimental: "strictNotNil".} + type + NilableObject = ref object + a: int + Object = NilableObject not nil + RequiresInit[T] = object + a {.requiresInit.}: T + + assert hasDefaultValue(NilableObject) + assert not hasDefaultValue(Object) + assert hasDefaultValue(string) + assert not hasDefaultValue(var string) + assert not hasDefaultValue(RequiresInit[int]) + proc isNamedTuple*(T: typedesc): bool {.magic: "TypeTrait".} = ## Returns true for named tuples, false for any other type. runnableExamples: @@ -113,6 +130,40 @@ template pointerBase*[T](_: typedesc[ptr T | ref T]): typedesc = assert (var s = "abc"; s[0].addr).typeof.pointerBase is char T +proc rangeBase*(T: typedesc[range]): typedesc {.magic: "TypeTrait".} = + ## Returns the base type for range types, or the type itself otherwise. + ## + ## **See also:** + ## * `rangeBase template <#rangeBase.t,T>`_ + runnableExamples: + type MyRange = range[0..5] + type MyEnum = enum a, b, c + type MyEnumRange = range[b..c] + doAssert rangeBase(MyRange) is int + doAssert rangeBase(MyEnumRange) is MyEnum + doAssert rangeBase(range['a'..'z']) is char + +template rangeBase*[T: range](a: T): untyped = + ## Overload of `rangeBase <#rangeBase,typedesc,static[bool]>`_ for values. + runnableExamples: + type MyRange = range[0..5] + type MyEnum = enum a, b, c + type MyEnumRange = range[b..c] + let x = MyRange(3) + doAssert rangeBase(x) is int + doAssert $typeof(rangeBase(x)) == "int" + doAssert rangeBase(x) == 3 + let y: set[MyEnumRange] = {c} + for e in y: + doAssert rangeBase(e) is MyEnum + doAssert $typeof(rangeBase(e)) == "MyEnum" + doAssert rangeBase(e) == c + let z: seq[range['a'..'z']] = @['c'] + doAssert rangeBase(z[0]) is char + doAssert $typeof(rangeBase(z[0])) == "char" + doAssert rangeBase(z[0]) == 'c' + rangeBase(typeof(T))(a) + proc distinctBase*(T: typedesc, recursive: static bool = true): typedesc {.magic: "TypeTrait".} = ## Returns the base type for distinct types, or the type itself otherwise. ## If `recursive` is false, only the immediate distinct base will be returned. @@ -187,7 +238,7 @@ since (1, 3, 5): typeof(block: (for ai in a: ai)) -import macros +import std/macros macro enumLen*(T: typedesc[enum]): int = ## Returns the number of items in the enum `T`. @@ -225,7 +276,7 @@ macro genericParamsImpl(T: typedesc): untyped = case ai.typeKind of ntyTypeDesc: ret = ai - of ntyStatic: doAssert false + of ntyStatic: raiseAssert "unreachable" else: # getType from a resolved symbol might return a typedesc symbol. # If so, use it directly instead of wrapping it in StaticParam. diff --git a/lib/pure/unicode.nim b/lib/pure/unicode.nim index 660be7814..8cbe117bb 100644 --- a/lib/pure/unicode.nim +++ b/lib/pure/unicode.nim @@ -21,6 +21,16 @@ ## * `encodings module <encodings.html>`_ include "system/inclrtl" +import std/strbasics +template toOa(s: string): auto = s.toOpenArray(0, s.high) + +proc substr(s: openArray[char] , first, last: int): string = + # Copied substr from system + let first = max(first, 0) + let L = max(min(last, high(s)) - first + 1, 0) + result = newString(L) + for i in 0 .. L-1: + result[i] = s[i+first] type RuneImpl = int32 # underlying type of Rune @@ -32,7 +42,7 @@ type template ones(n: untyped): untyped = ((1 shl n)-1) -proc runeLen*(s: string): int {.rtl, extern: "nuc$1".} = +proc runeLen*(s: openArray[char]): int {.rtl, extern: "nuc$1".} = ## Returns the number of runes of the string ``s``. runnableExamples: let a = "añyóng" @@ -51,7 +61,7 @@ proc runeLen*(s: string): int {.rtl, extern: "nuc$1".} = else: inc i inc(result) -proc runeLenAt*(s: string, i: Natural): int = +proc runeLenAt*(s: openArray[char], i: Natural): int = ## Returns the number of bytes the rune starting at ``s[i]`` takes. ## ## See also: @@ -71,7 +81,7 @@ proc runeLenAt*(s: string, i: Natural): int = const replRune = Rune(0xFFFD) -template fastRuneAt*(s: string, i: int, result: untyped, doInc = true) = +template fastRuneAt*(s: openArray[char] or string, i: int, result: untyped, doInc = true) = ## Returns the rune ``s[i]`` in ``result``. ## ## If ``doInc == true`` (default), ``i`` is incremented by the number @@ -149,7 +159,7 @@ template fastRuneAt*(s: string, i: int, result: untyped, doInc = true) = result = Rune(uint(s[i])) when doInc: inc(i) -proc runeAt*(s: string, i: Natural): Rune = +proc runeAt*(s: openArray[char], i: Natural): Rune = ## Returns the rune in ``s`` at **byte index** ``i``. ## ## See also: @@ -163,7 +173,7 @@ proc runeAt*(s: string, i: Natural): Rune = doAssert a.runeAt(3) == "y".runeAt(0) fastRuneAt(s, i, result, false) -proc validateUtf8*(s: string): int = +proc validateUtf8*(s: openArray[char]): int = ## Returns the position of the invalid byte in ``s`` if the string ``s`` does ## not hold valid UTF-8 data. Otherwise ``-1`` is returned. ## @@ -300,7 +310,7 @@ proc `$`*(runes: seq[Rune]): string = for rune in runes: result.add rune -proc runeOffset*(s: string, pos: Natural, start: Natural = 0): int = +proc runeOffset*(s: openArray[char], pos: Natural, start: Natural = 0): int = ## Returns the byte position of rune ## at position ``pos`` in ``s`` with an optional start byte position. ## Returns the special value -1 if it runs out of the string. @@ -327,7 +337,7 @@ proc runeOffset*(s: string, pos: Natural, start: Natural = 0): int = inc i return o -proc runeReverseOffset*(s: string, rev: Positive): (int, int) = +proc runeReverseOffset*(s: openArray[char], rev: Positive): (int, int) = ## Returns a tuple with the byte offset of the ## rune at position ``rev`` in ``s``, counting ## from the end (starting with 1) and the total @@ -355,7 +365,7 @@ proc runeReverseOffset*(s: string, rev: Positive): (int, int) = dec a result = if a > 0: (-a, rev.int-a) else: (x, -a+rev.int) -proc runeAtPos*(s: string, pos: int): Rune = +proc runeAtPos*(s: openArray[char], pos: int): Rune = ## Returns the rune at position ``pos``. ## ## **Beware:** This can lead to unoptimized code and slow execution! @@ -368,7 +378,7 @@ proc runeAtPos*(s: string, pos: int): Rune = ## * `fastRuneAt template <#fastRuneAt.t,string,int,untyped>`_ fastRuneAt(s, runeOffset(s, pos), result, false) -proc runeStrAtPos*(s: string, pos: Natural): string = +proc runeStrAtPos*(s: openArray[char], pos: Natural): string = ## Returns the rune at position ``pos`` as UTF8 String. ## ## **Beware:** This can lead to unoptimized code and slow execution! @@ -380,9 +390,9 @@ proc runeStrAtPos*(s: string, pos: Natural): string = ## * `runeAtPos proc <#runeAtPos,string,int>`_ ## * `fastRuneAt template <#fastRuneAt.t,string,int,untyped>`_ let o = runeOffset(s, pos) - s[o .. (o+runeLenAt(s, o)-1)] + substr(s.toOpenArray(o, (o+runeLenAt(s, o)-1))) -proc runeSubStr*(s: string, pos: int, len: int = int.high): string = +proc runeSubStr*(s: openArray[char], pos: int, len: int = int.high): string = ## Returns the UTF-8 substring starting at code point ``pos`` ## with ``len`` code points. ## @@ -401,7 +411,7 @@ proc runeSubStr*(s: string, pos: int, len: int = int.high): string = if pos < 0: let (o, rl) = runeReverseOffset(s, -pos) if len >= rl: - result = s.substr(o, s.len-1) + result = s.substr(o, s.high) elif len < 0: let e = rl + len if e < 0: @@ -454,7 +464,7 @@ proc `==`*(a, b: Rune): bool = include "includes/unicode_ranges" -proc binarySearch(c: RuneImpl, tab: openArray[int], len, stride: int): int = +proc binarySearch(c: RuneImpl, tab: openArray[int32], len, stride: int): int = var n = len var t = 0 while n > 1: @@ -626,7 +636,7 @@ template runeCheck(s, runeProc) = fastRuneAt(s, i, rune, doInc = true) result = runeProc(rune) and result -proc isAlpha*(s: string): bool {.noSideEffect, +proc isAlpha*(s: openArray[char]): bool {.noSideEffect, rtl, extern: "nuc$1Str".} = ## Returns true if ``s`` contains all alphabetic runes. runnableExamples: @@ -634,7 +644,7 @@ proc isAlpha*(s: string): bool {.noSideEffect, doAssert a.isAlpha runeCheck(s, isAlpha) -proc isSpace*(s: string): bool {.noSideEffect, +proc isSpace*(s: openArray[char]): bool {.noSideEffect, rtl, extern: "nuc$1Str".} = ## Returns true if ``s`` contains all whitespace runes. runnableExamples: @@ -655,21 +665,21 @@ template convertRune(s, runeProc) = rune = runeProc(rune) fastToUTF8Copy(rune, result, resultIndex, doInc = true) -proc toUpper*(s: string): string {.noSideEffect, +proc toUpper*(s: openArray[char]): string {.noSideEffect, rtl, extern: "nuc$1Str".} = ## Converts ``s`` into upper-case runes. runnableExamples: doAssert toUpper("abγ") == "ABΓ" convertRune(s, toUpper) -proc toLower*(s: string): string {.noSideEffect, +proc toLower*(s: openArray[char]): string {.noSideEffect, rtl, extern: "nuc$1Str".} = ## Converts ``s`` into lower-case runes. runnableExamples: doAssert toLower("ABΓ") == "abγ" convertRune(s, toLower) -proc swapCase*(s: string): string {.noSideEffect, +proc swapCase*(s: openArray[char]): string {.noSideEffect, rtl, extern: "nuc$1".} = ## Swaps the case of runes in ``s``. ## @@ -691,7 +701,7 @@ proc swapCase*(s: string): string {.noSideEffect, rune = rune.toUpper() fastToUTF8Copy(rune, result, resultIndex, doInc = true) -proc capitalize*(s: string): string {.noSideEffect, +proc capitalize*(s: openArray[char]): string {.noSideEffect, rtl, extern: "nuc$1".} = ## Converts the first character of ``s`` into an upper-case rune. runnableExamples: @@ -703,12 +713,12 @@ proc capitalize*(s: string): string {.noSideEffect, rune: Rune i = 0 fastRuneAt(s, i, rune, doInc = true) - result = $toUpper(rune) & substr(s, i) + result = $toUpper(rune) & substr(s.toOpenArray(i, s.high)) when not defined(nimHasEffectsOf): {.pragma: effectsOf.} -proc translate*(s: string, replacements: proc(key: string): string): string {. +proc translate*(s: openArray[char], replacements: proc(key: string): string): string {. rtl, extern: "nuc$1", effectsOf: replacements.} = ## Translates words in a string using the ``replacements`` proc to substitute ## words inside ``s`` with their replacements. @@ -743,7 +753,7 @@ proc translate*(s: string, replacements: proc(key: string): string): string {. if whiteSpace and inWord: # If we've reached the end of a word - let word = s[wordStart ..< lastIndex] + let word = substr(s.toOpenArray(wordStart, lastIndex - 1)) result.add(replacements(word)) result.add($rune) inWord = false @@ -758,10 +768,10 @@ proc translate*(s: string, replacements: proc(key: string): string): string {. if wordStart < len(s) and inWord: # Get the trailing word at the end - let word = s[wordStart .. ^1] + let word = substr(s.toOpenArray(wordStart, s.high)) result.add(replacements(word)) -proc title*(s: string): string {.noSideEffect, +proc title*(s: openArray[char]): string {.noSideEffect, rtl, extern: "nuc$1".} = ## Converts ``s`` to a unicode title. ## @@ -787,7 +797,7 @@ proc title*(s: string): string {.noSideEffect, fastToUTF8Copy(rune, result, resultIndex, doInc = true) -iterator runes*(s: string): Rune = +iterator runes*(s: openArray[char]): Rune = ## Iterates over any rune of the string ``s`` returning runes. var i = 0 @@ -796,7 +806,7 @@ iterator runes*(s: string): Rune = fastRuneAt(s, i, result, true) yield result -iterator utf8*(s: string): string = +iterator utf8*(s: openArray[char]): string = ## Iterates over any rune of the string ``s`` returning utf8 values. ## ## See also: @@ -807,10 +817,10 @@ iterator utf8*(s: string): string = var o = 0 while o < s.len: let n = runeLenAt(s, o) - yield s[o .. (o+n-1)] + yield substr(s.toOpenArray(o, (o+n-1))) o += n -proc toRunes*(s: string): seq[Rune] = +proc toRunes*(s: openArray[char]): seq[Rune] = ## Obtains a sequence containing the Runes in ``s``. ## ## See also: @@ -823,12 +833,12 @@ proc toRunes*(s: string): seq[Rune] = for r in s.runes: result.add(r) -proc cmpRunesIgnoreCase*(a, b: string): int {.rtl, extern: "nuc$1".} = +proc cmpRunesIgnoreCase*(a, b: openArray[char]): int {.rtl, extern: "nuc$1".} = ## Compares two UTF-8 strings and ignores the case. Returns: ## - ## | 0 if a == b - ## | < 0 if a < b - ## | > 0 if a > b + ## | `0` if a == b + ## | `< 0` if a < b + ## | `> 0` if a > b var i = 0 var j = 0 var ar, br: Rune @@ -836,11 +846,16 @@ proc cmpRunesIgnoreCase*(a, b: string): int {.rtl, extern: "nuc$1".} = # slow path: fastRuneAt(a, i, ar) fastRuneAt(b, j, br) - result = RuneImpl(toLower(ar)) - RuneImpl(toLower(br)) + when sizeof(int) < 4: + const lo = low(int).int32 + const hi = high(int).int32 + result = clamp(RuneImpl(toLower(ar)) - RuneImpl(toLower(br)), lo, hi).int + else: + result = RuneImpl(toLower(ar)) - RuneImpl(toLower(br)) if result != 0: return result = a.len - b.len -proc reversed*(s: string): string = +proc reversed*(s: openArray[char]): string = ## Returns the reverse of ``s``, interpreting it as runes. ## ## Unicode combining characters are correctly interpreted as well. @@ -875,9 +890,9 @@ proc reversed*(s: string): string = reverseUntil(len(s)) -proc graphemeLen*(s: string; i: Natural): Natural = +proc graphemeLen*(s: openArray[char]; i: Natural): Natural = ## The number of bytes belonging to byte index ``s[i]``, - ## including following combining code unit. + ## including following combining code units. runnableExamples: let a = "añyóng" doAssert a.graphemeLen(1) == 2 ## ñ @@ -894,7 +909,7 @@ proc graphemeLen*(s: string; i: Natural): Natural = if not isCombining(r2): break result = j-i -proc lastRune*(s: string; last: int): (Rune, int) = +proc lastRune*(s: openArray[char]; last: int): (Rune, int) = ## Length of the last rune in ``s[0..last]``. Returns the rune and its length ## in bytes. if s[last] <= chr(127): @@ -923,12 +938,12 @@ proc size*(r: Rune): int {.noSideEffect.} = else: result = 1 # --------- Private templates for different split separators ----------- -proc stringHasSep(s: string, index: int, seps: openArray[Rune]): bool = +proc stringHasSep(s: openArray[char], index: int, seps: openArray[Rune]): bool = var rune: Rune fastRuneAt(s, index, rune, false) return seps.contains(rune) -proc stringHasSep(s: string, index: int, sep: Rune): bool = +proc stringHasSep(s: openArray[char], index: int, sep: Rune): bool = var rune: Rune fastRuneAt(s, index, rune, false) return sep == rune @@ -946,12 +961,12 @@ template splitCommon(s, sep, maxsplit: untyped) = while last < sLen and not stringHasSep(s, last, sep): inc(last, runeLenAt(s, last)) if splits == 0: last = sLen - yield s[first .. (last - 1)] + yield substr(s.toOpenArray(first, (last - 1))) if splits == 0: break dec(splits) inc(last, if last < sLen: runeLenAt(s, last) else: 1) -iterator split*(s: string, seps: openArray[Rune] = unicodeSpaces, +iterator split*(s: openArray[char], seps: openArray[Rune] = unicodeSpaces, maxsplit: int = -1): string = ## Splits the unicode string ``s`` into substrings using a group of separators. ## @@ -977,7 +992,7 @@ iterator split*(s: string, seps: openArray[Rune] = unicodeSpaces, splitCommon(s, seps, maxsplit) -iterator splitWhitespace*(s: string): string = +iterator splitWhitespace*(s: openArray[char]): string = ## Splits a unicode string at whitespace runes. splitCommon(s, unicodeSpaces, -1) @@ -985,13 +1000,13 @@ template accResult(iter: untyped) = result = @[] for x in iter: add(result, x) -proc splitWhitespace*(s: string): seq[string] {.noSideEffect, +proc splitWhitespace*(s: openArray[char]): seq[string] {.noSideEffect, rtl, extern: "ncuSplitWhitespace".} = ## The same as the `splitWhitespace <#splitWhitespace.i,string>`_ ## iterator, but is a proc that returns a sequence of substrings. accResult(splitWhitespace(s)) -iterator split*(s: string, sep: Rune, maxsplit: int = -1): string = +iterator split*(s: openArray[char], sep: Rune, maxsplit: int = -1): string = ## Splits the unicode string ``s`` into substrings using a single separator. ## Substrings are separated by the rune ``sep``. runnableExamples: @@ -1002,19 +1017,19 @@ iterator split*(s: string, sep: Rune, maxsplit: int = -1): string = splitCommon(s, sep, maxsplit) -proc split*(s: string, seps: openArray[Rune] = unicodeSpaces, maxsplit: int = -1): +proc split*(s: openArray[char], seps: openArray[Rune] = unicodeSpaces, maxsplit: int = -1): seq[string] {.noSideEffect, rtl, extern: "nucSplitRunes".} = ## The same as the `split iterator <#split.i,string,openArray[Rune],int>`_, ## but is a proc that returns a sequence of substrings. accResult(split(s, seps, maxsplit)) -proc split*(s: string, sep: Rune, maxsplit: int = -1): seq[string] {.noSideEffect, +proc split*(s: openArray[char], sep: Rune, maxsplit: int = -1): seq[string] {.noSideEffect, rtl, extern: "nucSplitRune".} = ## The same as the `split iterator <#split.i,string,Rune,int>`_, but is a proc ## that returns a sequence of substrings. accResult(split(s, sep, maxsplit)) -proc strip*(s: string, leading = true, trailing = true, +proc strip*(s: openArray[char], leading = true, trailing = true, runes: openArray[Rune] = unicodeSpaces): string {.noSideEffect, rtl, extern: "nucStrip".} = ## Strips leading or trailing ``runes`` from ``s`` and returns @@ -1069,7 +1084,7 @@ proc strip*(s: string, leading = true, trailing = true, let newLen = eI - sI + 1 result = newStringOfCap(newLen) if newLen > 0: - result.add s[sI .. eI] + result.add substr(s.toOpenArray(sI, eI)) proc repeat*(c: Rune, count: Natural): string {.noSideEffect, rtl, extern: "nucRepeatRune".} = @@ -1085,7 +1100,7 @@ proc repeat*(c: Rune, count: Natural): string {.noSideEffect, for i in 0 ..< count: result.add s -proc align*(s: string, count: Natural, padding = ' '.Rune): string {. +proc align*(s: openArray[char], count: Natural, padding = ' '.Rune): string {. noSideEffect, rtl, extern: "nucAlignString".} = ## Aligns a unicode string ``s`` with ``padding``, so that it has a rune-length ## of ``count``. @@ -1110,9 +1125,9 @@ proc align*(s: string, count: Natural, padding = ' '.Rune): string {. for i in 0 ..< spaces: result.add padStr result.add s else: - result = s + result = s.substr -proc alignLeft*(s: string, count: Natural, padding = ' '.Rune): string {. +proc alignLeft*(s: openArray[char], count: Natural, padding = ' '.Rune): string {. noSideEffect.} = ## Left-aligns a unicode string ``s`` with ``padding``, so that it has a ## rune-length of ``count``. @@ -1136,4 +1151,365 @@ proc alignLeft*(s: string, count: Natural, padding = ' '.Rune): string {. for i in sLen ..< count: result.add padStr else: - result = s + result = s.substr + + +proc runeLen*(s: string): int {.inline.} = + ## Returns the number of runes of the string ``s``. + runnableExamples: + let a = "añyóng" + doAssert a.runeLen == 6 + ## note: a.len == 8 + runeLen(toOa(s)) + +proc runeLenAt*(s: string, i: Natural): int {.inline.} = + ## Returns the number of bytes the rune starting at ``s[i]`` takes. + ## + ## See also: + ## * `fastRuneAt template <#fastRuneAt.t,string,int,untyped>`_ + runnableExamples: + let a = "añyóng" + doAssert a.runeLenAt(0) == 1 + doAssert a.runeLenAt(1) == 2 + runeLenAt(toOa(s), i) + +proc runeAt*(s: string, i: Natural): Rune {.inline.} = + ## Returns the rune in ``s`` at **byte index** ``i``. + ## + ## See also: + ## * `runeAtPos proc <#runeAtPos,string,int>`_ + ## * `runeStrAtPos proc <#runeStrAtPos,string,Natural>`_ + ## * `fastRuneAt template <#fastRuneAt.t,string,int,untyped>`_ + runnableExamples: + let a = "añyóng" + doAssert a.runeAt(1) == "ñ".runeAt(0) + doAssert a.runeAt(2) == "ñ".runeAt(1) + doAssert a.runeAt(3) == "y".runeAt(0) + fastRuneAt(s, i, result, false) + +proc validateUtf8*(s: string): int {.inline.} = + ## Returns the position of the invalid byte in ``s`` if the string ``s`` does + ## not hold valid UTF-8 data. Otherwise ``-1`` is returned. + ## + ## See also: + ## * `toUTF8 proc <#toUTF8,Rune>`_ + ## * `$ proc <#$,Rune>`_ alias for `toUTF8` + ## * `fastToUTF8Copy template <#fastToUTF8Copy.t,Rune,string,int>`_ + validateUtf8(toOa(s)) + +proc runeOffset*(s: string, pos: Natural, start: Natural = 0): int {.inline.} = + ## Returns the byte position of rune + ## at position ``pos`` in ``s`` with an optional start byte position. + ## Returns the special value -1 if it runs out of the string. + ## + ## **Beware:** This can lead to unoptimized code and slow execution! + ## Most problems can be solved more efficiently by using an iterator + ## or conversion to a seq of Rune. + ## + ## See also: + ## * `runeReverseOffset proc <#runeReverseOffset,string,Positive>`_ + runnableExamples: + let a = "añyóng" + doAssert a.runeOffset(1) == 1 + doAssert a.runeOffset(3) == 4 + doAssert a.runeOffset(4) == 6 + runeOffset(toOa(s), pos, start) + +proc runeReverseOffset*(s: string, rev: Positive): (int, int) {.inline.} = + ## Returns a tuple with the byte offset of the + ## rune at position ``rev`` in ``s``, counting + ## from the end (starting with 1) and the total + ## number of runes in the string. + ## + ## Returns a negative value for offset if there are too few runes in + ## the string to satisfy the request. + ## + ## **Beware:** This can lead to unoptimized code and slow execution! + ## Most problems can be solved more efficiently by using an iterator + ## or conversion to a seq of Rune. + ## + ## See also: + ## * `runeOffset proc <#runeOffset,string,Natural,Natural>`_ + runeReverseOffset(toOa(s), rev) + +proc runeAtPos*(s: string, pos: int): Rune {.inline.} = + ## Returns the rune at position ``pos``. + ## + ## **Beware:** This can lead to unoptimized code and slow execution! + ## Most problems can be solved more efficiently by using an iterator + ## or conversion to a seq of Rune. + ## + ## See also: + ## * `runeAt proc <#runeAt,string,Natural>`_ + ## * `runeStrAtPos proc <#runeStrAtPos,string,Natural>`_ + ## * `fastRuneAt template <#fastRuneAt.t,string,int,untyped>`_ + fastRuneAt(toOa(s), runeOffset(s, pos), result, false) + +proc runeStrAtPos*(s: string, pos: Natural): string {.inline.} = + ## Returns the rune at position ``pos`` as UTF8 String. + ## + ## **Beware:** This can lead to unoptimized code and slow execution! + ## Most problems can be solved more efficiently by using an iterator + ## or conversion to a seq of Rune. + ## + ## See also: + ## * `runeAt proc <#runeAt,string,Natural>`_ + ## * `runeAtPos proc <#runeAtPos,string,int>`_ + ## * `fastRuneAt template <#fastRuneAt.t,string,int,untyped>`_ + let o = runeOffset(s, pos) + substr(s.toOpenArray(o, (o+runeLenAt(s, o)-1))) + +proc runeSubStr*(s: string, pos: int, len: int = int.high): string {.inline.} = + ## Returns the UTF-8 substring starting at code point ``pos`` + ## with ``len`` code points. + ## + ## If ``pos`` or ``len`` is negative they count from + ## the end of the string. If ``len`` is not given it means the longest + ## possible string. + runnableExamples: + let s = "Hänsel ««: 10,00€" + doAssert(runeSubStr(s, 0, 2) == "Hä") + doAssert(runeSubStr(s, 10, 1) == ":") + doAssert(runeSubStr(s, -6) == "10,00€") + doAssert(runeSubStr(s, 10) == ": 10,00€") + doAssert(runeSubStr(s, 12, 5) == "10,00") + doAssert(runeSubStr(s, -6, 3) == "10,") + runeSubStr(toOa(s), pos, len) + + +proc isAlpha*(s: string): bool {.noSideEffect, inline.} = + ## Returns true if ``s`` contains all alphabetic runes. + runnableExamples: + let a = "añyóng" + doAssert a.isAlpha + isAlpha(toOa(s)) + +proc isSpace*(s: string): bool {.noSideEffect, inline.} = + ## Returns true if ``s`` contains all whitespace runes. + runnableExamples: + let a = "\t\l \v\r\f" + doAssert a.isSpace + isSpace(toOa(s)) + + +proc toUpper*(s: string): string {.noSideEffect, inline.} = + ## Converts ``s`` into upper-case runes. + runnableExamples: + doAssert toUpper("abγ") == "ABΓ" + toUpper(toOa(s)) + +proc toLower*(s: string): string {.noSideEffect, inline.} = + ## Converts ``s`` into lower-case runes. + runnableExamples: + doAssert toLower("ABΓ") == "abγ" + toLower(toOa(s)) + +proc swapCase*(s: string): string {.noSideEffect, inline.} = + ## Swaps the case of runes in ``s``. + ## + ## Returns a new string such that the cases of all runes + ## are swapped if possible. + runnableExamples: + doAssert swapCase("Αlpha Βeta Γamma") == "αLPHA βETA γAMMA" + swapCase(toOa(s)) + +proc capitalize*(s: string): string {.noSideEffect.} = + ## Converts the first character of ``s`` into an upper-case rune. + runnableExamples: + doAssert capitalize("βeta") == "Βeta" + capitalize(toOa(s)) + + +proc translate*(s: string, replacements: proc(key: string): string): string {.effectsOf: replacements, inline.} = + ## Translates words in a string using the ``replacements`` proc to substitute + ## words inside ``s`` with their replacements. + ## + ## ``replacements`` is any proc that takes a word and returns + ## a new word to fill it's place. + runnableExamples: + proc wordToNumber(s: string): string = + case s + of "one": "1" + of "two": "2" + else: s + let a = "one two three four" + doAssert a.translate(wordToNumber) == "1 2 three four" + translate(toOa(s), replacements) + +proc title*(s: string): string {.noSideEffect, inline.} = + ## Converts ``s`` to a unicode title. + ## + ## Returns a new string such that the first character + ## in each word inside ``s`` is capitalized. + runnableExamples: + doAssert title("αlpha βeta γamma") == "Αlpha Βeta Γamma" + title(toOa(s)) + + +iterator runes*(s: string): Rune = + ## Iterates over any rune of the string ``s`` returning runes. + for rune in runes(toOa(s)): + yield rune + +iterator utf8*(s: string): string = + ## Iterates over any rune of the string ``s`` returning utf8 values. + ## + ## See also: + ## * `validateUtf8 proc <#validateUtf8,string>`_ + ## * `toUTF8 proc <#toUTF8,Rune>`_ + ## * `$ proc <#$,Rune>`_ alias for `toUTF8` + ## * `fastToUTF8Copy template <#fastToUTF8Copy.t,Rune,string,int>`_ + for str in utf8(toOa(s)): + yield str + +proc toRunes*(s: string): seq[Rune] {.inline.} = + ## Obtains a sequence containing the Runes in ``s``. + ## + ## See also: + ## * `$ proc <#$,Rune>`_ for a reverse operation + runnableExamples: + let a = toRunes("aáä") + doAssert a == @["a".runeAt(0), "á".runeAt(0), "ä".runeAt(0)] + toRunes(toOa(s)) + +proc cmpRunesIgnoreCase*(a, b: string): int {.inline.} = + ## Compares two UTF-8 strings and ignores the case. Returns: + ## + ## | `0` if a == b + ## | `< 0` if a < b + ## | `> 0` if a > b + cmpRunesIgnoreCase(a.toOa(), b.toOa()) + +proc reversed*(s: string): string {.inline.} = + ## Returns the reverse of ``s``, interpreting it as runes. + ## + ## Unicode combining characters are correctly interpreted as well. + runnableExamples: + assert reversed("Reverse this!") == "!siht esreveR" + assert reversed("先秦兩漢") == "漢兩秦先" + assert reversed("as⃝df̅") == "f̅ds⃝a" + assert reversed("a⃞b⃞c⃞") == "c⃞b⃞a⃞" + reversed(toOa(s)) + +proc graphemeLen*(s: string; i: Natural): Natural {.inline.} = + ## The number of bytes belonging to byte index ``s[i]``, + ## including following combining code unit. + runnableExamples: + let a = "añyóng" + doAssert a.graphemeLen(1) == 2 ## ñ + doAssert a.graphemeLen(2) == 1 + doAssert a.graphemeLen(4) == 2 ## ó + graphemeLen(toOa(s), i) + +proc lastRune*(s: string; last: int): (Rune, int) {.inline.} = + ## Length of the last rune in ``s[0..last]``. Returns the rune and its length + ## in bytes. + lastRune(toOa(s), last) + +iterator split*(s: string, seps: openArray[Rune] = unicodeSpaces, + maxsplit: int = -1): string = + ## Splits the unicode string ``s`` into substrings using a group of separators. + ## + ## Substrings are separated by a substring containing only ``seps``. + runnableExamples: + import std/sequtils + + assert toSeq("hÃllo\lthis\lis an\texample\l是".split) == + @["hÃllo", "this", "is", "an", "example", "是"] + + # And the following code splits the same string using a sequence of Runes. + assert toSeq(split("añyóng:hÃllo;是$example", ";:$".toRunes)) == + @["añyóng", "hÃllo", "是", "example"] + + # example with a `Rune` separator and unused one `;`: + assert toSeq(split("ab是de:f:", ";:是".toRunes)) == @["ab", "de", "f", ""] + + # Another example that splits a string containing a date. + let date = "2012-11-20T22:08:08.398990" + + assert toSeq(split(date, " -:T".toRunes)) == + @["2012", "11", "20", "22", "08", "08.398990"] + + splitCommon(toOa(s), seps, maxsplit) + +iterator splitWhitespace*(s: string): string = + ## Splits a unicode string at whitespace runes. + splitCommon(s.toOa(), unicodeSpaces, -1) + + +proc splitWhitespace*(s: string): seq[string] {.noSideEffect, inline.}= + ## The same as the `splitWhitespace <#splitWhitespace.i,string>`_ + ## iterator, but is a proc that returns a sequence of substrings. + accResult(splitWhitespace(toOa(s))) + +iterator split*(s: string, sep: Rune, maxsplit: int = -1): string = + ## Splits the unicode string ``s`` into substrings using a single separator. + ## Substrings are separated by the rune ``sep``. + runnableExamples: + import std/sequtils + + assert toSeq(split(";;hÃllo;this;is;an;;example;;;是", ";".runeAt(0))) == + @["", "", "hÃllo", "this", "is", "an", "", "example", "", "", "是"] + + splitCommon(toOa(s), sep, maxsplit) + +proc split*(s: string, seps: openArray[Rune] = unicodeSpaces, maxsplit: int = -1): + seq[string] {.noSideEffect, inline.} = + ## The same as the `split iterator <#split.i,string,openArray[Rune],int>`_, + ## but is a proc that returns a sequence of substrings. + accResult(split(toOa(s), seps, maxsplit)) + +proc split*(s: string, sep: Rune, maxsplit: int = -1): seq[string] {.noSideEffect, inline.} = + ## The same as the `split iterator <#split.i,string,Rune,int>`_, but is a proc + ## that returns a sequence of substrings. + accResult(split(toOa(s), sep, maxsplit)) + +proc strip*(s: string, leading = true, trailing = true, + runes: openArray[Rune] = unicodeSpaces): string {.noSideEffect, inline.} = + ## Strips leading or trailing ``runes`` from ``s`` and returns + ## the resulting string. + ## + ## If ``leading`` is true (default), leading ``runes`` are stripped. + ## If ``trailing`` is true (default), trailing ``runes`` are stripped. + ## If both are false, the string is returned unchanged. + runnableExamples: + let a = "\táñyóng " + doAssert a.strip == "áñyóng" + doAssert a.strip(leading = false) == "\táñyóng" + doAssert a.strip(trailing = false) == "áñyóng " + strip(toOa(s), leading, trailing, runes) + + +proc align*(s: string, count: Natural, padding = ' '.Rune): string {.noSideEffect, inline.} = + ## Aligns a unicode string ``s`` with ``padding``, so that it has a rune-length + ## of ``count``. + ## + ## ``padding`` characters (by default spaces) are added before ``s`` resulting in + ## right alignment. If ``s.runelen >= count``, no spaces are added and ``s`` is + ## returned unchanged. If you need to left align a string use the `alignLeft + ## proc <#alignLeft,string,Natural>`_. + runnableExamples: + assert align("abc", 4) == " abc" + assert align("a", 0) == "a" + assert align("1232", 6) == " 1232" + assert align("1232", 6, '#'.Rune) == "##1232" + assert align("Åge", 5) == " Åge" + assert align("×", 4, '_'.Rune) == "___×" + align(toOa(s), count, padding) + +proc alignLeft*(s: string, count: Natural, padding = ' '.Rune): string {.noSideEffect, inline.} = + ## Left-aligns a unicode string ``s`` with ``padding``, so that it has a + ## rune-length of ``count``. + ## + ## ``padding`` characters (by default spaces) are added after ``s`` resulting in + ## left alignment. If ``s.runelen >= count``, no spaces are added and ``s`` is + ## returned unchanged. If you need to right align a string use the `align + ## proc <#align,string,Natural>`_. + runnableExamples: + assert alignLeft("abc", 4) == "abc " + assert alignLeft("a", 0) == "a" + assert alignLeft("1232", 6) == "1232 " + assert alignLeft("1232", 6, '#'.Rune) == "1232##" + assert alignLeft("Åge", 5) == "Åge " + assert alignLeft("×", 4, '_'.Rune) == "×___" + alignLeft(toOa(s), count, padding) diff --git a/lib/pure/unittest.nim b/lib/pure/unittest.nim index 593bc1ca4..cfb762258 100644 --- a/lib/pure/unittest.nim +++ b/lib/pure/unittest.nim @@ -34,9 +34,9 @@ ## ## Specify the test name as a command line argument. ## -## .. code:: -## +## ```cmd ## nim c -r test "my test name" "another test" +## ``` ## ## Multiple arguments can be used. ## @@ -45,9 +45,9 @@ ## ## Specify the suite name delimited by `"::"`. ## -## .. code:: -## +## ```cmd ## nim c -r test "my test name::" +## ``` ## ## Selecting tests by pattern ## ========================== @@ -58,19 +58,18 @@ ## ## Tests matching **any** of the arguments are executed. ## -## .. code:: -## +## ```cmd ## nim c -r test fast_suite::mytest1 fast_suite::mytest2 ## nim c -r test "fast_suite::mytest*" ## nim c -r test "auth*::" "crypto::hashing*" ## # Run suites starting with 'bug #' and standalone tests starting with '#' ## nim c -r test 'bug #*::' '::#*' +## ``` ## ## Examples ## ======== ## -## .. code:: nim -## +## ```nim ## suite "description for this stuff": ## echo "suite setup: run once before the tests" ## @@ -96,6 +95,7 @@ ## discard v[4] ## ## echo "suite teardown: run once after the tests" +## ``` ## ## Limitations/Bugs ## ================ @@ -111,15 +111,15 @@ import std/exitprocs when defined(nimPreviewSlimSystem): import std/assertions -import macros, strutils, streams, times, sets, sequtils +import std/[macros, strutils, streams, times, sets, sequtils] when declared(stdout): - import os + import std/os const useTerminal = not defined(js) when useTerminal: - import terminal + import std/terminal type TestStatus* = enum ## The status of a test when it is done. @@ -235,7 +235,7 @@ proc colorOutput(): bool = else: result = false of "on": result = true of "off": result = false - else: doAssert false, $color + else: raiseAssert $color when declared(stdout): if existsEnv("NIMTEST_COLOR"): @@ -433,8 +433,6 @@ proc matchFilter(suiteName, testName, filter: string): bool = return glob(suiteName, suiteAndTestFilters[0]) and glob(testName, suiteAndTestFilters[1]) -when defined(testing): export matchFilter - proc shouldRun(currentSuiteName, testName: string): bool = ## Check if a test should be run by matching suiteName and testName against ## test filters. @@ -475,27 +473,26 @@ template suite*(name, body) {.dirty.} = ## common fixture (``setup``, ``teardown``). The fixture is executed ## for EACH test. ## - ## .. code-block:: nim - ## suite "test suite for addition": - ## setup: - ## let result = 4 + ## ```nim + ## suite "test suite for addition": + ## setup: + ## let result = 4 ## - ## test "2 + 2 = 4": - ## check(2+2 == result) + ## test "2 + 2 = 4": + ## check(2+2 == result) ## - ## test "(2 + -2) != 4": - ## check(2 + -2 != result) + ## test "(2 + -2) != 4": + ## check(2 + -2 != result) ## - ## # No teardown needed + ## # No teardown needed + ## ``` ## ## The suite will run the individual test cases in the order in which ## they were listed. With default global settings the above code prints: ## - ## .. code-block:: - ## - ## [Suite] test suite for addition - ## [OK] 2 + 2 = 4 - ## [OK] (2 + -2) != 4 + ## [Suite] test suite for addition + ## [OK] 2 + 2 = 4 + ## [OK] (2 + -2) != 4 bind formatters, ensureInitialized, suiteEnded block: @@ -521,20 +518,24 @@ proc exceptionTypeName(e: ref Exception): string {.inline.} = if e == nil: "<foreign exception>" else: $e.name +when not declared(setProgramResult): + {.warning: "setProgramResult not available on platform, unittest will not" & + " give failing exit code on test failure".} + template setProgramResult(a: int) = + discard + template test*(name, body) {.dirty.} = ## Define a single test case identified by `name`. ## - ## .. code-block:: nim - ## - ## test "roses are red": - ## let roses = "red" - ## check(roses == "red") + ## ```nim + ## test "roses are red": + ## let roses = "red" + ## check(roses == "red") + ## ``` ## ## The above code outputs: ## - ## .. code-block:: - ## - ## [OK] roses are red + ## [OK] roses are red bind shouldRun, checkpoints, formatters, ensureInitialized, testEnded, exceptionTypeName, setProgramResult ensureInitialized() @@ -546,11 +547,14 @@ template test*(name, body) {.dirty.} = for formatter in formatters: formatter.testStarted(name) + {.push warning[BareExcept]:off.} try: when declared(testSetupIMPLFlag): testSetupIMPL() when declared(testTeardownIMPLFlag): defer: testTeardownIMPL() + {.push warning[BareExcept]:on.} body + {.pop.} except: let e = getCurrentException() @@ -572,16 +576,17 @@ template test*(name, body) {.dirty.} = ) testEnded(testResult) checkpoints = @[] + {.pop.} proc checkpoint*(msg: string) = ## Set a checkpoint identified by `msg`. Upon test failure all ## checkpoints encountered so far are printed out. Example: ## - ## .. code-block:: nim - ## - ## checkpoint("Checkpoint A") - ## check((42, "the Answer to life and everything") == (1, "a")) - ## checkpoint("Checkpoint B") + ## ```nim + ## checkpoint("Checkpoint A") + ## check((42, "the Answer to life and everything") == (1, "a")) + ## checkpoint("Checkpoint B") + ## ``` ## ## outputs "Checkpoint A" once it fails. checkpoints.add(msg) @@ -593,11 +598,11 @@ template fail* = ## failed (change exit code and test status). This template is useful ## for debugging, but is otherwise mostly used internally. Example: ## - ## .. code-block:: nim - ## - ## checkpoint("Checkpoint A") - ## complicatedProcInThread() - ## fail() + ## ```nim + ## checkpoint("Checkpoint A") + ## complicatedProcInThread() + ## fail() + ## ``` ## ## outputs "Checkpoint A" before quitting. bind ensureInitialized, setProgramResult @@ -625,11 +630,10 @@ template skip* = ## for reasons depending on outer environment, ## or certain application logic conditions or configurations. ## The test code is still executed. - ## - ## .. code-block:: nim - ## - ## if not isGLContextCreated(): - ## skip() + ## ```nim + ## if not isGLContextCreated(): + ## skip() + ## ``` bind checkpoints testStatusIMPL = TestStatus.SKIPPED @@ -759,15 +763,20 @@ macro expect*(exceptions: varargs[typed], body: untyped): untyped = defectiveRobot() template expectBody(errorTypes, lineInfoLit, body): NimNode {.dirty.} = + {.push warning[BareExcept]:off.} try: + {.push warning[BareExcept]:on.} body + {.pop.} checkpoint(lineInfoLit & ": Expect Failed, no exception was thrown.") fail() except errorTypes: discard except: - checkpoint(lineInfoLit & ": Expect Failed, unexpected exception was thrown.") + let err = getCurrentException() + checkpoint(lineInfoLit & ": Expect Failed, " & $err.name & " was thrown.") fail() + {.pop.} var errorTypes = newNimNode(nnkBracket) for exp in exceptions: diff --git a/lib/pure/uri.nim b/lib/pure/uri.nim index 50b1b9445..725d5bbd9 100644 --- a/lib/pure/uri.nim +++ b/lib/pure/uri.nim @@ -14,6 +14,8 @@ ## as a locator, a name, or both. The term "Uniform Resource Locator" ## (URL) refers to the subset of URIs. ## +## .. warning:: URI parsers in this module do not perform security validation. +## ## # Basic usage @@ -32,11 +34,11 @@ runnableExamples: ## ## Data URI Base64 runnableExamples: - doAssert getDataUri("Hello World", "text/plain") == "data:text/plain;charset=utf-8;base64,SGVsbG8gV29ybGQ=" - doAssert getDataUri("Nim", "text/plain") == "data:text/plain;charset=utf-8;base64,Tmlt" + assert getDataUri("Hello World", "text/plain") == "data:text/plain;charset=utf-8;base64,SGVsbG8gV29ybGQ=" + assert getDataUri("Nim", "text/plain") == "data:text/plain;charset=utf-8;base64,Tmlt" -import strutils, parseutils, base64 +import std/[strutils, parseutils, base64] import std/private/[since, decode_helpers] when defined(nimPreviewSlimSystem): @@ -50,7 +52,7 @@ type scheme*, username*, password*: string hostname*, port*, path*, query*, anchor*: string opaque*: bool - isIpv6: bool # not expose it for compatibility. + isIpv6*: bool UriParseError* = object of ValueError @@ -494,42 +496,61 @@ func `$`*(u: Uri): string = ## Returns the string representation of the specified URI object. runnableExamples: assert $parseUri("https://nim-lang.org") == "https://nim-lang.org" - result = "" - if u.scheme.len > 0: - result.add(u.scheme) - if u.opaque: - result.add(":") - else: - result.add("://") - if u.username.len > 0: - result.add(u.username) - if u.password.len > 0: - result.add(":") - result.add(u.password) - result.add("@") + # Get the len of all the parts. + let schemeLen = u.scheme.len + let usernameLen = u.username.len + let passwordLen = u.password.len + let hostnameLen = u.hostname.len + let portLen = u.port.len + let pathLen = u.path.len + let queryLen = u.query.len + let anchorLen = u.anchor.len + # Prepare a string that fits all the parts and all punctuation chars. + # 12 is the max len required by all possible punctuation chars. + result = newStringOfCap( + schemeLen + usernameLen + passwordLen + hostnameLen + portLen + pathLen + queryLen + anchorLen + 12 + ) + # Insert to result. + if schemeLen > 0: + result.add u.scheme + result.add ':' + if not u.opaque: + result.add '/' + result.add '/' + if usernameLen > 0: + result.add u.username + if passwordLen > 0: + result.add ':' + result.add u.password + result.add '@' if u.hostname.endsWith('/'): if u.isIpv6: - result.add("[" & u.hostname[0 .. ^2] & "]") + result.add '[' + result.add u.hostname[0 .. ^2] + result.add ']' else: - result.add(u.hostname[0 .. ^2]) + result.add u.hostname[0 .. ^2] else: if u.isIpv6: - result.add("[" & u.hostname & "]") + result.add '[' + result.add u.hostname + result.add ']' else: - result.add(u.hostname) - if u.port.len > 0: - result.add(":") - result.add(u.port) - if u.path.len > 0: - if u.hostname.len > 0 and u.path[0] != '/': - result.add('/') - result.add(u.path) - if u.query.len > 0: - result.add("?") - result.add(u.query) - if u.anchor.len > 0: - result.add("#") - result.add(u.anchor) + result.add u.hostname + if portLen > 0: + result.add ':' + result.add u.port + if pathLen > 0: + if hostnameLen > 0 and u.path[0] != '/': + result.add '/' + result.add u.path + if queryLen > 0: + result.add '?' + result.add u.query + if anchorLen > 0: + result.add '#' + result.add u.anchor + proc getDataUri*(data, mime: string, encoding = "utf-8"): string {.since: (1, 3).} = ## Convenience proc for `base64.encode` returns a standard Base64 Data URI (RFC-2397) @@ -540,4 +561,12 @@ proc getDataUri*(data, mime: string, encoding = "utf-8"): string {.since: (1, 3) ## * https://en.wikipedia.org/wiki/Data_URI_scheme runnableExamples: static: assert getDataUri("Nim", "text/plain") == "data:text/plain;charset=utf-8;base64,Tmlt" assert encoding.len > 0 and mime.len > 0 # Must *not* be URL-Safe, see RFC-2397 - result = "data:" & mime & ";charset=" & encoding & ";base64," & base64.encode(data) + let base64encoded: string = base64.encode(data) + # ("data:".len + ";charset=".len + ";base64,".len) == 22 + result = newStringOfCap(22 + mime.len + encoding.len + base64encoded.len) + result.add "data:" + result.add mime + result.add ";charset=" + result.add encoding + result.add ";base64," + result.add base64encoded diff --git a/lib/pure/volatile.nim b/lib/pure/volatile.nim index f30d899df..a38247c7d 100644 --- a/lib/pure/volatile.nim +++ b/lib/pure/volatile.nim @@ -10,20 +10,18 @@ ## This module contains code for generating volatile loads and stores, ## which are useful in embedded and systems programming. -template volatileLoad*[T](src: ptr T): T = +proc volatileLoad*[T](src: ptr T): T {.inline, noinit.} = ## Generates a volatile load of the value stored in the container `src`. ## Note that this only effects code generation on `C` like backends. when nimvm: - src[] + result = src[] else: when defined(js): - src[] + result = src[] else: - var res: T - {.emit: [res, " = (*(", typeof(src[]), " volatile*)", src, ");"].} - res + {.emit: [result, " = (*(", typeof(src[]), " volatile*)", src, ");"].} -template volatileStore*[T](dest: ptr T, val: T) = +proc volatileStore*[T](dest: ptr T, val: T) {.inline.} = ## Generates a volatile store into the container `dest` of the value ## `val`. Note that this only effects code generation on `C` like ## backends. diff --git a/lib/pure/xmlparser.nim b/lib/pure/xmlparser.nim index 6785fa66e..2c1e4e37c 100644 --- a/lib/pure/xmlparser.nim +++ b/lib/pure/xmlparser.nim @@ -9,7 +9,7 @@ ## This module parses an XML document and creates its XML tree representation. -import streams, parsexml, strtabs, xmltree +import std/[streams, parsexml, strtabs, xmltree] when defined(nimPreviewSlimSystem): import std/syncio @@ -151,7 +151,7 @@ proc loadXml*(path: string, options: set[XmlParseOption] = {reportComments}): Xm when isMainModule: when not defined(testing): - import os + import std/os var errors: seq[string] = @[] var x = loadXml(paramStr(1), errors) diff --git a/lib/pure/xmltree.nim b/lib/pure/xmltree.nim index 72645ef96..5c0cbc5e4 100644 --- a/lib/pure/xmltree.nim +++ b/lib/pure/xmltree.nim @@ -31,7 +31,7 @@ runnableExamples: ## * `htmlgen module <htmlgen.html>`_ for html code generator import std/private/since -import macros, strtabs, strutils +import std/[macros, strtabs, strutils, sequtils] when defined(nimPreviewSlimSystem): import std/assertions @@ -70,6 +70,14 @@ const xmlHeader* = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n" ## Header to use for complete XML output. +template expect(node: XmlNode, kind: set[XmlNodeKind]) = + ## Check the node's kind is within a set of values + assert node.k in kind, "Got " & $node.k + +template expect(node: XmlNode, kind: XmlNodeKind) = + ## Check the node's kind equals a value + assert node.k == kind, "Got " & $node.k + proc newXmlNode(kind: XmlNodeKind): XmlNode = ## Creates a new ``XmlNode``. result = XmlNode(k: kind) @@ -182,7 +190,7 @@ proc text*(n: XmlNode): lent string {.inline.} = assert $c == "<!-- my comment -->" assert c.text == "my comment" - assert n.k in {xnText, xnComment, xnCData, xnEntity} + n.expect {xnText, xnVerbatimText, xnComment, xnCData, xnEntity} result = n.fText proc `text=`*(n: XmlNode, text: sink string) {.inline.} = @@ -200,7 +208,7 @@ proc `text=`*(n: XmlNode, text: sink string) {.inline.} = e.text = "a new entity text" assert $e == "&a new entity text;" - assert n.k in {xnText, xnComment, xnCData, xnEntity} + n.expect {xnText, xnVerbatimText, xnComment, xnCData, xnEntity} n.fText = text proc tag*(n: XmlNode): lent string {.inline.} = @@ -221,7 +229,7 @@ proc tag*(n: XmlNode): lent string {.inline.} = </firstTag>""" assert a.tag == "firstTag" - assert n.k == xnElement + n.expect xnElement result = n.fTag proc `tag=`*(n: XmlNode, tag: sink string) {.inline.} = @@ -244,7 +252,7 @@ proc `tag=`*(n: XmlNode, tag: sink string) {.inline.} = <childTag /> </newTag>""" - assert n.k == xnElement + n.expect xnElement n.fTag = tag proc rawText*(n: XmlNode): string {.inline.} = @@ -298,26 +306,60 @@ proc innerText*(n: XmlNode): string = proc add*(father, son: XmlNode) {.inline.} = ## Adds the child `son` to `father`. + ## `father` must be of `xnElement` type ## ## See also: + ## * `add proc <#add,XmlNode,openArray[XmlNode]>`_ ## * `insert proc <#insert,XmlNode,XmlNode,int>`_ + ## * `insert proc <#insert,XmlNode,openArray[XmlNode],int>`_ ## * `delete proc <#delete,XmlNode,Natural>`_ + ## * `delete proc <#delete.XmlNode,Slice[int]>`_ + ## * `replace proc <#replace.XmlNode,int,openArray[XmlNode]>`_ + ## * `replace proc <#replace.XmlNode,Slice[int],openArray[XmlNode]>`_ runnableExamples: var f = newElement("myTag") f.add newText("my text") f.add newElement("sonTag") f.add newEntity("my entity") assert $f == "<myTag>my text<sonTag />&my entity;</myTag>" + + father.expect xnElement add(father.s, son) +proc add*(father: XmlNode, sons: openArray[XmlNode]) {.inline.} = + ## Adds the children `sons` to `father`. + ## `father` must be of `xnElement` type + ## + ## See also: + ## * `add proc <#add,XmlNode,XmlNode>`_ + ## * `insert proc <#insert,XmlNode,XmlNode,int>`_ + ## * `insert proc <#insert,XmlNode,openArray[XmlNode],int>`_ + ## * `delete proc <#delete,XmlNode,Natural>`_ + ## * `delete proc <#delete.XmlNode,Slice[int]>`_ + ## * `replace proc <#replace.XmlNode,int,openArray[XmlNode]>`_ + ## * `replace proc <#replace.XmlNode,Slice[int],openArray[XmlNode]>`_ + runnableExamples: + var f = newElement("myTag") + f.add(@[newText("my text"), newElement("sonTag"), newEntity("my entity")]) + assert $f == "<myTag>my text<sonTag />&my entity;</myTag>" + + father.expect xnElement + add(father.s, sons) + + proc insert*(father, son: XmlNode, index: int) {.inline.} = ## Inserts the child `son` to a given position in `father`. ## - ## `father` and `son` must be of `xnElement` kind. + ## `father` must be of `xnElement` kind. ## ## See also: + ## * `insert proc <#insert,XmlNode,openArray[XmlNode],int>`_ ## * `add proc <#add,XmlNode,XmlNode>`_ + ## * `add proc <#add,XmlNode,openArray[XmlNode]>`_ ## * `delete proc <#delete,XmlNode,Natural>`_ + ## * `delete proc <#delete.XmlNode,Slice[int]>`_ + ## * `replace proc <#replace.XmlNode,int,openArray[XmlNode]>`_ + ## * `replace proc <#replace.XmlNode,Slice[int],openArray[XmlNode]>`_ runnableExamples: var f = newElement("myTag") f.add newElement("first") @@ -327,18 +369,52 @@ proc insert*(father, son: XmlNode, index: int) {.inline.} = <first /> </myTag>""" - assert father.k == xnElement and son.k == xnElement + father.expect xnElement if len(father.s) > index: insert(father.s, son, index) else: insert(father.s, son, len(father.s)) +proc insert*(father: XmlNode, sons: openArray[XmlNode], index: int) {.inline.} = + ## Inserts the children openArray[`sons`] to a given position in `father`. + ## + ## `father` must be of `xnElement` kind. + ## + ## See also: + ## * `insert proc <#insert,XmlNode,XmlNode,int>`_ + ## * `add proc <#add,XmlNode,XmlNode>`_ + ## * `add proc <#add,XmlNode,openArray[XmlNode]>`_ + ## * `delete proc <#delete,XmlNode,Natural>`_ + ## * `delete proc <#delete.XmlNode,Slice[int]>`_ + ## * `replace proc <#replace.XmlNode,int,openArray[XmlNode]>`_ + ## * `replace proc <#replace.XmlNode,Slice[int],openArray[XmlNode]>`_ + runnableExamples: + var f = newElement("myTag") + f.add newElement("first") + f.insert([newElement("second"), newElement("third")], 0) + assert $f == """<myTag> + <second /> + <third /> + <first /> +</myTag>""" + + father.expect xnElement + if len(father.s) > index: + insert(father.s, sons, index) + else: + insert(father.s, sons, len(father.s)) + proc delete*(n: XmlNode, i: Natural) = ## Deletes the `i`'th child of `n`. ## ## See also: + ## * `delete proc <#delete.XmlNode,Slice[int]>`_ ## * `add proc <#add,XmlNode,XmlNode>`_ + ## * `add proc <#add,XmlNode,openArray[XmlNode]>`_ ## * `insert proc <#insert,XmlNode,XmlNode,int>`_ + ## * `insert proc <#insert,XmlNode,openArray[XmlNode],int>`_ + ## * `replace proc <#replace.XmlNode,int,openArray[XmlNode]>`_ + ## * `replace proc <#replace.XmlNode,Slice[int],openArray[XmlNode]>`_ runnableExamples: var f = newElement("myTag") f.add newElement("first") @@ -348,9 +424,88 @@ proc delete*(n: XmlNode, i: Natural) = <first /> </myTag>""" - assert n.k == xnElement + n.expect xnElement n.s.delete(i) +proc delete*(n: XmlNode, slice: Slice[int]) = + ## Deletes the items `n[slice]` of `n`. + ## + ## See also: + ## * `delete proc <#delete.XmlNode,int>`_ + ## * `add proc <#add,XmlNode,XmlNode>`_ + ## * `add proc <#add,XmlNode,openArray[XmlNode]>`_ + ## * `insert proc <#insert,XmlNode,XmlNode,int>`_ + ## * `insert proc <#insert,XmlNode,openArray[XmlNode],int>`_ + ## * `replace proc <#replace.XmlNode,int,openArray[XmlNode]>`_ + ## * `replace proc <#replace.XmlNode,Slice[int],openArray[XmlNode]>`_ + runnableExamples: + var f = newElement("myTag") + f.add newElement("first") + f.insert([newElement("second"), newElement("third")], 0) + f.delete(0..1) + assert $f == """<myTag> + <first /> +</myTag>""" + + n.expect xnElement + n.s.delete(slice) + +proc replace*(n: XmlNode, i: Natural, replacement: openArray[XmlNode]) = + ## Replaces the `i`'th child of `n` with `replacement` openArray. + ## + ## `n` must be of `xnElement` kind. + ## + ## See also: + ## * `replace proc <#replace.XmlNode,Slice[int],openArray[XmlNode]>`_ + ## * `add proc <#add,XmlNode,XmlNode>`_ + ## * `add proc <#add,XmlNode,openArray[XmlNode]>`_ + ## * `delete proc <#delete,XmlNode,Natural>`_ + ## * `delete proc <#delete.XmlNode,Slice[int]>`_ + ## * `insert proc <#insert,XmlNode,XmlNode,int>`_ + ## * `insert proc <#insert,XmlNode,openArray[XmlNode],int>`_ + runnableExamples: + var f = newElement("myTag") + f.add newElement("first") + f.insert(newElement("second"), 0) + f.replace(0, @[newElement("third"), newElement("fourth")]) + assert $f == """<myTag> + <third /> + <fourth /> + <first /> +</myTag>""" + + n.expect xnElement + n.s.delete(i) + n.s.insert(replacement, i) + +proc replace*(n: XmlNode, slice: Slice[int], replacement: openArray[XmlNode]) = + ## Deletes the items `n[slice]` of `n`. + ## + ## `n` must be of `xnElement` kind. + ## + ## See also: + ## * `replace proc <#replace.XmlNode,int,openArray[XmlNode]>`_ + ## * `add proc <#add,XmlNode,XmlNode>`_ + ## * `add proc <#add,XmlNode,openArray[XmlNode]>`_ + ## * `delete proc <#delete,XmlNode,Natural>`_ + ## * `delete proc <#delete.XmlNode,Slice[int]>`_ + ## * `insert proc <#insert,XmlNode,XmlNode,int>`_ + ## * `insert proc <#insert,XmlNode,openArray[XmlNode],int>`_ + runnableExamples: + var f = newElement("myTag") + f.add newElement("first") + f.insert([newElement("second"), newElement("fifth")], 0) + f.replace(0..1, @[newElement("third"), newElement("fourth")]) + assert $f == """<myTag> + <third /> + <fourth /> + <first /> +</myTag>""" + + n.expect xnElement + n.s.delete(slice) + n.s.insert(replacement, slice.a) + proc len*(n: XmlNode): int {.inline.} = ## Returns the number of `n`'s children. runnableExamples: @@ -378,12 +533,12 @@ proc `[]`*(n: XmlNode, i: int): XmlNode {.inline.} = assert $f[1] == "<first />" assert $f[0] == "<second />" - assert n.k == xnElement + n.expect xnElement result = n.s[i] proc `[]`*(n: var XmlNode, i: int): var XmlNode {.inline.} = ## Returns the `i`'th child of `n` so that it can be modified. - assert n.k == xnElement + n.expect xnElement result = n.s[i] proc clear*(n: var XmlNode) = @@ -435,12 +590,12 @@ iterator items*(n: XmlNode): XmlNode {.inline.} = # <!-- this is comment --> # <secondTag>&some entity;<![CDATA[some cdata]]></secondTag> - assert n.k == xnElement + n.expect xnElement for i in 0 .. n.len-1: yield n[i] iterator mitems*(n: var XmlNode): var XmlNode {.inline.} = ## Iterates over all direct children of `n` so that they can be modified. - assert n.k == xnElement + n.expect xnElement for i in 0 .. n.len-1: yield n[i] proc toXmlAttributes*(keyValuePairs: varargs[tuple[key, @@ -472,7 +627,7 @@ proc attrs*(n: XmlNode): XmlAttributes {.inline.} = j.attrs = att assert j.attrs == att - assert n.k == xnElement + n.expect xnElement result = n.fAttr proc `attrs=`*(n: XmlNode, attr: XmlAttributes) {.inline.} = @@ -489,7 +644,7 @@ proc `attrs=`*(n: XmlNode, attr: XmlAttributes) {.inline.} = j.attrs = att assert j.attrs == att - assert n.k == xnElement + n.expect xnElement n.fAttr = attr proc attrsLen*(n: XmlNode): int {.inline.} = @@ -506,7 +661,7 @@ proc attrsLen*(n: XmlNode): int {.inline.} = j.attrs = att assert j.attrsLen == 2 - assert n.k == xnElement + n.expect xnElement if not isNil(n.fAttr): result = len(n.fAttr) proc attr*(n: XmlNode, name: string): string = @@ -524,7 +679,7 @@ proc attr*(n: XmlNode, name: string): string = assert j.attr("key1") == "first value" assert j.attr("key2") == "second value" - assert n.kind == xnElement + n.expect xnElement if n.attrs == nil: return "" return n.attrs.getOrDefault(name) @@ -576,23 +731,11 @@ proc addIndent(result: var string, indent: int, addNewLines: bool) = for i in 1 .. indent: result.add(' ') -proc add*(result: var string, n: XmlNode, indent = 0, indWidth = 2, - addNewLines = true) = - ## Adds the textual representation of `n` to string `result`. - runnableExamples: - var - a = newElement("firstTag") - b = newText("my text") - c = newComment("my comment") - s = "" - s.add(c) - s.add(a) - s.add(b) - assert s == "<!-- my comment --><firstTag />my text" - +proc addImpl(result: var string, n: XmlNode, indent = 0, indWidth = 2, + addNewLines = true, lastNodeIsText = false) = proc noWhitespace(n: XmlNode): bool = for i in 0 ..< n.len: - if n[i].kind in {xnText, xnEntity}: return true + if n[i].kind in {xnText, xnVerbatimText, xnEntity}: return true proc addEscapedAttr(result: var string, s: string) = # `addEscaped` alternative with less escaped characters. @@ -609,7 +752,7 @@ proc add*(result: var string, n: XmlNode, indent = 0, indWidth = 2, case n.k of xnElement: - if indent > 0: + if indent > 0 and not lastNodeIsText: result.addIndent(indent, addNewLines) let @@ -638,8 +781,10 @@ proc add*(result: var string, n: XmlNode, indent = 0, indWidth = 2, else: indent+indWidth result.add('>') + var lastNodeIsText = false for i in 0 ..< n.len: - result.add(n[i], indentNext, indWidth, addNewLines) + result.addImpl(n[i], indentNext, indWidth, addNewLines, lastNodeIsText) + lastNodeIsText = (n[i].kind == xnText) or (n[i].kind == xnVerbatimText) if not n.noWhitespace(): result.addIndent(indent, addNewLines) @@ -664,6 +809,21 @@ proc add*(result: var string, n: XmlNode, indent = 0, indWidth = 2, result.add(n.fText) result.add(';') +proc add*(result: var string, n: XmlNode, indent = 0, indWidth = 2, + addNewLines = true) {.inline.} = + ## Adds the textual representation of `n` to string `result`. + runnableExamples: + var + a = newElement("firstTag") + b = newText("my text") + c = newComment("my comment") + s = "" + s.add(c) + s.add(a) + s.add(b) + assert s == "<!-- my comment --><firstTag />my text" + result.addImpl(n, indent, indWidth, addNewLines) + proc `$`*(n: XmlNode): string = ## Converts `n` into its string representation. ## @@ -682,7 +842,7 @@ proc child*(n: XmlNode, name: string): XmlNode = f.add newElement("thirdSon") assert $(f.child("secondSon")) == "<secondSon />" - assert n.kind == xnElement + n.expect xnElement for i in items(n): if i.kind == xnElement: if i.tag == name: @@ -718,7 +878,7 @@ proc findAll*(n: XmlNode, tag: string, result: var seq[XmlNode], a.findAll("BAD", s, caseInsensitive = true) assert $s == "@[<bad>c text</bad>, <BAD>d text</BAD>]" - assert n.k == xnElement + n.expect xnElement for child in n.items(): if child.k != xnElement: continue @@ -775,11 +935,12 @@ proc xmlConstructor(a: NimNode): NimNode = macro `<>`*(x: untyped): untyped = ## Constructor macro for XML. Example usage: ## - ## .. code-block:: nim + ## ```nim ## <>a(href="http://nim-lang.org", newText("Nim rules.")) + ## ``` ## - ## Produces an XML tree for:: + ## Produces an XML tree for: ## - ## <a href="http://nim-lang.org">Nim rules.</a> + ## <a href="http://nim-lang.org">Nim rules.</a> ## result = xmlConstructor(x) |