diff options
Diffstat (limited to 'lib/std')
72 files changed, 10122 insertions, 1097 deletions
diff --git a/lib/std/appdirs.nim b/lib/std/appdirs.nim new file mode 100644 index 000000000..963451efe --- /dev/null +++ b/lib/std/appdirs.nim @@ -0,0 +1,94 @@ +## This module implements helpers for determining special directories used by apps. + +## .. importdoc:: paths.nim + +from std/private/osappdirs import nil +import std/paths +import std/envvars + +proc getHomeDir*(): Path {.inline, tags: [ReadEnvEffect, ReadIOEffect].} = + ## Returns the home directory of the current user. + ## + ## This proc is wrapped by the `expandTilde proc`_ + ## for the convenience of processing paths coming from user configuration files. + ## + ## See also: + ## * `getConfigDir proc`_ + ## * `getTempDir proc`_ + result = Path(osappdirs.getHomeDir()) + +proc getDataDir*(): Path {.inline, tags: [ReadEnvEffect, ReadIOEffect].} = + ## Returns the data directory of the current user for applications. + ## + ## On non-Windows OSs, this proc conforms to the XDG Base Directory + ## spec. Thus, this proc returns the value of the `XDG_DATA_HOME` environment + ## variable if it is set, otherwise it returns the default configuration + ## directory ("~/.local/share" or "~/Library/Application Support" on macOS). + ## + ## See also: + ## * `getHomeDir proc`_ + ## * `getConfigDir proc`_ + ## * `getTempDir proc`_ + ## * `expandTilde proc`_ + ## * `getCurrentDir proc`_ + ## * `setCurrentDir proc`_ + result = Path(osappdirs.getDataDir()) + +proc getConfigDir*(): Path {.inline, tags: [ReadEnvEffect, ReadIOEffect].} = + ## Returns the config directory of the current user for applications. + ## + ## On non-Windows OSs, this proc conforms to the XDG Base Directory + ## spec. Thus, this proc returns the value of the `XDG_CONFIG_HOME` environment + ## variable if it is set, otherwise it returns the default configuration + ## directory ("~/.config/"). + ## + ## An OS-dependent trailing slash is always present at the end of the + ## returned string: `\\` on Windows and `/` on all other OSs. + ## + ## See also: + ## * `getHomeDir proc`_ + ## * `getTempDir proc`_ + result = Path(osappdirs.getConfigDir()) + +proc getCacheDir*(): Path {.inline.} = + ## Returns the cache directory of the current user for applications. + ## + ## This makes use of the following environment variables: + ## + ## * On Windows: `getEnv("LOCALAPPDATA")` + ## + ## * On macOS: `getEnv("XDG_CACHE_HOME", getEnv("HOME") / "Library/Caches")` + ## + ## * On other platforms: `getEnv("XDG_CACHE_HOME", getEnv("HOME") / ".cache")` + ## + ## **See also:** + ## * `getHomeDir proc`_ + ## * `getTempDir proc`_ + ## * `getConfigDir proc`_ + # follows https://crates.io/crates/platform-dirs + result = Path(osappdirs.getCacheDir()) + +proc getCacheDir*(app: Path): Path {.inline.} = + ## Returns the cache directory for an application `app`. + ## + ## * On Windows, this uses: `getCacheDir() / app / "cache"` + ## * On other platforms, this uses: `getCacheDir() / app` + result = Path(osappdirs.getCacheDir(app.string)) + +proc getTempDir*(): Path {.inline, tags: [ReadEnvEffect, ReadIOEffect].} = + ## Returns the temporary directory of the current user for applications to + ## save temporary files in. + ## + ## On Windows, it calls [GetTempPath](https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-gettemppathw). + ## On Posix based platforms, it will check `TMPDIR`, `TEMP`, `TMP` and `TEMPDIR` environment variables in order. + ## On all platforms, `/tmp` will be returned if the procs fails. + ## + ## You can override this implementation + ## by adding `-d:tempDir=mytempname` to your compiler invocation. + ## + ## .. Note:: This proc does not check whether the returned path exists. + ## + ## See also: + ## * `getHomeDir proc`_ + ## * `getConfigDir proc`_ + result = Path(osappdirs.getTempDir()) diff --git a/lib/std/assertions.nim b/lib/std/assertions.nim new file mode 100644 index 000000000..56c37d205 --- /dev/null +++ b/lib/std/assertions.nim @@ -0,0 +1,122 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2022 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +when not defined(nimPreviewSlimSystem) and not declared(sysFatal): + include "system/rawquits" + include "system/fatal" + +## This module implements assertion handling. + +import std/private/miscdollars +# --------------------------------------------------------------------------- +# helpers + +type InstantiationInfo = tuple[filename: string, line: int, column: int] + +proc `$`(info: InstantiationInfo): string = + # The +1 is needed here + # instead of overriding `$` (and changing its meaning), consider explicit name. + result = "" + result.toLocation(info.filename, info.line, info.column + 1) + +# --------------------------------------------------------------------------- + + +proc raiseAssert*(msg: string) {.noinline, noreturn, nosinks.} = + ## Raises an `AssertionDefect` with `msg`. + when defined(nimPreviewSlimSystem): + raise newException(AssertionDefect, msg) + else: + sysFatal(AssertionDefect, msg) + +proc failedAssertImpl*(msg: string) {.raises: [], tags: [].} = + ## Raises an `AssertionDefect` with `msg`, but this is hidden + ## from the effect system. Called when an assertion failed. + raiseAssert(msg) + +template assertImpl(cond: bool, msg: string, expr: string, enabled: static[bool]) = + when enabled: + const + loc = instantiationInfo(fullPaths = compileOption("excessiveStackTrace")) + ploc = $loc + bind instantiationInfo + mixin failedAssertImpl + {.line: loc.}: + if not cond: + failedAssertImpl(ploc & " `" & expr & "` " & msg) + +template assert*(cond: untyped, msg = "") = + ## Raises `AssertionDefect` with `msg` if `cond` is false. Note + ## that `AssertionDefect` is hidden from the effect system, so it doesn't + ## produce `{.raises: [AssertionDefect].}`. This exception is only supposed + ## to be caught by unit testing frameworks. + ## + ## No code will be generated for `assert` when passing `-d:danger` (implied by `--assertions:off`). + ## See `command line switches <nimc.html#compiler-usage-commandminusline-switches>`_. + runnableExamples: assert 1 == 1 + runnableExamples("--assertions:off"): + assert 1 == 2 # no code generated, no failure here + runnableExamples("-d:danger"): assert 1 == 2 # ditto + assertImpl(cond, msg, astToStr(cond), compileOption("assertions")) + +template doAssert*(cond: untyped, msg = "") = + ## Similar to `assert <#assert.t,untyped,string>`_ but is always turned on regardless of `--assertions`. + runnableExamples: + doAssert 1 == 1 # generates code even when built with `-d:danger` or `--assertions:off` + assertImpl(cond, msg, astToStr(cond), true) + +template onFailedAssert*(msg, code: untyped): untyped {.dirty.} = + ## Sets an assertion failure handler that will intercept any assert + ## statements following `onFailedAssert` in the current scope. + runnableExamples: + type MyError = object of CatchableError + lineinfo: tuple[filename: string, line: int, column: int] + # block-wide policy to change the failed assert exception type in order to + # include a lineinfo + onFailedAssert(msg): + raise (ref MyError)(msg: msg, lineinfo: instantiationInfo(-2)) + doAssertRaises(MyError): doAssert false + when not defined(nimHasTemplateRedefinitionPragma): + {.pragma: redefine.} + template failedAssertImpl(msgIMPL: string): untyped {.dirty, redefine.} = + let msg = msgIMPL + code + +template doAssertRaises*(exception: typedesc, code: untyped) = + ## Raises `AssertionDefect` if specified `code` does not raise `exception`. + runnableExamples: + doAssertRaises(ValueError): raise newException(ValueError, "Hello World") + doAssertRaises(CatchableError): raise newException(ValueError, "Hello World") + doAssertRaises(AssertionDefect): doAssert false + var wrong = false + const begin = "expected raising '" & astToStr(exception) & "', instead" + const msgEnd = " by: " & astToStr(code) + template raisedForeign {.gensym.} = raiseAssert(begin & " raised foreign exception" & msgEnd) + {.push warning[BareExcept]:off.} + when Exception is exception: + try: + if true: + code + wrong = true + except Exception as e: discard + except: raisedForeign() + else: + try: + if true: + code + wrong = true + except exception: + discard + except Exception as e: + mixin `$` # alternatively, we could define $cstring in this module + raiseAssert(begin & " raised '" & $e.name & "'" & msgEnd) + except: raisedForeign() + {.pop.} + if wrong: + raiseAssert(begin & " nothing was raised" & msgEnd) diff --git a/lib/std/channels.nim b/lib/std/channels.nim deleted file mode 100644 index 3bbc8a6b6..000000000 --- a/lib/std/channels.nim +++ /dev/null @@ -1,498 +0,0 @@ -# -# -# The Nim Compiler -# (c) Copyright 2021 Andreas Prell, Mamy AndrĂ©-Ratsimbazafy & Nim Contributors -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - - -# Based on https://github.com/mratsim/weave/blob/5696d94e6358711e840f8c0b7c684fcc5cbd4472/unused/channels/channels_legacy.nim -# Those are translations of @aprell (Andreas Prell) original channels from C to Nim -# (https://github.com/aprell/tasking-2.0/blob/master/src/channel_shm/channel.c) -# And in turn they are an implementation of Michael & Scott lock-based queues -# (note the paper has 2 channels: lock-free and lock-based) with additional caching: -# Simple, Fast, and Practical Non-Blocking and Blocking Concurrent Queue Algorithms -# Maged M. Michael, Michael L. Scott, 1996 -# https://www.cs.rochester.edu/~scott/papers/1996_PODC_queues.pdf - -## This module only works with `--gc:arc` or `--gc:orc`. -## -## .. warning:: This module is experimental and its interface may change. -## -## The following is a simple example of two different ways to use channels: -## blocking and non-blocking. -## - -runnableExamples("--threads:on --gc:orc"): - import std/os - - # In this example a channel is declared at module scope. - # Channels are generic, and they include support for passing objects between - # threads. - # Note that isolated data passed through channels is moved around. - var chan = newChannel[string]() - - # This proc will be run in another thread using the threads module. - proc firstWorker() = - chan.send("Hello World!") - - # This is another proc to run in a background thread. This proc takes a while - # to send the message since it sleeps for 2 seconds (or 2000 milliseconds). - proc secondWorker() = - sleep(2000) - chan.send("Another message") - - # Launch the worker. - var worker1: Thread[void] - createThread(worker1, firstWorker) - - # Block until the message arrives, then print it out. - let dest = chan.recv() - assert dest == "Hello World!" - - # Wait for the thread to exit before moving on to the next example. - worker1.joinThread() - - # Launch the other worker. - var worker2: Thread[void] - createThread(worker2, secondWorker) - # This time, use a non-blocking approach with tryRecv. - # Since the main thread is not blocked, it could be used to perform other - # useful work while it waits for data to arrive on the channel. - var messages: seq[string] - while true: - var msg = "" - if chan.tryRecv(msg): - messages.add msg # "Another message" - break - - messages.add "Pretend I'm doing useful work..." - # For this example, sleep in order not to flood stdout with the above - # message. - sleep(400) - - # Wait for the second thread to exit before cleaning up the channel. - worker2.joinThread() - - # Clean up the channel. - assert chan.close() - - assert messages[^1] == "Another message" - assert messages.len >= 2 - - -when not defined(gcArc) and not defined(gcOrc) and not defined(nimdoc): - {.error: "This channel implementation requires --gc:arc or --gc:orc".} - -import std/[locks, atomics, isolation] -import system/ansi_c - -# Channel (Shared memory channels) -# ---------------------------------------------------------------------------------- - -const - cacheLineSize {.intdefine.} = 64 # TODO: some Samsung phone have 128 cache-line - nimChannelCacheSize* {.intdefine.} = 100 - -type - ChannelRaw = ptr ChannelObj - ChannelObj = object - headLock, tailLock: Lock - notFullCond, notEmptyCond: Cond - closed: Atomic[bool] - size: int - itemsize: int # up to itemsize bytes can be exchanged over this channel - head {.align: cacheLineSize.} : int # Items are taken from head and new items are inserted at tail - tail: int - buffer: ptr UncheckedArray[byte] - atomicCounter: Atomic[int] - - ChannelCache = ptr ChannelCacheObj - ChannelCacheObj = object - next: ChannelCache - chanSize: int - chanN: int - numCached: int - cache: array[nimChannelCacheSize, ChannelRaw] - -# ---------------------------------------------------------------------------------- - -proc numItems(chan: ChannelRaw): int {.inline.} = - result = chan.tail - chan.head - if result < 0: - inc(result, 2 * chan.size) - - assert result <= chan.size - -template isFull(chan: ChannelRaw): bool = - abs(chan.tail - chan.head) == chan.size - -template isEmpty(chan: ChannelRaw): bool = - chan.head == chan.tail - -# Unbuffered / synchronous channels -# ---------------------------------------------------------------------------------- - -template numItemsUnbuf(chan: ChannelRaw): int = - chan.head - -template isFullUnbuf(chan: ChannelRaw): bool = - chan.head == 1 - -template isEmptyUnbuf(chan: ChannelRaw): bool = - chan.head == 0 - -# ChannelRaw kinds -# ---------------------------------------------------------------------------------- - -func isUnbuffered(chan: ChannelRaw): bool = - chan.size - 1 == 0 - -# ChannelRaw status and properties -# ---------------------------------------------------------------------------------- - -proc isClosed(chan: ChannelRaw): bool {.inline.} = load(chan.closed, moRelaxed) - -proc peek(chan: ChannelRaw): int {.inline.} = - (if chan.isUnbuffered: numItemsUnbuf(chan) else: numItems(chan)) - -# Per-thread channel cache -# ---------------------------------------------------------------------------------- - -var channelCache {.threadvar.}: ChannelCache -var channelCacheLen {.threadvar.}: int - -proc allocChannelCache(size, n: int): bool = - ## Allocate a free list for storing channels of a given type - var p = channelCache - - # Avoid multiple free lists for the exact same type of channel - while not p.isNil: - if size == p.chanSize and n == p.chanN: - return false - p = p.next - - p = cast[ptr ChannelCacheObj](c_malloc(csize_t sizeof(ChannelCacheObj))) - if p.isNil: - raise newException(OutOfMemDefect, "Could not allocate memory") - - p.chanSize = size - p.chanN = n - p.numCached = 0 - - p.next = channelCache - channelCache = p - inc channelCacheLen - result = true - -proc freeChannelCache*() = - ## Frees the entire channel cache, including all channels - var p = channelCache - var q: ChannelCache - - while not p.isNil: - q = p.next - for i in 0 ..< p.numCached: - let chan = p.cache[i] - if not chan.buffer.isNil: - c_free(chan.buffer) - deinitLock(chan.headLock) - deinitLock(chan.tailLock) - deinitCond(chan.notFullCond) - deinitCond(chan.notEmptyCond) - c_free(chan) - c_free(p) - dec channelCacheLen - p = q - - assert(channelCacheLen == 0) - channelCache = nil - -# Channels memory ops -# ---------------------------------------------------------------------------------- - -proc allocChannel(size, n: int): ChannelRaw = - when nimChannelCacheSize > 0: - var p = channelCache - - while not p.isNil: - if size == p.chanSize and n == p.chanN: - # Check if free list contains channel - if p.numCached > 0: - dec p.numCached - result = p.cache[p.numCached] - assert(result.isEmpty) - return - else: - # All the other lists in cache won't match - break - p = p.next - - result = cast[ChannelRaw](c_malloc(csize_t sizeof(ChannelObj))) - if result.isNil: - raise newException(OutOfMemDefect, "Could not allocate memory") - - # To buffer n items, we allocate for n - result.buffer = cast[ptr UncheckedArray[byte]](c_malloc(csize_t n*size)) - if result.buffer.isNil: - raise newException(OutOfMemDefect, "Could not allocate memory") - - initLock(result.headLock) - initLock(result.tailLock) - initCond(result.notFullCond) - initCond(result.notEmptyCond) - - result.closed.store(false, moRelaxed) # We don't need atomic here, how to? - result.size = n - result.itemsize = size - result.head = 0 - result.tail = 0 - result.atomicCounter.store(0, moRelaxed) - - when nimChannelCacheSize > 0: - # Allocate a cache as well if one of the proper size doesn't exist - discard allocChannelCache(size, n) - -proc freeChannel(chan: ChannelRaw) = - if chan.isNil: - return - - when nimChannelCacheSize > 0: - var p = channelCache - while not p.isNil: - if chan.itemsize == p.chanSize and - chan.size == p.chanN: - if p.numCached < nimChannelCacheSize: - # If space left in cache, cache it - p.cache[p.numCached] = chan - inc p.numCached - return - else: - # All the other lists in cache won't match - break - p = p.next - - if not chan.buffer.isNil: - c_free(chan.buffer) - - deinitLock(chan.headLock) - deinitLock(chan.tailLock) - deinitCond(chan.notFullCond) - deinitCond(chan.notEmptyCond) - - c_free(chan) - -# MPMC Channels (Multi-Producer Multi-Consumer) -# ---------------------------------------------------------------------------------- - -proc sendUnbufferedMpmc(chan: ChannelRaw, data: sink pointer, size: int, nonBlocking: bool): bool = - if nonBlocking and chan.isFullUnbuf: - return false - - acquire(chan.headLock) - - if nonBlocking and chan.isFullUnbuf: - # Another thread was faster - release(chan.headLock) - return false - - while chan.isFullUnbuf: - wait(chan.notFullcond, chan.headLock) - - assert chan.isEmptyUnbuf - assert size <= chan.itemsize - copyMem(chan.buffer, data, size) - - chan.head = 1 - - release(chan.headLock) - signal(chan.notEmptyCond) - result = true - -proc sendMpmc(chan: ChannelRaw, data: sink pointer, size: int, nonBlocking: bool): bool = - assert not chan.isNil - assert not data.isNil - - if isUnbuffered(chan): - return sendUnbufferedMpmc(chan, data, size, nonBlocking) - - if nonBlocking and chan.isFull: - return false - - acquire(chan.tailLock) - - if nonBlocking and chan.isFull: - # Another thread was faster - release(chan.tailLock) - return false - - while chan.isFull: - wait(chan.notFullcond, chan.tailLock) - - assert not chan.isFull - assert size <= chan.itemsize - - let writeIdx = if chan.tail < chan.size: chan.tail - else: chan.tail - chan.size - - copyMem(chan.buffer[writeIdx * chan.itemsize].addr, data, size) - - inc chan.tail - if chan.tail == 2 * chan.size: - chan.tail = 0 - - release(chan.tailLock) - signal(chan.notEmptyCond) - result = true - -proc recvUnbufferedMpmc(chan: ChannelRaw, data: pointer, size: int, nonBlocking: bool): bool = - if nonBlocking and chan.isEmptyUnbuf: - return false - - acquire(chan.headLock) - - if nonBlocking and chan.isEmptyUnbuf: - # Another thread was faster - release(chan.headLock) - return false - - while chan.isEmptyUnbuf: - wait(chan.notEmptyCond, chan.headLock) - - assert chan.isFullUnbuf - assert size <= chan.itemsize - - copyMem(data, chan.buffer, size) - - chan.head = 0 - - release(chan.headLock) - signal(chan.notFullCond) - result = true - -proc recvMpmc(chan: ChannelRaw, data: pointer, size: int, nonBlocking: bool): bool = - assert not chan.isNil - assert not data.isNil - - if isUnbuffered(chan): - return recvUnbufferedMpmc(chan, data, size, nonBlocking) - - if nonBlocking and chan.isEmpty: - return false - - acquire(chan.headLock) - - if nonBlocking and chan.isEmpty: - # Another thread took the last data - release(chan.headLock) - return false - - while chan.isEmpty: - wait(chan.notEmptyCond, chan.headLock) - - assert not chan.isEmpty - assert size <= chan.itemsize - - let readIdx = if chan.head < chan.size: chan.head - else: chan.head - chan.size - - copyMem(data, chan.buffer[readIdx * chan.itemsize].addr, size) - - inc chan.head - if chan.head == 2 * chan.size: - chan.head = 0 - - release(chan.headLock) - signal(chan.notFullCond) - result = true - -proc channelCloseMpmc(chan: ChannelRaw): bool = - # Unsynchronized - - if chan.isClosed: - # ChannelRaw already closed - return false - - store(chan.closed, true, moRelaxed) - result = true - -proc channelOpenMpmc(chan: ChannelRaw): bool = - # Unsynchronized - - if not chan.isClosed: - # ChannelRaw already open - return false - - store(chan.closed, false, moRelaxed) - result = true - -# Public API -# ---------------------------------------------------------------------------------- - -type - Channel*[T] = object ## Typed channels - d: ChannelRaw - -proc `=destroy`*[T](c: var Channel[T]) = - if c.d != nil: - if load(c.d.atomicCounter, moAcquire) == 0: - if c.d.buffer != nil: - freeChannel(c.d) - else: - atomicDec(c.d.atomicCounter) - -proc `=`*[T](dest: var Channel[T], src: Channel[T]) = - ## Shares `Channel` by reference counting. - if src.d != nil: - atomicInc(src.d.atomicCounter) - - if dest.d != nil: - `=destroy`(dest) - dest.d = src.d - -func trySend*[T](c: Channel[T], src: var Isolated[T]): bool {.inline.} = - ## Sends item to the channel(non blocking). - var data = src.extract - result = sendMpmc(c.d, data.addr, sizeof(T), true) - if result: - wasMoved(data) - -template trySend*[T](c: Channel[T], src: T): bool = - ## Helper templates for `trySend`. - trySend(c, isolate(src)) - -func tryRecv*[T](c: Channel[T], dst: var T): bool {.inline.} = - ## Receives item from the channel(non blocking). - recvMpmc(c.d, dst.addr, sizeof(T), true) - -func send*[T](c: Channel[T], src: sink Isolated[T]) {.inline.} = - ## Sends item to the channel(blocking). - var data = src.extract - discard sendMpmc(c.d, data.addr, sizeof(T), false) - wasMoved(data) - -template send*[T](c: Channel[T]; src: T) = - ## Helper templates for `send`. - send(c, isolate(src)) - -func recv*[T](c: Channel[T]): T {.inline.} = - ## Receives item from the channel(blocking). - discard recvMpmc(c.d, result.addr, sizeof(result), false) - -func open*[T](c: Channel[T]): bool {.inline.} = - result = c.d.channelOpenMpmc() - -func close*[T](c: Channel[T]): bool {.inline.} = - result = c.d.channelCloseMpmc() - -func peek*[T](c: Channel[T]): int {.inline.} = peek(c.d) - -proc newChannel*[T](elements = 30): Channel[T] = - ## Returns a new `Channel`. `elements` should be positive. - ## `elements` is used to specify whether a channel is buffered or not. - ## If `elements` = 1, the channel is unbuffered. If `elements` > 1, the - ## channel is buffered. - assert elements >= 1, "Elements must be positive!" - result = Channel[T](d: allocChannel(sizeof(T), elements)) diff --git a/lib/std/cmdline.nim b/lib/std/cmdline.nim new file mode 100644 index 000000000..0ba4619e5 --- /dev/null +++ b/lib/std/cmdline.nim @@ -0,0 +1,313 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2022 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module contains system facilities for reading command +## line parameters. + +## **See also:** +## * `parseopt module <parseopt.html>`_ for command-line parser beyond +## `parseCmdLine proc`_ + + +include system/inclrtl + +when defined(nimPreviewSlimSystem): + import std/widestrs + +when defined(nodejs): + from std/private/oscommon import ReadDirEffect + + +const weirdTarget = defined(nimscript) or defined(js) + + +when weirdTarget: + discard +elif defined(windows): + import std/winlean +elif defined(posix): + import std/posix +else: + {.error: "The cmdline module has not been implemented for the target platform.".} + + +# Needed by windows in order to obtain the command line for targets +# other than command line targets +when defined(windows) and not weirdTarget: + template getCommandLine*(): untyped = getCommandLineW() + + +proc parseCmdLine*(c: string): seq[string] {. + noSideEffect, rtl, extern: "nos$1".} = + ## Splits a `command line`:idx: into several components. + ## + ## **Note**: This proc is only occasionally useful, better use the + ## `parseopt module <parseopt.html>`_. + ## + ## On Windows, it uses the `following parsing rules + ## <http://msdn.microsoft.com/en-us/library/17w5ykft.aspx>`_: + ## + ## * Arguments are delimited by white space, which is either a space or a tab. + ## * The caret character (^) is not recognized as an escape character or + ## delimiter. The character is handled completely by the command-line parser + ## in the operating system before being passed to the argv array in the + ## program. + ## * A string surrounded by double quotation marks ("string") is interpreted + ## as a single argument, regardless of white space contained within. A + ## quoted string can be embedded in an argument. + ## * A double quotation mark preceded by a backslash (\") is interpreted as a + ## literal double quotation mark character ("). + ## * Backslashes are interpreted literally, unless they immediately precede + ## a double quotation mark. + ## * If an even number of backslashes is followed by a double quotation mark, + ## one backslash is placed in the argv array for every pair of backslashes, + ## and the double quotation mark is interpreted as a string delimiter. + ## * If an odd number of backslashes is followed by a double quotation mark, + ## one backslash is placed in the argv array for every pair of backslashes, + ## and the double quotation mark is "escaped" by the remaining backslash, + ## causing a literal double quotation mark (") to be placed in argv. + ## + ## On Posix systems, it uses the following parsing rules: + ## Components are separated by whitespace unless the whitespace + ## occurs within ``"`` or ``'`` quotes. + ## + ## See also: + ## * `parseopt module <parseopt.html>`_ + ## * `paramCount proc`_ + ## * `paramStr proc`_ + ## * `commandLineParams proc`_ + + result = @[] + var i = 0 + var a = "" + while true: + setLen(a, 0) + # eat all delimiting whitespace + while i < c.len and c[i] in {' ', '\t', '\l', '\r'}: inc(i) + if i >= c.len: break + when defined(windows): + # parse a single argument according to the above rules: + var inQuote = false + while i < c.len: + case c[i] + of '\\': + var j = i + while j < c.len and c[j] == '\\': inc(j) + if j < c.len and c[j] == '"': + for k in 1..(j-i) div 2: a.add('\\') + if (j-i) mod 2 == 0: + i = j + else: + a.add('"') + i = j+1 + else: + a.add(c[i]) + inc(i) + of '"': + inc(i) + if not inQuote: inQuote = true + elif i < c.len and c[i] == '"': + a.add(c[i]) + inc(i) + else: + inQuote = false + break + of ' ', '\t': + if not inQuote: break + a.add(c[i]) + inc(i) + else: + a.add(c[i]) + inc(i) + else: + case c[i] + of '\'', '\"': + var delim = c[i] + inc(i) # skip ' or " + while i < c.len and c[i] != delim: + add a, c[i] + inc(i) + if i < c.len: inc(i) + else: + while i < c.len and c[i] > ' ': + add(a, c[i]) + inc(i) + add(result, move a) + +when defined(nimdoc): + # Common forward declaration docstring block for parameter retrieval procs. + proc paramCount*(): int {.tags: [ReadIOEffect].} = + ## Returns the number of `command line arguments`:idx: given to the + ## application. + ## + ## Unlike `argc`:idx: in C, if your binary was called without parameters this + ## will return zero. + ## You can query each individual parameter with `paramStr proc`_ + ## or retrieve all of them in one go with `commandLineParams proc`_. + ## + ## **Availability**: When generating a dynamic library (see `--app:lib`) on + ## Posix this proc is not defined. + ## Test for availability using `declared() <system.html#declared,untyped>`_. + ## + ## See also: + ## * `parseopt module <parseopt.html>`_ + ## * `parseCmdLine proc`_ + ## * `paramStr proc`_ + ## * `commandLineParams proc`_ + ## + ## **Examples:** + ## + ## ```nim + ## when declared(paramCount): + ## # Use paramCount() here + ## else: + ## # Do something else! + ## ``` + + proc paramStr*(i: int): string {.tags: [ReadIOEffect].} = + ## Returns the `i`-th `command line argument`:idx: given to the application. + ## + ## `i` should be in the range `1..paramCount()`, the `IndexDefect` + ## exception will be raised for invalid values. Instead of iterating + ## over `paramCount()`_ with this proc you can + ## call the convenience `commandLineParams()`_. + ## + ## Similarly to `argv`:idx: in C, + ## it is possible to call `paramStr(0)` but this will return OS specific + ## contents (usually the name of the invoked executable). You should avoid + ## this and call `getAppFilename() <os.html#getAppFilename>`_ instead. + ## + ## **Availability**: When generating a dynamic library (see `--app:lib`) on + ## Posix this proc is not defined. + ## Test for availability using `declared() <system.html#declared,untyped>`_. + ## + ## See also: + ## * `parseopt module <parseopt.html>`_ + ## * `parseCmdLine proc`_ + ## * `paramCount proc`_ + ## * `commandLineParams proc`_ + ## * `getAppFilename proc <os.html#getAppFilename>`_ + ## + ## **Examples:** + ## + ## ```nim + ## when declared(paramStr): + ## # Use paramStr() here + ## else: + ## # Do something else! + ## ``` + +elif defined(nimscript): discard +elif defined(nodejs): + type Argv = object of JsRoot + let argv {.importjs: "process.argv".} : Argv + proc len(argv: Argv): int {.importjs: "#.length".} + proc `[]`(argv: Argv, i: int): cstring {.importjs: "#[#]".} + + proc paramCount*(): int {.tags: [ReadDirEffect].} = + result = argv.len - 2 + + proc paramStr*(i: int): string {.tags: [ReadIOEffect].} = + let i = i + 1 + if i < argv.len and i >= 0: + result = $argv[i] + else: + raise newException(IndexDefect, formatErrorIndexBound(i - 1, argv.len - 2)) +elif defined(windows): + # Since we support GUI applications with Nim, we sometimes generate + # a WinMain entry proc. But a WinMain proc has no access to the parsed + # command line arguments. The way to get them differs. Thus we parse them + # ourselves. This has the additional benefit that the program's behaviour + # is always the same -- independent of the used C compiler. + var + ownArgv {.threadvar.}: seq[string] + ownParsedArgv {.threadvar.}: bool + + proc paramCount*(): int {.rtl, extern: "nos$1", tags: [ReadIOEffect].} = + # Docstring in nimdoc block. + if not ownParsedArgv: + ownArgv = parseCmdLine($getCommandLine()) + ownParsedArgv = true + result = ownArgv.len-1 + + proc paramStr*(i: int): string {.rtl, extern: "nos$1", + tags: [ReadIOEffect].} = + # Docstring in nimdoc block. + if not ownParsedArgv: + ownArgv = parseCmdLine($getCommandLine()) + ownParsedArgv = true + if i < ownArgv.len and i >= 0: + result = ownArgv[i] + else: + raise newException(IndexDefect, formatErrorIndexBound(i, ownArgv.len-1)) + +elif defined(genode): + proc paramStr*(i: int): string = + raise newException(OSError, "paramStr is not implemented on Genode") + + proc paramCount*(): int = + raise newException(OSError, "paramCount is not implemented on Genode") +elif weirdTarget or (defined(posix) and appType == "lib"): + proc paramStr*(i: int): string {.tags: [ReadIOEffect].} = + raise newException(OSError, "paramStr is not implemented on current platform") + + proc paramCount*(): int {.tags: [ReadIOEffect].} = + raise newException(OSError, "paramCount is not implemented on current platform") +elif not defined(createNimRtl) and + not(defined(posix) and appType == "lib"): + # On Posix, there is no portable way to get the command line from a DLL. + var + cmdCount {.importc: "cmdCount".}: cint + cmdLine {.importc: "cmdLine".}: cstringArray + + proc paramStr*(i: int): string {.tags: [ReadIOEffect].} = + # Docstring in nimdoc block. + if i < cmdCount and i >= 0: + result = $cmdLine[i] + else: + raise newException(IndexDefect, formatErrorIndexBound(i, cmdCount-1)) + + proc paramCount*(): int {.tags: [ReadIOEffect].} = + # Docstring in nimdoc block. + result = cmdCount-1 + +when declared(paramCount) or defined(nimdoc): + proc commandLineParams*(): seq[string] = + ## Convenience proc which returns the command line parameters. + ## + ## This returns **only** the parameters. If you want to get the application + ## executable filename, call `getAppFilename() <os.html#getAppFilename>`_. + ## + ## **Availability**: On Posix there is no portable way to get the command + ## line from a DLL and thus the proc isn't defined in this environment. You + ## can test for its availability with `declared() + ## <system.html#declared,untyped>`_. + ## + ## See also: + ## * `parseopt module <parseopt.html>`_ + ## * `parseCmdLine proc`_ + ## * `paramCount proc`_ + ## * `paramStr proc`_ + ## * `getAppFilename proc <os.html#getAppFilename>`_ + ## + ## **Examples:** + ## + ## ```nim + ## when declared(commandLineParams): + ## # Use commandLineParams() here + ## else: + ## # Do something else! + ## ``` + result = @[] + for i in 1..paramCount(): + result.add(paramStr(i)) +else: + proc commandLineParams*(): seq[string] {.error: + "commandLineParams() unsupported by dynamic libraries".} = + discard diff --git a/lib/std/compilesettings.nim b/lib/std/compilesettings.nim index 725b68acd..6d8bd22f4 100644 --- a/lib/std/compilesettings.nim +++ b/lib/std/compilesettings.nim @@ -1,6 +1,6 @@ # # -# The Nim Compiler +# Nim's Runtime Library # (c) Copyright 2020 Nim Contributors # # See the file "copying.txt", included in this @@ -8,7 +8,7 @@ # ## This module allows querying the compiler about -## diverse configuration settings. +## diverse configuration settings. See also `compileOption`. # Note: Only add new enum values at the end to ensure binary compatibility with # other Nim compiler versions! @@ -32,6 +32,8 @@ type backend ## the backend (eg: c|cpp|objc|js); both `nim doc --backend:js` ## and `nim js` would imply backend=js libPath ## the absolute path to the stdlib library, i.e. nim's `--lib`, since 1.5.1 + gc {.deprecated.} ## gc selected + mm ## memory management selected MultipleValueSetting* {.pure.} = enum ## \ ## settings resulting in a seq of string values diff --git a/lib/std/decls.nim b/lib/std/decls.nim index dd7d19da7..bb7ec3593 100644 --- a/lib/std/decls.nim +++ b/lib/std/decls.nim @@ -1,19 +1,31 @@ -# see `semLowerLetVarCustomPragma` for compiler support that enables these -# lowerings +## This module implements syntax sugar for some declarations. -template byaddr*(lhs, typ, ex) = - ## Allows a syntax for lvalue reference, exact analog to - ## `auto& a = ex;` in C++ +import std/macros + +macro byaddr*(sect) = + ## Allows a syntax for l-value references, being an exact analog to + ## `auto& a = ex;` in C++. + ## + ## .. warning:: This makes use of 2 experimental features, namely nullary + ## templates instantiated as symbols and variable macro pragmas. + ## For this reason, its behavior is not stable. The current implementation + ## allows redefinition, but this is not an intended consequence. runnableExamples: - var s = @[10,11,12] + var s = @[10, 11, 12] var a {.byaddr.} = s[0] - a+=100 - doAssert s == @[110,11,12] - doAssert a is int + a += 100 + assert s == @[110, 11, 12] + assert a is int var b {.byaddr.}: int = s[0] - doAssert a.addr == b.addr - when typ is typeof(nil): - let tmp = addr(ex) - else: - let tmp: ptr typ = addr(ex) - template lhs: untyped = tmp[] + assert a.addr == b.addr + expectLen sect, 1 + let def = sect[0] + let + lhs = def[0] + typ = def[1] + ex = def[2] + addrTyp = if typ.kind == nnkEmpty: typ else: newTree(nnkPtrTy, typ) + result = quote do: + let tmp: `addrTyp` = addr(`ex`) + template `lhs`: untyped = tmp[] + result.copyLineInfo(def) diff --git a/lib/std/dirs.nim b/lib/std/dirs.nim new file mode 100644 index 000000000..380d6d08f --- /dev/null +++ b/lib/std/dirs.nim @@ -0,0 +1,135 @@ +## This module implements directory handling. + +from std/paths import Path, ReadDirEffect, WriteDirEffect + +from std/private/osdirs import dirExists, createDir, existsOrCreateDir, removeDir, + moveDir, walkDir, setCurrentDir, + walkDirRec, PathComponent + +export PathComponent + +proc dirExists*(dir: Path): bool {.inline, tags: [ReadDirEffect], sideEffect.} = + ## Returns true if the directory `dir` exists. If `dir` is a file, false + ## is returned. Follows symlinks. + result = dirExists(dir.string) + +proc createDir*(dir: Path) {.inline, tags: [WriteDirEffect, ReadDirEffect].} = + ## Creates the `directory`:idx: `dir`. + ## + ## The directory may contain several subdirectories that do not exist yet. + ## The full path is created. If this fails, `OSError` is raised. + ## + ## It does **not** fail if the directory already exists because for + ## most usages this does not indicate an error. + ## + ## See also: + ## * `removeDir proc`_ + ## * `existsOrCreateDir proc`_ + ## * `moveDir proc`_ + createDir(dir.string) + +proc existsOrCreateDir*(dir: Path): bool {.inline, tags: [WriteDirEffect, ReadDirEffect].} = + ## Checks if a `directory`:idx: `dir` exists, and creates it otherwise. + ## + ## Does not create parent directories (raises `OSError` if parent directories do not exist). + ## Returns `true` if the directory already exists, and `false` otherwise. + ## + ## See also: + ## * `removeDir proc`_ + ## * `createDir proc`_ + ## * `moveDir proc`_ + result = existsOrCreateDir(dir.string) + +proc removeDir*(dir: Path, checkDir = false + ) {.inline, tags: [WriteDirEffect, ReadDirEffect].} = + ## Removes the directory `dir` including all subdirectories and files + ## in `dir` (recursively). + ## + ## If this fails, `OSError` is raised. This does not fail if the directory never + ## existed in the first place, unless `checkDir` = true. + ## + ## See also: + ## * `removeFile proc <files.html#removeFile>`_ + ## * `existsOrCreateDir proc`_ + ## * `createDir proc`_ + ## * `moveDir proc`_ + removeDir(dir.string, checkDir) + +proc moveDir*(source, dest: Path) {.inline, tags: [ReadIOEffect, WriteIOEffect].} = + ## Moves a directory from `source` to `dest`. + ## + ## Symlinks are not followed: if `source` contains symlinks, they themself are + ## moved, not their target. + ## + ## If this fails, `OSError` is raised. + ## + ## See also: + ## * `moveFile proc <files.html#moveFile>`_ + ## * `removeDir proc`_ + ## * `existsOrCreateDir proc`_ + ## * `createDir proc`_ + moveDir(source.string, dest.string) + +iterator walkDir*(dir: Path; relative = false, checkDir = false, + skipSpecial = false): + tuple[kind: PathComponent, path: Path] {.tags: [ReadDirEffect].} = + ## Walks over the directory `dir` and yields for each directory or file in + ## `dir`. The component type and full path for each item are returned. + ## + ## Walking is not recursive. + ## * If `relative` is true (default: false) + ## the resulting path is shortened to be relative to ``dir``, + ## otherwise the full path is returned. + ## * If `checkDir` is true, `OSError` is raised when `dir` + ## doesn't exist. + ## * If `skipSpecial` is true, then (besides all directories) only *regular* + ## files (**without** special "file" objects like FIFOs, device files, + ## etc) will be yielded on Unix. + for (k, p) in walkDir(dir.string, relative, checkDir, skipSpecial): + yield (k, Path(p)) + +iterator walkDirRec*(dir: Path, + yieldFilter = {pcFile}, followFilter = {pcDir}, + relative = false, checkDir = false, skipSpecial = false): + Path {.tags: [ReadDirEffect].} = + ## Recursively walks over the directory `dir` and yields for each file + ## or directory in `dir`. + ## + ## Options `relative`, `checkdir`, `skipSpecial` are explained in + ## [walkDir iterator] description. + ## + ## .. warning:: Modifying the directory structure while the iterator + ## is traversing may result in undefined behavior! + ## + ## Walking is recursive. `followFilter` controls the behaviour of the iterator: + ## + ## ===================== ============================================= + ## yieldFilter meaning + ## ===================== ============================================= + ## ``pcFile`` yield real files (default) + ## ``pcLinkToFile`` yield symbolic links to files + ## ``pcDir`` yield real directories + ## ``pcLinkToDir`` yield symbolic links to directories + ## ===================== ============================================= + ## + ## ===================== ============================================= + ## followFilter meaning + ## ===================== ============================================= + ## ``pcDir`` follow real directories (default) + ## ``pcLinkToDir`` follow symbolic links to directories + ## ===================== ============================================= + ## + ## + ## See also: + ## * `walkDir iterator`_ + for p in walkDirRec(dir.string, yieldFilter, followFilter, relative, + checkDir, skipSpecial): + yield Path(p) + +proc setCurrentDir*(newDir: Path) {.inline, tags: [].} = + ## Sets the `current working directory`:idx:; `OSError` + ## is raised if `newDir` cannot been set. + ## + ## See also: + ## * `getCurrentDir proc <paths.html#getCurrentDir>`_ + osdirs.setCurrentDir(newDir.string) diff --git a/lib/std/editdistance.nim b/lib/std/editdistance.nim index 9f29c5c05..40c0017ae 100644 --- a/lib/std/editdistance.nim +++ b/lib/std/editdistance.nim @@ -10,7 +10,7 @@ ## This module implements an algorithm to compute the ## `edit distance`:idx: between two Unicode strings. -import unicode +import std/unicode proc editDistance*(a, b: string): int {.noSideEffect.} = ## Returns the **unicode-rune** edit distance between `a` and `b`. @@ -18,7 +18,7 @@ proc editDistance*(a, b: string): int {.noSideEffect.} = ## This uses the `Levenshtein`:idx: distance algorithm with only a linear ## memory overhead. runnableExamples: static: doAssert editdistance("Kitten", "Bitten") == 1 - if len(a) > len(b): + if runeLen(a) > runeLen(b): # make `b` the longer string return editDistance(b, a) # strip common prefix diff --git a/lib/std/effecttraits.nim b/lib/std/effecttraits.nim index 358280db0..3d1b4ffd3 100644 --- a/lib/std/effecttraits.nim +++ b/lib/std/effecttraits.nim @@ -11,13 +11,14 @@ ## for Nim's macro system. ## **Since**: Version 1.4. ## -## One can test for the existance of this standard module +## One can test for the existence of this standard module ## via `defined(nimHasEffectTraitsModule)`. -import macros +import std/macros proc getRaisesListImpl(n: NimNode): NimNode = discard "see compiler/vmops.nim" proc getTagsListImpl(n: NimNode): NimNode = discard "see compiler/vmops.nim" +proc getForbidsListImpl(n: NimNode): NimNode = discard "see compiler/vmops.nim" proc isGcSafeImpl(n: NimNode): bool = discard "see compiler/vmops.nim" proc hasNoSideEffectsImpl(n: NimNode): bool = discard "see compiler/vmops.nim" @@ -37,6 +38,14 @@ proc getTagsList*(fn: NimNode): NimNode = expectKind fn, nnkSym result = getTagsListImpl(fn) +proc getForbidsList*(fn: NimNode): NimNode = + ## Extracts the `.forbids` list of the func/proc/etc `fn`. + ## `fn` has to be a resolved symbol of kind `nnkSym`. This + ## implies that the macro that calls this proc should accept `typed` + ## arguments and not `untyped` arguments. + expectKind fn, nnkSym + result = getForbidsListImpl(fn) + proc isGcSafe*(fn: NimNode): bool = ## Return true if the func/proc/etc `fn` is `gcsafe`. ## `fn` has to be a resolved symbol of kind `nnkSym`. This diff --git a/lib/std/enumerate.nim b/lib/std/enumerate.nim index a8f0e1ba7..beb65ed30 100644 --- a/lib/std/enumerate.nim +++ b/lib/std/enumerate.nim @@ -11,7 +11,7 @@ ## macro system. import std/private/since -import macros +import std/macros macro enumerate*(x: ForLoopStmt): untyped {.since: (1, 3).} = @@ -21,7 +21,7 @@ macro enumerate*(x: ForLoopStmt): untyped {.since: (1, 3).} = ## The default starting count `0` can be manually overridden if needed. runnableExamples: let a = [10, 20, 30] - var b: seq[(int, int)] + var b: seq[(int, int)] = @[] for i, x in enumerate(a): b.add((i, x)) assert b == @[(0, 10), (1, 20), (2, 30)] diff --git a/lib/std/enumutils.nim b/lib/std/enumutils.nim index 16dab9d1a..9c338817d 100644 --- a/lib/std/enumutils.nim +++ b/lib/std/enumutils.nim @@ -10,48 +10,63 @@ import std/macros from std/typetraits import OrdinalEnum, HoleyEnum +when defined(nimPreviewSlimSystem): + import std/assertions + + # xxx `genEnumCaseStmt` needs tests and runnableExamples -macro genEnumCaseStmt*(typ: typedesc, argSym: typed, default: typed, +macro genEnumCaseStmt*(typ: typedesc, argSym: typed, default: typed, userMin, userMax: static[int], normalizer: static[proc(s :string): string]): untyped = - # generates a case stmt, which assigns the correct enum field given + # Generates a case stmt, which assigns the correct enum field given # a normalized string comparison to the `argSym` input. - # string normalization is done using passed normalizer. - # NOTE: for an enum with fields Foo, Bar, ... we cannot generate - # `of "Foo".nimIdentNormalize: Foo`. - # This will fail, if the enum is not defined at top level (e.g. in a block). - # Thus we check for the field value of the (possible holed enum) and convert - # the integer value to the generic argument `typ`. + # string normalization is done using passed normalizer. let typ = typ.getTypeInst[1] - let impl = typ.getImpl[2] + let typSym = typ.getTypeImpl.getTypeInst # skip aliases etc to get type sym + let impl = typSym.getImpl[2] expectKind impl, nnkEnumTy let normalizerNode = quote: `normalizer` expectKind normalizerNode, nnkSym result = nnkCaseStmt.newTree(newCall(normalizerNode, argSym)) # stores all processed field strings to give error msg for ambiguous enums var foundFields: seq[string] = @[] + var fVal = "" var fStr = "" # string of current field var fNum = BiggestInt(0) # int value of current field for f in impl: case f.kind of nnkEmpty: continue # skip first node of `enumTy` - of nnkSym, nnkIdent: fStr = f.strVal + of nnkSym, nnkIdent: + fVal = f.strVal + fStr = fVal + of nnkAccQuoted: + fVal = "" + for ch in f: + fVal.add ch.strVal + fStr = fVal of nnkEnumFieldDef: + fVal = f[0].strVal case f[1].kind - of nnkStrLit: fStr = f[1].strVal + of nnkStrLit: + fStr = f[1].strVal of nnkTupleConstr: fStr = f[1][1].strVal fNum = f[1][0].intVal of nnkIntLit: fStr = f[0].strVal fNum = f[1].intVal - else: error("Invalid tuple syntax!", f[1]) - else: error("Invalid node for enum type!", f) + else: + let fAst = f[0].getImpl + if fAst.kind == nnkStrLit: + fStr = fAst.strVal + else: + error("Invalid tuple syntax!", f[1]) + else: error("Invalid node for enum type `" & $f.kind & "`!", f) # add field if string not already added if fNum >= userMin and fNum <= userMax: fStr = normalizer(fStr) if fStr notin foundFields: - result.add nnkOfBranch.newTree(newLit fStr, nnkCall.newTree(typ, newLit fNum)) + result.add nnkOfBranch.newTree(newLit fStr, newDotExpr(typ, ident fVal)) foundFields.add fStr else: error("Ambiguous enums cannot be parsed, field " & $fStr & @@ -67,7 +82,7 @@ macro genEnumCaseStmt*(typ: typedesc, argSym: typed, default: typed, result.add nnkElse.newTree(default) macro enumFullRange(a: typed): untyped = - newNimNode(nnkCurly).add(a.getType[1][1..^1]) + newNimNode(nnkBracket).add(a.getType[1][1..^1]) macro enumNames(a: typed): untyped = # this could be exported too; in particular this could be useful for enum with holes. @@ -79,15 +94,93 @@ macro enumNames(a: typed): untyped = iterator items*[T: HoleyEnum](E: typedesc[T]): T = ## Iterates over an enum with holes. runnableExamples: - type A = enum a0 = 2, a1 = 4, a2 - type B[T] = enum b0 = 2, b1 = 4 + type + A = enum + a0 = 2 + a1 = 4 + a2 + B[T] = enum + b0 = 2 + b1 = 4 from std/sequtils import toSeq assert A.toSeq == [a0, a1, a2] assert B[float].toSeq == [B[float].b0, B[float].b1] for a in enumFullRange(E): yield a -func symbolName*[T: OrdinalEnum](a: T): string = +func span(T: typedesc[HoleyEnum]): int = + (T.high.ord - T.low.ord) + 1 + +const invalidSlot = uint8.high + +proc genLookup[T: typedesc[HoleyEnum]](_: T): auto = + const n = span(T) + var i = 0 + assert n <= invalidSlot.int + var ret {.noinit.}: array[n, uint8] + for ai in mitems(ret): ai = invalidSlot + for ai in items(T): + ret[ai.ord - T.low.ord] = uint8(i) + inc(i) + return ret + +func symbolRankImpl[T](a: T): int {.inline.} = + const n = T.span + const thres = 255 # must be <= `invalidSlot`, but this should be tuned. + when n <= thres: + const lookup = genLookup(T) + let lookup2 {.global.} = lookup # xxx improve pending https://github.com/timotheecour/Nim/issues/553 + #[ + This could be optimized using a hash adapted to `T` (possible since it's known at CT) + to get better key distribution before indexing into the lookup table table. + ]# + {.noSideEffect.}: # because it's immutable + let ret = lookup2[ord(a) - T.low.ord] + if ret != invalidSlot: return ret.int + else: + var i = 0 + # we could also generate a case statement as optimization + for ai in items(T): + if ai == a: return i + inc(i) + raise newException(IndexDefect, $ord(a) & " invalid for " & $T) + +template symbolRank*[T: enum](a: T): int = + ## Returns the index in which `a` is listed in `T`. + ## + ## The cost for a `HoleyEnum` is implementation defined, currently optimized + ## for small enums, otherwise is `O(T.enumLen)`. + runnableExamples: + type + A = enum # HoleyEnum + a0 = -3 + a1 = 10 + a2 + a3 = (20, "f3Alt") + B = enum # OrdinalEnum + b0 + b1 + b2 + C = enum # OrdinalEnum + c0 = 10 + c1 + c2 + assert a2.symbolRank == 2 + assert b2.symbolRank == 2 + assert c2.symbolRank == 2 + assert c2.ord == 12 + assert a2.ord == 11 + var invalid = 7.A + doAssertRaises(IndexDefect): discard invalid.symbolRank + when T is Ordinal: ord(a) - T.low.ord.static + else: symbolRankImpl(a) + +proc rangeBase(T: typedesc): typedesc {.magic: "TypeTrait".} + # skip one level of range; return the base type of a range type + +func symbolName*[T: enum](a: T): string = ## Returns the symbol name of an enum. + ## + ## This uses `symbolRank`. runnableExamples: type B = enum b0 = (10, "kb0") @@ -97,5 +190,13 @@ func symbolName*[T: OrdinalEnum](a: T): string = assert b.symbolName == "b0" assert $b == "kb0" static: assert B.high.symbolName == "b2" - const names = enumNames(T) - names[a.ord - T.low.ord] + type C = enum # HoleyEnum + c0 = -3 + c1 = 4 + c2 = 20 + assert c1.symbolName == "c1" + when T is range: + const names = enumNames(rangeBase T) + else: + const names = enumNames(T) + names[a.symbolRank] diff --git a/lib/std/envvars.nim b/lib/std/envvars.nim new file mode 100644 index 000000000..a955077ea --- /dev/null +++ b/lib/std/envvars.nim @@ -0,0 +1,221 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2022 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + + +## The `std/envvars` module implements environment variable handling. +import std/oserrors + +type + ReadEnvEffect* = object of ReadIOEffect ## Effect that denotes a read + ## from an environment variable. + WriteEnvEffect* = object of WriteIOEffect ## Effect that denotes a write + ## to an environment variable. + + +when not defined(nimscript): + when defined(nodejs): + proc getEnv*(key: string, default = ""): string {.tags: [ReadEnvEffect].} = + var ret = default.cstring + let key2 = key.cstring + {.emit: "const value = process.env[`key2`];".} + {.emit: "if (value !== undefined) { `ret` = value };".} + result = $ret + + proc existsEnv*(key: string): bool {.tags: [ReadEnvEffect].} = + var key2 = key.cstring + var ret: bool + {.emit: "`ret` = `key2` in process.env;".} + result = ret + + proc putEnv*(key, val: string) {.tags: [WriteEnvEffect].} = + var key2 = key.cstring + var val2 = val.cstring + {.emit: "process.env[`key2`] = `val2`;".} + + proc delEnv*(key: string) {.tags: [WriteEnvEffect].} = + var key2 = key.cstring + {.emit: "delete process.env[`key2`];".} + + iterator envPairsImpl(): tuple[key, value: string] {.tags: [ReadEnvEffect].} = + var num: int + var keys: RootObj + {.emit: "`keys` = Object.keys(process.env); `num` = `keys`.length;".} + for i in 0..<num: + var key, value: cstring + {.emit: "`key` = `keys`[`i`]; `value` = process.env[`key`];".} + yield ($key, $value) + + # commented because it must keep working with js+VM + # elif defined(js): + # {.error: "requires -d:nodejs".} + + else: + + when defined(windows): + proc c_putenv(envstring: cstring): cint {.importc: "_putenv", header: "<stdlib.h>".} + from std/private/win_setenv import setEnvImpl + import std/winlean + when defined(nimPreviewSlimSystem): + import std/widestrs + + type wchar_t {.importc: "wchar_t", header: "<stdlib.h>".} = int16 + proc c_wgetenv(varname: ptr wchar_t): ptr wchar_t {.importc: "_wgetenv", + header: "<stdlib.h>".} + proc getEnvImpl(env: cstring): WideCString = + let r: WideCString = env.newWideCString + cast[WideCString](c_wgetenv(cast[ptr wchar_t](r))) + else: + proc c_getenv(env: cstring): cstring {. + importc: "getenv", header: "<stdlib.h>".} + proc c_setenv(envname: cstring, envval: cstring, overwrite: cint): cint {.importc: "setenv", header: "<stdlib.h>".} + proc c_unsetenv(env: cstring): cint {.importc: "unsetenv", header: "<stdlib.h>".} + proc getEnvImpl(env: cstring): cstring = c_getenv(env) + + proc getEnv*(key: string, default = ""): string {.tags: [ReadEnvEffect].} = + ## Returns the value of the `environment variable`:idx: named `key`. + ## + ## If the variable does not exist, `""` is returned. To distinguish + ## whether a variable exists or it's value is just `""`, call + ## `existsEnv(key) proc`_. + ## + ## See also: + ## * `existsEnv proc`_ + ## * `putEnv proc`_ + ## * `delEnv proc`_ + ## * `envPairs iterator`_ + runnableExamples: + assert getEnv("unknownEnv") == "" + assert getEnv("unknownEnv", "doesn't exist") == "doesn't exist" + + let env = getEnvImpl(key) + if env == nil: + result = default + else: + result = $env + + proc existsEnv*(key: string): bool {.tags: [ReadEnvEffect].} = + ## Checks whether the environment variable named `key` exists. + ## Returns true if it exists, false otherwise. + ## + ## See also: + ## * `getEnv proc`_ + ## * `putEnv proc`_ + ## * `delEnv proc`_ + ## * `envPairs iterator`_ + runnableExamples: + assert not existsEnv("unknownEnv") + + result = getEnvImpl(key) != nil + + proc putEnv*(key, val: string) {.tags: [WriteEnvEffect].} = + ## Sets the value of the `environment variable`:idx: named `key` to `val`. + ## If an error occurs, `OSError` is raised. + ## + ## See also: + ## * `getEnv proc`_ + ## * `existsEnv proc`_ + ## * `delEnv proc`_ + ## * `envPairs iterator`_ + when defined(windows): + if key.len == 0 or '=' in key: + raise newException(OSError, "invalid key, got: " & $(key, val)) + if setEnvImpl(key, val, 1'i32) != 0'i32: + raiseOSError(osLastError(), $(key, val)) + else: + if c_setenv(key, val, 1'i32) != 0'i32: + raiseOSError(osLastError(), $(key, val)) + + proc delEnv*(key: string) {.tags: [WriteEnvEffect].} = + ## Deletes the `environment variable`:idx: named `key`. + ## If an error occurs, `OSError` is raised. + ## + ## See also:ven + ## * `getEnv proc`_ + ## * `existsEnv proc`_ + ## * `putEnv proc`_ + ## * `envPairs iterator`_ + template bail = raiseOSError(osLastError(), key) + when defined(windows): + #[ + # https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/putenv-s-wputenv-s?view=msvc-160 + > You can remove a variable from the environment by specifying an empty string (that is, "") for value_string + note that nil is not legal + ]# + if key.len == 0 or '=' in key: + raise newException(OSError, "invalid key, got: " & key) + let envToDel = key & "=" + if c_putenv(cstring envToDel) != 0'i32: bail + else: + if c_unsetenv(key) != 0'i32: bail + + when defined(windows): + when defined(cpp): + proc strEnd(cstr: WideCString, c = 0'i32): WideCString {.importcpp: "(NI16*)wcschr((const wchar_t *)#, #)", + header: "<string.h>".} + else: + proc strEnd(cstr: WideCString, c = 0'i32): WideCString {.importc: "wcschr", + header: "<string.h>".} + elif defined(macosx) and not defined(ios) and not defined(emscripten): + # From the manual: + # Shared libraries and bundles don't have direct access to environ, + # which is only available to the loader ld(1) when a complete program + # is being linked. + # The environment routines can still be used, but if direct access to + # environ is needed, the _NSGetEnviron() routine, defined in + # <crt_externs.h>, can be used to retrieve the address of environ + # at runtime. + proc NSGetEnviron(): ptr cstringArray {.importc: "_NSGetEnviron", + header: "<crt_externs.h>".} + elif defined(haiku): + var gEnv {.importc: "environ", header: "<stdlib.h>".}: cstringArray + else: + var gEnv {.importc: "environ".}: cstringArray + + iterator envPairsImpl(): tuple[key, value: string] {.tags: [ReadEnvEffect].} = + when defined(windows): + let env = getEnvironmentStringsW() + var e = env + if e != nil: + while true: + let eend = strEnd(e) + let kv = $e + let p = find(kv, '=') + yield (substr(kv, 0, p-1), substr(kv, p+1)) + e = cast[WideCString](cast[ByteAddress](eend)+2) + if int(eend[1]) == 0: break + discard freeEnvironmentStringsW(env) + else: + var i = 0 + when defined(macosx) and not defined(ios) and not defined(emscripten): + var gEnv = NSGetEnviron()[] + while gEnv[i] != nil: + let kv = $gEnv[i] + inc(i) + let p = find(kv, '=') + yield (substr(kv, 0, p-1), substr(kv, p+1)) + +proc envPairsImplSeq(): seq[tuple[key, value: string]] = discard # vmops + +iterator envPairs*(): tuple[key, value: string] {.tags: [ReadEnvEffect].} = + ## Iterate over all `environments variables`:idx:. + ## + ## In the first component of the tuple is the name of the current variable stored, + ## in the second its value. + ## + ## Works in native backends, nodejs and vm, like the following APIs: + ## * `getEnv proc`_ + ## * `existsEnv proc`_ + ## * `putEnv proc`_ + ## * `delEnv proc`_ + when nimvm: + for ai in envPairsImplSeq(): yield ai + else: + when defined(nimscript): discard + else: + for ai in envPairsImpl(): yield ai diff --git a/lib/std/exitprocs.nim b/lib/std/exitprocs.nim index c6537f7f8..f26368f42 100644 --- a/lib/std/exitprocs.nim +++ b/lib/std/exitprocs.nim @@ -7,7 +7,11 @@ # distribution, for details about the copyright. # -import locks +## This module allows adding hooks to program exit. + +import std/locks +when defined(js) and not defined(nodejs): + import std/assertions type FunKind = enum kClosure, kNoconv # extend as needed @@ -18,20 +22,20 @@ type var gFunsLock: Lock - gFuns: seq[Fun] + gFuns {.cursor.}: seq[Fun] #Intentionally use the cursor to break up the lifetime trace and make it compatible with JS. initLock(gFunsLock) when defined(js): proc addAtExit(quitProc: proc() {.noconv.}) = when defined(nodejs): - asm """ + {.emit: """ process.on('exit', `quitProc`); - """ + """.} elif defined(js): - asm """ + {.emit: """ window.onbeforeunload = `quitProc`; - """ + """.} else: proc addAtExit(quitProc: proc() {.noconv.}) {. importc: "atexit", header: "<stdlib.h>".} @@ -43,6 +47,7 @@ proc callClosures() {.noconv.} = case fun.kind of kClosure: fun.fun1() of kNoconv: fun.fun2() + gFuns.setLen(0) template fun() = if gFuns.len == 0: @@ -64,24 +69,19 @@ proc addExitProc*(cl: proc() {.noconv.}) = fun() gFuns.add Fun(kind: kNoconv, fun2: cl) -when not defined(nimscript): +when not defined(nimscript) and (not defined(js) or defined(nodejs)): proc getProgramResult*(): int = when defined(js) and defined(nodejs): - asm """ + {.emit: """ `result` = process.exitCode; -""" - elif not defined(js): - result = programResult +""".} else: - doAssert false + result = programResult proc setProgramResult*(a: int) = - # pending https://github.com/nim-lang/Nim/issues/14674 when defined(js) and defined(nodejs): - asm """ + {.emit: """ process.exitCode = `a`; -""" - elif not defined(js): - programResult = a +""".} else: - doAssert false + programResult = a diff --git a/lib/std/files.nim b/lib/std/files.nim new file mode 100644 index 000000000..c4e0491c9 --- /dev/null +++ b/lib/std/files.nim @@ -0,0 +1,46 @@ +## This module implements file handling. +## +## **See also:** +## * `paths module <paths.html>`_ for path manipulation + +from std/paths import Path, ReadDirEffect, WriteDirEffect + +from std/private/osfiles import fileExists, removeFile, + moveFile + + +proc fileExists*(filename: Path): bool {.inline, tags: [ReadDirEffect], sideEffect.} = + ## Returns true if `filename` exists and is a regular file or symlink. + ## + ## Directories, device files, named pipes and sockets return false. + result = fileExists(filename.string) + +proc removeFile*(file: Path) {.inline, tags: [WriteDirEffect].} = + ## Removes the `file`. + ## + ## If this fails, `OSError` is raised. This does not fail + ## if the file never existed in the first place. + ## + ## On Windows, ignores the read-only attribute. + ## + ## See also: + ## * `removeDir proc <dirs.html#removeDir>`_ + ## * `moveFile proc`_ + removeFile(file.string) + +proc moveFile*(source, dest: Path) {.inline, + tags: [ReadDirEffect, ReadIOEffect, WriteIOEffect].} = + ## Moves a file from `source` to `dest`. + ## + ## Symlinks are not followed: if `source` is a symlink, it is itself moved, + ## not its target. + ## + ## If this fails, `OSError` is raised. + ## If `dest` already exists, it will be overwritten. + ## + ## Can be used to `rename files`:idx:. + ## + ## See also: + ## * `moveDir proc <dirs.html#moveDir>`_ + ## * `removeFile proc`_ + moveFile(source.string, dest.string) diff --git a/lib/std/formatfloat.nim b/lib/std/formatfloat.nim new file mode 100644 index 000000000..9258245f6 --- /dev/null +++ b/lib/std/formatfloat.nim @@ -0,0 +1,143 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2022 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module implements formatting floats as strings. + +when defined(nimPreviewSlimSystem): + import std/assertions + +proc c_memcpy(a, b: pointer, size: csize_t): pointer {.importc: "memcpy", header: "<string.h>", discardable.} + +proc addCstringN(result: var string, buf: cstring; buflen: int) = + # no nimvm support needed, so it doesn't need to be fast here either + let oldLen = result.len + let newLen = oldLen + buflen + result.setLen newLen + c_memcpy(result[oldLen].addr, buf, buflen.csize_t) + +import std/private/[dragonbox, schubfach] + +proc writeFloatToBufferRoundtrip*(buf: var array[65, char]; value: BiggestFloat): int = + ## This is the implementation to format floats. + ## + ## returns the amount of bytes written to `buf` not counting the + ## terminating '\0' character. + result = toChars(buf, value, forceTrailingDotZero=true).int + buf[result] = '\0' + +proc writeFloatToBufferRoundtrip*(buf: var array[65, char]; value: float32): int = + result = float32ToChars(buf, value, forceTrailingDotZero=true).int + buf[result] = '\0' + +proc c_snprintf(buf: cstring, n: csize_t, frmt: cstring): cint {.header: "<stdio.h>", + importc: "snprintf", varargs, noSideEffect.} + +proc writeToBuffer(buf: var array[65, char]; value: cstring) = + var i = 0 + while value[i] != '\0': + buf[i] = value[i] + inc i + +proc writeFloatToBufferSprintf*(buf: var array[65, char]; value: BiggestFloat): int = + ## This is the implementation to format floats. + ## + ## returns the amount of bytes written to `buf` not counting the + ## terminating '\0' character. + var n = c_snprintf(cast[cstring](addr buf), 65, "%.16g", value).int + var hasDot = false + for i in 0..n-1: + if buf[i] == ',': + buf[i] = '.' + hasDot = true + elif buf[i] in {'a'..'z', 'A'..'Z', '.'}: + hasDot = true + if not hasDot: + buf[n] = '.' + buf[n+1] = '0' + buf[n+2] = '\0' + result = n + 2 + else: + result = n + # On Windows nice numbers like '1.#INF', '-1.#INF' or '1.#NAN' or 'nan(ind)' + # of '-1.#IND' are produced. + # We want to get rid of these here: + if buf[n-1] in {'n', 'N', 'D', 'd', ')'}: + writeToBuffer(buf, "nan") + result = 3 + elif buf[n-1] == 'F': + if buf[0] == '-': + writeToBuffer(buf, "-inf") + result = 4 + else: + writeToBuffer(buf, "inf") + result = 3 + +proc writeFloatToBuffer*(buf: var array[65, char]; value: BiggestFloat | float32): int {.inline.} = + when defined(nimPreviewFloatRoundtrip) or defined(nimPreviewSlimSystem): + writeFloatToBufferRoundtrip(buf, value) + else: + writeFloatToBufferSprintf(buf, value) + +proc addFloatRoundtrip*(result: var string; x: float | float32) = + when nimvm: + raiseAssert "unreachable" + else: + var buffer {.noinit.}: array[65, char] + let n = writeFloatToBufferRoundtrip(buffer, x) + result.addCstringN(cast[cstring](buffer[0].addr), n) + +proc addFloatSprintf*(result: var string; x: float) = + when nimvm: + raiseAssert "unreachable" + else: + var buffer {.noinit.}: array[65, char] + let n = writeFloatToBufferSprintf(buffer, x) + result.addCstringN(cast[cstring](buffer[0].addr), n) + +when defined(js): + proc nimFloatToString(a: float): cstring = + ## ensures the result doesn't print like an integer, i.e. return 2.0, not 2 + # print `-0.0` properly + {.emit: """ + function nimOnlyDigitsOrMinus(n) { + return n.toString().match(/^-?\d+$/); + } + if (Number.isSafeInteger(`a`)) + `result` = `a` === 0 && 1 / `a` < 0 ? "-0.0" : `a`+".0"; + else { + `result` = `a`+""; + if(nimOnlyDigitsOrMinus(`result`)){ + `result` = `a`+".0"; + } + } + """.} + +proc addFloat*(result: var string; x: float | float32) {.inline.} = + ## Converts float to its string representation and appends it to `result`. + runnableExamples: + var + s = "foo:" + b = 45.67 + s.addFloat(45.67) + assert s == "foo:45.67" + template impl = + when defined(nimPreviewFloatRoundtrip) or defined(nimPreviewSlimSystem): + addFloatRoundtrip(result, x) + else: + addFloatSprintf(result, x) + when defined(js): + when nimvm: impl() + else: + result.add nimFloatToString(x) + else: impl() + +when defined(nimPreviewSlimSystem): + func `$`*(x: float | float32): string = + ## Outplace version of `addFloat`. + result.addFloat(x) diff --git a/lib/std/genasts.nim b/lib/std/genasts.nim new file mode 100644 index 000000000..d0f07c527 --- /dev/null +++ b/lib/std/genasts.nim @@ -0,0 +1,89 @@ +## This module implements AST generation using captured variables for macros. + +import std/macros + +type GenAstOpt* = enum + kDirtyTemplate, + # When set, uses a dirty template in implementation of `genAst`. This + # is occasionally useful as workaround for issues such as #8220, see + # `strformat limitations <strformat.html#limitations>`_ for details. + # Default is unset, to avoid hijacking of uncaptured local symbols by + # symbols in caller scope. + kNoNewLit, + # don't call call newLit automatically in `genAst` capture parameters + +macro genAstOpt*(options: static set[GenAstOpt], args: varargs[untyped]): untyped = + ## Accepts a list of captured variables `a=b` or `a` and a block and returns the + ## AST that represents it. Local `{.inject.}` symbols (e.g. procs) are captured + ## unless `kDirtyTemplate in options`. + runnableExamples: + # This example shows how one could write a simplified version of `unittest.check`. + import std/[macros, strutils] + macro check2(cond: bool): untyped = + assert cond.kind == nnkInfix, "$# not implemented" % $cond.kind + result = genAst(cond, s = repr(cond), lhs = cond[1], rhs = cond[2]): + # each local symbol we access must be explicitly captured + if not cond: + raiseAssert "'$#'' failed: lhs: '$#', rhs: '$#'" % [s, $lhs, $rhs] + let a = 3 + check2 a*2 == a+3 + if false: check2 a*2 < a+1 # would error with: 'a * 2 < a + 1'' failed: lhs: '6', rhs: '4' + + runnableExamples: + # This example goes in more details about the capture semantics. + macro fun(a: string, b: static bool): untyped = + let c = 'z' + var d = 11 # implicitly {.gensym.} and needs to be captured for use in `genAst`. + proc localFun(): auto = 12 # implicitly {.inject.}, doesn't need to be captured. + genAst(a, b, c = true): + # `a`, `b` are captured explicitly, `c` is a local definition masking `c = 'z'`. + const b2 = b # macro static param `b` is forwarded here as a static param. + # `echo d` would give: `var not init` because `d` is not captured. + (a & a, b, c, localFun()) # localFun can be called without capture. + assert fun("ab", false) == ("abab", false, true, 12) + + let params = newTree(nnkFormalParams, newEmptyNode()) + let pragmas = + if kDirtyTemplate in options: + nnkPragma.newTree(ident"dirty") + else: + newEmptyNode() + + template newLitMaybe(a): untyped = + when (a is type) or (typeof(a) is (proc | iterator | func | NimNode)): + a # `proc` actually also covers template, macro + else: newLit(a) + + # using `_` as workaround, see https://github.com/nim-lang/Nim/issues/2465#issuecomment-511076669 + let name = genSym(nskTemplate, "_fun") + let call = newCall(name) + for a in args[0..^2]: + var varName: NimNode + var varVal: NimNode + case a.kind + of nnkExprEqExpr: + varName = a[0] + varVal = a[1] + of nnkIdent: + varName = a + varVal = a + else: error("invalid argument kind: " & $a.kind, a) + if kNoNewLit notin options: varVal = newCall(bindSym"newLitMaybe", varVal) + + params.add newTree(nnkIdentDefs, varName, newEmptyNode(), newEmptyNode()) + call.add varVal + + result = newStmtList() + result.add nnkTemplateDef.newTree( + name, + newEmptyNode(), + newEmptyNode(), + params, + pragmas, + newEmptyNode(), + args[^1]) + result.add newCall(bindSym"getAst", call) + +template genAst*(args: varargs[untyped]): untyped = + ## Convenience wrapper around `genAstOpt`. + genAstOpt({}, args) diff --git a/lib/std/importutils.nim b/lib/std/importutils.nim new file mode 100644 index 000000000..d2da76ea8 --- /dev/null +++ b/lib/std/importutils.nim @@ -0,0 +1,44 @@ +##[ +Utilities related to import and symbol resolution. + +Experimental API, subject to change. +]## + +#[ +Possible future APIs: +* module symbols (https://github.com/nim-lang/Nim/pull/9560) +* whichModule (subsumes canImport / moduleExists) (https://github.com/timotheecour/Nim/issues/376) +* getCurrentPkgDir (https://github.com/nim-lang/Nim/pull/10530) +* import from a computed string + related APIs (https://github.com/nim-lang/Nim/pull/10527) +]# + +when defined(nimImportutilsExample): + type + Foo = object + f0: int # private + Goo*[T] = object + g0: int # private + proc initFoo*(): auto = Foo() + +proc privateAccess*(t: typedesc) {.magic: "PrivateAccess".} = + ## Enables access to private fields of `t` in current scope. + runnableExamples("-d:nimImportutilsExample"): + # here we're importing a module containing: + # type + # Foo = object + # f0: int # private + # Goo*[T] = object + # g0: int # private + # proc initFoo*(): auto = Foo() + var f = initFoo() + block: + assert not compiles(f.f0) + privateAccess(f.type) + f.f0 = 1 # accessible in this scope + block: + assert f.f0 == 1 # still in scope + assert not compiles(f.f0) + + # this also works with generics + privateAccess(Goo) + assert Goo[float](g0: 1).g0 == 1 diff --git a/lib/std/isolation.nim b/lib/std/isolation.nim index 8daca233b..b03e00651 100644 --- a/lib/std/isolation.nim +++ b/lib/std/isolation.nim @@ -15,10 +15,10 @@ ## type - Isolated*[T] = object ## Isolated data can only be moved, not copied. + Isolated*[T] {.sendable.} = object ## Isolated data can only be moved, not copied. value: T -proc `=`*[T](dest: var Isolated[T]; src: Isolated[T]) {.error.} +proc `=copy`*[T](dest: var Isolated[T]; src: Isolated[T]) {.error.} proc `=sink`*[T](dest: var Isolated[T]; src: Isolated[T]) {.inline.} = # delegate to value's sink operation @@ -38,9 +38,9 @@ func isolate*[T](value: sink T): Isolated[T] {.magic: "Isolate".} = func unsafeIsolate*[T](value: sink T): Isolated[T] = ## Creates an isolated subgraph from the expression `value`. - ## + ## ## .. warning:: The proc doesn't check whether `value` is isolated. - ## + ## Isolated[T](value: value) func extract*[T](src: var Isolated[T]): T = diff --git a/lib/std/jsbigints.nim b/lib/std/jsbigints.nim index ccf14080b..4e996ea7b 100644 --- a/lib/std/jsbigints.nim +++ b/lib/std/jsbigints.nim @@ -3,27 +3,36 @@ when not defined(js): {.fatal: "Module jsbigints is designed to be used with the JavaScript backend.".} +when defined(nimPreviewSlimSystem): + import std/assertions + type JsBigIntImpl {.importjs: "bigint".} = int # https://github.com/nim-lang/Nim/pull/16606 type JsBigInt* = distinct JsBigIntImpl ## Arbitrary precision integer for JavaScript target. func big*(integer: SomeInteger): JsBigInt {.importjs: "BigInt(#)".} = ## Constructor for `JsBigInt`. - when nimvm: doAssert false, "JsBigInt can not be used at compile-time nor static context" else: discard runnableExamples: doAssert big(1234567890) == big"1234567890" doAssert 0b1111100111.big == 0o1747.big and 0o1747.big == 999.big + when nimvm: raiseAssert "JsBigInt can not be used at compile-time nor static context" else: discard -func big*(integer: cstring): JsBigInt {.importjs: "BigInt(#)".} = +func `'big`*(num: cstring): JsBigInt {.importjs: "BigInt(#)".} = ## Constructor for `JsBigInt`. - when nimvm: doAssert false, "JsBigInt can not be used at compile-time nor static context" else: discard runnableExamples: - doAssert big"-1" == big"1" - big"2" + doAssert -1'big == 1'big - 2'big # supports decimal, binary, octal, hex: - doAssert big"12" == 12.big - doAssert big"0b101" == 0b101.big - doAssert big"0o701" == 0o701.big - doAssert big"0xdeadbeaf" == 0xdeadbeaf.big - doAssert big"0xffffffffffffffff" == (1.big shl 64.big) - 1.big + doAssert -12'big == big"-12" + doAssert 12'big == 12.big + doAssert 0b101'big == 0b101.big + doAssert 0o701'big == 0o701.big + doAssert 0xdeadbeaf'big == 0xdeadbeaf.big + doAssert 0xffffffffffffffff'big == (1'big shl 64'big) - 1'big + doAssert not compiles(static(12'big)) + when nimvm: raiseAssert "JsBigInt can not be used at compile-time nor static context" else: discard + +func big*(integer: cstring): JsBigInt {.importjs: "BigInt(#)".} = + ## Alias for `'big` + when nimvm: raiseAssert "JsBigInt can not be used at compile-time nor static context" else: discard func toCstring*(this: JsBigInt; radix: 2..36): cstring {.importjs: "#.toString(#)".} = ## Converts from `JsBigInt` to `cstring` representation. @@ -55,10 +64,10 @@ func wrapToUint*(this: JsBigInt; bits: Natural): JsBigInt {.importjs: runnableExamples: doAssert (big("3") + big("2") ** big("66")).wrapToUint(66) == big("3") -func toNumber*(this: JsBigInt): BiggestInt {.importjs: "Number(#)".} = +func toNumber*(this: JsBigInt): int {.importjs: "Number(#)".} = ## Does not do any bounds check and may or may not return an inexact representation. runnableExamples: - doAssert toNumber(big"2147483647") == 2147483647.BiggestInt + doAssert toNumber(big"2147483647") == 2147483647.int func `+`*(x, y: JsBigInt): JsBigInt {.importjs: "(# $1 #)".} = runnableExamples: @@ -101,7 +110,7 @@ func `==`*(x, y: JsBigInt): bool {.importjs: "(# == #)".} = doAssert big"42" == big"42" func `**`*(x, y: JsBigInt): JsBigInt {.importjs: "((#) $1 #)".} = - # (#) needed, refs https://github.com/nim-lang/Nim/pull/16409#issuecomment-760550812 + # (#) needed due to unary minus runnableExamples: doAssert big"2" ** big"64" == big"18446744073709551616" doAssert big"-2" ** big"3" == big"-8" @@ -111,8 +120,6 @@ func `**`*(x, y: JsBigInt): JsBigInt {.importjs: "((#) $1 #)".} = try: discard big"2" ** big"-1" # raises foreign `RangeError` except: ok = true doAssert ok - # pending https://github.com/nim-lang/Nim/pull/15940, simplify to: - # doAssertRaises: discard big"2" ** big"-1" # raises foreign `RangeError` func `and`*(x, y: JsBigInt): JsBigInt {.importjs: "(# & #)".} = runnableExamples: diff --git a/lib/std/jsfetch.nim b/lib/std/jsfetch.nim index 36bc772d5..219594619 100644 --- a/lib/std/jsfetch.nim +++ b/lib/std/jsfetch.nim @@ -1,122 +1,123 @@ ## - Fetch for the JavaScript target: https://developer.mozilla.org/docs/Web/API/Fetch_API -## .. Note:: jsfetch is Experimental. jsfetch module requires `-d:nimExperimentalJsfetch` when not defined(js): {.fatal: "Module jsfetch is designed to be used with the JavaScript backend.".} -when defined(nimExperimentalJsfetch) or defined(nimdoc): - import std/[asyncjs, jsheaders, jsformdata] - from std/httpcore import HttpMethod - from std/jsffi import JsObject - - type - FetchOptions* = ref object of JsRoot ## Options for Fetch API. - keepalive*: bool - metod* {.importjs: "method".}: cstring - body*, integrity*, referrer*, mode*, credentials*, cache*, redirect*, referrerPolicy*: cstring - - FetchModes* = enum ## Mode options. - fmCors = "cors" - fmNoCors = "no-cors" - fmSameOrigin = "same-origin" - - FetchCredentials* = enum ## Credential options. See https://developer.mozilla.org/en-US/docs/Web/API/Request/credentials - fcInclude = "include" - fcSameOrigin = "same-origin" - fcOmit = "omit" - - FetchCaches* = enum ## https://developer.mozilla.org/docs/Web/API/Request/cache - fchDefault = "default" - fchNoStore = "no-store" - fchReload = "reload" - fchNoCache = "no-cache" - fchForceCache = "force-cache" - - FetchRedirects* = enum ## Redirects options. - frFollow = "follow" - frError = "error" - frManual = "manual" - - FetchReferrerPolicies* = enum ## Referrer Policy options. - frpNoReferrer = "no-referrer" - frpNoReferrerWhenDowngrade = "no-referrer-when-downgrade" - frpOrigin = "origin" - frpOriginWhenCrossOrigin = "origin-when-cross-origin" - frpUnsafeUrl = "unsafe-url" - - Body* = ref object of JsRoot ## https://developer.mozilla.org/en-US/docs/Web/API/Body - bodyUsed*: bool - - Response* = ref object of JsRoot ## https://developer.mozilla.org/en-US/docs/Web/API/Response - bodyUsed*, ok*, redirected*: bool - typ* {.importjs: "type".}: cstring - url*, statusText*: cstring - status*: cint - headers*: Headers - body*: Body - - Request* = ref object of JsRoot ## https://developer.mozilla.org/en-US/docs/Web/API/Request - bodyUsed*, ok*, redirected*: bool - typ* {.importjs: "type".}: cstring - url*, statusText*: cstring - status*: cint - headers*: Headers - body*: Body - - - func newResponse*(body: cstring | FormData): Response {.importjs: "(new Response(#))".} - ## Constructor for `Response`. This does *not* call `fetch()`. Same as `new Response()`. - - func newRequest*(url: cstring): Request {.importjs: "(new Request(#))".} - ## Constructor for `Request`. This does *not* call `fetch()`. Same as `new Request()`. - - func clone*(self: Response | Request): Response {.importjs: "#.$1()".} - ## https://developer.mozilla.org/en-US/docs/Web/API/Response/clone - - proc text*(self: Response): Future[cstring] {.importjs: "#.$1()".} - ## https://developer.mozilla.org/en-US/docs/Web/API/Body/text - - proc json*(self: Response): Future[JsObject] {.importjs: "#.$1()".} - ## https://developer.mozilla.org/en-US/docs/Web/API/Body/json - - proc formData*(self: Body): Future[FormData] {.importjs: "#.$1()".} - ## https://developer.mozilla.org/en-US/docs/Web/API/Body/formData - - proc unsafeNewFetchOptions*(metod, body, mode, credentials, cache, referrerPolicy: cstring; - keepalive: bool; redirect = "follow".cstring; referrer = "client".cstring; integrity = "".cstring): FetchOptions {.importjs: - "{method: #, body: #, mode: #, credentials: #, cache: #, referrerPolicy: #, keepalive: #, redirect: #, referrer: #, integrity: #}".} - ## .. warning:: Unsafe `newfetchOptions`. - - func newfetchOptions*(metod: HttpMethod; body: cstring; - mode: FetchModes; credentials: FetchCredentials; cache: FetchCaches; referrerPolicy: FetchReferrerPolicies; - keepalive: bool; redirect = frFollow; referrer = "client".cstring; integrity = "".cstring): FetchOptions = - ## Constructor for `FetchOptions`. - result = FetchOptions( - body: body, mode: $mode, credentials: $credentials, cache: $cache, referrerPolicy: $referrerPolicy, - keepalive: keepalive, redirect: $redirect, referrer: referrer, integrity: integrity, - metod: (case metod - of HttpHead: "HEAD".cstring - of HttpGet: "GET".cstring - of HttpPost: "POST".cstring - of HttpPut: "PUT".cstring - of HttpDelete: "DELETE".cstring - of HttpPatch: "PATCH".cstring - else: "GET".cstring - ) +import std/[asyncjs, jsformdata, jsheaders] +export jsformdata, jsheaders +from std/httpcore import HttpMethod +from std/jsffi import JsObject + +type + FetchOptions* = ref object of JsRoot ## Options for Fetch API. + keepalive*: bool + metod* {.importjs: "method".}: cstring + body*, integrity*, referrer*, mode*, credentials*, cache*, redirect*, referrerPolicy*: cstring + headers*: Headers + + FetchModes* = enum ## Mode options. + fmCors = "cors" + fmNoCors = "no-cors" + fmSameOrigin = "same-origin" + + FetchCredentials* = enum ## Credential options. See https://developer.mozilla.org/en-US/docs/Web/API/Request/credentials + fcInclude = "include" + fcSameOrigin = "same-origin" + fcOmit = "omit" + + FetchCaches* = enum ## https://developer.mozilla.org/docs/Web/API/Request/cache + fchDefault = "default" + fchNoStore = "no-store" + fchReload = "reload" + fchNoCache = "no-cache" + fchForceCache = "force-cache" + + FetchRedirects* = enum ## Redirects options. + frFollow = "follow" + frError = "error" + frManual = "manual" + + FetchReferrerPolicies* = enum ## Referrer Policy options. + frpNoReferrer = "no-referrer" + frpNoReferrerWhenDowngrade = "no-referrer-when-downgrade" + frpOrigin = "origin" + frpOriginWhenCrossOrigin = "origin-when-cross-origin" + frpUnsafeUrl = "unsafe-url" + + Response* = ref object of JsRoot ## https://developer.mozilla.org/en-US/docs/Web/API/Response + bodyUsed*, ok*, redirected*: bool + typ* {.importjs: "type".}: cstring + url*, statusText*: cstring + status*: cint + headers*: Headers + body*: cstring + + Request* = ref object of JsRoot ## https://developer.mozilla.org/en-US/docs/Web/API/Request + bodyUsed*, ok*, redirected*: bool + typ* {.importjs: "type".}: cstring + url*, statusText*: cstring + status*: cint + headers*: Headers + body*: cstring + +func newResponse*(body: cstring | FormData): Response {.importjs: "(new Response(#))".} + ## Constructor for `Response`. This does *not* call `fetch()`. Same as `new Response()`. + +func newRequest*(url: cstring): Request {.importjs: "(new Request(#))".} + ## Constructor for `Request`. This does *not* call `fetch()`. Same as `new Request()`. + +func newRequest*(url: cstring; fetchOptions: FetchOptions): Request {.importjs: "(new Request(#, #))".} + ## Constructor for `Request` with `fetchOptions`. Same as `fetch(url, fetchOptions)`. + +func clone*(self: Response | Request): Response {.importjs: "#.$1()".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Response/clone + +proc text*(self: Response): Future[cstring] {.importjs: "#.$1()".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Response/text + +proc json*(self: Response): Future[JsObject] {.importjs: "#.$1()".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Response/json + +proc formData*(self: Response): Future[FormData] {.importjs: "#.$1()".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Response/formData + +proc unsafeNewFetchOptions*(metod, body, mode, credentials, cache, referrerPolicy: cstring; + keepalive: bool; redirect = "follow".cstring; referrer = "client".cstring; integrity = "".cstring; headers: Headers = newHeaders()): FetchOptions {.importjs: + "{method: #, body: #, mode: #, credentials: #, cache: #, referrerPolicy: #, keepalive: #, redirect: #, referrer: #, integrity: #, headers: #}".} + ## .. warning:: Unsafe `newfetchOptions`. + +func newfetchOptions*(metod = HttpGet; body: cstring = nil; + mode = fmCors; credentials = fcSameOrigin; cache = fchDefault; referrerPolicy = frpNoReferrerWhenDowngrade; + keepalive = false; redirect = frFollow; referrer = "client".cstring; integrity = "".cstring, + headers: Headers = newHeaders()): FetchOptions = + ## Constructor for `FetchOptions`. + result = FetchOptions( + body: if metod notin {HttpHead, HttpGet}: body else: nil, + mode: cstring($mode), credentials: cstring($credentials), cache: cstring($cache), referrerPolicy: cstring($referrerPolicy), + keepalive: keepalive, redirect: cstring($redirect), referrer: referrer, integrity: integrity, headers: headers, + metod: (case metod + of HttpHead: "HEAD".cstring + of HttpGet: "GET".cstring + of HttpPost: "POST".cstring + of HttpPut: "PUT".cstring + of HttpDelete: "DELETE".cstring + of HttpPatch: "PATCH".cstring + else: "GET".cstring ) + ) - proc fetch*(url: cstring | Request): Future[Response] {.importjs: "$1(#)".} - ## `fetch()` API, simple `GET` only, returns a `Future[Response]`. +proc fetch*(url: cstring | Request): Future[Response] {.importjs: "$1(#)".} + ## `fetch()` API, simple `GET` only, returns a `Future[Response]`. - proc fetch*(url: cstring | Request; options: FetchOptions): Future[Response] {.importjs: "$1(#, #)".} - ## `fetch()` API that takes a `FetchOptions`, returns a `Future[Response]`. +proc fetch*(url: cstring | Request; options: FetchOptions): Future[Response] {.importjs: "$1(#, #)".} + ## `fetch()` API that takes a `FetchOptions`, returns a `Future[Response]`. - func toCstring*(self: Request | Response | Body | FetchOptions): cstring {.importjs: "JSON.stringify(#)".} +func toCstring*(self: Request | Response | FetchOptions): cstring {.importjs: "JSON.stringify(#)".} - func `$`*(self: Request | Response | Body | FetchOptions): string = $toCstring(self) +func `$`*(self: Request | Response | FetchOptions): string = $toCstring(self) -runnableExamples("-d:nimExperimentalJsfetch -r:off"): - import std/[asyncjs, jsconsole, jsheaders, jsformdata] +runnableExamples("-r:off"): + import std/[asyncjs, jsconsole, jsformdata, jsheaders] from std/httpcore import HttpMethod from std/jsffi import JsObject from std/sugar import `=>` @@ -132,7 +133,8 @@ runnableExamples("-d:nimExperimentalJsfetch -r:off"): keepalive = false, redirect = "follow".cstring, referrer = "client".cstring, - integrity = "".cstring + integrity = "".cstring, + headers = newHeaders() ) assert options0.keepalive == false assert options0.metod == "POST".cstring @@ -144,6 +146,7 @@ runnableExamples("-d:nimExperimentalJsfetch -r:off"): assert options0.redirect == "follow".cstring assert options0.referrer == "client".cstring assert options0.integrity == "".cstring + assert options0.headers.len == 0 block: let options1: FetchOptions = newFetchOptions( @@ -156,7 +159,8 @@ runnableExamples("-d:nimExperimentalJsfetch -r:off"): keepalive = false, redirect = frFollow, referrer = "client".cstring, - integrity = "".cstring + integrity = "".cstring, + headers = newHeaders() ) assert options1.keepalive == false assert options1.metod == $HttpPost @@ -168,6 +172,7 @@ runnableExamples("-d:nimExperimentalJsfetch -r:off"): assert options1.redirect == $frFollow assert options1.referrer == "client".cstring assert options1.integrity == "".cstring + assert options1.headers.len == 0 block: let response: Response = newResponse(body = "-. .. --".cstring) @@ -183,16 +188,15 @@ runnableExamples("-d:nimExperimentalJsfetch -r:off"): assert response.ok assert response.status == 200.cint assert response.headers is Headers - assert response.body is Body + assert response.body is cstring discard example() - when defined(nimExperimentalAsyncjsThen): - block: - proc example2 {.async.} = - await fetch("https://api.github.com/users/torvalds".cstring) - .then((response: Response) => response.json()) - .then((json: JsObject) => console.log(json)) - .catch((err: Error) => console.log("Request Failed", err)) + block: + proc example2 {.async.} = + await fetch("https://api.github.com/users/torvalds".cstring) + .then((response: Response) => response.json()) + .then((json: JsObject) => console.log(json)) + .catch((err: Error) => console.log("Request Failed", err)) - discard example2() + discard example2() diff --git a/lib/std/jsformdata.nim b/lib/std/jsformdata.nim index 120f8742d..61dcc39a3 100644 --- a/lib/std/jsformdata.nim +++ b/lib/std/jsformdata.nim @@ -2,17 +2,21 @@ when not defined(js): {.fatal: "Module jsformdata is designed to be used with the JavaScript backend.".} +from std/dom import Blob + type FormData* = ref object of JsRoot ## FormData API. func newFormData*(): FormData {.importjs: "new FormData()".} -func add*(self: FormData; name: cstring; value: SomeNumber | bool | cstring) {.importjs: "#.append(#, #)".} +func add*(self: FormData; name: cstring; value: SomeNumber | bool | cstring | Blob) {.importjs: "#.append(#, #)".} ## https://developer.mozilla.org/en-US/docs/Web/API/FormData/append - ## Duplicate keys are allowed and order is preserved. + ## + ## .. hint:: Duplicate keys are allowed and order is preserved. -func add*(self: FormData; name: cstring; value: SomeNumber | bool | cstring, filename: cstring) {.importjs: "#.append(#, #, #)".} +func add*(self: FormData; name: cstring; value: SomeNumber | bool | cstring | Blob; filename: cstring) {.importjs: "#.append(#, #, #)".} ## https://developer.mozilla.org/en-US/docs/Web/API/FormData/append - ## Duplicate keys are allowed and order is preserved. + ## + ## .. hint:: Duplicate keys are allowed and order is preserved. func delete*(self: FormData; name: cstring) {.importjs: "#.$1(#)".} ## https://developer.mozilla.org/en-US/docs/Web/API/FormData/delete @@ -34,10 +38,10 @@ func values*(self: FormData): seq[cstring] {.importjs: "Array.from(#.$1())".} func pairs*(self: FormData): seq[tuple[key, val: cstring]] {.importjs: "Array.from(#.entries())".} ## https://developer.mozilla.org/en-US/docs/Web/API/FormData/entries -func put*(self: FormData; name, value, filename: cstring) {.importjs: "#.set(#, #, #)".} +func put*(self: FormData; name: cstring; value: SomeNumber | bool | cstring | Blob; filename: cstring) {.importjs: "#.set(#, #, #)".} ## https://developer.mozilla.org/en-US/docs/Web/API/FormData/set -func `[]=`*(self: FormData; name, value: cstring) {.importjs: "#.set(#, #)".} +func `[]=`*(self: FormData; name: cstring; value: SomeNumber | bool | cstring | Blob) {.importjs: "#.set(#, #)".} ## https://developer.mozilla.org/en-US/docs/Web/API/FormData/set func `[]`*(self: FormData; name: cstring): cstring {.importjs: "#.get(#)".} diff --git a/lib/std/jsonutils.nim b/lib/std/jsonutils.nim index fa61d79db..2d28748ce 100644 --- a/lib/std/jsonutils.nim +++ b/lib/std/jsonutils.nim @@ -1,5 +1,5 @@ ##[ -This module implements a hookable (de)serialization for arbitrary types. +This module implements a hookable (de)serialization for arbitrary types using JSON. Design goal: avoid importing modules where a custom serialization is needed; see strtabs.fromJsonHook,toJsonHook for an example. ]## @@ -11,9 +11,12 @@ runnableExamples: z1: int8 let a = (1.5'f32, (b: "b2", a: "a2"), 'x', @[Foo(t: true, z1: -3), nil], [{"name": "John"}.newStringTable]) let j = a.toJson - doAssert j.jsonTo(typeof(a)).toJson == j + assert j.jsonTo(typeof(a)).toJson == j + assert $[NaN, Inf, -Inf, 0.0, -0.0, 1.0, 1e-2].toJson == """["nan","inf","-inf",0.0,-0.0,1.0,0.01]""" + assert 0.0.toJson.kind == JFloat + assert Inf.toJson.kind == JString -import std/[json,strutils,tables,sets,strtabs,options] +import std/[json, strutils, tables, sets, strtabs, options, strformat] #[ Future directions: @@ -28,9 +31,17 @@ add a way to customize serialization, for e.g.: ]# import std/macros +from std/enumutils import symbolName +from std/typetraits import OrdinalEnum, tupleLen + +when defined(nimPreviewSlimSystem): + import std/assertions + + +proc isNamedTuple(T: typedesc): bool {.magic: "TypeTrait".} type - Joptions* = object + Joptions* = object # xxx rename FromJsonOptions ## Options controlling the behavior of `fromJson`. allowExtraKeys*: bool ## If `true` Nim's object to which the JSON is parsed is not required to @@ -39,10 +50,25 @@ type ## If `true` Nim's object to which JSON is parsed is allowed to have ## fields without corresponding JSON keys. # in future work: a key rename could be added - -proc isNamedTuple(T: typedesc): bool {.magic: "TypeTrait".} -proc distinctBase(T: typedesc): typedesc {.magic: "TypeTrait".} -template distinctBase[T](a: T): untyped = distinctBase(typeof(a))(a) + EnumMode* = enum + joptEnumOrd + joptEnumSymbol + joptEnumString + JsonNodeMode* = enum ## controls `toJson` for JsonNode types + joptJsonNodeAsRef ## returns the ref as is + joptJsonNodeAsCopy ## returns a deep copy of the JsonNode + joptJsonNodeAsObject ## treats JsonNode as a regular ref object + ToJsonOptions* = object + enumMode*: EnumMode + jsonNodeMode*: JsonNodeMode + # xxx charMode, etc + +proc initToJsonOptions*(): ToJsonOptions = + ## initializes `ToJsonOptions` with sane options. + ToJsonOptions(enumMode: joptEnumOrd, jsonNodeMode: joptJsonNodeAsRef) + +proc distinctBase(T: typedesc, recursive: static bool = true): typedesc {.magic: "TypeTrait".} +template distinctBase[T](a: T, recursive: static bool = true): untyped = distinctBase(typeof(a), recursive)(a) macro getDiscriminants(a: typedesc): seq[string] = ## return the discriminant keys @@ -52,19 +78,24 @@ macro getDiscriminants(a: typedesc): seq[string] = let sym = a[1] let t = sym.getTypeImpl let t2 = t[2] - doAssert t2.kind == nnkRecList - result = newTree(nnkBracket) - for ti in t2: - if ti.kind == nnkRecCase: - let key = ti[0][0] - let typ = ti[0][1] - result.add newLit key.strVal - if result.len > 0: + case t2.kind + of nnkEmpty: # allow empty objects result = quote do: - @`result` + seq[string].default + of nnkRecList: + result = newTree(nnkBracket) + for ti in t2: + if ti.kind == nnkRecCase: + let key = ti[0][0] + result.add newLit key.strVal + if result.len > 0: + result = quote do: + @`result` + else: + result = quote do: + seq[string].default else: - result = quote do: - seq[string].default + raiseAssert "unexpected kind: " & $t2.kind macro initCaseObject(T: typedesc, fun: untyped): untyped = ## does the minimum to construct a valid case object, only initializing @@ -78,7 +109,7 @@ macro initCaseObject(T: typedesc, fun: untyped): untyped = case t.kind of nnkObjectTy: t2 = t[2] of nnkRefTy: t2 = t[0].getTypeImpl[2] - else: doAssert false, $t.kind # xxx `nnkPtrTy` could be handled too + else: raiseAssert $t.kind # xxx `nnkPtrTy` could be handled too doAssert t2.kind == nnkRecList result = newTree(nnkObjConstr) result.add sym @@ -91,14 +122,14 @@ macro initCaseObject(T: typedesc, fun: untyped): untyped = `fun`(`key2`, typedesc[`typ`]) result.add newTree(nnkExprColonExpr, key, val) -proc checkJsonImpl(cond: bool, condStr: string, msg = "") = - if not cond: - # just pick 1 exception type for simplicity; other choices would be: - # JsonError, JsonParser, JsonKindError - raise newException(ValueError, msg) +proc raiseJsonException(condStr: string, msg: string) {.noinline.} = + # just pick 1 exception type for simplicity; other choices would be: + # JsonError, JsonParser, JsonKindError + raise newException(ValueError, condStr & " failed: " & msg) template checkJson(cond: untyped, msg = "") = - checkJsonImpl(cond, astToStr(cond), msg) + if not cond: + raiseJsonException(astToStr(cond), msg) proc hasField[T](obj: T, field: string): bool = for k, _ in fieldPairs(obj): @@ -106,7 +137,7 @@ proc hasField[T](obj: T, field: string): bool = return true return false -macro accessField(obj: typed, name: static string): untyped = +macro accessField(obj: typed, name: static string): untyped = newDotExpr(obj, ident(name)) template fromJsonFields(newObj, oldObj, json, discKeys, opt) = @@ -128,7 +159,7 @@ template fromJsonFields(newObj, oldObj, json, discKeys, opt) = if discKeys.len == 0 or hasField(oldObj, key): val = accessField(oldObj, key) else: - checkJson false, $($T, key, json) + checkJson false, "key '$1' for $2 not in $3" % [key, $T, json.pretty()] else: if json.hasKey key: numMatched.inc @@ -146,8 +177,8 @@ template fromJsonFields(newObj, oldObj, json, discKeys, opt) = json.len == numMatched else: json.len == num and num == numMatched - - checkJson ok, $(json.len, num, numMatched, $T, json) + + checkJson ok, "There were $1 keys (expecting $2) for $3 with $4" % [$json.len, $num, $T, json.pretty()] proc fromJson*[T](a: var T, b: JsonNode, opt = Joptions()) @@ -180,23 +211,24 @@ proc fromJson*[T](a: var T, b: JsonNode, opt = Joptions()) = adding "json path" leading to `b` can be added in future work. ]# checkJson b != nil, $($T, b) - when compiles(fromJsonHook(a, b)): fromJsonHook(a, b) + when compiles(fromJsonHook(a, b, opt)): fromJsonHook(a, b, opt) + elif compiles(fromJsonHook(a, b)): fromJsonHook(a, b) elif T is bool: a = to(b,T) elif T is enum: case b.kind of JInt: a = T(b.getBiggestInt()) of JString: a = parseEnum[T](b.getStr()) - else: checkJson false, $($T, " ", b) + else: checkJson false, fmt"Expecting int/string for {$T} got {b.pretty()}" elif T is uint|uint64: a = T(to(b, uint64)) elif T is Ordinal: a = cast[T](to(b, int)) elif T is pointer: a = cast[pointer](to(b, int)) - elif T is distinct: - when nimvm: - # bug, potentially related to https://github.com/nim-lang/Nim/issues/12282 - a = T(jsonTo(b, distinctBase(T))) - else: - a.distinctBase.fromJson(b) + elif T is distinct: a.distinctBase.fromJson(b) elif T is string|SomeNumber: a = to(b,T) + elif T is cstring: + case b.kind + of JNull: a = nil + of JString: a = b.str + else: checkJson false, fmt"Expecting null/string for {$T} got {b.pretty()}" elif T is JsonNode: a = b elif T is ref | ptr: if b.kind == JNull: a = nil @@ -204,11 +236,15 @@ proc fromJson*[T](a: var T, b: JsonNode, opt = Joptions()) = a = T() fromJson(a[], b, opt) elif T is array: - checkJson a.len == b.len, $(a.len, b.len, $T) + checkJson a.len == b.len, fmt"Json array size doesn't match for {$T}" var i = 0 for ai in mitems(a): fromJson(ai, b[i], opt) i.inc + elif T is set: + type E = typeof(for ai in a: ai) + for val in b.getElems: + incl a, jsonTo(val, E) elif T is seq: a.setLen b.len for i, val in b.getElems: @@ -236,49 +272,83 @@ proc fromJson*[T](a: var T, b: JsonNode, opt = Joptions()) = fromJsonFields(a, nil, b, seq[string].default, opt) else: checkJson b.kind == JArray, $(b.kind) # we could customize whether to allow JNull + + when compiles(tupleLen(T)): + let tupleSize = tupleLen(T) + else: + # Tuple len isn't in csources_v1 so using tupleLen would fail. + # Else branch basically never runs (tupleLen added in 1.1 and jsonutils in 1.4), but here for consistency + var tupleSize = 0 + for val in fields(a): + tupleSize.inc + + checkJson b.len == tupleSize, fmt"Json doesn't match expected length of {tupleSize}, got {b.pretty()}" var i = 0 for val in fields(a): fromJson(val, b[i], opt) i.inc - checkJson b.len == i, $(b.len, i, $T, b) # could customize else: # checkJson not appropriate here - static: doAssert false, "not yet implemented: " & $T + static: raiseAssert "not yet implemented: " & $T proc jsonTo*(b: JsonNode, T: typedesc, opt = Joptions()): T = ## reverse of `toJson` fromJson(result, b, opt) -proc toJson*[T](a: T): JsonNode = +proc toJson*[T](a: T, opt = initToJsonOptions()): JsonNode = ## serializes `a` to json; uses `toJsonHook(a: T)` if it's in scope to ## customize serialization, see strtabs.toJsonHook for an example. - when compiles(toJsonHook(a)): result = toJsonHook(a) + ## + ## .. note:: With `-d:nimPreviewJsonutilsHoleyEnum`, `toJson` now can + ## serialize/deserialize holey enums as regular enums (via `ord`) instead of as strings. + ## It is expected that this behavior becomes the new default in upcoming versions. + when compiles(toJsonHook(a, opt)): result = toJsonHook(a, opt) + elif compiles(toJsonHook(a)): result = toJsonHook(a) elif T is object | tuple: when T is object or isNamedTuple(T): result = newJObject() - for k, v in a.fieldPairs: result[k] = toJson(v) + for k, v in a.fieldPairs: result[k] = toJson(v, opt) else: result = newJArray() - for v in a.fields: result.add toJson(v) + for v in a.fields: result.add toJson(v, opt) elif T is ref | ptr: - if system.`==`(a, nil): result = newJNull() - else: result = toJson(a[]) - elif T is array | seq: + template impl = + if system.`==`(a, nil): result = newJNull() + else: result = toJson(a[], opt) + when T is JsonNode: + case opt.jsonNodeMode + of joptJsonNodeAsRef: result = a + of joptJsonNodeAsCopy: result = copy(a) + of joptJsonNodeAsObject: impl() + else: impl() + elif T is array | seq | set: result = newJArray() - for ai in a: result.add toJson(ai) - elif T is pointer: result = toJson(cast[int](a)) + for ai in a: result.add toJson(ai, opt) + elif T is pointer: result = toJson(cast[int](a), opt) # edge case: `a == nil` could've also led to `newJNull()`, but this results # in simpler code for `toJson` and `fromJson`. - elif T is distinct: result = toJson(a.distinctBase) + elif T is distinct: result = toJson(a.distinctBase, opt) elif T is bool: result = %(a) elif T is SomeInteger: result = %a + elif T is enum: + case opt.enumMode + of joptEnumOrd: + when T is Ordinal or defined(nimPreviewJsonutilsHoleyEnum): %(a.ord) + else: toJson($a, opt) + of joptEnumSymbol: + when T is OrdinalEnum: + toJson(symbolName(a), opt) + else: + toJson($a, opt) + of joptEnumString: toJson($a, opt) elif T is Ordinal: result = %(a.ord) + elif T is cstring: (if a == nil: result = newJNull() else: result = % $a) else: result = %a -proc fromJsonHook*[K, V](t: var (Table[K, V] | OrderedTable[K, V]), - jsonNode: JsonNode) = +proc fromJsonHook*[K: string|cstring, V](t: var (Table[K, V] | OrderedTable[K, V]), + jsonNode: JsonNode, opt = Joptions()) = ## Enables `fromJson` for `Table` and `OrderedTable` types. - ## + ## ## See also: ## * `toJsonHook proc<#toJsonHook>`_ runnableExamples: @@ -294,27 +364,32 @@ proc fromJsonHook*[K, V](t: var (Table[K, V] | OrderedTable[K, V]), "type is `" & $jsonNode.kind & "`." clear(t) for k, v in jsonNode: - t[k] = jsonTo(v, V) + t[k] = jsonTo(v, V, opt) -proc toJsonHook*[K, V](t: (Table[K, V] | OrderedTable[K, V])): JsonNode = +proc toJsonHook*[K: string|cstring, V](t: (Table[K, V] | OrderedTable[K, V]), opt = initToJsonOptions()): JsonNode = ## Enables `toJson` for `Table` and `OrderedTable` types. ## ## See also: ## * `fromJsonHook proc<#fromJsonHook,,JsonNode>`_ runnableExamples: - import std/[tables, json] + import std/[tables, json, sugar] let foo = ( t: [("two", 2)].toTable, ot: [("one", 1), ("three", 3)].toOrderedTable) assert $toJson(foo) == """{"t":{"two":2},"ot":{"one":1,"three":3}}""" + # if keys are not string|cstring, you can use this: + let a = {10: "foo", 11: "bar"}.newOrderedTable + let a2 = collect: (for k,v in a: (k,v)) + assert $toJson(a2) == """[[10,"foo"],[11,"bar"]]""" result = newJObject() for k, v in pairs(t): - result[k] = toJson(v) + # not sure if $k has overhead for string + result[(when K is string: k else: $k)] = toJson(v, opt) -proc fromJsonHook*[A](s: var SomeSet[A], jsonNode: JsonNode) = +proc fromJsonHook*[A](s: var SomeSet[A], jsonNode: JsonNode, opt = Joptions()) = ## Enables `fromJson` for `HashSet` and `OrderedSet` types. - ## + ## ## See also: ## * `toJsonHook proc<#toJsonHook,SomeSet[A]>`_ runnableExamples: @@ -330,9 +405,9 @@ proc fromJsonHook*[A](s: var SomeSet[A], jsonNode: JsonNode) = "type is `" & $jsonNode.kind & "`." clear(s) for v in jsonNode: - incl(s, jsonTo(v, A)) + incl(s, jsonTo(v, A, opt)) -proc toJsonHook*[A](s: SomeSet[A]): JsonNode = +proc toJsonHook*[A](s: SomeSet[A], opt = initToJsonOptions()): JsonNode = ## Enables `toJson` for `HashSet` and `OrderedSet` types. ## ## See also: @@ -344,11 +419,11 @@ proc toJsonHook*[A](s: SomeSet[A]): JsonNode = result = newJArray() for k in s: - add(result, toJson(k)) + add(result, toJson(k, opt)) -proc fromJsonHook*[T](self: var Option[T], jsonNode: JsonNode) = +proc fromJsonHook*[T](self: var Option[T], jsonNode: JsonNode, opt = Joptions()) = ## Enables `fromJson` for `Option` types. - ## + ## ## See also: ## * `toJsonHook proc<#toJsonHook,Option[T]>`_ runnableExamples: @@ -360,11 +435,11 @@ proc fromJsonHook*[T](self: var Option[T], jsonNode: JsonNode) = assert isNone(opt) if jsonNode.kind != JNull: - self = some(jsonTo(jsonNode, T)) + self = some(jsonTo(jsonNode, T, opt)) else: self = none[T]() -proc toJsonHook*[T](self: Option[T]): JsonNode = +proc toJsonHook*[T](self: Option[T], opt = initToJsonOptions()): JsonNode = ## Enables `toJson` for `Option` types. ## ## See also: @@ -377,13 +452,13 @@ proc toJsonHook*[T](self: Option[T]): JsonNode = assert $toJson(optNone) == "null" if isSome(self): - toJson(get(self)) + toJson(get(self), opt) else: newJNull() proc fromJsonHook*(a: var StringTableRef, b: JsonNode) = ## Enables `fromJson` for `StringTableRef` type. - ## + ## ## See also: ## * `toJsonHook proc<#toJsonHook,StringTableRef>`_ runnableExamples: @@ -401,7 +476,7 @@ proc fromJsonHook*(a: var StringTableRef, b: JsonNode) = proc toJsonHook*(a: StringTableRef): JsonNode = ## Enables `toJson` for `StringTableRef` type. - ## + ## ## See also: ## * `fromJsonHook proc<#fromJsonHook,StringTableRef,JsonNode>`_ runnableExamples: diff --git a/lib/std/monotimes.nim b/lib/std/monotimes.nim index 78736d719..bf6dc776b 100644 --- a/lib/std/monotimes.nim +++ b/lib/std/monotimes.nim @@ -10,17 +10,14 @@ ##[ The `std/monotimes` module implements monotonic timestamps. A monotonic timestamp represents the time that has passed since some system defined -point in time. The monotonic timestamps are guaranteed to always increase, +point in time. The monotonic timestamps are guaranteed not to decrease, meaning that that the following is guaranteed to work: ]## runnableExamples: - import std/os - let a = getMonoTime() - sleep(10) let b = getMonoTime() - assert a < b + assert a <= b ##[ This is not guaranteed for the `times.Time` type! This means that the @@ -79,6 +76,10 @@ when defined(js): elif defined(posix) and not defined(osx): import std/posix +when defined(zephyr): + proc k_uptime_ticks(): int64 {.importc: "k_uptime_ticks", header: "<kernel.h>".} + proc k_ticks_to_ns_floor64(ticks: int64): int64 {.importc: "k_ticks_to_ns_floor64", header: "<kernel.h>".} + elif defined(windows): proc QueryPerformanceCounter(res: var uint64) {. importc: "QueryPerformanceCounter", stdcall, dynlib: "kernel32".} @@ -101,6 +102,9 @@ proc getMonoTime*(): MonoTime {.tags: [TimeEffect].} = mach_timebase_info(machAbsoluteTimeFreq) result = MonoTime(ticks: ticks * machAbsoluteTimeFreq.numer div machAbsoluteTimeFreq.denom) + elif defined(zephyr): + let ticks = k_ticks_to_ns_floor64(k_uptime_ticks()) + result = MonoTime(ticks: ticks) elif defined(posix): var ts: Timespec discard clock_gettime(CLOCK_MONOTONIC, ts) diff --git a/lib/std/objectdollar.nim b/lib/std/objectdollar.nim new file mode 100644 index 000000000..86ce9afc8 --- /dev/null +++ b/lib/std/objectdollar.nim @@ -0,0 +1,13 @@ +## This module implements a generic `$` operator to convert objects to strings. + +import std/private/miscdollars + +proc `$`*[T: object](x: T): string = + ## Generic `$` operator for objects with similar output to + ## `$` for named tuples. + runnableExamples: + type Foo = object + a, b: int + let x = Foo(a: 23, b: 45) + assert $x == "(a: 23, b: 45)" + tupleObjectDollar(result, x) diff --git a/lib/std/oserrors.nim b/lib/std/oserrors.nim new file mode 100644 index 000000000..7b11c5e8e --- /dev/null +++ b/lib/std/oserrors.nim @@ -0,0 +1,117 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2022 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + + +## The `std/oserrors` module implements OS error reporting. + +type + OSErrorCode* = distinct int32 ## Specifies an OS Error Code. + +when not defined(nimscript): + when defined(windows): + import std/winlean + when defined(nimPreviewSlimSystem): + import std/widestrs + else: + var errno {.importc, header: "<errno.h>".}: cint + + proc c_strerror(errnum: cint): cstring {. + importc: "strerror", header: "<string.h>".} + +proc `==`*(err1, err2: OSErrorCode): bool {.borrow.} +proc `$`*(err: OSErrorCode): string {.borrow.} + +proc osErrorMsg*(errorCode: OSErrorCode): string = + ## Converts an OS error code into a human readable string. + ## + ## The error code can be retrieved using the `osLastError proc`_. + ## + ## If conversion fails, or `errorCode` is `0` then `""` will be + ## returned. + ## + ## See also: + ## * `raiseOSError proc`_ + ## * `osLastError proc`_ + runnableExamples: + when defined(linux): + assert osErrorMsg(OSErrorCode(0)) == "" + assert osErrorMsg(OSErrorCode(1)) == "Operation not permitted" + assert osErrorMsg(OSErrorCode(2)) == "No such file or directory" + + result = "" + when defined(nimscript): + discard + elif defined(windows): + if errorCode != OSErrorCode(0'i32): + var msgbuf: WideCString + if formatMessageW(0x00000100 or 0x00001000 or 0x00000200, + nil, errorCode.int32, 0, addr(msgbuf), 0, nil) != 0'i32: + result = $msgbuf + if msgbuf != nil: localFree(cast[pointer](msgbuf)) + else: + if errorCode != OSErrorCode(0'i32): + result = $c_strerror(errorCode.int32) + +proc newOSError*( + errorCode: OSErrorCode, additionalInfo = "" +): owned(ref OSError) {.noinline.} = + ## Creates a new `OSError exception <system.html#OSError>`_. + ## + ## The `errorCode` will determine the + ## message, `osErrorMsg proc`_ will be used + ## to get this message. + ## + ## The error code can be retrieved using the `osLastError proc`_. + ## + ## If the error code is `0` or an error message could not be retrieved, + ## the message `unknown OS error` will be used. + ## + ## See also: + ## * `osErrorMsg proc`_ + ## * `osLastError proc`_ + result = (ref OSError)(errorCode: errorCode.int32, msg: osErrorMsg(errorCode)) + if additionalInfo.len > 0: + if result.msg.len > 0 and result.msg[^1] != '\n': result.msg.add '\n' + result.msg.add "Additional info: " + result.msg.add additionalInfo + # don't add trailing `.` etc, which negatively impacts "jump to file" in IDEs. + if result.msg == "": + result.msg = "unknown OS error" + +proc raiseOSError*(errorCode: OSErrorCode, additionalInfo = "") {.noinline.} = + ## Raises an `OSError exception <system.html#OSError>`_. + ## + ## Read the description of the `newOSError proc`_ to learn + ## how the exception object is created. + raise newOSError(errorCode, additionalInfo) + +{.push stackTrace:off.} +proc osLastError*(): OSErrorCode {.sideEffect.} = + ## Retrieves the last operating system error code. + ## + ## This procedure is useful in the event when an OS call fails. In that case + ## this procedure will return the error code describing the reason why the + ## OS call failed. The `OSErrorMsg` procedure can then be used to convert + ## this code into a string. + ## + ## .. warning:: The behaviour of this procedure varies between Windows and POSIX systems. + ## On Windows some OS calls can reset the error code to `0` causing this + ## procedure to return `0`. It is therefore advised to call this procedure + ## immediately after an OS call fails. On POSIX systems this is not a problem. + ## + ## See also: + ## * `osErrorMsg proc`_ + ## * `raiseOSError proc`_ + when defined(nimscript): + discard + elif defined(windows): + result = cast[OSErrorCode](getLastError()) + else: + result = OSErrorCode(errno) +{.pop.} diff --git a/lib/std/outparams.nim b/lib/std/outparams.nim new file mode 100644 index 000000000..a471fbaa7 --- /dev/null +++ b/lib/std/outparams.nim @@ -0,0 +1,38 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2022 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## `outParamsAt` macro for easy writing code that works with both 2.0 and 1.x. + +import std/macros + +macro outParamsAt*(positions: static openArray[int]; n: untyped): untyped = + ## Use this macro to annotate `out` parameters in a portable way. + runnableExamples: + proc p(x: var int) {.outParamsAt: [1].} = + discard "x is really an 'out int' if the Nim compiler supports 'out' parameters" + + result = n + when defined(nimHasOutParams): + var p = n.params + for po in positions: + p[po][^2].expectKind nnkVarTy + p[po][^2] = newTree(nnkOutTy, p[po][^2][0]) + +when isMainModule: + {.experimental: "strictDefs".} + + proc main(x: var int) {.outParamsAt: [1].} = + x = 3 + + proc us = + var x: int + main x + echo x + + us() diff --git a/lib/std/packedsets.nim b/lib/std/packedsets.nim index 2b5896be0..3320558f2 100644 --- a/lib/std/packedsets.nim +++ b/lib/std/packedsets.nim @@ -12,17 +12,15 @@ ## ## Supports any Ordinal type. ## -## **Note**: Currently the assignment operator `=` for `PackedSet[A]` -## performs some rather meaningless shallow copy. Since Nim currently does -## not allow the assignment operator to be overloaded, use the `assign proc -## <#assign,PackedSet[A],PackedSet[A]>`_ to get a deep copy. -## ## See also ## ======== ## * `sets module <sets.html>`_ for more general hash sets import std/private/since -import hashes +import std/hashes + +when defined(nimPreviewSlimSystem): + import std/assertions type BitScalar = uint @@ -38,7 +36,7 @@ const IntMask = 1 shl IntShift - 1 type - Trunk = ref object + Trunk {.acyclic.} = ref object next: Trunk # all nodes are connected with this pointer key: int # start address at bit 0 bits: array[0..IntsPerTrunk - 1, BitScalar] # a bit vector @@ -111,7 +109,6 @@ proc intSetPut[A](t: var PackedSet[A], key: int): Trunk = t.data[h] = result proc bitincl[A](s: var PackedSet[A], key: int) {.inline.} = - var ret: Trunk var t = intSetPut(s, key shr TrunkShift) var u = key and TrunkMask t.bits[u shr IntShift] = t.bits[u shr IntShift] or @@ -201,6 +198,7 @@ proc contains*[A](s: PackedSet[A], key: A): bool = assert B notin letters if s.elems <= s.a.len: + result = false for i in 0..<s.elems: if s.a[i] == ord(key): return true else: @@ -409,18 +407,9 @@ proc isNil*[A](x: PackedSet[A]): bool {.inline.} = x.head.isNil and x.elems == 0 -proc assign*[A](dest: var PackedSet[A], src: PackedSet[A]) = +proc `=copy`*[A](dest: var PackedSet[A], src: PackedSet[A]) = ## Copies `src` to `dest`. ## `dest` does not need to be initialized by the `initPackedSet proc <#initPackedSet>`_. - runnableExamples: - var - a = initPackedSet[int]() - b = initPackedSet[int]() - b.incl(5) - b.incl(7) - a.assign(b) - assert len(a) == 2 - if src.elems <= src.a.len: dest.data = @[] dest.max = 0 @@ -449,6 +438,19 @@ proc assign*[A](dest: var PackedSet[A], src: PackedSet[A]) = dest.data[h] = n it = it.next +proc assign*[A](dest: var PackedSet[A], src: PackedSet[A]) {.inline, deprecated.} = + ## Copies `src` to `dest`. + ## `dest` does not need to be initialized by the `initPackedSet proc <#initPackedSet>`_. + runnableExamples: + var + a = initPackedSet[int]() + b = initPackedSet[int]() + b.incl(5) + b.incl(7) + a.assign(b) + assert len(a) == 2 + `=copy`(dest, src) + proc union*[A](s1, s2: PackedSet[A]): PackedSet[A] = ## Returns the union of the sets `s1` and `s2`. ## diff --git a/lib/std/paths.nim b/lib/std/paths.nim new file mode 100644 index 000000000..664dedd31 --- /dev/null +++ b/lib/std/paths.nim @@ -0,0 +1,302 @@ +## This module implements path handling. +## +## **See also:** +## * `files module <files.html>`_ for file access + +import std/private/osseps +export osseps + +import std/envvars +import std/private/osappdirs + +import std/[pathnorm, hashes, sugar, strutils] + +from std/private/ospaths2 import joinPath, splitPath, + ReadDirEffect, WriteDirEffect, + isAbsolute, relativePath, + normalizePathEnd, isRelativeTo, parentDir, + tailDir, isRootDir, parentDirs, `/../`, + extractFilename, lastPathPart, + changeFileExt, addFileExt, cmpPaths, splitFile, + unixToNativePath, absolutePath, normalizeExe, + normalizePath +export ReadDirEffect, WriteDirEffect + +type + Path* = distinct string + +func hash*(x: Path): Hash = + let x = x.string.dup(normalizePath) + if FileSystemCaseSensitive: + result = x.hash + else: + result = x.toLowerAscii.hash + +template `$`*(x: Path): string = + string(x) + +func `==`*(x, y: Path): bool {.inline.} = + ## Compares two paths. + ## + ## On a case-sensitive filesystem this is done + ## case-sensitively otherwise case-insensitively. + result = cmpPaths(x.string, y.string) == 0 + +template endsWith(a: string, b: set[char]): bool = + a.len > 0 and a[^1] in b + +func add(x: var string, tail: string) = + var state = 0 + let trailingSep = tail.endsWith({DirSep, AltSep}) or tail.len == 0 and x.endsWith({DirSep, AltSep}) + normalizePathEnd(x, trailingSep=false) + addNormalizePath(tail, x, state, DirSep) + normalizePathEnd(x, trailingSep=trailingSep) + +func add*(x: var Path, y: Path) {.borrow.} + +func `/`*(head, tail: Path): Path {.inline.} = + ## Joins two directory names to one. + ## + ## returns normalized path concatenation of `head` and `tail`, preserving + ## whether or not `tail` has a trailing slash (or, if tail if empty, whether + ## head has one). + ## + ## See also: + ## * `splitPath proc`_ + ## * `uri.combine proc <uri.html#combine,Uri,Uri>`_ + ## * `uri./ proc <uri.html#/,Uri,string>`_ + Path(joinPath(head.string, tail.string)) + +func splitPath*(path: Path): tuple[head, tail: Path] {.inline.} = + ## Splits a directory into `(head, tail)` tuple, so that + ## ``head / tail == path`` (except for edge cases like "/usr"). + ## + ## See also: + ## * `add proc`_ + ## * `/ proc`_ + ## * `/../ proc`_ + ## * `relativePath proc`_ + let res = splitPath(path.string) + result = (Path(res.head), Path(res.tail)) + +func splitFile*(path: Path): tuple[dir, name: Path, ext: string] {.inline.} = + ## Splits a filename into `(dir, name, extension)` tuple. + ## + ## `dir` does not end in DirSep unless it's `/`. + ## `extension` includes the leading dot. + ## + ## If `path` has no extension, `ext` is the empty string. + ## If `path` has no directory component, `dir` is the empty string. + ## If `path` has no filename component, `name` and `ext` are empty strings. + ## + ## See also: + ## * `extractFilename proc`_ + ## * `lastPathPart proc`_ + ## * `changeFileExt proc`_ + ## * `addFileExt proc`_ + let res = splitFile(path.string) + result = (Path(res.dir), Path(res.name), res.ext) + +func isAbsolute*(path: Path): bool {.inline, raises: [].} = + ## Checks whether a given `path` is absolute. + ## + ## On Windows, network paths are considered absolute too. + result = isAbsolute(path.string) + +proc relativePath*(path, base: Path, sep = DirSep): Path {.inline.} = + ## Converts `path` to a path relative to `base`. + ## + ## The `sep` (default: DirSep) is used for the path normalizations, + ## this can be useful to ensure the relative path only contains `'/'` + ## so that it can be used for URL constructions. + ## + ## On Windows, if a root of `path` and a root of `base` are different, + ## returns `path` as is because it is impossible to make a relative path. + ## That means an absolute path can be returned. + ## + ## See also: + ## * `splitPath proc`_ + ## * `parentDir proc`_ + ## * `tailDir proc`_ + result = Path(relativePath(path.string, base.string, sep)) + +proc isRelativeTo*(path: Path, base: Path): bool {.inline.} = + ## Returns true if `path` is relative to `base`. + result = isRelativeTo(path.string, base.string) + + +func parentDir*(path: Path): Path {.inline.} = + ## Returns the parent directory of `path`. + ## + ## This is similar to ``splitPath(path).head`` when ``path`` doesn't end + ## in a dir separator, but also takes care of path normalizations. + ## The remainder can be obtained with `lastPathPart(path) proc`_. + ## + ## See also: + ## * `relativePath proc`_ + ## * `splitPath proc`_ + ## * `tailDir proc`_ + ## * `parentDirs iterator`_ + result = Path(parentDir(path.string)) + +func tailDir*(path: Path): Path {.inline.} = + ## Returns the tail part of `path`. + ## + ## See also: + ## * `relativePath proc`_ + ## * `splitPath proc`_ + ## * `parentDir proc`_ + result = Path(tailDir(path.string)) + +func isRootDir*(path: Path): bool {.inline.} = + ## Checks whether a given `path` is a root directory. + result = isRootDir(path.string) + +iterator parentDirs*(path: Path, fromRoot=false, inclusive=true): Path = + ## Walks over all parent directories of a given `path`. + ## + ## If `fromRoot` is true (default: false), the traversal will start from + ## the file system root directory. + ## If `inclusive` is true (default), the original argument will be included + ## in the traversal. + ## + ## Relative paths won't be expanded by this iterator. Instead, it will traverse + ## only the directories appearing in the relative path. + ## + ## See also: + ## * `parentDir proc`_ + ## + for p in parentDirs(path.string, fromRoot, inclusive): + yield Path(p) + +func `/../`*(head, tail: Path): Path {.inline.} = + ## The same as ``parentDir(head) / tail``, unless there is no parent + ## directory. Then ``head / tail`` is performed instead. + ## + ## See also: + ## * `/ proc`_ + ## * `parentDir proc`_ + Path(`/../`(head.string, tail.string)) + +func extractFilename*(path: Path): Path {.inline.} = + ## Extracts the filename of a given `path`. + ## + ## This is the same as ``name & ext`` from `splitFile(path) proc`_. + ## + ## See also: + ## * `splitFile proc`_ + ## * `lastPathPart proc`_ + ## * `changeFileExt proc`_ + ## * `addFileExt proc`_ + result = Path(extractFilename(path.string)) + +func lastPathPart*(path: Path): Path {.inline.} = + ## Like `extractFilename proc`_, but ignores + ## trailing dir separator; aka: `baseName`:idx: in some other languages. + ## + ## See also: + ## * `splitFile proc`_ + ## * `extractFilename proc`_ + ## * `changeFileExt proc`_ + ## * `addFileExt proc`_ + result = Path(lastPathPart(path.string)) + +func changeFileExt*(filename: Path, ext: string): Path {.inline.} = + ## Changes the file extension to `ext`. + ## + ## If the `filename` has no extension, `ext` will be added. + ## If `ext` == "" then any extension is removed. + ## + ## `Ext` should be given without the leading `'.'`, because some + ## filesystems may use a different character. (Although I know + ## of none such beast.) + ## + ## See also: + ## * `splitFile proc`_ + ## * `extractFilename proc`_ + ## * `lastPathPart proc`_ + ## * `addFileExt proc`_ + result = Path(changeFileExt(filename.string, ext)) + +func addFileExt*(filename: Path, ext: string): Path {.inline.} = + ## Adds the file extension `ext` to `filename`, unless + ## `filename` already has an extension. + ## + ## `Ext` should be given without the leading `'.'`, because some + ## filesystems may use a different character. + ## (Although I know of none such beast.) + ## + ## See also: + ## * `splitFile proc`_ + ## * `extractFilename proc`_ + ## * `lastPathPart proc`_ + ## * `changeFileExt proc`_ + result = Path(addFileExt(filename.string, ext)) + +func unixToNativePath*(path: Path, drive=Path("")): Path {.inline.} = + ## Converts an UNIX-like path to a native one. + ## + ## On an UNIX system this does nothing. Else it converts + ## `'/'`, `'.'`, `'..'` to the appropriate things. + ## + ## On systems with a concept of "drives", `drive` is used to determine + ## which drive label to use during absolute path conversion. + ## `drive` defaults to the drive of the current working directory, and is + ## ignored on systems that do not have a concept of "drives". + result = Path(unixToNativePath(path.string, drive.string)) + +proc getCurrentDir*(): Path {.inline, tags: [].} = + ## Returns the `current working directory`:idx: i.e. where the built + ## binary is run. + ## + ## So the path returned by this proc is determined at run time. + ## + ## See also: + ## * `getHomeDir proc <appdirs.html#getHomeDir>`_ + ## * `getConfigDir proc <appdirs.html#getConfigDir>`_ + ## * `getTempDir proc <appdirs.html#getTempDir>`_ + ## * `setCurrentDir proc <dirs.html#setCurrentDir>`_ + ## * `currentSourcePath template <system.html#currentSourcePath.t>`_ + ## * `getProjectPath proc <macros.html#getProjectPath>`_ + result = Path(ospaths2.getCurrentDir()) + +proc normalizeExe*(file: var Path) {.borrow.} + +proc normalizePath*(path: var Path) {.borrow.} + +proc normalizePathEnd*(path: var Path, trailingSep = false) {.borrow.} + +proc absolutePath*(path: Path, root = getCurrentDir()): Path = + ## Returns the absolute path of `path`, rooted at `root` (which must be absolute; + ## default: current directory). + ## If `path` is absolute, return it, ignoring `root`. + ## + ## See also: + ## * `normalizePath proc`_ + result = Path(absolutePath(path.string, root.string)) + +proc expandTildeImpl(path: string): string {. + tags: [ReadEnvEffect, ReadIOEffect].} = + if len(path) == 0 or path[0] != '~': + result = path + elif len(path) == 1: + result = getHomeDir() + elif (path[1] in {DirSep, AltSep}): + result = joinPath(getHomeDir(), path.substr(2)) + else: + # TODO: handle `~bob` and `~bob/` which means home of bob + result = path + +proc expandTilde*(path: Path): Path {.inline, + tags: [ReadEnvEffect, ReadIOEffect].} = + ## Expands ``~`` or a path starting with ``~/`` to a full path, replacing + ## ``~`` with `getHomeDir() <appdirs.html#getHomeDir>`_ (otherwise returns ``path`` unmodified). + ## + ## Windows: this is still supported despite the Windows platform not having this + ## convention; also, both ``~/`` and ``~\`` are handled. + runnableExamples: + import std/appdirs + assert expandTilde(Path("~") / Path("appname.cfg")) == getHomeDir() / Path("appname.cfg") + assert expandTilde(Path("~/foo/bar")) == getHomeDir() / Path("foo/bar") + assert expandTilde(Path("/foo/bar")) == Path("/foo/bar") + result = Path(expandTildeImpl(path.string)) diff --git a/lib/std/private/bitops_utils.nim b/lib/std/private/bitops_utils.nim new file mode 100644 index 000000000..0b9484416 --- /dev/null +++ b/lib/std/private/bitops_utils.nim @@ -0,0 +1,22 @@ +template forwardImpl*(impl, arg) {.dirty.} = + when sizeof(x) <= 4: + when x is SomeSignedInt: + impl(cast[uint32](x.int32)) + else: + impl(x.uint32) + else: + when x is SomeSignedInt: + impl(cast[uint64](x.int64)) + else: + impl(x.uint64) + +# this could also be implemented via: +# import std/typetraits +# template castToUnsigned*(x: SomeInteger): auto = cast[toUnsigned(typeof(x))](x) + +template castToUnsigned*(x: int8): uint8 = cast[uint8](x) +template castToUnsigned*(x: int16): uint16 = cast[uint16](x) +template castToUnsigned*(x: int32): uint32 = cast[uint32](x) +template castToUnsigned*(x: int64): uint64 = cast[uint64](x) +template castToUnsigned*(x: int): uint = cast[uint](x) +template castToUnsigned*[T: SomeUnsignedInt](x: T): T = x diff --git a/lib/std/private/digitsutils.nim b/lib/std/private/digitsutils.nim new file mode 100644 index 000000000..f2d0d25cb --- /dev/null +++ b/lib/std/private/digitsutils.nim @@ -0,0 +1,116 @@ +const + trailingZeros100: array[100, int8] = [2'i8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0] + + digits100: array[200, char] = ['0', '0', '0', '1', '0', '2', '0', '3', '0', '4', '0', '5', + '0', '6', '0', '7', '0', '8', '0', '9', '1', '0', '1', '1', '1', '2', '1', '3', '1', '4', + '1', '5', '1', '6', '1', '7', '1', '8', '1', '9', '2', '0', '2', '1', '2', '2', '2', '3', + '2', '4', '2', '5', '2', '6', '2', '7', '2', '8', '2', '9', '3', '0', '3', '1', '3', '2', + '3', '3', '3', '4', '3', '5', '3', '6', '3', '7', '3', '8', '3', '9', '4', '0', '4', '1', + '4', '2', '4', '3', '4', '4', '4', '5', '4', '6', '4', '7', '4', '8', '4', '9', '5', '0', + '5', '1', '5', '2', '5', '3', '5', '4', '5', '5', '5', '6', '5', '7', '5', '8', '5', '9', + '6', '0', '6', '1', '6', '2', '6', '3', '6', '4', '6', '5', '6', '6', '6', '7', '6', '8', + '6', '9', '7', '0', '7', '1', '7', '2', '7', '3', '7', '4', '7', '5', '7', '6', '7', '7', + '7', '8', '7', '9', '8', '0', '8', '1', '8', '2', '8', '3', '8', '4', '8', '5', '8', '6', + '8', '7', '8', '8', '8', '9', '9', '0', '9', '1', '9', '2', '9', '3', '9', '4', '9', '5', + '9', '6', '9', '7', '9', '8', '9', '9'] + +# Inspired by https://engineering.fb.com/2013/03/15/developer-tools/three-optimization-tips-for-c +# Generates: +# ```nim +# var res = "" +# for i in 0 .. 99: +# if i < 10: +# res.add "0" & $i +# else: +# res.add $i +# doAssert res == digits100 +# ``` + +proc utoa2Digits*(buf: var openArray[char]; pos: int; digits: uint32) {.inline.} = + buf[pos] = digits100[2 * digits] + buf[pos+1] = digits100[2 * digits + 1] + #copyMem(buf, unsafeAddr(digits100[2 * digits]), 2 * sizeof((char))) + +proc trailingZeros2Digits*(digits: uint32): int {.inline.} = + trailingZeros100[digits] + +when defined(js): + proc numToString(a: SomeInteger): cstring {.importjs: "((#) + \"\")".} + +func addChars[T](result: var string, x: T, start: int, n: int) {.inline.} = + let old = result.len + result.setLen old + n + template impl = + for i in 0..<n: result[old + i] = x[start + i] + when nimvm: impl + else: + when defined(js) or defined(nimscript): impl + else: + {.noSideEffect.}: + copyMem result[old].addr, x[start].unsafeAddr, n + +func addChars[T](result: var string, x: T) {.inline.} = + addChars(result, x, 0, x.len) + +func addIntImpl(result: var string, x: uint64) {.inline.} = + var tmp {.noinit.}: array[24, char] + var num = x + var next = tmp.len - 1 + const nbatch = 100 + + while num >= nbatch: + let originNum = num + num = num div nbatch + let index = int16((originNum - num * nbatch) shl 1) + tmp[next] = digits100[index + 1] + tmp[next - 1] = digits100[index] + dec(next, 2) + + # process last 1-2 digits + if num < 10: + tmp[next] = chr(ord('0') + num.uint8) + else: + let index = num * 2 + tmp[next] = digits100[index + 1] + tmp[next - 1] = digits100[index] + dec next + addChars(result, tmp, next, tmp.len - next) + +when not defined(nimHasEnforceNoRaises): + {.pragma: enforceNoRaises.} + +func addInt*(result: var string, x: uint64) {.enforceNoRaises.} = + when nimvm: addIntImpl(result, x) + else: + when not defined(js): addIntImpl(result, x) + else: + addChars(result, numToString(x)) + +proc addInt*(result: var string; x: int64) {.enforceNoRaises.} = + ## Converts integer to its string representation and appends it to `result`. + runnableExamples: + var s = "foo" + s.addInt(45) + assert s == "foo45" + template impl = + var num: uint64 + if x < 0: + if x == low(int64): + num = cast[uint64](x) + else: + num = uint64(-x) + result.add '-' + else: + num = uint64(x) + addInt(result, num) + when nimvm: impl() + else: + when defined(js): + addChars(result, numToString(x)) + else: impl() + +proc addInt*(result: var string; x: int) {.inline, enforceNoRaises.} = + addInt(result, int64(x)) diff --git a/lib/std/private/dragonbox.nim b/lib/std/private/dragonbox.nim new file mode 100644 index 000000000..85ffea84a --- /dev/null +++ b/lib/std/private/dragonbox.nim @@ -0,0 +1,1325 @@ +## Copyright 2020 Junekey Jeon +## Copyright 2020 Alexander Bolz +## +## Distributed under the Boost Software License, Version 1.0. +## (See accompanying file LICENSE_1_0.txt or copy at https://www.boost.org/LICENSE_1_0.txt) + +## char* output_end = Dtoa(buffer, value); +## +## Converts the given double-precision number into decimal form and stores the result in the given +## buffer. +## +## The buffer must be large enough, i.e. >= DtoaMinBufferLength. +## The output format is similar to printf("%g"). +## The output is _not_ null-terminted. +## +## The output is optimal, i.e. the output string +## 1. rounds back to the input number when read in (using round-to-nearest-even) +## 2. is as short as possible, +## 3. is as close to the input number as possible. +## +## Note: +## This function may temporarily write up to DtoaMinBufferLength characters into the buffer. + + +import std/private/digitsutils + +when defined(nimPreviewSlimSystem): + import std/assertions + +const + dtoaMinBufferLength*: cint = 64 + +## This file contains an implementation of Junekey Jeon's Dragonbox algorithm. +## +## It is a simplified version of the reference implementation found here: +## https://github.com/jk-jeon/dragonbox +## +## The reference implementation also works with single-precision floating-point numbers and +## has options to configure the rounding mode. + +template dragonbox_Assert*(x: untyped): untyped = + assert(x) + +# ================================================================================================== +# +# ================================================================================================== + +type + ValueType* = float + BitsType* = uint64 + +type + Double* = object + bits*: BitsType + +const ## = p (includes the hidden bit) + significandSize*: int32 = 53 + +const ## static constexpr int32_t MaxExponent = 1024 - 1 - (SignificandSize - 1); + ## static constexpr int32_t MinExponent = std::numeric_limits<value_type>::min_exponent - 1 - (SignificandSize - 1); + exponentBias*: int32 = 1024 - 1 + (significandSize - 1) + +const + maxIeeeExponent*: BitsType = BitsType(2 * 1024 - 1) + +const ## = 2^(p-1) + hiddenBit*: BitsType = BitsType(1) shl (significandSize - 1) + +const ## = 2^(p-1) - 1 + significandMask*: BitsType = hiddenBit - 1 + +const + exponentMask*: BitsType = maxIeeeExponent shl (significandSize - 1) + +const + signMask*: BitsType = not (not BitsType(0) shr 1) + +proc constructDouble*(bits: BitsType): Double = + result.bits = bits + +proc constructDouble*(value: ValueType): Double = + result.bits = cast[typeof(result.bits)](value) + +proc physicalSignificand*(this: Double): BitsType {.noSideEffect.} = + return this.bits and significandMask + +proc physicalExponent*(this: Double): BitsType {.noSideEffect.} = + return (this.bits and exponentMask) shr (significandSize - 1) + +proc isFinite*(this: Double): bool {.noSideEffect.} = + return (this.bits and exponentMask) != exponentMask + +proc isInf*(this: Double): bool {.noSideEffect.} = + return (this.bits and exponentMask) == exponentMask and + (this.bits and significandMask) == 0 + +proc isNaN*(this: Double): bool {.noSideEffect.} = + return (this.bits and exponentMask) == exponentMask and + (this.bits and significandMask) != 0 + +proc isZero*(this: Double): bool {.noSideEffect.} = + return (this.bits and not signMask) == 0 + +proc signBit*(this: Double): int {.noSideEffect.} = + return ord((this.bits and signMask) != 0) + + +# ================================================================================================== +# +# ================================================================================================== +## namespace +## Returns floor(x / 2^n). +## +## Technically, right-shift of negative integers is implementation defined... +## Should easily be optimized into SAR (or equivalent) instruction. + +proc floorDivPow2*(x: int32; n: int32): int32 {.inline.} = + return x shr n + +proc floorLog2Pow10*(e: int32): int32 {.inline.} = + dragonbox_Assert(e >= -1233) + dragonbox_Assert(e <= 1233) + return floorDivPow2(e * 1741647, 19) + +proc floorLog10Pow2*(e: int32): int32 {.inline.} = + dragonbox_Assert(e >= -1500) + dragonbox_Assert(e <= 1500) + return floorDivPow2(e * 1262611, 22) + +proc floorLog10ThreeQuartersPow2*(e: int32): int32 {.inline.} = + dragonbox_Assert(e >= -1500) + dragonbox_Assert(e <= 1500) + return floorDivPow2(e * 1262611 - 524031, 22) + +# ================================================================================================== +# +# ================================================================================================== + +type + uint64x2* {.bycopy.} = object + hi*: uint64 + lo*: uint64 + + +proc computePow10*(k: int32): uint64x2 {.inline.} = + const + kMin: int32 = -292 + const + kMax: int32 = 326 + const + pow10: array[kMax - kMin + 1, uint64x2] = [ + uint64x2(hi: 0xFF77B1FCBEBCDC4F'u, lo: 0x25E8E89C13BB0F7B'u), + uint64x2(hi: 0x9FAACF3DF73609B1'u, lo: 0x77B191618C54E9AD'u), + uint64x2(hi: 0xC795830D75038C1D'u, lo: 0xD59DF5B9EF6A2418'u), + uint64x2(hi: 0xF97AE3D0D2446F25'u, lo: 0x4B0573286B44AD1E'u), + uint64x2(hi: 0x9BECCE62836AC577'u, lo: 0x4EE367F9430AEC33'u), + uint64x2(hi: 0xC2E801FB244576D5'u, lo: 0x229C41F793CDA740'u), + uint64x2(hi: 0xF3A20279ED56D48A'u, lo: 0x6B43527578C11110'u), + uint64x2(hi: 0x9845418C345644D6'u, lo: 0x830A13896B78AAAA'u), + uint64x2(hi: 0xBE5691EF416BD60C'u, lo: 0x23CC986BC656D554'u), + uint64x2(hi: 0xEDEC366B11C6CB8F'u, lo: 0x2CBFBE86B7EC8AA9'u), + uint64x2(hi: 0x94B3A202EB1C3F39'u, lo: 0x7BF7D71432F3D6AA'u), + uint64x2(hi: 0xB9E08A83A5E34F07'u, lo: 0xDAF5CCD93FB0CC54'u), + uint64x2(hi: 0xE858AD248F5C22C9'u, lo: 0xD1B3400F8F9CFF69'u), + uint64x2(hi: 0x91376C36D99995BE'u, lo: 0x23100809B9C21FA2'u), + uint64x2(hi: 0xB58547448FFFFB2D'u, lo: 0xABD40A0C2832A78B'u), + uint64x2(hi: 0xE2E69915B3FFF9F9'u, lo: 0x16C90C8F323F516D'u), + uint64x2(hi: 0x8DD01FAD907FFC3B'u, lo: 0xAE3DA7D97F6792E4'u), + uint64x2(hi: 0xB1442798F49FFB4A'u, lo: 0x99CD11CFDF41779D'u), + uint64x2(hi: 0xDD95317F31C7FA1D'u, lo: 0x40405643D711D584'u), + uint64x2(hi: 0x8A7D3EEF7F1CFC52'u, lo: 0x482835EA666B2573'u), + uint64x2(hi: 0xAD1C8EAB5EE43B66'u, lo: 0xDA3243650005EED0'u), + uint64x2(hi: 0xD863B256369D4A40'u, lo: 0x90BED43E40076A83'u), + uint64x2(hi: 0x873E4F75E2224E68'u, lo: 0x5A7744A6E804A292'u), + uint64x2(hi: 0xA90DE3535AAAE202'u, lo: 0x711515D0A205CB37'u), + uint64x2(hi: 0xD3515C2831559A83'u, lo: 0x0D5A5B44CA873E04'u), + uint64x2(hi: 0x8412D9991ED58091'u, lo: 0xE858790AFE9486C3'u), + uint64x2(hi: 0xA5178FFF668AE0B6'u, lo: 0x626E974DBE39A873'u), + uint64x2(hi: 0xCE5D73FF402D98E3'u, lo: 0xFB0A3D212DC81290'u), + uint64x2(hi: 0x80FA687F881C7F8E'u, lo: 0x7CE66634BC9D0B9A'u), + uint64x2(hi: 0xA139029F6A239F72'u, lo: 0x1C1FFFC1EBC44E81'u), + uint64x2(hi: 0xC987434744AC874E'u, lo: 0xA327FFB266B56221'u), + uint64x2(hi: 0xFBE9141915D7A922'u, lo: 0x4BF1FF9F0062BAA9'u), + uint64x2(hi: 0x9D71AC8FADA6C9B5'u, lo: 0x6F773FC3603DB4AA'u), + uint64x2(hi: 0xC4CE17B399107C22'u, lo: 0xCB550FB4384D21D4'u), + uint64x2(hi: 0xF6019DA07F549B2B'u, lo: 0x7E2A53A146606A49'u), + uint64x2(hi: 0x99C102844F94E0FB'u, lo: 0x2EDA7444CBFC426E'u), + uint64x2(hi: 0xC0314325637A1939'u, lo: 0xFA911155FEFB5309'u), + uint64x2(hi: 0xF03D93EEBC589F88'u, lo: 0x793555AB7EBA27CB'u), + uint64x2(hi: 0x96267C7535B763B5'u, lo: 0x4BC1558B2F3458DF'u), + uint64x2(hi: 0xBBB01B9283253CA2'u, lo: 0x9EB1AAEDFB016F17'u), + uint64x2(hi: 0xEA9C227723EE8BCB'u, lo: 0x465E15A979C1CADD'u), + uint64x2(hi: 0x92A1958A7675175F'u, lo: 0x0BFACD89EC191ECA'u), + uint64x2(hi: 0xB749FAED14125D36'u, lo: 0xCEF980EC671F667C'u), + uint64x2(hi: 0xE51C79A85916F484'u, lo: 0x82B7E12780E7401B'u), + uint64x2(hi: 0x8F31CC0937AE58D2'u, lo: 0xD1B2ECB8B0908811'u), + uint64x2(hi: 0xB2FE3F0B8599EF07'u, lo: 0x861FA7E6DCB4AA16'u), + uint64x2(hi: 0xDFBDCECE67006AC9'u, lo: 0x67A791E093E1D49B'u), + uint64x2(hi: 0x8BD6A141006042BD'u, lo: 0xE0C8BB2C5C6D24E1'u), + uint64x2(hi: 0xAECC49914078536D'u, lo: 0x58FAE9F773886E19'u), + uint64x2(hi: 0xDA7F5BF590966848'u, lo: 0xAF39A475506A899F'u), + uint64x2(hi: 0x888F99797A5E012D'u, lo: 0x6D8406C952429604'u), + uint64x2(hi: 0xAAB37FD7D8F58178'u, lo: 0xC8E5087BA6D33B84'u), + uint64x2(hi: 0xD5605FCDCF32E1D6'u, lo: 0xFB1E4A9A90880A65'u), + uint64x2(hi: 0x855C3BE0A17FCD26'u, lo: 0x5CF2EEA09A550680'u), + uint64x2(hi: 0xA6B34AD8C9DFC06F'u, lo: 0xF42FAA48C0EA481F'u), + uint64x2(hi: 0xD0601D8EFC57B08B'u, lo: 0xF13B94DAF124DA27'u), + uint64x2(hi: 0x823C12795DB6CE57'u, lo: 0x76C53D08D6B70859'u), + uint64x2(hi: 0xA2CB1717B52481ED'u, lo: 0x54768C4B0C64CA6F'u), + uint64x2(hi: 0xCB7DDCDDA26DA268'u, lo: 0xA9942F5DCF7DFD0A'u), + uint64x2(hi: 0xFE5D54150B090B02'u, lo: 0xD3F93B35435D7C4D'u), + uint64x2(hi: 0x9EFA548D26E5A6E1'u, lo: 0xC47BC5014A1A6DB0'u), + uint64x2(hi: 0xC6B8E9B0709F109A'u, lo: 0x359AB6419CA1091C'u), + uint64x2(hi: 0xF867241C8CC6D4C0'u, lo: 0xC30163D203C94B63'u), + uint64x2(hi: 0x9B407691D7FC44F8'u, lo: 0x79E0DE63425DCF1E'u), + uint64x2(hi: 0xC21094364DFB5636'u, lo: 0x985915FC12F542E5'u), + uint64x2(hi: 0xF294B943E17A2BC4'u, lo: 0x3E6F5B7B17B2939E'u), + uint64x2(hi: 0x979CF3CA6CEC5B5A'u, lo: 0xA705992CEECF9C43'u), + uint64x2(hi: 0xBD8430BD08277231'u, lo: 0x50C6FF782A838354'u), + uint64x2(hi: 0xECE53CEC4A314EBD'u, lo: 0xA4F8BF5635246429'u), + uint64x2(hi: 0x940F4613AE5ED136'u, lo: 0x871B7795E136BE9A'u), + uint64x2(hi: 0xB913179899F68584'u, lo: 0x28E2557B59846E40'u), + uint64x2(hi: 0xE757DD7EC07426E5'u, lo: 0x331AEADA2FE589D0'u), + uint64x2(hi: 0x9096EA6F3848984F'u, lo: 0x3FF0D2C85DEF7622'u), + uint64x2(hi: 0xB4BCA50B065ABE63'u, lo: 0x0FED077A756B53AA'u), + uint64x2(hi: 0xE1EBCE4DC7F16DFB'u, lo: 0xD3E8495912C62895'u), + uint64x2(hi: 0x8D3360F09CF6E4BD'u, lo: 0x64712DD7ABBBD95D'u), + uint64x2(hi: 0xB080392CC4349DEC'u, lo: 0xBD8D794D96AACFB4'u), + uint64x2(hi: 0xDCA04777F541C567'u, lo: 0xECF0D7A0FC5583A1'u), + uint64x2(hi: 0x89E42CAAF9491B60'u, lo: 0xF41686C49DB57245'u), + uint64x2(hi: 0xAC5D37D5B79B6239'u, lo: 0x311C2875C522CED6'u), + uint64x2(hi: 0xD77485CB25823AC7'u, lo: 0x7D633293366B828C'u), + uint64x2(hi: 0x86A8D39EF77164BC'u, lo: 0xAE5DFF9C02033198'u), + uint64x2(hi: 0xA8530886B54DBDEB'u, lo: 0xD9F57F830283FDFD'u), + uint64x2(hi: 0xD267CAA862A12D66'u, lo: 0xD072DF63C324FD7C'u), + uint64x2(hi: 0x8380DEA93DA4BC60'u, lo: 0x4247CB9E59F71E6E'u), + uint64x2(hi: 0xA46116538D0DEB78'u, lo: 0x52D9BE85F074E609'u), + uint64x2(hi: 0xCD795BE870516656'u, lo: 0x67902E276C921F8C'u), + uint64x2(hi: 0x806BD9714632DFF6'u, lo: 0x00BA1CD8A3DB53B7'u), + uint64x2(hi: 0xA086CFCD97BF97F3'u, lo: 0x80E8A40ECCD228A5'u), + uint64x2(hi: 0xC8A883C0FDAF7DF0'u, lo: 0x6122CD128006B2CE'u), + uint64x2(hi: 0xFAD2A4B13D1B5D6C'u, lo: 0x796B805720085F82'u), + uint64x2(hi: 0x9CC3A6EEC6311A63'u, lo: 0xCBE3303674053BB1'u), + uint64x2(hi: 0xC3F490AA77BD60FC'u, lo: 0xBEDBFC4411068A9D'u), + uint64x2(hi: 0xF4F1B4D515ACB93B'u, lo: 0xEE92FB5515482D45'u), + uint64x2(hi: 0x991711052D8BF3C5'u, lo: 0x751BDD152D4D1C4B'u), + uint64x2(hi: 0xBF5CD54678EEF0B6'u, lo: 0xD262D45A78A0635E'u), + uint64x2(hi: 0xEF340A98172AACE4'u, lo: 0x86FB897116C87C35'u), + uint64x2(hi: 0x9580869F0E7AAC0E'u, lo: 0xD45D35E6AE3D4DA1'u), + uint64x2(hi: 0xBAE0A846D2195712'u, lo: 0x8974836059CCA10A'u), + uint64x2(hi: 0xE998D258869FACD7'u, lo: 0x2BD1A438703FC94C'u), + uint64x2(hi: 0x91FF83775423CC06'u, lo: 0x7B6306A34627DDD0'u), + uint64x2(hi: 0xB67F6455292CBF08'u, lo: 0x1A3BC84C17B1D543'u), + uint64x2(hi: 0xE41F3D6A7377EECA'u, lo: 0x20CABA5F1D9E4A94'u), + uint64x2(hi: 0x8E938662882AF53E'u, lo: 0x547EB47B7282EE9D'u), + uint64x2(hi: 0xB23867FB2A35B28D'u, lo: 0xE99E619A4F23AA44'u), + uint64x2(hi: 0xDEC681F9F4C31F31'u, lo: 0x6405FA00E2EC94D5'u), + uint64x2(hi: 0x8B3C113C38F9F37E'u, lo: 0xDE83BC408DD3DD05'u), + uint64x2(hi: 0xAE0B158B4738705E'u, lo: 0x9624AB50B148D446'u), + uint64x2(hi: 0xD98DDAEE19068C76'u, lo: 0x3BADD624DD9B0958'u), + uint64x2(hi: 0x87F8A8D4CFA417C9'u, lo: 0xE54CA5D70A80E5D7'u), + uint64x2(hi: 0xA9F6D30A038D1DBC'u, lo: 0x5E9FCF4CCD211F4D'u), + uint64x2(hi: 0xD47487CC8470652B'u, lo: 0x7647C32000696720'u), + uint64x2(hi: 0x84C8D4DFD2C63F3B'u, lo: 0x29ECD9F40041E074'u), + uint64x2(hi: 0xA5FB0A17C777CF09'u, lo: 0xF468107100525891'u), + uint64x2(hi: 0xCF79CC9DB955C2CC'u, lo: 0x7182148D4066EEB5'u), + uint64x2(hi: 0x81AC1FE293D599BF'u, lo: 0xC6F14CD848405531'u), + uint64x2(hi: 0xA21727DB38CB002F'u, lo: 0xB8ADA00E5A506A7D'u), + uint64x2(hi: 0xCA9CF1D206FDC03B'u, lo: 0xA6D90811F0E4851D'u), + uint64x2(hi: 0xFD442E4688BD304A'u, lo: 0x908F4A166D1DA664'u), + uint64x2(hi: 0x9E4A9CEC15763E2E'u, lo: 0x9A598E4E043287FF'u), + uint64x2(hi: 0xC5DD44271AD3CDBA'u, lo: 0x40EFF1E1853F29FE'u), + uint64x2(hi: 0xF7549530E188C128'u, lo: 0xD12BEE59E68EF47D'u), + uint64x2(hi: 0x9A94DD3E8CF578B9'u, lo: 0x82BB74F8301958CF'u), + uint64x2(hi: 0xC13A148E3032D6E7'u, lo: 0xE36A52363C1FAF02'u), + uint64x2(hi: 0xF18899B1BC3F8CA1'u, lo: 0xDC44E6C3CB279AC2'u), + uint64x2(hi: 0x96F5600F15A7B7E5'u, lo: 0x29AB103A5EF8C0BA'u), + uint64x2(hi: 0xBCB2B812DB11A5DE'u, lo: 0x7415D448F6B6F0E8'u), + uint64x2(hi: 0xEBDF661791D60F56'u, lo: 0x111B495B3464AD22'u), + uint64x2(hi: 0x936B9FCEBB25C995'u, lo: 0xCAB10DD900BEEC35'u), + uint64x2(hi: 0xB84687C269EF3BFB'u, lo: 0x3D5D514F40EEA743'u), + uint64x2(hi: 0xE65829B3046B0AFA'u, lo: 0x0CB4A5A3112A5113'u), + uint64x2(hi: 0x8FF71A0FE2C2E6DC'u, lo: 0x47F0E785EABA72AC'u), + uint64x2(hi: 0xB3F4E093DB73A093'u, lo: 0x59ED216765690F57'u), + uint64x2(hi: 0xE0F218B8D25088B8'u, lo: 0x306869C13EC3532D'u), + uint64x2(hi: 0x8C974F7383725573'u, lo: 0x1E414218C73A13FC'u), + uint64x2(hi: 0xAFBD2350644EEACF'u, lo: 0xE5D1929EF90898FB'u), + uint64x2(hi: 0xDBAC6C247D62A583'u, lo: 0xDF45F746B74ABF3A'u), + uint64x2(hi: 0x894BC396CE5DA772'u, lo: 0x6B8BBA8C328EB784'u), + uint64x2(hi: 0xAB9EB47C81F5114F'u, lo: 0x066EA92F3F326565'u), + uint64x2(hi: 0xD686619BA27255A2'u, lo: 0xC80A537B0EFEFEBE'u), + uint64x2(hi: 0x8613FD0145877585'u, lo: 0xBD06742CE95F5F37'u), + uint64x2(hi: 0xA798FC4196E952E7'u, lo: 0x2C48113823B73705'u), + uint64x2(hi: 0xD17F3B51FCA3A7A0'u, lo: 0xF75A15862CA504C6'u), + uint64x2(hi: 0x82EF85133DE648C4'u, lo: 0x9A984D73DBE722FC'u), + uint64x2(hi: 0xA3AB66580D5FDAF5'u, lo: 0xC13E60D0D2E0EBBB'u), + uint64x2(hi: 0xCC963FEE10B7D1B3'u, lo: 0x318DF905079926A9'u), + uint64x2(hi: 0xFFBBCFE994E5C61F'u, lo: 0xFDF17746497F7053'u), + uint64x2(hi: 0x9FD561F1FD0F9BD3'u, lo: 0xFEB6EA8BEDEFA634'u), + uint64x2(hi: 0xC7CABA6E7C5382C8'u, lo: 0xFE64A52EE96B8FC1'u), + uint64x2(hi: 0xF9BD690A1B68637B'u, lo: 0x3DFDCE7AA3C673B1'u), + uint64x2(hi: 0x9C1661A651213E2D'u, lo: 0x06BEA10CA65C084F'u), + uint64x2(hi: 0xC31BFA0FE5698DB8'u, lo: 0x486E494FCFF30A63'u), + uint64x2(hi: 0xF3E2F893DEC3F126'u, lo: 0x5A89DBA3C3EFCCFB'u), + uint64x2(hi: 0x986DDB5C6B3A76B7'u, lo: 0xF89629465A75E01D'u), + uint64x2(hi: 0xBE89523386091465'u, lo: 0xF6BBB397F1135824'u), + uint64x2(hi: 0xEE2BA6C0678B597F'u, lo: 0x746AA07DED582E2D'u), + uint64x2(hi: 0x94DB483840B717EF'u, lo: 0xA8C2A44EB4571CDD'u), + uint64x2(hi: 0xBA121A4650E4DDEB'u, lo: 0x92F34D62616CE414'u), + uint64x2(hi: 0xE896A0D7E51E1566'u, lo: 0x77B020BAF9C81D18'u), + uint64x2(hi: 0x915E2486EF32CD60'u, lo: 0x0ACE1474DC1D122F'u), + uint64x2(hi: 0xB5B5ADA8AAFF80B8'u, lo: 0x0D819992132456BB'u), + uint64x2(hi: 0xE3231912D5BF60E6'u, lo: 0x10E1FFF697ED6C6A'u), + uint64x2(hi: 0x8DF5EFABC5979C8F'u, lo: 0xCA8D3FFA1EF463C2'u), + uint64x2(hi: 0xB1736B96B6FD83B3'u, lo: 0xBD308FF8A6B17CB3'u), + uint64x2(hi: 0xDDD0467C64BCE4A0'u, lo: 0xAC7CB3F6D05DDBDF'u), + uint64x2(hi: 0x8AA22C0DBEF60EE4'u, lo: 0x6BCDF07A423AA96C'u), + uint64x2(hi: 0xAD4AB7112EB3929D'u, lo: 0x86C16C98D2C953C7'u), + uint64x2(hi: 0xD89D64D57A607744'u, lo: 0xE871C7BF077BA8B8'u), + uint64x2(hi: 0x87625F056C7C4A8B'u, lo: 0x11471CD764AD4973'u), + uint64x2(hi: 0xA93AF6C6C79B5D2D'u, lo: 0xD598E40D3DD89BD0'u), + uint64x2(hi: 0xD389B47879823479'u, lo: 0x4AFF1D108D4EC2C4'u), + uint64x2(hi: 0x843610CB4BF160CB'u, lo: 0xCEDF722A585139BB'u), + uint64x2(hi: 0xA54394FE1EEDB8FE'u, lo: 0xC2974EB4EE658829'u), + uint64x2(hi: 0xCE947A3DA6A9273E'u, lo: 0x733D226229FEEA33'u), + uint64x2(hi: 0x811CCC668829B887'u, lo: 0x0806357D5A3F5260'u), + uint64x2(hi: 0xA163FF802A3426A8'u, lo: 0xCA07C2DCB0CF26F8'u), + uint64x2(hi: 0xC9BCFF6034C13052'u, lo: 0xFC89B393DD02F0B6'u), + uint64x2(hi: 0xFC2C3F3841F17C67'u, lo: 0xBBAC2078D443ACE3'u), + uint64x2(hi: 0x9D9BA7832936EDC0'u, lo: 0xD54B944B84AA4C0E'u), + uint64x2(hi: 0xC5029163F384A931'u, lo: 0x0A9E795E65D4DF12'u), + uint64x2(hi: 0xF64335BCF065D37D'u, lo: 0x4D4617B5FF4A16D6'u), + uint64x2(hi: 0x99EA0196163FA42E'u, lo: 0x504BCED1BF8E4E46'u), + uint64x2(hi: 0xC06481FB9BCF8D39'u, lo: 0xE45EC2862F71E1D7'u), + uint64x2(hi: 0xF07DA27A82C37088'u, lo: 0x5D767327BB4E5A4D'u), + uint64x2(hi: 0x964E858C91BA2655'u, lo: 0x3A6A07F8D510F870'u), + uint64x2(hi: 0xBBE226EFB628AFEA'u, lo: 0x890489F70A55368C'u), + uint64x2(hi: 0xEADAB0ABA3B2DBE5'u, lo: 0x2B45AC74CCEA842F'u), + uint64x2(hi: 0x92C8AE6B464FC96F'u, lo: 0x3B0B8BC90012929E'u), + uint64x2(hi: 0xB77ADA0617E3BBCB'u, lo: 0x09CE6EBB40173745'u), + uint64x2(hi: 0xE55990879DDCAABD'u, lo: 0xCC420A6A101D0516'u), + uint64x2(hi: 0x8F57FA54C2A9EAB6'u, lo: 0x9FA946824A12232E'u), + uint64x2(hi: 0xB32DF8E9F3546564'u, lo: 0x47939822DC96ABFA'u), + uint64x2(hi: 0xDFF9772470297EBD'u, lo: 0x59787E2B93BC56F8'u), + uint64x2(hi: 0x8BFBEA76C619EF36'u, lo: 0x57EB4EDB3C55B65B'u), + uint64x2(hi: 0xAEFAE51477A06B03'u, lo: 0xEDE622920B6B23F2'u), + uint64x2(hi: 0xDAB99E59958885C4'u, lo: 0xE95FAB368E45ECEE'u), + uint64x2(hi: 0x88B402F7FD75539B'u, lo: 0x11DBCB0218EBB415'u), + uint64x2(hi: 0xAAE103B5FCD2A881'u, lo: 0xD652BDC29F26A11A'u), + uint64x2(hi: 0xD59944A37C0752A2'u, lo: 0x4BE76D3346F04960'u), + uint64x2(hi: 0x857FCAE62D8493A5'u, lo: 0x6F70A4400C562DDC'u), + uint64x2(hi: 0xA6DFBD9FB8E5B88E'u, lo: 0xCB4CCD500F6BB953'u), + uint64x2(hi: 0xD097AD07A71F26B2'u, lo: 0x7E2000A41346A7A8'u), + uint64x2(hi: 0x825ECC24C873782F'u, lo: 0x8ED400668C0C28C9'u), + uint64x2(hi: 0xA2F67F2DFA90563B'u, lo: 0x728900802F0F32FB'u), + uint64x2(hi: 0xCBB41EF979346BCA'u, lo: 0x4F2B40A03AD2FFBA'u), + uint64x2(hi: 0xFEA126B7D78186BC'u, lo: 0xE2F610C84987BFA9'u), + uint64x2(hi: 0x9F24B832E6B0F436'u, lo: 0x0DD9CA7D2DF4D7CA'u), + uint64x2(hi: 0xC6EDE63FA05D3143'u, lo: 0x91503D1C79720DBC'u), + uint64x2(hi: 0xF8A95FCF88747D94'u, lo: 0x75A44C6397CE912B'u), + uint64x2(hi: 0x9B69DBE1B548CE7C'u, lo: 0xC986AFBE3EE11ABB'u), + uint64x2(hi: 0xC24452DA229B021B'u, lo: 0xFBE85BADCE996169'u), + uint64x2(hi: 0xF2D56790AB41C2A2'u, lo: 0xFAE27299423FB9C4'u), + uint64x2(hi: 0x97C560BA6B0919A5'u, lo: 0xDCCD879FC967D41B'u), + uint64x2(hi: 0xBDB6B8E905CB600F'u, lo: 0x5400E987BBC1C921'u), + uint64x2(hi: 0xED246723473E3813'u, lo: 0x290123E9AAB23B69'u), + uint64x2(hi: 0x9436C0760C86E30B'u, lo: 0xF9A0B6720AAF6522'u), + uint64x2(hi: 0xB94470938FA89BCE'u, lo: 0xF808E40E8D5B3E6A'u), + uint64x2(hi: 0xE7958CB87392C2C2'u, lo: 0xB60B1D1230B20E05'u), + uint64x2(hi: 0x90BD77F3483BB9B9'u, lo: 0xB1C6F22B5E6F48C3'u), + uint64x2(hi: 0xB4ECD5F01A4AA828'u, lo: 0x1E38AEB6360B1AF4'u), + uint64x2(hi: 0xE2280B6C20DD5232'u, lo: 0x25C6DA63C38DE1B1'u), + uint64x2(hi: 0x8D590723948A535F'u, lo: 0x579C487E5A38AD0F'u), + uint64x2(hi: 0xB0AF48EC79ACE837'u, lo: 0x2D835A9DF0C6D852'u), + uint64x2(hi: 0xDCDB1B2798182244'u, lo: 0xF8E431456CF88E66'u), + uint64x2(hi: 0x8A08F0F8BF0F156B'u, lo: 0x1B8E9ECB641B5900'u), + uint64x2(hi: 0xAC8B2D36EED2DAC5'u, lo: 0xE272467E3D222F40'u), + uint64x2(hi: 0xD7ADF884AA879177'u, lo: 0x5B0ED81DCC6ABB10'u), + uint64x2(hi: 0x86CCBB52EA94BAEA'u, lo: 0x98E947129FC2B4EA'u), + uint64x2(hi: 0xA87FEA27A539E9A5'u, lo: 0x3F2398D747B36225'u), + uint64x2(hi: 0xD29FE4B18E88640E'u, lo: 0x8EEC7F0D19A03AAE'u), + uint64x2(hi: 0x83A3EEEEF9153E89'u, lo: 0x1953CF68300424AD'u), + uint64x2(hi: 0xA48CEAAAB75A8E2B'u, lo: 0x5FA8C3423C052DD8'u), + uint64x2(hi: 0xCDB02555653131B6'u, lo: 0x3792F412CB06794E'u), + uint64x2(hi: 0x808E17555F3EBF11'u, lo: 0xE2BBD88BBEE40BD1'u), + uint64x2(hi: 0xA0B19D2AB70E6ED6'u, lo: 0x5B6ACEAEAE9D0EC5'u), + uint64x2(hi: 0xC8DE047564D20A8B'u, lo: 0xF245825A5A445276'u), + uint64x2(hi: 0xFB158592BE068D2E'u, lo: 0xEED6E2F0F0D56713'u), + uint64x2(hi: 0x9CED737BB6C4183D'u, lo: 0x55464DD69685606C'u), + uint64x2(hi: 0xC428D05AA4751E4C'u, lo: 0xAA97E14C3C26B887'u), + uint64x2(hi: 0xF53304714D9265DF'u, lo: 0xD53DD99F4B3066A9'u), + uint64x2(hi: 0x993FE2C6D07B7FAB'u, lo: 0xE546A8038EFE402A'u), + uint64x2(hi: 0xBF8FDB78849A5F96'u, lo: 0xDE98520472BDD034'u), + uint64x2(hi: 0xEF73D256A5C0F77C'u, lo: 0x963E66858F6D4441'u), + uint64x2(hi: 0x95A8637627989AAD'u, lo: 0xDDE7001379A44AA9'u), + uint64x2(hi: 0xBB127C53B17EC159'u, lo: 0x5560C018580D5D53'u), + uint64x2(hi: 0xE9D71B689DDE71AF'u, lo: 0xAAB8F01E6E10B4A7'u), + uint64x2(hi: 0x9226712162AB070D'u, lo: 0xCAB3961304CA70E9'u), + uint64x2(hi: 0xB6B00D69BB55C8D1'u, lo: 0x3D607B97C5FD0D23'u), + uint64x2(hi: 0xE45C10C42A2B3B05'u, lo: 0x8CB89A7DB77C506B'u), + uint64x2(hi: 0x8EB98A7A9A5B04E3'u, lo: 0x77F3608E92ADB243'u), + uint64x2(hi: 0xB267ED1940F1C61C'u, lo: 0x55F038B237591ED4'u), + uint64x2(hi: 0xDF01E85F912E37A3'u, lo: 0x6B6C46DEC52F6689'u), + uint64x2(hi: 0x8B61313BBABCE2C6'u, lo: 0x2323AC4B3B3DA016'u), + uint64x2(hi: 0xAE397D8AA96C1B77'u, lo: 0xABEC975E0A0D081B'u), + uint64x2(hi: 0xD9C7DCED53C72255'u, lo: 0x96E7BD358C904A22'u), + uint64x2(hi: 0x881CEA14545C7575'u, lo: 0x7E50D64177DA2E55'u), + uint64x2(hi: 0xAA242499697392D2'u, lo: 0xDDE50BD1D5D0B9EA'u), + uint64x2(hi: 0xD4AD2DBFC3D07787'u, lo: 0x955E4EC64B44E865'u), + uint64x2(hi: 0x84EC3C97DA624AB4'u, lo: 0xBD5AF13BEF0B113F'u), + uint64x2(hi: 0xA6274BBDD0FADD61'u, lo: 0xECB1AD8AEACDD58F'u), + uint64x2(hi: 0xCFB11EAD453994BA'u, lo: 0x67DE18EDA5814AF3'u), + uint64x2(hi: 0x81CEB32C4B43FCF4'u, lo: 0x80EACF948770CED8'u), + uint64x2(hi: 0xA2425FF75E14FC31'u, lo: 0xA1258379A94D028E'u), + uint64x2(hi: 0xCAD2F7F5359A3B3E'u, lo: 0x096EE45813A04331'u), + uint64x2(hi: 0xFD87B5F28300CA0D'u, lo: 0x8BCA9D6E188853FD'u), + uint64x2(hi: 0x9E74D1B791E07E48'u, lo: 0x775EA264CF55347E'u), + uint64x2(hi: 0xC612062576589DDA'u, lo: 0x95364AFE032A819E'u), + uint64x2(hi: 0xF79687AED3EEC551'u, lo: 0x3A83DDBD83F52205'u), + uint64x2(hi: 0x9ABE14CD44753B52'u, lo: 0xC4926A9672793543'u), + uint64x2(hi: 0xC16D9A0095928A27'u, lo: 0x75B7053C0F178294'u), + uint64x2(hi: 0xF1C90080BAF72CB1'u, lo: 0x5324C68B12DD6339'u), + uint64x2(hi: 0x971DA05074DA7BEE'u, lo: 0xD3F6FC16EBCA5E04'u), + uint64x2(hi: 0xBCE5086492111AEA'u, lo: 0x88F4BB1CA6BCF585'u), + uint64x2(hi: 0xEC1E4A7DB69561A5'u, lo: 0x2B31E9E3D06C32E6'u), + uint64x2(hi: 0x9392EE8E921D5D07'u, lo: 0x3AFF322E62439FD0'u), + uint64x2(hi: 0xB877AA3236A4B449'u, lo: 0x09BEFEB9FAD487C3'u), + uint64x2(hi: 0xE69594BEC44DE15B'u, lo: 0x4C2EBE687989A9B4'u), + uint64x2(hi: 0x901D7CF73AB0ACD9'u, lo: 0x0F9D37014BF60A11'u), + uint64x2(hi: 0xB424DC35095CD80F'u, lo: 0x538484C19EF38C95'u), + uint64x2(hi: 0xE12E13424BB40E13'u, lo: 0x2865A5F206B06FBA'u), + uint64x2(hi: 0x8CBCCC096F5088CB'u, lo: 0xF93F87B7442E45D4'u), + uint64x2(hi: 0xAFEBFF0BCB24AAFE'u, lo: 0xF78F69A51539D749'u), + uint64x2(hi: 0xDBE6FECEBDEDD5BE'u, lo: 0xB573440E5A884D1C'u), + uint64x2(hi: 0x89705F4136B4A597'u, lo: 0x31680A88F8953031'u), + uint64x2(hi: 0xABCC77118461CEFC'u, lo: 0xFDC20D2B36BA7C3E'u), + uint64x2(hi: 0xD6BF94D5E57A42BC'u, lo: 0x3D32907604691B4D'u), + uint64x2(hi: 0x8637BD05AF6C69B5'u, lo: 0xA63F9A49C2C1B110'u), + uint64x2(hi: 0xA7C5AC471B478423'u, lo: 0x0FCF80DC33721D54'u), + uint64x2(hi: 0xD1B71758E219652B'u, lo: 0xD3C36113404EA4A9'u), + uint64x2(hi: 0x83126E978D4FDF3B'u, lo: 0x645A1CAC083126EA'u), + uint64x2(hi: 0xA3D70A3D70A3D70A'u, lo: 0x3D70A3D70A3D70A4'u), + uint64x2(hi: 0xCCCCCCCCCCCCCCCC'u, lo: 0xCCCCCCCCCCCCCCCD'u), + uint64x2(hi: 0x8000000000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xA000000000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xC800000000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xFA00000000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0x9C40000000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xC350000000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xF424000000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0x9896800000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xBEBC200000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xEE6B280000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0x9502F90000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xBA43B74000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xE8D4A51000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0x9184E72A00000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xB5E620F480000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xE35FA931A0000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0x8E1BC9BF04000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xB1A2BC2EC5000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xDE0B6B3A76400000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0x8AC7230489E80000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xAD78EBC5AC620000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xD8D726B7177A8000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0x878678326EAC9000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xA968163F0A57B400'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xD3C21BCECCEDA100'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0x84595161401484A0'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xA56FA5B99019A5C8'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xCECB8F27F4200F3A'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0x813F3978F8940984'u, lo: 0x4000000000000000'u), + uint64x2(hi: 0xA18F07D736B90BE5'u, lo: 0x5000000000000000'u), + uint64x2(hi: 0xC9F2C9CD04674EDE'u, lo: 0xA400000000000000'u), + uint64x2(hi: 0xFC6F7C4045812296'u, lo: 0x4D00000000000000'u), + uint64x2(hi: 0x9DC5ADA82B70B59D'u, lo: 0xF020000000000000'u), + uint64x2(hi: 0xC5371912364CE305'u, lo: 0x6C28000000000000'u), + uint64x2(hi: 0xF684DF56C3E01BC6'u, lo: 0xC732000000000000'u), + uint64x2(hi: 0x9A130B963A6C115C'u, lo: 0x3C7F400000000000'u), + uint64x2(hi: 0xC097CE7BC90715B3'u, lo: 0x4B9F100000000000'u), + uint64x2(hi: 0xF0BDC21ABB48DB20'u, lo: 0x1E86D40000000000'u), + uint64x2(hi: 0x96769950B50D88F4'u, lo: 0x1314448000000000'u), + uint64x2(hi: 0xBC143FA4E250EB31'u, lo: 0x17D955A000000000'u), + uint64x2(hi: 0xEB194F8E1AE525FD'u, lo: 0x5DCFAB0800000000'u), + uint64x2(hi: 0x92EFD1B8D0CF37BE'u, lo: 0x5AA1CAE500000000'u), + uint64x2(hi: 0xB7ABC627050305AD'u, lo: 0xF14A3D9E40000000'u), + uint64x2(hi: 0xE596B7B0C643C719'u, lo: 0x6D9CCD05D0000000'u), + uint64x2(hi: 0x8F7E32CE7BEA5C6F'u, lo: 0xE4820023A2000000'u), + uint64x2(hi: 0xB35DBF821AE4F38B'u, lo: 0xDDA2802C8A800000'u), + uint64x2(hi: 0xE0352F62A19E306E'u, lo: 0xD50B2037AD200000'u), + uint64x2(hi: 0x8C213D9DA502DE45'u, lo: 0x4526F422CC340000'u), + uint64x2(hi: 0xAF298D050E4395D6'u, lo: 0x9670B12B7F410000'u), + uint64x2(hi: 0xDAF3F04651D47B4C'u, lo: 0x3C0CDD765F114000'u), + uint64x2(hi: 0x88D8762BF324CD0F'u, lo: 0xA5880A69FB6AC800'u), + uint64x2(hi: 0xAB0E93B6EFEE0053'u, lo: 0x8EEA0D047A457A00'u), + uint64x2(hi: 0xD5D238A4ABE98068'u, lo: 0x72A4904598D6D880'u), + uint64x2(hi: 0x85A36366EB71F041'u, lo: 0x47A6DA2B7F864750'u), + uint64x2(hi: 0xA70C3C40A64E6C51'u, lo: 0x999090B65F67D924'u), + uint64x2(hi: 0xD0CF4B50CFE20765'u, lo: 0xFFF4B4E3F741CF6D'u), + uint64x2(hi: 0x82818F1281ED449F'u, lo: 0xBFF8F10E7A8921A4'u), + uint64x2(hi: 0xA321F2D7226895C7'u, lo: 0xAFF72D52192B6A0D'u), + uint64x2(hi: 0xCBEA6F8CEB02BB39'u, lo: 0x9BF4F8A69F764490'u), + uint64x2(hi: 0xFEE50B7025C36A08'u, lo: 0x02F236D04753D5B4'u), + uint64x2(hi: 0x9F4F2726179A2245'u, lo: 0x01D762422C946590'u), + uint64x2(hi: 0xC722F0EF9D80AAD6'u, lo: 0x424D3AD2B7B97EF5'u), + uint64x2(hi: 0xF8EBAD2B84E0D58B'u, lo: 0xD2E0898765A7DEB2'u), + uint64x2(hi: 0x9B934C3B330C8577'u, lo: 0x63CC55F49F88EB2F'u), + uint64x2(hi: 0xC2781F49FFCFA6D5'u, lo: 0x3CBF6B71C76B25FB'u), + uint64x2(hi: 0xF316271C7FC3908A'u, lo: 0x8BEF464E3945EF7A'u), + uint64x2(hi: 0x97EDD871CFDA3A56'u, lo: 0x97758BF0E3CBB5AC'u), + uint64x2(hi: 0xBDE94E8E43D0C8EC'u, lo: 0x3D52EEED1CBEA317'u), + uint64x2(hi: 0xED63A231D4C4FB27'u, lo: 0x4CA7AAA863EE4BDD'u), + uint64x2(hi: 0x945E455F24FB1CF8'u, lo: 0x8FE8CAA93E74EF6A'u), + uint64x2(hi: 0xB975D6B6EE39E436'u, lo: 0xB3E2FD538E122B44'u), + uint64x2(hi: 0xE7D34C64A9C85D44'u, lo: 0x60DBBCA87196B616'u), + uint64x2(hi: 0x90E40FBEEA1D3A4A'u, lo: 0xBC8955E946FE31CD'u), + uint64x2(hi: 0xB51D13AEA4A488DD'u, lo: 0x6BABAB6398BDBE41'u), + uint64x2(hi: 0xE264589A4DCDAB14'u, lo: 0xC696963C7EED2DD1'u), + uint64x2(hi: 0x8D7EB76070A08AEC'u, lo: 0xFC1E1DE5CF543CA2'u), + uint64x2(hi: 0xB0DE65388CC8ADA8'u, lo: 0x3B25A55F43294BCB'u), + uint64x2(hi: 0xDD15FE86AFFAD912'u, lo: 0x49EF0EB713F39EBE'u), + uint64x2(hi: 0x8A2DBF142DFCC7AB'u, lo: 0x6E3569326C784337'u), + uint64x2(hi: 0xACB92ED9397BF996'u, lo: 0x49C2C37F07965404'u), + uint64x2(hi: 0xD7E77A8F87DAF7FB'u, lo: 0xDC33745EC97BE906'u), + uint64x2(hi: 0x86F0AC99B4E8DAFD'u, lo: 0x69A028BB3DED71A3'u), + uint64x2(hi: 0xA8ACD7C0222311BC'u, lo: 0xC40832EA0D68CE0C'u), + uint64x2(hi: 0xD2D80DB02AABD62B'u, lo: 0xF50A3FA490C30190'u), + uint64x2(hi: 0x83C7088E1AAB65DB'u, lo: 0x792667C6DA79E0FA'u), + uint64x2(hi: 0xA4B8CAB1A1563F52'u, lo: 0x577001B891185938'u), + uint64x2(hi: 0xCDE6FD5E09ABCF26'u, lo: 0xED4C0226B55E6F86'u), + uint64x2(hi: 0x80B05E5AC60B6178'u, lo: 0x544F8158315B05B4'u), + uint64x2(hi: 0xA0DC75F1778E39D6'u, lo: 0x696361AE3DB1C721'u), + uint64x2(hi: 0xC913936DD571C84C'u, lo: 0x03BC3A19CD1E38E9'u), + uint64x2(hi: 0xFB5878494ACE3A5F'u, lo: 0x04AB48A04065C723'u), + uint64x2(hi: 0x9D174B2DCEC0E47B'u, lo: 0x62EB0D64283F9C76'u), + uint64x2(hi: 0xC45D1DF942711D9A'u, lo: 0x3BA5D0BD324F8394'u), + uint64x2(hi: 0xF5746577930D6500'u, lo: 0xCA8F44EC7EE36479'u), + uint64x2(hi: 0x9968BF6ABBE85F20'u, lo: 0x7E998B13CF4E1ECB'u), + uint64x2(hi: 0xBFC2EF456AE276E8'u, lo: 0x9E3FEDD8C321A67E'u), + uint64x2(hi: 0xEFB3AB16C59B14A2'u, lo: 0xC5CFE94EF3EA101E'u), + uint64x2(hi: 0x95D04AEE3B80ECE5'u, lo: 0xBBA1F1D158724A12'u), + uint64x2(hi: 0xBB445DA9CA61281F'u, lo: 0x2A8A6E45AE8EDC97'u), + uint64x2(hi: 0xEA1575143CF97226'u, lo: 0xF52D09D71A3293BD'u), + uint64x2(hi: 0x924D692CA61BE758'u, lo: 0x593C2626705F9C56'u), + uint64x2(hi: 0xB6E0C377CFA2E12E'u, lo: 0x6F8B2FB00C77836C'u), + uint64x2(hi: 0xE498F455C38B997A'u, lo: 0x0B6DFB9C0F956447'u), + uint64x2(hi: 0x8EDF98B59A373FEC'u, lo: 0x4724BD4189BD5EAC'u), + uint64x2(hi: 0xB2977EE300C50FE7'u, lo: 0x58EDEC91EC2CB657'u), + uint64x2(hi: 0xDF3D5E9BC0F653E1'u, lo: 0x2F2967B66737E3ED'u), + uint64x2(hi: 0x8B865B215899F46C'u, lo: 0xBD79E0D20082EE74'u), + uint64x2(hi: 0xAE67F1E9AEC07187'u, lo: 0xECD8590680A3AA11'u), + uint64x2(hi: 0xDA01EE641A708DE9'u, lo: 0xE80E6F4820CC9495'u), + uint64x2(hi: 0x884134FE908658B2'u, lo: 0x3109058D147FDCDD'u), + uint64x2(hi: 0xAA51823E34A7EEDE'u, lo: 0xBD4B46F0599FD415'u), + uint64x2(hi: 0xD4E5E2CDC1D1EA96'u, lo: 0x6C9E18AC7007C91A'u), + uint64x2(hi: 0x850FADC09923329E'u, lo: 0x03E2CF6BC604DDB0'u), + uint64x2(hi: 0xA6539930BF6BFF45'u, lo: 0x84DB8346B786151C'u), + uint64x2(hi: 0xCFE87F7CEF46FF16'u, lo: 0xE612641865679A63'u), + uint64x2(hi: 0x81F14FAE158C5F6E'u, lo: 0x4FCB7E8F3F60C07E'u), + uint64x2(hi: 0xA26DA3999AEF7749'u, lo: 0xE3BE5E330F38F09D'u), + uint64x2(hi: 0xCB090C8001AB551C'u, lo: 0x5CADF5BFD3072CC5'u), + uint64x2(hi: 0xFDCB4FA002162A63'u, lo: 0x73D9732FC7C8F7F6'u), + uint64x2(hi: 0x9E9F11C4014DDA7E'u, lo: 0x2867E7FDDCDD9AFA'u), + uint64x2(hi: 0xC646D63501A1511D'u, lo: 0xB281E1FD541501B8'u), + uint64x2(hi: 0xF7D88BC24209A565'u, lo: 0x1F225A7CA91A4226'u), + uint64x2(hi: 0x9AE757596946075F'u, lo: 0x3375788DE9B06958'u), + uint64x2(hi: 0xC1A12D2FC3978937'u, lo: 0x0052D6B1641C83AE'u), + uint64x2(hi: 0xF209787BB47D6B84'u, lo: 0xC0678C5DBD23A49A'u), + uint64x2(hi: 0x9745EB4D50CE6332'u, lo: 0xF840B7BA963646E0'u), + uint64x2(hi: 0xBD176620A501FBFF'u, lo: 0xB650E5A93BC3D898'u), + uint64x2(hi: 0xEC5D3FA8CE427AFF'u, lo: 0xA3E51F138AB4CEBE'u), + uint64x2(hi: 0x93BA47C980E98CDF'u, lo: 0xC66F336C36B10137'u), + uint64x2(hi: 0xB8A8D9BBE123F017'u, lo: 0xB80B0047445D4184'u), + uint64x2(hi: 0xE6D3102AD96CEC1D'u, lo: 0xA60DC059157491E5'u), + uint64x2(hi: 0x9043EA1AC7E41392'u, lo: 0x87C89837AD68DB2F'u), + uint64x2(hi: 0xB454E4A179DD1877'u, lo: 0x29BABE4598C311FB'u), + uint64x2(hi: 0xE16A1DC9D8545E94'u, lo: 0xF4296DD6FEF3D67A'u), + uint64x2(hi: 0x8CE2529E2734BB1D'u, lo: 0x1899E4A65F58660C'u), + uint64x2(hi: 0xB01AE745B101E9E4'u, lo: 0x5EC05DCFF72E7F8F'u), + uint64x2(hi: 0xDC21A1171D42645D'u, lo: 0x76707543F4FA1F73'u), + uint64x2(hi: 0x899504AE72497EBA'u, lo: 0x6A06494A791C53A8'u), + uint64x2(hi: 0xABFA45DA0EDBDE69'u, lo: 0x0487DB9D17636892'u), + uint64x2(hi: 0xD6F8D7509292D603'u, lo: 0x45A9D2845D3C42B6'u), + uint64x2(hi: 0x865B86925B9BC5C2'u, lo: 0x0B8A2392BA45A9B2'u), + uint64x2(hi: 0xA7F26836F282B732'u, lo: 0x8E6CAC7768D7141E'u), + uint64x2(hi: 0xD1EF0244AF2364FF'u, lo: 0x3207D795430CD926'u), + uint64x2(hi: 0x8335616AED761F1F'u, lo: 0x7F44E6BD49E807B8'u), + uint64x2(hi: 0xA402B9C5A8D3A6E7'u, lo: 0x5F16206C9C6209A6'u), + uint64x2(hi: 0xCD036837130890A1'u, lo: 0x36DBA887C37A8C0F'u), + uint64x2(hi: 0x802221226BE55A64'u, lo: 0xC2494954DA2C9789'u), + uint64x2(hi: 0xA02AA96B06DEB0FD'u, lo: 0xF2DB9BAA10B7BD6C'u), + uint64x2(hi: 0xC83553C5C8965D3D'u, lo: 0x6F92829494E5ACC7'u), + uint64x2(hi: 0xFA42A8B73ABBF48C'u, lo: 0xCB772339BA1F17F9'u), + uint64x2(hi: 0x9C69A97284B578D7'u, lo: 0xFF2A760414536EFB'u), + uint64x2(hi: 0xC38413CF25E2D70D'u, lo: 0xFEF5138519684ABA'u), + uint64x2(hi: 0xF46518C2EF5B8CD1'u, lo: 0x7EB258665FC25D69'u), + uint64x2(hi: 0x98BF2F79D5993802'u, lo: 0xEF2F773FFBD97A61'u), + uint64x2(hi: 0xBEEEFB584AFF8603'u, lo: 0xAAFB550FFACFD8FA'u), + uint64x2(hi: 0xEEAABA2E5DBF6784'u, lo: 0x95BA2A53F983CF38'u), + uint64x2(hi: 0x952AB45CFA97A0B2'u, lo: 0xDD945A747BF26183'u), + uint64x2(hi: 0xBA756174393D88DF'u, lo: 0x94F971119AEEF9E4'u), + uint64x2(hi: 0xE912B9D1478CEB17'u, lo: 0x7A37CD5601AAB85D'u), + uint64x2(hi: 0x91ABB422CCB812EE'u, lo: 0xAC62E055C10AB33A'u), + uint64x2(hi: 0xB616A12B7FE617AA'u, lo: 0x577B986B314D6009'u), + uint64x2(hi: 0xE39C49765FDF9D94'u, lo: 0xED5A7E85FDA0B80B'u), + uint64x2(hi: 0x8E41ADE9FBEBC27D'u, lo: 0x14588F13BE847307'u), + uint64x2(hi: 0xB1D219647AE6B31C'u, lo: 0x596EB2D8AE258FC8'u), + uint64x2(hi: 0xDE469FBD99A05FE3'u, lo: 0x6FCA5F8ED9AEF3BB'u), + uint64x2(hi: 0x8AEC23D680043BEE'u, lo: 0x25DE7BB9480D5854'u), + uint64x2(hi: 0xADA72CCC20054AE9'u, lo: 0xAF561AA79A10AE6A'u), + uint64x2(hi: 0xD910F7FF28069DA4'u, lo: 0x1B2BA1518094DA04'u), + uint64x2(hi: 0x87AA9AFF79042286'u, lo: 0x90FB44D2F05D0842'u), + uint64x2(hi: 0xA99541BF57452B28'u, lo: 0x353A1607AC744A53'u), + uint64x2(hi: 0xD3FA922F2D1675F2'u, lo: 0x42889B8997915CE8'u), + uint64x2(hi: 0x847C9B5D7C2E09B7'u, lo: 0x69956135FEBADA11'u), + uint64x2(hi: 0xA59BC234DB398C25'u, lo: 0x43FAB9837E699095'u), + uint64x2(hi: 0xCF02B2C21207EF2E'u, lo: 0x94F967E45E03F4BB'u), + uint64x2(hi: 0x8161AFB94B44F57D'u, lo: 0x1D1BE0EEBAC278F5'u), + uint64x2(hi: 0xA1BA1BA79E1632DC'u, lo: 0x6462D92A69731732'u), + uint64x2(hi: 0xCA28A291859BBF93'u, lo: 0x7D7B8F7503CFDCFE'u), + uint64x2(hi: 0xFCB2CB35E702AF78'u, lo: 0x5CDA735244C3D43E'u), + uint64x2(hi: 0x9DEFBF01B061ADAB'u, lo: 0x3A0888136AFA64A7'u), + uint64x2(hi: 0xC56BAEC21C7A1916'u, lo: 0x088AAA1845B8FDD0'u), + uint64x2(hi: 0xF6C69A72A3989F5B'u, lo: 0x8AAD549E57273D45'u), + uint64x2(hi: 0x9A3C2087A63F6399'u, lo: 0x36AC54E2F678864B'u), + uint64x2(hi: 0xC0CB28A98FCF3C7F'u, lo: 0x84576A1BB416A7DD'u), + uint64x2(hi: 0xF0FDF2D3F3C30B9F'u, lo: 0x656D44A2A11C51D5'u), + uint64x2(hi: 0x969EB7C47859E743'u, lo: 0x9F644AE5A4B1B325'u), + uint64x2(hi: 0xBC4665B596706114'u, lo: 0x873D5D9F0DDE1FEE'u), + uint64x2(hi: 0xEB57FF22FC0C7959'u, lo: 0xA90CB506D155A7EA'u), + uint64x2(hi: 0x9316FF75DD87CBD8'u, lo: 0x09A7F12442D588F2'u), + uint64x2(hi: 0xB7DCBF5354E9BECE'u, lo: 0x0C11ED6D538AEB2F'u), + uint64x2(hi: 0xE5D3EF282A242E81'u, lo: 0x8F1668C8A86DA5FA'u), + uint64x2(hi: 0x8FA475791A569D10'u, lo: 0xF96E017D694487BC'u), + uint64x2(hi: 0xB38D92D760EC4455'u, lo: 0x37C981DCC395A9AC'u), + uint64x2(hi: 0xE070F78D3927556A'u, lo: 0x85BBE253F47B1417'u), + uint64x2(hi: 0x8C469AB843B89562'u, lo: 0x93956D7478CCEC8E'u), + uint64x2(hi: 0xAF58416654A6BABB'u, lo: 0x387AC8D1970027B2'u), + uint64x2(hi: 0xDB2E51BFE9D0696A'u, lo: 0x06997B05FCC0319E'u), + uint64x2(hi: 0x88FCF317F22241E2'u, lo: 0x441FECE3BDF81F03'u), + uint64x2(hi: 0xAB3C2FDDEEAAD25A'u, lo: 0xD527E81CAD7626C3'u), + uint64x2(hi: 0xD60B3BD56A5586F1'u, lo: 0x8A71E223D8D3B074'u), + uint64x2(hi: 0x85C7056562757456'u, lo: 0xF6872D5667844E49'u), + uint64x2(hi: 0xA738C6BEBB12D16C'u, lo: 0xB428F8AC016561DB'u), + uint64x2(hi: 0xD106F86E69D785C7'u, lo: 0xE13336D701BEBA52'u), + uint64x2(hi: 0x82A45B450226B39C'u, lo: 0xECC0024661173473'u), + uint64x2(hi: 0xA34D721642B06084'u, lo: 0x27F002D7F95D0190'u), + uint64x2(hi: 0xCC20CE9BD35C78A5'u, lo: 0x31EC038DF7B441F4'u), + uint64x2(hi: 0xFF290242C83396CE'u, lo: 0x7E67047175A15271'u), + uint64x2(hi: 0x9F79A169BD203E41'u, lo: 0x0F0062C6E984D386'u), + uint64x2(hi: 0xC75809C42C684DD1'u, lo: 0x52C07B78A3E60868'u), + uint64x2(hi: 0xF92E0C3537826145'u, lo: 0xA7709A56CCDF8A82'u), + uint64x2(hi: 0x9BBCC7A142B17CCB'u, lo: 0x88A66076400BB691'u), + uint64x2(hi: 0xC2ABF989935DDBFE'u, lo: 0x6ACFF893D00EA435'u), + uint64x2(hi: 0xF356F7EBF83552FE'u, lo: 0x0583F6B8C4124D43'u), + uint64x2(hi: 0x98165AF37B2153DE'u, lo: 0xC3727A337A8B704A'u), + uint64x2(hi: 0xBE1BF1B059E9A8D6'u, lo: 0x744F18C0592E4C5C'u), + uint64x2(hi: 0xEDA2EE1C7064130C'u, lo: 0x1162DEF06F79DF73'u), + uint64x2(hi: 0x9485D4D1C63E8BE7'u, lo: 0x8ADDCB5645AC2BA8'u), + uint64x2(hi: 0xB9A74A0637CE2EE1'u, lo: 0x6D953E2BD7173692'u), + uint64x2(hi: 0xE8111C87C5C1BA99'u, lo: 0xC8FA8DB6CCDD0437'u), + uint64x2(hi: 0x910AB1D4DB9914A0'u, lo: 0x1D9C9892400A22A2'u), + uint64x2(hi: 0xB54D5E4A127F59C8'u, lo: 0x2503BEB6D00CAB4B'u), + uint64x2(hi: 0xE2A0B5DC971F303A'u, lo: 0x2E44AE64840FD61D'u), + uint64x2(hi: 0x8DA471A9DE737E24'u, lo: 0x5CEAECFED289E5D2'u), + uint64x2(hi: 0xB10D8E1456105DAD'u, lo: 0x7425A83E872C5F47'u), + uint64x2(hi: 0xDD50F1996B947518'u, lo: 0xD12F124E28F77719'u), + uint64x2(hi: 0x8A5296FFE33CC92F'u, lo: 0x82BD6B70D99AAA6F'u), + uint64x2(hi: 0xACE73CBFDC0BFB7B'u, lo: 0x636CC64D1001550B'u), + uint64x2(hi: 0xD8210BEFD30EFA5A'u, lo: 0x3C47F7E05401AA4E'u), + uint64x2(hi: 0x8714A775E3E95C78'u, lo: 0x65ACFAEC34810A71'u), + uint64x2(hi: 0xA8D9D1535CE3B396'u, lo: 0x7F1839A741A14D0D'u), + uint64x2(hi: 0xD31045A8341CA07C'u, lo: 0x1EDE48111209A050'u), + uint64x2(hi: 0x83EA2B892091E44D'u, lo: 0x934AED0AAB460432'u), + uint64x2(hi: 0xA4E4B66B68B65D60'u, lo: 0xF81DA84D5617853F'u), + uint64x2(hi: 0xCE1DE40642E3F4B9'u, lo: 0x36251260AB9D668E'u), + uint64x2(hi: 0x80D2AE83E9CE78F3'u, lo: 0xC1D72B7C6B426019'u), + uint64x2(hi: 0xA1075A24E4421730'u, lo: 0xB24CF65B8612F81F'u), + uint64x2(hi: 0xC94930AE1D529CFC'u, lo: 0xDEE033F26797B627'u), + uint64x2(hi: 0xFB9B7CD9A4A7443C'u, lo: 0x169840EF017DA3B1'u), + uint64x2(hi: 0x9D412E0806E88AA5'u, lo: 0x8E1F289560EE864E'u), + uint64x2(hi: 0xC491798A08A2AD4E'u, lo: 0xF1A6F2BAB92A27E2'u), + uint64x2(hi: 0xF5B5D7EC8ACB58A2'u, lo: 0xAE10AF696774B1DB'u), + uint64x2(hi: 0x9991A6F3D6BF1765'u, lo: 0xACCA6DA1E0A8EF29'u), + uint64x2(hi: 0xBFF610B0CC6EDD3F'u, lo: 0x17FD090A58D32AF3'u), + uint64x2(hi: 0xEFF394DCFF8A948E'u, lo: 0xDDFC4B4CEF07F5B0'u), + uint64x2(hi: 0x95F83D0A1FB69CD9'u, lo: 0x4ABDAF101564F98E'u), + uint64x2(hi: 0xBB764C4CA7A4440F'u, lo: 0x9D6D1AD41ABE37F1'u), + uint64x2(hi: 0xEA53DF5FD18D5513'u, lo: 0x84C86189216DC5ED'u), + uint64x2(hi: 0x92746B9BE2F8552C'u, lo: 0x32FD3CF5B4E49BB4'u), + uint64x2(hi: 0xB7118682DBB66A77'u, lo: 0x3FBC8C33221DC2A1'u), + uint64x2(hi: 0xE4D5E82392A40515'u, lo: 0x0FABAF3FEAA5334A'u), + uint64x2(hi: 0x8F05B1163BA6832D'u, lo: 0x29CB4D87F2A7400E'u), + uint64x2(hi: 0xB2C71D5BCA9023F8'u, lo: 0x743E20E9EF511012'u), + uint64x2(hi: 0xDF78E4B2BD342CF6'u, lo: 0x914DA9246B255416'u), + uint64x2(hi: 0x8BAB8EEFB6409C1A'u, lo: 0x1AD089B6C2F7548E'u), + uint64x2(hi: 0xAE9672ABA3D0C320'u, lo: 0xA184AC2473B529B1'u), + uint64x2(hi: 0xDA3C0F568CC4F3E8'u, lo: 0xC9E5D72D90A2741E'u), + uint64x2(hi: 0x8865899617FB1871'u, lo: 0x7E2FA67C7A658892'u), + uint64x2(hi: 0xAA7EEBFB9DF9DE8D'u, lo: 0xDDBB901B98FEEAB7'u), + uint64x2(hi: 0xD51EA6FA85785631'u, lo: 0x552A74227F3EA565'u), + uint64x2(hi: 0x8533285C936B35DE'u, lo: 0xD53A88958F87275F'u), + uint64x2(hi: 0xA67FF273B8460356'u, lo: 0x8A892ABAF368F137'u), + uint64x2(hi: 0xD01FEF10A657842C'u, lo: 0x2D2B7569B0432D85'u), + uint64x2(hi: 0x8213F56A67F6B29B'u, lo: 0x9C3B29620E29FC73'u), + uint64x2(hi: 0xA298F2C501F45F42'u, lo: 0x8349F3BA91B47B8F'u), + uint64x2(hi: 0xCB3F2F7642717713'u, lo: 0x241C70A936219A73'u), + uint64x2(hi: 0xFE0EFB53D30DD4D7'u, lo: 0xED238CD383AA0110'u), + uint64x2(hi: 0x9EC95D1463E8A506'u, lo: 0xF4363804324A40AA'u), + uint64x2(hi: 0xC67BB4597CE2CE48'u, lo: 0xB143C6053EDCD0D5'u), + uint64x2(hi: 0xF81AA16FDC1B81DA'u, lo: 0xDD94B7868E94050A'u), + uint64x2(hi: 0x9B10A4E5E9913128'u, lo: 0xCA7CF2B4191C8326'u), + uint64x2(hi: 0xC1D4CE1F63F57D72'u, lo: 0xFD1C2F611F63A3F0'u), + uint64x2(hi: 0xF24A01A73CF2DCCF'u, lo: 0xBC633B39673C8CEC'u), + uint64x2(hi: 0x976E41088617CA01'u, lo: 0xD5BE0503E085D813'u), + uint64x2(hi: 0xBD49D14AA79DBC82'u, lo: 0x4B2D8644D8A74E18'u), + uint64x2(hi: 0xEC9C459D51852BA2'u, lo: 0xDDF8E7D60ED1219E'u), + uint64x2(hi: 0x93E1AB8252F33B45'u, lo: 0xCABB90E5C942B503'u), + uint64x2(hi: 0xB8DA1662E7B00A17'u, lo: 0x3D6A751F3B936243'u), + uint64x2(hi: 0xE7109BFBA19C0C9D'u, lo: 0x0CC512670A783AD4'u), + uint64x2(hi: 0x906A617D450187E2'u, lo: 0x27FB2B80668B24C5'u), + uint64x2(hi: 0xB484F9DC9641E9DA'u, lo: 0xB1F9F660802DEDF6'u), + uint64x2(hi: 0xE1A63853BBD26451'u, lo: 0x5E7873F8A0396973'u), + uint64x2(hi: 0x8D07E33455637EB2'u, lo: 0xDB0B487B6423E1E8'u), + uint64x2(hi: 0xB049DC016ABC5E5F'u, lo: 0x91CE1A9A3D2CDA62'u), + uint64x2(hi: 0xDC5C5301C56B75F7'u, lo: 0x7641A140CC7810FB'u), + uint64x2(hi: 0x89B9B3E11B6329BA'u, lo: 0xA9E904C87FCB0A9D'u), + uint64x2(hi: 0xAC2820D9623BF429'u, lo: 0x546345FA9FBDCD44'u), + uint64x2(hi: 0xD732290FBACAF133'u, lo: 0xA97C177947AD4095'u), + uint64x2(hi: 0x867F59A9D4BED6C0'u, lo: 0x49ED8EABCCCC485D'u), + uint64x2(hi: 0xA81F301449EE8C70'u, lo: 0x5C68F256BFFF5A74'u), + uint64x2(hi: 0xD226FC195C6A2F8C'u, lo: 0x73832EEC6FFF3111'u), + uint64x2(hi: 0x83585D8FD9C25DB7'u, lo: 0xC831FD53C5FF7EAB'u), + uint64x2(hi: 0xA42E74F3D032F525'u, lo: 0xBA3E7CA8B77F5E55'u), + uint64x2(hi: 0xCD3A1230C43FB26F'u, lo: 0x28CE1BD2E55F35EB'u), + uint64x2(hi: 0x80444B5E7AA7CF85'u, lo: 0x7980D163CF5B81B3'u), + uint64x2(hi: 0xA0555E361951C366'u, lo: 0xD7E105BCC332621F'u), + uint64x2(hi: 0xC86AB5C39FA63440'u, lo: 0x8DD9472BF3FEFAA7'u), + uint64x2(hi: 0xFA856334878FC150'u, lo: 0xB14F98F6F0FEB951'u), + uint64x2(hi: 0x9C935E00D4B9D8D2'u, lo: 0x6ED1BF9A569F33D3'u), + uint64x2(hi: 0xC3B8358109E84F07'u, lo: 0x0A862F80EC4700C8'u), + uint64x2(hi: 0xF4A642E14C6262C8'u, lo: 0xCD27BB612758C0FA'u), + uint64x2(hi: 0x98E7E9CCCFBD7DBD'u, lo: 0x8038D51CB897789C'u), + uint64x2(hi: 0xBF21E44003ACDD2C'u, lo: 0xE0470A63E6BD56C3'u), + uint64x2(hi: 0xEEEA5D5004981478'u, lo: 0x1858CCFCE06CAC74'u), + uint64x2(hi: 0x95527A5202DF0CCB'u, lo: 0x0F37801E0C43EBC8'u), + uint64x2(hi: 0xBAA718E68396CFFD'u, lo: 0xD30560258F54E6BA'u), + uint64x2(hi: 0xE950DF20247C83FD'u, lo: 0x47C6B82EF32A2069'u), + uint64x2(hi: 0x91D28B7416CDD27E'u, lo: 0x4CDC331D57FA5441'u), + uint64x2(hi: 0xB6472E511C81471D'u, lo: 0xE0133FE4ADF8E952'u), + uint64x2(hi: 0xE3D8F9E563A198E5'u, lo: 0x58180FDDD97723A6'u), + uint64x2(hi: 0x8E679C2F5E44FF8F'u, lo: 0x570F09EAA7EA7648'u), + uint64x2(hi: 0xB201833B35D63F73'u, lo: 0x2CD2CC6551E513DA'u), + uint64x2(hi: 0xDE81E40A034BCF4F'u, lo: 0xF8077F7EA65E58D1'u), + uint64x2(hi: 0x8B112E86420F6191'u, lo: 0xFB04AFAF27FAF782'u), + uint64x2(hi: 0xADD57A27D29339F6'u, lo: 0x79C5DB9AF1F9B563'u), + uint64x2(hi: 0xD94AD8B1C7380874'u, lo: 0x18375281AE7822BC'u), + uint64x2(hi: 0x87CEC76F1C830548'u, lo: 0x8F2293910D0B15B5'u), + uint64x2(hi: 0xA9C2794AE3A3C69A'u, lo: 0xB2EB3875504DDB22'u), + uint64x2(hi: 0xD433179D9C8CB841'u, lo: 0x5FA60692A46151EB'u), + uint64x2(hi: 0x849FEEC281D7F328'u, lo: 0xDBC7C41BA6BCD333'u), + uint64x2(hi: 0xA5C7EA73224DEFF3'u, lo: 0x12B9B522906C0800'u), + uint64x2(hi: 0xCF39E50FEAE16BEF'u, lo: 0xD768226B34870A00'u), + uint64x2(hi: 0x81842F29F2CCE375'u, lo: 0xE6A1158300D46640'u), + uint64x2(hi: 0xA1E53AF46F801C53'u, lo: 0x60495AE3C1097FD0'u), + uint64x2(hi: 0xCA5E89B18B602368'u, lo: 0x385BB19CB14BDFC4'u), + uint64x2(hi: 0xFCF62C1DEE382C42'u, lo: 0x46729E03DD9ED7B5'u), + uint64x2(hi: 0x9E19DB92B4E31BA9'u, lo: 0x6C07A2C26A8346D1'u), + uint64x2(hi: 0xC5A05277621BE293'u, lo: 0xC7098B7305241885'u), + uint64x2(hi: 0xF70867153AA2DB38'u, lo: 0xB8CBEE4FC66D1EA7'u)] + dragonbox_Assert(k >= kMin) + dragonbox_Assert(k <= kMax) + return pow10[k - kMin] + +## Returns whether value is divisible by 2^e2 + +proc multipleOfPow2*(value: uint64; e2: int32): bool {.inline.} = + dragonbox_Assert(e2 >= 0) + return e2 < 64 and (value and ((uint64(1) shl e2) - 1)) == 0 + +## Returns whether value is divisible by 5^e5 + +proc multipleOfPow5*(value: uint64; e5: int32): bool {.inline.} = + type + MulCmp {.bycopy.} = object + mul: uint64 + cmp: uint64 + + const + mod5 = [MulCmp(mul: 0x0000000000000001'u, cmp: 0xFFFFFFFFFFFFFFFF'u), + MulCmp(mul: 0xCCCCCCCCCCCCCCCD'u, cmp: 0x3333333333333333'u), + MulCmp(mul: 0x8F5C28F5C28F5C29'u, cmp: 0x0A3D70A3D70A3D70'u), + MulCmp(mul: 0x1CAC083126E978D5'u, cmp: 0x020C49BA5E353F7C'u), + MulCmp(mul: 0xD288CE703AFB7E91'u, cmp: 0x0068DB8BAC710CB2'u), + MulCmp(mul: 0x5D4E8FB00BCBE61D'u, cmp: 0x0014F8B588E368F0'u), + MulCmp(mul: 0x790FB65668C26139'u, cmp: 0x000431BDE82D7B63'u), + MulCmp(mul: 0xE5032477AE8D46A5'u, cmp: 0x0000D6BF94D5E57A'u), + MulCmp(mul: 0xC767074B22E90E21'u, cmp: 0x00002AF31DC46118'u), + MulCmp(mul: 0x8E47CE423A2E9C6D'u, cmp: 0x0000089705F4136B'u), + MulCmp(mul: 0x4FA7F60D3ED61F49'u, cmp: 0x000001B7CDFD9D7B'u), + MulCmp(mul: 0x0FEE64690C913975'u, cmp: 0x00000057F5FF85E5'u), + MulCmp(mul: 0x3662E0E1CF503EB1'u, cmp: 0x000000119799812D'u), + MulCmp(mul: 0xA47A2CF9F6433FBD'u, cmp: 0x0000000384B84D09'u), + MulCmp(mul: 0x54186F653140A659'u, cmp: 0x00000000B424DC35'u), + MulCmp(mul: 0x7738164770402145'u, cmp: 0x0000000024075F3D'u), + MulCmp(mul: 0xE4A4D1417CD9A041'u, cmp: 0x000000000734ACA5'u), + MulCmp(mul: 0xC75429D9E5C5200D'u, cmp: 0x000000000170EF54'u), + MulCmp(mul: 0xC1773B91FAC10669'u, cmp: 0x000000000049C977'u), + MulCmp(mul: 0x26B172506559CE15'u, cmp: 0x00000000000EC1E4'u), + MulCmp(mul: 0xD489E3A9ADDEC2D1'u, cmp: 0x000000000002F394'u), + MulCmp(mul: 0x90E860BB892C8D5D'u, cmp: 0x000000000000971D'u), + MulCmp(mul: 0x502E79BF1B6F4F79'u, cmp: 0x0000000000001E39'u), + MulCmp(mul: 0xDCD618596BE30FE5'u, cmp: 0x000000000000060B'u), + MulCmp(mul: 0x2C2AD1AB7BFA3661'u, cmp: 0x0000000000000135'u)] + dragonbox_Assert(e5 >= 0) + dragonbox_Assert(e5 <= 24) + let m5: MulCmp = mod5[e5] + return value * m5.mul <= m5.cmp + +type + FloatingDecimal64* {.bycopy.} = object + significand*: uint64 + exponent*: int32 + + +proc toDecimal64AsymmetricInterval*(e2: int32): FloatingDecimal64 {.inline.} = + ## NB: + ## accept_lower_endpoint = true + ## accept_upper_endpoint = true + const + P: int32 = significandSize + ## Compute k and beta + let minusK: int32 = floorLog10ThreeQuartersPow2(e2) + let betaMinus1: int32 = e2 + floorLog2Pow10(-minusK) + ## Compute xi and zi + let pow10: uint64x2 = computePow10(-minusK) + let lowerEndpoint: uint64 = (pow10.hi - (pow10.hi shr (P + 1))) shr + (64 - P - betaMinus1) + let upperEndpoint: uint64 = (pow10.hi + (pow10.hi shr (P + 0))) shr + (64 - P - betaMinus1) + ## If we don't accept the left endpoint (but we do!) or + ## if the left endpoint is not an integer, increase it + let lowerEndpointIsInteger: bool = (2 <= e2 and e2 <= 3) + let xi: uint64 = lowerEndpoint + uint64(not lowerEndpointIsInteger) + let zi: uint64 = upperEndpoint + ## Try bigger divisor + var q: uint64 = zi div 10 + if q * 10 >= xi: + return FloatingDecimal64(significand: q, exponent: minusK + 1) + q = ((pow10.hi shr (64 - (P + 1) - betaMinus1)) + 1) div 2 + ## When tie occurs, choose one of them according to the rule + if e2 == -77: + dec(q, ord(q mod 2 != 0)) + ## Round to even. + else: + inc(q, ord(q < xi)) + return FloatingDecimal64(significand: q, exponent: minusK) + +proc computeDelta*(pow10: uint64x2; betaMinus1: int32): uint32 {.inline.} = + dragonbox_Assert(betaMinus1 >= 0) + dragonbox_Assert(betaMinus1 <= 63) + return cast[uint32](pow10.hi shr (64 - 1 - betaMinus1)) + +when defined(sizeof_Int128): + proc mul128*(x: uint64; y: uint64): uint64x2 {.inline.} = + ## 1 mulx + type + uint128T = uint128 + let p: uint128T = uint128T(x) * y + let hi: uint64 = cast[uint64](p shr 64) + let lo: uint64 = cast[uint64](p) + return (hi, lo) + +elif defined(vcc) and defined(cpu64): + proc umul128(x, y: uint64, z: ptr uint64): uint64 {.importc: "_umul128", header: "<intrin.h>".} + proc mul128*(x: uint64; y: uint64): uint64x2 {.inline.} = + var hi: uint64 = 0 + var lo: uint64 = umul128(x, y, addr(hi)) + return uint64x2(hi: hi, lo: lo) + +else: + proc lo32*(x: uint64): uint32 {.inline.} = + return cast[uint32](x) + + proc hi32*(x: uint64): uint32 {.inline.} = + return cast[uint32](x shr 32) + + proc mul128*(a: uint64; b: uint64): uint64x2 {.inline.} = + let b00: uint64 = uint64(lo32(a)) * lo32(b) + let b01: uint64 = uint64(lo32(a)) * hi32(b) + let b10: uint64 = uint64(hi32(a)) * lo32(b) + let b11: uint64 = uint64(hi32(a)) * hi32(b) + let mid1: uint64 = b10 + hi32(b00) + let mid2: uint64 = b01 + lo32(mid1) + let hi: uint64 = b11 + hi32(mid1) + hi32(mid2) + let lo: uint64 = lo32(b00) or uint64(lo32(mid2)) shl 32 + return uint64x2(hi: hi, lo: lo) + +## Returns (x * y) / 2^128 + +proc mulShift*(x: uint64; y: uint64x2): uint64 {.inline.} = + ## 2 mulx + var p1: uint64x2 = mul128(x, y.hi) + var p0: uint64x2 = mul128(x, y.lo) + p1.lo += p0.hi + inc(p1.hi, ord(p1.lo < p0.hi)) + return p1.hi + +proc mulParity*(twoF: uint64; pow10: uint64x2; betaMinus1: int32): bool {.inline.} = + ## 1 mulx, 1 mul + dragonbox_Assert(betaMinus1 >= 1) + dragonbox_Assert(betaMinus1 <= 63) + let p01: uint64 = twoF * pow10.hi + let p10: uint64 = mul128(twoF, pow10.lo).hi + let mid: uint64 = p01 + p10 + return (mid and (uint64(1) shl (64 - betaMinus1))) != 0 + +proc isIntegralEndpoint*(twoF: uint64; e2: int32; minusK: int32): bool {.inline.} = + if e2 < -2: + return false + if e2 <= 9: + return true + if e2 <= 86: + return multipleOfPow5(twoF, minusK) + return false + +proc isIntegralMidpoint*(twoF: uint64; e2: int32; minusK: int32): bool {.inline.} = + if e2 < -4: + return multipleOfPow2(twoF, minusK - e2 + 1) + if e2 <= 9: + return true + if e2 <= 86: + return multipleOfPow5(twoF, minusK) + return false + +proc toDecimal64*(ieeeSignificand: uint64; ieeeExponent: uint64): FloatingDecimal64 {. + inline.} = + const + kappa: int32 = 2 + const + bigDivisor: uint32 = 1000 + ## 10^(kappa + 1) + const + smallDivisor: uint32 = 100 + ## 10^(kappa) + ## + ## Step 1: + ## integer promotion & Schubfach multiplier calculation. + ## + var m2: uint64 + var e2: int32 + if ieeeExponent != 0: + m2 = hiddenBit or ieeeSignificand + e2 = cast[int32](ieeeExponent) - exponentBias + if 0 <= -e2 and -e2 < significandSize and multipleOfPow2(m2, -e2): + ## Small integer. + return FloatingDecimal64(significand: m2 shr -e2, exponent: 0) + if ieeeSignificand == 0 and ieeeExponent > 1: + ## Shorter interval case; proceed like Schubfach. + return toDecimal64AsymmetricInterval(e2) + else: + ## Subnormal case; interval is always regular. + m2 = ieeeSignificand + e2 = 1 - exponentBias + let isEven: bool = (m2 mod 2 == 0) + let acceptLower: bool = isEven + let acceptUpper: bool = isEven + ## Compute k and beta. + let minusK: int32 = floorLog10Pow2(e2) - kappa + let betaMinus1: int32 = e2 + floorLog2Pow10(-minusK) + dragonbox_Assert(betaMinus1 >= 6) + dragonbox_Assert(betaMinus1 <= 9) + let pow10: uint64x2 = computePow10(-minusK) + ## Compute delta + ## 10^kappa <= delta < 10^(kappa + 1) + ## 100 <= delta < 1000 + let delta: uint32 = computeDelta(pow10, betaMinus1) + dragonbox_Assert(delta >= smallDivisor) + dragonbox_Assert(delta < bigDivisor) + let twoFl: uint64 = 2 * m2 - 1 + let twoFc: uint64 = 2 * m2 + let twoFr: uint64 = 2 * m2 + 1 + ## (54 bits) + ## Compute zi + ## (54 + 9 = 63 bits) + let zi: uint64 = mulShift(twoFr shl betaMinus1, pow10) + ## 2 mulx + ## + ## Step 2: + ## Try larger divisor. + ## + var q: uint64 = zi div bigDivisor + ## uint64_t q = Mul128(zi, 0x83126E978D4FDF3Cu).hi >> 9; // 1 mulx + var r: uint32 = cast[uint32](zi) - bigDivisor * cast[uint32](q) + ## r = zi % BigDivisor + ## 0 <= r < 1000 + if r < delta: ## likely ~50% ?! + ## (r > deltai) + ## Exclude the right endpoint if necessary + if r != 0 or acceptUpper or not isIntegralEndpoint(twoFr, e2, minusK): + return FloatingDecimal64(significand: q, exponent: minusK + kappa + 1) + dragonbox_Assert(q != 0) + dec(q) + r = bigDivisor + elif r == delta: ## unlikely + ## Compare fractional parts. + ## Check conditions in the order different from the paper + ## to take advantage of short-circuiting + if (acceptLower and isIntegralEndpoint(twoFl, e2, minusK)) or + mulParity(twoFl, pow10, betaMinus1): + return FloatingDecimal64(significand: q, exponent: minusK + kappa + 1) + else: + discard + ## + ## Step 3: + ## Find the significand with the smaller divisor + ## + q = q * 10 + ## 1 hmul + ## 0 <= r <= 1000 + let dist: uint32 = r - (delta div 2) + (smallDivisor div 2) + let distQ: uint32 = dist div 100 + ## 1 mul + ## const uint32_t dist_r = dist % 100; + q += distQ + ## if /*likely*/ (dist_r == 0) + if dist == distQ * 100: + ## const bool approx_y_parity = ((dist ^ (SmallDivisor / 2)) & 1) != 0; + let approxYParity: bool = (dist and 1) != 0 + ## Check z^(f) >= epsilon^(f) + ## We have either yi == zi - epsiloni or yi == (zi - epsiloni) - 1, + ## where yi == zi - epsiloni if and only if z^(f) >= epsilon^(f) + ## Since there are only 2 possibilities, we only need to care about the + ## parity. Also, zi and r should have the same parity since the divisor + ## is an even number + if mulParity(twoFc, pow10, betaMinus1) != approxYParity: + dec(q) + elif q mod 2 != 0 and isIntegralMidpoint(twoFc, e2, minusK): + dec(q) + return FloatingDecimal64(significand: q, exponent: minusK + kappa) + +# ================================================================================================== +# ToChars +# ================================================================================================== + +when false: + template `+!`(x: cstring; offset: int): cstring = cast[cstring](cast[uint](x) + uint(offset)) + + template dec(x: cstring; offset=1) = x = cast[cstring](cast[uint](x) - uint(offset)) + template inc(x: cstring; offset=1) = x = cast[cstring](cast[uint](x) + uint(offset)) + + proc memset(x: cstring; ch: char; L: int) {.importc, nodecl.} + proc memmove(a, b: cstring; L: int) {.importc, nodecl.} + +proc utoa8DigitsSkipTrailingZeros*(buf: var openArray[char]; pos: int; digits: uint32): int {.inline.} = + dragonbox_Assert(digits >= 1) + dragonbox_Assert(digits <= 99999999'u32) + let q: uint32 = digits div 10000 + let r: uint32 = digits mod 10000 + let qH: uint32 = q div 100 + let qL: uint32 = q mod 100 + utoa2Digits(buf, pos, qH) + utoa2Digits(buf, pos + 2, qL) + if r == 0: + return trailingZeros2Digits(if qL == 0: qH else: qL) + (if qL == 0: 6 else: 4) + else: + let rH: uint32 = r div 100 + let rL: uint32 = r mod 100 + utoa2Digits(buf, pos + 4, rH) + utoa2Digits(buf, pos + 6, rL) + return trailingZeros2Digits(if rL == 0: rH else: rL) + (if rL == 0: 2 else: 0) + +proc printDecimalDigitsBackwards*(buf: var openArray[char]; pos: int; output64: uint64): int {.inline.} = + var pos = pos + var output64 = output64 + var tz = 0 + ## number of trailing zeros removed. + var nd = 0 + ## number of decimal digits processed. + ## At most 17 digits remaining + if output64 >= 100000000'u64: + let q: uint64 = output64 div 100000000'u64 + let r: uint32 = cast[uint32](output64 mod 100000000'u64) + output64 = q + dec(pos, 8) + if r != 0: + tz = utoa8DigitsSkipTrailingZeros(buf, pos, r) + dragonbox_Assert(tz >= 0) + dragonbox_Assert(tz <= 7) + else: + tz = 8 + nd = 8 + dragonbox_Assert(output64 <= high(uint32)) + var output = cast[uint32](output64) + if output >= 10000: + let q: uint32 = output div 10000 + let r: uint32 = output mod 10000 + output = q + dec(pos, 4) + if r != 0: + let rH: uint32 = r div 100 + let rL: uint32 = r mod 100 + utoa2Digits(buf, pos, rH) + utoa2Digits(buf, pos + 2, rL) + if tz == nd: + inc(tz, trailingZeros2Digits(if rL == 0: rH else: rL) + + (if rL == 0: 2 else: 0)) + else: + if tz == nd: + inc(tz, 4) + else: + for i in 0..3: buf[pos+i] = '0' + ## (actually not required...) + inc(nd, 4) + if output >= 100: + let q: uint32 = output div 100 + let r: uint32 = output mod 100 + output = q + dec(pos, 2) + utoa2Digits(buf, pos, r) + if tz == nd: + inc(tz, trailingZeros2Digits(r)) + inc(nd, 2) + if output >= 100: + let q2: uint32 = output div 100 + let r2: uint32 = output mod 100 + output = q2 + dec(pos, 2) + utoa2Digits(buf, pos, r2) + if tz == nd: + inc(tz, trailingZeros2Digits(r2)) + inc(nd, 2) + dragonbox_Assert(output >= 1) + dragonbox_Assert(output <= 99) + if output >= 10: + let q: uint32 = output + dec(pos, 2) + utoa2Digits(buf, pos, q) + if tz == nd: + inc(tz, trailingZeros2Digits(q)) + else: + let q: uint32 = output + dragonbox_Assert(q >= 1) + dragonbox_Assert(q <= 9) + dec(pos) + buf[pos] = chr(ord('0') + q) + return tz + +proc decimalLength*(v: uint64): int {.inline.} = + dragonbox_Assert(v >= 1) + dragonbox_Assert(v <= 99999999999999999'u64) + if cast[uint32](v shr 32) != 0: + if v >= 10000000000000000'u64: + return 17 + if v >= 1000000000000000'u64: + return 16 + if v >= 100000000000000'u64: + return 15 + if v >= 10000000000000'u64: + return 14 + if v >= 1000000000000'u64: + return 13 + if v >= 100000000000'u64: + return 12 + if v >= 10000000000'u64: + return 11 + return 10 + let v32: uint32 = cast[uint32](v) + if v32 >= 1000000000'u32: + return 10 + if v32 >= 100000000'u32: + return 9 + if v32 >= 10000000'u32: + return 8 + if v32 >= 1000000'u32: + return 7 + if v32 >= 100000'u32: + return 6 + if v32 >= 10000'u32: + return 5 + if v32 >= 1000'u32: + return 4 + if v32 >= 100'u32: + return 3 + if v32 >= 10'u32: + return 2 + return 1 + +proc formatDigits*[T: Ordinal](buffer: var openArray[char]; pos: T; digits: uint64; decimalExponent: int; + forceTrailingDotZero = false): int {.inline.} = + const + minFixedDecimalPoint = -6 + const + maxFixedDecimalPoint = 17 + var pos:int = pos.int + assert(minFixedDecimalPoint <= -1, "internal error") + assert(maxFixedDecimalPoint >= 17, "internal error") + dragonbox_Assert(digits >= 1) + dragonbox_Assert(digits <= 99999999999999999'u64) + dragonbox_Assert(decimalExponent >= -999) + dragonbox_Assert(decimalExponent <= 999) + var numDigits = decimalLength(digits) + let decimalPoint = numDigits + decimalExponent + let useFixed: bool = minFixedDecimalPoint <= decimalPoint and + decimalPoint <= maxFixedDecimalPoint + ## Prepare the buffer. + for i in 0..<32: buffer[pos+i] = '0' + assert(minFixedDecimalPoint >= -30, "internal error") + assert(maxFixedDecimalPoint <= 32, "internal error") + var decimalDigitsPosition: int + if useFixed: + if decimalPoint <= 0: + ## 0.[000]digits + decimalDigitsPosition = 2 - decimalPoint + else: + ## dig.its + ## digits[000] + decimalDigitsPosition = 0 + else: + ## dE+123 or d.igitsE+123 + decimalDigitsPosition = 1 + var digitsEnd = pos + int(decimalDigitsPosition + numDigits) + let tz = printDecimalDigitsBackwards(buffer, digitsEnd, digits) + dec(digitsEnd, tz) + dec(numDigits, tz) + ## decimal_exponent += tz; // => decimal_point unchanged. + if useFixed: + if decimalPoint <= 0: + ## 0.[000]digits + buffer[pos+1] = '.' + pos = digitsEnd + elif decimalPoint < numDigits: + ## dig.its + when true: #defined(vcc) and not defined(clang): + ## VC does not inline the memmove call below. (Even if compiled with /arch:AVX2.) + ## However, memcpy will be inlined. + var tmp: array[16, char] + for i in 0..<16: tmp[i] = buffer[i+pos+decimalPoint] + for i in 0..<16: buffer[i+pos+decimalPoint+1] = tmp[i] + else: + memmove(buffer +! (decimalPoint + 1), buffer +! decimalPoint, 16) + buffer[pos+decimalPoint] = '.' + pos = digitsEnd + 1 + else: + ## digits[000] + inc(pos, decimalPoint) + if forceTrailingDotZero: + buffer[pos] = '.' + buffer[pos+1] = '0' + inc(pos, 2) + else: + ## Copy the first digit one place to the left. + buffer[pos] = buffer[pos+1] + if numDigits == 1: + ## dE+123 + inc(pos) + else: + ## d.igitsE+123 + buffer[pos+1] = '.' + pos = digitsEnd + let scientificExponent: int = decimalPoint - 1 + ## SF_ASSERT(scientific_exponent != 0); + buffer[pos] = 'e' + buffer[pos+1] = if scientificExponent < 0: '-' else: '+' + inc(pos, 2) + let k: uint32 = cast[uint32](if scientificExponent < 0: -scientificExponent else: scientificExponent) + if k < 10: + buffer[pos] = chr(ord('0') + k) + inc(pos) + elif k < 100: + utoa2Digits(buffer, pos, k) + inc(pos, 2) + else: + let q: uint32 = k div 100 + let r: uint32 = k mod 100 + buffer[pos] = chr(ord('0') + q) + inc(pos) + utoa2Digits(buffer, pos, r) + inc(pos, 2) + return pos + +proc toChars*(buffer: var openArray[char]; v: float; forceTrailingDotZero = false): int {. + inline.} = + var pos = 0 + let significand: uint64 = physicalSignificand(constructDouble(v)) + let exponent: uint64 = physicalExponent(constructDouble(v)) + if exponent != maxIeeeExponent: + ## Finite + buffer[pos] = '-' + inc(pos, signBit(constructDouble(v))) + if exponent != 0 or significand != 0: + ## != 0 + let dec = toDecimal64(significand, exponent) + return formatDigits(buffer, pos, dec.significand, dec.exponent.int, + forceTrailingDotZero) + else: + buffer[pos] = '0' + buffer[pos+1] = '.' + buffer[pos+2] = '0' + buffer[pos+3] = ' ' + inc(pos, if forceTrailingDotZero: 3 else: 1) + return pos + if significand == 0: + buffer[pos] = '-' + inc(pos, signBit(constructDouble(v))) + buffer[pos] = 'i' + buffer[pos+1] = 'n' + buffer[pos+2] = 'f' + buffer[pos+3] = ' ' + return pos + 3 + else: + buffer[pos] = 'n' + buffer[pos+1] = 'a' + buffer[pos+2] = 'n' + buffer[pos+3] = ' ' + return pos + 3 + +when false: + proc toString*(value: float): string = + var buffer: array[dtoaMinBufferLength, char] + let last = toChars(addr buffer, value) + let L = cast[int](last) - cast[int](addr(buffer)) + result = newString(L) + copyMem(addr result[0], addr buffer[0], L) + diff --git a/lib/std/private/gitutils.nim b/lib/std/private/gitutils.nim index bf5e7cb1f..6dc9c8f3b 100644 --- a/lib/std/private/gitutils.nim +++ b/lib/std/private/gitutils.nim @@ -4,7 +4,10 @@ internal API for now, API subject to change # xxx move other git utilities here; candidate for stdlib. -import std/[os, osproc, strutils] +import std/[os, paths, osproc, strutils, tempfiles] + +when defined(nimPreviewSlimSystem): + import std/[assertions, syncio] const commitHead* = "HEAD" @@ -29,12 +32,43 @@ template retryCall*(maxRetry = 3, backoffDuration = 1.0, call: untyped): bool = result proc isGitRepo*(dir: string): bool = - ## This command is used to get the relative path to the root of the repository. - ## Using this, we can verify whether a folder is a git repository by checking - ## whether the command success and if the output is empty. - let (output, status) = execCmdEx("git rev-parse --show-cdup", workingDir = dir) - # On Windows there will be a trailing newline on success, remove it. - # The value of a successful call typically won't have a whitespace (it's - # usually a series of ../), so we know that it's safe to unconditionally - # remove trailing whitespaces from the result. - result = status == 0 and output.strip() == "" + ## Avoid calling git since it depends on /bin/sh existing and fails in Nix. + return fileExists(dir/".git/HEAD") + +proc diffFiles*(path1, path2: string): tuple[output: string, same: bool] = + ## Returns a human readable diff of files `path1`, `path2`, the exact form of + ## which is implementation defined. + # This could be customized, e.g. non-git diff with `diff -uNdr`, or with + # git diff options (e.g. --color-moved, --word-diff). + # in general, `git diff` has more options than `diff`. + var status = 0 + (result.output, status) = execCmdEx("git diff --no-index $1 $2" % [path1.quoteShell, path2.quoteShell]) + doAssert (status == 0) or (status == 1) + result.same = status == 0 + +proc diffStrings*(a, b: string): tuple[output: string, same: bool] = + ## Returns a human readable diff of `a`, `b`, the exact form of which is + ## implementation defined. + ## See also `experimental.diff`. + runnableExamples: + let a = "ok1\nok2\nok3\n" + let b = "ok1\nok2 alt\nok3\nok4\n" + let (c, same) = diffStrings(a, b) + doAssert not same + let (c2, same2) = diffStrings(a, a) + doAssert same2 + runnableExamples("-r:off"): + let a = "ok1\nok2\nok3\n" + let b = "ok1\nok2 alt\nok3\nok4\n" + echo diffStrings(a, b).output + + template tmpFileImpl(prefix, str): auto = + let path = genTempPath(prefix, "") + writeFile(path, str) + path + let patha = tmpFileImpl("diffStrings_a_", a) + let pathb = tmpFileImpl("diffStrings_b_", b) + defer: + removeFile(patha) + removeFile(pathb) + result = diffFiles(patha, pathb) diff --git a/lib/std/private/globs.nim b/lib/std/private/globs.nim index b98a7808b..a6d088558 100644 --- a/lib/std/private/globs.nim +++ b/lib/std/private/globs.nim @@ -4,9 +4,18 @@ this can eventually be moved to std/os and `walkDirRec` can be implemented in te to avoid duplication ]## -import std/[os] +import std/os when defined(windows): - from strutils import replace + from std/strutils import replace + +when defined(nimPreviewSlimSystem): + import std/[assertions, objectdollar] + + +when defined(nimHasEffectsOf): + {.experimental: "strictEffects".} +else: + {.pragma: effectsOf.} type PathEntry* = object @@ -14,7 +23,7 @@ type path*: string iterator walkDirRecFilter*(dir: string, follow: proc(entry: PathEntry): bool = nil, - relative = false, checkDir = true): PathEntry {.tags: [ReadDirEffect].} = + relative = false, checkDir = true): PathEntry {.tags: [ReadDirEffect], effectsOf: follow.} = ## Improved `os.walkDirRec`. #[ note: a yieldFilter isn't needed because caller can filter at call site, without @@ -45,12 +54,17 @@ iterator walkDirRecFilter*(dir: string, follow: proc(entry: PathEntry): bool = n proc nativeToUnixPath*(path: string): string = # pending https://github.com/nim-lang/Nim/pull/13265 - doAssert not path.isAbsolute # not implemented here; absolute files need more care for the drive + result = path + when defined(windows): + if path.len >= 2 and path[0] in {'a'..'z', 'A'..'Z'} and path[1] == ':': + result[0] = '/' + result[1] = path[0] + if path.len > 2 and path[2] != '\\': + raiseAssert "paths like `C:foo` are currently unsupported, path: " & path when DirSep == '\\': - result = replace(path, '\\', '/') - else: result = path + result = replace(result, '\\', '/') when isMainModule: - import sugar - for a in walkDirRecFilter(".", follow = a=>a.path.lastPathPart notin ["nimcache", ".git", ".csources", "bin"]): + import std/sugar + for a in walkDirRecFilter(".", follow = a=>a.path.lastPathPart notin ["nimcache", ".git", "csources_v1", "csources", "bin"]): echo a diff --git a/lib/std/private/jsutils.nim b/lib/std/private/jsutils.nim index 836b3512a..5f79eab27 100644 --- a/lib/std/private/jsutils.nim +++ b/lib/std/private/jsutils.nim @@ -37,13 +37,13 @@ when defined(js): let a = array[2, float64].default assert jsConstructorName(a) == "Float64Array" assert jsConstructorName(a.toJs) == "Float64Array" - asm """`result` = `a`.constructor.name""" + {.emit: """`result` = `a`.constructor.name;""".} proc hasJsBigInt*(): bool = - asm """`result` = typeof BigInt != 'undefined'""" + {.emit: """`result` = typeof BigInt != 'undefined';""".} proc hasBigUint64Array*(): bool = - asm """`result` = typeof BigUint64Array != 'undefined'""" + {.emit: """`result` = typeof BigUint64Array != 'undefined';""".} proc getProtoName*[T](a: T): cstring {.importjs: "Object.prototype.toString.call(#)".} = runnableExamples: @@ -79,5 +79,18 @@ when defined(js): assert not "123".toJs.isSafeInteger assert 123.isSafeInteger assert 123.toJs.isSafeInteger - assert 9007199254740991.toJs.isSafeInteger - assert not 9007199254740992.toJs.isSafeInteger + when false: + assert 9007199254740991.toJs.isSafeInteger + assert not 9007199254740992.toJs.isSafeInteger + +template whenJsNoBigInt64*(no64, yes64): untyped = + when defined(js): + when compiles(compileOption("jsbigint64")): + when compileOption("jsbigint64"): + yes64 + else: + no64 + else: + no64 + else: + no64 diff --git a/lib/std/private/miscdollars.nim b/lib/std/private/miscdollars.nim index a41cf1bc1..06fda6fa1 100644 --- a/lib/std/private/miscdollars.nim +++ b/lib/std/private/miscdollars.nim @@ -1,3 +1,5 @@ +from std/private/digitsutils import addInt + template toLocation*(result: var string, file: string | cstring, line: int, col: int) = ## avoids spurious allocations # Hopefully this can be re-used everywhere so that if a user needs to customize, @@ -5,11 +7,33 @@ template toLocation*(result: var string, file: string | cstring, line: int, col: result.add file if line > 0: result.add "(" - # simplify this after moving moving `include strmantle` above import assertions` - when declared(addInt): result.addInt line - else: result.add $line + addInt(result, line) if col > 0: result.add ", " - when declared(addInt): result.addInt col - else: result.add $col + addInt(result, col) result.add ")" + +proc isNamedTuple(T: typedesc): bool {.magic: "TypeTrait".} + +template tupleObjectDollar*[T: tuple | object](result: var string, x: T) = + result = "(" + const isNamed = T is object or isNamedTuple(typeof(T)) + var count {.used.} = 0 + for name, value in fieldPairs(x): + if count > 0: result.add(", ") + when isNamed: + result.add(name) + result.add(": ") + count.inc + when compiles($value): + when value isnot string and value isnot seq and compiles(value.isNil): + if value.isNil: result.add "nil" + else: result.addQuoted(value) + else: + result.addQuoted(value) + else: + result.add("...") + when not isNamed: + if count == 1: + result.add(",") # $(1,) should print as the semantically legal (1,) + result.add(")") diff --git a/lib/std/private/ntpath.nim b/lib/std/private/ntpath.nim new file mode 100644 index 000000000..7c8661bb7 --- /dev/null +++ b/lib/std/private/ntpath.nim @@ -0,0 +1,61 @@ +# This module is inspired by Python's `ntpath.py` module. + +import std/[ + strutils, +] + +# Adapted `splitdrive` function from the following commits in Python source +# code: +# 5a607a3ee5e81bdcef3f886f9d20c1376a533df4 (2009): Initial UNC handling (by Mark Hammond) +# 2ba0fd5767577954f331ecbd53596cd8035d7186 (2022): Support for "UNC"-device paths (by Barney Gale) +# +# FAQ: Why use `strip` below? `\\?\UNC` is the start of a "UNC symbolic link", +# which is a special UNC form. Running `strip` differentiates `\\?\UNC\` (a UNC +# symbolic link) from e.g. `\\?\UNCD` (UNCD is the server in the UNC path). +func splitDrive*(p: string): tuple[drive, path: string] = + ## Splits a Windows path into a drive and path part. The drive can be e.g. + ## `C:`. It can also be a UNC path (`\\server\drive\path`). + ## + ## The equivalent `splitDrive` for POSIX systems always returns empty drive. + ## Therefore this proc is only necessary on DOS-like file systems (together + ## with Nim's `doslikeFileSystem` conditional variable). + ## + ## This proc's use case is to extract `path` such that it can be manipulated + ## like a POSIX path. + runnableExamples: + doAssert splitDrive("C:") == ("C:", "") + doAssert splitDrive(r"C:\") == (r"C:", r"\") + doAssert splitDrive(r"\\server\drive\foo\bar") == (r"\\server\drive", r"\foo\bar") + doAssert splitDrive(r"\\?\UNC\server\share\dir") == (r"\\?\UNC\server\share", r"\dir") + + result = ("", p) + if p.len < 2: + return + const sep = '\\' + let normp = p.replace('/', sep) + if p.len > 2 and normp[0] == sep and normp[1] == sep and normp[2] != sep: + + # is a UNC path: + # vvvvvvvvvvvvvvvvvvvv drive letter or UNC path + # \\machine\mountpoint\directory\etc\... + # directory ^^^^^^^^^^^^^^^ + let start = block: + const unc = "\\\\?\\UNC" # Length is 7 + let idx = min(8, normp.len) + if unc == normp[0..<idx].strip(chars = {sep}, leading = false).toUpperAscii: + 8 + else: + 2 + let index = normp.find(sep, start) + if index == -1: + return + var index2 = normp.find(sep, index + 1) + + # a UNC path can't have two slashes in a row (after the initial two) + if index2 == index + 1: + return + if index2 == -1: + index2 = p.len + return (p[0..<index2], p[index2..^1]) + if p[1] == ':': + return (p[0..1], p[2..^1]) diff --git a/lib/std/private/osappdirs.nim b/lib/std/private/osappdirs.nim new file mode 100644 index 000000000..07a6809bb --- /dev/null +++ b/lib/std/private/osappdirs.nim @@ -0,0 +1,176 @@ +## .. importdoc:: paths.nim, dirs.nim + +include system/inclrtl +import std/envvars +import std/private/ospaths2 + +proc getHomeDir*(): string {.rtl, extern: "nos$1", + tags: [ReadEnvEffect, ReadIOEffect].} = + ## Returns the home directory of the current user. + ## + ## This proc is wrapped by the `expandTilde proc`_ + ## for the convenience of processing paths coming from user configuration files. + ## + ## See also: + ## * `getDataDir proc`_ + ## * `getConfigDir proc`_ + ## * `getTempDir proc`_ + ## * `expandTilde proc`_ + ## * `getCurrentDir proc`_ + ## * `setCurrentDir proc`_ + runnableExamples: + import std/os + assert getHomeDir() == expandTilde("~") + + when defined(windows): return getEnv("USERPROFILE") & "\\" + else: return getEnv("HOME") & "/" + +proc getDataDir*(): string {.rtl, extern: "nos$1" + tags: [ReadEnvEffect, ReadIOEffect].} = + ## Returns the data directory of the current user for applications. + ## + ## On non-Windows OSs, this proc conforms to the XDG Base Directory + ## spec. Thus, this proc returns the value of the `XDG_DATA_HOME` environment + ## variable if it is set, otherwise it returns the default configuration + ## directory ("~/.local/share" or "~/Library/Application Support" on macOS). + ## + ## See also: + ## * `getHomeDir proc`_ + ## * `getConfigDir proc`_ + ## * `getTempDir proc`_ + ## * `expandTilde proc`_ + ## * `getCurrentDir proc`_ + ## * `setCurrentDir proc`_ + when defined(windows): + result = getEnv("APPDATA") + elif defined(macosx): + result = getEnv("XDG_DATA_HOME", getEnv("HOME") / "Library" / "Application Support") + else: + result = getEnv("XDG_DATA_HOME", getEnv("HOME") / ".local" / "share") + result.normalizePathEnd(trailingSep = true) + +proc getConfigDir*(): string {.rtl, extern: "nos$1", + tags: [ReadEnvEffect, ReadIOEffect].} = + ## Returns the config directory of the current user for applications. + ## + ## On non-Windows OSs, this proc conforms to the XDG Base Directory + ## spec. Thus, this proc returns the value of the `XDG_CONFIG_HOME` environment + ## variable if it is set, otherwise it returns the default configuration + ## directory ("~/.config/"). + ## + ## An OS-dependent trailing slash is always present at the end of the + ## returned string: `\\` on Windows and `/` on all other OSs. + ## + ## See also: + ## * `getHomeDir proc`_ + ## * `getDataDir proc`_ + ## * `getTempDir proc`_ + ## * `expandTilde proc`_ + ## * `getCurrentDir proc`_ + ## * `setCurrentDir proc`_ + when defined(windows): + result = getEnv("APPDATA") + else: + result = getEnv("XDG_CONFIG_HOME", getEnv("HOME") / ".config") + result.normalizePathEnd(trailingSep = true) + +proc getCacheDir*(): string = + ## Returns the cache directory of the current user for applications. + ## + ## This makes use of the following environment variables: + ## + ## * On Windows: `getEnv("LOCALAPPDATA")` + ## + ## * On macOS: `getEnv("XDG_CACHE_HOME", getEnv("HOME") / "Library/Caches")` + ## + ## * On other platforms: `getEnv("XDG_CACHE_HOME", getEnv("HOME") / ".cache")` + ## + ## **See also:** + ## * `getHomeDir proc`_ + ## * `getTempDir proc`_ + ## * `getConfigDir proc`_ + ## * `getDataDir proc`_ + # follows https://crates.io/crates/platform-dirs + when defined(windows): + result = getEnv("LOCALAPPDATA") + elif defined(osx): + result = getEnv("XDG_CACHE_HOME", getEnv("HOME") / "Library/Caches") + else: + result = getEnv("XDG_CACHE_HOME", getEnv("HOME") / ".cache") + result.normalizePathEnd(false) + +proc getCacheDir*(app: string): string = + ## Returns the cache directory for an application `app`. + ## + ## * On Windows, this uses: `getCacheDir() / app / "cache"` + ## * On other platforms, this uses: `getCacheDir() / app` + when defined(windows): + getCacheDir() / app / "cache" + else: + getCacheDir() / app + + +when defined(windows): + type DWORD = uint32 + + when defined(nimPreviewSlimSystem): + import std/widestrs + + proc getTempPath( + nBufferLength: DWORD, lpBuffer: WideCString + ): DWORD {.stdcall, dynlib: "kernel32.dll", importc: "GetTempPathW".} = + ## Retrieves the path of the directory designated for temporary files. + +template getEnvImpl(result: var string, tempDirList: openArray[string]) = + for dir in tempDirList: + if existsEnv(dir): + result = getEnv(dir) + break + +template getTempDirImpl(result: var string) = + when defined(windows): + getEnvImpl(result, ["TMP", "TEMP", "USERPROFILE"]) + else: + getEnvImpl(result, ["TMPDIR", "TEMP", "TMP", "TEMPDIR"]) + +proc getTempDir*(): string {.rtl, extern: "nos$1", + tags: [ReadEnvEffect, ReadIOEffect].} = + ## Returns the temporary directory of the current user for applications to + ## save temporary files in. + ## + ## On Windows, it calls [GetTempPath](https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-gettemppathw). + ## On Posix based platforms, it will check `TMPDIR`, `TEMP`, `TMP` and `TEMPDIR` environment variables in order. + ## On all platforms, `/tmp` will be returned if the procs fails. + ## + ## You can override this implementation + ## by adding `-d:tempDir=mytempname` to your compiler invocation. + ## + ## **Note:** This proc does not check whether the returned path exists. + ## + ## See also: + ## * `getHomeDir proc`_ + ## * `getConfigDir proc`_ + ## * `expandTilde proc`_ + ## * `getCurrentDir proc`_ + ## * `setCurrentDir proc`_ + const tempDirDefault = "/tmp" + when defined(tempDir): + const tempDir {.strdefine.}: string = tempDirDefault + result = tempDir + else: + when nimvm: + getTempDirImpl(result) + else: + when defined(windows): + let size = getTempPath(0, nil) + # If the function fails, the return value is zero. + if size > 0: + let buffer = newWideCString(size.int) + if getTempPath(size, buffer) > 0: + result = $buffer + elif defined(android): result = "/data/local/tmp" + else: + getTempDirImpl(result) + if result.len == 0: + result = tempDirDefault + normalizePathEnd(result, trailingSep=true) diff --git a/lib/std/private/oscommon.nim b/lib/std/private/oscommon.nim new file mode 100644 index 000000000..c49d52ef2 --- /dev/null +++ b/lib/std/private/oscommon.nim @@ -0,0 +1,186 @@ +include system/inclrtl + +import std/[oserrors] + +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions, widestrs] + +## .. importdoc:: osdirs.nim, os.nim + +const weirdTarget* = defined(nimscript) or defined(js) + + +type + ReadDirEffect* = object of ReadIOEffect ## Effect that denotes a read + ## operation from the directory + ## structure. + WriteDirEffect* = object of WriteIOEffect ## Effect that denotes a write + ## operation to + ## the directory structure. + + +when weirdTarget: + discard +elif defined(windows): + import std/[winlean, times] +elif defined(posix): + import std/posix + proc c_rename(oldname, newname: cstring): cint {. + importc: "rename", header: "<stdio.h>".} +else: + {.error: "OS module not ported to your operating system!".} + + +when weirdTarget: + {.pragma: noWeirdTarget, error: "this proc is not available on the NimScript/js target".} +else: + {.pragma: noWeirdTarget.} + + +when defined(nimscript): + # for procs already defined in scriptconfig.nim + template noNimJs(body): untyped = discard +elif defined(js): + {.pragma: noNimJs, error: "this proc is not available on the js target".} +else: + {.pragma: noNimJs.} + + +when defined(windows) and not weirdTarget: + template wrapUnary*(varname, winApiProc, arg: untyped) = + var varname = winApiProc(newWideCString(arg)) + + template wrapBinary*(varname, winApiProc, arg, arg2: untyped) = + var varname = winApiProc(newWideCString(arg), arg2) + proc findFirstFile*(a: string, b: var WIN32_FIND_DATA): Handle = + result = findFirstFileW(newWideCString(a), b) + template findNextFile*(a, b: untyped): untyped = findNextFileW(a, b) + + template getFilename*(f: untyped): untyped = + $cast[WideCString](addr(f.cFileName[0])) + + proc skipFindData*(f: WIN32_FIND_DATA): bool {.inline.} = + # Note - takes advantage of null delimiter in the cstring + const dot = ord('.') + result = f.cFileName[0].int == dot and (f.cFileName[1].int == 0 or + f.cFileName[1].int == dot and f.cFileName[2].int == 0) + + +type + PathComponent* = enum ## Enumeration specifying a path component. + ## + ## See also: + ## * `walkDirRec iterator`_ + ## * `FileInfo object`_ + pcFile, ## path refers to a file + pcLinkToFile, ## path refers to a symbolic link to a file + pcDir, ## path refers to a directory + pcLinkToDir ## path refers to a symbolic link to a directory + + +when defined(posix) and not weirdTarget: + proc getSymlinkFileKind*(path: string): + tuple[pc: PathComponent, isSpecial: bool] = + # Helper function. + var s: Stat + assert(path != "") + result = (pcLinkToFile, false) + if stat(path, s) == 0'i32: + if S_ISDIR(s.st_mode): + result = (pcLinkToDir, false) + elif not S_ISREG(s.st_mode): + result = (pcLinkToFile, true) + +proc tryMoveFSObject*(source, dest: string, isDir: bool): bool {.noWeirdTarget.} = + ## Moves a file (or directory if `isDir` is true) from `source` to `dest`. + ## + ## Returns false in case of `EXDEV` error or `AccessDeniedError` on Windows (if `isDir` is true). + ## In case of other errors `OSError` is raised. + ## Returns true in case of success. + when defined(windows): + let s = newWideCString(source) + let d = newWideCString(dest) + result = moveFileExW(s, d, MOVEFILE_COPY_ALLOWED or MOVEFILE_REPLACE_EXISTING) != 0'i32 + else: + result = c_rename(source, dest) == 0'i32 + + if not result: + let err = osLastError() + let isAccessDeniedError = + when defined(windows): + const AccessDeniedError = OSErrorCode(5) + isDir and err == AccessDeniedError + else: + err == EXDEV.OSErrorCode + if not isAccessDeniedError: + raiseOSError(err, $(source, dest)) + +when not defined(windows): + const maxSymlinkLen* = 1024 + +proc fileExists*(filename: string): bool {.rtl, extern: "nos$1", + tags: [ReadDirEffect], noNimJs, sideEffect.} = + ## Returns true if `filename` exists and is a regular file or symlink. + ## + ## Directories, device files, named pipes and sockets return false. + ## + ## See also: + ## * `dirExists proc`_ + ## * `symlinkExists proc`_ + when defined(windows): + wrapUnary(a, getFileAttributesW, filename) + if a != -1'i32: + result = (a and FILE_ATTRIBUTE_DIRECTORY) == 0'i32 + else: + var res: Stat + return stat(filename, res) >= 0'i32 and S_ISREG(res.st_mode) + + +proc dirExists*(dir: string): bool {.rtl, extern: "nos$1", tags: [ReadDirEffect], + noNimJs, sideEffect.} = + ## Returns true if the directory `dir` exists. If `dir` is a file, false + ## is returned. Follows symlinks. + ## + ## See also: + ## * `fileExists proc`_ + ## * `symlinkExists proc`_ + when defined(windows): + wrapUnary(a, getFileAttributesW, dir) + if a != -1'i32: + result = (a and FILE_ATTRIBUTE_DIRECTORY) != 0'i32 + else: + var res: Stat + result = stat(dir, res) >= 0'i32 and S_ISDIR(res.st_mode) + + +proc symlinkExists*(link: string): bool {.rtl, extern: "nos$1", + tags: [ReadDirEffect], + noWeirdTarget, sideEffect.} = + ## Returns true if the symlink `link` exists. Will return true + ## regardless of whether the link points to a directory or file. + ## + ## See also: + ## * `fileExists proc`_ + ## * `dirExists proc`_ + when defined(windows): + wrapUnary(a, getFileAttributesW, link) + if a != -1'i32: + # xxx see: bug #16784 (bug9); checking `IO_REPARSE_TAG_SYMLINK` + # may also be needed. + result = (a and FILE_ATTRIBUTE_REPARSE_POINT) != 0'i32 + else: + var res: Stat + result = lstat(link, res) >= 0'i32 and S_ISLNK(res.st_mode) + +when defined(windows) and not weirdTarget: + proc openHandle*(path: string, followSymlink=true, writeAccess=false): Handle = + var flags = FILE_FLAG_BACKUP_SEMANTICS or FILE_ATTRIBUTE_NORMAL + if not followSymlink: + flags = flags or FILE_FLAG_OPEN_REPARSE_POINT + let access = if writeAccess: GENERIC_WRITE else: 0'i32 + + result = createFileW( + newWideCString(path), access, + FILE_SHARE_DELETE or FILE_SHARE_READ or FILE_SHARE_WRITE, + nil, OPEN_EXISTING, flags, 0 + ) diff --git a/lib/std/private/osdirs.nim b/lib/std/private/osdirs.nim new file mode 100644 index 000000000..a44cad7d9 --- /dev/null +++ b/lib/std/private/osdirs.nim @@ -0,0 +1,570 @@ +## .. importdoc:: osfiles.nim, appdirs.nim, paths.nim + +include system/inclrtl +import std/oserrors + + +import ospaths2, osfiles +import oscommon +export dirExists, PathComponent + + +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions, widestrs] + + +when weirdTarget: + discard +elif defined(windows): + import std/[winlean, times] +elif defined(posix): + import std/[posix, times] + +else: + {.error: "OS module not ported to your operating system!".} + + +when weirdTarget: + {.pragma: noWeirdTarget, error: "this proc is not available on the NimScript/js target".} +else: + {.pragma: noWeirdTarget.} + + +when defined(nimscript): + # for procs already defined in scriptconfig.nim + template noNimJs(body): untyped = discard +elif defined(js): + {.pragma: noNimJs, error: "this proc is not available on the js target".} +else: + {.pragma: noNimJs.} + +# Templates for filtering directories and files +when defined(windows) and not weirdTarget: + template isDir(f: WIN32_FIND_DATA): bool = + (f.dwFileAttributes and FILE_ATTRIBUTE_DIRECTORY) != 0'i32 + template isFile(f: WIN32_FIND_DATA): bool = + not isDir(f) +else: + template isDir(f: string): bool {.dirty.} = + dirExists(f) + template isFile(f: string): bool {.dirty.} = + fileExists(f) + +template defaultWalkFilter(item): bool = + ## Walk filter used to return true on both + ## files and directories + true + +template walkCommon(pattern: string, filter) = + ## Common code for getting the files and directories with the + ## specified `pattern` + when defined(windows): + var + f: WIN32_FIND_DATA + res: int + res = findFirstFile(pattern, f) + if res != -1: + defer: findClose(res) + let dotPos = searchExtPos(pattern) + while true: + if not skipFindData(f) and filter(f): + # Windows bug/gotcha: 't*.nim' matches 'tfoo.nims' -.- so we check + # that the file extensions have the same length ... + let ff = getFilename(f) + let idx = ff.len - pattern.len + dotPos + if dotPos < 0 or idx >= ff.len or (idx >= 0 and ff[idx] == '.') or + (dotPos >= 0 and dotPos+1 < pattern.len and pattern[dotPos+1] == '*'): + yield splitFile(pattern).dir / extractFilename(ff) + if findNextFile(res, f) == 0'i32: + let errCode = getLastError() + if errCode == ERROR_NO_MORE_FILES: break + else: raiseOSError(errCode.OSErrorCode) + else: # here we use glob + var + f: Glob + res: int + f.gl_offs = 0 + f.gl_pathc = 0 + f.gl_pathv = nil + res = glob(pattern, 0, nil, addr(f)) + defer: globfree(addr(f)) + if res == 0: + for i in 0.. f.gl_pathc - 1: + assert(f.gl_pathv[i] != nil) + let path = $f.gl_pathv[i] + if filter(path): + yield path + +iterator walkPattern*(pattern: string): string {.tags: [ReadDirEffect], noWeirdTarget.} = + ## Iterate over all the files and directories that match the `pattern`. + ## + ## On POSIX this uses the `glob`:idx: call. + ## `pattern` is OS dependent, but at least the `"*.ext"` + ## notation is supported. + ## + ## See also: + ## * `walkFiles iterator`_ + ## * `walkDirs iterator`_ + ## * `walkDir iterator`_ + ## * `walkDirRec iterator`_ + runnableExamples: + import std/os + import std/sequtils + let paths = toSeq(walkPattern("lib/pure/*")) # works on Windows too + assert "lib/pure/concurrency".unixToNativePath in paths + assert "lib/pure/os.nim".unixToNativePath in paths + walkCommon(pattern, defaultWalkFilter) + +iterator walkFiles*(pattern: string): string {.tags: [ReadDirEffect], noWeirdTarget.} = + ## Iterate over all the files that match the `pattern`. + ## + ## On POSIX this uses the `glob`:idx: call. + ## `pattern` is OS dependent, but at least the `"*.ext"` + ## notation is supported. + ## + ## See also: + ## * `walkPattern iterator`_ + ## * `walkDirs iterator`_ + ## * `walkDir iterator`_ + ## * `walkDirRec iterator`_ + runnableExamples: + import std/os + import std/sequtils + assert "lib/pure/os.nim".unixToNativePath in toSeq(walkFiles("lib/pure/*.nim")) # works on Windows too + walkCommon(pattern, isFile) + +iterator walkDirs*(pattern: string): string {.tags: [ReadDirEffect], noWeirdTarget.} = + ## Iterate over all the directories that match the `pattern`. + ## + ## On POSIX this uses the `glob`:idx: call. + ## `pattern` is OS dependent, but at least the `"*.ext"` + ## notation is supported. + ## + ## See also: + ## * `walkPattern iterator`_ + ## * `walkFiles iterator`_ + ## * `walkDir iterator`_ + ## * `walkDirRec iterator`_ + runnableExamples: + import std/os + import std/sequtils + let paths = toSeq(walkDirs("lib/pure/*")) # works on Windows too + assert "lib/pure/concurrency".unixToNativePath in paths + walkCommon(pattern, isDir) + +proc staticWalkDir(dir: string; relative: bool): seq[ + tuple[kind: PathComponent, path: string]] = + discard + +iterator walkDir*(dir: string; relative = false, checkDir = false, + skipSpecial = false): + tuple[kind: PathComponent, path: string] {.tags: [ReadDirEffect].} = + ## Walks over the directory `dir` and yields for each directory or file in + ## `dir`. The component type and full path for each item are returned. + ## + ## Walking is not recursive. + ## * If `relative` is true (default: false) + ## the resulting path is shortened to be relative to ``dir``, + ## otherwise the full path is returned. + ## * If `checkDir` is true, `OSError` is raised when `dir` + ## doesn't exist. + ## * If `skipSpecial` is true, then (besides all directories) only *regular* + ## files (**without** special "file" objects like FIFOs, device files, + ## etc) will be yielded on Unix. + ## + ## **Example:** + ## + ## This directory structure: + ## + ## dirA / dirB / fileB1.txt + ## / dirC + ## / fileA1.txt + ## / fileA2.txt + ## + ## and this code: + runnableExamples("-r:off"): + import std/[strutils, sugar] + # note: order is not guaranteed + # this also works at compile time + assert collect(for k in walkDir("dirA"): k.path).join(" ") == + "dirA/dirB dirA/dirC dirA/fileA2.txt dirA/fileA1.txt" + ## See also: + ## * `walkPattern iterator`_ + ## * `walkFiles iterator`_ + ## * `walkDirs iterator`_ + ## * `walkDirRec iterator`_ + + when nimvm: + for k, v in items(staticWalkDir(dir, relative)): + yield (k, v) + else: + when weirdTarget: + for k, v in items(staticWalkDir(dir, relative)): + yield (k, v) + elif defined(windows): + var f: WIN32_FIND_DATA + var h = findFirstFile(dir / "*", f) + if h == -1: + if checkDir: + raiseOSError(osLastError(), dir) + else: + defer: findClose(h) + while true: + var k = pcFile + if not skipFindData(f): + if (f.dwFileAttributes and FILE_ATTRIBUTE_DIRECTORY) != 0'i32: + k = pcDir + if (f.dwFileAttributes and FILE_ATTRIBUTE_REPARSE_POINT) != 0'i32: + k = succ(k) + let xx = if relative: extractFilename(getFilename(f)) + else: dir / extractFilename(getFilename(f)) + yield (k, xx) + if findNextFile(h, f) == 0'i32: + let errCode = getLastError() + if errCode == ERROR_NO_MORE_FILES: break + else: raiseOSError(errCode.OSErrorCode) + else: + var d = opendir(dir) + if d == nil: + if checkDir: + raiseOSError(osLastError(), dir) + else: + defer: discard closedir(d) + while true: + var x = readdir(d) + if x == nil: break + var y = $cast[cstring](addr x.d_name) + if y != "." and y != "..": + var s: Stat + let path = dir / y + if not relative: + y = path + var k = pcFile + + template resolveSymlink() = + var isSpecial: bool + (k, isSpecial) = getSymlinkFileKind(path) + if skipSpecial and isSpecial: continue + + template kSetGeneric() = # pure Posix component `k` resolution + if lstat(path.cstring, s) < 0'i32: continue # don't yield + elif S_ISDIR(s.st_mode): + k = pcDir + elif S_ISLNK(s.st_mode): + resolveSymlink() + elif skipSpecial and not S_ISREG(s.st_mode): continue + + when defined(linux) or defined(macosx) or + defined(bsd) or defined(genode) or defined(nintendoswitch): + case x.d_type + of DT_DIR: k = pcDir + of DT_LNK: + resolveSymlink() + of DT_UNKNOWN: + kSetGeneric() + else: # DT_REG or special "files" like FIFOs + if skipSpecial and x.d_type != DT_REG: continue + else: discard # leave it as pcFile + else: # assuming that field `d_type` is not present + kSetGeneric() + + yield (k, y) + +iterator walkDirRec*(dir: string, + yieldFilter = {pcFile}, followFilter = {pcDir}, + relative = false, checkDir = false, skipSpecial = false): + string {.tags: [ReadDirEffect].} = + ## Recursively walks over the directory `dir` and yields for each file + ## or directory in `dir`. + ## + ## Options `relative`, `checkdir`, `skipSpecial` are explained in + ## [walkDir iterator] description. + ## + ## .. warning:: Modifying the directory structure while the iterator + ## is traversing may result in undefined behavior! + ## + ## Walking is recursive. `followFilter` controls the behaviour of the iterator: + ## + ## ===================== ============================================= + ## yieldFilter meaning + ## ===================== ============================================= + ## ``pcFile`` yield real files (default) + ## ``pcLinkToFile`` yield symbolic links to files + ## ``pcDir`` yield real directories + ## ``pcLinkToDir`` yield symbolic links to directories + ## ===================== ============================================= + ## + ## ===================== ============================================= + ## followFilter meaning + ## ===================== ============================================= + ## ``pcDir`` follow real directories (default) + ## ``pcLinkToDir`` follow symbolic links to directories + ## ===================== ============================================= + ## + ## + ## See also: + ## * `walkPattern iterator`_ + ## * `walkFiles iterator`_ + ## * `walkDirs iterator`_ + ## * `walkDir iterator`_ + + var stack = @[""] + var checkDir = checkDir + while stack.len > 0: + let d = stack.pop() + for k, p in walkDir(dir / d, relative = true, checkDir = checkDir, + skipSpecial = skipSpecial): + let rel = d / p + if k in {pcDir, pcLinkToDir} and k in followFilter: + stack.add rel + if k in yieldFilter: + yield if relative: rel else: dir / rel + checkDir = false + # We only check top-level dir, otherwise if a subdir is invalid (eg. wrong + # permissions), it'll abort iteration and there would be no way to + # continue iteration. + # Future work can provide a way to customize this and do error reporting. + + +proc rawRemoveDir(dir: string) {.noWeirdTarget.} = + when defined(windows): + wrapUnary(res, removeDirectoryW, dir) + let lastError = osLastError() + if res == 0'i32 and lastError.int32 != 3'i32 and + lastError.int32 != 18'i32 and lastError.int32 != 2'i32: + raiseOSError(lastError, dir) + else: + if rmdir(dir) != 0'i32 and errno != ENOENT: raiseOSError(osLastError(), dir) + +proc removeDir*(dir: string, checkDir = false) {.rtl, extern: "nos$1", tags: [ + WriteDirEffect, ReadDirEffect], benign, noWeirdTarget.} = + ## Removes the directory `dir` including all subdirectories and files + ## in `dir` (recursively). + ## + ## If this fails, `OSError` is raised. This does not fail if the directory never + ## existed in the first place, unless `checkDir` = true. + ## + ## See also: + ## * `tryRemoveFile proc`_ + ## * `removeFile proc`_ + ## * `existsOrCreateDir proc`_ + ## * `createDir proc`_ + ## * `copyDir proc`_ + ## * `copyDirWithPermissions proc`_ + ## * `moveDir proc`_ + for kind, path in walkDir(dir, checkDir = checkDir): + case kind + of pcFile, pcLinkToFile, pcLinkToDir: removeFile(path) + of pcDir: removeDir(path, true) + # for subdirectories there is no benefit in `checkDir = false` + # (unless perhaps for edge case of concurrent processes also deleting + # the same files) + rawRemoveDir(dir) + +proc rawCreateDir(dir: string): bool {.noWeirdTarget.} = + # Try to create one directory (not the whole path). + # returns `true` for success, `false` if the path has previously existed + # + # This is a thin wrapper over mkDir (or alternatives on other systems), + # so in case of a pre-existing path we don't check that it is a directory. + when defined(solaris): + let res = mkdir(dir, 0o777) + if res == 0'i32: + result = true + elif errno in {EEXIST, ENOSYS}: + result = false + else: + raiseOSError(osLastError(), dir) + elif defined(haiku): + let res = mkdir(dir, 0o777) + if res == 0'i32: + result = true + elif errno == EEXIST or errno == EROFS: + result = false + else: + raiseOSError(osLastError(), dir) + elif defined(posix): + let res = mkdir(dir, 0o777) + if res == 0'i32: + result = true + elif errno == EEXIST: + result = false + else: + #echo res + raiseOSError(osLastError(), dir) + else: + wrapUnary(res, createDirectoryW, dir) + + if res != 0'i32: + result = true + elif getLastError() == 183'i32: + result = false + else: + raiseOSError(osLastError(), dir) + +proc existsOrCreateDir*(dir: string): bool {.rtl, extern: "nos$1", + tags: [WriteDirEffect, ReadDirEffect], noWeirdTarget.} = + ## Checks if a `directory`:idx: `dir` exists, and creates it otherwise. + ## + ## Does not create parent directories (raises `OSError` if parent directories do not exist). + ## Returns `true` if the directory already exists, and `false` otherwise. + ## + ## See also: + ## * `removeDir proc`_ + ## * `createDir proc`_ + ## * `copyDir proc`_ + ## * `copyDirWithPermissions proc`_ + ## * `moveDir proc`_ + result = not rawCreateDir(dir) + if result: + # path already exists - need to check that it is indeed a directory + if not dirExists(dir): + raise newException(IOError, "Failed to create '" & dir & "'") + +proc createDir*(dir: string) {.rtl, extern: "nos$1", + tags: [WriteDirEffect, ReadDirEffect], noWeirdTarget.} = + ## Creates the `directory`:idx: `dir`. + ## + ## The directory may contain several subdirectories that do not exist yet. + ## The full path is created. If this fails, `OSError` is raised. + ## + ## It does **not** fail if the directory already exists because for + ## most usages this does not indicate an error. + ## + ## See also: + ## * `removeDir proc`_ + ## * `existsOrCreateDir proc`_ + ## * `copyDir proc`_ + ## * `copyDirWithPermissions proc`_ + ## * `moveDir proc`_ + if dir == "": + return + var omitNext = isAbsolute(dir) + for p in parentDirs(dir, fromRoot=true): + if omitNext: + omitNext = false + else: + discard existsOrCreateDir(p) + +proc copyDir*(source, dest: string, skipSpecial = false) {.rtl, extern: "nos$1", + tags: [ReadDirEffect, WriteIOEffect, ReadIOEffect], benign, noWeirdTarget.} = + ## Copies a directory from `source` to `dest`. + ## + ## On non-Windows OSes, symlinks are copied as symlinks. On Windows, symlinks + ## are skipped. + ## + ## If `skipSpecial` is true, then (besides all directories) only *regular* + ## files (**without** special "file" objects like FIFOs, device files, + ## etc) will be copied on Unix. + ## + ## If this fails, `OSError` is raised. + ## + ## On the Windows platform this proc will copy the attributes from + ## `source` into `dest`. + ## + ## On other platforms created files and directories will inherit the + ## default permissions of a newly created file/directory for the user. + ## Use `copyDirWithPermissions proc`_ + ## to preserve attributes recursively on these platforms. + ## + ## See also: + ## * `copyDirWithPermissions proc`_ + ## * `copyFile proc`_ + ## * `copyFileWithPermissions proc`_ + ## * `removeDir proc`_ + ## * `existsOrCreateDir proc`_ + ## * `createDir proc`_ + ## * `moveDir proc`_ + createDir(dest) + for kind, path in walkDir(source, skipSpecial = skipSpecial): + var noSource = splitPath(path).tail + if kind == pcDir: + copyDir(path, dest / noSource, skipSpecial = skipSpecial) + else: + copyFile(path, dest / noSource, {cfSymlinkAsIs}) + + +proc copyDirWithPermissions*(source, dest: string, + ignorePermissionErrors = true, + skipSpecial = false) + {.rtl, extern: "nos$1", tags: [ReadDirEffect, WriteIOEffect, ReadIOEffect], + benign, noWeirdTarget.} = + ## Copies a directory from `source` to `dest` preserving file permissions. + ## + ## On non-Windows OSes, symlinks are copied as symlinks. On Windows, symlinks + ## are skipped. + ## + ## If `skipSpecial` is true, then (besides all directories) only *regular* + ## files (**without** special "file" objects like FIFOs, device files, + ## etc) will be copied on Unix. + ## + ## If this fails, `OSError` is raised. This is a wrapper proc around + ## `copyDir`_ and `copyFileWithPermissions`_ procs + ## on non-Windows platforms. + ## + ## On Windows this proc is just a wrapper for `copyDir proc`_ since + ## that proc already copies attributes. + ## + ## On non-Windows systems permissions are copied after the file or directory + ## itself has been copied, which won't happen atomically and could lead to a + ## race condition. If `ignorePermissionErrors` is true (default), errors while + ## reading/setting file attributes will be ignored, otherwise will raise + ## `OSError`. + ## + ## See also: + ## * `copyDir proc`_ + ## * `copyFile proc`_ + ## * `copyFileWithPermissions proc`_ + ## * `removeDir proc`_ + ## * `moveDir proc`_ + ## * `existsOrCreateDir proc`_ + ## * `createDir proc`_ + createDir(dest) + when not defined(windows): + try: + setFilePermissions(dest, getFilePermissions(source), followSymlinks = + false) + except: + if not ignorePermissionErrors: + raise + for kind, path in walkDir(source, skipSpecial = skipSpecial): + var noSource = splitPath(path).tail + if kind == pcDir: + copyDirWithPermissions(path, dest / noSource, ignorePermissionErrors, skipSpecial = skipSpecial) + else: + copyFileWithPermissions(path, dest / noSource, ignorePermissionErrors, {cfSymlinkAsIs}) + +proc moveDir*(source, dest: string) {.tags: [ReadIOEffect, WriteIOEffect], noWeirdTarget.} = + ## Moves a directory from `source` to `dest`. + ## + ## Symlinks are not followed: if `source` contains symlinks, they themself are + ## moved, not their target. + ## + ## If this fails, `OSError` is raised. + ## + ## See also: + ## * `moveFile proc`_ + ## * `copyDir proc`_ + ## * `copyDirWithPermissions proc`_ + ## * `removeDir proc`_ + ## * `existsOrCreateDir proc`_ + ## * `createDir proc`_ + if not tryMoveFSObject(source, dest, isDir = true): + # Fallback to copy & del + copyDir(source, dest) + removeDir(source) + +proc setCurrentDir*(newDir: string) {.inline, tags: [], noWeirdTarget.} = + ## Sets the `current working directory`:idx:; `OSError` + ## is raised if `newDir` cannot been set. + ## + ## See also: + ## * `getHomeDir proc`_ + ## * `getConfigDir proc`_ + ## * `getTempDir proc`_ + ## * `getCurrentDir proc`_ + when defined(windows): + if setCurrentDirectoryW(newWideCString(newDir)) == 0'i32: + raiseOSError(osLastError(), newDir) + else: + if chdir(newDir) != 0'i32: raiseOSError(osLastError(), newDir) diff --git a/lib/std/private/osfiles.nim b/lib/std/private/osfiles.nim new file mode 100644 index 000000000..37d8eabca --- /dev/null +++ b/lib/std/private/osfiles.nim @@ -0,0 +1,416 @@ +include system/inclrtl +import std/private/since +import std/oserrors + +import oscommon +export fileExists + +import ospaths2, ossymlinks + +## .. importdoc:: osdirs.nim, os.nim + +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions, widestrs] + +when weirdTarget: + discard +elif defined(windows): + import std/winlean +elif defined(posix): + import std/[posix, times] + + proc toTime(ts: Timespec): times.Time {.inline.} = + result = initTime(ts.tv_sec.int64, ts.tv_nsec.int) +else: + {.error: "OS module not ported to your operating system!".} + + +when weirdTarget: + {.pragma: noWeirdTarget, error: "this proc is not available on the NimScript/js target".} +else: + {.pragma: noWeirdTarget.} + + +when defined(nimscript): + # for procs already defined in scriptconfig.nim + template noNimJs(body): untyped = discard +elif defined(js): + {.pragma: noNimJs, error: "this proc is not available on the js target".} +else: + {.pragma: noNimJs.} + + +type + FilePermission* = enum ## File access permission, modelled after UNIX. + ## + ## See also: + ## * `getFilePermissions`_ + ## * `setFilePermissions`_ + ## * `FileInfo object`_ + fpUserExec, ## execute access for the file owner + fpUserWrite, ## write access for the file owner + fpUserRead, ## read access for the file owner + fpGroupExec, ## execute access for the group + fpGroupWrite, ## write access for the group + fpGroupRead, ## read access for the group + fpOthersExec, ## execute access for others + fpOthersWrite, ## write access for others + fpOthersRead ## read access for others + +proc getFilePermissions*(filename: string): set[FilePermission] {. + rtl, extern: "nos$1", tags: [ReadDirEffect], noWeirdTarget.} = + ## Retrieves file permissions for `filename`. + ## + ## `OSError` is raised in case of an error. + ## On Windows, only the ``readonly`` flag is checked, every other + ## permission is available in any case. + ## + ## See also: + ## * `setFilePermissions proc`_ + ## * `FilePermission enum`_ + when defined(posix): + var a: Stat + if stat(filename, a) < 0'i32: raiseOSError(osLastError(), filename) + result = {} + if (a.st_mode and S_IRUSR.Mode) != 0.Mode: result.incl(fpUserRead) + if (a.st_mode and S_IWUSR.Mode) != 0.Mode: result.incl(fpUserWrite) + if (a.st_mode and S_IXUSR.Mode) != 0.Mode: result.incl(fpUserExec) + + if (a.st_mode and S_IRGRP.Mode) != 0.Mode: result.incl(fpGroupRead) + if (a.st_mode and S_IWGRP.Mode) != 0.Mode: result.incl(fpGroupWrite) + if (a.st_mode and S_IXGRP.Mode) != 0.Mode: result.incl(fpGroupExec) + + if (a.st_mode and S_IROTH.Mode) != 0.Mode: result.incl(fpOthersRead) + if (a.st_mode and S_IWOTH.Mode) != 0.Mode: result.incl(fpOthersWrite) + if (a.st_mode and S_IXOTH.Mode) != 0.Mode: result.incl(fpOthersExec) + else: + wrapUnary(res, getFileAttributesW, filename) + if res == -1'i32: raiseOSError(osLastError(), filename) + if (res and FILE_ATTRIBUTE_READONLY) != 0'i32: + result = {fpUserExec, fpUserRead, fpGroupExec, fpGroupRead, + fpOthersExec, fpOthersRead} + else: + result = {fpUserExec..fpOthersRead} + +proc setFilePermissions*(filename: string, permissions: set[FilePermission], + followSymlinks = true) + {.rtl, extern: "nos$1", tags: [ReadDirEffect, WriteDirEffect], + noWeirdTarget.} = + ## Sets the file permissions for `filename`. + ## + ## If `followSymlinks` set to true (default) and ``filename`` points to a + ## symlink, permissions are set to the file symlink points to. + ## `followSymlinks` set to false is a noop on Windows and some POSIX + ## systems (including Linux) on which `lchmod` is either unavailable or always + ## fails, given that symlinks permissions there are not observed. + ## + ## `OSError` is raised in case of an error. + ## On Windows, only the ``readonly`` flag is changed, depending on + ## ``fpUserWrite`` permission. + ## + ## See also: + ## * `getFilePermissions proc`_ + ## * `FilePermission enum`_ + when defined(posix): + var p = 0.Mode + if fpUserRead in permissions: p = p or S_IRUSR.Mode + if fpUserWrite in permissions: p = p or S_IWUSR.Mode + if fpUserExec in permissions: p = p or S_IXUSR.Mode + + if fpGroupRead in permissions: p = p or S_IRGRP.Mode + if fpGroupWrite in permissions: p = p or S_IWGRP.Mode + if fpGroupExec in permissions: p = p or S_IXGRP.Mode + + if fpOthersRead in permissions: p = p or S_IROTH.Mode + if fpOthersWrite in permissions: p = p or S_IWOTH.Mode + if fpOthersExec in permissions: p = p or S_IXOTH.Mode + + if not followSymlinks and filename.symlinkExists: + when declared(lchmod): + if lchmod(filename, cast[Mode](p)) != 0: + raiseOSError(osLastError(), $(filename, permissions)) + else: + if chmod(filename, cast[Mode](p)) != 0: + raiseOSError(osLastError(), $(filename, permissions)) + else: + wrapUnary(res, getFileAttributesW, filename) + if res == -1'i32: raiseOSError(osLastError(), filename) + if fpUserWrite in permissions: + res = res and not FILE_ATTRIBUTE_READONLY + else: + res = res or FILE_ATTRIBUTE_READONLY + wrapBinary(res2, setFileAttributesW, filename, res) + if res2 == - 1'i32: raiseOSError(osLastError(), $(filename, permissions)) + + +const hasCCopyfile = defined(osx) and not defined(nimLegacyCopyFile) + # xxx instead of `nimLegacyCopyFile`, support something like: `when osxVersion >= (10, 5)` + +when hasCCopyfile: + # `copyfile` API available since osx 10.5. + {.push nodecl, header: "<copyfile.h>".} + type + copyfile_state_t {.nodecl.} = pointer + copyfile_flags_t = cint + proc copyfile_state_alloc(): copyfile_state_t + proc copyfile_state_free(state: copyfile_state_t): cint + proc c_copyfile(src, dst: cstring, state: copyfile_state_t, flags: copyfile_flags_t): cint {.importc: "copyfile".} + when (NimMajor, NimMinor) >= (1, 4): + let + COPYFILE_DATA {.nodecl.}: copyfile_flags_t + COPYFILE_XATTR {.nodecl.}: copyfile_flags_t + else: + var + COPYFILE_DATA {.nodecl.}: copyfile_flags_t + COPYFILE_XATTR {.nodecl.}: copyfile_flags_t + {.pop.} + +type + CopyFlag* = enum ## Copy options. + cfSymlinkAsIs, ## Copy symlinks as symlinks + cfSymlinkFollow, ## Copy the files symlinks point to + cfSymlinkIgnore ## Ignore symlinks + +const copyFlagSymlink = {cfSymlinkAsIs, cfSymlinkFollow, cfSymlinkIgnore} + +proc copyFile*(source, dest: string, options = {cfSymlinkFollow}; bufferSize = 16_384) {.rtl, + extern: "nos$1", tags: [ReadDirEffect, ReadIOEffect, WriteIOEffect], + noWeirdTarget.} = + ## Copies a file from `source` to `dest`, where `dest.parentDir` must exist. + ## + ## On non-Windows OSes, `options` specify the way file is copied; by default, + ## if `source` is a symlink, copies the file symlink points to. `options` is + ## ignored on Windows: symlinks are skipped. + ## + ## If this fails, `OSError` is raised. + ## + ## On the Windows platform this proc will + ## copy the source file's attributes into dest. + ## + ## On other platforms you need + ## to use `getFilePermissions`_ and + ## `setFilePermissions`_ + ## procs + ## to copy them by hand (or use the convenience `copyFileWithPermissions + ## proc`_), + ## otherwise `dest` will inherit the default permissions of a newly + ## created file for the user. + ## + ## If `dest` already exists, the file attributes + ## will be preserved and the content overwritten. + ## + ## On OSX, `copyfile` C api will be used (available since OSX 10.5) unless + ## `-d:nimLegacyCopyFile` is used. + ## + ## `copyFile` allows to specify `bufferSize` to improve I/O performance. + ## + ## See also: + ## * `CopyFlag enum`_ + ## * `copyDir proc`_ + ## * `copyFileWithPermissions proc`_ + ## * `tryRemoveFile proc`_ + ## * `removeFile proc`_ + ## * `moveFile proc`_ + + doAssert card(copyFlagSymlink * options) == 1, "There should be exactly one cfSymlink* in options" + let isSymlink = source.symlinkExists + if isSymlink and (cfSymlinkIgnore in options or defined(windows)): + return + when defined(windows): + let s = newWideCString(source) + let d = newWideCString(dest) + if copyFileW(s, d, 0'i32) == 0'i32: + raiseOSError(osLastError(), $(source, dest)) + else: + if isSymlink and cfSymlinkAsIs in options: + createSymlink(expandSymlink(source), dest) + else: + when hasCCopyfile: + let state = copyfile_state_alloc() + # xxx `COPYFILE_STAT` could be used for one-shot + # `copyFileWithPermissions`. + let status = c_copyfile(source.cstring, dest.cstring, state, + COPYFILE_DATA) + if status != 0: + let err = osLastError() + discard copyfile_state_free(state) + raiseOSError(err, $(source, dest)) + let status2 = copyfile_state_free(state) + if status2 != 0: raiseOSError(osLastError(), $(source, dest)) + else: + # generic version of copyFile which works for any platform: + var d, s: File + if not open(s, source): raiseOSError(osLastError(), source) + if not open(d, dest, fmWrite): + close(s) + raiseOSError(osLastError(), dest) + + # Hints for kernel-level aggressive sequential low-fragmentation read-aheads: + # https://pubs.opengroup.org/onlinepubs/9699919799/functions/posix_fadvise.html + when defined(linux) or defined(osx): + discard posix_fadvise(getFileHandle(d), 0.cint, 0.cint, POSIX_FADV_SEQUENTIAL) + discard posix_fadvise(getFileHandle(s), 0.cint, 0.cint, POSIX_FADV_SEQUENTIAL) + + var buf = alloc(bufferSize) + while true: + var bytesread = readBuffer(s, buf, bufferSize) + if bytesread > 0: + var byteswritten = writeBuffer(d, buf, bytesread) + if bytesread != byteswritten: + dealloc(buf) + close(s) + close(d) + raiseOSError(osLastError(), dest) + if bytesread != bufferSize: break + dealloc(buf) + close(s) + flushFile(d) + close(d) + +proc copyFileToDir*(source, dir: string, options = {cfSymlinkFollow}; bufferSize = 16_384) + {.noWeirdTarget, since: (1,3,7).} = + ## Copies a file `source` into directory `dir`, which must exist. + ## + ## On non-Windows OSes, `options` specify the way file is copied; by default, + ## if `source` is a symlink, copies the file symlink points to. `options` is + ## ignored on Windows: symlinks are skipped. + ## + ## `copyFileToDir` allows to specify `bufferSize` to improve I/O performance. + ## + ## See also: + ## * `CopyFlag enum`_ + ## * `copyFile proc`_ + if dir.len == 0: # treating "" as "." is error prone + raise newException(ValueError, "dest is empty") + copyFile(source, dir / source.lastPathPart, options, bufferSize) + + +proc copyFileWithPermissions*(source, dest: string, + ignorePermissionErrors = true, + options = {cfSymlinkFollow}) {.noWeirdTarget.} = + ## Copies a file from `source` to `dest` preserving file permissions. + ## + ## On non-Windows OSes, `options` specify the way file is copied; by default, + ## if `source` is a symlink, copies the file symlink points to. `options` is + ## ignored on Windows: symlinks are skipped. + ## + ## This is a wrapper proc around `copyFile`_, + ## `getFilePermissions`_ and `setFilePermissions`_ + ## procs on non-Windows platforms. + ## + ## On Windows this proc is just a wrapper for `copyFile proc`_ since + ## that proc already copies attributes. + ## + ## On non-Windows systems permissions are copied after the file itself has + ## been copied, which won't happen atomically and could lead to a race + ## condition. If `ignorePermissionErrors` is true (default), errors while + ## reading/setting file attributes will be ignored, otherwise will raise + ## `OSError`. + ## + ## See also: + ## * `CopyFlag enum`_ + ## * `copyFile proc`_ + ## * `copyDir proc`_ + ## * `tryRemoveFile proc`_ + ## * `removeFile proc`_ + ## * `moveFile proc`_ + ## * `copyDirWithPermissions proc`_ + copyFile(source, dest, options) + when not defined(windows): + try: + setFilePermissions(dest, getFilePermissions(source), followSymlinks = + (cfSymlinkFollow in options)) + except: + if not ignorePermissionErrors: + raise + +when not declared(ENOENT) and not defined(windows): + when defined(nimscript): + when not defined(haiku): + const ENOENT = cint(2) # 2 on most systems including Solaris + else: + const ENOENT = cint(-2147459069) + else: + var ENOENT {.importc, header: "<errno.h>".}: cint + +when defined(windows) and not weirdTarget: + template deleteFile(file: untyped): untyped = deleteFileW(file) + template setFileAttributes(file, attrs: untyped): untyped = + setFileAttributesW(file, attrs) + +proc tryRemoveFile*(file: string): bool {.rtl, extern: "nos$1", tags: [WriteDirEffect], noWeirdTarget.} = + ## Removes the `file`. + ## + ## If this fails, returns `false`. This does not fail + ## if the file never existed in the first place. + ## + ## On Windows, ignores the read-only attribute. + ## + ## See also: + ## * `copyFile proc`_ + ## * `copyFileWithPermissions proc`_ + ## * `removeFile proc`_ + ## * `moveFile proc`_ + result = true + when defined(windows): + let f = newWideCString(file) + if deleteFile(f) == 0: + result = false + let err = getLastError() + if err == ERROR_FILE_NOT_FOUND or err == ERROR_PATH_NOT_FOUND: + result = true + elif err == ERROR_ACCESS_DENIED and + setFileAttributes(f, FILE_ATTRIBUTE_NORMAL) != 0 and + deleteFile(f) != 0: + result = true + else: + if unlink(file) != 0'i32 and errno != ENOENT: + result = false + +proc removeFile*(file: string) {.rtl, extern: "nos$1", tags: [WriteDirEffect], noWeirdTarget.} = + ## Removes the `file`. + ## + ## If this fails, `OSError` is raised. This does not fail + ## if the file never existed in the first place. + ## + ## On Windows, ignores the read-only attribute. + ## + ## See also: + ## * `removeDir proc`_ + ## * `copyFile proc`_ + ## * `copyFileWithPermissions proc`_ + ## * `tryRemoveFile proc`_ + ## * `moveFile proc`_ + if not tryRemoveFile(file): + raiseOSError(osLastError(), file) + +proc moveFile*(source, dest: string) {.rtl, extern: "nos$1", + tags: [ReadDirEffect, ReadIOEffect, WriteIOEffect], noWeirdTarget.} = + ## Moves a file from `source` to `dest`. + ## + ## Symlinks are not followed: if `source` is a symlink, it is itself moved, + ## not its target. + ## + ## If this fails, `OSError` is raised. + ## If `dest` already exists, it will be overwritten. + ## + ## Can be used to `rename files`:idx:. + ## + ## See also: + ## * `moveDir proc`_ + ## * `copyFile proc`_ + ## * `copyFileWithPermissions proc`_ + ## * `removeFile proc`_ + ## * `tryRemoveFile proc`_ + + if not tryMoveFSObject(source, dest, isDir = false): + when defined(windows): + raiseAssert "unreachable" + else: + # Fallback to copy & del + copyFileWithPermissions(source, dest, options={cfSymlinkAsIs}) + try: + removeFile(source) + except: + discard tryRemoveFile(dest) + raise diff --git a/lib/std/private/ospaths2.nim b/lib/std/private/ospaths2.nim new file mode 100644 index 000000000..bc69ff725 --- /dev/null +++ b/lib/std/private/ospaths2.nim @@ -0,0 +1,1030 @@ +include system/inclrtl +import std/private/since + +import std/[strutils, pathnorm] +import std/oserrors + +import oscommon +export ReadDirEffect, WriteDirEffect + +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions, widestrs] + +## .. importdoc:: osappdirs.nim, osdirs.nim, osseps.nim, os.nim + +const weirdTarget = defined(nimscript) or defined(js) + +when weirdTarget: + discard +elif defined(windows): + import std/winlean +elif defined(posix): + import std/posix, system/ansi_c +else: + {.error: "OS module not ported to your operating system!".} + +when weirdTarget: + {.pragma: noWeirdTarget, error: "this proc is not available on the NimScript/js target".} +else: + {.pragma: noWeirdTarget.} + +when defined(nimscript): + # for procs already defined in scriptconfig.nim + template noNimJs(body): untyped = discard +elif defined(js): + {.pragma: noNimJs, error: "this proc is not available on the js target".} +else: + {.pragma: noNimJs.} + + +proc normalizePathAux(path: var string){.inline, raises: [], noSideEffect.} + + +import std/private/osseps +export osseps + +proc absolutePathInternal(path: string): string {.gcsafe.} + +proc normalizePathEnd*(path: var string, trailingSep = false) = + ## Ensures ``path`` has exactly 0 or 1 trailing `DirSep`, depending on + ## ``trailingSep``, and taking care of edge cases: it preservers whether + ## a path is absolute or relative, and makes sure trailing sep is `DirSep`, + ## not `AltSep`. Trailing `/.` are compressed, see examples. + if path.len == 0: return + var i = path.len + while i >= 1: + if path[i-1] in {DirSep, AltSep}: dec(i) + elif path[i-1] == '.' and i >= 2 and path[i-2] in {DirSep, AltSep}: dec(i) + else: break + if trailingSep: + # foo// => foo + path.setLen(i) + # foo => foo/ + path.add DirSep + elif i > 0: + # foo// => foo + path.setLen(i) + else: + # // => / (empty case was already taken care of) + path = $DirSep + +proc normalizePathEnd*(path: string, trailingSep = false): string = + ## outplace overload + runnableExamples: + when defined(posix): + assert normalizePathEnd("/lib//.//", trailingSep = true) == "/lib/" + assert normalizePathEnd("lib/./.", trailingSep = false) == "lib" + assert normalizePathEnd(".//./.", trailingSep = false) == "." + assert normalizePathEnd("", trailingSep = true) == "" # not / ! + assert normalizePathEnd("/", trailingSep = false) == "/" # not "" ! + result = path + result.normalizePathEnd(trailingSep) + +template endsWith(a: string, b: set[char]): bool = + a.len > 0 and a[^1] in b + +proc joinPathImpl(result: var string, state: var int, tail: string) = + let trailingSep = tail.endsWith({DirSep, AltSep}) or tail.len == 0 and result.endsWith({DirSep, AltSep}) + normalizePathEnd(result, trailingSep=false) + addNormalizePath(tail, result, state, DirSep) + normalizePathEnd(result, trailingSep=trailingSep) + +proc joinPath*(head, tail: string): string {. + noSideEffect, rtl, extern: "nos$1".} = + ## Joins two directory names to one. + ## + ## returns normalized path concatenation of `head` and `tail`, preserving + ## whether or not `tail` has a trailing slash (or, if tail if empty, whether + ## head has one). + ## + ## See also: + ## * `joinPath(parts: varargs[string]) proc`_ + ## * `/ proc`_ + ## * `splitPath proc`_ + ## * `uri.combine proc <uri.html#combine,Uri,Uri>`_ + ## * `uri./ proc <uri.html#/,Uri,string>`_ + runnableExamples: + when defined(posix): + assert joinPath("usr", "lib") == "usr/lib" + assert joinPath("usr", "lib/") == "usr/lib/" + assert joinPath("usr", "") == "usr" + assert joinPath("usr/", "") == "usr/" + assert joinPath("", "") == "" + assert joinPath("", "lib") == "lib" + assert joinPath("", "/lib") == "/lib" + assert joinPath("usr/", "/lib") == "usr/lib" + assert joinPath("usr/lib", "../bin") == "usr/bin" + + result = newStringOfCap(head.len + tail.len) + var state = 0 + joinPathImpl(result, state, head) + joinPathImpl(result, state, tail) + when false: + if len(head) == 0: + result = tail + elif head[len(head)-1] in {DirSep, AltSep}: + if tail.len > 0 and tail[0] in {DirSep, AltSep}: + result = head & substr(tail, 1) + else: + result = head & tail + else: + if tail.len > 0 and tail[0] in {DirSep, AltSep}: + result = head & tail + else: + result = head & DirSep & tail + +proc joinPath*(parts: varargs[string]): string {.noSideEffect, + rtl, extern: "nos$1OpenArray".} = + ## The same as `joinPath(head, tail) proc`_, + ## but works with any number of directory parts. + ## + ## You need to pass at least one element or the proc + ## will assert in debug builds and crash on release builds. + ## + ## See also: + ## * `joinPath(head, tail) proc`_ + ## * `/ proc`_ + ## * `/../ proc`_ + ## * `splitPath proc`_ + runnableExamples: + when defined(posix): + assert joinPath("a") == "a" + assert joinPath("a", "b", "c") == "a/b/c" + assert joinPath("usr/lib", "../../var", "log") == "var/log" + + var estimatedLen = 0 + for p in parts: estimatedLen += p.len + result = newStringOfCap(estimatedLen) + var state = 0 + for i in 0..high(parts): + joinPathImpl(result, state, parts[i]) + +proc `/`*(head, tail: string): string {.noSideEffect, inline.} = + ## The same as `joinPath(head, tail) proc`_. + ## + ## See also: + ## * `/../ proc`_ + ## * `joinPath(head, tail) proc`_ + ## * `joinPath(parts: varargs[string]) proc`_ + ## * `splitPath proc`_ + ## * `uri.combine proc <uri.html#combine,Uri,Uri>`_ + ## * `uri./ proc <uri.html#/,Uri,string>`_ + runnableExamples: + when defined(posix): + assert "usr" / "" == "usr" + assert "" / "lib" == "lib" + assert "" / "/lib" == "/lib" + assert "usr/" / "/lib/" == "usr/lib/" + assert "usr" / "lib" / "../bin" == "usr/bin" + + result = joinPath(head, tail) + +when doslikeFileSystem: + import std/private/ntpath + +proc splitPath*(path: string): tuple[head, tail: string] {. + noSideEffect, rtl, extern: "nos$1".} = + ## Splits a directory into `(head, tail)` tuple, so that + ## ``head / tail == path`` (except for edge cases like "/usr"). + ## + ## See also: + ## * `joinPath(head, tail) proc`_ + ## * `joinPath(parts: varargs[string]) proc`_ + ## * `/ proc`_ + ## * `/../ proc`_ + ## * `relativePath proc`_ + runnableExamples: + assert splitPath("usr/local/bin") == ("usr/local", "bin") + assert splitPath("usr/local/bin/") == ("usr/local/bin", "") + assert splitPath("/bin/") == ("/bin", "") + when (NimMajor, NimMinor) <= (1, 0): + assert splitPath("/bin") == ("", "bin") + else: + assert splitPath("/bin") == ("/", "bin") + assert splitPath("bin") == ("", "bin") + assert splitPath("") == ("", "") + + when doslikeFileSystem: + let (drive, splitpath) = splitDrive(path) + let stop = drive.len + else: + const stop = 0 + + var sepPos = -1 + for i in countdown(len(path)-1, stop): + if path[i] in {DirSep, AltSep}: + sepPos = i + break + if sepPos >= 0: + result.head = substr(path, 0, + when (NimMajor, NimMinor) <= (1, 0): + sepPos-1 + else: + if likely(sepPos >= 1): sepPos-1 else: 0 + ) + result.tail = substr(path, sepPos+1) + else: + when doslikeFileSystem: + result.head = drive + result.tail = splitpath + else: + result.head = "" + result.tail = path + +proc isAbsolute*(path: string): bool {.rtl, noSideEffect, extern: "nos$1", raises: [].} = + ## Checks whether a given `path` is absolute. + ## + ## On Windows, network paths are considered absolute too. + runnableExamples: + assert not "".isAbsolute + assert not ".".isAbsolute + when defined(posix): + assert "/".isAbsolute + assert not "a/".isAbsolute + assert "/a/".isAbsolute + + if len(path) == 0: return false + + when doslikeFileSystem: + var len = len(path) + result = (path[0] in {'/', '\\'}) or + (len > 1 and path[0] in {'a'..'z', 'A'..'Z'} and path[1] == ':') + elif defined(macos): + # according to https://perldoc.perl.org/File/Spec/Mac.html `:a` is a relative path + result = path[0] != ':' + elif defined(RISCOS): + result = path[0] == '$' + elif defined(posix): + result = path[0] == '/' + elif defined(nodejs): + {.emit: [result," = require(\"path\").isAbsolute(",path.cstring,");"].} + else: + raiseAssert "unreachable" # if ever hits here, adapt as needed + +when FileSystemCaseSensitive: + template `!=?`(a, b: char): bool = a != b +else: + template `!=?`(a, b: char): bool = toLowerAscii(a) != toLowerAscii(b) + +when doslikeFileSystem: + proc isAbsFromCurrentDrive(path: string): bool {.noSideEffect, raises: [].} = + ## An absolute path from the root of the current drive (e.g. "\foo") + path.len > 0 and + (path[0] == AltSep or + (path[0] == DirSep and + (path.len == 1 or path[1] notin {DirSep, AltSep, ':'}))) + + proc sameRoot(path1, path2: string): bool {.noSideEffect, raises: [].} = + ## Return true if path1 and path2 have a same root. + ## + ## Detail of Windows path formats: + ## https://docs.microsoft.com/en-us/dotnet/standard/io/file-path-formats + + assert(isAbsolute(path1)) + assert(isAbsolute(path2)) + + if isAbsFromCurrentDrive(path1) and isAbsFromCurrentDrive(path2): + result = true + elif cmpIgnoreCase(splitDrive(path1).drive, splitDrive(path2).drive) == 0: + result = true + else: + result = false + +proc relativePath*(path, base: string, sep = DirSep): string {. + rtl, extern: "nos$1".} = + ## Converts `path` to a path relative to `base`. + ## + ## The `sep` (default: DirSep_) is used for the path normalizations, + ## this can be useful to ensure the relative path only contains `'/'` + ## so that it can be used for URL constructions. + ## + ## On Windows, if a root of `path` and a root of `base` are different, + ## returns `path` as is because it is impossible to make a relative path. + ## That means an absolute path can be returned. + ## + ## See also: + ## * `splitPath proc`_ + ## * `parentDir proc`_ + ## * `tailDir proc`_ + runnableExamples: + assert relativePath("/Users/me/bar/z.nim", "/Users/other/bad", '/') == "../../me/bar/z.nim" + assert relativePath("/Users/me/bar/z.nim", "/Users/other", '/') == "../me/bar/z.nim" + when not doslikeFileSystem: # On Windows, UNC-paths start with `//` + assert relativePath("/Users///me/bar//z.nim", "//Users/", '/') == "me/bar/z.nim" + assert relativePath("/Users/me/bar/z.nim", "/Users/me", '/') == "bar/z.nim" + assert relativePath("", "/users/moo", '/') == "" + assert relativePath("foo", ".", '/') == "foo" + assert relativePath("foo", "foo", '/') == "." + + if path.len == 0: return "" + var base = if base == ".": "" else: base + var path = path + path.normalizePathAux + base.normalizePathAux + let a1 = isAbsolute(path) + let a2 = isAbsolute(base) + if a1 and not a2: + base = absolutePathInternal(base) + elif a2 and not a1: + path = absolutePathInternal(path) + + when doslikeFileSystem: + if isAbsolute(path) and isAbsolute(base): + if not sameRoot(path, base): + return path + + var f = default PathIter + var b = default PathIter + var ff = (0, -1) + var bb = (0, -1) # (int, int) + result = newStringOfCap(path.len) + # skip the common prefix: + while f.hasNext(path) and b.hasNext(base): + ff = next(f, path) + bb = next(b, base) + let diff = ff[1] - ff[0] + if diff != bb[1] - bb[0]: break + var same = true + for i in 0..diff: + if path[i + ff[0]] !=? base[i + bb[0]]: + same = false + break + if not same: break + ff = (0, -1) + bb = (0, -1) + # for i in 0..diff: + # result.add base[i + bb[0]] + + # /foo/bar/xxx/ -- base + # /foo/bar/baz -- path path + # ../baz + # every directory that is in 'base', needs to add '..' + while true: + if bb[1] >= bb[0]: + if result.len > 0 and result[^1] != sep: + result.add sep + result.add ".." + if not b.hasNext(base): break + bb = b.next(base) + + # add the rest of 'path': + while true: + if ff[1] >= ff[0]: + if result.len > 0 and result[^1] != sep: + result.add sep + for i in 0..ff[1] - ff[0]: + result.add path[i + ff[0]] + if not f.hasNext(path): break + ff = f.next(path) + + when not defined(nimOldRelativePathBehavior): + if result.len == 0: result.add "." + +proc isRelativeTo*(path: string, base: string): bool {.since: (1, 1).} = + ## Returns true if `path` is relative to `base`. + runnableExamples: + doAssert isRelativeTo("./foo//bar", "foo") + doAssert isRelativeTo("foo/bar", ".") + doAssert isRelativeTo("/foo/bar.nim", "/foo/bar.nim") + doAssert not isRelativeTo("foo/bar.nims", "foo/bar.nim") + let path = path.normalizePath + let base = base.normalizePath + let ret = relativePath(path, base) + result = path.len > 0 and not ret.startsWith ".." + +proc parentDirPos(path: string): int = + var q = 1 + if len(path) >= 1 and path[len(path)-1] in {DirSep, AltSep}: q = 2 + for i in countdown(len(path)-q, 0): + if path[i] in {DirSep, AltSep}: return i + result = -1 + +proc parentDir*(path: string): string {. + noSideEffect, rtl, extern: "nos$1".} = + ## Returns the parent directory of `path`. + ## + ## This is similar to ``splitPath(path).head`` when ``path`` doesn't end + ## in a dir separator, but also takes care of path normalizations. + ## The remainder can be obtained with `lastPathPart(path) proc`_. + ## + ## See also: + ## * `relativePath proc`_ + ## * `splitPath proc`_ + ## * `tailDir proc`_ + ## * `parentDirs iterator`_ + runnableExamples: + assert parentDir("") == "" + when defined(posix): + assert parentDir("/usr/local/bin") == "/usr/local" + assert parentDir("foo/bar//") == "foo" + assert parentDir("//foo//bar//.") == "/foo" + assert parentDir("./foo") == "." + assert parentDir("/./foo//./") == "/" + assert parentDir("a//./") == "." + assert parentDir("a/b/c/..") == "a" + result = pathnorm.normalizePath(path) + when doslikeFileSystem: + let (drive, splitpath) = splitDrive(result) + result = splitpath + var sepPos = parentDirPos(result) + if sepPos >= 0: + result = substr(result, 0, sepPos) + normalizePathEnd(result) + elif result == ".." or result == "." or result.len == 0 or result[^1] in {DirSep, AltSep}: + # `.` => `..` and .. => `../..`(etc) would be a sensible alternative + # `/` => `/` (as done with splitFile) would be a sensible alternative + result = "" + else: + result = "." + when doslikeFileSystem: + if result.len == 0: + discard + elif drive.len > 0 and result.len == 1 and result[0] in {DirSep, AltSep}: + result = drive + else: + result = drive & result + +proc tailDir*(path: string): string {. + noSideEffect, rtl, extern: "nos$1".} = + ## Returns the tail part of `path`. + ## + ## See also: + ## * `relativePath proc`_ + ## * `splitPath proc`_ + ## * `parentDir proc`_ + runnableExamples: + assert tailDir("/bin") == "bin" + assert tailDir("bin") == "" + assert tailDir("bin/") == "" + assert tailDir("/usr/local/bin") == "usr/local/bin" + assert tailDir("//usr//local//bin//") == "usr//local//bin//" + assert tailDir("./usr/local/bin") == "usr/local/bin" + assert tailDir("usr/local/bin") == "local/bin" + + var i = 0 + when doslikeFileSystem: + let (drive, splitpath) = path.splitDrive + if drive != "": + return splitpath.strip(chars = {DirSep, AltSep}, trailing = false) + while i < len(path): + if path[i] in {DirSep, AltSep}: + while i < len(path) and path[i] in {DirSep, AltSep}: inc i + return substr(path, i) + inc i + result = "" + +proc isRootDir*(path: string): bool {. + noSideEffect, rtl, extern: "nos$1".} = + ## Checks whether a given `path` is a root directory. + runnableExamples: + assert isRootDir("") + assert isRootDir(".") + assert isRootDir("/") + assert isRootDir("a") + assert not isRootDir("/a") + assert not isRootDir("a/b/c") + + when doslikeFileSystem: + if splitDrive(path).path == "": + return true + result = parentDirPos(path) < 0 + +iterator parentDirs*(path: string, fromRoot=false, inclusive=true): string = + ## Walks over all parent directories of a given `path`. + ## + ## If `fromRoot` is true (default: false), the traversal will start from + ## the file system root directory. + ## If `inclusive` is true (default), the original argument will be included + ## in the traversal. + ## + ## Relative paths won't be expanded by this iterator. Instead, it will traverse + ## only the directories appearing in the relative path. + ## + ## See also: + ## * `parentDir proc`_ + ## + runnableExamples: + let g = "a/b/c" + + for p in g.parentDirs: + echo p + # a/b/c + # a/b + # a + + for p in g.parentDirs(fromRoot=true): + echo p + # a/ + # a/b/ + # a/b/c + + for p in g.parentDirs(inclusive=false): + echo p + # a/b + # a + + if not fromRoot: + var current = path + if inclusive: yield path + while true: + if current.isRootDir: break + current = current.parentDir + yield current + else: + when doslikeFileSystem: + let start = path.splitDrive.drive.len + else: + const start = 0 + for i in countup(start, path.len - 2): # ignore the last / + # deal with non-normalized paths such as /foo//bar//baz + if path[i] in {DirSep, AltSep} and + (i == 0 or path[i-1] notin {DirSep, AltSep}): + yield path.substr(0, i) + + if inclusive: yield path + +proc `/../`*(head, tail: string): string {.noSideEffect.} = + ## The same as ``parentDir(head) / tail``, unless there is no parent + ## directory. Then ``head / tail`` is performed instead. + ## + ## See also: + ## * `/ proc`_ + ## * `parentDir proc`_ + runnableExamples: + when defined(posix): + assert "a/b/c" /../ "d/e" == "a/b/d/e" + assert "a" /../ "d/e" == "a/d/e" + + when doslikeFileSystem: + let (drive, head) = splitDrive(head) + let sepPos = parentDirPos(head) + if sepPos >= 0: + result = substr(head, 0, sepPos-1) / tail + else: + result = head / tail + when doslikeFileSystem: + result = drive / result + +proc normExt(ext: string): string = + if ext == "" or ext[0] == ExtSep: result = ext # no copy needed here + else: result = ExtSep & ext + +proc searchExtPos*(path: string): int = + ## Returns index of the `'.'` char in `path` if it signifies the beginning + ## of the file extension. Returns -1 otherwise. + ## + ## See also: + ## * `splitFile proc`_ + ## * `extractFilename proc`_ + ## * `lastPathPart proc`_ + ## * `changeFileExt proc`_ + ## * `addFileExt proc`_ + runnableExamples: + assert searchExtPos("a/b/c") == -1 + assert searchExtPos("c.nim") == 1 + assert searchExtPos("a/b/c.nim") == 5 + assert searchExtPos("a.b.c.nim") == 5 + assert searchExtPos(".nim") == -1 + assert searchExtPos("..nim") == -1 + assert searchExtPos("a..nim") == 2 + + # Unless there is any char that is not `ExtSep` before last `ExtSep` in the file name, + # it is not a file extension. + const DirSeps = when doslikeFileSystem: {DirSep, AltSep, ':'} else: {DirSep, AltSep} + result = -1 + var i = path.high + while i >= 1: + if path[i] == ExtSep: + break + elif path[i] in DirSeps: + return -1 # do not skip over path + dec i + + for j in countdown(i - 1, 0): + if path[j] in DirSeps: + return -1 + elif path[j] != ExtSep: + result = i + break + +proc splitFile*(path: string): tuple[dir, name, ext: string] {. + noSideEffect, rtl, extern: "nos$1".} = + ## Splits a filename into `(dir, name, extension)` tuple. + ## + ## `dir` does not end in DirSep_ unless it's `/`. + ## `extension` includes the leading dot. + ## + ## If `path` has no extension, `ext` is the empty string. + ## If `path` has no directory component, `dir` is the empty string. + ## If `path` has no filename component, `name` and `ext` are empty strings. + ## + ## See also: + ## * `searchExtPos proc`_ + ## * `extractFilename proc`_ + ## * `lastPathPart proc`_ + ## * `changeFileExt proc`_ + ## * `addFileExt proc`_ + runnableExamples: + var (dir, name, ext) = splitFile("usr/local/nimc.html") + assert dir == "usr/local" + assert name == "nimc" + assert ext == ".html" + (dir, name, ext) = splitFile("/usr/local/os") + assert dir == "/usr/local" + assert name == "os" + assert ext == "" + (dir, name, ext) = splitFile("/usr/local/") + assert dir == "/usr/local" + assert name == "" + assert ext == "" + (dir, name, ext) = splitFile("/tmp.txt") + assert dir == "/" + assert name == "tmp" + assert ext == ".txt" + + var namePos = 0 + var dotPos = 0 + when doslikeFileSystem: + let (drive, _) = splitDrive(path) + let stop = len(drive) + result.dir = drive + else: + const stop = 0 + for i in countdown(len(path) - 1, stop): + if path[i] in {DirSep, AltSep} or i == 0: + if path[i] in {DirSep, AltSep}: + result.dir = substr(path, 0, if likely(i >= 1): i - 1 else: 0) + namePos = i + 1 + if dotPos > i: + result.name = substr(path, namePos, dotPos - 1) + result.ext = substr(path, dotPos) + else: + result.name = substr(path, namePos) + break + elif path[i] == ExtSep and i > 0 and i < len(path) - 1 and + path[i - 1] notin {DirSep, AltSep} and + path[i + 1] != ExtSep and dotPos == 0: + dotPos = i + +proc extractFilename*(path: string): string {. + noSideEffect, rtl, extern: "nos$1".} = + ## Extracts the filename of a given `path`. + ## + ## This is the same as ``name & ext`` from `splitFile(path) proc`_. + ## + ## See also: + ## * `searchExtPos proc`_ + ## * `splitFile proc`_ + ## * `lastPathPart proc`_ + ## * `changeFileExt proc`_ + ## * `addFileExt proc`_ + runnableExamples: + assert extractFilename("foo/bar/") == "" + assert extractFilename("foo/bar") == "bar" + assert extractFilename("foo/bar.baz") == "bar.baz" + + if path.len == 0 or path[path.len-1] in {DirSep, AltSep}: + result = "" + else: + result = splitPath(path).tail + +proc lastPathPart*(path: string): string {. + noSideEffect, rtl, extern: "nos$1".} = + ## Like `extractFilename proc`_, but ignores + ## trailing dir separator; aka: `baseName`:idx: in some other languages. + ## + ## See also: + ## * `searchExtPos proc`_ + ## * `splitFile proc`_ + ## * `extractFilename proc`_ + ## * `changeFileExt proc`_ + ## * `addFileExt proc`_ + runnableExamples: + assert lastPathPart("foo/bar/") == "bar" + assert lastPathPart("foo/bar") == "bar" + + let path = path.normalizePathEnd(trailingSep = false) + result = extractFilename(path) + +proc changeFileExt*(filename, ext: string): string {. + noSideEffect, rtl, extern: "nos$1".} = + ## Changes the file extension to `ext`. + ## + ## If the `filename` has no extension, `ext` will be added. + ## If `ext` == "" then any extension is removed. + ## + ## `Ext` should be given without the leading `'.'`, because some + ## filesystems may use a different character. (Although I know + ## of none such beast.) + ## + ## See also: + ## * `searchExtPos proc`_ + ## * `splitFile proc`_ + ## * `extractFilename proc`_ + ## * `lastPathPart proc`_ + ## * `addFileExt proc`_ + runnableExamples: + assert changeFileExt("foo.bar", "baz") == "foo.baz" + assert changeFileExt("foo.bar", "") == "foo" + assert changeFileExt("foo", "baz") == "foo.baz" + + var extPos = searchExtPos(filename) + if extPos < 0: result = filename & normExt(ext) + else: result = substr(filename, 0, extPos-1) & normExt(ext) + +proc addFileExt*(filename, ext: string): string {. + noSideEffect, rtl, extern: "nos$1".} = + ## Adds the file extension `ext` to `filename`, unless + ## `filename` already has an extension. + ## + ## `Ext` should be given without the leading `'.'`, because some + ## filesystems may use a different character. + ## (Although I know of none such beast.) + ## + ## See also: + ## * `searchExtPos proc`_ + ## * `splitFile proc`_ + ## * `extractFilename proc`_ + ## * `lastPathPart proc`_ + ## * `changeFileExt proc`_ + runnableExamples: + assert addFileExt("foo.bar", "baz") == "foo.bar" + assert addFileExt("foo.bar", "") == "foo.bar" + assert addFileExt("foo", "baz") == "foo.baz" + + var extPos = searchExtPos(filename) + if extPos < 0: result = filename & normExt(ext) + else: result = filename + +proc cmpPaths*(pathA, pathB: string): int {. + noSideEffect, rtl, extern: "nos$1".} = + ## Compares two paths. + ## + ## On a case-sensitive filesystem this is done + ## case-sensitively otherwise case-insensitively. Returns: + ## + ## | `0` if pathA == pathB + ## | `< 0` if pathA < pathB + ## | `> 0` if pathA > pathB + runnableExamples: + when defined(macosx): + assert cmpPaths("foo", "Foo") == 0 + elif defined(posix): + assert cmpPaths("foo", "Foo") > 0 + + let a = normalizePath(pathA) + let b = normalizePath(pathB) + if FileSystemCaseSensitive: + result = cmp(a, b) + else: + when defined(nimscript): + result = cmpic(a, b) + elif defined(nimdoc): discard + else: + result = cmpIgnoreCase(a, b) + +proc unixToNativePath*(path: string, drive=""): string {. + noSideEffect, rtl, extern: "nos$1".} = + ## Converts an UNIX-like path to a native one. + ## + ## On an UNIX system this does nothing. Else it converts + ## `'/'`, `'.'`, `'..'` to the appropriate things. + ## + ## On systems with a concept of "drives", `drive` is used to determine + ## which drive label to use during absolute path conversion. + ## `drive` defaults to the drive of the current working directory, and is + ## ignored on systems that do not have a concept of "drives". + when defined(unix): + result = path + else: + if path.len == 0: return "" + + var start: int + if path[0] == '/': + # an absolute path + when doslikeFileSystem: + if drive != "": + result = drive & ":" & DirSep + else: + result = $DirSep + elif defined(macos): + result = "" # must not start with ':' + else: + result = $DirSep + start = 1 + elif path[0] == '.' and (path.len == 1 or path[1] == '/'): + # current directory + result = $CurDir + start = when doslikeFileSystem: 1 else: 2 + else: + result = "" + start = 0 + + var i = start + while i < len(path): # ../../../ --> :::: + if i+2 < path.len and path[i] == '.' and path[i+1] == '.' and path[i+2] == '/': + # parent directory + when defined(macos): + if result[high(result)] == ':': + add result, ':' + else: + add result, ParDir + else: + add result, ParDir & DirSep + inc(i, 3) + elif path[i] == '/': + add result, DirSep + inc(i) + else: + add result, path[i] + inc(i) + + +when not defined(nimscript): + proc getCurrentDir*(): string {.rtl, extern: "nos$1", tags: [].} = + ## Returns the `current working directory`:idx: i.e. where the built + ## binary is run. + ## + ## So the path returned by this proc is determined at run time. + ## + ## See also: + ## * `getHomeDir proc`_ + ## * `getConfigDir proc`_ + ## * `getTempDir proc`_ + ## * `setCurrentDir proc`_ + ## * `currentSourcePath template <system.html#currentSourcePath.t>`_ + ## * `getProjectPath proc <macros.html#getProjectPath>`_ + when defined(nodejs): + var ret: cstring + {.emit: "`ret` = process.cwd();".} + return $ret + elif defined(js): + raiseAssert "use -d:nodejs to have `getCurrentDir` defined" + elif defined(windows): + var bufsize = MAX_PATH.int32 + var res = newWideCString(bufsize) + while true: + var L = getCurrentDirectoryW(bufsize, res) + if L == 0'i32: + raiseOSError(osLastError()) + elif L > bufsize: + res = newWideCString(L) + bufsize = L + else: + result = res$L + break + else: + var bufsize = 1024 # should be enough + result = newString(bufsize) + while true: + if getcwd(result.cstring, bufsize) != nil: + setLen(result, c_strlen(result.cstring)) + break + else: + let err = osLastError() + if err.int32 == ERANGE: + bufsize = bufsize shl 1 + doAssert(bufsize >= 0) + result = newString(bufsize) + else: + raiseOSError(osLastError()) + +proc absolutePath*(path: string, root = getCurrentDir()): string = + ## Returns the absolute path of `path`, rooted at `root` (which must be absolute; + ## default: current directory). + ## If `path` is absolute, return it, ignoring `root`. + ## + ## See also: + ## * `normalizedPath proc`_ + ## * `normalizePath proc`_ + runnableExamples: + assert absolutePath("a") == getCurrentDir() / "a" + + if isAbsolute(path): path + else: + if not root.isAbsolute: + raise newException(ValueError, "The specified root is not absolute: " & root) + joinPath(root, path) + +proc absolutePathInternal(path: string): string = + absolutePath(path, getCurrentDir()) + + +proc normalizePath*(path: var string) {.rtl, extern: "nos$1", tags: [].} = + ## Normalize a path. + ## + ## Consecutive directory separators are collapsed, including an initial double slash. + ## + ## On relative paths, double dot (`..`) sequences are collapsed if possible. + ## On absolute paths they are always collapsed. + ## + ## .. warning:: URL-encoded and Unicode attempts at directory traversal are not detected. + ## Triple dot is not handled. + ## + ## See also: + ## * `absolutePath proc`_ + ## * `normalizedPath proc`_ for outplace version + ## * `normalizeExe proc`_ + runnableExamples: + when defined(posix): + var a = "a///b//..//c///d" + a.normalizePath() + assert a == "a/c/d" + + path = pathnorm.normalizePath(path) + when false: + let isAbs = isAbsolute(path) + var stack: seq[string] = @[] + for p in split(path, {DirSep}): + case p + of "", ".": + continue + of "..": + if stack.len == 0: + if isAbs: + discard # collapse all double dots on absoluta paths + else: + stack.add(p) + elif stack[^1] == "..": + stack.add(p) + else: + discard stack.pop() + else: + stack.add(p) + + if isAbs: + path = DirSep & join(stack, $DirSep) + elif stack.len > 0: + path = join(stack, $DirSep) + else: + path = "." + +proc normalizePathAux(path: var string) = normalizePath(path) + +proc normalizedPath*(path: string): string {.rtl, extern: "nos$1", tags: [].} = + ## Returns a normalized path for the current OS. + ## + ## See also: + ## * `absolutePath proc`_ + ## * `normalizePath proc`_ for the in-place version + runnableExamples: + when defined(posix): + assert normalizedPath("a///b//..//c///d") == "a/c/d" + result = pathnorm.normalizePath(path) + +proc normalizeExe*(file: var string) {.since: (1, 3, 5).} = + ## on posix, prepends `./` if `file` doesn't contain `/` and is not `"", ".", ".."`. + runnableExamples: + import std/sugar + when defined(posix): + doAssert "foo".dup(normalizeExe) == "./foo" + doAssert "foo/../bar".dup(normalizeExe) == "foo/../bar" + doAssert "".dup(normalizeExe) == "" + when defined(posix): + if file.len > 0 and DirSep notin file and file != "." and file != "..": + file = "./" & file + +proc sameFile*(path1, path2: string): bool {.rtl, extern: "nos$1", + tags: [ReadDirEffect], noWeirdTarget.} = + ## Returns true if both pathname arguments refer to the same physical + ## file or directory. + ## + ## Raises `OSError` if any of the files does not + ## exist or information about it can not be obtained. + ## + ## This proc will return true if given two alternative hard-linked or + ## sym-linked paths to the same file or directory. + ## + ## See also: + ## * `sameFileContent proc`_ + when defined(windows): + var success = true + var f1 = openHandle(path1) + var f2 = openHandle(path2) + + var lastErr: OSErrorCode + if f1 != INVALID_HANDLE_VALUE and f2 != INVALID_HANDLE_VALUE: + var fi1, fi2: BY_HANDLE_FILE_INFORMATION + + if getFileInformationByHandle(f1, addr(fi1)) != 0 and + getFileInformationByHandle(f2, addr(fi2)) != 0: + result = fi1.dwVolumeSerialNumber == fi2.dwVolumeSerialNumber and + fi1.nFileIndexHigh == fi2.nFileIndexHigh and + fi1.nFileIndexLow == fi2.nFileIndexLow + else: + lastErr = osLastError() + success = false + else: + lastErr = osLastError() + success = false + + discard closeHandle(f1) + discard closeHandle(f2) + + if not success: raiseOSError(lastErr, $(path1, path2)) + else: + var a, b: Stat + if stat(path1, a) < 0'i32 or stat(path2, b) < 0'i32: + raiseOSError(osLastError(), $(path1, path2)) + else: + result = a.st_dev == b.st_dev and a.st_ino == b.st_ino diff --git a/lib/std/private/osseps.nim b/lib/std/private/osseps.nim new file mode 100644 index 000000000..f2d49d886 --- /dev/null +++ b/lib/std/private/osseps.nim @@ -0,0 +1,113 @@ +# Include file that implements 'DirSep' and friends. Do not import this when +# you also import `os.nim`! + +# Improved based on info in 'compiler/platform.nim' + +## .. importdoc:: ospaths2.nim + +const + doslikeFileSystem* = defined(windows) or defined(OS2) or defined(DOS) + +const + CurDir* = + when defined(macos): ':' + elif defined(genode): '/' + else: '.' + ## The constant character used by the operating system to refer to the + ## current directory. + ## + ## For example: `'.'` for POSIX or `':'` for the classic Macintosh. + + ParDir* = + when defined(macos): "::" + else: ".." + ## The constant string used by the operating system to refer to the + ## parent directory. + ## + ## For example: `".."` for POSIX or `"::"` for the classic Macintosh. + + DirSep* = + when defined(macos): ':' + elif doslikeFileSystem or defined(vxworks): '\\' + elif defined(RISCOS): '.' + else: '/' + ## The character used by the operating system to separate pathname + ## components, for example: `'/'` for POSIX, `':'` for the classic + ## Macintosh, and `'\\'` on Windows. + + AltSep* = + when doslikeFileSystem: '/' + else: DirSep + ## An alternative character used by the operating system to separate + ## pathname components, or the same as DirSep_ if only one separator + ## character exists. This is set to `'/'` on Windows systems + ## where DirSep_ is a backslash (`'\\'`). + + PathSep* = + when defined(macos) or defined(RISCOS): ',' + elif doslikeFileSystem or defined(vxworks): ';' + elif defined(PalmOS) or defined(MorphOS): ':' # platform has ':' but osseps has ';' + else: ':' + ## The character conventionally used by the operating system to separate + ## search path components (as in PATH), such as `':'` for POSIX + ## or `';'` for Windows. + + FileSystemCaseSensitive* = + when defined(macos) or defined(macosx) or doslikeFileSystem or defined(vxworks) or + defined(PalmOS) or defined(MorphOS): false + else: true + ## True if the file system is case sensitive, false otherwise. Used by + ## `cmpPaths proc`_ to compare filenames properly. + + ExeExt* = + when doslikeFileSystem: "exe" + elif defined(atari): "tpp" + elif defined(netware): "nlm" + elif defined(vxworks): "vxe" + elif defined(nintendoswitch): "elf" + else: "" + ## The file extension of native executables. For example: + ## `""` for POSIX, `"exe"` on Windows (without a dot). + + ScriptExt* = + when doslikeFileSystem: "bat" + else: "" + ## The file extension of a script file. For example: `""` for POSIX, + ## `"bat"` on Windows. + + DynlibFormat* = + when defined(macos): "$1.dylib" # platform has $1Lib + elif defined(macosx): "lib$1.dylib" + elif doslikeFileSystem or defined(atari): "$1.dll" + elif defined(MorphOS): "$1.prc" + elif defined(PalmOS): "$1.prc" # platform has lib$1.so + elif defined(genode): "$1.lib.so" + elif defined(netware): "$1.nlm" + elif defined(amiga): "$1.Library" + else: "lib$1.so" + ## The format string to turn a filename into a `DLL`:idx: file (also + ## called `shared object`:idx: on some operating systems). + + ExtSep* = '.' + ## The character which separates the base filename from the extension; + ## for example, the `'.'` in `os.nim`. + + # MacOS paths + # =========== + # MacOS directory separator is a colon ":" which is the only character not + # allowed in filenames. + # + # A path containing no colon or which begins with a colon is a partial + # path. + # E.g. ":kalle:petter" ":kalle" "kalle" + # + # All other paths are full (absolute) paths. E.g. "HD:kalle:" "HD:" + # When generating paths, one is safe if one ensures that all partial paths + # begin with a colon, and all full paths end with a colon. + # In full paths the first name (e g HD above) is the name of a mounted + # volume. + # These names are not unique, because, for instance, two diskettes with the + # same names could be inserted. This means that paths on MacOS are not + # waterproof. In case of equal names the first volume found will do. + # Two colons "::" are the relative path to the parent. Three is to the + # grandparent etc. diff --git a/lib/std/private/ossymlinks.nim b/lib/std/private/ossymlinks.nim new file mode 100644 index 000000000..c1760c42e --- /dev/null +++ b/lib/std/private/ossymlinks.nim @@ -0,0 +1,78 @@ +include system/inclrtl +import std/oserrors + +import oscommon +export symlinkExists + +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions, widestrs] + +when weirdTarget: + discard +elif defined(windows): + import std/[winlean, times] +elif defined(posix): + import std/posix +else: + {.error: "OS module not ported to your operating system!".} + + +when weirdTarget: + {.pragma: noWeirdTarget, error: "this proc is not available on the NimScript/js target".} +else: + {.pragma: noWeirdTarget.} + + +when defined(nimscript): + # for procs already defined in scriptconfig.nim + template noNimJs(body): untyped = discard +elif defined(js): + {.pragma: noNimJs, error: "this proc is not available on the js target".} +else: + {.pragma: noNimJs.} + +## .. importdoc:: os.nim + +proc createSymlink*(src, dest: string) {.noWeirdTarget.} = + ## Create a symbolic link at `dest` which points to the item specified + ## by `src`. On most operating systems, will fail if a link already exists. + ## + ## .. warning:: Some OS's (such as Microsoft Windows) restrict the creation + ## of symlinks to root users (administrators) or users with developer mode enabled. + ## + ## See also: + ## * `createHardlink proc`_ + ## * `expandSymlink proc`_ + + when defined(windows): + const SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE = 2 + # allows anyone with developer mode on to create a link + let flag = dirExists(src).int32 or SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE + var wSrc = newWideCString(src) + var wDst = newWideCString(dest) + if createSymbolicLinkW(wDst, wSrc, flag) == 0 or getLastError() != 0: + raiseOSError(osLastError(), $(src, dest)) + else: + if symlink(src, dest) != 0: + raiseOSError(osLastError(), $(src, dest)) + +proc expandSymlink*(symlinkPath: string): string {.noWeirdTarget.} = + ## Returns a string representing the path to which the symbolic link points. + ## + ## On Windows this is a noop, `symlinkPath` is simply returned. + ## + ## See also: + ## * `createSymlink proc`_ + when defined(windows) or defined(nintendoswitch): + result = symlinkPath + else: + var bufLen = 1024 + while true: + result = newString(bufLen) + let len = readlink(symlinkPath.cstring, result.cstring, bufLen) + if len < 0: + raiseOSError(osLastError(), symlinkPath) + if len < bufLen: + result.setLen(len) + break + bufLen = bufLen shl 1 diff --git a/lib/std/private/schubfach.nim b/lib/std/private/schubfach.nim new file mode 100644 index 000000000..b8c85d2bc --- /dev/null +++ b/lib/std/private/schubfach.nim @@ -0,0 +1,436 @@ +## Copyright 2020 Alexander Bolz +## +## Distributed under the Boost Software License, Version 1.0. +## (See accompanying file LICENSE_1_0.txt or copy at https://www.boost.org/LICENSE_1_0.txt) + +# -------------------------------------------------------------------------------------------------- +## This file contains an implementation of the Schubfach algorithm as described in +## +## \[1] Raffaello Giulietti, "The Schubfach way to render doubles", +## https://drive.google.com/open?id=1luHhyQF9zKlM8yJ1nebU0OgVYhfC6CBN +# -------------------------------------------------------------------------------------------------- + +import std/private/digitsutils + +when defined(nimPreviewSlimSystem): + import std/assertions + + +template sf_Assert(x: untyped): untyped = + assert(x) + +# ================================================================================================== +# +# ================================================================================================== + +type + ValueType = float32 + BitsType = uint32 + Single {.bycopy.} = object + bits: BitsType + +const + significandSize: int32 = 24 + MaxExponent = 128 + exponentBias: int32 = MaxExponent - 1 + (significandSize - 1) + maxIeeeExponent: BitsType = BitsType(2 * MaxExponent - 1) + hiddenBit: BitsType = BitsType(1) shl (significandSize - 1) + significandMask: BitsType = hiddenBit - 1 + exponentMask: BitsType = maxIeeeExponent shl (significandSize - 1) + signMask: BitsType = not (not BitsType(0) shr 1) + +proc constructSingle(bits: BitsType): Single = + result.bits = bits + +proc constructSingle(value: ValueType): Single = + result.bits = cast[typeof(result.bits)](value) + +proc physicalSignificand(this: Single): BitsType {.noSideEffect.} = + return this.bits and significandMask + +proc physicalExponent(this: Single): BitsType {.noSideEffect.} = + return (this.bits and exponentMask) shr (significandSize - 1) + +proc isFinite(this: Single): bool {.noSideEffect.} = + return (this.bits and exponentMask) != exponentMask + +proc isInf(this: Single): bool {.noSideEffect.} = + return (this.bits and exponentMask) == exponentMask and + (this.bits and significandMask) == 0 + +proc isNaN(this: Single): bool {.noSideEffect.} = + return (this.bits and exponentMask) == exponentMask and + (this.bits and significandMask) != 0 + +proc isZero(this: Single): bool {.noSideEffect.} = + return (this.bits and not signMask) == 0 + +proc signBit(this: Single): int {.noSideEffect.} = + return int((this.bits and signMask) != 0) + +# ================================================================================================== +## Returns floor(x / 2^n). +## +## Technically, right-shift of negative integers is implementation defined... +## Should easily be optimized into SAR (or equivalent) instruction. + +proc floorDivPow2(x: int32; n: int32): int32 {.inline.} = + return x shr n + +## Returns floor(log_10(2^e)) +## ```c +## static inline int32_t FloorLog10Pow2(int32_t e) +## { +## SF_ASSERT(e >= -1500); +## SF_ASSERT(e <= 1500); +## return FloorDivPow2(e * 1262611, 22); +## } +## ``` +## Returns floor(log_10(3/4 2^e)) +## ```c +## static inline int32_t FloorLog10ThreeQuartersPow2(int32_t e) +## { +## SF_ASSERT(e >= -1500); +## SF_ASSERT(e <= 1500); +## return FloorDivPow2(e * 1262611 - 524031, 22); +## } +## ``` +## Returns floor(log_2(10^e)) + +proc floorLog2Pow10(e: int32): int32 {.inline.} = + sf_Assert(e >= -1233) + sf_Assert(e <= 1233) + return floorDivPow2(e * 1741647, 19) + +const + kMin: int32 = -31 + kMax: int32 = 45 + g: array[kMax - kMin + 1, uint64] = [0x81CEB32C4B43FCF5'u64, 0xA2425FF75E14FC32'u64, + 0xCAD2F7F5359A3B3F'u64, 0xFD87B5F28300CA0E'u64, 0x9E74D1B791E07E49'u64, + 0xC612062576589DDB'u64, 0xF79687AED3EEC552'u64, 0x9ABE14CD44753B53'u64, + 0xC16D9A0095928A28'u64, 0xF1C90080BAF72CB2'u64, 0x971DA05074DA7BEF'u64, + 0xBCE5086492111AEB'u64, 0xEC1E4A7DB69561A6'u64, 0x9392EE8E921D5D08'u64, + 0xB877AA3236A4B44A'u64, 0xE69594BEC44DE15C'u64, 0x901D7CF73AB0ACDA'u64, + 0xB424DC35095CD810'u64, 0xE12E13424BB40E14'u64, 0x8CBCCC096F5088CC'u64, + 0xAFEBFF0BCB24AAFF'u64, 0xDBE6FECEBDEDD5BF'u64, 0x89705F4136B4A598'u64, + 0xABCC77118461CEFD'u64, 0xD6BF94D5E57A42BD'u64, 0x8637BD05AF6C69B6'u64, + 0xA7C5AC471B478424'u64, 0xD1B71758E219652C'u64, 0x83126E978D4FDF3C'u64, + 0xA3D70A3D70A3D70B'u64, 0xCCCCCCCCCCCCCCCD'u64, 0x8000000000000000'u64, + 0xA000000000000000'u64, 0xC800000000000000'u64, 0xFA00000000000000'u64, + 0x9C40000000000000'u64, 0xC350000000000000'u64, 0xF424000000000000'u64, + 0x9896800000000000'u64, 0xBEBC200000000000'u64, 0xEE6B280000000000'u64, + 0x9502F90000000000'u64, 0xBA43B74000000000'u64, 0xE8D4A51000000000'u64, + 0x9184E72A00000000'u64, 0xB5E620F480000000'u64, 0xE35FA931A0000000'u64, + 0x8E1BC9BF04000000'u64, 0xB1A2BC2EC5000000'u64, 0xDE0B6B3A76400000'u64, + 0x8AC7230489E80000'u64, 0xAD78EBC5AC620000'u64, 0xD8D726B7177A8000'u64, + 0x878678326EAC9000'u64, 0xA968163F0A57B400'u64, 0xD3C21BCECCEDA100'u64, + 0x84595161401484A0'u64, 0xA56FA5B99019A5C8'u64, 0xCECB8F27F4200F3A'u64, + 0x813F3978F8940985'u64, 0xA18F07D736B90BE6'u64, 0xC9F2C9CD04674EDF'u64, + 0xFC6F7C4045812297'u64, 0x9DC5ADA82B70B59E'u64, 0xC5371912364CE306'u64, + 0xF684DF56C3E01BC7'u64, 0x9A130B963A6C115D'u64, 0xC097CE7BC90715B4'u64, + 0xF0BDC21ABB48DB21'u64, 0x96769950B50D88F5'u64, 0xBC143FA4E250EB32'u64, + 0xEB194F8E1AE525FE'u64, 0x92EFD1B8D0CF37BF'u64, 0xB7ABC627050305AE'u64, + 0xE596B7B0C643C71A'u64, 0x8F7E32CE7BEA5C70'u64, 0xB35DBF821AE4F38C'u64] + +proc computePow10Single(k: int32): uint64 {.inline.} = + ## There are unique beta and r such that 10^k = beta 2^r and + ## 2^63 <= beta < 2^64, namely r = floor(log_2 10^k) - 63 and + ## beta = 2^-r 10^k. + ## Let g = ceil(beta), so (g-1) 2^r < 10^k <= g 2^r, with the latter + ## value being a pretty good overestimate for 10^k. + ## NB: Since for all the required exponents k, we have g < 2^64, + ## all constants can be stored in 128-bit integers. + sf_Assert(k >= kMin) + sf_Assert(k <= kMax) + return g[k - kMin] + +proc lo32(x: uint64): uint32 {.inline.} = + return cast[uint32](x) + +proc hi32(x: uint64): uint32 {.inline.} = + return cast[uint32](x shr 32) + +when defined(sizeof_Int128): + proc roundToOdd(g: uint64; cp: uint32): uint32 {.inline.} = + let p: uint128 = uint128(g) * cp + let y1: uint32 = lo32(cast[uint64](p shr 64)) + let y0: uint32 = hi32(cast[uint64](p)) + return y1 or uint32(y0 > 1) + +elif defined(vcc) and defined(cpu64): + proc umul128(x, y: uint64, z: ptr uint64): uint64 {.importc: "_umul128", header: "<intrin.h>".} + proc roundToOdd(g: uint64; cpHi: uint32): uint32 {.inline.} = + var p1: uint64 = 0 + var p0: uint64 = umul128(g, cpHi, addr(p1)) + let y1: uint32 = lo32(p1) + let y0: uint32 = hi32(p0) + return y1 or uint32(y0 > 1) + +else: + proc roundToOdd(g: uint64; cp: uint32): uint32 {.inline.} = + let b01: uint64 = uint64(lo32(g)) * cp + let b11: uint64 = uint64(hi32(g)) * cp + let hi: uint64 = b11 + hi32(b01) + let y1: uint32 = hi32(hi) + let y0: uint32 = lo32(hi) + return y1 or uint32(y0 > 1) + +## Returns whether value is divisible by 2^e2 + +proc multipleOfPow2(value: uint32; e2: int32): bool {.inline.} = + sf_Assert(e2 >= 0) + sf_Assert(e2 <= 31) + return (value and ((uint32(1) shl e2) - 1)) == 0 + +type + FloatingDecimal32 {.bycopy.} = object + digits: uint32 ## num_digits <= 9 + exponent: int32 + +proc toDecimal32(ieeeSignificand: uint32; ieeeExponent: uint32): FloatingDecimal32 {. + inline.} = + var c: uint32 + var q: int32 + if ieeeExponent != 0: + c = hiddenBit or ieeeSignificand + q = cast[int32](ieeeExponent) - exponentBias + if 0 <= -q and -q < significandSize and multipleOfPow2(c, -q): + return FloatingDecimal32(digits: c shr -q, exponent: 0'i32) + else: + c = ieeeSignificand + q = 1 - exponentBias + let isEven: bool = (c mod 2 == 0) + let lowerBoundaryIsCloser: bool = (ieeeSignificand == 0 and ieeeExponent > 1) + ## const int32_t qb = q - 2; + let cbl: uint32 = 4 * c - 2 + uint32(lowerBoundaryIsCloser) + let cb: uint32 = 4 * c + let cbr: uint32 = 4 * c + 2 + ## (q * 1262611 ) >> 22 == floor(log_10( 2^q)) + ## (q * 1262611 - 524031) >> 22 == floor(log_10(3/4 2^q)) + sf_Assert(q >= -1500) + sf_Assert(q <= 1500) + let k: int32 = floorDivPow2(q * 1262611 - (if lowerBoundaryIsCloser: 524031 else: 0), 22) + let h: int32 = q + floorLog2Pow10(-k) + 1 + sf_Assert(h >= 1) + sf_Assert(h <= 4) + let pow10: uint64 = computePow10Single(-k) + let vbl: uint32 = roundToOdd(pow10, cbl shl h) + let vb: uint32 = roundToOdd(pow10, cb shl h) + let vbr: uint32 = roundToOdd(pow10, cbr shl h) + let lower: uint32 = vbl + uint32(not isEven) + let upper: uint32 = vbr - uint32(not isEven) + ## See Figure 4 in [1]. + ## And the modifications in Figure 6. + let s: uint32 = vb div 4 + ## NB: 4 * s == vb & ~3 == vb & -4 + if s >= 10: + let sp: uint32 = s div 10 + ## = vb / 40 + let upInside: bool = lower <= 40 * sp + let wpInside: bool = 40 * sp + 40 <= upper + ## if (up_inside || wp_inside) // NB: At most one of u' and w' is in R_v. + if upInside != wpInside: + return FloatingDecimal32(digits: sp + uint32(wpInside), exponent: k + 1) + let uInside: bool = lower <= 4 * s + let wInside: bool = 4 * s + 4 <= upper + if uInside != wInside: + return FloatingDecimal32(digits: s + uint32(wInside), exponent: k) + let mid: uint32 = 4 * s + 2 + ## = 2(s + t) + let roundUp: bool = vb > mid or (vb == mid and (s and 1) != 0) + return FloatingDecimal32(digits: s + uint32(roundUp), exponent: k) + +## ================================================================================================== +## ToChars +## ================================================================================================== + +proc printDecimalDigitsBackwards[T: Ordinal](buf: var openArray[char]; pos: T; output: uint32): int {.inline.} = + var output = output + var pos = pos + var tz = 0 + ## number of trailing zeros removed. + var nd = 0 + ## number of decimal digits processed. + ## At most 9 digits remaining + if output >= 10000: + let q: uint32 = output div 10000 + let r: uint32 = output mod 10000 + output = q + dec(pos, 4) + if r != 0: + let rH: uint32 = r div 100 + let rL: uint32 = r mod 100 + utoa2Digits(buf, pos, rH) + utoa2Digits(buf, pos + 2, rL) + tz = trailingZeros2Digits(if rL == 0: rH else: rL) + (if rL == 0: 2 else: 0) + else: + tz = 4 + nd = 4 + if output >= 100: + let q: uint32 = output div 100 + let r: uint32 = output mod 100 + output = q + dec(pos, 2) + utoa2Digits(buf, pos, r) + if tz == nd: + inc(tz, trailingZeros2Digits(r)) + inc(nd, 2) + if output >= 100: + let q2: uint32 = output div 100 + let r2: uint32 = output mod 100 + output = q2 + dec(pos, 2) + utoa2Digits(buf, pos, r2) + if tz == nd: + inc(tz, trailingZeros2Digits(r2)) + inc(nd, 2) + sf_Assert(output >= 1) + sf_Assert(output <= 99) + if output >= 10: + let q: uint32 = output + dec(pos, 2) + utoa2Digits(buf, pos, q) + if tz == nd: + inc(tz, trailingZeros2Digits(q)) + else: + let q: uint32 = output + sf_Assert(q >= 1) + sf_Assert(q <= 9) + dec(pos) + buf[pos] = chr(uint32('0') + q) + return tz + +proc decimalLength(v: uint32): int {.inline.} = + sf_Assert(v >= 1) + sf_Assert(v <= 999999999'u) + if v >= 100000000'u: + return 9 + if v >= 10000000'u: + return 8 + if v >= 1000000'u: + return 7 + if v >= 100000'u: + return 6 + if v >= 10000'u: + return 5 + if v >= 1000'u: + return 4 + if v >= 100'u: + return 3 + if v >= 10'u: + return 2 + return 1 + +proc formatDigits[T: Ordinal](buffer: var openArray[char]; pos: T; digits: uint32; decimalExponent: int; + forceTrailingDotZero: bool = false): int {.inline.} = + const + minFixedDecimalPoint: int32 = -4 + maxFixedDecimalPoint: int32 = 9 + var pos = pos + assert(minFixedDecimalPoint <= -1, "internal error") + assert(maxFixedDecimalPoint >= 1, "internal error") + sf_Assert(digits >= 1) + sf_Assert(digits <= 999999999'u) + sf_Assert(decimalExponent >= -99) + sf_Assert(decimalExponent <= 99) + var numDigits = decimalLength(digits) + let decimalPoint = numDigits + decimalExponent + let useFixed: bool = minFixedDecimalPoint <= decimalPoint and + decimalPoint <= maxFixedDecimalPoint + ## Prepare the buffer. + ## Avoid calling memset/memcpy with variable arguments below... + for i in 0..<32: buffer[pos+i] = '0' + assert(minFixedDecimalPoint >= -30, "internal error") + assert(maxFixedDecimalPoint <= 32, "internal error") + var decimalDigitsPosition: int + if useFixed: + if decimalPoint <= 0: + ## 0.[000]digits + decimalDigitsPosition = 2 - decimalPoint + else: + ## dig.its + ## digits[000] + decimalDigitsPosition = 0 + else: + ## dE+123 or d.igitsE+123 + decimalDigitsPosition = 1 + var digitsEnd = pos + decimalDigitsPosition + numDigits + let tz = printDecimalDigitsBackwards(buffer, digitsEnd, digits) + dec(digitsEnd, tz) + dec(numDigits, tz) + ## decimal_exponent += tz; // => decimal_point unchanged. + if useFixed: + if decimalPoint <= 0: + ## 0.[000]digits + buffer[pos+1] = '.' + pos = digitsEnd + elif decimalPoint < numDigits: + ## dig.its + for i in countdown(7, 0): + buffer[i + decimalPoint + 1] = buffer[i + decimalPoint] + buffer[pos+decimalPoint] = '.' + pos = digitsEnd + 1 + else: + ## digits[000] + inc(pos, decimalPoint) + if forceTrailingDotZero: + buffer[pos] = '.' + buffer[pos+1] = '0' + inc(pos, 2) + else: + buffer[pos] = buffer[pos+1] + if numDigits == 1: + ## dE+123 + inc(pos) + else: + ## d.igitsE+123 + buffer[pos+1] = '.' + pos = digitsEnd + let scientificExponent = decimalPoint - 1 + ## SF_ASSERT(scientific_exponent != 0); + buffer[pos] = 'e' + buffer[pos+1] = if scientificExponent < 0: '-' else: '+' + inc(pos, 2) + let k: uint32 = cast[uint32](if scientificExponent < 0: -scientificExponent else: scientificExponent) + if k < 10: + buffer[pos] = chr(uint32('0') + k) + inc pos + else: + utoa2Digits(buffer, pos, k) + inc(pos, 2) + return pos + +proc float32ToChars*(buffer: var openArray[char]; v: float32; forceTrailingDotZero = false): int {. + inline.} = + let significand: uint32 = physicalSignificand(constructSingle(v)) + let exponent: uint32 = physicalExponent(constructSingle(v)) + var pos = 0 + if exponent != maxIeeeExponent: + ## Finite + buffer[pos] = '-' + inc(pos, signBit(constructSingle(v))) + if exponent != 0 or significand != 0: + ## != 0 + let dec: auto = toDecimal32(significand, exponent) + return formatDigits(buffer, pos, dec.digits, dec.exponent.int, forceTrailingDotZero) + else: + buffer[pos] = '0' + buffer[pos+1] = '.' + buffer[pos+2] = '0' + buffer[pos+3] = ' ' + inc(pos, if forceTrailingDotZero: 3 else: 1) + return pos + if significand == 0: + buffer[pos] = '-' + inc(pos, signBit(constructSingle(v))) + buffer[pos] = 'i' + buffer[pos+1] = 'n' + buffer[pos+2] = 'f' + buffer[pos+3] = ' ' + return pos + 3 + else: + buffer[pos] = 'n' + buffer[pos+1] = 'a' + buffer[pos+2] = 'n' + buffer[pos+3] = ' ' + return pos + 3 diff --git a/lib/std/private/since.nim b/lib/std/private/since.nim index 5b22b6391..720120f11 100644 --- a/lib/std/private/since.nim +++ b/lib/std/private/since.nim @@ -1,5 +1,5 @@ ##[ -`since` is used to emulate older versions of nim stdlib with `--useVersion`, +`since` is used to emulate older versions of nim stdlib, see `tuse_version.nim`. If a symbol `foo` is added in version `(1,3,5)`, use `{.since: (1.3.5).}`, not @@ -15,19 +15,19 @@ The emulation cannot be 100% faithful and to avoid adding too much complexity, template since*(version: (int, int), body: untyped) {.dirty.} = ## Evaluates `body` if the ``(NimMajor, NimMinor)`` is greater than ## or equal to `version`. Usage: - ## - ## .. code-block:: Nim + ## ```Nim ## proc fun*() {.since: (1, 3).} ## since (1, 3): fun() + ## ``` when (NimMajor, NimMinor) >= version: body template since*(version: (int, int, int), body: untyped) {.dirty.} = ## Evaluates `body` if ``(NimMajor, NimMinor, NimPatch)`` is greater than ## or equal to `version`. Usage: - ## - ## .. code-block:: Nim + ## ```Nim ## proc fun*() {.since: (1, 3, 1).} ## since (1, 3, 1): fun() + ## ``` when (NimMajor, NimMinor, NimPatch) >= version: body diff --git a/lib/std/private/strimpl.nim b/lib/std/private/strimpl.nim index 7d42a7cf8..f8c9236a5 100644 --- a/lib/std/private/strimpl.nim +++ b/lib/std/private/strimpl.nim @@ -74,3 +74,40 @@ template endsWithImpl*[T: string | cstring](s, suffix: T) = func cmpNimIdentifier*[T: string | cstring](a, b: T): int = cmpIgnoreStyleImpl(a, b, true) + +func c_memchr(cstr: pointer, c: char, n: csize_t): pointer {. + importc: "memchr", header: "<string.h>".} +func c_strstr(haystack, needle: cstring): cstring {. + importc: "strstr", header: "<string.h>".} + + +func find*(s: cstring, sub: char, start: Natural = 0, last = 0): int = + ## Searches for `sub` in `s` inside the range `start..last` (both ends included). + ## If `last` is unspecified, it defaults to `s.high` (the last element). + ## + ## Searching is case-sensitive. If `sub` is not in `s`, -1 is returned. + ## Otherwise the index returned is relative to `s[0]`, not `start`. + ## Use `s[start..last].rfind` for a `start`-origin index. + let last = if last == 0: s.high else: last + let L = last-start+1 + if L > 0: + let found = c_memchr(s[start].unsafeAddr, sub, cast[csize_t](L)) + if not found.isNil: + return cast[int](found) -% cast[int](s) + return -1 + +func find*(s, sub: cstring, start: Natural = 0, last = 0): int = + ## Searches for `sub` in `s` inside the range `start..last` (both ends included). + ## If `last` is unspecified, it defaults to `s.high` (the last element). + ## + ## Searching is case-sensitive. If `sub` is not in `s`, -1 is returned. + ## Otherwise the index returned is relative to `s[0]`, not `start`. + ## Use `s[start..last].find` for a `start`-origin index. + if sub.len > s.len - start: return -1 + if sub.len == 1: return find(s, sub[0], start, last) + if last == 0 and s.len > start: + let found = c_strstr(cast[cstring](s[start].unsafeAddr), sub) + if not found.isNil: + result = cast[int](found) -% cast[int](s) + else: + result = -1 diff --git a/lib/std/private/syslocks.nim b/lib/std/private/syslocks.nim new file mode 100644 index 000000000..e19ec2c04 --- /dev/null +++ b/lib/std/private/syslocks.nim @@ -0,0 +1,234 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2012 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# Low level system locks and condition vars. + +{.push stackTrace: off.} + +when defined(windows): + type + Handle = int + + SysLock* {.importc: "CRITICAL_SECTION", + header: "<windows.h>", final, pure, byref.} = object # CRITICAL_SECTION in WinApi + DebugInfo: pointer + LockCount: int32 + RecursionCount: int32 + OwningThread: int + LockSemaphore: int + SpinCount: int + + SysCond* {.importc: "RTL_CONDITION_VARIABLE", header: "<windows.h>", byref.} = object + thePtr {.importc: "Ptr".} : Handle + + proc initSysLock*(L: var SysLock) {.importc: "InitializeCriticalSection", + header: "<windows.h>".} + ## Initializes the lock `L`. + + proc tryAcquireSysAux(L: var SysLock): int32 {.importc: "TryEnterCriticalSection", + header: "<windows.h>".} + ## Tries to acquire the lock `L`. + + proc tryAcquireSys*(L: var SysLock): bool {.inline.} = + result = tryAcquireSysAux(L) != 0'i32 + + proc acquireSys*(L: var SysLock) {.importc: "EnterCriticalSection", + header: "<windows.h>".} + ## Acquires the lock `L`. + + proc releaseSys*(L: var SysLock) {.importc: "LeaveCriticalSection", + header: "<windows.h>".} + ## Releases the lock `L`. + + proc deinitSys*(L: SysLock) {.importc: "DeleteCriticalSection", + header: "<windows.h>".} + + proc initializeConditionVariable( + conditionVariable: var SysCond + ) {.stdcall, noSideEffect, dynlib: "kernel32", importc: "InitializeConditionVariable".} + + proc sleepConditionVariableCS( + conditionVariable: var SysCond, + PCRITICAL_SECTION: var SysLock, + dwMilliseconds: int + ): int32 {.stdcall, noSideEffect, dynlib: "kernel32", importc: "SleepConditionVariableCS".} + + + proc signalSysCond*(hEvent: var SysCond) {.stdcall, noSideEffect, + dynlib: "kernel32", importc: "WakeConditionVariable".} + + proc broadcastSysCond*(hEvent: var SysCond) {.stdcall, noSideEffect, + dynlib: "kernel32", importc: "WakeAllConditionVariable".} + + proc initSysCond*(cond: var SysCond) {.inline.} = + initializeConditionVariable(cond) + proc deinitSysCond*(cond: SysCond) {.inline.} = + discard + proc waitSysCond*(cond: var SysCond, lock: var SysLock) = + discard sleepConditionVariableCS(cond, lock, -1'i32) + +elif defined(genode): + const + Header = "genode_cpp/syslocks.h" + type + SysLock* {.importcpp: "Nim::SysLock", pure, final, + header: Header.} = object + SysCond* {.importcpp: "Nim::SysCond", pure, final, + header: Header.} = object + + proc initSysLock*(L: var SysLock) = discard + proc deinitSys*(L: SysLock) = discard + proc acquireSys*(L: var SysLock) {.noSideEffect, importcpp.} + proc tryAcquireSys*(L: var SysLock): bool {.noSideEffect, importcpp.} + proc releaseSys*(L: var SysLock) {.noSideEffect, importcpp.} + + proc initSysCond*(L: var SysCond) = discard + proc deinitSysCond*(L: SysCond) = discard + proc waitSysCond*(cond: var SysCond, lock: var SysLock) {. + noSideEffect, importcpp.} + proc signalSysCond*(cond: var SysCond) {. + noSideEffect, importcpp.} + proc broadcastSysCond*(cond: var SysCond) {. + noSideEffect, importcpp.} + +else: + type + SysLockObj {.importc: "pthread_mutex_t", pure, final, + header: """#include <sys/types.h> + #include <pthread.h>""", byref.} = object + when defined(linux) and defined(amd64): + abi: array[40 div sizeof(clong), clong] + + SysLockAttr* {.importc: "pthread_mutexattr_t", pure, final + header: """#include <sys/types.h> + #include <pthread.h>""".} = object + when defined(linux) and defined(amd64): + abi: array[4 div sizeof(cint), cint] # actually a cint + + SysCondObj {.importc: "pthread_cond_t", pure, final, + header: """#include <sys/types.h> + #include <pthread.h>""", byref.} = object + when defined(linux) and defined(amd64): + abi: array[48 div sizeof(clonglong), clonglong] + + SysCondAttr {.importc: "pthread_condattr_t", pure, final + header: """#include <sys/types.h> + #include <pthread.h>""".} = object + when defined(linux) and defined(amd64): + abi: array[4 div sizeof(cint), cint] # actually a cint + + SysLockType = distinct cint + + proc initSysLockAux(L: var SysLockObj, attr: ptr SysLockAttr) {. + importc: "pthread_mutex_init", header: "<pthread.h>", noSideEffect.} + proc deinitSysAux(L: SysLockObj) {.noSideEffect, + importc: "pthread_mutex_destroy", header: "<pthread.h>".} + + proc acquireSysAux(L: var SysLockObj) {.noSideEffect, + importc: "pthread_mutex_lock", header: "<pthread.h>".} + proc tryAcquireSysAux(L: var SysLockObj): cint {.noSideEffect, + importc: "pthread_mutex_trylock", header: "<pthread.h>".} + + proc releaseSysAux(L: var SysLockObj) {.noSideEffect, + importc: "pthread_mutex_unlock", header: "<pthread.h>".} + + when defined(ios): + # iOS will behave badly if sync primitives are moved in memory. In order + # to prevent this once and for all, we're doing an extra malloc when + # initializing the primitive. + type + SysLock* = ptr SysLockObj + SysCond* = ptr SysCondObj + + when not declared(c_malloc): + proc c_malloc(size: csize_t): pointer {. + importc: "malloc", header: "<stdlib.h>".} + proc c_free(p: pointer) {. + importc: "free", header: "<stdlib.h>".} + + proc initSysLock*(L: var SysLock, attr: ptr SysLockAttr = nil) = + L = cast[SysLock](c_malloc(csize_t(sizeof(SysLockObj)))) + initSysLockAux(L[], attr) + + proc deinitSys*(L: SysLock) = + deinitSysAux(L[]) + c_free(L) + + template acquireSys*(L: var SysLock) = + acquireSysAux(L[]) + template tryAcquireSys*(L: var SysLock): bool = + tryAcquireSysAux(L[]) == 0'i32 + template releaseSys*(L: var SysLock) = + releaseSysAux(L[]) + else: + type + SysLock* = SysLockObj + SysCond* = SysCondObj + + template initSysLock*(L: var SysLock, attr: ptr SysLockAttr = nil) = + initSysLockAux(L, attr) + template deinitSys*(L: SysLock) = + deinitSysAux(L) + template acquireSys*(L: var SysLock) = + acquireSysAux(L) + template tryAcquireSys*(L: var SysLock): bool = + tryAcquireSysAux(L) == 0'i32 + template releaseSys*(L: var SysLock) = + releaseSysAux(L) + + # rlocks + var SysLockType_Reentrant* {.importc: "PTHREAD_MUTEX_RECURSIVE", + header: "<pthread.h>".}: SysLockType + proc initSysLockAttr*(a: var SysLockAttr) {. + importc: "pthread_mutexattr_init", header: "<pthread.h>", noSideEffect.} + proc setSysLockType*(a: var SysLockAttr, t: SysLockType) {. + importc: "pthread_mutexattr_settype", header: "<pthread.h>", noSideEffect.} + + # locks + proc initSysCondAux(cond: var SysCondObj, cond_attr: ptr SysCondAttr = nil) {. + importc: "pthread_cond_init", header: "<pthread.h>", noSideEffect.} + proc deinitSysCondAux(cond: SysCondObj) {.noSideEffect, + importc: "pthread_cond_destroy", header: "<pthread.h>".} + + proc waitSysCondAux(cond: var SysCondObj, lock: var SysLockObj): cint {. + importc: "pthread_cond_wait", header: "<pthread.h>", noSideEffect.} + proc signalSysCondAux(cond: var SysCondObj) {. + importc: "pthread_cond_signal", header: "<pthread.h>", noSideEffect.} + proc broadcastSysCondAux(cond: var SysCondObj) {. + importc: "pthread_cond_broadcast", header: "<pthread.h>", noSideEffect.} + + when defined(ios): + proc initSysCond*(cond: var SysCond, cond_attr: ptr SysCondAttr = nil) = + cond = cast[SysCond](c_malloc(csize_t(sizeof(SysCondObj)))) + initSysCondAux(cond[], cond_attr) + + proc deinitSysCond*(cond: SysCond) = + deinitSysCondAux(cond[]) + c_free(cond) + + template waitSysCond*(cond: var SysCond, lock: var SysLock) = + discard waitSysCondAux(cond[], lock[]) + template signalSysCond*(cond: var SysCond) = + signalSysCondAux(cond[]) + template broadcastSysCond*(cond: var SysCond) = + broadcastSysCondAux(cond[]) + else: + template initSysCond*(cond: var SysCond, cond_attr: ptr SysCondAttr = nil) = + initSysCondAux(cond, cond_attr) + template deinitSysCond*(cond: SysCond) = + deinitSysCondAux(cond) + + template waitSysCond*(cond: var SysCond, lock: var SysLock) = + discard waitSysCondAux(cond, lock) + template signalSysCond*(cond: var SysCond) = + signalSysCondAux(cond) + template broadcastSysCond*(cond: var SysCond) = + broadcastSysCondAux(cond) + +{.pop.} diff --git a/lib/std/private/threadtypes.nim b/lib/std/private/threadtypes.nim new file mode 100644 index 000000000..a1cdf21dc --- /dev/null +++ b/lib/std/private/threadtypes.nim @@ -0,0 +1,176 @@ +include system/inclrtl + +const hasSharedHeap* = defined(boehmgc) or defined(gogc) # don't share heaps; every thread has its own + +when defined(windows): + type + Handle* = int + SysThread* = Handle + WinThreadProc* = proc (x: pointer): int32 {.stdcall.} + + proc createThread*(lpThreadAttributes: pointer, dwStackSize: int32, + lpStartAddress: WinThreadProc, + lpParameter: pointer, + dwCreationFlags: int32, + lpThreadId: var int32): SysThread {. + stdcall, dynlib: "kernel32", importc: "CreateThread".} + + proc winSuspendThread*(hThread: SysThread): int32 {. + stdcall, dynlib: "kernel32", importc: "SuspendThread".} + + proc winResumeThread*(hThread: SysThread): int32 {. + stdcall, dynlib: "kernel32", importc: "ResumeThread".} + + proc waitForSingleObject*(hHandle: SysThread, dwMilliseconds: int32): int32 {. + stdcall, dynlib: "kernel32", importc: "WaitForSingleObject".} + + proc waitForMultipleObjects*(nCount: int32, + lpHandles: ptr SysThread, + bWaitAll: int32, + dwMilliseconds: int32): int32 {. + stdcall, dynlib: "kernel32", importc: "WaitForMultipleObjects".} + + proc terminateThread*(hThread: SysThread, dwExitCode: int32): int32 {. + stdcall, dynlib: "kernel32", importc: "TerminateThread".} + + proc setThreadAffinityMask*(hThread: SysThread, dwThreadAffinityMask: uint) {. + importc: "SetThreadAffinityMask", stdcall, header: "<windows.h>".} + +elif defined(genode): + const + GenodeHeader* = "genode_cpp/threads.h" + type + SysThread* {.importcpp: "Nim::SysThread", + header: GenodeHeader, final, pure.} = object + GenodeThreadProc* = proc (x: pointer) {.noconv.} + + proc initThread*(s: var SysThread, + env: GenodeEnv, + stackSize: culonglong, + entry: GenodeThreadProc, + arg: pointer, + affinity: cuint) {. + importcpp: "#.initThread(@)".} + + +else: + when not (defined(macosx) or defined(haiku)): + {.passl: "-pthread".} + + when not defined(haiku): + {.passc: "-pthread".} + + const + schedh = "#define _GNU_SOURCE\n#include <sched.h>" + pthreadh* = "#define _GNU_SOURCE\n#include <pthread.h>" + + when not declared(Time): + when defined(linux): + type Time = clong + else: + type Time = int + + when (defined(linux) or defined(nintendoswitch)) and defined(amd64): + type + SysThread* {.importc: "pthread_t", + header: "<sys/types.h>" .} = distinct culong + Pthread_attr* {.importc: "pthread_attr_t", + header: "<sys/types.h>".} = object + abi: array[56 div sizeof(clong), clong] + elif defined(openbsd) and defined(amd64): + type + SysThread* {.importc: "pthread_t", header: "<pthread.h>".} = object + Pthread_attr* {.importc: "pthread_attr_t", + header: "<pthread.h>".} = object + else: + type + SysThread* {.importc: "pthread_t", header: "<sys/types.h>".} = int + Pthread_attr* {.importc: "pthread_attr_t", + header: "<sys/types.h>".} = object + type + Timespec* {.importc: "struct timespec", header: "<time.h>".} = object + tv_sec*: Time + tv_nsec*: clong + + proc pthread_attr_init*(a1: var Pthread_attr): cint {. + importc, header: pthreadh.} + proc pthread_attr_setstack*(a1: ptr Pthread_attr, a2: pointer, a3: int): cint {. + importc, header: pthreadh.} + proc pthread_attr_setstacksize*(a1: var Pthread_attr, a2: int): cint {. + importc, header: pthreadh.} + proc pthread_attr_destroy*(a1: var Pthread_attr): cint {. + importc, header: pthreadh.} + + proc pthread_create*(a1: var SysThread, a2: var Pthread_attr, + a3: proc (x: pointer): pointer {.noconv.}, + a4: pointer): cint {.importc: "pthread_create", + header: pthreadh.} + proc pthread_join*(a1: SysThread, a2: ptr pointer): cint {. + importc, header: pthreadh.} + + proc pthread_cancel*(a1: SysThread): cint {. + importc: "pthread_cancel", header: pthreadh.} + + type CpuSet* {.importc: "cpu_set_t", header: schedh.} = object + when defined(linux) and defined(amd64): + abi: array[1024 div (8 * sizeof(culong)), culong] + + proc cpusetZero*(s: var CpuSet) {.importc: "CPU_ZERO", header: schedh.} + proc cpusetIncl*(cpu: cint; s: var CpuSet) {. + importc: "CPU_SET", header: schedh.} + + when defined(android): + # libc of android doesn't implement pthread_setaffinity_np, + # it exposes pthread_gettid_np though, so we can use that in combination + # with sched_setaffinity to set the thread affinity. + type Pid* {.importc: "pid_t", header: "<sys/types.h>".} = int32 # From posix_other.nim + + proc setAffinityTID*(tid: Pid; setsize: csize_t; s: var CpuSet) {. + importc: "sched_setaffinity", header: schedh.} + + proc pthread_gettid_np*(thread: SysThread): Pid {. + importc: "pthread_gettid_np", header: pthreadh.} + + proc setAffinity*(thread: SysThread; setsize: csize_t; s: var CpuSet) = + setAffinityTID(pthread_gettid_np(thread), setsize, s) + else: + proc setAffinity*(thread: SysThread; setsize: csize_t; s: var CpuSet) {. + importc: "pthread_setaffinity_np", header: pthreadh.} + + +const + emulatedThreadVars* = compileOption("tlsEmulation") +# we preallocate a fixed size for thread local storage, so that no heap +# allocations are needed. Currently less than 16K are used on a 64bit machine. +# We use `float` for proper alignment: +const nimTlsSize {.intdefine.} = 16000 +type + ThreadLocalStorage* = array[0..(nimTlsSize div sizeof(float)), float] + PGcThread* = ptr GcThread + GcThread* {.pure, inheritable.} = object + when emulatedThreadVars: + tls*: ThreadLocalStorage + else: + nil + when hasSharedHeap: + next*, prev*: PGcThread + stackBottom*, stackTop*: pointer + stackSize*: int + else: + nil + +const hasAllocStack* = defined(zephyr) # maybe freertos too? + +type + Thread*[TArg] = object + core*: PGcThread + sys*: SysThread + when TArg is void: + dataFn*: proc () {.nimcall, gcsafe.} + else: + dataFn*: proc (m: TArg) {.nimcall, gcsafe.} + data*: TArg + when hasAllocStack: + rawStack*: pointer + +proc `=copy`*[TArg](x: var Thread[TArg], y: Thread[TArg]) {.error.} diff --git a/lib/std/private/underscored_calls.nim b/lib/std/private/underscored_calls.nim index 6d0a99ab5..f853572b5 100644 --- a/lib/std/private/underscored_calls.nim +++ b/lib/std/private/underscored_calls.nim @@ -10,7 +10,9 @@ ## This is an internal helper module. Do not use. -import macros +import std/macros + +proc underscoredCalls*(result, calls, arg0: NimNode) proc underscoredCall(n, arg0: NimNode): NimNode = proc underscorePos(n: NimNode): int = @@ -19,13 +21,19 @@ proc underscoredCall(n, arg0: NimNode): NimNode = return 0 if n.kind in nnkCallKinds: - result = copyNimNode(n) - result.add n[0] + if n[0].kind in {nnkIdent, nnkSym} and n[0].eqIdent("with"): + expectKind n[1], {nnkIdent, nnkSym} - let u = underscorePos(n) - for i in 1..u-1: result.add n[i] - result.add arg0 - for i in u+1..n.len-1: result.add n[i] + result = newStmtList() + underscoredCalls(result, n[2 .. ^1].newStmtList, newDotExpr(arg0, n[1])) + else: + result = copyNimNode(n) + result.add n[0] + + let u = underscorePos(n) + for i in 1..u-1: result.add n[i] + result.add arg0 + for i in u+1..n.len-1: result.add n[i] elif n.kind in {nnkAsgn, nnkExprEqExpr}: var field = n[0] if n[0].kind == nnkDotExpr and n[0][0].eqIdent("_"): @@ -39,7 +47,7 @@ proc underscoredCall(n, arg0: NimNode): NimNode = result.add arg0 proc underscoredCalls*(result, calls, arg0: NimNode) = - expectKind calls, {nnkArglist, nnkStmtList, nnkStmtListExpr} + expectKind calls, {nnkArgList, nnkStmtList, nnkStmtListExpr} for call in calls: if call.kind in {nnkStmtList, nnkStmtListExpr}: diff --git a/lib/std/private/vmutils.nim b/lib/std/private/vmutils.nim deleted file mode 100644 index d54977b4e..000000000 --- a/lib/std/private/vmutils.nim +++ /dev/null @@ -1,17 +0,0 @@ -template forwardImpl*(impl, arg) {.dirty.} = - when sizeof(x) <= 4: - when x is SomeSignedInt: - impl(cast[uint32](x.int32)) - else: - impl(x.uint32) - else: - when x is SomeSignedInt: - impl(cast[uint64](x.int64)) - else: - impl(x.uint64) - -template toUnsigned*(x: int8): uint8 = cast[uint8](x) -template toUnsigned*(x: int16): uint16 = cast[uint16](x) -template toUnsigned*(x: int32): uint32 = cast[uint32](x) -template toUnsigned*(x: int64): uint64 = cast[uint64](x) -template toUnsigned*(x: int): uint = cast[uint](x) diff --git a/lib/std/private/win_getsysteminfo.nim b/lib/std/private/win_getsysteminfo.nim new file mode 100644 index 000000000..b98478231 --- /dev/null +++ b/lib/std/private/win_getsysteminfo.nim @@ -0,0 +1,15 @@ +type + SystemInfo* = object + u1: uint32 + dwPageSize: uint32 + lpMinimumApplicationAddress: pointer + lpMaximumApplicationAddress: pointer + dwActiveProcessorMask: ptr uint32 + dwNumberOfProcessors*: uint32 + dwProcessorType: uint32 + dwAllocationGranularity*: uint32 + wProcessorLevel: uint16 + wProcessorRevision: uint16 + +proc getSystemInfo*(lpSystemInfo: ptr SystemInfo) {.stdcall, + dynlib: "kernel32", importc: "GetSystemInfo".} diff --git a/lib/std/private/win_setenv.nim b/lib/std/private/win_setenv.nim new file mode 100644 index 000000000..66e199dfe --- /dev/null +++ b/lib/std/private/win_setenv.nim @@ -0,0 +1,106 @@ +#[ +Copyright (c) Facebook, Inc. and its affiliates. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Adapted `setenv` from https://github.com/facebook/folly/blob/master/folly/portability/Stdlib.cpp +translated from C to nim. +]# + +#[ +Introduced in https://github.com/facebook/folly/commit/5d8ca09a3f96afefb44e35808f03651a096ab9c7 + +TODO: +check errno_t vs cint +]# + +when not defined(windows): discard +else: + when defined(nimPreviewSlimSystem): + import std/widestrs + + type wchar_t {.importc: "wchar_t".} = int16 + + proc setEnvironmentVariableW*(lpName, lpValue: WideCString): int32 {. + stdcall, dynlib: "kernel32", importc: "SetEnvironmentVariableW", sideEffect.} + # same as winlean.setEnvironmentVariableA + + proc c_getenv(varname: cstring): cstring {.importc: "getenv", header: "<stdlib.h>".} + proc c_wputenv(envstring: ptr wchar_t): cint {.importc: "_wputenv", header: "<stdlib.h>".} + proc c_wgetenv(varname: ptr wchar_t): ptr wchar_t {.importc: "_wgetenv", header: "<stdlib.h>".} + + var errno {.importc, header: "<errno.h>".}: cint + var genviron {.importc: "_environ".}: ptr ptr char + # xxx `ptr UncheckedArray[WideCString]` did not work + + proc wcstombs(wcstr: ptr char, mbstr: ptr wchar_t, count: csize_t): csize_t {.importc, header: "<stdlib.h>".} + # xxx cint vs errno_t? + + proc setEnvImpl*(name: string, value: string, overwrite: cint): cint = + const EINVAL = cint(22) + let wideName: WideCString = newWideCString(name) + if overwrite == 0 and c_wgetenv(cast[ptr wchar_t](wideName)) != nil: + return 0 + + if value != "": + let envstring: WideCString = newWideCString(name & "=" & value) + let e = c_wputenv(cast[ptr wchar_t](envstring)) + if e != 0: + errno = EINVAL + return -1 + return 0 + #[ + We are trying to set the value to an empty string, but `_putenv` deletes + entries if the value is an empty string, and just calling + SetEnvironmentVariableA doesn't update `_environ`, + so we have to do these terrible things. + ]# + let envstring: WideCString = newWideCString(name & "= ") + if c_wputenv(cast[ptr wchar_t](envstring)) != 0: + errno = EINVAL + return -1 + # Here lies the documentation we blatently ignore to make this work. + var s = cast[WideCString](c_wgetenv(cast[ptr wchar_t](wideName))) + s[0] = Utf16Char('\0') + #[ + This would result in a double null termination, which normally signifies the + end of the environment variable list, so we stick a completely empty + environment variable into the list instead. + ]# + s = cast[WideCString](c_wgetenv(cast[ptr wchar_t](wideName))) + s[1] = Utf16Char('=') + #[ + If genviron is null, the MBCS environment has not been initialized + yet, and we don't need to try to update it. We have to do this otherwise + we'd be forcing the initialization and maintenance of the MBCS environment + even though it's never actually used in most programs. + ]# + if genviron != nil: + + # wcstombs returns `high(csize_t)` if any characters cannot be represented + # in the current codepage. Skip updating MBCS environment in this case. + # For some reason, second `wcstombs` can find non-convertible characters + # that the first `wcstombs` cannot. + let requiredSizeS = wcstombs(nil, cast[ptr wchar_t](wideName), 0) + if requiredSizeS != high(csize_t): + let requiredSize = requiredSizeS.int + var buf = newSeq[char](requiredSize + 1) + let buf2 = buf[0].addr + if wcstombs(buf2, cast[ptr wchar_t](wideName), csize_t(requiredSize + 1)) != high(csize_t): + var ptrToEnv = c_getenv(cast[cstring](buf2)) + ptrToEnv[0] = '\0' + ptrToEnv = c_getenv(cast[cstring](buf2)) + ptrToEnv[1] = '=' + + # And now, we have to update the outer environment to have a proper empty value. + if setEnvironmentVariableW(wideName, value.newWideCString) == 0: + errno = EINVAL + return -1 + return 0 diff --git a/lib/std/setutils.nim b/lib/std/setutils.nim index c7fac0a54..8e7bc6a92 100644 --- a/lib/std/setutils.nim +++ b/lib/std/setutils.nim @@ -1,6 +1,6 @@ # # -# The Nim Compiler +# Nim's Runtime Library # (c) Copyright 2020 Nim Contributors # # See the file "copying.txt", included in this @@ -36,7 +36,7 @@ template toSet*(iter: untyped): untyped = incl(result, x) result -macro enmRange(enm: typed): untyped = result = newNimNode(nnkCurly).add(enm.getType[1][1..^1]) +macro enumElementsAsSet(enm: typed): untyped = result = newNimNode(nnkCurly).add(enm.getType[1][1..^1]) # func fullSet*(T: typedesc): set[T] {.inline.} = # xxx would give: Error: ordinal type expected func fullSet*[T](U: typedesc[T]): set[T] {.inline.} = @@ -49,7 +49,7 @@ func fullSet*[T](U: typedesc[T]): set[T] {.inline.} = when T is Ordinal: {T.low..T.high} else: # Hole filled enum - enmRange(T) + enumElementsAsSet(T) func complement*[T](s: set[T]): set[T] {.inline.} = ## Returns the set complement of `a`. diff --git a/lib/std/sha1.nim b/lib/std/sha1.nim index b74b285f8..213af4229 100644 --- a/lib/std/sha1.nim +++ b/lib/std/sha1.nim @@ -1,41 +1,40 @@ # # -# The Nim Compiler +# Nim's Runtime Library # (c) Copyright 2015 Nim Contributors # # See the file "copying.txt", included in this # distribution, for details about the copyright. # -## **Note:** Import `std/sha1` to use this module. -## ## [SHA-1 (Secure Hash Algorithm 1)](https://en.wikipedia.org/wiki/SHA-1) ## is a cryptographic hash function which takes an input and produces ## a 160-bit (20-byte) hash value known as a message digest. ## -## Basic usage -## =========== -## +## See also +## ======== +## * `base64 module<base64.html>`_ for a Base64 encoder and decoder +## * `hashes module<hashes.html>`_ for efficient computations of hash values for diverse Nim types +## * `md5 module<md5.html>`_ for the MD5 checksum algorithm + runnableExamples: let accessName = secureHash("John Doe") assert $accessName == "AE6E4D1209F17B460503904FAD297B31E9CF6362" -## .. code-block:: -## let -## a = secureHashFile("myFile.nim") -## b = parseSecureHash("10DFAEBF6BFDBC7939957068E2EFACEC4972933C") -## -## if a == b: -## echo "Files match" -## -## See also -## ======== -## * `base64 module<base64.html>`_ implements a Base64 encoder and decoder -## * `hashes module<hashes.html>`_ for efficient computations of hash values for diverse Nim types -## * `md5 module<md5.html>`_ implements the MD5 checksum algorithm +runnableExamples("-r:off"): + let + a = secureHashFile("myFile.nim") + b = parseSecureHash("10DFAEBF6BFDBC7939957068E2EFACEC4972933C") + assert a == b, "files don't match" + + +{.deprecated: "use command `nimble install checksums` and import `checksums/sha1` instead".} import std/strutils from std/endians import bigEndian32, bigEndian64 +when defined(nimPreviewSlimSystem): + import std/syncio + const Sha1DigestSize = 20 type @@ -282,3 +281,7 @@ proc `==`*(a, b: SecureHash): bool = # Not a constant-time comparison, but that's acceptable in this context Sha1Digest(a) == Sha1Digest(b) + +proc isValidSha1Hash*(s: string): bool = + ## Checks if a string is a valid sha1 hash sum. + s.len == 40 and allCharsInSet(s, HexDigits) \ No newline at end of file diff --git a/lib/std/socketstreams.nim b/lib/std/socketstreams.nim index 5c882858d..45e906795 100644 --- a/lib/std/socketstreams.nim +++ b/lib/std/socketstreams.nim @@ -31,39 +31,40 @@ ## Examples ## ======== ## -## .. code-block:: Nim -## import std/socketstreams +## ```Nim +## import std/socketstreams ## -## var -## socket = newSocket(AF_INET, SOCK_DGRAM, IPPROTO_UDP) -## stream = newReadSocketStream(socket) -## socket.sendTo("127.0.0.1", Port(12345), "SOME REQUEST") -## echo stream.readLine() # Will call `recv` -## stream.setPosition(0) -## echo stream.readLine() # Will return the read line from the buffer -## stream.resetStream() # Buffer is now empty, position is 0 -## echo stream.readLine() # Will call `recv` again -## stream.close() # Closes the socket +## var +## socket = newSocket(AF_INET, SOCK_DGRAM, IPPROTO_UDP) +## stream = newReadSocketStream(socket) +## socket.sendTo("127.0.0.1", Port(12345), "SOME REQUEST") +## echo stream.readLine() # Will call `recv` +## stream.setPosition(0) +## echo stream.readLine() # Will return the read line from the buffer +## stream.resetStream() # Buffer is now empty, position is 0 +## echo stream.readLine() # Will call `recv` again +## stream.close() # Closes the socket +## ``` ## -## .. code-block:: Nim +## ```Nim +## import std/socketstreams ## -## import std/socketstreams -## -## var socket = newSocket(AF_INET, SOCK_DGRAM, IPPROTO_UDP) -## socket.connect("127.0.0.1", Port(12345)) -## var sendStream = newWriteSocketStream(socket) -## sendStream.write "NOM" -## sendStream.setPosition(1) -## echo sendStream.peekStr(2) # OM -## sendStream.write "I" -## sendStream.setPosition(0) -## echo sendStream.readStr(3) # NIM -## echo sendStream.getPosition() # 3 -## sendStream.flush() # This actually performs the writing to the socket -## sendStream.setPosition(1) -## sendStream.write "I" # Throws an error as we can't write into an already sent buffer - -import net, streams +## var socket = newSocket(AF_INET, SOCK_DGRAM, IPPROTO_UDP) +## socket.connect("127.0.0.1", Port(12345)) +## var sendStream = newWriteSocketStream(socket) +## sendStream.write "NOM" +## sendStream.setPosition(1) +## echo sendStream.peekStr(2) # OM +## sendStream.write "I" +## sendStream.setPosition(0) +## echo sendStream.readStr(3) # NIM +## echo sendStream.getPosition() # 3 +## sendStream.flush() # This actually performs the writing to the socket +## sendStream.setPosition(1) +## sendStream.write "I" # Throws an error as we can't write into an already sent buffer +## ``` + +import std/[net, streams] type ReadSocketStream* = ref ReadSocketStreamObj @@ -146,7 +147,7 @@ proc wsFlush(s: Stream) = s.lastFlush = s.buf.len proc rsClose(s: Stream) = - {.cast(tags: []).}: + {.cast(raises: [IOError, OSError]), cast(tags: []).}: # todo fixme maybe do something? var s = ReadSocketStream(s) s.data.close() diff --git a/lib/std/staticos.nim b/lib/std/staticos.nim new file mode 100644 index 000000000..2617c6913 --- /dev/null +++ b/lib/std/staticos.nim @@ -0,0 +1,13 @@ +## This module implements path handling like os module but works at only compile-time. +## This module works even when cross compiling to OS that is not supported by os module. + +proc staticFileExists*(filename: string): bool {.compileTime.} = + ## Returns true if `filename` exists and is a regular file or symlink. + ## + ## Directories, device files, named pipes and sockets return false. + discard + +proc staticDirExists*(dir: string): bool {.compileTime.} = + ## Returns true if the directory `dir` exists. If `dir` is a file, false + ## is returned. Follows symlinks. + discard diff --git a/lib/std/strbasics.nim b/lib/std/strbasics.nim index 9232204ab..b2c36a4be 100644 --- a/lib/std/strbasics.nim +++ b/lib/std/strbasics.nim @@ -1,6 +1,6 @@ # # -# The Nim Compiler +# Nim's Runtime Library # (c) Copyright 2021 Nim Contributors # # See the file "copying.txt", included in this @@ -11,6 +11,10 @@ ## ## Experimental API, subject to change. +when defined(nimPreviewSlimSystem): + import std/assertions + + const whitespaces = {' ', '\t', '\v', '\r', '\l', '\f'} proc add*(x: var string, y: openArray[char]) = @@ -19,8 +23,8 @@ proc add*(x: var string, y: openArray[char]) = # Use `{.noalias.}` ? let n = x.len x.setLen n + y.len - # pending https://github.com/nim-lang/Nim/issues/14655#issuecomment-643671397 - # use x.setLen(n + y.len, isInit = false) + # pending #19727 + # setLen unnecessarily zeros memory var i = 0 while i < y.len: x[n + i] = y[i] @@ -46,7 +50,7 @@ func setSlice*(s: var string, slice: Slice[int]) = import std/sugar var a = "Hello, Nim!" - doassert a.dup(setSlice(7 .. 9)) == "Nim" + doAssert a.dup(setSlice(7 .. 9)) == "Nim" doAssert a.dup(setSlice(0 .. 0)) == "H" doAssert a.dup(setSlice(0 .. 1)) == "He" doAssert a.dup(setSlice(0 .. 10)) == a diff --git a/lib/std/sums.nim b/lib/std/sums.nim deleted file mode 100644 index b68858ef7..000000000 --- a/lib/std/sums.nim +++ /dev/null @@ -1,78 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2019 b3liever -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. - -## Accurate summation functions. - -runnableExamples: - import std/math - - template `~=`(x, y: float): bool = abs(x - y) < 1e-4 - - let - n = 1_000_000 - first = 1e10 - small = 0.1 - var data = @[first] - for _ in 1 .. n: - data.add(small) - - let result = first + small * n.float - - doAssert abs(sum(data) - result) > 0.3 - doAssert sumKbn(data) ~= result - doAssert sumPairs(data) ~= result - -## See also -## ======== -## * `math module <math.html>`_ for a standard `sum proc <math.html#sum,openArray[T]>`_ - -func sumKbn*[T](x: openArray[T]): T = - ## Kahan-BabuÅ¡ka-Neumaier summation: O(1) error growth, at the expense - ## of a considerable increase in computational cost. - ## - ## See: - ## * https://en.wikipedia.org/wiki/Kahan_summation_algorithm#Further_enhancements - if len(x) == 0: return - var sum = x[0] - var c = T(0) - for i in 1 ..< len(x): - let xi = x[i] - let t = sum + xi - if abs(sum) >= abs(xi): - c += (sum - t) + xi - else: - c += (xi - t) + sum - sum = t - result = sum + c - -func sumPairwise[T](x: openArray[T], i0, n: int): T = - if n < 128: - result = x[i0] - for i in i0 + 1 ..< i0 + n: - result += x[i] - else: - let n2 = n div 2 - result = sumPairwise(x, i0, n2) + sumPairwise(x, i0 + n2, n - n2) - -func sumPairs*[T](x: openArray[T]): T = - ## Pairwise (cascade) summation of `x[i0:i0+n-1]`, with O(log n) error growth - ## (vs O(n) for a simple loop) with negligible performance cost if - ## the base case is large enough. - ## - ## See, e.g.: - ## * https://en.wikipedia.org/wiki/Pairwise_summation - ## * Higham, Nicholas J. (1993), "The accuracy of floating point - ## summation", SIAM Journal on Scientific Computing 14 (4): 783–799. - ## - ## In fact, the root-mean-square error growth, assuming random roundoff - ## errors, is only O(sqrt(log n)), which is nearly indistinguishable from O(1) - ## in practice. See: - ## * Manfred Tasche and Hansmartin Zeuner, Handbook of - ## Analytic-Computational Methods in Applied Mathematics (2000). - let n = len(x) - if n == 0: T(0) else: sumPairwise(x, 0, n) diff --git a/lib/std/symlinks.nim b/lib/std/symlinks.nim new file mode 100644 index 000000000..dbe908612 --- /dev/null +++ b/lib/std/symlinks.nim @@ -0,0 +1,33 @@ +## This module implements symlink (symbolic link) handling. + +## .. importdoc:: os.nim + +from std/paths import Path, ReadDirEffect + +from std/private/ossymlinks import symlinkExists, createSymlink, expandSymlink + +proc symlinkExists*(link: Path): bool {.inline, tags: [ReadDirEffect], sideEffect.} = + ## Returns true if the symlink `link` exists. Will return true + ## regardless of whether the link points to a directory or file. + result = symlinkExists(link.string) + +proc createSymlink*(src, dest: Path) {.inline.} = + ## Create a symbolic link at `dest` which points to the item specified + ## by `src`. On most operating systems, will fail if a link already exists. + ## + ## .. warning:: Some OS's (such as Microsoft Windows) restrict the creation + ## of symlinks to root users (administrators) or users with developer mode enabled. + ## + ## See also: + ## * `createHardlink proc`_ + ## * `expandSymlink proc`_ + createSymlink(src.string, dest.string) + +proc expandSymlink*(symlinkPath: Path): Path {.inline.} = + ## Returns a string representing the path to which the symbolic link points. + ## + ## On Windows this is a noop, `symlinkPath` is simply returned. + ## + ## See also: + ## * `createSymlink proc`_ + result = Path(expandSymlink(symlinkPath.string)) diff --git a/lib/std/syncio.nim b/lib/std/syncio.nim new file mode 100644 index 000000000..c34a025af --- /dev/null +++ b/lib/std/syncio.nim @@ -0,0 +1,942 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2022 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module implements various synchronized I/O operations. + +include system/inclrtl +import std/private/since +import std/formatfloat +when defined(windows): + import std/widestrs + +# ----------------- IO Part ------------------------------------------------ +type + CFile {.importc: "FILE", header: "<stdio.h>", + incompleteStruct.} = object + File* = ptr CFile ## The type representing a file handle. + + FileMode* = enum ## The file mode when opening a file. + fmRead, ## Open the file for read access only. + ## If the file does not exist, it will not + ## be created. + fmWrite, ## Open the file for write access only. + ## If the file does not exist, it will be + ## created. Existing files will be cleared! + fmReadWrite, ## Open the file for read and write access. + ## If the file does not exist, it will be + ## created. Existing files will be cleared! + fmReadWriteExisting, ## Open the file for read and write access. + ## If the file does not exist, it will not be + ## created. The existing file will not be cleared. + fmAppend ## Open the file for writing only; append data + ## at the end. If the file does not exist, it + ## will be created. + + FileHandle* = cint ## The type that represents an OS file handle; this is + ## useful for low-level file access. + + FileSeekPos* = enum ## Position relative to which seek should happen. + # The values are ordered so that they match with stdio + # SEEK_SET, SEEK_CUR and SEEK_END respectively. + fspSet ## Seek to absolute value + fspCur ## Seek relative to current position + fspEnd ## Seek relative to end + +# text file handling: +when not defined(nimscript) and not defined(js): + # duplicated between io and ansi_c + const stdioUsesMacros = (defined(osx) or defined(freebsd) or defined( + dragonfly)) and not defined(emscripten) + const stderrName = when stdioUsesMacros: "__stderrp" else: "stderr" + const stdoutName = when stdioUsesMacros: "__stdoutp" else: "stdout" + const stdinName = when stdioUsesMacros: "__stdinp" else: "stdin" + + var + stdin* {.importc: stdinName, header: "<stdio.h>".}: File + ## The standard input stream. + stdout* {.importc: stdoutName, header: "<stdio.h>".}: File + ## The standard output stream. + stderr* {.importc: stderrName, header: "<stdio.h>".}: File + ## The standard error stream. + +when defined(useStdoutAsStdmsg): + template stdmsg*: File = stdout +else: + template stdmsg*: File = stderr + ## Template which expands to either stdout or stderr depending on + ## `useStdoutAsStdmsg` compile-time switch. + +when defined(windows): + proc c_fileno(f: File): cint {. + importc: "_fileno", header: "<stdio.h>".} +else: + proc c_fileno(f: File): cint {. + importc: "fileno", header: "<fcntl.h>".} + +when defined(windows): + proc c_fdopen(filehandle: cint, mode: cstring): File {. + importc: "_fdopen", header: "<stdio.h>".} +else: + proc c_fdopen(filehandle: cint, mode: cstring): File {. + importc: "fdopen", header: "<stdio.h>".} +proc c_fputs(c: cstring, f: File): cint {. + importc: "fputs", header: "<stdio.h>", tags: [WriteIOEffect].} +proc c_fgets(c: cstring, n: cint, f: File): cstring {. + importc: "fgets", header: "<stdio.h>", tags: [ReadIOEffect].} +proc c_fgetc(stream: File): cint {. + importc: "fgetc", header: "<stdio.h>", tags: [].} +proc c_ungetc(c: cint, f: File): cint {. + importc: "ungetc", header: "<stdio.h>", tags: [].} +proc c_putc(c: cint, stream: File): cint {. + importc: "putc", header: "<stdio.h>", tags: [WriteIOEffect].} +proc c_fflush(f: File): cint {. + importc: "fflush", header: "<stdio.h>".} +proc c_fclose(f: File): cint {. + importc: "fclose", header: "<stdio.h>".} +proc c_clearerr(f: File) {. + importc: "clearerr", header: "<stdio.h>".} +proc c_feof(f: File): cint {. + importc: "feof", header: "<stdio.h>".} + +when not declared(c_fwrite): + proc c_fwrite(buf: pointer, size, n: csize_t, f: File): csize_t {. + importc: "fwrite", header: "<stdio.h>".} + +# C routine that is used here: +proc c_fread(buf: pointer, size, n: csize_t, f: File): csize_t {. + importc: "fread", header: "<stdio.h>", tags: [ReadIOEffect].} +when defined(windows): + when not defined(amd64): + proc c_fseek(f: File, offset: int64, whence: cint): cint {. + importc: "fseek", header: "<stdio.h>", tags: [].} + proc c_ftell(f: File): int64 {. + importc: "ftell", header: "<stdio.h>", tags: [].} + else: + proc c_fseek(f: File, offset: int64, whence: cint): cint {. + importc: "_fseeki64", header: "<stdio.h>", tags: [].} + when defined(tcc): + proc c_fsetpos(f: File, pos: var int64): int32 {. + importc: "fsetpos", header: "<stdio.h>", tags: [].} + proc c_fgetpos(f: File, pos: var int64): int32 {. + importc: "fgetpos", header: "<stdio.h>", tags: [].} + proc c_telli64(f: cint): int64 {. + importc: "_telli64", header: "<io.h>", tags: [].} + proc c_ftell(f: File): int64 = + # Taken from https://pt.osdn.net/projects/mingw/scm/git/mingw-org-wsl/blobs/5.4-trunk/mingwrt/mingwex/stdio/ftelli64.c + result = -1'i64 + var pos: int64 + if c_fgetpos(f, pos) == 0 and c_fsetpos(f, pos) == 0: + result = c_telli64(c_fileno(f)) + else: + proc c_ftell(f: File): int64 {. + importc: "_ftelli64", header: "<stdio.h>", tags: [].} +else: + proc c_fseek(f: File, offset: int64, whence: cint): cint {. + importc: "fseeko", header: "<stdio.h>", tags: [].} + proc c_ftell(f: File): int64 {. + importc: "ftello", header: "<stdio.h>", tags: [].} +proc c_ferror(f: File): cint {. + importc: "ferror", header: "<stdio.h>", tags: [].} +proc c_setvbuf(f: File, buf: pointer, mode: cint, size: csize_t): cint {. + importc: "setvbuf", header: "<stdio.h>", tags: [].} + +proc c_fprintf(f: File, frmt: cstring): cint {. + importc: "fprintf", header: "<stdio.h>", varargs, discardable.} +proc c_fputc(c: char, f: File): cint {. + importc: "fputc", header: "<stdio.h>".} + +proc raiseEIO(msg: string) {.noinline, noreturn.} = + raise newException(IOError, msg) + +proc raiseEOF() {.noinline, noreturn.} = + raise newException(EOFError, "EOF reached") + +proc strerror(errnum: cint): cstring {.importc, header: "<string.h>".} + +when not defined(nimscript): + var + errno {.importc, header: "<errno.h>".}: cint ## error variable + EINTR {.importc: "EINTR", header: "<errno.h>".}: cint + +proc checkErr(f: File) = + when not defined(nimscript): + if c_ferror(f) != 0: + let msg = "errno: " & $errno & " `" & $strerror(errno) & "`" + c_clearerr(f) + raiseEIO(msg) + else: + # shouldn't happen + quit(1) + +{.push stackTrace: off, profiler: off.} +proc readBuffer*(f: File, buffer: pointer, len: Natural): int {. + tags: [ReadIOEffect], benign.} = + ## Reads `len` bytes into the buffer pointed to by `buffer`. Returns + ## the actual number of bytes that have been read which may be less than + ## `len` (if not as many bytes are remaining), but not greater. + result = cast[int](c_fread(buffer, 1, cast[csize_t](len), f)) + if result != len: checkErr(f) + +proc readBytes*(f: File, a: var openArray[int8|uint8], start, + len: Natural): int {. + tags: [ReadIOEffect], benign.} = + ## Reads `len` bytes into the buffer `a` starting at `a[start]`. Returns + ## the actual number of bytes that have been read which may be less than + ## `len` (if not as many bytes are remaining), but not greater. + result = readBuffer(f, addr(a[start]), len) + +proc readChars*(f: File, a: var openArray[char]): int {.tags: [ReadIOEffect], benign.} = + ## Reads up to `a.len` bytes into the buffer `a`. Returns + ## the actual number of bytes that have been read which may be less than + ## `a.len` (if not as many bytes are remaining), but not greater. + result = readBuffer(f, addr(a[0]), a.len) + +proc readChars*(f: File, a: var openArray[char], start, len: Natural): int {. + tags: [ReadIOEffect], benign, deprecated: + "use other `readChars` overload, possibly via: readChars(toOpenArray(buf, start, len-1))".} = + ## Reads `len` bytes into the buffer `a` starting at `a[start]`. Returns + ## the actual number of bytes that have been read which may be less than + ## `len` (if not as many bytes are remaining), but not greater. + if (start + len) > len(a): + raiseEIO("buffer overflow: (start+len) > length of openarray buffer") + result = readBuffer(f, addr(a[start]), len) + +proc write*(f: File, c: cstring) {.tags: [WriteIOEffect], benign.} = + ## Writes a value to the file `f`. May throw an IO exception. + discard c_fputs(c, f) + checkErr(f) + +proc writeBuffer*(f: File, buffer: pointer, len: Natural): int {. + tags: [WriteIOEffect], benign.} = + ## Writes the bytes of buffer pointed to by the parameter `buffer` to the + ## file `f`. Returns the number of actual written bytes, which may be less + ## than `len` in case of an error. + result = cast[int](c_fwrite(buffer, 1, cast[csize_t](len), f)) + checkErr(f) + +proc writeBytes*(f: File, a: openArray[int8|uint8], start, len: Natural): int {. + tags: [WriteIOEffect], benign.} = + ## Writes the bytes of `a[start..start+len-1]` to the file `f`. Returns + ## the number of actual written bytes, which may be less than `len` in case + ## of an error. + var x = cast[ptr UncheckedArray[int8]](a) + result = writeBuffer(f, addr(x[int(start)]), len) + +proc writeChars*(f: File, a: openArray[char], start, len: Natural): int {. + tags: [WriteIOEffect], benign.} = + ## Writes the bytes of `a[start..start+len-1]` to the file `f`. Returns + ## the number of actual written bytes, which may be less than `len` in case + ## of an error. + var x = cast[ptr UncheckedArray[int8]](a) + result = writeBuffer(f, addr(x[int(start)]), len) + +when defined(windows): + proc writeWindows(f: File; s: string; doRaise = false) = + # Don't ask why but the 'printf' family of function is the only thing + # that writes utf-8 strings reliably on Windows. At least on my Win 10 + # machine. We also enable `setConsoleOutputCP(65001)` now by default. + # But we cannot call printf directly as the string might contain \0. + # So we have to loop over all the sections separated by potential \0s. + var i = int c_fprintf(f, "%s", s) + while i < s.len: + if s[i] == '\0': + let w = c_fputc('\0', f) + if w != 0: + if doRaise: raiseEIO("cannot write string to file") + break + inc i + else: + let w = c_fprintf(f, "%s", unsafeAddr s[i]) + if w <= 0: + if doRaise: raiseEIO("cannot write string to file") + break + inc i, w + +proc write*(f: File, s: string) {.tags: [WriteIOEffect], benign.} = + when defined(windows): + writeWindows(f, s, doRaise = true) + else: + if writeBuffer(f, cstring(s), s.len) != s.len: + raiseEIO("cannot write string to file") +{.pop.} + +when defined(nimscript): + when defined(windows): + const + IOFBF = cint(0) + IONBF = cint(4) + else: + # On all systems I could find, including Linux, Mac OS X, and the BSDs + const + IOFBF = cint(0) + IONBF = cint(2) +else: + var + IOFBF {.importc: "_IOFBF", nodecl.}: cint + IONBF {.importc: "_IONBF", nodecl.}: cint + +const SupportIoctlInheritCtl = (defined(linux) or defined(bsd)) and + not defined(nimscript) +when SupportIoctlInheritCtl: + var + FIOCLEX {.importc, header: "<sys/ioctl.h>".}: cint + FIONCLEX {.importc, header: "<sys/ioctl.h>".}: cint + + proc c_ioctl(fd: cint, request: cint): cint {. + importc: "ioctl", header: "<sys/ioctl.h>", varargs.} +elif defined(posix) and not defined(lwip) and not defined(nimscript): + var + F_GETFD {.importc, header: "<fcntl.h>".}: cint + F_SETFD {.importc, header: "<fcntl.h>".}: cint + FD_CLOEXEC {.importc, header: "<fcntl.h>".}: cint + + proc c_fcntl(fd: cint, cmd: cint): cint {. + importc: "fcntl", header: "<fcntl.h>", varargs.} +elif defined(windows): + type + WinDWORD = culong + WinBOOL = cint + + const HANDLE_FLAG_INHERIT = 1.WinDWORD + + proc getOsfhandle(fd: cint): int {. + importc: "_get_osfhandle", header: "<io.h>".} + + type + IoHandle = distinct pointer + ## Windows' HANDLE type. Defined as an untyped pointer but is **not** + ## one. Named like this to avoid collision with other `system` modules. + + proc setHandleInformation(hObject: IoHandle, dwMask, dwFlags: WinDWORD): + WinBOOL {.stdcall, dynlib: "kernel32", + importc: "SetHandleInformation".} + +const + BufSize = 4000 + +proc close*(f: File) {.tags: [], gcsafe, sideEffect.} = + ## Closes the file. + if not f.isNil: + discard c_fclose(f) + +proc readChar*(f: File): char {.tags: [ReadIOEffect].} = + ## Reads a single character from the stream `f`. Should not be used in + ## performance sensitive code. + let x = c_fgetc(f) + if x < 0: + checkErr(f) + raiseEOF() + result = char(x) + +proc flushFile*(f: File) {.tags: [WriteIOEffect].} = + ## Flushes `f`'s buffer. + discard c_fflush(f) + +proc getFileHandle*(f: File): FileHandle = + ## Returns the file handle of the file `f`. This is only useful for + ## platform specific programming. + ## Note that on Windows this doesn't return the Windows-specific handle, + ## but the C library's notion of a handle, whatever that means. + ## Use `getOsFileHandle` instead. + c_fileno(f) + +proc getOsFileHandle*(f: File): FileHandle = + ## Returns the OS file handle of the file `f`. This is only useful for + ## platform specific programming. + when defined(windows): + result = FileHandle getOsfhandle(cint getFileHandle(f)) + else: + result = c_fileno(f) + +when defined(nimdoc) or (defined(posix) and not defined(nimscript)) or defined(windows): + proc setInheritable*(f: FileHandle, inheritable: bool): bool = + ## Controls whether a file handle can be inherited by child processes. Returns + ## `true` on success. This requires the OS file handle, which can be + ## retrieved via `getOsFileHandle <#getOsFileHandle,File>`_. + ## + ## This procedure is not guaranteed to be available for all platforms. Test for + ## availability with `declared() <system.html#declared,untyped>`_. + when SupportIoctlInheritCtl: + result = c_ioctl(f, if inheritable: FIONCLEX else: FIOCLEX) != -1 + elif defined(freertos) or defined(zephyr): + result = true + elif defined(posix): + var flags = c_fcntl(f, F_GETFD) + if flags == -1: + return false + flags = if inheritable: flags and not FD_CLOEXEC else: flags or FD_CLOEXEC + result = c_fcntl(f, F_SETFD, flags) != -1 + else: + result = setHandleInformation(cast[IoHandle](f), HANDLE_FLAG_INHERIT, + inheritable.WinDWORD) != 0 + +proc readLine*(f: File, line: var string): bool {.tags: [ReadIOEffect], + benign.} = + ## Reads a line of text from the file `f` into `line`. May throw an IO + ## exception. + ## A line of text may be delimited by `LF` or `CRLF`. The newline + ## character(s) are not part of the returned string. Returns `false` + ## if the end of the file has been reached, `true` otherwise. If + ## `false` is returned `line` contains no new data. + proc c_memchr(s: pointer, c: cint, n: csize_t): pointer {. + importc: "memchr", header: "<string.h>".} + + when defined(windows): + proc readConsole(hConsoleInput: FileHandle, lpBuffer: pointer, + nNumberOfCharsToRead: int32, + lpNumberOfCharsRead: ptr int32, + pInputControl: pointer): int32 {. + importc: "ReadConsoleW", stdcall, dynlib: "kernel32".} + + proc getLastError(): int32 {. + importc: "GetLastError", stdcall, dynlib: "kernel32", sideEffect.} + + proc formatMessageW(dwFlags: int32, lpSource: pointer, + dwMessageId, dwLanguageId: int32, + lpBuffer: pointer, nSize: int32, + arguments: pointer): int32 {. + importc: "FormatMessageW", stdcall, dynlib: "kernel32".} + + proc localFree(p: pointer) {. + importc: "LocalFree", stdcall, dynlib: "kernel32".} + + proc isatty(f: File): bool = + when defined(posix): + proc isatty(fildes: FileHandle): cint {. + importc: "isatty", header: "<unistd.h>".} + else: + proc isatty(fildes: FileHandle): cint {. + importc: "_isatty", header: "<io.h>".} + result = isatty(getFileHandle(f)) != 0'i32 + + # this implies the file is open + if f.isatty: + const numberOfCharsToRead = 2048 + var numberOfCharsRead = 0'i32 + var buffer = newWideCString(numberOfCharsToRead) + if readConsole(getOsFileHandle(f), addr(buffer[0]), + numberOfCharsToRead, addr(numberOfCharsRead), nil) == 0: + var error = getLastError() + var errorMsg: string + var msgbuf: WideCString + if formatMessageW(0x00000100 or 0x00001000 or 0x00000200, + nil, error, 0, addr(msgbuf), 0, nil) != 0'i32: + errorMsg = $msgbuf + if msgbuf != nil: + localFree(cast[pointer](msgbuf)) + raiseEIO("error: " & $error & " `" & errorMsg & "`") + # input always ends with "\r\n" + numberOfCharsRead -= 2 + # handle Ctrl+Z as EOF + for i in 0..<numberOfCharsRead: + if buffer[i].uint16 == 26: #Ctrl+Z + close(f) #has the same effect as setting EOF + if i == 0: + line = "" + return false + numberOfCharsRead = i + break + buffer[numberOfCharsRead] = 0.Utf16Char + when defined(nimv2): + line = $toWideCString(buffer) + else: + line = $buffer + return(true) + + var pos = 0 + + # Use the currently reserved space for a first try + var sp = max(line.len, 80) + line.setLen(sp) + + while true: + # memset to \L so that we can tell how far fgets wrote, even on EOF, where + # fgets doesn't append an \L + for i in 0..<sp: line[pos+i] = '\L' + + var fgetsSuccess: bool + while true: + # fixes #9634; this pattern may need to be abstracted as a template if reused; + # likely other io procs need this for correctness. + fgetsSuccess = c_fgets(cast[cstring](addr line[pos]), sp.cint, f) != nil + if fgetsSuccess: break + when not defined(nimscript): + if errno == EINTR: + errno = 0 + c_clearerr(f) + continue + checkErr(f) + break + + let m = c_memchr(addr line[pos], '\L'.ord, cast[csize_t](sp)) + if m != nil: + # \l found: Could be our own or the one by fgets, in any case, we're done + var last = cast[int](m) - cast[int](addr line[0]) + if last > 0 and line[last-1] == '\c': + line.setLen(last-1) + return last > 1 or fgetsSuccess + elif last > 0 and line[last-1] == '\0': + # We have to distinguish among three possible cases: + # \0\l\0 => line ending in a null character. + # \0\l\l => last line without newline, null was put there by fgets. + # \0\l => last line without newline, null was put there by fgets. + if last >= pos + sp - 1 or line[last+1] != '\0': # bug #21273 + dec last + line.setLen(last) + return last > 0 or fgetsSuccess + else: + # fgets will have inserted a null byte at the end of the string. + dec sp + # No \l found: Increase buffer and read more + inc pos, sp + sp = 128 # read in 128 bytes at a time + line.setLen(pos+sp) + +proc readLine*(f: File): string {.tags: [ReadIOEffect], benign.} = + ## Reads a line of text from the file `f`. May throw an IO exception. + ## A line of text may be delimited by `LF` or `CRLF`. The newline + ## character(s) are not part of the returned string. + result = newStringOfCap(80) + if not readLine(f, result): raiseEOF() + +proc write*(f: File, i: int) {.tags: [WriteIOEffect], benign.} = + when sizeof(int) == 8: + if c_fprintf(f, "%lld", i) < 0: checkErr(f) + else: + if c_fprintf(f, "%ld", i) < 0: checkErr(f) + +proc write*(f: File, i: BiggestInt) {.tags: [WriteIOEffect], benign.} = + when sizeof(BiggestInt) == 8: + if c_fprintf(f, "%lld", i) < 0: checkErr(f) + else: + if c_fprintf(f, "%ld", i) < 0: checkErr(f) + +proc write*(f: File, b: bool) {.tags: [WriteIOEffect], benign.} = + if b: write(f, "true") + else: write(f, "false") + +proc write*(f: File, r: float32) {.tags: [WriteIOEffect], benign.} = + var buffer {.noinit.}: array[65, char] + discard writeFloatToBuffer(buffer, r) + if c_fprintf(f, "%s", buffer[0].addr) < 0: checkErr(f) + +proc write*(f: File, r: BiggestFloat) {.tags: [WriteIOEffect], benign.} = + var buffer {.noinit.}: array[65, char] + discard writeFloatToBuffer(buffer, r) + if c_fprintf(f, "%s", buffer[0].addr) < 0: checkErr(f) + +proc write*(f: File, c: char) {.tags: [WriteIOEffect], benign.} = + discard c_putc(cint(c), f) + +proc write*(f: File, a: varargs[string, `$`]) {.tags: [WriteIOEffect], benign.} = + for x in items(a): write(f, x) + +proc readAllBuffer(file: File): string = + # This proc is for File we want to read but don't know how many + # bytes we need to read before the buffer is empty. + result = "" + var buffer = newString(BufSize) + while true: + var bytesRead = readBuffer(file, addr(buffer[0]), BufSize) + if bytesRead == BufSize: + result.add(buffer) + else: + buffer.setLen(bytesRead) + result.add(buffer) + break + +proc rawFileSize(file: File): int64 = + # this does not raise an error opposed to `getFileSize` + var oldPos = c_ftell(file) + discard c_fseek(file, 0, 2) # seek the end of the file + result = c_ftell(file) + discard c_fseek(file, oldPos, 0) + +proc endOfFile*(f: File): bool {.tags: [], benign.} = + ## Returns true if `f` is at the end. + var c = c_fgetc(f) + discard c_ungetc(c, f) + return c < 0'i32 + #result = c_feof(f) != 0 + +proc readAllFile(file: File, len: int64): string = + # We acquire the filesize beforehand and hope it doesn't change. + # Speeds things up. + result = newString(len) + let bytes = readBuffer(file, addr(result[0]), len) + if endOfFile(file): + if bytes.int64 < len: + result.setLen(bytes) + else: + # We read all the bytes but did not reach the EOF + # Try to read it as a buffer + result.add(readAllBuffer(file)) + +proc readAllFile(file: File): string = + var len = rawFileSize(file) + result = readAllFile(file, len) + +proc readAll*(file: File): string {.tags: [ReadIOEffect], benign.} = + ## Reads all data from the stream `file`. + ## + ## Raises an IO exception in case of an error. It is an error if the + ## current file position is not at the beginning of the file. + + # Separate handling needed because we need to buffer when we + # don't know the overall length of the File. + when declared(stdin): + let len = if file != stdin: rawFileSize(file) else: -1 + else: + let len = rawFileSize(file) + if len > 0: + result = readAllFile(file, len) + else: + result = readAllBuffer(file) + +proc writeLine*[Ty](f: File, x: varargs[Ty, `$`]) {.inline, + tags: [WriteIOEffect], benign.} = + ## Writes the values `x` to `f` and then writes "\\n". + ## May throw an IO exception. + for i in items(x): + write(f, i) + write(f, "\n") + +# interface to the C procs: + +when defined(windows): + when defined(cpp): + proc wfopen(filename, mode: WideCString): pointer {. + importcpp: "_wfopen((const wchar_t*)#, (const wchar_t*)#)", nodecl.} + proc wfreopen(filename, mode: WideCString, stream: File): File {. + importcpp: "_wfreopen((const wchar_t*)#, (const wchar_t*)#, #)", nodecl.} + else: + proc wfopen(filename, mode: WideCString): pointer {. + importc: "_wfopen", nodecl.} + proc wfreopen(filename, mode: WideCString, stream: File): File {. + importc: "_wfreopen", nodecl.} + + proc fopen(filename, mode: cstring): pointer = + var f = newWideCString(filename) + var m = newWideCString(mode) + result = wfopen(f, m) + + proc freopen(filename, mode: cstring, stream: File): File = + var f = newWideCString(filename) + var m = newWideCString(mode) + result = wfreopen(f, m, stream) + +else: + proc fopen(filename, mode: cstring): pointer {.importc: "fopen", nodecl.} + proc freopen(filename, mode: cstring, stream: File): File {. + importc: "freopen", nodecl.} + +const + NoInheritFlag = + # Platform specific flag for creating a File without inheritance. + when not defined(nimInheritHandles): + when defined(windows): + "N" + elif defined(linux) or defined(bsd): + "e" + else: + "" + else: + "" + RawFormatOpen: array[FileMode, cstring] = [ + # used for open by FileHandle, which calls `fdopen` + cstring("rb"), "wb", "w+b", "r+b", "ab"] + FormatOpen: array[FileMode, cstring] = [ + cstring("rb" & NoInheritFlag), "wb" & NoInheritFlag, "w+b" & NoInheritFlag, + "r+b" & NoInheritFlag, "ab" & NoInheritFlag + ] + #"rt", "wt", "w+t", "r+t", "at" + # we always use binary here as for Nim the OS line ending + # should not be translated. + +when defined(posix) and not defined(nimscript): + when defined(linux) and defined(amd64): + type + Mode {.importc: "mode_t", header: "<sys/types.h>".} = cint + + # fillers ensure correct size & offsets + Stat {.importc: "struct stat", + header: "<sys/stat.h>", final, pure.} = object ## struct stat + filler_1: array[24, char] + st_mode: Mode ## Mode of file + filler_2: array[144 - 24 - 4, char] + + proc modeIsDir(m: Mode): bool = + ## Test for a directory. + (m and 0o170000) == 0o40000 + + else: + type + Mode {.importc: "mode_t", header: "<sys/types.h>".} = cint + + Stat {.importc: "struct stat", + header: "<sys/stat.h>", final, pure.} = object ## struct stat + st_mode: Mode ## Mode of file + + proc modeIsDir(m: Mode): bool {.importc: "S_ISDIR", header: "<sys/stat.h>".} + ## Test for a directory. + + proc c_fstat(a1: cint, a2: var Stat): cint {. + importc: "fstat", header: "<sys/stat.h>".} + + +proc open*(f: var File, filename: string, + mode: FileMode = fmRead, + bufSize: int = -1): bool {.tags: [], raises: [], benign.} = + ## Opens a file named `filename` with given `mode`. + ## + ## Default mode is readonly. Returns true if the file could be opened. + ## This throws no exception if the file could not be opened. + ## + ## The file handle associated with the resulting `File` is not inheritable. + var p = fopen(filename.cstring, FormatOpen[mode]) + if p != nil: + var f2 = cast[File](p) + when defined(posix) and not defined(nimscript): + # How `fopen` handles opening a directory is not specified in ISO C and + # POSIX. We do not want to handle directories as regular files that can + # be opened. + var res {.noinit.}: Stat + if c_fstat(getFileHandle(f2), res) >= 0'i32 and modeIsDir(res.st_mode): + close(f2) + return false + when not defined(nimInheritHandles) and declared(setInheritable) and + NoInheritFlag.len == 0: + if not setInheritable(getOsFileHandle(f2), false): + close(f2) + return false + + result = true + f = cast[File](p) + if bufSize > 0 and bufSize.uint <= high(uint): + discard c_setvbuf(f, nil, IOFBF, cast[csize_t](bufSize)) + elif bufSize == 0: + discard c_setvbuf(f, nil, IONBF, 0) + +proc reopen*(f: File, filename: string, mode: FileMode = fmRead): bool {. + tags: [], benign.} = + ## Reopens the file `f` with given `filename` and `mode`. This + ## is often used to redirect the `stdin`, `stdout` or `stderr` + ## file variables. + ## + ## Default mode is readonly. Returns true if the file could be reopened. + ## + ## The file handle associated with `f` won't be inheritable. + if freopen(filename.cstring, FormatOpen[mode], f) != nil: + when not defined(nimInheritHandles) and declared(setInheritable) and + NoInheritFlag.len == 0: + if not setInheritable(getOsFileHandle(f), false): + close(f) + return false + result = true + +proc open*(f: var File, filehandle: FileHandle, + mode: FileMode = fmRead): bool {.tags: [], raises: [], benign.} = + ## Creates a `File` from a `filehandle` with given `mode`. + ## + ## Default mode is readonly. Returns true if the file could be opened. + ## + ## The passed file handle will no longer be inheritable. + when not defined(nimInheritHandles) and declared(setInheritable): + let oshandle = when defined(windows): FileHandle getOsfhandle( + filehandle) else: filehandle + if not setInheritable(oshandle, false): + return false + f = c_fdopen(filehandle, RawFormatOpen[mode]) + result = f != nil + +proc open*(filename: string, + mode: FileMode = fmRead, bufSize: int = -1): File = + ## Opens a file named `filename` with given `mode`. + ## + ## Default mode is readonly. Raises an `IOError` if the file + ## could not be opened. + ## + ## The file handle associated with the resulting `File` is not inheritable. + if not open(result, filename, mode, bufSize): + raise newException(IOError, "cannot open: " & filename) + +proc setFilePos*(f: File, pos: int64, relativeTo: FileSeekPos = fspSet) {.benign, sideEffect.} = + ## Sets the position of the file pointer that is used for read/write + ## operations. The file's first byte has the index zero. + if c_fseek(f, pos, cint(relativeTo)) != 0: + raiseEIO("cannot set file position") + +proc getFilePos*(f: File): int64 {.benign.} = + ## Retrieves the current position of the file pointer that is used to + ## read from the file `f`. The file's first byte has the index zero. + result = c_ftell(f) + if result < 0: raiseEIO("cannot retrieve file position") + +proc getFileSize*(f: File): int64 {.tags: [ReadIOEffect], benign.} = + ## Retrieves the file size (in bytes) of `f`. + let oldPos = getFilePos(f) + discard c_fseek(f, 0, 2) # seek the end of the file + result = getFilePos(f) + setFilePos(f, oldPos) + +proc setStdIoUnbuffered*() {.tags: [], benign.} = + ## Configures `stdin`, `stdout` and `stderr` to be unbuffered. + when declared(stdout): + discard c_setvbuf(stdout, nil, IONBF, 0) + when declared(stderr): + discard c_setvbuf(stderr, nil, IONBF, 0) + when declared(stdin): + discard c_setvbuf(stdin, nil, IONBF, 0) + + +when defined(windows) and not defined(nimscript) and not defined(js): + # work-around C's sucking abstraction: + # BUGFIX: stdin and stdout should be binary files! + proc c_setmode(handle, mode: cint) {. + importc: when defined(bcc): "setmode" else: "_setmode", + header: "<io.h>".} + var + O_BINARY {.importc: "_O_BINARY", header: "<fcntl.h>".}: cint + + # we use binary mode on Windows: + c_setmode(c_fileno(stdin), O_BINARY) + c_setmode(c_fileno(stdout), O_BINARY) + c_setmode(c_fileno(stderr), O_BINARY) + +when defined(windows) and appType == "console" and + not defined(nimDontSetUtf8CodePage) and not defined(nimscript): + import std/exitprocs + + proc setConsoleOutputCP(codepage: cuint): int32 {.stdcall, dynlib: "kernel32", + importc: "SetConsoleOutputCP".} + proc setConsoleCP(wCodePageID: cuint): int32 {.stdcall, dynlib: "kernel32", + importc: "SetConsoleCP".} + proc getConsoleOutputCP(): cuint {.stdcall, dynlib: "kernel32", + importc: "GetConsoleOutputCP".} + proc getConsoleCP(): cuint {.stdcall, dynlib: "kernel32", + importc: "GetConsoleCP".} + + const Utf8codepage = 65001'u32 + + let + consoleOutputCP = getConsoleOutputCP() + consoleCP = getConsoleCP() + + proc restoreConsoleOutputCP() = discard setConsoleOutputCP(consoleOutputCP) + proc restoreConsoleCP() = discard setConsoleCP(consoleCP) + + if consoleOutputCP != Utf8codepage: + discard setConsoleOutputCP(Utf8codepage) + addExitProc(restoreConsoleOutputCP) + + if consoleCP != Utf8codepage: + discard setConsoleCP(Utf8codepage) + addExitProc(restoreConsoleCP) + +proc readFile*(filename: string): string {.tags: [ReadIOEffect], benign.} = + ## Opens a file named `filename` for reading, calls `readAll + ## <#readAll,File>`_ and closes the file afterwards. Returns the string. + ## Raises an IO exception in case of an error. If you need to call + ## this inside a compile time macro you can use `staticRead + ## <system.html#staticRead,string>`_. + var f: File = nil + if open(f, filename): + try: + result = readAll(f) + finally: + close(f) + else: + raise newException(IOError, "cannot open: " & filename) + +proc writeFile*(filename, content: string) {.tags: [WriteIOEffect], benign.} = + ## Opens a file named `filename` for writing. Then writes the + ## `content` completely to the file and closes the file afterwards. + ## Raises an IO exception in case of an error. + var f: File = nil + if open(f, filename, fmWrite): + try: + f.write(content) + finally: + close(f) + else: + raise newException(IOError, "cannot open: " & filename) + +proc writeFile*(filename: string, content: openArray[byte]) {.since: (1, 1).} = + ## Opens a file named `filename` for writing. Then writes the + ## `content` completely to the file and closes the file afterwards. + ## Raises an IO exception in case of an error. + var f: File = nil + if open(f, filename, fmWrite): + try: + discard f.writeBuffer(unsafeAddr content[0], content.len) + finally: + close(f) + else: + raise newException(IOError, "cannot open: " & filename) + +proc readLines*(filename: string, n: Natural): seq[string] = + ## Reads `n` lines from the file named `filename`. Raises an IO exception + ## in case of an error. Raises EOF if file does not contain at least `n` lines. + ## Available at compile time. A line of text may be delimited by `LF` or `CRLF`. + ## The newline character(s) are not part of the returned strings. + var f: File = nil + if open(f, filename): + try: + result = newSeq[string](n) + for i in 0 .. n - 1: + if not readLine(f, result[i]): + raiseEOF() + finally: + close(f) + else: + raise newException(IOError, "cannot open: " & filename) + +template readLines*(filename: string): seq[ + string] {.deprecated: "use readLines with two arguments".} = + readLines(filename, 1) + +iterator lines*(filename: string): string {.tags: [ReadIOEffect].} = + ## Iterates over any line in the file named `filename`. + ## + ## If the file does not exist `IOError` is raised. The trailing newline + ## character(s) are removed from the iterated lines. Example: + ## + runnableExamples: + import std/strutils + + proc transformLetters(filename: string) = + var buffer = "" + for line in filename.lines: + buffer.add(line.replace("a", "0") & '\n') + writeFile(filename, buffer) + var f = open(filename, bufSize = 8000) + try: + var res = newStringOfCap(80) + while f.readLine(res): yield res + finally: + close(f) + +iterator lines*(f: File): string {.tags: [ReadIOEffect].} = + ## Iterates over any line in the file `f`. + ## + ## The trailing newline character(s) are removed from the iterated lines. + ## + runnableExamples: + proc countZeros(filename: File): tuple[lines, zeros: int] = + for line in filename.lines: + for letter in line: + if letter == '0': + result.zeros += 1 + result.lines += 1 + var res = newStringOfCap(80) + while f.readLine(res): yield res + +template `&=`*(f: File, x: typed) = + ## An alias for `write`. + write(f, x) diff --git a/lib/std/sysatomics.nim b/lib/std/sysatomics.nim new file mode 100644 index 000000000..2f203b3eb --- /dev/null +++ b/lib/std/sysatomics.nim @@ -0,0 +1,376 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2015 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +when defined(nimPreviewSlimSystem): + {.deprecated: "use `std/atomics` instead".} + +# Atomic operations for Nim. +{.push stackTrace:off, profiler:off.} + +const + hasThreadSupport = compileOption("threads") and not defined(nimscript) +const someGcc = defined(gcc) or defined(llvm_gcc) or defined(clang) or defined(nintendoswitch) +const someVcc = defined(vcc) or defined(clang_cl) + +type + AtomType* = SomeNumber|pointer|ptr|char|bool + ## Type Class representing valid types for use with atomic procs + +when someGcc: + type AtomMemModel* = distinct cint + + var ATOMIC_RELAXED* {.importc: "__ATOMIC_RELAXED", nodecl.}: AtomMemModel + ## No barriers or synchronization. + var ATOMIC_CONSUME* {.importc: "__ATOMIC_CONSUME", nodecl.}: AtomMemModel + ## Data dependency only for both barrier and + ## synchronization with another thread. + var ATOMIC_ACQUIRE* {.importc: "__ATOMIC_ACQUIRE", nodecl.}: AtomMemModel + ## Barrier to hoisting of code and synchronizes with + ## release (or stronger) + ## semantic stores from another thread. + var ATOMIC_RELEASE* {.importc: "__ATOMIC_RELEASE", nodecl.}: AtomMemModel + ## Barrier to sinking of code and synchronizes with + ## acquire (or stronger) + ## semantic loads from another thread. + var ATOMIC_ACQ_REL* {.importc: "__ATOMIC_ACQ_REL", nodecl.}: AtomMemModel + ## Full barrier in both directions and synchronizes + ## with acquire loads + ## and release stores in another thread. + var ATOMIC_SEQ_CST* {.importc: "__ATOMIC_SEQ_CST", nodecl.}: AtomMemModel + ## Full barrier in both directions and synchronizes + ## with acquire loads + ## and release stores in all threads. + + proc atomicLoadN*[T: AtomType](p: ptr T, mem: AtomMemModel): T {. + importc: "__atomic_load_n", nodecl.} + ## This proc implements an atomic load operation. It returns the contents at p. + ## ATOMIC_RELAXED, ATOMIC_SEQ_CST, ATOMIC_ACQUIRE, ATOMIC_CONSUME. + + proc atomicLoad*[T: AtomType](p, ret: ptr T, mem: AtomMemModel) {. + importc: "__atomic_load", nodecl.} + ## This is the generic version of an atomic load. It returns the contents at p in ret. + + proc atomicStoreN*[T: AtomType](p: ptr T, val: T, mem: AtomMemModel) {. + importc: "__atomic_store_n", nodecl.} + ## This proc implements an atomic store operation. It writes val at p. + ## ATOMIC_RELAXED, ATOMIC_SEQ_CST, and ATOMIC_RELEASE. + + proc atomicStore*[T: AtomType](p, val: ptr T, mem: AtomMemModel) {. + importc: "__atomic_store", nodecl.} + ## This is the generic version of an atomic store. It stores the value of val at p + + proc atomicExchangeN*[T: AtomType](p: ptr T, val: T, mem: AtomMemModel): T {. + importc: "__atomic_exchange_n", nodecl.} + ## This proc implements an atomic exchange operation. It writes val at p, + ## and returns the previous contents at p. + ## ATOMIC_RELAXED, ATOMIC_SEQ_CST, ATOMIC_ACQUIRE, ATOMIC_RELEASE, ATOMIC_ACQ_REL + + proc atomicExchange*[T: AtomType](p, val, ret: ptr T, mem: AtomMemModel) {. + importc: "__atomic_exchange", nodecl.} + ## This is the generic version of an atomic exchange. It stores the contents at val at p. + ## The original value at p is copied into ret. + + proc atomicCompareExchangeN*[T: AtomType](p, expected: ptr T, desired: T, + weak: bool, success_memmodel: AtomMemModel, failure_memmodel: AtomMemModel): bool {. + importc: "__atomic_compare_exchange_n", nodecl.} + ## This proc implements an atomic compare and exchange operation. This compares the + ## contents at p with the contents at expected and if equal, writes desired at p. + ## If they are not equal, the current contents at p is written into expected. + ## Weak is true for weak compare_exchange, and false for the strong variation. + ## Many targets only offer the strong variation and ignore the parameter. + ## When in doubt, use the strong variation. + ## True is returned if desired is written at p and the execution is considered + ## to conform to the memory model specified by success_memmodel. There are no + ## restrictions on what memory model can be used here. False is returned otherwise, + ## and the execution is considered to conform to failure_memmodel. This memory model + ## cannot be __ATOMIC_RELEASE nor __ATOMIC_ACQ_REL. It also cannot be a stronger model + ## than that specified by success_memmodel. + + proc atomicCompareExchange*[T: AtomType](p, expected, desired: ptr T, + weak: bool, success_memmodel: AtomMemModel, failure_memmodel: AtomMemModel): bool {. + importc: "__atomic_compare_exchange", nodecl.} + ## This proc implements the generic version of atomic_compare_exchange. + ## The proc is virtually identical to atomic_compare_exchange_n, except the desired + ## value is also a pointer. + + ## Perform the operation return the new value, all memory models are valid + proc atomicAddFetch*[T: AtomType](p: ptr T, val: T, mem: AtomMemModel): T {. + importc: "__atomic_add_fetch", nodecl.} + proc atomicSubFetch*[T: AtomType](p: ptr T, val: T, mem: AtomMemModel): T {. + importc: "__atomic_sub_fetch", nodecl.} + proc atomicOrFetch*[T: AtomType](p: ptr T, val: T, mem: AtomMemModel): T {. + importc: "__atomic_or_fetch", nodecl.} + proc atomicAndFetch*[T: AtomType](p: ptr T, val: T, mem: AtomMemModel): T {. + importc: "__atomic_and_fetch", nodecl.} + proc atomicXorFetch*[T: AtomType](p: ptr T, val: T, mem: AtomMemModel): T {. + importc: "__atomic_xor_fetch", nodecl.} + proc atomicNandFetch*[T: AtomType](p: ptr T, val: T, mem: AtomMemModel): T {. + importc: "__atomic_nand_fetch", nodecl.} + + ## Perform the operation return the old value, all memory models are valid + proc atomicFetchAdd*[T: AtomType](p: ptr T, val: T, mem: AtomMemModel): T {. + importc: "__atomic_fetch_add", nodecl.} + proc atomicFetchSub*[T: AtomType](p: ptr T, val: T, mem: AtomMemModel): T {. + importc: "__atomic_fetch_sub", nodecl.} + proc atomicFetchOr*[T: AtomType](p: ptr T, val: T, mem: AtomMemModel): T {. + importc: "__atomic_fetch_or", nodecl.} + proc atomicFetchAnd*[T: AtomType](p: ptr T, val: T, mem: AtomMemModel): T {. + importc: "__atomic_fetch_and", nodecl.} + proc atomicFetchXor*[T: AtomType](p: ptr T, val: T, mem: AtomMemModel): T {. + importc: "__atomic_fetch_xor", nodecl.} + proc atomicFetchNand*[T: AtomType](p: ptr T, val: T, mem: AtomMemModel): T {. + importc: "__atomic_fetch_nand", nodecl.} + + proc atomicTestAndSet*(p: pointer, mem: AtomMemModel): bool {. + importc: "__atomic_test_and_set", nodecl.} + ## This built-in function performs an atomic test-and-set operation on the byte at p. + ## The byte is set to some implementation defined nonzero "set" value and the return + ## value is true if and only if the previous contents were "set". + ## All memory models are valid. + + proc atomicClear*(p: pointer, mem: AtomMemModel) {. + importc: "__atomic_clear", nodecl.} + ## This built-in function performs an atomic clear operation at p. + ## After the operation, at p contains 0. + ## ATOMIC_RELAXED, ATOMIC_SEQ_CST, ATOMIC_RELEASE + + proc atomicThreadFence*(mem: AtomMemModel) {. + importc: "__atomic_thread_fence", nodecl.} + ## This built-in function acts as a synchronization fence between threads based + ## on the specified memory model. All memory orders are valid. + + proc atomicSignalFence*(mem: AtomMemModel) {. + importc: "__atomic_signal_fence", nodecl.} + ## This built-in function acts as a synchronization fence between a thread and + ## signal handlers based in the same thread. All memory orders are valid. + + proc atomicAlwaysLockFree*(size: int, p: pointer): bool {. + importc: "__atomic_always_lock_free", nodecl.} + ## This built-in function returns true if objects of size bytes always generate + ## lock free atomic instructions for the target architecture. size must resolve + ## to a compile-time constant and the result also resolves to a compile-time constant. + ## ptr is an optional pointer to the object that may be used to determine alignment. + ## A value of 0 indicates typical alignment should be used. The compiler may also + ## ignore this parameter. + + proc atomicIsLockFree*(size: int, p: pointer): bool {. + importc: "__atomic_is_lock_free", nodecl.} + ## This built-in function returns true if objects of size bytes always generate + ## lock free atomic instructions for the target architecture. If it is not known + ## to be lock free a call is made to a runtime routine named __atomic_is_lock_free. + ## ptr is an optional pointer to the object that may be used to determine alignment. + ## A value of 0 indicates typical alignment should be used. The compiler may also + ## ignore this parameter. + + template fence*() = atomicThreadFence(ATOMIC_SEQ_CST) +elif someVcc: + type AtomMemModel* = distinct cint + + const + ATOMIC_RELAXED* = 0.AtomMemModel + ATOMIC_CONSUME* = 1.AtomMemModel + ATOMIC_ACQUIRE* = 2.AtomMemModel + ATOMIC_RELEASE* = 3.AtomMemModel + ATOMIC_ACQ_REL* = 4.AtomMemModel + ATOMIC_SEQ_CST* = 5.AtomMemModel + + proc `==`(x1, x2: AtomMemModel): bool {.borrow.} + + proc readBarrier() {.importc: "_ReadBarrier", header: "<intrin.h>".} + proc writeBarrier() {.importc: "_WriteBarrier", header: "<intrin.h>".} + proc fence*() {.importc: "_ReadWriteBarrier", header: "<intrin.h>".} + + when defined(cpp): + proc interlockedCompareExchange64(p: pointer; exchange, comparand: int64): int64 + {.importcpp: "_InterlockedCompareExchange64(static_cast<NI64 volatile *>(#), #, #)", header: "<intrin.h>".} + proc interlockedCompareExchange32(p: pointer; exchange, comparand: int32): int32 + {.importcpp: "_InterlockedCompareExchange(static_cast<long volatile *>(#), #, #)", header: "<intrin.h>".} + proc interlockedCompareExchange8(p: pointer; exchange, comparand: byte): byte + {.importcpp: "_InterlockedCompareExchange8(static_cast<char volatile *>(#), #, #)", header: "<intrin.h>".} + proc interlockedExchange8(location: pointer; desired: int8): int8 {.importcpp: "_InterlockedExchange8(static_cast<NI8 volatile *>(#), #)", header: "<intrin.h>".} + proc interlockedExchange16(location: pointer; desired: int16): int16 {.importcpp: "_InterlockedExchange16(static_cast<NI16 volatile *>(#), #)", header: "<intrin.h>".} + proc interlockedExchange32(location: pointer; desired: int32): int32 {.importcpp: "_InterlockedExchange(static_cast<long volatile *>(#), #)", header: "<intrin.h>".} + proc interlockedExchange64(location: pointer; desired: int64): int64 {.importcpp: "_InterlockedExchange64(static_cast<NI64 volatile *>(#), #)", header: "<intrin.h>".} + else: + proc interlockedCompareExchange64(p: pointer; exchange, comparand: int64): int64 + {.importc: "_InterlockedCompareExchange64", header: "<intrin.h>".} + proc interlockedCompareExchange32(p: pointer; exchange, comparand: int32): int32 + {.importc: "_InterlockedCompareExchange", header: "<intrin.h>".} + proc interlockedCompareExchange8(p: pointer; exchange, comparand: byte): byte + {.importc: "_InterlockedCompareExchange8", header: "<intrin.h>".} + + proc interlockedExchange8(location: pointer; desired: int8): int8 {.importc: "_InterlockedExchange8", header: "<intrin.h>".} + proc interlockedExchange16(location: pointer; desired: int16): int16 {.importc: "_InterlockedExchange16", header: "<intrin.h>".} + proc interlockedExchange32(location: pointer; desired: int32): int32 {.importc: "_InterlockedExchange", header: "<intrin.h>".} + proc interlockedExchange64(location: pointer; desired: int64): int64 {.importc: "_InterlockedExchange64", header: "<intrin.h>".} + + + template barrier(mem: AtomMemModel) = + when mem == ATOMIC_RELAXED: discard + elif mem == ATOMIC_CONSUME: readBarrier() + elif mem == ATOMIC_ACQUIRE: writeBarrier() + elif mem == ATOMIC_RELEASE: fence() + elif mem == ATOMIC_ACQ_REL: fence() + elif mem == ATOMIC_SEQ_CST: fence() + + proc atomicStoreN*[T: AtomType](p: ptr T, val: T, mem: static[AtomMemModel]) = + barrier(mem) + p[] = val + + proc atomicLoadN*[T: AtomType](p: ptr T, mem: static[AtomMemModel]): T = + result = p[] + barrier(mem) + + proc atomicCompareExchangeN*[T: ptr](p, expected: ptr T, desired: T, + weak: bool, success_memmodel: AtomMemModel, failure_memmodel: AtomMemModel): bool = + when sizeof(T) == 8: + interlockedCompareExchange64(p, cast[int64](desired), cast[int64](expected)) == + cast[int64](expected) + elif sizeof(T) == 4: + interlockedCompareExchange32(p, cast[int32](desired), cast[int32](expected)) == + cast[int32](expected) + + proc atomicExchangeN*[T: ptr](p: ptr T, val: T, mem: AtomMemModel): T = + when sizeof(T) == 8: + cast[T](interlockedExchange64(p, cast[int64](val))) + elif sizeof(T) == 4: + cast[T](interlockedExchange32(p, cast[int32](val))) + when defined(cpp): + when sizeof(int) == 8: + proc addAndFetch*(p: ptr int, val: int): int {. + importcpp: "_InterlockedExchangeAdd64(static_cast<NI volatile *>(#), #)", + header: "<intrin.h>".} + else: + proc addAndFetch*(p: ptr int, val: int): int {. + importcpp: "_InterlockedExchangeAdd(reinterpret_cast<long volatile *>(#), static_cast<long>(#))", + header: "<intrin.h>".} + else: + when sizeof(int) == 8: + proc addAndFetch*(p: ptr int, val: int): int {. + importc: "_InterlockedExchangeAdd64", header: "<intrin.h>".} + else: + proc addAndFetch*(p: ptr int, val: int): int {. + importc: "_InterlockedExchangeAdd", header: "<intrin.h>".} + +else: + proc addAndFetch*(p: ptr int, val: int): int {.inline.} = + inc(p[], val) + result = p[] + + +proc atomicInc*(memLoc: var int, x: int = 1): int {.inline, discardable, raises: [], tags: [].} = + ## Atomically increments the integer by some `x`. It returns the new value. + when someGcc and hasThreadSupport: + result = atomicAddFetch(memLoc.addr, x, ATOMIC_SEQ_CST) + elif someVcc and hasThreadSupport: + result = addAndFetch(memLoc.addr, x) + inc(result, x) + else: + inc(memLoc, x) + result = memLoc + +proc atomicDec*(memLoc: var int, x: int = 1): int {.inline, discardable, raises: [], tags: [].} = + ## Atomically decrements the integer by some `x`. It returns the new value. + when someGcc and hasThreadSupport: + when declared(atomicSubFetch): + result = atomicSubFetch(memLoc.addr, x, ATOMIC_SEQ_CST) + else: + result = atomicAddFetch(memLoc.addr, -x, ATOMIC_SEQ_CST) + elif someVcc and hasThreadSupport: + result = addAndFetch(memLoc.addr, -x) + dec(result, x) + else: + dec(memLoc, x) + result = memLoc + +when someVcc: + proc cas*[T: bool|int|ptr](p: ptr T; oldValue, newValue: T): bool = + when sizeof(T) == 8: + interlockedCompareExchange64(p, cast[int64](newValue), cast[int64](oldValue)) == + cast[int64](oldValue) + elif sizeof(T) == 4: + interlockedCompareExchange32(p, cast[int32](newValue), cast[int32](oldValue)) == + cast[int32](oldValue) + elif sizeof(T) == 1: + interlockedCompareExchange8(p, cast[byte](newValue), cast[byte](oldValue)) == + cast[byte](oldValue) + else: + {.error: "invalid CAS instruction".} + +elif defined(tcc): + when defined(amd64): + {.emit:""" +static int __tcc_cas(int *ptr, int oldVal, int newVal) +{ + unsigned char ret; + __asm__ __volatile__ ( + " lock\n" + " cmpxchgq %2,%1\n" + " sete %0\n" + : "=q" (ret), "=m" (*ptr) + : "r" (newVal), "m" (*ptr), "a" (oldVal) + : "memory"); + + return ret; +} +""".} + else: + #assert sizeof(int) == 4 + {.emit:""" +static int __tcc_cas(int *ptr, int oldVal, int newVal) +{ + unsigned char ret; + __asm__ __volatile__ ( + " lock\n" + " cmpxchgl %2,%1\n" + " sete %0\n" + : "=q" (ret), "=m" (*ptr) + : "r" (newVal), "m" (*ptr), "a" (oldVal) + : "memory"); + + return ret; +} +""".} + + proc tcc_cas(p: ptr int; oldValue, newValue: int): bool + {.importc: "__tcc_cas", nodecl.} + proc cas*[T: bool|int|ptr](p: ptr T; oldValue, newValue: T): bool = + tcc_cas(cast[ptr int](p), cast[int](oldValue), cast[int](newValue)) +elif declared(atomicCompareExchangeN): + proc cas*[T: bool|int|ptr](p: ptr T; oldValue, newValue: T): bool = + atomicCompareExchangeN(p, oldValue.unsafeAddr, newValue, false, ATOMIC_SEQ_CST, ATOMIC_SEQ_CST) +else: + # this is valid for GCC and Intel C++ + proc cas*[T: bool|int|ptr](p: ptr T; oldValue, newValue: T): bool + {.importc: "__sync_bool_compare_and_swap", nodecl.} + # XXX is this valid for 'int'? + + +when (defined(x86) or defined(amd64)) and someVcc: + proc cpuRelax* {.importc: "YieldProcessor", header: "<windows.h>".} +elif (defined(x86) or defined(amd64)) and (someGcc or defined(bcc)): + proc cpuRelax* {.inline.} = + {.emit: """asm volatile("pause" ::: "memory");""".} +elif someGcc or defined(tcc): + proc cpuRelax* {.inline.} = + {.emit: """asm volatile("" ::: "memory");""".} +elif defined(icl): + proc cpuRelax* {.importc: "_mm_pause", header: "xmmintrin.h".} +elif false: + from std/os import sleep + + proc cpuRelax* {.inline.} = os.sleep(1) + +when not declared(fence) and hasThreadSupport: + # XXX fixme + proc fence*() {.inline.} = + var dummy: bool + discard cas(addr dummy, false, true) + +{.pop.} diff --git a/lib/std/sysrand.nim b/lib/std/sysrand.nim index c9908143c..6f2c6b0c1 100644 --- a/lib/std/sysrand.nim +++ b/lib/std/sysrand.nim @@ -10,23 +10,23 @@ ## .. warning:: This module was added in Nim 1.6. If you are using it for cryptographic purposes, ## keep in mind that so far this has not been audited by any security professionals, ## therefore may not be secure. -## +## ## `std/sysrand` generates random numbers from a secure source provided by the operating system. -## It is also called Cryptographically secure pseudorandom number generator. -## It should be unpredictable enough for cryptographic applications, +## It is a cryptographically secure pseudorandom number generator +## and should be unpredictable enough for cryptographic applications, ## though its exact quality depends on the OS implementation. ## -## | Targets | Implementation| -## | :--- | ----: | -## | Windows | `BCryptGenRandom`_ | -## | Linux | `getrandom`_ | -## | MacOSX | `getentropy`_ | -## | IOS | `SecRandomCopyBytes`_ | -## | OpenBSD | `getentropy openbsd`_ | -## | FreeBSD | `getrandom freebsd`_ | -## | JS(Web Browser) | `getRandomValues`_ | -## | Nodejs | `randomFillSync`_ | -## | Other Unix platforms | `/dev/urandom`_ | +## | Targets | Implementation | +## | :--- | ----: | +## | Windows | `BCryptGenRandom`_ | +## | Linux | `getrandom`_ | +## | MacOSX | `SecRandomCopyBytes`_ | +## | iOS | `SecRandomCopyBytes`_ | +## | OpenBSD | `getentropy openbsd`_ | +## | FreeBSD | `getrandom freebsd`_ | +## | JS (Web Browser) | `getRandomValues`_ | +## | Node.js | `randomFillSync`_ | +## | Other Unix platforms | `/dev/urandom`_ | ## ## .. _BCryptGenRandom: https://docs.microsoft.com/en-us/windows/win32/api/bcrypt/nf-bcrypt-bcryptgenrandom ## .. _getrandom: https://man7.org/linux/man-pages/man2/getrandom.2.html @@ -38,6 +38,11 @@ ## .. _randomFillSync: https://nodejs.org/api/crypto.html#crypto_crypto_randomfillsync_buffer_offset_size ## .. _/dev/urandom: https://en.wikipedia.org/wiki//dev/random ## +## On a Linux target, a call to the `getrandom` syscall can be avoided (e.g. +## for targets running kernel version < 3.17) by passing a compile flag of +## `-d:nimNoGetRandom`. If this flag is passed, sysrand will use `/dev/urandom` +## as with any other POSIX compliant OS. +## runnableExamples: doAssert urandom(0).len == 0 @@ -52,13 +57,16 @@ runnableExamples: when not defined(js): - import std/os + import std/oserrors when defined(posix): import std/posix +when defined(nimPreviewSlimSystem): + import std/assertions + const - batchImplOS = defined(freebsd) or defined(openbsd) or (defined(macosx) and not defined(ios)) + batchImplOS = defined(freebsd) or defined(openbsd) or defined(zephyr) batchSize {.used.} = 256 when batchImplOS: @@ -132,7 +140,7 @@ elif defined(windows): type PVOID = pointer BCRYPT_ALG_HANDLE = PVOID - PUCHAR = ptr cuchar + PUCHAR = ptr uint8 NTSTATUS = clong ULONG = culong @@ -159,15 +167,16 @@ elif defined(windows): result = randomBytes(addr dest[0], size) -elif defined(linux): - # TODO using let, pending bootstrap >= 1.4.0 - var SYS_getrandom {.importc: "SYS_getrandom", header: "<sys/syscall.h>".}: clong +elif defined(linux) and not defined(nimNoGetRandom) and not defined(emscripten): + when (NimMajor, NimMinor) >= (1, 4): + let SYS_getrandom {.importc: "SYS_getrandom", header: "<sys/syscall.h>".}: clong + else: + var SYS_getrandom {.importc: "SYS_getrandom", header: "<sys/syscall.h>".}: clong const syscallHeader = """#include <unistd.h> #include <sys/syscall.h>""" - proc syscall( - n: clong, buf: pointer, bufLen: cint, flags: cuint - ): clong {.importc: "syscall", header: syscallHeader.} + proc syscall(n: clong): clong {. + importc: "syscall", varargs, header: syscallHeader.} # When reading from the urandom source (GRND_RANDOM is not set), # getrandom() will block until the entropy pool has been # initialized (unless the GRND_NONBLOCK flag was specified). If a @@ -183,25 +192,34 @@ elif defined(linux): while result < size: let readBytes = syscall(SYS_getrandom, addr dest[result], cint(size - result), 0).int if readBytes == 0: - doAssert false + raiseAssert "unreachable" elif readBytes > 0: inc(result, readBytes) else: - if osLastError().int in {EINTR, EAGAIN}: - discard + if osLastError().cint in [EINTR, EAGAIN]: discard else: result = -1 break elif defined(openbsd): proc getentropy(p: pointer, size: cint): cint {.importc: "getentropy", header: "<unistd.h>".} - # fills a buffer with high-quality entropy, + # Fills a buffer with high-quality entropy, # which can be used as input for process-context pseudorandom generators like `arc4random`. # The maximum buffer size permitted is 256 bytes. proc getRandomImpl(p: pointer, size: int): int {.inline.} = result = getentropy(p, cint(size)).int +elif defined(zephyr): + proc sys_csrand_get(dst: pointer, length: csize_t): cint {.importc: "sys_csrand_get", header: "<random/rand32.h>".} + # Fill the destination buffer with cryptographically secure + # random data values + # + + proc getRandomImpl(p: pointer, size: int): int {.inline.} = + # 0 if success, -EIO if entropy reseed error + result = sys_csrand_get(p, csize_t(size)).int + elif defined(freebsd): type cssize_t {.importc: "ssize_t", header: "<sys/types.h>".} = int @@ -214,8 +232,8 @@ elif defined(freebsd): proc getRandomImpl(p: pointer, size: int): int {.inline.} = result = getrandom(p, csize_t(size), 0) -elif defined(ios): - {.passL: "-framework Security".} +elif defined(ios) or defined(macosx): + {.passl: "-framework Security".} const errSecSuccess = 0 ## No error. @@ -227,7 +245,7 @@ elif defined(ios): proc secRandomCopyBytes( rnd: SecRandomRef, count: csize_t, bytes: pointer - ): cint {.importc: "SecRandomCopyBytes", header: "<Security/SecRandom.h>".} + ): cint {.importc: "SecRandomCopyBytes", header: "<Security/SecRandom.h>".} ## https://developer.apple.com/documentation/security/1399291-secrandomcopybytes template urandomImpl(result: var int, dest: var openArray[byte]) = @@ -237,19 +255,6 @@ elif defined(ios): result = secRandomCopyBytes(nil, csize_t(size), addr dest[0]) -elif defined(macosx): - const sysrandomHeader = """#include <Availability.h> -#include <sys/random.h> -""" - - proc getentropy(p: pointer, size: csize_t): cint {.importc: "getentropy", header: sysrandomHeader.} - # getentropy() fills a buffer with random data, which can be used as input - # for process-context pseudorandom generators like arc4random(3). - # The maximum buffer size permitted is 256 bytes. - - proc getRandomImpl(p: pointer, size: int): int {.inline.} = - result = getentropy(p, csize_t(size)).int - else: template urandomImpl(result: var int, dest: var openArray[byte]) = let size = dest.len @@ -292,7 +297,7 @@ proc urandom*(dest: var openArray[byte]): bool = ## If the call succeeds, returns `true`. ## ## If `dest` is empty, `urandom` immediately returns success, - ## without calling underlying operating system api. + ## without calling the underlying operating system API. ## ## .. warning:: The code hasn't been audited by cryptography experts and ## is provided as-is without guarantees. Use at your own risks. For production @@ -310,7 +315,7 @@ proc urandom*(dest: var openArray[byte]): bool = proc urandom*(size: Natural): seq[byte] {.inline.} = ## Returns random bytes suitable for cryptographic use. - ## + ## ## .. warning:: The code hasn't been audited by cryptography experts and ## is provided as-is without guarantees. Use at your own risks. For production ## systems we advise you to request an external audit. diff --git a/lib/std/tasks.nim b/lib/std/tasks.nim new file mode 100644 index 000000000..7e59747f5 --- /dev/null +++ b/lib/std/tasks.nim @@ -0,0 +1,312 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2021 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module provides basic primitives for creating parallel programs. +## A `Task` should be only owned by a single Thread, it cannot be shared by threads. + +import std/[macros, isolation, typetraits] + +when defined(nimPreviewSlimSystem): + import std/assertions + +export isolation + + +when compileOption("threads"): + from std/effecttraits import isGcSafe + + +# +# proc hello(a: int, b: string) = +# echo $a & b +# +# let literal = "Nim" +# let t = toTask(hello(521, literal)) +# +# +# is roughly converted to +# +# type +# ScratchObj_369098780 = object +# a: int +# b: string +# +# let scratch_369098762 = cast[ptr ScratchObj_369098780](c_calloc(csize_t 1, +# csize_t sizeof(ScratchObj_369098780))) +# if scratch_369098762.isNil: +# raise newException(OutOfMemDefect, "Could not allocate memory") +# block: +# var isolate_369098776 = isolate(521) +# scratch_369098762.a = extract(isolate_369098776) +# var isolate_369098778 = isolate(literal) +# scratch_369098762.b = extract(isolate_369098778) +# proc hello_369098781(args`gensym3: pointer) {.nimcall.} = +# let objTemp_369098775 = cast[ptr ScratchObj_369098780](args`gensym3) +# let :tmp_369098777 = objTemp_369098775.a +# let :tmp_369098779 = objTemp_369098775.b +# hello(a = :tmp_369098777, b = :tmp_369098779) +# +# proc destroyScratch_369098782(args`gensym3: pointer) {.nimcall.} = +# let obj_369098783 = cast[ptr ScratchObj_369098780](args`gensym3) +# =destroy(obj_369098783[]) +# let t = Task(callback: hello_369098781, args: scratch_369098762, destroy: destroyScratch_369098782) +# + + +type + Task* = object ## `Task` contains the callback and its arguments. + callback: proc (args, res: pointer) {.nimcall, gcsafe.} + args: pointer + destroy: proc (args: pointer) {.nimcall, gcsafe.} + + +proc `=copy`*(x: var Task, y: Task) {.error.} + +const arcLike = defined(gcArc) or defined(gcAtomicArc) or defined(gcOrc) +when defined(nimAllowNonVarDestructor) and arcLike: + proc `=destroy`*(t: Task) {.inline, gcsafe.} = + ## Frees the resources allocated for a `Task`. + if t.args != nil: + if t.destroy != nil: + t.destroy(t.args) + deallocShared(t.args) +else: + proc `=destroy`*(t: var Task) {.inline, gcsafe.} = + ## Frees the resources allocated for a `Task`. + if t.args != nil: + if t.destroy != nil: + t.destroy(t.args) + deallocShared(t.args) + +proc invoke*(task: Task; res: pointer = nil) {.inline, gcsafe.} = + ## Invokes the `task`. + assert task.callback != nil + task.callback(task.args, res) + +template checkIsolate(scratchAssignList: seq[NimNode], procParam, scratchDotExpr: NimNode) = + # block: + # var isoTempA = isolate(521) + # scratch.a = extract(isolateA) + # var isoTempB = isolate(literal) + # scratch.b = extract(isolateB) + let isolatedTemp = genSym(nskTemp, "isoTemp") + scratchAssignList.add newVarStmt(isolatedTemp, newCall(newIdentNode("isolate"), procParam)) + scratchAssignList.add newAssignment(scratchDotExpr, + newCall(newIdentNode("extract"), isolatedTemp)) + +template addAllNode(assignParam: NimNode, procParam: NimNode) = + let scratchDotExpr = newDotExpr(scratchIdent, formalParams[i][0]) + + checkIsolate(scratchAssignList, procParam, scratchDotExpr) + + let tempNode = genSym(kind = nskTemp, ident = formalParams[i][0].strVal) + callNode.add nnkExprEqExpr.newTree(formalParams[i][0], tempNode) + tempAssignList.add newLetStmt(tempNode, newDotExpr(objTemp, formalParams[i][0])) + scratchRecList.add newIdentDefs(newIdentNode(formalParams[i][0].strVal), assignParam) + +proc analyseRootSym(s: NimNode): NimNode = + result = s + while true: + case result.kind + of nnkBracketExpr, nnkDerefExpr, nnkHiddenDeref, + nnkAddr, nnkHiddenAddr, + nnkObjDownConv, nnkObjUpConv: + result = result[0] + of nnkDotExpr, nnkCheckedFieldExpr, nnkHiddenStdConv, nnkHiddenSubConv: + result = result[1] + else: + break + +macro toTask*(e: typed{nkCall | nkInfix | nkPrefix | nkPostfix | nkCommand | nkCallStrLit}): Task = + ## Converts the call and its arguments to `Task`. + runnableExamples: + proc hello(a: int) = echo a + + let b = toTask hello(13) + assert b is Task + + let retType = getTypeInst(e) + let returnsVoid = retType.typeKind == ntyVoid + + let rootSym = analyseRootSym(e[0]) + expectKind rootSym, nnkSym + + when compileOption("threads"): + if not isGcSafe(rootSym): + error("'toTask' takes a GC safe call expression", e) + + if hasClosure(rootSym): + error("closure call is not allowed", e) + + if e.len > 1: + let scratchIdent = genSym(kind = nskTemp, ident = "scratch") + let impl = e[0].getTypeInst + + when defined(nimTasksDebug): + echo impl.treeRepr + echo e.treeRepr + let formalParams = impl[0] + + var + scratchRecList = newNimNode(nnkRecList) + scratchAssignList: seq[NimNode] + tempAssignList: seq[NimNode] + callNode: seq[NimNode] + + let + objTemp = genSym(nskTemp, ident = "objTemp") + + for i in 1 ..< formalParams.len: + var param = formalParams[i][1] + + if param.kind == nnkBracketExpr and param[0].eqIdent("sink"): + param = param[0] + + if param.typeKind in {ntyExpr, ntyStmt}: + error("'toTask'ed function cannot have a 'typed' or 'untyped' parameter", e) + + case param.kind + of nnkVarTy: + error("'toTask'ed function cannot have a 'var' parameter", e) + of nnkBracketExpr: + if param[0].typeKind == ntyTypeDesc: + callNode.add nnkExprEqExpr.newTree(formalParams[i][0], e[i]) + elif param[0].typeKind in {ntyVarargs, ntyOpenArray}: + if param[1].typeKind in {ntyExpr, ntyStmt}: + error("'toTask'ed function cannot have a 'typed' or 'untyped' parameter", e) + let + seqType = nnkBracketExpr.newTree(newIdentNode("seq"), param[1]) + seqCallNode = newCall("@", e[i]) + addAllNode(seqType, seqCallNode) + else: + addAllNode(param, e[i]) + of nnkBracket, nnkObjConstr: + # passing by static parameters + # so we pass them directly instead of passing by scratchObj + callNode.add nnkExprEqExpr.newTree(formalParams[i][0], e[i]) + of nnkSym, nnkPtrTy, nnkProcTy, nnkTupleConstr: + addAllNode(param, e[i]) + of nnkCharLit..nnkNilLit: + callNode.add nnkExprEqExpr.newTree(formalParams[i][0], e[i]) + else: + error("'toTask'ed function cannot have a parameter of " & $param.kind & " kind", e) + + let scratchObjType = genSym(kind = nskType, ident = "ScratchObj") + let scratchObj = nnkTypeSection.newTree( + nnkTypeDef.newTree( + scratchObjType, + newEmptyNode(), + nnkObjectTy.newTree( + newEmptyNode(), + newEmptyNode(), + scratchRecList + ) + ) + ) + + + let scratchObjPtrType = quote do: + cast[ptr `scratchObjType`](allocShared0(sizeof(`scratchObjType`))) + + let scratchLetSection = newLetStmt(scratchIdent, scratchObjPtrType) + + var stmtList = newStmtList() + stmtList.add(scratchObj) + stmtList.add(scratchLetSection) + stmtList.add(nnkBlockStmt.newTree(newEmptyNode(), newStmtList(scratchAssignList))) + + var functionStmtList = newStmtList() + let funcCall = newCall(e[0], callNode) + functionStmtList.add tempAssignList + + let funcName = genSym(nskProc, rootSym.strVal) + let destroyName = genSym(nskProc, "destroyScratch") + let objTemp2 = genSym(ident = "obj") + let tempNode = quote("@") do: + `=destroy`(@objTemp2[]) + + var funcDecl: NimNode + if returnsVoid: + funcDecl = quote do: + proc `funcName`(args, res: pointer) {.gcsafe, nimcall.} = + let `objTemp` = cast[ptr `scratchObjType`](args) + `functionStmtList` + `funcCall` + else: + funcDecl = quote do: + proc `funcName`(args, res: pointer) {.gcsafe, nimcall.} = + let `objTemp` = cast[ptr `scratchObjType`](args) + `functionStmtList` + cast[ptr `retType`](res)[] = `funcCall` + + result = quote do: + `stmtList` + + `funcDecl` + + proc `destroyName`(args: pointer) {.gcsafe, nimcall.} = + let `objTemp2` = cast[ptr `scratchObjType`](args) + `tempNode` + + Task(callback: `funcName`, args: `scratchIdent`, destroy: `destroyName`) + else: + let funcCall = newCall(e[0]) + let funcName = genSym(nskProc, rootSym.strVal) + + if returnsVoid: + result = quote do: + proc `funcName`(args, res: pointer) {.gcsafe, nimcall.} = + `funcCall` + + Task(callback: `funcName`, args: nil) + else: + result = quote do: + proc `funcName`(args, res: pointer) {.gcsafe, nimcall.} = + cast[ptr `retType`](res)[] = `funcCall` + + Task(callback: `funcName`, args: nil) + + + when defined(nimTasksDebug): + echo result.repr + +runnableExamples: + block: + var num = 0 + proc hello(a: int) = inc num, a + + let b = toTask hello(13) + b.invoke() + assert num == 13 + # A task can be invoked multiple times + b.invoke() + assert num == 26 + + block: + type + Runnable = ref object + data: int + + var data: int + proc hello(a: Runnable) {.nimcall.} = + a.data += 2 + data = a.data + + + when false: + # the parameters of call must be isolated. + let x = Runnable(data: 12) + let b = toTask hello(x) # error ----> expression cannot be isolated: x + b.invoke() + + let b = toTask(hello(Runnable(data: 12))) + b.invoke() + assert data == 14 + b.invoke() + assert data == 16 diff --git a/lib/std/tempfiles.nim b/lib/std/tempfiles.nim new file mode 100644 index 000000000..539305bde --- /dev/null +++ b/lib/std/tempfiles.nim @@ -0,0 +1,192 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2021 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module creates temporary files and directories. +## +## Experimental API, subject to change. + +#[ +See also: +* `GetTempFileName` (on windows), refs https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-gettempfilenamea +* `mkstemp` (posix), refs https://man7.org/linux/man-pages/man3/mkstemp.3.html +]# + +import std / [os, random] + +when defined(nimPreviewSlimSystem): + import std/syncio + +const + maxRetry = 10000 + letters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + nimTempPathLength {.intdefine.} = 8 + + +when defined(windows): + import std/winlean + when defined(nimPreviewSlimSystem): + import std/widestrs + + var O_RDWR {.importc: "_O_RDWR", header: "<fcntl.h>".}: cint + + proc c_fdopen( + filehandle: cint, + mode: cstring + ): File {.importc: "_fdopen",header: "<stdio.h>".} + + proc open_osfhandle(osh: Handle, mode: cint): cint {. + importc: "_open_osfhandle", header: "<io.h>".} + + proc close_osfandle(fd: cint): cint {. + importc: "_close", header: "<io.h>".} +else: + import std/posix + + proc c_fdopen( + filehandle: cint, + mode: cstring + ): File {.importc: "fdopen",header: "<stdio.h>".} + + +proc safeOpen(filename: string): File = + ## Open files exclusively; returns `nil` if the file already exists. + # xxx this should be clarified; it doesn't in particular prevent other processes + # from opening the file, at least currently. + when defined(windows): + let dwShareMode = FILE_SHARE_DELETE or FILE_SHARE_READ or FILE_SHARE_WRITE + let dwCreation = CREATE_NEW + let dwFlags = FILE_FLAG_BACKUP_SEMANTICS or FILE_ATTRIBUTE_NORMAL + let handle = createFileW(newWideCString(filename), GENERIC_READ or GENERIC_WRITE, dwShareMode, + nil, dwCreation, dwFlags, Handle(0)) + + if handle == INVALID_HANDLE_VALUE: + if getLastError() == ERROR_FILE_EXISTS: + return nil + else: + raiseOSError(osLastError(), filename) + + let fileHandle = open_osfhandle(handle, O_RDWR) + if fileHandle == -1: + discard closeHandle(handle) + raiseOSError(osLastError(), filename) + + result = c_fdopen(fileHandle, "w+") + if result == nil: + discard close_osfandle(fileHandle) + raiseOSError(osLastError(), filename) + else: + # xxx we need a `proc toMode(a: FilePermission): Mode`, possibly by + # exposing fusion/filepermissions.fromFilePermissions to stdlib; then we need + # to expose a `perm` param so users can customize this (e.g. the temp file may + # need execute permissions), and figure out how to make the API cross platform. + let mode = Mode(S_IRUSR or S_IWUSR) + let flags = posix.O_RDWR or posix.O_CREAT or posix.O_EXCL + let fileHandle = posix.open(filename, flags, mode) + if fileHandle == -1: + if errno == EEXIST: + # xxx `getLastError()` should be defined on posix too and resolve to `errno`? + return nil + else: + raiseOSError(osLastError(), filename) + + result = c_fdopen(fileHandle, "w+") + if result == nil: + discard posix.close(fileHandle) # TODO handles failure when closing file + raiseOSError(osLastError(), filename) + + +type + NimTempPathState = object + state: Rand + isInit: bool + +var nimTempPathState {.threadvar.}: NimTempPathState + +template randomPathName(length: Natural): string = + var res = newString(length) + if not nimTempPathState.isInit: + nimTempPathState.isInit = true + nimTempPathState.state = initRand() + + for i in 0 ..< length: + res[i] = nimTempPathState.state.sample(letters) + res + +proc getTempDirImpl(dir: string): string {.inline.} = + result = dir + if result.len == 0: + result = getTempDir() + +proc genTempPath*(prefix, suffix: string, dir = ""): string = + ## Generates a path name in `dir`. + ## + ## The path begins with `prefix` and ends with `suffix`. + ## + ## .. note:: `dir` must exist (empty `dir` will resolve to `getTempDir <os.html#getTempDir>`_). + let dir = getTempDirImpl(dir) + result = dir / (prefix & randomPathName(nimTempPathLength) & suffix) + +proc createTempFile*(prefix, suffix: string, dir = ""): tuple[cfile: File, path: string] = + ## Creates a new temporary file in the directory `dir`. + ## + ## This generates a path name using `genTempPath(prefix, suffix, dir)` and + ## returns a file handle to an open file and the path of that file, possibly after + ## retrying to ensure it doesn't already exist. + ## + ## If failing to create a temporary file, `OSError` will be raised. + ## + ## .. note:: It is the caller's responsibility to close `result.cfile` and + ## remove `result.file` when no longer needed. + ## .. note:: `dir` must exist (empty `dir` will resolve to `getTempDir <os.html#getTempDir>`_). + runnableExamples: + import std/os + doAssertRaises(OSError): discard createTempFile("", "", "nonexistent") + let (cfile, path) = createTempFile("tmpprefix_", "_end.tmp") + # path looks like: getTempDir() / "tmpprefix_FDCIRZA0_end.tmp" + cfile.write "foo" + cfile.setFilePos 0 + assert readAll(cfile) == "foo" + close cfile + assert readFile(path) == "foo" + removeFile(path) + # xxx why does above work without `cfile.flushFile` ? + let dir = getTempDirImpl(dir) + for i in 0 ..< maxRetry: + result.path = genTempPath(prefix, suffix, dir) + result.cfile = safeOpen(result.path) + if result.cfile != nil: + return + + raise newException(OSError, "Failed to create a temporary file under directory " & dir) + +proc createTempDir*(prefix, suffix: string, dir = ""): string = + ## Creates a new temporary directory in the directory `dir`. + ## + ## This generates a dir name using `genTempPath(prefix, suffix, dir)`, creates + ## the directory and returns it, possibly after retrying to ensure it doesn't + ## already exist. + ## + ## If failing to create a temporary directory, `OSError` will be raised. + ## + ## .. note:: It is the caller's responsibility to remove the directory when no longer needed. + ## .. note:: `dir` must exist (empty `dir` will resolve to `getTempDir <os.html#getTempDir>`_). + runnableExamples: + import std/os + doAssertRaises(OSError): discard createTempDir("", "", "nonexistent") + let dir = createTempDir("tmpprefix_", "_end") + # dir looks like: getTempDir() / "tmpprefix_YEl9VuVj_end" + assert dirExists(dir) + removeDir(dir) + let dir = getTempDirImpl(dir) + for i in 0 ..< maxRetry: + result = genTempPath(prefix, suffix, dir) + if not existsOrCreateDir(result): + return + + raise newException(OSError, "Failed to create a temporary directory under directory " & dir) diff --git a/lib/std/time_t.nim b/lib/std/time_t.nim index 7fb6e6d46..de051b135 100644 --- a/lib/std/time_t.nim +++ b/lib/std/time_t.nim @@ -14,10 +14,10 @@ when defined(nimdoc): ## Wrapper for `time_t`. On posix, this is an alias to `posix.Time`. elif defined(windows): when defined(i386) and defined(gcc): - type Time* {.importc: "time_t", header: "<time.h>".} = distinct int32 + type Time* {.importc: "time_t", header: "<time.h>".} = distinct clong else: # newest version of Visual C++ defines time_t to be of 64 bits type Time* {.importc: "time_t", header: "<time.h>".} = distinct int64 elif defined(posix): - import posix + import std/posix export posix.Time \ No newline at end of file diff --git a/lib/std/typedthreads.nim b/lib/std/typedthreads.nim new file mode 100644 index 000000000..7b0b81968 --- /dev/null +++ b/lib/std/typedthreads.nim @@ -0,0 +1,305 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2012 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +##[ +Thread support for Nim. Threads allow multiple functions to execute concurrently. + +In Nim, threads are a low-level construct and using a library like `malebolgia`, `taskpools` or `weave` is recommended. + +When creating a thread, you can pass arguments to it. As Nim's garbage collector does not use atomic references, sharing +`ref` and other variables managed by the garbage collector between threads is not supported. +Use global variables to do so, or pointers. + +Memory allocated using [`sharedAlloc`](./system.html#allocShared.t%2CNatural) can be used and shared between threads. + +To communicate between threads, consider using [channels](./system.html#Channel) + +Examples +======== + +```Nim +import std/locks + +var + thr: array[0..4, Thread[tuple[a,b: int]]] + L: Lock + +proc threadFunc(interval: tuple[a,b: int]) {.thread.} = + for i in interval.a..interval.b: + acquire(L) # lock stdout + echo i + release(L) + +initLock(L) + +for i in 0..high(thr): + createThread(thr[i], threadFunc, (i*10, i*10+5)) +joinThreads(thr) + +deinitLock(L) +``` + +When using a memory management strategy that supports shared heaps like `arc` or `boehm`, +you can pass pointer to threads and share memory between them, but the memory must outlive the thread. +The default memory management strategy, `orc`, supports this. +The example below is **not valid** for memory management strategies that use local heaps like `refc`! + +```Nim +import locks + +var l: Lock + +proc threadFunc(obj: ptr seq[int]) {.thread.} = + withLock l: + for i in 0..<100: + obj[].add(obj[].len * obj[].len) + +proc threadHandler() = + var thr: array[0..4, Thread[ptr seq[int]]] + var s = newSeq[int]() + + for i in 0..high(thr): + createThread(thr[i], threadFunc, s.addr) + joinThreads(thr) + echo s + +initLock(l) +threadHandler() +deinitLock(l) +``` +]## + + +import std/private/[threadtypes] +export Thread + +import system/ansi_c + +when defined(nimPreviewSlimSystem): + import std/assertions + +when defined(genode): + import genode/env + +when hostOS == "any": + {.error: "Threads not implemented for os:any. Please compile with --threads:off.".} + +when hasAllocStack or defined(zephyr) or defined(freertos) or defined(nuttx) or + defined(cpu16) or defined(cpu8): + const + nimThreadStackSize {.intdefine.} = 8192 + nimThreadStackGuard {.intdefine.} = 128 + + StackGuardSize = nimThreadStackGuard + ThreadStackSize = nimThreadStackSize - nimThreadStackGuard +else: + const + StackGuardSize = 4096 + ThreadStackMask = + when defined(genode): + 1024*64*sizeof(int)-1 + else: + 1024*256*sizeof(int)-1 + + ThreadStackSize = ThreadStackMask+1 - StackGuardSize + + +when defined(gcDestructors): + proc allocThreadStorage(size: int): pointer = + result = c_malloc(csize_t size) + zeroMem(result, size) +else: + template allocThreadStorage(size: untyped): untyped = allocShared0(size) + +#const globalsSlot = ThreadVarSlot(0) +#sysAssert checkSlot.int == globalsSlot.int + +# Zephyr doesn't include this properly without some help +when defined(zephyr): + {.emit: """/*INCLUDESECTION*/ + #include <pthread.h> + """.} + + +# We jump through some hops here to ensure that Nim thread procs can have +# the Nim calling convention. This is needed because thread procs are +# ``stdcall`` on Windows and ``noconv`` on UNIX. Alternative would be to just +# use ``stdcall`` since it is mapped to ``noconv`` on UNIX anyway. + + + +{.push stack_trace:off.} +when defined(windows): + proc threadProcWrapper[TArg](closure: pointer): int32 {.stdcall.} = + nimThreadProcWrapperBody(closure) + # implicitly return 0 +elif defined(genode): + proc threadProcWrapper[TArg](closure: pointer) {.noconv.} = + nimThreadProcWrapperBody(closure) +else: + proc threadProcWrapper[TArg](closure: pointer): pointer {.noconv.} = + nimThreadProcWrapperBody(closure) +{.pop.} + +proc running*[TArg](t: Thread[TArg]): bool {.inline.} = + ## Returns true if `t` is running. + result = t.dataFn != nil + +proc handle*[TArg](t: Thread[TArg]): SysThread {.inline.} = + ## Returns the thread handle of `t`. + result = t.sys + +when hostOS == "windows": + const MAXIMUM_WAIT_OBJECTS = 64 + + proc joinThread*[TArg](t: Thread[TArg]) {.inline.} = + ## Waits for the thread `t` to finish. + discard waitForSingleObject(t.sys, -1'i32) + + proc joinThreads*[TArg](t: varargs[Thread[TArg]]) = + ## Waits for every thread in `t` to finish. + var a: array[MAXIMUM_WAIT_OBJECTS, SysThread] + var k = 0 + while k < len(t): + var count = min(len(t) - k, MAXIMUM_WAIT_OBJECTS) + for i in 0..(count - 1): a[i] = t[i + k].sys + discard waitForMultipleObjects(int32(count), + cast[ptr SysThread](addr(a)), 1, -1) + inc(k, MAXIMUM_WAIT_OBJECTS) + +elif defined(genode): + proc joinThread*[TArg](t: Thread[TArg]) {.importcpp.} + ## Waits for the thread `t` to finish. + + proc joinThreads*[TArg](t: varargs[Thread[TArg]]) = + ## Waits for every thread in `t` to finish. + for i in 0..t.high: joinThread(t[i]) + +else: + proc joinThread*[TArg](t: Thread[TArg]) {.inline.} = + ## Waits for the thread `t` to finish. + discard pthread_join(t.sys, nil) + + proc joinThreads*[TArg](t: varargs[Thread[TArg]]) = + ## Waits for every thread in `t` to finish. + for i in 0..t.high: joinThread(t[i]) + +when false: + # XXX a thread should really release its heap here somehow: + proc destroyThread*[TArg](t: var Thread[TArg]) = + ## Forces the thread `t` to terminate. This is potentially dangerous if + ## you don't have full control over `t` and its acquired resources. + when hostOS == "windows": + discard TerminateThread(t.sys, 1'i32) + else: + discard pthread_cancel(t.sys) + when declared(registerThread): unregisterThread(addr(t)) + t.dataFn = nil + ## if thread `t` already exited, `t.core` will be `null`. + if not isNil(t.core): + deallocThreadStorage(t.core) + t.core = nil + +when hostOS == "windows": + proc createThread*[TArg](t: var Thread[TArg], + tp: proc (arg: TArg) {.thread, nimcall.}, + param: TArg) = + ## Creates a new thread `t` and starts its execution. + ## + ## Entry point is the proc `tp`. + ## `param` is passed to `tp`. `TArg` can be `void` if you + ## don't need to pass any data to the thread. + t.core = cast[PGcThread](allocThreadStorage(sizeof(GcThread))) + + when TArg isnot void: t.data = param + t.dataFn = tp + when hasSharedHeap: t.core.stackSize = ThreadStackSize + var dummyThreadId: int32 + t.sys = createThread(nil, ThreadStackSize, threadProcWrapper[TArg], + addr(t), 0'i32, dummyThreadId) + if t.sys <= 0: + raise newException(ResourceExhaustedError, "cannot create thread") + + proc pinToCpu*[Arg](t: var Thread[Arg]; cpu: Natural) = + ## Pins a thread to a `CPU`:idx:. + ## + ## In other words sets a thread's `affinity`:idx:. + ## If you don't know what this means, you shouldn't use this proc. + setThreadAffinityMask(t.sys, uint(1 shl cpu)) + +elif defined(genode): + var affinityOffset: cuint = 1 + ## CPU affinity offset for next thread, safe to roll-over. + + proc createThread*[TArg](t: var Thread[TArg], + tp: proc (arg: TArg) {.thread, nimcall.}, + param: TArg) = + t.core = cast[PGcThread](allocThreadStorage(sizeof(GcThread))) + + when TArg isnot void: t.data = param + t.dataFn = tp + when hasSharedHeap: t.stackSize = ThreadStackSize + t.sys.initThread( + runtimeEnv, + ThreadStackSize.culonglong, + threadProcWrapper[TArg], addr(t), affinityOffset) + inc affinityOffset + + proc pinToCpu*[Arg](t: var Thread[Arg]; cpu: Natural) = + {.hint: "cannot change Genode thread CPU affinity after initialization".} + discard + +else: + proc createThread*[TArg](t: var Thread[TArg], + tp: proc (arg: TArg) {.thread, nimcall.}, + param: TArg) = + ## Creates a new thread `t` and starts its execution. + ## + ## Entry point is the proc `tp`. `param` is passed to `tp`. + ## `TArg` can be `void` if you + ## don't need to pass any data to the thread. + t.core = cast[PGcThread](allocThreadStorage(sizeof(GcThread))) + + when TArg isnot void: t.data = param + t.dataFn = tp + when hasSharedHeap: t.core.stackSize = ThreadStackSize + var a {.noinit.}: Pthread_attr + doAssert pthread_attr_init(a) == 0 + when hasAllocStack: + var + rawstk = allocThreadStorage(ThreadStackSize + StackGuardSize) + stk = cast[pointer](cast[uint](rawstk) + StackGuardSize) + let setstacksizeResult = pthread_attr_setstack(addr a, stk, ThreadStackSize) + t.rawStack = rawstk + else: + let setstacksizeResult = pthread_attr_setstacksize(a, ThreadStackSize) + + when not defined(ios): + # This fails on iOS + doAssert(setstacksizeResult == 0) + if pthread_create(t.sys, a, threadProcWrapper[TArg], addr(t)) != 0: + raise newException(ResourceExhaustedError, "cannot create thread") + doAssert pthread_attr_destroy(a) == 0 + + proc pinToCpu*[Arg](t: var Thread[Arg]; cpu: Natural) = + ## Pins a thread to a `CPU`:idx:. + ## + ## In other words sets a thread's `affinity`:idx:. + ## If you don't know what this means, you shouldn't use this proc. + when not defined(macosx): + var s {.noinit.}: CpuSet + cpusetZero(s) + cpusetIncl(cpu.cint, s) + setAffinity(t.sys, csize_t(sizeof(s)), s) + +proc createThread*(t: var Thread[void], tp: proc () {.thread, nimcall.}) = + createThread[void](t, tp) + +when not defined(gcOrc): + include system/threadids diff --git a/lib/std/varints.nim b/lib/std/varints.nim index 0d18b9069..32fe2fffb 100644 --- a/lib/std/varints.nim +++ b/lib/std/varints.nim @@ -82,29 +82,29 @@ proc writeVu64*(z: var openArray[byte], x: uint64): int = z[3] = cast[uint8](y) return 4 z[0] = 251 - varintWrite32(toOpenArray(z, 1, z.high-1), y) + varintWrite32(toOpenArray(z, 1, 4), y) return 5 if w <= 255: z[0] = 252 z[1] = cast[uint8](w) - varintWrite32(toOpenArray(z, 2, z.high-2), y) + varintWrite32(toOpenArray(z, 2, 5), y) return 6 if w <= 65535: z[0] = 253 z[1] = cast[uint8](w shr 8) z[2] = cast[uint8](w) - varintWrite32(toOpenArray(z, 3, z.high-3), y) + varintWrite32(toOpenArray(z, 3, 6), y) return 7 if w <= 16777215: z[0] = 254 z[1] = cast[uint8](w shr 16) z[2] = cast[uint8](w shr 8) z[3] = cast[uint8](w) - varintWrite32(toOpenArray(z, 4, z.high-4), y) + varintWrite32(toOpenArray(z, 4, 7), y) return 8 z[0] = 255 - varintWrite32(toOpenArray(z, 1, z.high-1), w) - varintWrite32(toOpenArray(z, 5, z.high-5), y) + varintWrite32(toOpenArray(z, 1, 4), w) + varintWrite32(toOpenArray(z, 5, 8), y) return 9 proc sar(a, b: int64): int64 = diff --git a/lib/std/vmutils.nim b/lib/std/vmutils.nim new file mode 100644 index 000000000..e16912a3c --- /dev/null +++ b/lib/std/vmutils.nim @@ -0,0 +1,11 @@ +##[ +Experimental API, subject to change. +]## + +proc vmTrace*(on: bool) {.compileTime.} = + runnableExamples: + static: vmTrace(true) + proc fn = + var a = 1 + vmTrace(false) + static: fn() diff --git a/lib/std/widestrs.nim b/lib/std/widestrs.nim new file mode 100644 index 000000000..2ddf80d14 --- /dev/null +++ b/lib/std/widestrs.nim @@ -0,0 +1,239 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2012 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Nim support for C/C++'s `wide strings`:idx:. + +#when not declared(ThisIsSystem): +# {.error: "You must not import this module explicitly".} + +type + Utf16Char* = distinct int16 + +when not (defined(cpu16) or defined(cpu8)): + when defined(nimv2): + + type + WideCString* = ptr UncheckedArray[Utf16Char] + + WideCStringObj* = object + bytes: int + data: WideCString + + const arcLike = defined(gcArc) or defined(gcAtomicArc) or defined(gcOrc) + when defined(nimAllowNonVarDestructor) and arcLike: + proc `=destroy`(a: WideCStringObj) = + if a.data != nil: + when compileOption("threads"): + deallocShared(a.data) + else: + dealloc(a.data) + else: + proc `=destroy`(a: var WideCStringObj) = + if a.data != nil: + when compileOption("threads"): + deallocShared(a.data) + else: + dealloc(a.data) + + proc `=copy`(a: var WideCStringObj; b: WideCStringObj) {.error.} + + proc `=sink`(a: var WideCStringObj; b: WideCStringObj) = + a.bytes = b.bytes + a.data = b.data + + proc createWide(a: var WideCStringObj; bytes: int) = + a.bytes = bytes + when compileOption("threads"): + a.data = cast[typeof(a.data)](allocShared0(bytes)) + else: + a.data = cast[typeof(a.data)](alloc0(bytes)) + + template `[]`*(a: WideCStringObj; idx: int): Utf16Char = a.data[idx] + template `[]=`*(a: WideCStringObj; idx: int; val: Utf16Char) = a.data[idx] = val + + template nullWide(): untyped = WideCStringObj(bytes: 0, data: nil) + + converter toWideCString*(x: WideCStringObj): WideCString {.inline.} = + result = x.data + + else: + template nullWide(): untyped = nil + + type + WideCString* = ref UncheckedArray[Utf16Char] + WideCStringObj* = WideCString + + template createWide(a; L) = + unsafeNew(a, L) + + proc ord(arg: Utf16Char): int = int(cast[uint16](arg)) + + proc len*(w: WideCString): int = + ## returns the length of a widestring. This traverses the whole string to + ## find the binary zero end marker! + result = 0 + while int16(w[result]) != 0'i16: inc result + + const + UNI_REPLACEMENT_CHAR = Utf16Char(0xFFFD'i16) + UNI_MAX_BMP = 0x0000FFFF + UNI_MAX_UTF16 = 0x0010FFFF + # UNI_MAX_UTF32 = 0x7FFFFFFF + # UNI_MAX_LEGAL_UTF32 = 0x0010FFFF + + halfShift = 10 + halfBase = 0x0010000 + halfMask = 0x3FF + + UNI_SUR_HIGH_START = 0xD800 + UNI_SUR_HIGH_END = 0xDBFF + UNI_SUR_LOW_START = 0xDC00 + UNI_SUR_LOW_END = 0xDFFF + UNI_REPL = 0xFFFD + + template ones(n: untyped): untyped = ((1 shl n)-1) + + template fastRuneAt(s: cstring, i, L: int, result: untyped, doInc = true) = + ## Returns the unicode character `s[i]` in `result`. If `doInc == true` + ## `i` is incremented by the number of bytes that have been processed. + bind ones + + if ord(s[i]) <= 127: + result = ord(s[i]) + when doInc: inc(i) + elif ord(s[i]) shr 5 == 0b110: + #assert(ord(s[i+1]) shr 6 == 0b10) + if i <= L - 2: + result = (ord(s[i]) and (ones(5))) shl 6 or (ord(s[i+1]) and ones(6)) + when doInc: inc(i, 2) + else: + result = UNI_REPL + when doInc: inc(i) + elif ord(s[i]) shr 4 == 0b1110: + if i <= L - 3: + #assert(ord(s[i+1]) shr 6 == 0b10) + #assert(ord(s[i+2]) shr 6 == 0b10) + result = (ord(s[i]) and ones(4)) shl 12 or + (ord(s[i+1]) and ones(6)) shl 6 or + (ord(s[i+2]) and ones(6)) + when doInc: inc(i, 3) + else: + result = UNI_REPL + when doInc: inc(i) + elif ord(s[i]) shr 3 == 0b11110: + if i <= L - 4: + #assert(ord(s[i+1]) shr 6 == 0b10) + #assert(ord(s[i+2]) shr 6 == 0b10) + #assert(ord(s[i+3]) shr 6 == 0b10) + result = (ord(s[i]) and ones(3)) shl 18 or + (ord(s[i+1]) and ones(6)) shl 12 or + (ord(s[i+2]) and ones(6)) shl 6 or + (ord(s[i+3]) and ones(6)) + when doInc: inc(i, 4) + else: + result = UNI_REPL + when doInc: inc(i) + else: + result = 0xFFFD + when doInc: inc(i) + + iterator runes(s: cstring, L: int): int = + var + i = 0 + result: int + while i < L: + fastRuneAt(s, i, L, result, true) + yield result + + proc newWideCString*(size: int): WideCStringObj = + createWide(result, size * 2 + 2) + + proc newWideCString*(source: cstring, L: int): WideCStringObj = + ## Warning:: `source` needs to be preallocated with the length `L` + createWide(result, L * 2 + 2) + var d = 0 + for ch in runes(source, L): + + if ch <= UNI_MAX_BMP: + if ch >= UNI_SUR_HIGH_START and ch <= UNI_SUR_LOW_END: + result[d] = UNI_REPLACEMENT_CHAR + else: + result[d] = cast[Utf16Char](uint16(ch)) + elif ch > UNI_MAX_UTF16: + result[d] = UNI_REPLACEMENT_CHAR + else: + let ch = ch - halfBase + result[d] = cast[Utf16Char](uint16((ch shr halfShift) + UNI_SUR_HIGH_START)) + inc d + result[d] = cast[Utf16Char](uint16((ch and halfMask) + UNI_SUR_LOW_START)) + inc d + result[d] = Utf16Char(0) + + proc newWideCString*(s: cstring): WideCStringObj = + if s.isNil: return nullWide + + result = newWideCString(s, s.len) + + proc newWideCString*(s: string): WideCStringObj = + result = newWideCString(cstring s, s.len) + + proc `$`*(w: WideCString, estimate: int, replacement: int = 0xFFFD): string = + result = newStringOfCap(estimate + estimate shr 2) + + var i = 0 + while w[i].int16 != 0'i16: + var ch = ord(w[i]) + inc i + if ch >= UNI_SUR_HIGH_START and ch <= UNI_SUR_HIGH_END: + # If the 16 bits following the high surrogate are in the source buffer... + let ch2 = ord(w[i]) + + # If it's a low surrogate, convert to UTF32: + if ch2 >= UNI_SUR_LOW_START and ch2 <= UNI_SUR_LOW_END: + ch = (((ch and halfMask) shl halfShift) + (ch2 and halfMask)) + halfBase + inc i + else: + #invalid UTF-16 + ch = replacement + elif ch >= UNI_SUR_LOW_START and ch <= UNI_SUR_LOW_END: + #invalid UTF-16 + ch = replacement + + if ch < 0x80: + result.add chr(ch) + elif ch < 0x800: + result.add chr((ch shr 6) or 0xc0) + result.add chr((ch and 0x3f) or 0x80) + elif ch < 0x10000: + result.add chr((ch shr 12) or 0xe0) + result.add chr(((ch shr 6) and 0x3f) or 0x80) + result.add chr((ch and 0x3f) or 0x80) + elif ch <= 0x10FFFF: + result.add chr((ch shr 18) or 0xf0) + result.add chr(((ch shr 12) and 0x3f) or 0x80) + result.add chr(((ch shr 6) and 0x3f) or 0x80) + result.add chr((ch and 0x3f) or 0x80) + else: + # replacement char(in case user give very large number): + result.add chr(0xFFFD shr 12 or 0b1110_0000) + result.add chr(0xFFFD shr 6 and ones(6) or 0b10_0000_00) + result.add chr(0xFFFD and ones(6) or 0b10_0000_00) + + proc `$`*(s: WideCString): string = + result = s $ 80 + + when defined(nimv2): + proc `$`*(s: WideCStringObj, estimate: int, replacement: int = 0xFFFD): string = + `$`(s.data, estimate, replacement) + + proc `$`*(s: WideCStringObj): string = + $(s.data) + + proc len*(w: WideCStringObj): int {.inline.} = + len(w.data) diff --git a/lib/std/with.nim b/lib/std/with.nim index 79afd61a4..c2eaa4bef 100644 --- a/lib/std/with.nim +++ b/lib/std/with.nim @@ -12,15 +12,16 @@ ## and https://github.com/nim-lang/RFCs/issues/192 for details leading to this ## particular design. ## -## **Since** version 1.2. +## **Since:** version 1.2. -import macros, private / underscored_calls +import std/[macros, private / underscored_calls] macro with*(arg: typed; calls: varargs[untyped]): untyped = - ## This macro provides the `chaining`:idx: of function calls. + ## This macro provides `chaining`:idx: of function calls. ## It does so by patching every call in `calls` to ## use `arg` as the first argument. - ## **This evaluates `arg` multiple times!** + ## + ## .. caution:: This evaluates `arg` multiple times! runnableExamples: var x = "yay" with x: @@ -34,5 +35,14 @@ macro with*(arg: typed; calls: varargs[untyped]): untyped = -= 5 doAssert a == 43 + # Nesting works for object types too! + var foo = (bar: 1, qux: (baz: 2)) + with foo: + bar = 2 + with qux: + baz = 3 + doAssert foo.bar == 2 + doAssert foo.qux.baz == 3 + result = newNimNode(nnkStmtList, arg) underscoredCalls(result, calls, arg) diff --git a/lib/std/wordwrap.nim b/lib/std/wordwrap.nim index 7dcfc7f59..9333f880b 100644 --- a/lib/std/wordwrap.nim +++ b/lib/std/wordwrap.nim @@ -9,7 +9,7 @@ ## This module contains an algorithm to wordwrap a Unicode string. -import strutils, unicode +import std/[strutils, unicode] proc olen(s: string; start, lastExclusive: int): int = var i = start diff --git a/lib/std/wrapnils.nim b/lib/std/wrapnils.nim index 3ff48fbfe..0b75c270e 100644 --- a/lib/std/wrapnils.nim +++ b/lib/std/wrapnils.nim @@ -1,9 +1,20 @@ -## This module allows chains of field-access and indexing where the LHS can be nil. -## This simplifies code by reducing need for if-else branches around intermediate values -## that may be nil. +## This module allows evaluating expressions safely against the following conditions: +## * nil dereferences +## * field accesses with incorrect discriminant in case objects +## +## `default(T)` is returned in those cases when evaluating an expression of type `T`. +## This simplifies code by reducing need for if-else branches. ## ## Note: experimental module, unstable API. +#[ +TODO: +consider handling indexing operations, eg: +doAssert ?.default(seq[int])[3] == default(int) +]# + +import std/macros + runnableExamples: type Foo = ref object x1: string @@ -24,8 +35,125 @@ runnableExamples: assert (?.f2.x2.x2).x3 == nil # this terminates ?. early +runnableExamples: + # ?. also allows case object + type B = object + b0: int + case cond: bool + of false: discard + of true: + b1: float + + var b = B(cond: false, b0: 3) + doAssertRaises(FieldDefect): discard b.b1 # wrong discriminant + doAssert ?.b.b1 == 0.0 # safe + b = B(cond: true, b1: 4.5) + doAssert ?.b.b1 == 4.5 + + # lvalue semantics are preserved: + if (let p = ?.b.b1.addr; p != nil): p[] = 4.7 + doAssert b.b1 == 4.7 + +proc finalize(n: NimNode, lhs: NimNode, level: int): NimNode = + if level == 0: + result = quote: `lhs` = `n` + else: + result = quote: (let `lhs` = `n`) + +proc process(n: NimNode, lhs: NimNode, label: NimNode, level: int): NimNode = + var n = n.copyNimTree + var it = n + let addr2 = bindSym"addr" + var old: tuple[n: NimNode, index: int] + while true: + if it.len == 0: + result = finalize(n, lhs, level) + break + elif it.kind == nnkCheckedFieldExpr: + let dot = it[0] + let obj = dot[0] + let objRef = quote do: `addr2`(`obj`) + # avoids a copy and preserves lvalue semantics, see tests + let check = it[1] + let okSet = check[1] + let kind1 = check[2] + let tmp = genSym(nskLet, "tmpCase") + let body = process(objRef, tmp, label, level + 1) + let tmp3 = nnkDerefExpr.newTree(tmp) + it[0][0] = tmp3 + let dot2 = nnkDotExpr.newTree(@[tmp, dot[1]]) + if old.n != nil: old.n[old.index] = dot2 + else: n = dot2 + let assgn = finalize(n, lhs, level) + result = quote do: + `body` + if `tmp3`.`kind1` notin `okSet`: break `label` + `assgn` + break + elif it.kind in {nnkHiddenDeref, nnkDerefExpr}: + let tmp = genSym(nskLet, "tmp") + let body = process(it[0], tmp, label, level + 1) + it[0] = tmp + let assgn = finalize(n, lhs, level) + result = quote do: + `body` + if `tmp` == nil: break `label` + `assgn` + break + elif it.kind == nnkCall: # consider extending to `nnkCallKinds` + # `copyNimTree` needed to avoid `typ = nil` issues + old = (it, 1) + it = it[1].copyNimTree + else: + old = (it, 0) + it = it[0] + +macro `?.`*(a: typed): auto = + ## Transforms `a` into an expression that can be safely evaluated even in + ## presence of intermediate nil pointers/references, in which case a default + ## value is produced. + let lhs = genSym(nskVar, "lhs") + let label = genSym(nskLabel, "label") + let body = process(a, lhs, label, 0) + result = quote do: + var `lhs`: type(`a`) + block `label`: + `body` + `lhs` + +# the code below is not needed for `?.` from std/options import Option, isSome, get, option, unsafeGet, UnpackDefect -export options.get, options.isSome, options.isNone + +macro `??.`*(a: typed): Option = + ## Same as `?.` but returns an `Option`. + runnableExamples: + import std/options + type Foo = ref object + x1: ref int + x2: int + # `?.` can't distinguish between a valid vs invalid default value, but `??.` can: + var f1 = Foo(x1: int.new, x2: 2) + doAssert (??.f1.x1[]).get == 0 # not enough to tell when the chain was valid. + doAssert (??.f1.x1[]).isSome # a nil didn't occur in the chain + doAssert (??.f1.x2).get == 2 + + var f2: Foo + doAssert not (??.f2.x1[]).isSome # f2 was nil + + doAssertRaises(UnpackDefect): discard (??.f2.x1[]).get + doAssert ?.f2.x1[] == 0 # in contrast, this returns default(int) + + let lhs = genSym(nskVar, "lhs") + let lhs2 = genSym(nskVar, "lhs") + let label = genSym(nskLabel, "label") + let body = process(a, lhs2, label, 0) + result = quote do: + var `lhs`: Option[type(`a`)] + block `label`: + var `lhs2`: type(`a`) + `body` + `lhs` = option(`lhs2`) + `lhs` template fakeDot*(a: Option, b): untyped = ## See top-level example. @@ -59,50 +187,7 @@ func `[]`*[U](a: Option[U]): auto {.inline.} = if a2 != nil: result = option(a2[]) -import std/macros - -func replace(n: NimNode): NimNode = - if n.kind == nnkDotExpr: - result = newCall(bindSym"fakeDot", replace(n[0]), n[1]) - elif n.kind == nnkPar: - doAssert n.len == 1 - result = newCall(bindSym"option", n[0]) - elif n.kind in {nnkCall, nnkObjConstr}: - result = newCall(bindSym"option", n) - elif n.len == 0: - result = newCall(bindSym"option", n) - else: - n[0] = replace(n[0]) - result = n - -proc safeGet[T](a: Option[T]): T {.inline.} = - get(a, default(T)) - -macro `?.`*(a: untyped): auto = - ## Transforms `a` into an expression that can be safely evaluated even in - ## presence of intermediate nil pointers/references, in which case a default - ## value is produced. - result = replace(a) - result = quote do: - # `result`.val # TODO: expose a way to do this directly in std/options, e.g.: `getAsIs` - safeGet(`result`) - -macro `??.`*(a: untyped): Option = - ## Same as `?.` but returns an `Option`. - runnableExamples: - type Foo = ref object - x1: ref int - x2: int - # `?.` can't distinguish between a valid vs invalid default value, but `??.` can: - var f1 = Foo(x1: int.new, x2: 2) - doAssert (??.f1.x1[]).get == 0 # not enough to tell when the chain was valid. - doAssert (??.f1.x1[]).isSome # a nil didn't occur in the chain - doAssert (??.f1.x2).get == 2 - - var f2: Foo - doAssert not (??.f2.x1[]).isSome # f2 was nil - from std/options import UnpackDefect - doAssertRaises(UnpackDefect): discard (??.f2.x1[]).get - doAssert ?.f2.x1[] == 0 # in contrast, this returns default(int) - - result = replace(a) +when false: + # xxx: expose a way to do this directly in std/options, e.g.: `getAsIs` + proc safeGet[T](a: Option[T]): T {.inline.} = + get(a, default(T)) |