diff options
Diffstat (limited to 'lib/std')
65 files changed, 5487 insertions, 632 deletions
diff --git a/lib/std/appdirs.nim b/lib/std/appdirs.nim new file mode 100644 index 000000000..963451efe --- /dev/null +++ b/lib/std/appdirs.nim @@ -0,0 +1,94 @@ +## This module implements helpers for determining special directories used by apps. + +## .. importdoc:: paths.nim + +from std/private/osappdirs import nil +import std/paths +import std/envvars + +proc getHomeDir*(): Path {.inline, tags: [ReadEnvEffect, ReadIOEffect].} = + ## Returns the home directory of the current user. + ## + ## This proc is wrapped by the `expandTilde proc`_ + ## for the convenience of processing paths coming from user configuration files. + ## + ## See also: + ## * `getConfigDir proc`_ + ## * `getTempDir proc`_ + result = Path(osappdirs.getHomeDir()) + +proc getDataDir*(): Path {.inline, tags: [ReadEnvEffect, ReadIOEffect].} = + ## Returns the data directory of the current user for applications. + ## + ## On non-Windows OSs, this proc conforms to the XDG Base Directory + ## spec. Thus, this proc returns the value of the `XDG_DATA_HOME` environment + ## variable if it is set, otherwise it returns the default configuration + ## directory ("~/.local/share" or "~/Library/Application Support" on macOS). + ## + ## See also: + ## * `getHomeDir proc`_ + ## * `getConfigDir proc`_ + ## * `getTempDir proc`_ + ## * `expandTilde proc`_ + ## * `getCurrentDir proc`_ + ## * `setCurrentDir proc`_ + result = Path(osappdirs.getDataDir()) + +proc getConfigDir*(): Path {.inline, tags: [ReadEnvEffect, ReadIOEffect].} = + ## Returns the config directory of the current user for applications. + ## + ## On non-Windows OSs, this proc conforms to the XDG Base Directory + ## spec. Thus, this proc returns the value of the `XDG_CONFIG_HOME` environment + ## variable if it is set, otherwise it returns the default configuration + ## directory ("~/.config/"). + ## + ## An OS-dependent trailing slash is always present at the end of the + ## returned string: `\\` on Windows and `/` on all other OSs. + ## + ## See also: + ## * `getHomeDir proc`_ + ## * `getTempDir proc`_ + result = Path(osappdirs.getConfigDir()) + +proc getCacheDir*(): Path {.inline.} = + ## Returns the cache directory of the current user for applications. + ## + ## This makes use of the following environment variables: + ## + ## * On Windows: `getEnv("LOCALAPPDATA")` + ## + ## * On macOS: `getEnv("XDG_CACHE_HOME", getEnv("HOME") / "Library/Caches")` + ## + ## * On other platforms: `getEnv("XDG_CACHE_HOME", getEnv("HOME") / ".cache")` + ## + ## **See also:** + ## * `getHomeDir proc`_ + ## * `getTempDir proc`_ + ## * `getConfigDir proc`_ + # follows https://crates.io/crates/platform-dirs + result = Path(osappdirs.getCacheDir()) + +proc getCacheDir*(app: Path): Path {.inline.} = + ## Returns the cache directory for an application `app`. + ## + ## * On Windows, this uses: `getCacheDir() / app / "cache"` + ## * On other platforms, this uses: `getCacheDir() / app` + result = Path(osappdirs.getCacheDir(app.string)) + +proc getTempDir*(): Path {.inline, tags: [ReadEnvEffect, ReadIOEffect].} = + ## Returns the temporary directory of the current user for applications to + ## save temporary files in. + ## + ## On Windows, it calls [GetTempPath](https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-gettemppathw). + ## On Posix based platforms, it will check `TMPDIR`, `TEMP`, `TMP` and `TEMPDIR` environment variables in order. + ## On all platforms, `/tmp` will be returned if the procs fails. + ## + ## You can override this implementation + ## by adding `-d:tempDir=mytempname` to your compiler invocation. + ## + ## .. Note:: This proc does not check whether the returned path exists. + ## + ## See also: + ## * `getHomeDir proc`_ + ## * `getConfigDir proc`_ + result = Path(osappdirs.getTempDir()) diff --git a/lib/std/assertions.nim b/lib/std/assertions.nim index 0bc4653f2..56c37d205 100644 --- a/lib/std/assertions.nim +++ b/lib/std/assertions.nim @@ -7,11 +7,12 @@ # distribution, for details about the copyright. # -## This module implements assertion handling. - -when not declared(sysFatal): +when not defined(nimPreviewSlimSystem) and not declared(sysFatal): + include "system/rawquits" include "system/fatal" +## This module implements assertion handling. + import std/private/miscdollars # --------------------------------------------------------------------------- # helpers @@ -26,22 +27,18 @@ proc `$`(info: InstantiationInfo): string = # --------------------------------------------------------------------------- -when not defined(nimHasSinkInference): - {.pragma: nosinks.} proc raiseAssert*(msg: string) {.noinline, noreturn, nosinks.} = ## Raises an `AssertionDefect` with `msg`. - sysFatal(AssertionDefect, msg) + when defined(nimPreviewSlimSystem): + raise newException(AssertionDefect, msg) + else: + sysFatal(AssertionDefect, msg) proc failedAssertImpl*(msg: string) {.raises: [], tags: [].} = ## Raises an `AssertionDefect` with `msg`, but this is hidden ## from the effect system. Called when an assertion failed. - # trick the compiler to not list `AssertionDefect` when called - # by `assert`. - # xxx simplify this pending bootstrap >= 1.4.0, after which cast not needed - # anymore since `Defect` can't be raised. - type Hide = proc (msg: string) {.noinline, raises: [], noSideEffect, tags: [].} - cast[Hide](raiseAssert)(msg) + raiseAssert(msg) template assertImpl(cond: bool, msg: string, expr: string, enabled: static[bool]) = when enabled: @@ -101,6 +98,7 @@ template doAssertRaises*(exception: typedesc, code: untyped) = const begin = "expected raising '" & astToStr(exception) & "', instead" const msgEnd = " by: " & astToStr(code) template raisedForeign {.gensym.} = raiseAssert(begin & " raised foreign exception" & msgEnd) + {.push warning[BareExcept]:off.} when Exception is exception: try: if true: @@ -119,5 +117,6 @@ template doAssertRaises*(exception: typedesc, code: untyped) = mixin `$` # alternatively, we could define $cstring in this module raiseAssert(begin & " raised '" & $e.name & "'" & msgEnd) except: raisedForeign() + {.pop.} if wrong: raiseAssert(begin & " nothing was raised" & msgEnd) diff --git a/lib/std/cmdline.nim b/lib/std/cmdline.nim new file mode 100644 index 000000000..0ba4619e5 --- /dev/null +++ b/lib/std/cmdline.nim @@ -0,0 +1,313 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2022 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module contains system facilities for reading command +## line parameters. + +## **See also:** +## * `parseopt module <parseopt.html>`_ for command-line parser beyond +## `parseCmdLine proc`_ + + +include system/inclrtl + +when defined(nimPreviewSlimSystem): + import std/widestrs + +when defined(nodejs): + from std/private/oscommon import ReadDirEffect + + +const weirdTarget = defined(nimscript) or defined(js) + + +when weirdTarget: + discard +elif defined(windows): + import std/winlean +elif defined(posix): + import std/posix +else: + {.error: "The cmdline module has not been implemented for the target platform.".} + + +# Needed by windows in order to obtain the command line for targets +# other than command line targets +when defined(windows) and not weirdTarget: + template getCommandLine*(): untyped = getCommandLineW() + + +proc parseCmdLine*(c: string): seq[string] {. + noSideEffect, rtl, extern: "nos$1".} = + ## Splits a `command line`:idx: into several components. + ## + ## **Note**: This proc is only occasionally useful, better use the + ## `parseopt module <parseopt.html>`_. + ## + ## On Windows, it uses the `following parsing rules + ## <http://msdn.microsoft.com/en-us/library/17w5ykft.aspx>`_: + ## + ## * Arguments are delimited by white space, which is either a space or a tab. + ## * The caret character (^) is not recognized as an escape character or + ## delimiter. The character is handled completely by the command-line parser + ## in the operating system before being passed to the argv array in the + ## program. + ## * A string surrounded by double quotation marks ("string") is interpreted + ## as a single argument, regardless of white space contained within. A + ## quoted string can be embedded in an argument. + ## * A double quotation mark preceded by a backslash (\") is interpreted as a + ## literal double quotation mark character ("). + ## * Backslashes are interpreted literally, unless they immediately precede + ## a double quotation mark. + ## * If an even number of backslashes is followed by a double quotation mark, + ## one backslash is placed in the argv array for every pair of backslashes, + ## and the double quotation mark is interpreted as a string delimiter. + ## * If an odd number of backslashes is followed by a double quotation mark, + ## one backslash is placed in the argv array for every pair of backslashes, + ## and the double quotation mark is "escaped" by the remaining backslash, + ## causing a literal double quotation mark (") to be placed in argv. + ## + ## On Posix systems, it uses the following parsing rules: + ## Components are separated by whitespace unless the whitespace + ## occurs within ``"`` or ``'`` quotes. + ## + ## See also: + ## * `parseopt module <parseopt.html>`_ + ## * `paramCount proc`_ + ## * `paramStr proc`_ + ## * `commandLineParams proc`_ + + result = @[] + var i = 0 + var a = "" + while true: + setLen(a, 0) + # eat all delimiting whitespace + while i < c.len and c[i] in {' ', '\t', '\l', '\r'}: inc(i) + if i >= c.len: break + when defined(windows): + # parse a single argument according to the above rules: + var inQuote = false + while i < c.len: + case c[i] + of '\\': + var j = i + while j < c.len and c[j] == '\\': inc(j) + if j < c.len and c[j] == '"': + for k in 1..(j-i) div 2: a.add('\\') + if (j-i) mod 2 == 0: + i = j + else: + a.add('"') + i = j+1 + else: + a.add(c[i]) + inc(i) + of '"': + inc(i) + if not inQuote: inQuote = true + elif i < c.len and c[i] == '"': + a.add(c[i]) + inc(i) + else: + inQuote = false + break + of ' ', '\t': + if not inQuote: break + a.add(c[i]) + inc(i) + else: + a.add(c[i]) + inc(i) + else: + case c[i] + of '\'', '\"': + var delim = c[i] + inc(i) # skip ' or " + while i < c.len and c[i] != delim: + add a, c[i] + inc(i) + if i < c.len: inc(i) + else: + while i < c.len and c[i] > ' ': + add(a, c[i]) + inc(i) + add(result, move a) + +when defined(nimdoc): + # Common forward declaration docstring block for parameter retrieval procs. + proc paramCount*(): int {.tags: [ReadIOEffect].} = + ## Returns the number of `command line arguments`:idx: given to the + ## application. + ## + ## Unlike `argc`:idx: in C, if your binary was called without parameters this + ## will return zero. + ## You can query each individual parameter with `paramStr proc`_ + ## or retrieve all of them in one go with `commandLineParams proc`_. + ## + ## **Availability**: When generating a dynamic library (see `--app:lib`) on + ## Posix this proc is not defined. + ## Test for availability using `declared() <system.html#declared,untyped>`_. + ## + ## See also: + ## * `parseopt module <parseopt.html>`_ + ## * `parseCmdLine proc`_ + ## * `paramStr proc`_ + ## * `commandLineParams proc`_ + ## + ## **Examples:** + ## + ## ```nim + ## when declared(paramCount): + ## # Use paramCount() here + ## else: + ## # Do something else! + ## ``` + + proc paramStr*(i: int): string {.tags: [ReadIOEffect].} = + ## Returns the `i`-th `command line argument`:idx: given to the application. + ## + ## `i` should be in the range `1..paramCount()`, the `IndexDefect` + ## exception will be raised for invalid values. Instead of iterating + ## over `paramCount()`_ with this proc you can + ## call the convenience `commandLineParams()`_. + ## + ## Similarly to `argv`:idx: in C, + ## it is possible to call `paramStr(0)` but this will return OS specific + ## contents (usually the name of the invoked executable). You should avoid + ## this and call `getAppFilename() <os.html#getAppFilename>`_ instead. + ## + ## **Availability**: When generating a dynamic library (see `--app:lib`) on + ## Posix this proc is not defined. + ## Test for availability using `declared() <system.html#declared,untyped>`_. + ## + ## See also: + ## * `parseopt module <parseopt.html>`_ + ## * `parseCmdLine proc`_ + ## * `paramCount proc`_ + ## * `commandLineParams proc`_ + ## * `getAppFilename proc <os.html#getAppFilename>`_ + ## + ## **Examples:** + ## + ## ```nim + ## when declared(paramStr): + ## # Use paramStr() here + ## else: + ## # Do something else! + ## ``` + +elif defined(nimscript): discard +elif defined(nodejs): + type Argv = object of JsRoot + let argv {.importjs: "process.argv".} : Argv + proc len(argv: Argv): int {.importjs: "#.length".} + proc `[]`(argv: Argv, i: int): cstring {.importjs: "#[#]".} + + proc paramCount*(): int {.tags: [ReadDirEffect].} = + result = argv.len - 2 + + proc paramStr*(i: int): string {.tags: [ReadIOEffect].} = + let i = i + 1 + if i < argv.len and i >= 0: + result = $argv[i] + else: + raise newException(IndexDefect, formatErrorIndexBound(i - 1, argv.len - 2)) +elif defined(windows): + # Since we support GUI applications with Nim, we sometimes generate + # a WinMain entry proc. But a WinMain proc has no access to the parsed + # command line arguments. The way to get them differs. Thus we parse them + # ourselves. This has the additional benefit that the program's behaviour + # is always the same -- independent of the used C compiler. + var + ownArgv {.threadvar.}: seq[string] + ownParsedArgv {.threadvar.}: bool + + proc paramCount*(): int {.rtl, extern: "nos$1", tags: [ReadIOEffect].} = + # Docstring in nimdoc block. + if not ownParsedArgv: + ownArgv = parseCmdLine($getCommandLine()) + ownParsedArgv = true + result = ownArgv.len-1 + + proc paramStr*(i: int): string {.rtl, extern: "nos$1", + tags: [ReadIOEffect].} = + # Docstring in nimdoc block. + if not ownParsedArgv: + ownArgv = parseCmdLine($getCommandLine()) + ownParsedArgv = true + if i < ownArgv.len and i >= 0: + result = ownArgv[i] + else: + raise newException(IndexDefect, formatErrorIndexBound(i, ownArgv.len-1)) + +elif defined(genode): + proc paramStr*(i: int): string = + raise newException(OSError, "paramStr is not implemented on Genode") + + proc paramCount*(): int = + raise newException(OSError, "paramCount is not implemented on Genode") +elif weirdTarget or (defined(posix) and appType == "lib"): + proc paramStr*(i: int): string {.tags: [ReadIOEffect].} = + raise newException(OSError, "paramStr is not implemented on current platform") + + proc paramCount*(): int {.tags: [ReadIOEffect].} = + raise newException(OSError, "paramCount is not implemented on current platform") +elif not defined(createNimRtl) and + not(defined(posix) and appType == "lib"): + # On Posix, there is no portable way to get the command line from a DLL. + var + cmdCount {.importc: "cmdCount".}: cint + cmdLine {.importc: "cmdLine".}: cstringArray + + proc paramStr*(i: int): string {.tags: [ReadIOEffect].} = + # Docstring in nimdoc block. + if i < cmdCount and i >= 0: + result = $cmdLine[i] + else: + raise newException(IndexDefect, formatErrorIndexBound(i, cmdCount-1)) + + proc paramCount*(): int {.tags: [ReadIOEffect].} = + # Docstring in nimdoc block. + result = cmdCount-1 + +when declared(paramCount) or defined(nimdoc): + proc commandLineParams*(): seq[string] = + ## Convenience proc which returns the command line parameters. + ## + ## This returns **only** the parameters. If you want to get the application + ## executable filename, call `getAppFilename() <os.html#getAppFilename>`_. + ## + ## **Availability**: On Posix there is no portable way to get the command + ## line from a DLL and thus the proc isn't defined in this environment. You + ## can test for its availability with `declared() + ## <system.html#declared,untyped>`_. + ## + ## See also: + ## * `parseopt module <parseopt.html>`_ + ## * `parseCmdLine proc`_ + ## * `paramCount proc`_ + ## * `paramStr proc`_ + ## * `getAppFilename proc <os.html#getAppFilename>`_ + ## + ## **Examples:** + ## + ## ```nim + ## when declared(commandLineParams): + ## # Use commandLineParams() here + ## else: + ## # Do something else! + ## ``` + result = @[] + for i in 1..paramCount(): + result.add(paramStr(i)) +else: + proc commandLineParams*(): seq[string] {.error: + "commandLineParams() unsupported by dynamic libraries".} = + discard diff --git a/lib/std/decls.nim b/lib/std/decls.nim index 7b907f5e1..bb7ec3593 100644 --- a/lib/std/decls.nim +++ b/lib/std/decls.nim @@ -1,16 +1,15 @@ -# see `semLowerLetVarCustomPragma` for compiler support that enables these -# lowerings +## This module implements syntax sugar for some declarations. -import macros +import std/macros macro byaddr*(sect) = ## Allows a syntax for l-value references, being an exact analog to ## `auto& a = ex;` in C++. ## - ## Warning: This makes use of 2 experimental features, namely nullary - ## templates instantiated as symbols and variable macro pragmas. - ## For this reason, its behavior is not stable. The current implementation - ## allows redefinition, but this is not an intended consequence. + ## .. warning:: This makes use of 2 experimental features, namely nullary + ## templates instantiated as symbols and variable macro pragmas. + ## For this reason, its behavior is not stable. The current implementation + ## allows redefinition, but this is not an intended consequence. runnableExamples: var s = @[10, 11, 12] var a {.byaddr.} = s[0] diff --git a/lib/std/dirs.nim b/lib/std/dirs.nim new file mode 100644 index 000000000..380d6d08f --- /dev/null +++ b/lib/std/dirs.nim @@ -0,0 +1,135 @@ +## This module implements directory handling. + +from std/paths import Path, ReadDirEffect, WriteDirEffect + +from std/private/osdirs import dirExists, createDir, existsOrCreateDir, removeDir, + moveDir, walkDir, setCurrentDir, + walkDirRec, PathComponent + +export PathComponent + +proc dirExists*(dir: Path): bool {.inline, tags: [ReadDirEffect], sideEffect.} = + ## Returns true if the directory `dir` exists. If `dir` is a file, false + ## is returned. Follows symlinks. + result = dirExists(dir.string) + +proc createDir*(dir: Path) {.inline, tags: [WriteDirEffect, ReadDirEffect].} = + ## Creates the `directory`:idx: `dir`. + ## + ## The directory may contain several subdirectories that do not exist yet. + ## The full path is created. If this fails, `OSError` is raised. + ## + ## It does **not** fail if the directory already exists because for + ## most usages this does not indicate an error. + ## + ## See also: + ## * `removeDir proc`_ + ## * `existsOrCreateDir proc`_ + ## * `moveDir proc`_ + createDir(dir.string) + +proc existsOrCreateDir*(dir: Path): bool {.inline, tags: [WriteDirEffect, ReadDirEffect].} = + ## Checks if a `directory`:idx: `dir` exists, and creates it otherwise. + ## + ## Does not create parent directories (raises `OSError` if parent directories do not exist). + ## Returns `true` if the directory already exists, and `false` otherwise. + ## + ## See also: + ## * `removeDir proc`_ + ## * `createDir proc`_ + ## * `moveDir proc`_ + result = existsOrCreateDir(dir.string) + +proc removeDir*(dir: Path, checkDir = false + ) {.inline, tags: [WriteDirEffect, ReadDirEffect].} = + ## Removes the directory `dir` including all subdirectories and files + ## in `dir` (recursively). + ## + ## If this fails, `OSError` is raised. This does not fail if the directory never + ## existed in the first place, unless `checkDir` = true. + ## + ## See also: + ## * `removeFile proc <files.html#removeFile>`_ + ## * `existsOrCreateDir proc`_ + ## * `createDir proc`_ + ## * `moveDir proc`_ + removeDir(dir.string, checkDir) + +proc moveDir*(source, dest: Path) {.inline, tags: [ReadIOEffect, WriteIOEffect].} = + ## Moves a directory from `source` to `dest`. + ## + ## Symlinks are not followed: if `source` contains symlinks, they themself are + ## moved, not their target. + ## + ## If this fails, `OSError` is raised. + ## + ## See also: + ## * `moveFile proc <files.html#moveFile>`_ + ## * `removeDir proc`_ + ## * `existsOrCreateDir proc`_ + ## * `createDir proc`_ + moveDir(source.string, dest.string) + +iterator walkDir*(dir: Path; relative = false, checkDir = false, + skipSpecial = false): + tuple[kind: PathComponent, path: Path] {.tags: [ReadDirEffect].} = + ## Walks over the directory `dir` and yields for each directory or file in + ## `dir`. The component type and full path for each item are returned. + ## + ## Walking is not recursive. + ## * If `relative` is true (default: false) + ## the resulting path is shortened to be relative to ``dir``, + ## otherwise the full path is returned. + ## * If `checkDir` is true, `OSError` is raised when `dir` + ## doesn't exist. + ## * If `skipSpecial` is true, then (besides all directories) only *regular* + ## files (**without** special "file" objects like FIFOs, device files, + ## etc) will be yielded on Unix. + for (k, p) in walkDir(dir.string, relative, checkDir, skipSpecial): + yield (k, Path(p)) + +iterator walkDirRec*(dir: Path, + yieldFilter = {pcFile}, followFilter = {pcDir}, + relative = false, checkDir = false, skipSpecial = false): + Path {.tags: [ReadDirEffect].} = + ## Recursively walks over the directory `dir` and yields for each file + ## or directory in `dir`. + ## + ## Options `relative`, `checkdir`, `skipSpecial` are explained in + ## [walkDir iterator] description. + ## + ## .. warning:: Modifying the directory structure while the iterator + ## is traversing may result in undefined behavior! + ## + ## Walking is recursive. `followFilter` controls the behaviour of the iterator: + ## + ## ===================== ============================================= + ## yieldFilter meaning + ## ===================== ============================================= + ## ``pcFile`` yield real files (default) + ## ``pcLinkToFile`` yield symbolic links to files + ## ``pcDir`` yield real directories + ## ``pcLinkToDir`` yield symbolic links to directories + ## ===================== ============================================= + ## + ## ===================== ============================================= + ## followFilter meaning + ## ===================== ============================================= + ## ``pcDir`` follow real directories (default) + ## ``pcLinkToDir`` follow symbolic links to directories + ## ===================== ============================================= + ## + ## + ## See also: + ## * `walkDir iterator`_ + for p in walkDirRec(dir.string, yieldFilter, followFilter, relative, + checkDir, skipSpecial): + yield Path(p) + +proc setCurrentDir*(newDir: Path) {.inline, tags: [].} = + ## Sets the `current working directory`:idx:; `OSError` + ## is raised if `newDir` cannot been set. + ## + ## See also: + ## * `getCurrentDir proc <paths.html#getCurrentDir>`_ + osdirs.setCurrentDir(newDir.string) diff --git a/lib/std/editdistance.nim b/lib/std/editdistance.nim index 9f29c5c05..40c0017ae 100644 --- a/lib/std/editdistance.nim +++ b/lib/std/editdistance.nim @@ -10,7 +10,7 @@ ## This module implements an algorithm to compute the ## `edit distance`:idx: between two Unicode strings. -import unicode +import std/unicode proc editDistance*(a, b: string): int {.noSideEffect.} = ## Returns the **unicode-rune** edit distance between `a` and `b`. @@ -18,7 +18,7 @@ proc editDistance*(a, b: string): int {.noSideEffect.} = ## This uses the `Levenshtein`:idx: distance algorithm with only a linear ## memory overhead. runnableExamples: static: doAssert editdistance("Kitten", "Bitten") == 1 - if len(a) > len(b): + if runeLen(a) > runeLen(b): # make `b` the longer string return editDistance(b, a) # strip common prefix diff --git a/lib/std/effecttraits.nim b/lib/std/effecttraits.nim index fb057a669..3d1b4ffd3 100644 --- a/lib/std/effecttraits.nim +++ b/lib/std/effecttraits.nim @@ -14,7 +14,7 @@ ## One can test for the existence of this standard module ## via `defined(nimHasEffectTraitsModule)`. -import macros +import std/macros proc getRaisesListImpl(n: NimNode): NimNode = discard "see compiler/vmops.nim" proc getTagsListImpl(n: NimNode): NimNode = discard "see compiler/vmops.nim" diff --git a/lib/std/enumerate.nim b/lib/std/enumerate.nim index a8f0e1ba7..beb65ed30 100644 --- a/lib/std/enumerate.nim +++ b/lib/std/enumerate.nim @@ -11,7 +11,7 @@ ## macro system. import std/private/since -import macros +import std/macros macro enumerate*(x: ForLoopStmt): untyped {.since: (1, 3).} = @@ -21,7 +21,7 @@ macro enumerate*(x: ForLoopStmt): untyped {.since: (1, 3).} = ## The default starting count `0` can be manually overridden if needed. runnableExamples: let a = [10, 20, 30] - var b: seq[(int, int)] + var b: seq[(int, int)] = @[] for i, x in enumerate(a): b.add((i, x)) assert b == @[(0, 10), (1, 20), (2, 30)] diff --git a/lib/std/enumutils.nim b/lib/std/enumutils.nim index 9d4ff1bcf..9c338817d 100644 --- a/lib/std/enumutils.nim +++ b/lib/std/enumutils.nim @@ -7,8 +7,8 @@ # distribution, for details about the copyright. # -import macros -from typetraits import OrdinalEnum, HoleyEnum +import std/macros +from std/typetraits import OrdinalEnum, HoleyEnum when defined(nimPreviewSlimSystem): import std/assertions @@ -21,32 +21,34 @@ macro genEnumCaseStmt*(typ: typedesc, argSym: typed, default: typed, # Generates a case stmt, which assigns the correct enum field given # a normalized string comparison to the `argSym` input. # string normalization is done using passed normalizer. - # NOTE: for an enum with fields Foo, Bar, ... we cannot generate - # `of "Foo".nimIdentNormalize: Foo`. - # This will fail, if the enum is not defined at top level (e.g. in a block). - # Thus we check for the field value of the (possible holed enum) and convert - # the integer value to the generic argument `typ`. let typ = typ.getTypeInst[1] - let impl = typ.getImpl[2] + let typSym = typ.getTypeImpl.getTypeInst # skip aliases etc to get type sym + let impl = typSym.getImpl[2] expectKind impl, nnkEnumTy let normalizerNode = quote: `normalizer` expectKind normalizerNode, nnkSym result = nnkCaseStmt.newTree(newCall(normalizerNode, argSym)) # stores all processed field strings to give error msg for ambiguous enums var foundFields: seq[string] = @[] + var fVal = "" var fStr = "" # string of current field var fNum = BiggestInt(0) # int value of current field for f in impl: case f.kind of nnkEmpty: continue # skip first node of `enumTy` - of nnkSym, nnkIdent: fStr = f.strVal + of nnkSym, nnkIdent: + fVal = f.strVal + fStr = fVal of nnkAccQuoted: - fStr = "" + fVal = "" for ch in f: - fStr.add ch.strVal + fVal.add ch.strVal + fStr = fVal of nnkEnumFieldDef: + fVal = f[0].strVal case f[1].kind - of nnkStrLit: fStr = f[1].strVal + of nnkStrLit: + fStr = f[1].strVal of nnkTupleConstr: fStr = f[1][1].strVal fNum = f[1][0].intVal @@ -64,7 +66,7 @@ macro genEnumCaseStmt*(typ: typedesc, argSym: typed, default: typed, if fNum >= userMin and fNum <= userMax: fStr = normalizer(fStr) if fStr notin foundFields: - result.add nnkOfBranch.newTree(newLit fStr, nnkCall.newTree(typ, newLit fNum)) + result.add nnkOfBranch.newTree(newLit fStr, newDotExpr(typ, ident fVal)) foundFields.add fStr else: error("Ambiguous enums cannot be parsed, field " & $fStr & @@ -80,7 +82,7 @@ macro genEnumCaseStmt*(typ: typedesc, argSym: typed, default: typed, result.add nnkElse.newTree(default) macro enumFullRange(a: typed): untyped = - newNimNode(nnkCurly).add(a.getType[1][1..^1]) + newNimNode(nnkBracket).add(a.getType[1][1..^1]) macro enumNames(a: typed): untyped = # this could be exported too; in particular this could be useful for enum with holes. @@ -112,9 +114,9 @@ const invalidSlot = uint8.high proc genLookup[T: typedesc[HoleyEnum]](_: T): auto = const n = span(T) - var ret: array[n, uint8] var i = 0 assert n <= invalidSlot.int + var ret {.noinit.}: array[n, uint8] for ai in mitems(ret): ai = invalidSlot for ai in items(T): ret[ai.ord - T.low.ord] = uint8(i) @@ -172,6 +174,9 @@ template symbolRank*[T: enum](a: T): int = when T is Ordinal: ord(a) - T.low.ord.static else: symbolRankImpl(a) +proc rangeBase(T: typedesc): typedesc {.magic: "TypeTrait".} + # skip one level of range; return the base type of a range type + func symbolName*[T: enum](a: T): string = ## Returns the symbol name of an enum. ## @@ -190,5 +195,8 @@ func symbolName*[T: enum](a: T): string = c1 = 4 c2 = 20 assert c1.symbolName == "c1" - const names = enumNames(T) + when T is range: + const names = enumNames(rangeBase T) + else: + const names = enumNames(T) names[a.symbolRank] diff --git a/lib/std/envvars.nim b/lib/std/envvars.nim index d7706c17d..a955077ea 100644 --- a/lib/std/envvars.nim +++ b/lib/std/envvars.nim @@ -8,7 +8,7 @@ # -## The `std/envvars` module implements environment variables handling. +## The `std/envvars` module implements environment variable handling. import std/oserrors type @@ -60,10 +60,16 @@ when not defined(nimscript): when defined(windows): proc c_putenv(envstring: cstring): cint {.importc: "_putenv", header: "<stdlib.h>".} from std/private/win_setenv import setEnvImpl - import winlean - proc c_wgetenv(varname: WideCString): WideCString {.importc: "_wgetenv", + import std/winlean + when defined(nimPreviewSlimSystem): + import std/widestrs + + type wchar_t {.importc: "wchar_t", header: "<stdlib.h>".} = int16 + proc c_wgetenv(varname: ptr wchar_t): ptr wchar_t {.importc: "_wgetenv", header: "<stdlib.h>".} - proc getEnvImpl(env: cstring): WideCString = c_wgetenv(env.newWideCString) + proc getEnvImpl(env: cstring): WideCString = + let r: WideCString = env.newWideCString + cast[WideCString](c_wgetenv(cast[ptr wchar_t](r))) else: proc c_getenv(env: cstring): cstring {. importc: "getenv", header: "<stdlib.h>".} @@ -88,8 +94,10 @@ when not defined(nimscript): assert getEnv("unknownEnv", "doesn't exist") == "doesn't exist" let env = getEnvImpl(key) - if env == nil: return default - result = $env + if env == nil: + result = default + else: + result = $env proc existsEnv*(key: string): bool {.tags: [ReadEnvEffect].} = ## Checks whether the environment variable named `key` exists. @@ -103,7 +111,7 @@ when not defined(nimscript): runnableExamples: assert not existsEnv("unknownEnv") - return getEnvImpl(key) != nil + result = getEnvImpl(key) != nil proc putEnv*(key, val: string) {.tags: [WriteEnvEffect].} = ## Sets the value of the `environment variable`:idx: named `key` to `val`. @@ -134,7 +142,7 @@ when not defined(nimscript): ## * `envPairs iterator`_ template bail = raiseOSError(osLastError(), key) when defined(windows): - #[ + #[ # https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/putenv-s-wputenv-s?view=msvc-160 > You can remove a variable from the environment by specifying an empty string (that is, "") for value_string note that nil is not legal @@ -171,20 +179,17 @@ when not defined(nimscript): iterator envPairsImpl(): tuple[key, value: string] {.tags: [ReadEnvEffect].} = when defined(windows): - block: - template impl(get_fun, typ, size, zero, free_fun) = - let env = get_fun() - var e = env - if e == nil: break - while true: - let eend = strEnd(e) - let kv = $e - let p = find(kv, '=') - yield (substr(kv, 0, p-1), substr(kv, p+1)) - e = cast[typ](cast[ByteAddress](eend)+size) - if typeof(zero)(eend[1]) == zero: break - discard free_fun(env) - impl(getEnvironmentStringsW, WideCString, 2, 0, freeEnvironmentStringsW) + let env = getEnvironmentStringsW() + var e = env + if e != nil: + while true: + let eend = strEnd(e) + let kv = $e + let p = find(kv, '=') + yield (substr(kv, 0, p-1), substr(kv, p+1)) + e = cast[WideCString](cast[ByteAddress](eend)+2) + if int(eend[1]) == 0: break + discard freeEnvironmentStringsW(env) else: var i = 0 when defined(macosx) and not defined(ios) and not defined(emscripten): diff --git a/lib/std/exitprocs.nim b/lib/std/exitprocs.nim index c6537f7f8..f26368f42 100644 --- a/lib/std/exitprocs.nim +++ b/lib/std/exitprocs.nim @@ -7,7 +7,11 @@ # distribution, for details about the copyright. # -import locks +## This module allows adding hooks to program exit. + +import std/locks +when defined(js) and not defined(nodejs): + import std/assertions type FunKind = enum kClosure, kNoconv # extend as needed @@ -18,20 +22,20 @@ type var gFunsLock: Lock - gFuns: seq[Fun] + gFuns {.cursor.}: seq[Fun] #Intentionally use the cursor to break up the lifetime trace and make it compatible with JS. initLock(gFunsLock) when defined(js): proc addAtExit(quitProc: proc() {.noconv.}) = when defined(nodejs): - asm """ + {.emit: """ process.on('exit', `quitProc`); - """ + """.} elif defined(js): - asm """ + {.emit: """ window.onbeforeunload = `quitProc`; - """ + """.} else: proc addAtExit(quitProc: proc() {.noconv.}) {. importc: "atexit", header: "<stdlib.h>".} @@ -43,6 +47,7 @@ proc callClosures() {.noconv.} = case fun.kind of kClosure: fun.fun1() of kNoconv: fun.fun2() + gFuns.setLen(0) template fun() = if gFuns.len == 0: @@ -64,24 +69,19 @@ proc addExitProc*(cl: proc() {.noconv.}) = fun() gFuns.add Fun(kind: kNoconv, fun2: cl) -when not defined(nimscript): +when not defined(nimscript) and (not defined(js) or defined(nodejs)): proc getProgramResult*(): int = when defined(js) and defined(nodejs): - asm """ + {.emit: """ `result` = process.exitCode; -""" - elif not defined(js): - result = programResult +""".} else: - doAssert false + result = programResult proc setProgramResult*(a: int) = - # pending https://github.com/nim-lang/Nim/issues/14674 when defined(js) and defined(nodejs): - asm """ + {.emit: """ process.exitCode = `a`; -""" - elif not defined(js): - programResult = a +""".} else: - doAssert false + programResult = a diff --git a/lib/std/files.nim b/lib/std/files.nim new file mode 100644 index 000000000..c4e0491c9 --- /dev/null +++ b/lib/std/files.nim @@ -0,0 +1,46 @@ +## This module implements file handling. +## +## **See also:** +## * `paths module <paths.html>`_ for path manipulation + +from std/paths import Path, ReadDirEffect, WriteDirEffect + +from std/private/osfiles import fileExists, removeFile, + moveFile + + +proc fileExists*(filename: Path): bool {.inline, tags: [ReadDirEffect], sideEffect.} = + ## Returns true if `filename` exists and is a regular file or symlink. + ## + ## Directories, device files, named pipes and sockets return false. + result = fileExists(filename.string) + +proc removeFile*(file: Path) {.inline, tags: [WriteDirEffect].} = + ## Removes the `file`. + ## + ## If this fails, `OSError` is raised. This does not fail + ## if the file never existed in the first place. + ## + ## On Windows, ignores the read-only attribute. + ## + ## See also: + ## * `removeDir proc <dirs.html#removeDir>`_ + ## * `moveFile proc`_ + removeFile(file.string) + +proc moveFile*(source, dest: Path) {.inline, + tags: [ReadDirEffect, ReadIOEffect, WriteIOEffect].} = + ## Moves a file from `source` to `dest`. + ## + ## Symlinks are not followed: if `source` is a symlink, it is itself moved, + ## not its target. + ## + ## If this fails, `OSError` is raised. + ## If `dest` already exists, it will be overwritten. + ## + ## Can be used to `rename files`:idx:. + ## + ## See also: + ## * `moveDir proc <dirs.html#moveDir>`_ + ## * `removeFile proc`_ + moveFile(source.string, dest.string) diff --git a/lib/std/formatfloat.nim b/lib/std/formatfloat.nim index 6f2383760..9258245f6 100644 --- a/lib/std/formatfloat.nim +++ b/lib/std/formatfloat.nim @@ -7,10 +7,10 @@ # distribution, for details about the copyright. # +## This module implements formatting floats as strings. + when defined(nimPreviewSlimSystem): import std/assertions -else: - {.deprecated: "formatfloat is about to move out of system; use `-d:nimPreviewSlimSystem` and import `std/formatfloat`".} proc c_memcpy(a, b: pointer, size: csize_t): pointer {.importc: "memcpy", header: "<string.h>", discardable.} @@ -28,15 +28,15 @@ proc writeFloatToBufferRoundtrip*(buf: var array[65, char]; value: BiggestFloat) ## ## returns the amount of bytes written to `buf` not counting the ## terminating '\0' character. - result = toChars(buf, value, forceTrailingDotZero=true) + result = toChars(buf, value, forceTrailingDotZero=true).int buf[result] = '\0' proc writeFloatToBufferRoundtrip*(buf: var array[65, char]; value: float32): int = - result = float32ToChars(buf, value, forceTrailingDotZero=true) + result = float32ToChars(buf, value, forceTrailingDotZero=true).int buf[result] = '\0' -proc c_sprintf(buf, frmt: cstring): cint {.header: "<stdio.h>", - importc: "sprintf", varargs, noSideEffect.} +proc c_snprintf(buf: cstring, n: csize_t, frmt: cstring): cint {.header: "<stdio.h>", + importc: "snprintf", varargs, noSideEffect.} proc writeToBuffer(buf: var array[65, char]; value: cstring) = var i = 0 @@ -49,7 +49,7 @@ proc writeFloatToBufferSprintf*(buf: var array[65, char]; value: BiggestFloat): ## ## returns the amount of bytes written to `buf` not counting the ## terminating '\0' character. - var n: int = c_sprintf(addr buf, "%.16g", value) + var n = c_snprintf(cast[cstring](addr buf), 65, "%.16g", value).int var hasDot = false for i in 0..n-1: if buf[i] == ',': @@ -86,36 +86,37 @@ proc writeFloatToBuffer*(buf: var array[65, char]; value: BiggestFloat | float32 proc addFloatRoundtrip*(result: var string; x: float | float32) = when nimvm: - doAssert false + raiseAssert "unreachable" else: var buffer {.noinit.}: array[65, char] let n = writeFloatToBufferRoundtrip(buffer, x) - result.addCstringN(cstring(buffer[0].addr), n) + result.addCstringN(cast[cstring](buffer[0].addr), n) proc addFloatSprintf*(result: var string; x: float) = when nimvm: - doAssert false + raiseAssert "unreachable" else: var buffer {.noinit.}: array[65, char] let n = writeFloatToBufferSprintf(buffer, x) - result.addCstringN(cstring(buffer[0].addr), n) - -proc nimFloatToString(a: float): cstring = - ## ensures the result doesn't print like an integer, i.e. return 2.0, not 2 - # print `-0.0` properly - asm """ - function nimOnlyDigitsOrMinus(n) { - return n.toString().match(/^-?\d+$/); - } - if (Number.isSafeInteger(`a`)) - `result` = `a` === 0 && 1 / `a` < 0 ? "-0.0" : `a`+".0" - else { - `result` = `a`+"" - if(nimOnlyDigitsOrMinus(`result`)){ - `result` = `a`+".0" + result.addCstringN(cast[cstring](buffer[0].addr), n) + +when defined(js): + proc nimFloatToString(a: float): cstring = + ## ensures the result doesn't print like an integer, i.e. return 2.0, not 2 + # print `-0.0` properly + {.emit: """ + function nimOnlyDigitsOrMinus(n) { + return n.toString().match(/^-?\d+$/); + } + if (Number.isSafeInteger(`a`)) + `result` = `a` === 0 && 1 / `a` < 0 ? "-0.0" : `a`+".0"; + else { + `result` = `a`+""; + if(nimOnlyDigitsOrMinus(`result`)){ + `result` = `a`+".0"; + } } - } - """ + """.} proc addFloat*(result: var string; x: float | float32) {.inline.} = ## Converts float to its string representation and appends it to `result`. diff --git a/lib/std/genasts.nim b/lib/std/genasts.nim index c5f51e5d9..d0f07c527 100644 --- a/lib/std/genasts.nim +++ b/lib/std/genasts.nim @@ -1,4 +1,6 @@ -import macros +## This module implements AST generation using captured variables for macros. + +import std/macros type GenAstOpt* = enum kDirtyTemplate, @@ -22,7 +24,7 @@ macro genAstOpt*(options: static set[GenAstOpt], args: varargs[untyped]): untype result = genAst(cond, s = repr(cond), lhs = cond[1], rhs = cond[2]): # each local symbol we access must be explicitly captured if not cond: - doAssert false, "'$#'' failed: lhs: '$#', rhs: '$#'" % [s, $lhs, $rhs] + raiseAssert "'$#'' failed: lhs: '$#', rhs: '$#'" % [s, $lhs, $rhs] let a = 3 check2 a*2 == a+3 if false: check2 a*2 < a+1 # would error with: 'a * 2 < a + 1'' failed: lhs: '6', rhs: '4' diff --git a/lib/std/isolation.nim b/lib/std/isolation.nim index 7d6ac6092..b03e00651 100644 --- a/lib/std/isolation.nim +++ b/lib/std/isolation.nim @@ -15,7 +15,7 @@ ## type - Isolated*[T] = object ## Isolated data can only be moved, not copied. + Isolated*[T] {.sendable.} = object ## Isolated data can only be moved, not copied. value: T proc `=copy`*[T](dest: var Isolated[T]; src: Isolated[T]) {.error.} @@ -38,9 +38,9 @@ func isolate*[T](value: sink T): Isolated[T] {.magic: "Isolate".} = func unsafeIsolate*[T](value: sink T): Isolated[T] = ## Creates an isolated subgraph from the expression `value`. - ## + ## ## .. warning:: The proc doesn't check whether `value` is isolated. - ## + ## Isolated[T](value: value) func extract*[T](src: var Isolated[T]): T = diff --git a/lib/std/jsbigints.nim b/lib/std/jsbigints.nim index 04578fc87..4e996ea7b 100644 --- a/lib/std/jsbigints.nim +++ b/lib/std/jsbigints.nim @@ -14,7 +14,7 @@ func big*(integer: SomeInteger): JsBigInt {.importjs: "BigInt(#)".} = runnableExamples: doAssert big(1234567890) == big"1234567890" doAssert 0b1111100111.big == 0o1747.big and 0o1747.big == 999.big - when nimvm: doAssert false, "JsBigInt can not be used at compile-time nor static context" else: discard + when nimvm: raiseAssert "JsBigInt can not be used at compile-time nor static context" else: discard func `'big`*(num: cstring): JsBigInt {.importjs: "BigInt(#)".} = ## Constructor for `JsBigInt`. @@ -28,11 +28,11 @@ func `'big`*(num: cstring): JsBigInt {.importjs: "BigInt(#)".} = doAssert 0xdeadbeaf'big == 0xdeadbeaf.big doAssert 0xffffffffffffffff'big == (1'big shl 64'big) - 1'big doAssert not compiles(static(12'big)) - when nimvm: doAssert false, "JsBigInt can not be used at compile-time nor static context" else: discard + when nimvm: raiseAssert "JsBigInt can not be used at compile-time nor static context" else: discard func big*(integer: cstring): JsBigInt {.importjs: "BigInt(#)".} = ## Alias for `'big` - when nimvm: doAssert false, "JsBigInt can not be used at compile-time nor static context" else: discard + when nimvm: raiseAssert "JsBigInt can not be used at compile-time nor static context" else: discard func toCstring*(this: JsBigInt; radix: 2..36): cstring {.importjs: "#.toString(#)".} = ## Converts from `JsBigInt` to `cstring` representation. @@ -64,10 +64,10 @@ func wrapToUint*(this: JsBigInt; bits: Natural): JsBigInt {.importjs: runnableExamples: doAssert (big("3") + big("2") ** big("66")).wrapToUint(66) == big("3") -func toNumber*(this: JsBigInt): BiggestInt {.importjs: "Number(#)".} = +func toNumber*(this: JsBigInt): int {.importjs: "Number(#)".} = ## Does not do any bounds check and may or may not return an inexact representation. runnableExamples: - doAssert toNumber(big"2147483647") == 2147483647.BiggestInt + doAssert toNumber(big"2147483647") == 2147483647.int func `+`*(x, y: JsBigInt): JsBigInt {.importjs: "(# $1 #)".} = runnableExamples: @@ -110,7 +110,7 @@ func `==`*(x, y: JsBigInt): bool {.importjs: "(# == #)".} = doAssert big"42" == big"42" func `**`*(x, y: JsBigInt): JsBigInt {.importjs: "((#) $1 #)".} = - # (#) needed, refs https://github.com/nim-lang/Nim/pull/16409#issuecomment-760550812 + # (#) needed due to unary minus runnableExamples: doAssert big"2" ** big"64" == big"18446744073709551616" doAssert big"-2" ** big"3" == big"-8" @@ -120,8 +120,6 @@ func `**`*(x, y: JsBigInt): JsBigInt {.importjs: "((#) $1 #)".} = try: discard big"2" ** big"-1" # raises foreign `RangeError` except: ok = true doAssert ok - # pending https://github.com/nim-lang/Nim/pull/15940, simplify to: - # doAssertRaises: discard big"2" ** big"-1" # raises foreign `RangeError` func `and`*(x, y: JsBigInt): JsBigInt {.importjs: "(# & #)".} = runnableExamples: diff --git a/lib/std/jsfetch.nim b/lib/std/jsfetch.nim index 7fe154325..219594619 100644 --- a/lib/std/jsfetch.nim +++ b/lib/std/jsfetch.nim @@ -2,7 +2,8 @@ when not defined(js): {.fatal: "Module jsfetch is designed to be used with the JavaScript backend.".} -import std/[asyncjs, jsheaders, jsformdata] +import std/[asyncjs, jsformdata, jsheaders] +export jsformdata, jsheaders from std/httpcore import HttpMethod from std/jsffi import JsObject @@ -84,9 +85,9 @@ proc unsafeNewFetchOptions*(metod, body, mode, credentials, cache, referrerPolic "{method: #, body: #, mode: #, credentials: #, cache: #, referrerPolicy: #, keepalive: #, redirect: #, referrer: #, integrity: #, headers: #}".} ## .. warning:: Unsafe `newfetchOptions`. -func newfetchOptions*(metod: HttpMethod; body: cstring; - mode: FetchModes; credentials: FetchCredentials; cache: FetchCaches; referrerPolicy: FetchReferrerPolicies; - keepalive: bool; redirect = frFollow; referrer = "client".cstring; integrity = "".cstring, +func newfetchOptions*(metod = HttpGet; body: cstring = nil; + mode = fmCors; credentials = fcSameOrigin; cache = fchDefault; referrerPolicy = frpNoReferrerWhenDowngrade; + keepalive = false; redirect = frFollow; referrer = "client".cstring; integrity = "".cstring, headers: Headers = newHeaders()): FetchOptions = ## Constructor for `FetchOptions`. result = FetchOptions( @@ -116,7 +117,7 @@ func `$`*(self: Request | Response | FetchOptions): string = $toCstring(self) runnableExamples("-r:off"): - import std/[asyncjs, jsconsole, jsheaders, jsformdata] + import std/[asyncjs, jsconsole, jsformdata, jsheaders] from std/httpcore import HttpMethod from std/jsffi import JsObject from std/sugar import `=>` diff --git a/lib/std/jsformdata.nim b/lib/std/jsformdata.nim index 120f8742d..61dcc39a3 100644 --- a/lib/std/jsformdata.nim +++ b/lib/std/jsformdata.nim @@ -2,17 +2,21 @@ when not defined(js): {.fatal: "Module jsformdata is designed to be used with the JavaScript backend.".} +from std/dom import Blob + type FormData* = ref object of JsRoot ## FormData API. func newFormData*(): FormData {.importjs: "new FormData()".} -func add*(self: FormData; name: cstring; value: SomeNumber | bool | cstring) {.importjs: "#.append(#, #)".} +func add*(self: FormData; name: cstring; value: SomeNumber | bool | cstring | Blob) {.importjs: "#.append(#, #)".} ## https://developer.mozilla.org/en-US/docs/Web/API/FormData/append - ## Duplicate keys are allowed and order is preserved. + ## + ## .. hint:: Duplicate keys are allowed and order is preserved. -func add*(self: FormData; name: cstring; value: SomeNumber | bool | cstring, filename: cstring) {.importjs: "#.append(#, #, #)".} +func add*(self: FormData; name: cstring; value: SomeNumber | bool | cstring | Blob; filename: cstring) {.importjs: "#.append(#, #, #)".} ## https://developer.mozilla.org/en-US/docs/Web/API/FormData/append - ## Duplicate keys are allowed and order is preserved. + ## + ## .. hint:: Duplicate keys are allowed and order is preserved. func delete*(self: FormData; name: cstring) {.importjs: "#.$1(#)".} ## https://developer.mozilla.org/en-US/docs/Web/API/FormData/delete @@ -34,10 +38,10 @@ func values*(self: FormData): seq[cstring] {.importjs: "Array.from(#.$1())".} func pairs*(self: FormData): seq[tuple[key, val: cstring]] {.importjs: "Array.from(#.entries())".} ## https://developer.mozilla.org/en-US/docs/Web/API/FormData/entries -func put*(self: FormData; name, value, filename: cstring) {.importjs: "#.set(#, #, #)".} +func put*(self: FormData; name: cstring; value: SomeNumber | bool | cstring | Blob; filename: cstring) {.importjs: "#.set(#, #, #)".} ## https://developer.mozilla.org/en-US/docs/Web/API/FormData/set -func `[]=`*(self: FormData; name, value: cstring) {.importjs: "#.set(#, #)".} +func `[]=`*(self: FormData; name: cstring; value: SomeNumber | bool | cstring | Blob) {.importjs: "#.set(#, #)".} ## https://developer.mozilla.org/en-US/docs/Web/API/FormData/set func `[]`*(self: FormData; name: cstring): cstring {.importjs: "#.get(#)".} diff --git a/lib/std/jsonutils.nim b/lib/std/jsonutils.nim index 722ea49b5..2d28748ce 100644 --- a/lib/std/jsonutils.nim +++ b/lib/std/jsonutils.nim @@ -1,5 +1,5 @@ ##[ -This module implements a hookable (de)serialization for arbitrary types. +This module implements a hookable (de)serialization for arbitrary types using JSON. Design goal: avoid importing modules where a custom serialization is needed; see strtabs.fromJsonHook,toJsonHook for an example. ]## @@ -16,7 +16,7 @@ runnableExamples: assert 0.0.toJson.kind == JFloat assert Inf.toJson.kind == JString -import json, strutils, tables, sets, strtabs, options +import std/[json, strutils, tables, sets, strtabs, options, strformat] #[ Future directions: @@ -30,29 +30,15 @@ add a way to customize serialization, for e.g.: objects. ]# -import macros -from enumutils import symbolName -from typetraits import OrdinalEnum +import std/macros +from std/enumutils import symbolName +from std/typetraits import OrdinalEnum, tupleLen when defined(nimPreviewSlimSystem): import std/assertions -when not defined(nimFixedForwardGeneric): - # xxx remove pending csources_v1 update >= 1.2.0 - proc to[T](node: JsonNode, t: typedesc[T]): T = - when T is string: node.getStr - elif T is bool: node.getBool - else: static: doAssert false, $T # support as needed (only needed during bootstrap) - proc isNamedTuple(T: typedesc): bool = # old implementation - when T isnot tuple: result = false - else: - var t: T - for name, _ in t.fieldPairs: - when name == "Field0": return compiles(t.Field0) - else: return true - return false -else: - proc isNamedTuple(T: typedesc): bool {.magic: "TypeTrait".} + +proc isNamedTuple(T: typedesc): bool {.magic: "TypeTrait".} type Joptions* = object # xxx rename FromJsonOptions @@ -92,19 +78,24 @@ macro getDiscriminants(a: typedesc): seq[string] = let sym = a[1] let t = sym.getTypeImpl let t2 = t[2] - doAssert t2.kind == nnkRecList - result = newTree(nnkBracket) - for ti in t2: - if ti.kind == nnkRecCase: - let key = ti[0][0] - let typ = ti[0][1] - result.add newLit key.strVal - if result.len > 0: + case t2.kind + of nnkEmpty: # allow empty objects result = quote do: - @`result` + seq[string].default + of nnkRecList: + result = newTree(nnkBracket) + for ti in t2: + if ti.kind == nnkRecCase: + let key = ti[0][0] + result.add newLit key.strVal + if result.len > 0: + result = quote do: + @`result` + else: + result = quote do: + seq[string].default else: - result = quote do: - seq[string].default + raiseAssert "unexpected kind: " & $t2.kind macro initCaseObject(T: typedesc, fun: untyped): untyped = ## does the minimum to construct a valid case object, only initializing @@ -118,7 +109,7 @@ macro initCaseObject(T: typedesc, fun: untyped): untyped = case t.kind of nnkObjectTy: t2 = t[2] of nnkRefTy: t2 = t[0].getTypeImpl[2] - else: doAssert false, $t.kind # xxx `nnkPtrTy` could be handled too + else: raiseAssert $t.kind # xxx `nnkPtrTy` could be handled too doAssert t2.kind == nnkRecList result = newTree(nnkObjConstr) result.add sym @@ -168,7 +159,7 @@ template fromJsonFields(newObj, oldObj, json, discKeys, opt) = if discKeys.len == 0 or hasField(oldObj, key): val = accessField(oldObj, key) else: - checkJson false, $($T, key, json) + checkJson false, "key '$1' for $2 not in $3" % [key, $T, json.pretty()] else: if json.hasKey key: numMatched.inc @@ -187,7 +178,7 @@ template fromJsonFields(newObj, oldObj, json, discKeys, opt) = else: json.len == num and num == numMatched - checkJson ok, $(json.len, num, numMatched, $T, json) + checkJson ok, "There were $1 keys (expecting $2) for $3 with $4" % [$json.len, $num, $T, json.pretty()] proc fromJson*[T](a: var T, b: JsonNode, opt = Joptions()) @@ -220,28 +211,24 @@ proc fromJson*[T](a: var T, b: JsonNode, opt = Joptions()) = adding "json path" leading to `b` can be added in future work. ]# checkJson b != nil, $($T, b) - when compiles(fromJsonHook(a, b)): fromJsonHook(a, b) + when compiles(fromJsonHook(a, b, opt)): fromJsonHook(a, b, opt) + elif compiles(fromJsonHook(a, b)): fromJsonHook(a, b) elif T is bool: a = to(b,T) elif T is enum: case b.kind of JInt: a = T(b.getBiggestInt()) of JString: a = parseEnum[T](b.getStr()) - else: checkJson false, $($T, " ", b) + else: checkJson false, fmt"Expecting int/string for {$T} got {b.pretty()}" elif T is uint|uint64: a = T(to(b, uint64)) elif T is Ordinal: a = cast[T](to(b, int)) elif T is pointer: a = cast[pointer](to(b, int)) - elif T is distinct: - when nimvm: - # bug, potentially related to https://github.com/nim-lang/Nim/issues/12282 - a = T(jsonTo(b, distinctBase(T))) - else: - a.distinctBase.fromJson(b) + elif T is distinct: a.distinctBase.fromJson(b) elif T is string|SomeNumber: a = to(b,T) elif T is cstring: case b.kind of JNull: a = nil of JString: a = b.str - else: checkJson false, $($T, " ", b) + else: checkJson false, fmt"Expecting null/string for {$T} got {b.pretty()}" elif T is JsonNode: a = b elif T is ref | ptr: if b.kind == JNull: a = nil @@ -249,7 +236,7 @@ proc fromJson*[T](a: var T, b: JsonNode, opt = Joptions()) = a = T() fromJson(a[], b, opt) elif T is array: - checkJson a.len == b.len, $(a.len, b.len, $T) + checkJson a.len == b.len, fmt"Json array size doesn't match for {$T}" var i = 0 for ai in mitems(a): fromJson(ai, b[i], opt) @@ -285,14 +272,24 @@ proc fromJson*[T](a: var T, b: JsonNode, opt = Joptions()) = fromJsonFields(a, nil, b, seq[string].default, opt) else: checkJson b.kind == JArray, $(b.kind) # we could customize whether to allow JNull + + when compiles(tupleLen(T)): + let tupleSize = tupleLen(T) + else: + # Tuple len isn't in csources_v1 so using tupleLen would fail. + # Else branch basically never runs (tupleLen added in 1.1 and jsonutils in 1.4), but here for consistency + var tupleSize = 0 + for val in fields(a): + tupleSize.inc + + checkJson b.len == tupleSize, fmt"Json doesn't match expected length of {tupleSize}, got {b.pretty()}" var i = 0 for val in fields(a): fromJson(val, b[i], opt) i.inc - checkJson b.len == i, $(b.len, i, $T, b) # could customize else: # checkJson not appropriate here - static: doAssert false, "not yet implemented: " & $T + static: raiseAssert "not yet implemented: " & $T proc jsonTo*(b: JsonNode, T: typedesc, opt = Joptions()): T = ## reverse of `toJson` @@ -305,7 +302,8 @@ proc toJson*[T](a: T, opt = initToJsonOptions()): JsonNode = ## .. note:: With `-d:nimPreviewJsonutilsHoleyEnum`, `toJson` now can ## serialize/deserialize holey enums as regular enums (via `ord`) instead of as strings. ## It is expected that this behavior becomes the new default in upcoming versions. - when compiles(toJsonHook(a)): result = toJsonHook(a) + when compiles(toJsonHook(a, opt)): result = toJsonHook(a, opt) + elif compiles(toJsonHook(a)): result = toJsonHook(a) elif T is object | tuple: when T is object or isNamedTuple(T): result = newJObject() @@ -348,7 +346,7 @@ proc toJson*[T](a: T, opt = initToJsonOptions()): JsonNode = else: result = %a proc fromJsonHook*[K: string|cstring, V](t: var (Table[K, V] | OrderedTable[K, V]), - jsonNode: JsonNode) = + jsonNode: JsonNode, opt = Joptions()) = ## Enables `fromJson` for `Table` and `OrderedTable` types. ## ## See also: @@ -366,14 +364,13 @@ proc fromJsonHook*[K: string|cstring, V](t: var (Table[K, V] | OrderedTable[K, V "type is `" & $jsonNode.kind & "`." clear(t) for k, v in jsonNode: - t[k] = jsonTo(v, V) + t[k] = jsonTo(v, V, opt) -proc toJsonHook*[K: string|cstring, V](t: (Table[K, V] | OrderedTable[K, V])): JsonNode = +proc toJsonHook*[K: string|cstring, V](t: (Table[K, V] | OrderedTable[K, V]), opt = initToJsonOptions()): JsonNode = ## Enables `toJson` for `Table` and `OrderedTable` types. ## ## See also: ## * `fromJsonHook proc<#fromJsonHook,,JsonNode>`_ - # pending PR #9217 use: toSeq(a) instead of `collect` in `runnableExamples`. runnableExamples: import std/[tables, json, sugar] let foo = ( @@ -388,9 +385,9 @@ proc toJsonHook*[K: string|cstring, V](t: (Table[K, V] | OrderedTable[K, V])): J result = newJObject() for k, v in pairs(t): # not sure if $k has overhead for string - result[(when K is string: k else: $k)] = toJson(v) + result[(when K is string: k else: $k)] = toJson(v, opt) -proc fromJsonHook*[A](s: var SomeSet[A], jsonNode: JsonNode) = +proc fromJsonHook*[A](s: var SomeSet[A], jsonNode: JsonNode, opt = Joptions()) = ## Enables `fromJson` for `HashSet` and `OrderedSet` types. ## ## See also: @@ -408,9 +405,9 @@ proc fromJsonHook*[A](s: var SomeSet[A], jsonNode: JsonNode) = "type is `" & $jsonNode.kind & "`." clear(s) for v in jsonNode: - incl(s, jsonTo(v, A)) + incl(s, jsonTo(v, A, opt)) -proc toJsonHook*[A](s: SomeSet[A]): JsonNode = +proc toJsonHook*[A](s: SomeSet[A], opt = initToJsonOptions()): JsonNode = ## Enables `toJson` for `HashSet` and `OrderedSet` types. ## ## See also: @@ -422,9 +419,9 @@ proc toJsonHook*[A](s: SomeSet[A]): JsonNode = result = newJArray() for k in s: - add(result, toJson(k)) + add(result, toJson(k, opt)) -proc fromJsonHook*[T](self: var Option[T], jsonNode: JsonNode) = +proc fromJsonHook*[T](self: var Option[T], jsonNode: JsonNode, opt = Joptions()) = ## Enables `fromJson` for `Option` types. ## ## See also: @@ -438,11 +435,11 @@ proc fromJsonHook*[T](self: var Option[T], jsonNode: JsonNode) = assert isNone(opt) if jsonNode.kind != JNull: - self = some(jsonTo(jsonNode, T)) + self = some(jsonTo(jsonNode, T, opt)) else: self = none[T]() -proc toJsonHook*[T](self: Option[T]): JsonNode = +proc toJsonHook*[T](self: Option[T], opt = initToJsonOptions()): JsonNode = ## Enables `toJson` for `Option` types. ## ## See also: @@ -455,7 +452,7 @@ proc toJsonHook*[T](self: Option[T]): JsonNode = assert $toJson(optNone) == "null" if isSome(self): - toJson(get(self)) + toJson(get(self), opt) else: newJNull() diff --git a/lib/std/monotimes.nim b/lib/std/monotimes.nim index 5c67a5d4c..bf6dc776b 100644 --- a/lib/std/monotimes.nim +++ b/lib/std/monotimes.nim @@ -36,7 +36,7 @@ See also * `times module <times.html>`_ ]## -import times +import std/times type MonoTime* = object ## Represents a monotonic timestamp. @@ -74,7 +74,7 @@ when defined(js): {.pop.} elif defined(posix) and not defined(osx): - import posix + import std/posix when defined(zephyr): proc k_uptime_ticks(): int64 {.importc: "k_uptime_ticks", header: "<kernel.h>".} diff --git a/lib/std/objectdollar.nim b/lib/std/objectdollar.nim index f413bbc46..86ce9afc8 100644 --- a/lib/std/objectdollar.nim +++ b/lib/std/objectdollar.nim @@ -1,3 +1,5 @@ +## This module implements a generic `$` operator to convert objects to strings. + import std/private/miscdollars proc `$`*[T: object](x: T): string = diff --git a/lib/std/oserrors.nim b/lib/std/oserrors.nim index 9c2649eab..7b11c5e8e 100644 --- a/lib/std/oserrors.nim +++ b/lib/std/oserrors.nim @@ -15,7 +15,9 @@ type when not defined(nimscript): when defined(windows): - import winlean + import std/winlean + when defined(nimPreviewSlimSystem): + import std/widestrs else: var errno {.importc, header: "<errno.h>".}: cint @@ -73,17 +75,14 @@ proc newOSError*( ## See also: ## * `osErrorMsg proc`_ ## * `osLastError proc`_ - var e: owned(ref OSError); new(e) - e.errorCode = errorCode.int32 - e.msg = osErrorMsg(errorCode) + result = (ref OSError)(errorCode: errorCode.int32, msg: osErrorMsg(errorCode)) if additionalInfo.len > 0: - if e.msg.len > 0 and e.msg[^1] != '\n': e.msg.add '\n' - e.msg.add "Additional info: " - e.msg.add additionalInfo + if result.msg.len > 0 and result.msg[^1] != '\n': result.msg.add '\n' + result.msg.add "Additional info: " + result.msg.add additionalInfo # don't add trailing `.` etc, which negatively impacts "jump to file" in IDEs. - if e.msg == "": - e.msg = "unknown OS error" - return e + if result.msg == "": + result.msg = "unknown OS error" proc raiseOSError*(errorCode: OSErrorCode, additionalInfo = "") {.noinline.} = ## Raises an `OSError exception <system.html#OSError>`_. diff --git a/lib/std/outparams.nim b/lib/std/outparams.nim new file mode 100644 index 000000000..a471fbaa7 --- /dev/null +++ b/lib/std/outparams.nim @@ -0,0 +1,38 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2022 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## `outParamsAt` macro for easy writing code that works with both 2.0 and 1.x. + +import std/macros + +macro outParamsAt*(positions: static openArray[int]; n: untyped): untyped = + ## Use this macro to annotate `out` parameters in a portable way. + runnableExamples: + proc p(x: var int) {.outParamsAt: [1].} = + discard "x is really an 'out int' if the Nim compiler supports 'out' parameters" + + result = n + when defined(nimHasOutParams): + var p = n.params + for po in positions: + p[po][^2].expectKind nnkVarTy + p[po][^2] = newTree(nnkOutTy, p[po][^2][0]) + +when isMainModule: + {.experimental: "strictDefs".} + + proc main(x: var int) {.outParamsAt: [1].} = + x = 3 + + proc us = + var x: int + main x + echo x + + us() diff --git a/lib/std/packedsets.nim b/lib/std/packedsets.nim index 1e2892658..3320558f2 100644 --- a/lib/std/packedsets.nim +++ b/lib/std/packedsets.nim @@ -12,17 +12,12 @@ ## ## Supports any Ordinal type. ## -## .. note:: Currently the assignment operator `=` for `PackedSet[A]` -## performs some rather meaningless shallow copy. Since Nim currently does -## not allow the assignment operator to be overloaded, use the `assign proc -## <#assign,PackedSet[A],PackedSet[A]>`_ to get a deep copy. -## ## See also ## ======== ## * `sets module <sets.html>`_ for more general hash sets import std/private/since -import hashes +import std/hashes when defined(nimPreviewSlimSystem): import std/assertions @@ -114,7 +109,6 @@ proc intSetPut[A](t: var PackedSet[A], key: int): Trunk = t.data[h] = result proc bitincl[A](s: var PackedSet[A], key: int) {.inline.} = - var ret: Trunk var t = intSetPut(s, key shr TrunkShift) var u = key and TrunkMask t.bits[u shr IntShift] = t.bits[u shr IntShift] or @@ -204,6 +198,7 @@ proc contains*[A](s: PackedSet[A], key: A): bool = assert B notin letters if s.elems <= s.a.len: + result = false for i in 0..<s.elems: if s.a[i] == ord(key): return true else: @@ -412,18 +407,9 @@ proc isNil*[A](x: PackedSet[A]): bool {.inline.} = x.head.isNil and x.elems == 0 -proc assign*[A](dest: var PackedSet[A], src: PackedSet[A]) = +proc `=copy`*[A](dest: var PackedSet[A], src: PackedSet[A]) = ## Copies `src` to `dest`. ## `dest` does not need to be initialized by the `initPackedSet proc <#initPackedSet>`_. - runnableExamples: - var - a = initPackedSet[int]() - b = initPackedSet[int]() - b.incl(5) - b.incl(7) - a.assign(b) - assert len(a) == 2 - if src.elems <= src.a.len: dest.data = @[] dest.max = 0 @@ -452,6 +438,19 @@ proc assign*[A](dest: var PackedSet[A], src: PackedSet[A]) = dest.data[h] = n it = it.next +proc assign*[A](dest: var PackedSet[A], src: PackedSet[A]) {.inline, deprecated.} = + ## Copies `src` to `dest`. + ## `dest` does not need to be initialized by the `initPackedSet proc <#initPackedSet>`_. + runnableExamples: + var + a = initPackedSet[int]() + b = initPackedSet[int]() + b.incl(5) + b.incl(7) + a.assign(b) + assert len(a) == 2 + `=copy`(dest, src) + proc union*[A](s1, s2: PackedSet[A]): PackedSet[A] = ## Returns the union of the sets `s1` and `s2`. ## diff --git a/lib/std/paths.nim b/lib/std/paths.nim new file mode 100644 index 000000000..664dedd31 --- /dev/null +++ b/lib/std/paths.nim @@ -0,0 +1,302 @@ +## This module implements path handling. +## +## **See also:** +## * `files module <files.html>`_ for file access + +import std/private/osseps +export osseps + +import std/envvars +import std/private/osappdirs + +import std/[pathnorm, hashes, sugar, strutils] + +from std/private/ospaths2 import joinPath, splitPath, + ReadDirEffect, WriteDirEffect, + isAbsolute, relativePath, + normalizePathEnd, isRelativeTo, parentDir, + tailDir, isRootDir, parentDirs, `/../`, + extractFilename, lastPathPart, + changeFileExt, addFileExt, cmpPaths, splitFile, + unixToNativePath, absolutePath, normalizeExe, + normalizePath +export ReadDirEffect, WriteDirEffect + +type + Path* = distinct string + +func hash*(x: Path): Hash = + let x = x.string.dup(normalizePath) + if FileSystemCaseSensitive: + result = x.hash + else: + result = x.toLowerAscii.hash + +template `$`*(x: Path): string = + string(x) + +func `==`*(x, y: Path): bool {.inline.} = + ## Compares two paths. + ## + ## On a case-sensitive filesystem this is done + ## case-sensitively otherwise case-insensitively. + result = cmpPaths(x.string, y.string) == 0 + +template endsWith(a: string, b: set[char]): bool = + a.len > 0 and a[^1] in b + +func add(x: var string, tail: string) = + var state = 0 + let trailingSep = tail.endsWith({DirSep, AltSep}) or tail.len == 0 and x.endsWith({DirSep, AltSep}) + normalizePathEnd(x, trailingSep=false) + addNormalizePath(tail, x, state, DirSep) + normalizePathEnd(x, trailingSep=trailingSep) + +func add*(x: var Path, y: Path) {.borrow.} + +func `/`*(head, tail: Path): Path {.inline.} = + ## Joins two directory names to one. + ## + ## returns normalized path concatenation of `head` and `tail`, preserving + ## whether or not `tail` has a trailing slash (or, if tail if empty, whether + ## head has one). + ## + ## See also: + ## * `splitPath proc`_ + ## * `uri.combine proc <uri.html#combine,Uri,Uri>`_ + ## * `uri./ proc <uri.html#/,Uri,string>`_ + Path(joinPath(head.string, tail.string)) + +func splitPath*(path: Path): tuple[head, tail: Path] {.inline.} = + ## Splits a directory into `(head, tail)` tuple, so that + ## ``head / tail == path`` (except for edge cases like "/usr"). + ## + ## See also: + ## * `add proc`_ + ## * `/ proc`_ + ## * `/../ proc`_ + ## * `relativePath proc`_ + let res = splitPath(path.string) + result = (Path(res.head), Path(res.tail)) + +func splitFile*(path: Path): tuple[dir, name: Path, ext: string] {.inline.} = + ## Splits a filename into `(dir, name, extension)` tuple. + ## + ## `dir` does not end in DirSep unless it's `/`. + ## `extension` includes the leading dot. + ## + ## If `path` has no extension, `ext` is the empty string. + ## If `path` has no directory component, `dir` is the empty string. + ## If `path` has no filename component, `name` and `ext` are empty strings. + ## + ## See also: + ## * `extractFilename proc`_ + ## * `lastPathPart proc`_ + ## * `changeFileExt proc`_ + ## * `addFileExt proc`_ + let res = splitFile(path.string) + result = (Path(res.dir), Path(res.name), res.ext) + +func isAbsolute*(path: Path): bool {.inline, raises: [].} = + ## Checks whether a given `path` is absolute. + ## + ## On Windows, network paths are considered absolute too. + result = isAbsolute(path.string) + +proc relativePath*(path, base: Path, sep = DirSep): Path {.inline.} = + ## Converts `path` to a path relative to `base`. + ## + ## The `sep` (default: DirSep) is used for the path normalizations, + ## this can be useful to ensure the relative path only contains `'/'` + ## so that it can be used for URL constructions. + ## + ## On Windows, if a root of `path` and a root of `base` are different, + ## returns `path` as is because it is impossible to make a relative path. + ## That means an absolute path can be returned. + ## + ## See also: + ## * `splitPath proc`_ + ## * `parentDir proc`_ + ## * `tailDir proc`_ + result = Path(relativePath(path.string, base.string, sep)) + +proc isRelativeTo*(path: Path, base: Path): bool {.inline.} = + ## Returns true if `path` is relative to `base`. + result = isRelativeTo(path.string, base.string) + + +func parentDir*(path: Path): Path {.inline.} = + ## Returns the parent directory of `path`. + ## + ## This is similar to ``splitPath(path).head`` when ``path`` doesn't end + ## in a dir separator, but also takes care of path normalizations. + ## The remainder can be obtained with `lastPathPart(path) proc`_. + ## + ## See also: + ## * `relativePath proc`_ + ## * `splitPath proc`_ + ## * `tailDir proc`_ + ## * `parentDirs iterator`_ + result = Path(parentDir(path.string)) + +func tailDir*(path: Path): Path {.inline.} = + ## Returns the tail part of `path`. + ## + ## See also: + ## * `relativePath proc`_ + ## * `splitPath proc`_ + ## * `parentDir proc`_ + result = Path(tailDir(path.string)) + +func isRootDir*(path: Path): bool {.inline.} = + ## Checks whether a given `path` is a root directory. + result = isRootDir(path.string) + +iterator parentDirs*(path: Path, fromRoot=false, inclusive=true): Path = + ## Walks over all parent directories of a given `path`. + ## + ## If `fromRoot` is true (default: false), the traversal will start from + ## the file system root directory. + ## If `inclusive` is true (default), the original argument will be included + ## in the traversal. + ## + ## Relative paths won't be expanded by this iterator. Instead, it will traverse + ## only the directories appearing in the relative path. + ## + ## See also: + ## * `parentDir proc`_ + ## + for p in parentDirs(path.string, fromRoot, inclusive): + yield Path(p) + +func `/../`*(head, tail: Path): Path {.inline.} = + ## The same as ``parentDir(head) / tail``, unless there is no parent + ## directory. Then ``head / tail`` is performed instead. + ## + ## See also: + ## * `/ proc`_ + ## * `parentDir proc`_ + Path(`/../`(head.string, tail.string)) + +func extractFilename*(path: Path): Path {.inline.} = + ## Extracts the filename of a given `path`. + ## + ## This is the same as ``name & ext`` from `splitFile(path) proc`_. + ## + ## See also: + ## * `splitFile proc`_ + ## * `lastPathPart proc`_ + ## * `changeFileExt proc`_ + ## * `addFileExt proc`_ + result = Path(extractFilename(path.string)) + +func lastPathPart*(path: Path): Path {.inline.} = + ## Like `extractFilename proc`_, but ignores + ## trailing dir separator; aka: `baseName`:idx: in some other languages. + ## + ## See also: + ## * `splitFile proc`_ + ## * `extractFilename proc`_ + ## * `changeFileExt proc`_ + ## * `addFileExt proc`_ + result = Path(lastPathPart(path.string)) + +func changeFileExt*(filename: Path, ext: string): Path {.inline.} = + ## Changes the file extension to `ext`. + ## + ## If the `filename` has no extension, `ext` will be added. + ## If `ext` == "" then any extension is removed. + ## + ## `Ext` should be given without the leading `'.'`, because some + ## filesystems may use a different character. (Although I know + ## of none such beast.) + ## + ## See also: + ## * `splitFile proc`_ + ## * `extractFilename proc`_ + ## * `lastPathPart proc`_ + ## * `addFileExt proc`_ + result = Path(changeFileExt(filename.string, ext)) + +func addFileExt*(filename: Path, ext: string): Path {.inline.} = + ## Adds the file extension `ext` to `filename`, unless + ## `filename` already has an extension. + ## + ## `Ext` should be given without the leading `'.'`, because some + ## filesystems may use a different character. + ## (Although I know of none such beast.) + ## + ## See also: + ## * `splitFile proc`_ + ## * `extractFilename proc`_ + ## * `lastPathPart proc`_ + ## * `changeFileExt proc`_ + result = Path(addFileExt(filename.string, ext)) + +func unixToNativePath*(path: Path, drive=Path("")): Path {.inline.} = + ## Converts an UNIX-like path to a native one. + ## + ## On an UNIX system this does nothing. Else it converts + ## `'/'`, `'.'`, `'..'` to the appropriate things. + ## + ## On systems with a concept of "drives", `drive` is used to determine + ## which drive label to use during absolute path conversion. + ## `drive` defaults to the drive of the current working directory, and is + ## ignored on systems that do not have a concept of "drives". + result = Path(unixToNativePath(path.string, drive.string)) + +proc getCurrentDir*(): Path {.inline, tags: [].} = + ## Returns the `current working directory`:idx: i.e. where the built + ## binary is run. + ## + ## So the path returned by this proc is determined at run time. + ## + ## See also: + ## * `getHomeDir proc <appdirs.html#getHomeDir>`_ + ## * `getConfigDir proc <appdirs.html#getConfigDir>`_ + ## * `getTempDir proc <appdirs.html#getTempDir>`_ + ## * `setCurrentDir proc <dirs.html#setCurrentDir>`_ + ## * `currentSourcePath template <system.html#currentSourcePath.t>`_ + ## * `getProjectPath proc <macros.html#getProjectPath>`_ + result = Path(ospaths2.getCurrentDir()) + +proc normalizeExe*(file: var Path) {.borrow.} + +proc normalizePath*(path: var Path) {.borrow.} + +proc normalizePathEnd*(path: var Path, trailingSep = false) {.borrow.} + +proc absolutePath*(path: Path, root = getCurrentDir()): Path = + ## Returns the absolute path of `path`, rooted at `root` (which must be absolute; + ## default: current directory). + ## If `path` is absolute, return it, ignoring `root`. + ## + ## See also: + ## * `normalizePath proc`_ + result = Path(absolutePath(path.string, root.string)) + +proc expandTildeImpl(path: string): string {. + tags: [ReadEnvEffect, ReadIOEffect].} = + if len(path) == 0 or path[0] != '~': + result = path + elif len(path) == 1: + result = getHomeDir() + elif (path[1] in {DirSep, AltSep}): + result = joinPath(getHomeDir(), path.substr(2)) + else: + # TODO: handle `~bob` and `~bob/` which means home of bob + result = path + +proc expandTilde*(path: Path): Path {.inline, + tags: [ReadEnvEffect, ReadIOEffect].} = + ## Expands ``~`` or a path starting with ``~/`` to a full path, replacing + ## ``~`` with `getHomeDir() <appdirs.html#getHomeDir>`_ (otherwise returns ``path`` unmodified). + ## + ## Windows: this is still supported despite the Windows platform not having this + ## convention; also, both ``~/`` and ``~\`` are handled. + runnableExamples: + import std/appdirs + assert expandTilde(Path("~") / Path("appname.cfg")) == getHomeDir() / Path("appname.cfg") + assert expandTilde(Path("~/foo/bar")) == getHomeDir() / Path("foo/bar") + assert expandTilde(Path("/foo/bar")) == Path("/foo/bar") + result = Path(expandTildeImpl(path.string)) diff --git a/lib/std/private/dbutils.nim b/lib/std/private/dbutils.nim deleted file mode 100644 index 0ae3b3702..000000000 --- a/lib/std/private/dbutils.nim +++ /dev/null @@ -1,15 +0,0 @@ -import db_common - - -template dbFormatImpl*(formatstr: SqlQuery, dbQuote: proc (s: string): string, args: varargs[string]): string = - var res = "" - var a = 0 - for c in items(string(formatstr)): - if c == '?': - if a == args.len: - dbError("""The number of "?" given exceeds the number of parameters present in the query.""") - add(res, dbQuote(args[a])) - inc(a) - else: - add(res, c) - res diff --git a/lib/std/private/digitsutils.nim b/lib/std/private/digitsutils.nim index 588bcaec0..f2d0d25cb 100644 --- a/lib/std/private/digitsutils.nim +++ b/lib/std/private/digitsutils.nim @@ -19,7 +19,7 @@ const # Inspired by https://engineering.fb.com/2013/03/15/developer-tools/three-optimization-tips-for-c # Generates: -# .. code-block:: nim +# ```nim # var res = "" # for i in 0 .. 99: # if i < 10: @@ -27,14 +27,15 @@ const # else: # res.add $i # doAssert res == digits100 +# ``` proc utoa2Digits*(buf: var openArray[char]; pos: int; digits: uint32) {.inline.} = buf[pos] = digits100[2 * digits] buf[pos+1] = digits100[2 * digits + 1] #copyMem(buf, unsafeAddr(digits100[2 * digits]), 2 * sizeof((char))) -proc trailingZeros2Digits*(digits: uint32): int32 {.inline.} = - return trailingZeros100[digits] +proc trailingZeros2Digits*(digits: uint32): int {.inline.} = + trailingZeros100[digits] when defined(js): proc numToString(a: SomeInteger): cstring {.importjs: "((#) + \"\")".} @@ -63,14 +64,14 @@ func addIntImpl(result: var string, x: uint64) {.inline.} = while num >= nbatch: let originNum = num num = num div nbatch - let index = (originNum - num * nbatch) shl 1 + let index = int16((originNum - num * nbatch) shl 1) tmp[next] = digits100[index + 1] tmp[next - 1] = digits100[index] dec(next, 2) # process last 1-2 digits if num < 10: - tmp[next] = chr(ord('0') + num) + tmp[next] = chr(ord('0') + num.uint8) else: let index = num * 2 tmp[next] = digits100[index + 1] @@ -101,9 +102,7 @@ proc addInt*(result: var string; x: int64) {.enforceNoRaises.} = num = cast[uint64](x) else: num = uint64(-x) - let base = result.len - setLen(result, base + 1) - result[base] = '-' + result.add '-' else: num = uint64(x) addInt(result, num) diff --git a/lib/std/private/dragonbox.nim b/lib/std/private/dragonbox.nim index 23adff385..85ffea84a 100644 --- a/lib/std/private/dragonbox.nim +++ b/lib/std/private/dragonbox.nim @@ -75,10 +75,10 @@ const const signMask*: BitsType = not (not BitsType(0) shr 1) -proc constructDouble*(bits: BitsType): Double {.constructor.} = +proc constructDouble*(bits: BitsType): Double = result.bits = bits -proc constructDouble*(value: ValueType): Double {.constructor.} = +proc constructDouble*(value: ValueType): Double = result.bits = cast[typeof(result.bits)](value) proc physicalSignificand*(this: Double): BitsType {.noSideEffect.} = @@ -1052,7 +1052,7 @@ when false: proc memset(x: cstring; ch: char; L: int) {.importc, nodecl.} proc memmove(a, b: cstring; L: int) {.importc, nodecl.} -proc utoa8DigitsSkipTrailingZeros*(buf: var openArray[char]; pos: int; digits: uint32): int32 {.inline.} = +proc utoa8DigitsSkipTrailingZeros*(buf: var openArray[char]; pos: int; digits: uint32): int {.inline.} = dragonbox_Assert(digits >= 1) dragonbox_Assert(digits <= 99999999'u32) let q: uint32 = digits div 10000 @@ -1070,12 +1070,12 @@ proc utoa8DigitsSkipTrailingZeros*(buf: var openArray[char]; pos: int; digits: u utoa2Digits(buf, pos + 6, rL) return trailingZeros2Digits(if rL == 0: rH else: rL) + (if rL == 0: 2 else: 0) -proc printDecimalDigitsBackwards*(buf: var openArray[char]; pos: int; output64: uint64): int32 {.inline.} = +proc printDecimalDigitsBackwards*(buf: var openArray[char]; pos: int; output64: uint64): int {.inline.} = var pos = pos var output64 = output64 - var tz: int32 = 0 + var tz = 0 ## number of trailing zeros removed. - var nd: int32 = 0 + var nd = 0 ## number of decimal digits processed. ## At most 17 digits remaining if output64 >= 100000000'u64: @@ -1146,7 +1146,7 @@ proc printDecimalDigitsBackwards*(buf: var openArray[char]; pos: int; output64: buf[pos] = chr(ord('0') + q) return tz -proc decimalLength*(v: uint64): int32 {.inline.} = +proc decimalLength*(v: uint64): int {.inline.} = dragonbox_Assert(v >= 1) dragonbox_Assert(v <= 99999999999999999'u64) if cast[uint32](v shr 32) != 0: @@ -1166,48 +1166,48 @@ proc decimalLength*(v: uint64): int32 {.inline.} = return 11 return 10 let v32: uint32 = cast[uint32](v) - if v32 >= 1000000000'u: + if v32 >= 1000000000'u32: return 10 - if v32 >= 100000000'u: + if v32 >= 100000000'u32: return 9 - if v32 >= 10000000'u: + if v32 >= 10000000'u32: return 8 - if v32 >= 1000000'u: + if v32 >= 1000000'u32: return 7 - if v32 >= 100000'u: + if v32 >= 100000'u32: return 6 - if v32 >= 10000'u: + if v32 >= 10000'u32: return 5 - if v32 >= 1000'u: + if v32 >= 1000'u32: return 4 - if v32 >= 100'u: + if v32 >= 100'u32: return 3 - if v32 >= 10'u: + if v32 >= 10'u32: return 2 return 1 -proc formatDigits*(buffer: var openArray[char]; pos: int; digits: uint64; decimalExponent: int32; +proc formatDigits*[T: Ordinal](buffer: var openArray[char]; pos: T; digits: uint64; decimalExponent: int; forceTrailingDotZero = false): int {.inline.} = const - minFixedDecimalPoint: int32 = -6 + minFixedDecimalPoint = -6 const - maxFixedDecimalPoint: int32 = 17 - var pos = pos + maxFixedDecimalPoint = 17 + var pos:int = pos.int assert(minFixedDecimalPoint <= -1, "internal error") assert(maxFixedDecimalPoint >= 17, "internal error") dragonbox_Assert(digits >= 1) dragonbox_Assert(digits <= 99999999999999999'u64) dragonbox_Assert(decimalExponent >= -999) dragonbox_Assert(decimalExponent <= 999) - var numDigits: int32 = decimalLength(digits) - let decimalPoint: int32 = numDigits + decimalExponent + var numDigits = decimalLength(digits) + let decimalPoint = numDigits + decimalExponent let useFixed: bool = minFixedDecimalPoint <= decimalPoint and decimalPoint <= maxFixedDecimalPoint ## Prepare the buffer. for i in 0..<32: buffer[pos+i] = '0' assert(minFixedDecimalPoint >= -30, "internal error") assert(maxFixedDecimalPoint <= 32, "internal error") - var decimalDigitsPosition: int32 + var decimalDigitsPosition: int if useFixed: if decimalPoint <= 0: ## 0.[000]digits @@ -1220,7 +1220,7 @@ proc formatDigits*(buffer: var openArray[char]; pos: int; digits: uint64; decima ## dE+123 or d.igitsE+123 decimalDigitsPosition = 1 var digitsEnd = pos + int(decimalDigitsPosition + numDigits) - let tz: int32 = printDecimalDigitsBackwards(buffer, digitsEnd, digits) + let tz = printDecimalDigitsBackwards(buffer, digitsEnd, digits) dec(digitsEnd, tz) dec(numDigits, tz) ## decimal_exponent += tz; // => decimal_point unchanged. @@ -1258,7 +1258,7 @@ proc formatDigits*(buffer: var openArray[char]; pos: int; digits: uint64; decima ## d.igitsE+123 buffer[pos+1] = '.' pos = digitsEnd - let scientificExponent: int32 = decimalPoint - 1 + let scientificExponent: int = decimalPoint - 1 ## SF_ASSERT(scientific_exponent != 0); buffer[pos] = 'e' buffer[pos+1] = if scientificExponent < 0: '-' else: '+' @@ -1291,7 +1291,7 @@ proc toChars*(buffer: var openArray[char]; v: float; forceTrailingDotZero = fals if exponent != 0 or significand != 0: ## != 0 let dec = toDecimal64(significand, exponent) - return formatDigits(buffer, pos, dec.significand, dec.exponent, + return formatDigits(buffer, pos, dec.significand, dec.exponent.int, forceTrailingDotZero) else: buffer[pos] = '0' diff --git a/lib/std/private/gitutils.nim b/lib/std/private/gitutils.nim index 5bcd9e377..6dc9c8f3b 100644 --- a/lib/std/private/gitutils.nim +++ b/lib/std/private/gitutils.nim @@ -4,7 +4,10 @@ internal API for now, API subject to change # xxx move other git utilities here; candidate for stdlib. -import std/[os, osproc, strutils, tempfiles] +import std/[os, paths, osproc, strutils, tempfiles] + +when defined(nimPreviewSlimSystem): + import std/[assertions, syncio] const commitHead* = "HEAD" @@ -29,15 +32,8 @@ template retryCall*(maxRetry = 3, backoffDuration = 1.0, call: untyped): bool = result proc isGitRepo*(dir: string): bool = - ## This command is used to get the relative path to the root of the repository. - ## Using this, we can verify whether a folder is a git repository by checking - ## whether the command success and if the output is empty. - let (output, status) = execCmdEx("git rev-parse --show-cdup", workingDir = dir) - # On Windows there will be a trailing newline on success, remove it. - # The value of a successful call typically won't have a whitespace (it's - # usually a series of ../), so we know that it's safe to unconditionally - # remove trailing whitespaces from the result. - result = status == 0 and output.strip() == "" + ## Avoid calling git since it depends on /bin/sh existing and fails in Nix. + return fileExists(dir/".git/HEAD") proc diffFiles*(path1, path2: string): tuple[output: string, same: bool] = ## Returns a human readable diff of files `path1`, `path2`, the exact form of diff --git a/lib/std/private/globs.nim b/lib/std/private/globs.nim index 28a810372..a6d088558 100644 --- a/lib/std/private/globs.nim +++ b/lib/std/private/globs.nim @@ -4,12 +4,12 @@ this can eventually be moved to std/os and `walkDirRec` can be implemented in te to avoid duplication ]## -import os +import std/os when defined(windows): - from strutils import replace + from std/strutils import replace when defined(nimPreviewSlimSystem): - import std/assertions + import std/[assertions, objectdollar] when defined(nimHasEffectsOf): @@ -60,11 +60,11 @@ proc nativeToUnixPath*(path: string): string = result[0] = '/' result[1] = path[0] if path.len > 2 and path[2] != '\\': - doAssert false, "paths like `C:foo` are currently unsupported, path: " & path + raiseAssert "paths like `C:foo` are currently unsupported, path: " & path when DirSep == '\\': result = replace(result, '\\', '/') when isMainModule: - import sugar + import std/sugar for a in walkDirRecFilter(".", follow = a=>a.path.lastPathPart notin ["nimcache", ".git", "csources_v1", "csources", "bin"]): echo a diff --git a/lib/std/private/jsutils.nim b/lib/std/private/jsutils.nim index 836b3512a..5f79eab27 100644 --- a/lib/std/private/jsutils.nim +++ b/lib/std/private/jsutils.nim @@ -37,13 +37,13 @@ when defined(js): let a = array[2, float64].default assert jsConstructorName(a) == "Float64Array" assert jsConstructorName(a.toJs) == "Float64Array" - asm """`result` = `a`.constructor.name""" + {.emit: """`result` = `a`.constructor.name;""".} proc hasJsBigInt*(): bool = - asm """`result` = typeof BigInt != 'undefined'""" + {.emit: """`result` = typeof BigInt != 'undefined';""".} proc hasBigUint64Array*(): bool = - asm """`result` = typeof BigUint64Array != 'undefined'""" + {.emit: """`result` = typeof BigUint64Array != 'undefined';""".} proc getProtoName*[T](a: T): cstring {.importjs: "Object.prototype.toString.call(#)".} = runnableExamples: @@ -79,5 +79,18 @@ when defined(js): assert not "123".toJs.isSafeInteger assert 123.isSafeInteger assert 123.toJs.isSafeInteger - assert 9007199254740991.toJs.isSafeInteger - assert not 9007199254740992.toJs.isSafeInteger + when false: + assert 9007199254740991.toJs.isSafeInteger + assert not 9007199254740992.toJs.isSafeInteger + +template whenJsNoBigInt64*(no64, yes64): untyped = + when defined(js): + when compiles(compileOption("jsbigint64")): + when compileOption("jsbigint64"): + yes64 + else: + no64 + else: + no64 + else: + no64 diff --git a/lib/std/private/miscdollars.nim b/lib/std/private/miscdollars.nim index 47b788ee9..06fda6fa1 100644 --- a/lib/std/private/miscdollars.nim +++ b/lib/std/private/miscdollars.nim @@ -13,21 +13,7 @@ template toLocation*(result: var string, file: string | cstring, line: int, col: addInt(result, col) result.add ")" -when defined(nimHasIsNamedTuple): - proc isNamedTuple(T: typedesc): bool {.magic: "TypeTrait".} -else: - # for bootstrap; remove after release 1.2 - proc isNamedTuple(T: typedesc): bool = - # Taken from typetraits. - when T isnot tuple: result = false - else: - var t: T - for name, _ in t.fieldPairs: - when name == "Field0": - return compiles(t.Field0) - else: - return true - return false +proc isNamedTuple(T: typedesc): bool {.magic: "TypeTrait".} template tupleObjectDollar*[T: tuple | object](result: var string, x: T) = result = "(" diff --git a/lib/std/private/osappdirs.nim b/lib/std/private/osappdirs.nim new file mode 100644 index 000000000..07a6809bb --- /dev/null +++ b/lib/std/private/osappdirs.nim @@ -0,0 +1,176 @@ +## .. importdoc:: paths.nim, dirs.nim + +include system/inclrtl +import std/envvars +import std/private/ospaths2 + +proc getHomeDir*(): string {.rtl, extern: "nos$1", + tags: [ReadEnvEffect, ReadIOEffect].} = + ## Returns the home directory of the current user. + ## + ## This proc is wrapped by the `expandTilde proc`_ + ## for the convenience of processing paths coming from user configuration files. + ## + ## See also: + ## * `getDataDir proc`_ + ## * `getConfigDir proc`_ + ## * `getTempDir proc`_ + ## * `expandTilde proc`_ + ## * `getCurrentDir proc`_ + ## * `setCurrentDir proc`_ + runnableExamples: + import std/os + assert getHomeDir() == expandTilde("~") + + when defined(windows): return getEnv("USERPROFILE") & "\\" + else: return getEnv("HOME") & "/" + +proc getDataDir*(): string {.rtl, extern: "nos$1" + tags: [ReadEnvEffect, ReadIOEffect].} = + ## Returns the data directory of the current user for applications. + ## + ## On non-Windows OSs, this proc conforms to the XDG Base Directory + ## spec. Thus, this proc returns the value of the `XDG_DATA_HOME` environment + ## variable if it is set, otherwise it returns the default configuration + ## directory ("~/.local/share" or "~/Library/Application Support" on macOS). + ## + ## See also: + ## * `getHomeDir proc`_ + ## * `getConfigDir proc`_ + ## * `getTempDir proc`_ + ## * `expandTilde proc`_ + ## * `getCurrentDir proc`_ + ## * `setCurrentDir proc`_ + when defined(windows): + result = getEnv("APPDATA") + elif defined(macosx): + result = getEnv("XDG_DATA_HOME", getEnv("HOME") / "Library" / "Application Support") + else: + result = getEnv("XDG_DATA_HOME", getEnv("HOME") / ".local" / "share") + result.normalizePathEnd(trailingSep = true) + +proc getConfigDir*(): string {.rtl, extern: "nos$1", + tags: [ReadEnvEffect, ReadIOEffect].} = + ## Returns the config directory of the current user for applications. + ## + ## On non-Windows OSs, this proc conforms to the XDG Base Directory + ## spec. Thus, this proc returns the value of the `XDG_CONFIG_HOME` environment + ## variable if it is set, otherwise it returns the default configuration + ## directory ("~/.config/"). + ## + ## An OS-dependent trailing slash is always present at the end of the + ## returned string: `\\` on Windows and `/` on all other OSs. + ## + ## See also: + ## * `getHomeDir proc`_ + ## * `getDataDir proc`_ + ## * `getTempDir proc`_ + ## * `expandTilde proc`_ + ## * `getCurrentDir proc`_ + ## * `setCurrentDir proc`_ + when defined(windows): + result = getEnv("APPDATA") + else: + result = getEnv("XDG_CONFIG_HOME", getEnv("HOME") / ".config") + result.normalizePathEnd(trailingSep = true) + +proc getCacheDir*(): string = + ## Returns the cache directory of the current user for applications. + ## + ## This makes use of the following environment variables: + ## + ## * On Windows: `getEnv("LOCALAPPDATA")` + ## + ## * On macOS: `getEnv("XDG_CACHE_HOME", getEnv("HOME") / "Library/Caches")` + ## + ## * On other platforms: `getEnv("XDG_CACHE_HOME", getEnv("HOME") / ".cache")` + ## + ## **See also:** + ## * `getHomeDir proc`_ + ## * `getTempDir proc`_ + ## * `getConfigDir proc`_ + ## * `getDataDir proc`_ + # follows https://crates.io/crates/platform-dirs + when defined(windows): + result = getEnv("LOCALAPPDATA") + elif defined(osx): + result = getEnv("XDG_CACHE_HOME", getEnv("HOME") / "Library/Caches") + else: + result = getEnv("XDG_CACHE_HOME", getEnv("HOME") / ".cache") + result.normalizePathEnd(false) + +proc getCacheDir*(app: string): string = + ## Returns the cache directory for an application `app`. + ## + ## * On Windows, this uses: `getCacheDir() / app / "cache"` + ## * On other platforms, this uses: `getCacheDir() / app` + when defined(windows): + getCacheDir() / app / "cache" + else: + getCacheDir() / app + + +when defined(windows): + type DWORD = uint32 + + when defined(nimPreviewSlimSystem): + import std/widestrs + + proc getTempPath( + nBufferLength: DWORD, lpBuffer: WideCString + ): DWORD {.stdcall, dynlib: "kernel32.dll", importc: "GetTempPathW".} = + ## Retrieves the path of the directory designated for temporary files. + +template getEnvImpl(result: var string, tempDirList: openArray[string]) = + for dir in tempDirList: + if existsEnv(dir): + result = getEnv(dir) + break + +template getTempDirImpl(result: var string) = + when defined(windows): + getEnvImpl(result, ["TMP", "TEMP", "USERPROFILE"]) + else: + getEnvImpl(result, ["TMPDIR", "TEMP", "TMP", "TEMPDIR"]) + +proc getTempDir*(): string {.rtl, extern: "nos$1", + tags: [ReadEnvEffect, ReadIOEffect].} = + ## Returns the temporary directory of the current user for applications to + ## save temporary files in. + ## + ## On Windows, it calls [GetTempPath](https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-gettemppathw). + ## On Posix based platforms, it will check `TMPDIR`, `TEMP`, `TMP` and `TEMPDIR` environment variables in order. + ## On all platforms, `/tmp` will be returned if the procs fails. + ## + ## You can override this implementation + ## by adding `-d:tempDir=mytempname` to your compiler invocation. + ## + ## **Note:** This proc does not check whether the returned path exists. + ## + ## See also: + ## * `getHomeDir proc`_ + ## * `getConfigDir proc`_ + ## * `expandTilde proc`_ + ## * `getCurrentDir proc`_ + ## * `setCurrentDir proc`_ + const tempDirDefault = "/tmp" + when defined(tempDir): + const tempDir {.strdefine.}: string = tempDirDefault + result = tempDir + else: + when nimvm: + getTempDirImpl(result) + else: + when defined(windows): + let size = getTempPath(0, nil) + # If the function fails, the return value is zero. + if size > 0: + let buffer = newWideCString(size.int) + if getTempPath(size, buffer) > 0: + result = $buffer + elif defined(android): result = "/data/local/tmp" + else: + getTempDirImpl(result) + if result.len == 0: + result = tempDirDefault + normalizePathEnd(result, trailingSep=true) diff --git a/lib/std/private/oscommon.nim b/lib/std/private/oscommon.nim new file mode 100644 index 000000000..c49d52ef2 --- /dev/null +++ b/lib/std/private/oscommon.nim @@ -0,0 +1,186 @@ +include system/inclrtl + +import std/[oserrors] + +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions, widestrs] + +## .. importdoc:: osdirs.nim, os.nim + +const weirdTarget* = defined(nimscript) or defined(js) + + +type + ReadDirEffect* = object of ReadIOEffect ## Effect that denotes a read + ## operation from the directory + ## structure. + WriteDirEffect* = object of WriteIOEffect ## Effect that denotes a write + ## operation to + ## the directory structure. + + +when weirdTarget: + discard +elif defined(windows): + import std/[winlean, times] +elif defined(posix): + import std/posix + proc c_rename(oldname, newname: cstring): cint {. + importc: "rename", header: "<stdio.h>".} +else: + {.error: "OS module not ported to your operating system!".} + + +when weirdTarget: + {.pragma: noWeirdTarget, error: "this proc is not available on the NimScript/js target".} +else: + {.pragma: noWeirdTarget.} + + +when defined(nimscript): + # for procs already defined in scriptconfig.nim + template noNimJs(body): untyped = discard +elif defined(js): + {.pragma: noNimJs, error: "this proc is not available on the js target".} +else: + {.pragma: noNimJs.} + + +when defined(windows) and not weirdTarget: + template wrapUnary*(varname, winApiProc, arg: untyped) = + var varname = winApiProc(newWideCString(arg)) + + template wrapBinary*(varname, winApiProc, arg, arg2: untyped) = + var varname = winApiProc(newWideCString(arg), arg2) + proc findFirstFile*(a: string, b: var WIN32_FIND_DATA): Handle = + result = findFirstFileW(newWideCString(a), b) + template findNextFile*(a, b: untyped): untyped = findNextFileW(a, b) + + template getFilename*(f: untyped): untyped = + $cast[WideCString](addr(f.cFileName[0])) + + proc skipFindData*(f: WIN32_FIND_DATA): bool {.inline.} = + # Note - takes advantage of null delimiter in the cstring + const dot = ord('.') + result = f.cFileName[0].int == dot and (f.cFileName[1].int == 0 or + f.cFileName[1].int == dot and f.cFileName[2].int == 0) + + +type + PathComponent* = enum ## Enumeration specifying a path component. + ## + ## See also: + ## * `walkDirRec iterator`_ + ## * `FileInfo object`_ + pcFile, ## path refers to a file + pcLinkToFile, ## path refers to a symbolic link to a file + pcDir, ## path refers to a directory + pcLinkToDir ## path refers to a symbolic link to a directory + + +when defined(posix) and not weirdTarget: + proc getSymlinkFileKind*(path: string): + tuple[pc: PathComponent, isSpecial: bool] = + # Helper function. + var s: Stat + assert(path != "") + result = (pcLinkToFile, false) + if stat(path, s) == 0'i32: + if S_ISDIR(s.st_mode): + result = (pcLinkToDir, false) + elif not S_ISREG(s.st_mode): + result = (pcLinkToFile, true) + +proc tryMoveFSObject*(source, dest: string, isDir: bool): bool {.noWeirdTarget.} = + ## Moves a file (or directory if `isDir` is true) from `source` to `dest`. + ## + ## Returns false in case of `EXDEV` error or `AccessDeniedError` on Windows (if `isDir` is true). + ## In case of other errors `OSError` is raised. + ## Returns true in case of success. + when defined(windows): + let s = newWideCString(source) + let d = newWideCString(dest) + result = moveFileExW(s, d, MOVEFILE_COPY_ALLOWED or MOVEFILE_REPLACE_EXISTING) != 0'i32 + else: + result = c_rename(source, dest) == 0'i32 + + if not result: + let err = osLastError() + let isAccessDeniedError = + when defined(windows): + const AccessDeniedError = OSErrorCode(5) + isDir and err == AccessDeniedError + else: + err == EXDEV.OSErrorCode + if not isAccessDeniedError: + raiseOSError(err, $(source, dest)) + +when not defined(windows): + const maxSymlinkLen* = 1024 + +proc fileExists*(filename: string): bool {.rtl, extern: "nos$1", + tags: [ReadDirEffect], noNimJs, sideEffect.} = + ## Returns true if `filename` exists and is a regular file or symlink. + ## + ## Directories, device files, named pipes and sockets return false. + ## + ## See also: + ## * `dirExists proc`_ + ## * `symlinkExists proc`_ + when defined(windows): + wrapUnary(a, getFileAttributesW, filename) + if a != -1'i32: + result = (a and FILE_ATTRIBUTE_DIRECTORY) == 0'i32 + else: + var res: Stat + return stat(filename, res) >= 0'i32 and S_ISREG(res.st_mode) + + +proc dirExists*(dir: string): bool {.rtl, extern: "nos$1", tags: [ReadDirEffect], + noNimJs, sideEffect.} = + ## Returns true if the directory `dir` exists. If `dir` is a file, false + ## is returned. Follows symlinks. + ## + ## See also: + ## * `fileExists proc`_ + ## * `symlinkExists proc`_ + when defined(windows): + wrapUnary(a, getFileAttributesW, dir) + if a != -1'i32: + result = (a and FILE_ATTRIBUTE_DIRECTORY) != 0'i32 + else: + var res: Stat + result = stat(dir, res) >= 0'i32 and S_ISDIR(res.st_mode) + + +proc symlinkExists*(link: string): bool {.rtl, extern: "nos$1", + tags: [ReadDirEffect], + noWeirdTarget, sideEffect.} = + ## Returns true if the symlink `link` exists. Will return true + ## regardless of whether the link points to a directory or file. + ## + ## See also: + ## * `fileExists proc`_ + ## * `dirExists proc`_ + when defined(windows): + wrapUnary(a, getFileAttributesW, link) + if a != -1'i32: + # xxx see: bug #16784 (bug9); checking `IO_REPARSE_TAG_SYMLINK` + # may also be needed. + result = (a and FILE_ATTRIBUTE_REPARSE_POINT) != 0'i32 + else: + var res: Stat + result = lstat(link, res) >= 0'i32 and S_ISLNK(res.st_mode) + +when defined(windows) and not weirdTarget: + proc openHandle*(path: string, followSymlink=true, writeAccess=false): Handle = + var flags = FILE_FLAG_BACKUP_SEMANTICS or FILE_ATTRIBUTE_NORMAL + if not followSymlink: + flags = flags or FILE_FLAG_OPEN_REPARSE_POINT + let access = if writeAccess: GENERIC_WRITE else: 0'i32 + + result = createFileW( + newWideCString(path), access, + FILE_SHARE_DELETE or FILE_SHARE_READ or FILE_SHARE_WRITE, + nil, OPEN_EXISTING, flags, 0 + ) diff --git a/lib/std/private/osdirs.nim b/lib/std/private/osdirs.nim new file mode 100644 index 000000000..a44cad7d9 --- /dev/null +++ b/lib/std/private/osdirs.nim @@ -0,0 +1,570 @@ +## .. importdoc:: osfiles.nim, appdirs.nim, paths.nim + +include system/inclrtl +import std/oserrors + + +import ospaths2, osfiles +import oscommon +export dirExists, PathComponent + + +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions, widestrs] + + +when weirdTarget: + discard +elif defined(windows): + import std/[winlean, times] +elif defined(posix): + import std/[posix, times] + +else: + {.error: "OS module not ported to your operating system!".} + + +when weirdTarget: + {.pragma: noWeirdTarget, error: "this proc is not available on the NimScript/js target".} +else: + {.pragma: noWeirdTarget.} + + +when defined(nimscript): + # for procs already defined in scriptconfig.nim + template noNimJs(body): untyped = discard +elif defined(js): + {.pragma: noNimJs, error: "this proc is not available on the js target".} +else: + {.pragma: noNimJs.} + +# Templates for filtering directories and files +when defined(windows) and not weirdTarget: + template isDir(f: WIN32_FIND_DATA): bool = + (f.dwFileAttributes and FILE_ATTRIBUTE_DIRECTORY) != 0'i32 + template isFile(f: WIN32_FIND_DATA): bool = + not isDir(f) +else: + template isDir(f: string): bool {.dirty.} = + dirExists(f) + template isFile(f: string): bool {.dirty.} = + fileExists(f) + +template defaultWalkFilter(item): bool = + ## Walk filter used to return true on both + ## files and directories + true + +template walkCommon(pattern: string, filter) = + ## Common code for getting the files and directories with the + ## specified `pattern` + when defined(windows): + var + f: WIN32_FIND_DATA + res: int + res = findFirstFile(pattern, f) + if res != -1: + defer: findClose(res) + let dotPos = searchExtPos(pattern) + while true: + if not skipFindData(f) and filter(f): + # Windows bug/gotcha: 't*.nim' matches 'tfoo.nims' -.- so we check + # that the file extensions have the same length ... + let ff = getFilename(f) + let idx = ff.len - pattern.len + dotPos + if dotPos < 0 or idx >= ff.len or (idx >= 0 and ff[idx] == '.') or + (dotPos >= 0 and dotPos+1 < pattern.len and pattern[dotPos+1] == '*'): + yield splitFile(pattern).dir / extractFilename(ff) + if findNextFile(res, f) == 0'i32: + let errCode = getLastError() + if errCode == ERROR_NO_MORE_FILES: break + else: raiseOSError(errCode.OSErrorCode) + else: # here we use glob + var + f: Glob + res: int + f.gl_offs = 0 + f.gl_pathc = 0 + f.gl_pathv = nil + res = glob(pattern, 0, nil, addr(f)) + defer: globfree(addr(f)) + if res == 0: + for i in 0.. f.gl_pathc - 1: + assert(f.gl_pathv[i] != nil) + let path = $f.gl_pathv[i] + if filter(path): + yield path + +iterator walkPattern*(pattern: string): string {.tags: [ReadDirEffect], noWeirdTarget.} = + ## Iterate over all the files and directories that match the `pattern`. + ## + ## On POSIX this uses the `glob`:idx: call. + ## `pattern` is OS dependent, but at least the `"*.ext"` + ## notation is supported. + ## + ## See also: + ## * `walkFiles iterator`_ + ## * `walkDirs iterator`_ + ## * `walkDir iterator`_ + ## * `walkDirRec iterator`_ + runnableExamples: + import std/os + import std/sequtils + let paths = toSeq(walkPattern("lib/pure/*")) # works on Windows too + assert "lib/pure/concurrency".unixToNativePath in paths + assert "lib/pure/os.nim".unixToNativePath in paths + walkCommon(pattern, defaultWalkFilter) + +iterator walkFiles*(pattern: string): string {.tags: [ReadDirEffect], noWeirdTarget.} = + ## Iterate over all the files that match the `pattern`. + ## + ## On POSIX this uses the `glob`:idx: call. + ## `pattern` is OS dependent, but at least the `"*.ext"` + ## notation is supported. + ## + ## See also: + ## * `walkPattern iterator`_ + ## * `walkDirs iterator`_ + ## * `walkDir iterator`_ + ## * `walkDirRec iterator`_ + runnableExamples: + import std/os + import std/sequtils + assert "lib/pure/os.nim".unixToNativePath in toSeq(walkFiles("lib/pure/*.nim")) # works on Windows too + walkCommon(pattern, isFile) + +iterator walkDirs*(pattern: string): string {.tags: [ReadDirEffect], noWeirdTarget.} = + ## Iterate over all the directories that match the `pattern`. + ## + ## On POSIX this uses the `glob`:idx: call. + ## `pattern` is OS dependent, but at least the `"*.ext"` + ## notation is supported. + ## + ## See also: + ## * `walkPattern iterator`_ + ## * `walkFiles iterator`_ + ## * `walkDir iterator`_ + ## * `walkDirRec iterator`_ + runnableExamples: + import std/os + import std/sequtils + let paths = toSeq(walkDirs("lib/pure/*")) # works on Windows too + assert "lib/pure/concurrency".unixToNativePath in paths + walkCommon(pattern, isDir) + +proc staticWalkDir(dir: string; relative: bool): seq[ + tuple[kind: PathComponent, path: string]] = + discard + +iterator walkDir*(dir: string; relative = false, checkDir = false, + skipSpecial = false): + tuple[kind: PathComponent, path: string] {.tags: [ReadDirEffect].} = + ## Walks over the directory `dir` and yields for each directory or file in + ## `dir`. The component type and full path for each item are returned. + ## + ## Walking is not recursive. + ## * If `relative` is true (default: false) + ## the resulting path is shortened to be relative to ``dir``, + ## otherwise the full path is returned. + ## * If `checkDir` is true, `OSError` is raised when `dir` + ## doesn't exist. + ## * If `skipSpecial` is true, then (besides all directories) only *regular* + ## files (**without** special "file" objects like FIFOs, device files, + ## etc) will be yielded on Unix. + ## + ## **Example:** + ## + ## This directory structure: + ## + ## dirA / dirB / fileB1.txt + ## / dirC + ## / fileA1.txt + ## / fileA2.txt + ## + ## and this code: + runnableExamples("-r:off"): + import std/[strutils, sugar] + # note: order is not guaranteed + # this also works at compile time + assert collect(for k in walkDir("dirA"): k.path).join(" ") == + "dirA/dirB dirA/dirC dirA/fileA2.txt dirA/fileA1.txt" + ## See also: + ## * `walkPattern iterator`_ + ## * `walkFiles iterator`_ + ## * `walkDirs iterator`_ + ## * `walkDirRec iterator`_ + + when nimvm: + for k, v in items(staticWalkDir(dir, relative)): + yield (k, v) + else: + when weirdTarget: + for k, v in items(staticWalkDir(dir, relative)): + yield (k, v) + elif defined(windows): + var f: WIN32_FIND_DATA + var h = findFirstFile(dir / "*", f) + if h == -1: + if checkDir: + raiseOSError(osLastError(), dir) + else: + defer: findClose(h) + while true: + var k = pcFile + if not skipFindData(f): + if (f.dwFileAttributes and FILE_ATTRIBUTE_DIRECTORY) != 0'i32: + k = pcDir + if (f.dwFileAttributes and FILE_ATTRIBUTE_REPARSE_POINT) != 0'i32: + k = succ(k) + let xx = if relative: extractFilename(getFilename(f)) + else: dir / extractFilename(getFilename(f)) + yield (k, xx) + if findNextFile(h, f) == 0'i32: + let errCode = getLastError() + if errCode == ERROR_NO_MORE_FILES: break + else: raiseOSError(errCode.OSErrorCode) + else: + var d = opendir(dir) + if d == nil: + if checkDir: + raiseOSError(osLastError(), dir) + else: + defer: discard closedir(d) + while true: + var x = readdir(d) + if x == nil: break + var y = $cast[cstring](addr x.d_name) + if y != "." and y != "..": + var s: Stat + let path = dir / y + if not relative: + y = path + var k = pcFile + + template resolveSymlink() = + var isSpecial: bool + (k, isSpecial) = getSymlinkFileKind(path) + if skipSpecial and isSpecial: continue + + template kSetGeneric() = # pure Posix component `k` resolution + if lstat(path.cstring, s) < 0'i32: continue # don't yield + elif S_ISDIR(s.st_mode): + k = pcDir + elif S_ISLNK(s.st_mode): + resolveSymlink() + elif skipSpecial and not S_ISREG(s.st_mode): continue + + when defined(linux) or defined(macosx) or + defined(bsd) or defined(genode) or defined(nintendoswitch): + case x.d_type + of DT_DIR: k = pcDir + of DT_LNK: + resolveSymlink() + of DT_UNKNOWN: + kSetGeneric() + else: # DT_REG or special "files" like FIFOs + if skipSpecial and x.d_type != DT_REG: continue + else: discard # leave it as pcFile + else: # assuming that field `d_type` is not present + kSetGeneric() + + yield (k, y) + +iterator walkDirRec*(dir: string, + yieldFilter = {pcFile}, followFilter = {pcDir}, + relative = false, checkDir = false, skipSpecial = false): + string {.tags: [ReadDirEffect].} = + ## Recursively walks over the directory `dir` and yields for each file + ## or directory in `dir`. + ## + ## Options `relative`, `checkdir`, `skipSpecial` are explained in + ## [walkDir iterator] description. + ## + ## .. warning:: Modifying the directory structure while the iterator + ## is traversing may result in undefined behavior! + ## + ## Walking is recursive. `followFilter` controls the behaviour of the iterator: + ## + ## ===================== ============================================= + ## yieldFilter meaning + ## ===================== ============================================= + ## ``pcFile`` yield real files (default) + ## ``pcLinkToFile`` yield symbolic links to files + ## ``pcDir`` yield real directories + ## ``pcLinkToDir`` yield symbolic links to directories + ## ===================== ============================================= + ## + ## ===================== ============================================= + ## followFilter meaning + ## ===================== ============================================= + ## ``pcDir`` follow real directories (default) + ## ``pcLinkToDir`` follow symbolic links to directories + ## ===================== ============================================= + ## + ## + ## See also: + ## * `walkPattern iterator`_ + ## * `walkFiles iterator`_ + ## * `walkDirs iterator`_ + ## * `walkDir iterator`_ + + var stack = @[""] + var checkDir = checkDir + while stack.len > 0: + let d = stack.pop() + for k, p in walkDir(dir / d, relative = true, checkDir = checkDir, + skipSpecial = skipSpecial): + let rel = d / p + if k in {pcDir, pcLinkToDir} and k in followFilter: + stack.add rel + if k in yieldFilter: + yield if relative: rel else: dir / rel + checkDir = false + # We only check top-level dir, otherwise if a subdir is invalid (eg. wrong + # permissions), it'll abort iteration and there would be no way to + # continue iteration. + # Future work can provide a way to customize this and do error reporting. + + +proc rawRemoveDir(dir: string) {.noWeirdTarget.} = + when defined(windows): + wrapUnary(res, removeDirectoryW, dir) + let lastError = osLastError() + if res == 0'i32 and lastError.int32 != 3'i32 and + lastError.int32 != 18'i32 and lastError.int32 != 2'i32: + raiseOSError(lastError, dir) + else: + if rmdir(dir) != 0'i32 and errno != ENOENT: raiseOSError(osLastError(), dir) + +proc removeDir*(dir: string, checkDir = false) {.rtl, extern: "nos$1", tags: [ + WriteDirEffect, ReadDirEffect], benign, noWeirdTarget.} = + ## Removes the directory `dir` including all subdirectories and files + ## in `dir` (recursively). + ## + ## If this fails, `OSError` is raised. This does not fail if the directory never + ## existed in the first place, unless `checkDir` = true. + ## + ## See also: + ## * `tryRemoveFile proc`_ + ## * `removeFile proc`_ + ## * `existsOrCreateDir proc`_ + ## * `createDir proc`_ + ## * `copyDir proc`_ + ## * `copyDirWithPermissions proc`_ + ## * `moveDir proc`_ + for kind, path in walkDir(dir, checkDir = checkDir): + case kind + of pcFile, pcLinkToFile, pcLinkToDir: removeFile(path) + of pcDir: removeDir(path, true) + # for subdirectories there is no benefit in `checkDir = false` + # (unless perhaps for edge case of concurrent processes also deleting + # the same files) + rawRemoveDir(dir) + +proc rawCreateDir(dir: string): bool {.noWeirdTarget.} = + # Try to create one directory (not the whole path). + # returns `true` for success, `false` if the path has previously existed + # + # This is a thin wrapper over mkDir (or alternatives on other systems), + # so in case of a pre-existing path we don't check that it is a directory. + when defined(solaris): + let res = mkdir(dir, 0o777) + if res == 0'i32: + result = true + elif errno in {EEXIST, ENOSYS}: + result = false + else: + raiseOSError(osLastError(), dir) + elif defined(haiku): + let res = mkdir(dir, 0o777) + if res == 0'i32: + result = true + elif errno == EEXIST or errno == EROFS: + result = false + else: + raiseOSError(osLastError(), dir) + elif defined(posix): + let res = mkdir(dir, 0o777) + if res == 0'i32: + result = true + elif errno == EEXIST: + result = false + else: + #echo res + raiseOSError(osLastError(), dir) + else: + wrapUnary(res, createDirectoryW, dir) + + if res != 0'i32: + result = true + elif getLastError() == 183'i32: + result = false + else: + raiseOSError(osLastError(), dir) + +proc existsOrCreateDir*(dir: string): bool {.rtl, extern: "nos$1", + tags: [WriteDirEffect, ReadDirEffect], noWeirdTarget.} = + ## Checks if a `directory`:idx: `dir` exists, and creates it otherwise. + ## + ## Does not create parent directories (raises `OSError` if parent directories do not exist). + ## Returns `true` if the directory already exists, and `false` otherwise. + ## + ## See also: + ## * `removeDir proc`_ + ## * `createDir proc`_ + ## * `copyDir proc`_ + ## * `copyDirWithPermissions proc`_ + ## * `moveDir proc`_ + result = not rawCreateDir(dir) + if result: + # path already exists - need to check that it is indeed a directory + if not dirExists(dir): + raise newException(IOError, "Failed to create '" & dir & "'") + +proc createDir*(dir: string) {.rtl, extern: "nos$1", + tags: [WriteDirEffect, ReadDirEffect], noWeirdTarget.} = + ## Creates the `directory`:idx: `dir`. + ## + ## The directory may contain several subdirectories that do not exist yet. + ## The full path is created. If this fails, `OSError` is raised. + ## + ## It does **not** fail if the directory already exists because for + ## most usages this does not indicate an error. + ## + ## See also: + ## * `removeDir proc`_ + ## * `existsOrCreateDir proc`_ + ## * `copyDir proc`_ + ## * `copyDirWithPermissions proc`_ + ## * `moveDir proc`_ + if dir == "": + return + var omitNext = isAbsolute(dir) + for p in parentDirs(dir, fromRoot=true): + if omitNext: + omitNext = false + else: + discard existsOrCreateDir(p) + +proc copyDir*(source, dest: string, skipSpecial = false) {.rtl, extern: "nos$1", + tags: [ReadDirEffect, WriteIOEffect, ReadIOEffect], benign, noWeirdTarget.} = + ## Copies a directory from `source` to `dest`. + ## + ## On non-Windows OSes, symlinks are copied as symlinks. On Windows, symlinks + ## are skipped. + ## + ## If `skipSpecial` is true, then (besides all directories) only *regular* + ## files (**without** special "file" objects like FIFOs, device files, + ## etc) will be copied on Unix. + ## + ## If this fails, `OSError` is raised. + ## + ## On the Windows platform this proc will copy the attributes from + ## `source` into `dest`. + ## + ## On other platforms created files and directories will inherit the + ## default permissions of a newly created file/directory for the user. + ## Use `copyDirWithPermissions proc`_ + ## to preserve attributes recursively on these platforms. + ## + ## See also: + ## * `copyDirWithPermissions proc`_ + ## * `copyFile proc`_ + ## * `copyFileWithPermissions proc`_ + ## * `removeDir proc`_ + ## * `existsOrCreateDir proc`_ + ## * `createDir proc`_ + ## * `moveDir proc`_ + createDir(dest) + for kind, path in walkDir(source, skipSpecial = skipSpecial): + var noSource = splitPath(path).tail + if kind == pcDir: + copyDir(path, dest / noSource, skipSpecial = skipSpecial) + else: + copyFile(path, dest / noSource, {cfSymlinkAsIs}) + + +proc copyDirWithPermissions*(source, dest: string, + ignorePermissionErrors = true, + skipSpecial = false) + {.rtl, extern: "nos$1", tags: [ReadDirEffect, WriteIOEffect, ReadIOEffect], + benign, noWeirdTarget.} = + ## Copies a directory from `source` to `dest` preserving file permissions. + ## + ## On non-Windows OSes, symlinks are copied as symlinks. On Windows, symlinks + ## are skipped. + ## + ## If `skipSpecial` is true, then (besides all directories) only *regular* + ## files (**without** special "file" objects like FIFOs, device files, + ## etc) will be copied on Unix. + ## + ## If this fails, `OSError` is raised. This is a wrapper proc around + ## `copyDir`_ and `copyFileWithPermissions`_ procs + ## on non-Windows platforms. + ## + ## On Windows this proc is just a wrapper for `copyDir proc`_ since + ## that proc already copies attributes. + ## + ## On non-Windows systems permissions are copied after the file or directory + ## itself has been copied, which won't happen atomically and could lead to a + ## race condition. If `ignorePermissionErrors` is true (default), errors while + ## reading/setting file attributes will be ignored, otherwise will raise + ## `OSError`. + ## + ## See also: + ## * `copyDir proc`_ + ## * `copyFile proc`_ + ## * `copyFileWithPermissions proc`_ + ## * `removeDir proc`_ + ## * `moveDir proc`_ + ## * `existsOrCreateDir proc`_ + ## * `createDir proc`_ + createDir(dest) + when not defined(windows): + try: + setFilePermissions(dest, getFilePermissions(source), followSymlinks = + false) + except: + if not ignorePermissionErrors: + raise + for kind, path in walkDir(source, skipSpecial = skipSpecial): + var noSource = splitPath(path).tail + if kind == pcDir: + copyDirWithPermissions(path, dest / noSource, ignorePermissionErrors, skipSpecial = skipSpecial) + else: + copyFileWithPermissions(path, dest / noSource, ignorePermissionErrors, {cfSymlinkAsIs}) + +proc moveDir*(source, dest: string) {.tags: [ReadIOEffect, WriteIOEffect], noWeirdTarget.} = + ## Moves a directory from `source` to `dest`. + ## + ## Symlinks are not followed: if `source` contains symlinks, they themself are + ## moved, not their target. + ## + ## If this fails, `OSError` is raised. + ## + ## See also: + ## * `moveFile proc`_ + ## * `copyDir proc`_ + ## * `copyDirWithPermissions proc`_ + ## * `removeDir proc`_ + ## * `existsOrCreateDir proc`_ + ## * `createDir proc`_ + if not tryMoveFSObject(source, dest, isDir = true): + # Fallback to copy & del + copyDir(source, dest) + removeDir(source) + +proc setCurrentDir*(newDir: string) {.inline, tags: [], noWeirdTarget.} = + ## Sets the `current working directory`:idx:; `OSError` + ## is raised if `newDir` cannot been set. + ## + ## See also: + ## * `getHomeDir proc`_ + ## * `getConfigDir proc`_ + ## * `getTempDir proc`_ + ## * `getCurrentDir proc`_ + when defined(windows): + if setCurrentDirectoryW(newWideCString(newDir)) == 0'i32: + raiseOSError(osLastError(), newDir) + else: + if chdir(newDir) != 0'i32: raiseOSError(osLastError(), newDir) diff --git a/lib/std/private/osfiles.nim b/lib/std/private/osfiles.nim new file mode 100644 index 000000000..37d8eabca --- /dev/null +++ b/lib/std/private/osfiles.nim @@ -0,0 +1,416 @@ +include system/inclrtl +import std/private/since +import std/oserrors + +import oscommon +export fileExists + +import ospaths2, ossymlinks + +## .. importdoc:: osdirs.nim, os.nim + +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions, widestrs] + +when weirdTarget: + discard +elif defined(windows): + import std/winlean +elif defined(posix): + import std/[posix, times] + + proc toTime(ts: Timespec): times.Time {.inline.} = + result = initTime(ts.tv_sec.int64, ts.tv_nsec.int) +else: + {.error: "OS module not ported to your operating system!".} + + +when weirdTarget: + {.pragma: noWeirdTarget, error: "this proc is not available on the NimScript/js target".} +else: + {.pragma: noWeirdTarget.} + + +when defined(nimscript): + # for procs already defined in scriptconfig.nim + template noNimJs(body): untyped = discard +elif defined(js): + {.pragma: noNimJs, error: "this proc is not available on the js target".} +else: + {.pragma: noNimJs.} + + +type + FilePermission* = enum ## File access permission, modelled after UNIX. + ## + ## See also: + ## * `getFilePermissions`_ + ## * `setFilePermissions`_ + ## * `FileInfo object`_ + fpUserExec, ## execute access for the file owner + fpUserWrite, ## write access for the file owner + fpUserRead, ## read access for the file owner + fpGroupExec, ## execute access for the group + fpGroupWrite, ## write access for the group + fpGroupRead, ## read access for the group + fpOthersExec, ## execute access for others + fpOthersWrite, ## write access for others + fpOthersRead ## read access for others + +proc getFilePermissions*(filename: string): set[FilePermission] {. + rtl, extern: "nos$1", tags: [ReadDirEffect], noWeirdTarget.} = + ## Retrieves file permissions for `filename`. + ## + ## `OSError` is raised in case of an error. + ## On Windows, only the ``readonly`` flag is checked, every other + ## permission is available in any case. + ## + ## See also: + ## * `setFilePermissions proc`_ + ## * `FilePermission enum`_ + when defined(posix): + var a: Stat + if stat(filename, a) < 0'i32: raiseOSError(osLastError(), filename) + result = {} + if (a.st_mode and S_IRUSR.Mode) != 0.Mode: result.incl(fpUserRead) + if (a.st_mode and S_IWUSR.Mode) != 0.Mode: result.incl(fpUserWrite) + if (a.st_mode and S_IXUSR.Mode) != 0.Mode: result.incl(fpUserExec) + + if (a.st_mode and S_IRGRP.Mode) != 0.Mode: result.incl(fpGroupRead) + if (a.st_mode and S_IWGRP.Mode) != 0.Mode: result.incl(fpGroupWrite) + if (a.st_mode and S_IXGRP.Mode) != 0.Mode: result.incl(fpGroupExec) + + if (a.st_mode and S_IROTH.Mode) != 0.Mode: result.incl(fpOthersRead) + if (a.st_mode and S_IWOTH.Mode) != 0.Mode: result.incl(fpOthersWrite) + if (a.st_mode and S_IXOTH.Mode) != 0.Mode: result.incl(fpOthersExec) + else: + wrapUnary(res, getFileAttributesW, filename) + if res == -1'i32: raiseOSError(osLastError(), filename) + if (res and FILE_ATTRIBUTE_READONLY) != 0'i32: + result = {fpUserExec, fpUserRead, fpGroupExec, fpGroupRead, + fpOthersExec, fpOthersRead} + else: + result = {fpUserExec..fpOthersRead} + +proc setFilePermissions*(filename: string, permissions: set[FilePermission], + followSymlinks = true) + {.rtl, extern: "nos$1", tags: [ReadDirEffect, WriteDirEffect], + noWeirdTarget.} = + ## Sets the file permissions for `filename`. + ## + ## If `followSymlinks` set to true (default) and ``filename`` points to a + ## symlink, permissions are set to the file symlink points to. + ## `followSymlinks` set to false is a noop on Windows and some POSIX + ## systems (including Linux) on which `lchmod` is either unavailable or always + ## fails, given that symlinks permissions there are not observed. + ## + ## `OSError` is raised in case of an error. + ## On Windows, only the ``readonly`` flag is changed, depending on + ## ``fpUserWrite`` permission. + ## + ## See also: + ## * `getFilePermissions proc`_ + ## * `FilePermission enum`_ + when defined(posix): + var p = 0.Mode + if fpUserRead in permissions: p = p or S_IRUSR.Mode + if fpUserWrite in permissions: p = p or S_IWUSR.Mode + if fpUserExec in permissions: p = p or S_IXUSR.Mode + + if fpGroupRead in permissions: p = p or S_IRGRP.Mode + if fpGroupWrite in permissions: p = p or S_IWGRP.Mode + if fpGroupExec in permissions: p = p or S_IXGRP.Mode + + if fpOthersRead in permissions: p = p or S_IROTH.Mode + if fpOthersWrite in permissions: p = p or S_IWOTH.Mode + if fpOthersExec in permissions: p = p or S_IXOTH.Mode + + if not followSymlinks and filename.symlinkExists: + when declared(lchmod): + if lchmod(filename, cast[Mode](p)) != 0: + raiseOSError(osLastError(), $(filename, permissions)) + else: + if chmod(filename, cast[Mode](p)) != 0: + raiseOSError(osLastError(), $(filename, permissions)) + else: + wrapUnary(res, getFileAttributesW, filename) + if res == -1'i32: raiseOSError(osLastError(), filename) + if fpUserWrite in permissions: + res = res and not FILE_ATTRIBUTE_READONLY + else: + res = res or FILE_ATTRIBUTE_READONLY + wrapBinary(res2, setFileAttributesW, filename, res) + if res2 == - 1'i32: raiseOSError(osLastError(), $(filename, permissions)) + + +const hasCCopyfile = defined(osx) and not defined(nimLegacyCopyFile) + # xxx instead of `nimLegacyCopyFile`, support something like: `when osxVersion >= (10, 5)` + +when hasCCopyfile: + # `copyfile` API available since osx 10.5. + {.push nodecl, header: "<copyfile.h>".} + type + copyfile_state_t {.nodecl.} = pointer + copyfile_flags_t = cint + proc copyfile_state_alloc(): copyfile_state_t + proc copyfile_state_free(state: copyfile_state_t): cint + proc c_copyfile(src, dst: cstring, state: copyfile_state_t, flags: copyfile_flags_t): cint {.importc: "copyfile".} + when (NimMajor, NimMinor) >= (1, 4): + let + COPYFILE_DATA {.nodecl.}: copyfile_flags_t + COPYFILE_XATTR {.nodecl.}: copyfile_flags_t + else: + var + COPYFILE_DATA {.nodecl.}: copyfile_flags_t + COPYFILE_XATTR {.nodecl.}: copyfile_flags_t + {.pop.} + +type + CopyFlag* = enum ## Copy options. + cfSymlinkAsIs, ## Copy symlinks as symlinks + cfSymlinkFollow, ## Copy the files symlinks point to + cfSymlinkIgnore ## Ignore symlinks + +const copyFlagSymlink = {cfSymlinkAsIs, cfSymlinkFollow, cfSymlinkIgnore} + +proc copyFile*(source, dest: string, options = {cfSymlinkFollow}; bufferSize = 16_384) {.rtl, + extern: "nos$1", tags: [ReadDirEffect, ReadIOEffect, WriteIOEffect], + noWeirdTarget.} = + ## Copies a file from `source` to `dest`, where `dest.parentDir` must exist. + ## + ## On non-Windows OSes, `options` specify the way file is copied; by default, + ## if `source` is a symlink, copies the file symlink points to. `options` is + ## ignored on Windows: symlinks are skipped. + ## + ## If this fails, `OSError` is raised. + ## + ## On the Windows platform this proc will + ## copy the source file's attributes into dest. + ## + ## On other platforms you need + ## to use `getFilePermissions`_ and + ## `setFilePermissions`_ + ## procs + ## to copy them by hand (or use the convenience `copyFileWithPermissions + ## proc`_), + ## otherwise `dest` will inherit the default permissions of a newly + ## created file for the user. + ## + ## If `dest` already exists, the file attributes + ## will be preserved and the content overwritten. + ## + ## On OSX, `copyfile` C api will be used (available since OSX 10.5) unless + ## `-d:nimLegacyCopyFile` is used. + ## + ## `copyFile` allows to specify `bufferSize` to improve I/O performance. + ## + ## See also: + ## * `CopyFlag enum`_ + ## * `copyDir proc`_ + ## * `copyFileWithPermissions proc`_ + ## * `tryRemoveFile proc`_ + ## * `removeFile proc`_ + ## * `moveFile proc`_ + + doAssert card(copyFlagSymlink * options) == 1, "There should be exactly one cfSymlink* in options" + let isSymlink = source.symlinkExists + if isSymlink and (cfSymlinkIgnore in options or defined(windows)): + return + when defined(windows): + let s = newWideCString(source) + let d = newWideCString(dest) + if copyFileW(s, d, 0'i32) == 0'i32: + raiseOSError(osLastError(), $(source, dest)) + else: + if isSymlink and cfSymlinkAsIs in options: + createSymlink(expandSymlink(source), dest) + else: + when hasCCopyfile: + let state = copyfile_state_alloc() + # xxx `COPYFILE_STAT` could be used for one-shot + # `copyFileWithPermissions`. + let status = c_copyfile(source.cstring, dest.cstring, state, + COPYFILE_DATA) + if status != 0: + let err = osLastError() + discard copyfile_state_free(state) + raiseOSError(err, $(source, dest)) + let status2 = copyfile_state_free(state) + if status2 != 0: raiseOSError(osLastError(), $(source, dest)) + else: + # generic version of copyFile which works for any platform: + var d, s: File + if not open(s, source): raiseOSError(osLastError(), source) + if not open(d, dest, fmWrite): + close(s) + raiseOSError(osLastError(), dest) + + # Hints for kernel-level aggressive sequential low-fragmentation read-aheads: + # https://pubs.opengroup.org/onlinepubs/9699919799/functions/posix_fadvise.html + when defined(linux) or defined(osx): + discard posix_fadvise(getFileHandle(d), 0.cint, 0.cint, POSIX_FADV_SEQUENTIAL) + discard posix_fadvise(getFileHandle(s), 0.cint, 0.cint, POSIX_FADV_SEQUENTIAL) + + var buf = alloc(bufferSize) + while true: + var bytesread = readBuffer(s, buf, bufferSize) + if bytesread > 0: + var byteswritten = writeBuffer(d, buf, bytesread) + if bytesread != byteswritten: + dealloc(buf) + close(s) + close(d) + raiseOSError(osLastError(), dest) + if bytesread != bufferSize: break + dealloc(buf) + close(s) + flushFile(d) + close(d) + +proc copyFileToDir*(source, dir: string, options = {cfSymlinkFollow}; bufferSize = 16_384) + {.noWeirdTarget, since: (1,3,7).} = + ## Copies a file `source` into directory `dir`, which must exist. + ## + ## On non-Windows OSes, `options` specify the way file is copied; by default, + ## if `source` is a symlink, copies the file symlink points to. `options` is + ## ignored on Windows: symlinks are skipped. + ## + ## `copyFileToDir` allows to specify `bufferSize` to improve I/O performance. + ## + ## See also: + ## * `CopyFlag enum`_ + ## * `copyFile proc`_ + if dir.len == 0: # treating "" as "." is error prone + raise newException(ValueError, "dest is empty") + copyFile(source, dir / source.lastPathPart, options, bufferSize) + + +proc copyFileWithPermissions*(source, dest: string, + ignorePermissionErrors = true, + options = {cfSymlinkFollow}) {.noWeirdTarget.} = + ## Copies a file from `source` to `dest` preserving file permissions. + ## + ## On non-Windows OSes, `options` specify the way file is copied; by default, + ## if `source` is a symlink, copies the file symlink points to. `options` is + ## ignored on Windows: symlinks are skipped. + ## + ## This is a wrapper proc around `copyFile`_, + ## `getFilePermissions`_ and `setFilePermissions`_ + ## procs on non-Windows platforms. + ## + ## On Windows this proc is just a wrapper for `copyFile proc`_ since + ## that proc already copies attributes. + ## + ## On non-Windows systems permissions are copied after the file itself has + ## been copied, which won't happen atomically and could lead to a race + ## condition. If `ignorePermissionErrors` is true (default), errors while + ## reading/setting file attributes will be ignored, otherwise will raise + ## `OSError`. + ## + ## See also: + ## * `CopyFlag enum`_ + ## * `copyFile proc`_ + ## * `copyDir proc`_ + ## * `tryRemoveFile proc`_ + ## * `removeFile proc`_ + ## * `moveFile proc`_ + ## * `copyDirWithPermissions proc`_ + copyFile(source, dest, options) + when not defined(windows): + try: + setFilePermissions(dest, getFilePermissions(source), followSymlinks = + (cfSymlinkFollow in options)) + except: + if not ignorePermissionErrors: + raise + +when not declared(ENOENT) and not defined(windows): + when defined(nimscript): + when not defined(haiku): + const ENOENT = cint(2) # 2 on most systems including Solaris + else: + const ENOENT = cint(-2147459069) + else: + var ENOENT {.importc, header: "<errno.h>".}: cint + +when defined(windows) and not weirdTarget: + template deleteFile(file: untyped): untyped = deleteFileW(file) + template setFileAttributes(file, attrs: untyped): untyped = + setFileAttributesW(file, attrs) + +proc tryRemoveFile*(file: string): bool {.rtl, extern: "nos$1", tags: [WriteDirEffect], noWeirdTarget.} = + ## Removes the `file`. + ## + ## If this fails, returns `false`. This does not fail + ## if the file never existed in the first place. + ## + ## On Windows, ignores the read-only attribute. + ## + ## See also: + ## * `copyFile proc`_ + ## * `copyFileWithPermissions proc`_ + ## * `removeFile proc`_ + ## * `moveFile proc`_ + result = true + when defined(windows): + let f = newWideCString(file) + if deleteFile(f) == 0: + result = false + let err = getLastError() + if err == ERROR_FILE_NOT_FOUND or err == ERROR_PATH_NOT_FOUND: + result = true + elif err == ERROR_ACCESS_DENIED and + setFileAttributes(f, FILE_ATTRIBUTE_NORMAL) != 0 and + deleteFile(f) != 0: + result = true + else: + if unlink(file) != 0'i32 and errno != ENOENT: + result = false + +proc removeFile*(file: string) {.rtl, extern: "nos$1", tags: [WriteDirEffect], noWeirdTarget.} = + ## Removes the `file`. + ## + ## If this fails, `OSError` is raised. This does not fail + ## if the file never existed in the first place. + ## + ## On Windows, ignores the read-only attribute. + ## + ## See also: + ## * `removeDir proc`_ + ## * `copyFile proc`_ + ## * `copyFileWithPermissions proc`_ + ## * `tryRemoveFile proc`_ + ## * `moveFile proc`_ + if not tryRemoveFile(file): + raiseOSError(osLastError(), file) + +proc moveFile*(source, dest: string) {.rtl, extern: "nos$1", + tags: [ReadDirEffect, ReadIOEffect, WriteIOEffect], noWeirdTarget.} = + ## Moves a file from `source` to `dest`. + ## + ## Symlinks are not followed: if `source` is a symlink, it is itself moved, + ## not its target. + ## + ## If this fails, `OSError` is raised. + ## If `dest` already exists, it will be overwritten. + ## + ## Can be used to `rename files`:idx:. + ## + ## See also: + ## * `moveDir proc`_ + ## * `copyFile proc`_ + ## * `copyFileWithPermissions proc`_ + ## * `removeFile proc`_ + ## * `tryRemoveFile proc`_ + + if not tryMoveFSObject(source, dest, isDir = false): + when defined(windows): + raiseAssert "unreachable" + else: + # Fallback to copy & del + copyFileWithPermissions(source, dest, options={cfSymlinkAsIs}) + try: + removeFile(source) + except: + discard tryRemoveFile(dest) + raise diff --git a/lib/std/private/ospaths2.nim b/lib/std/private/ospaths2.nim new file mode 100644 index 000000000..bc69ff725 --- /dev/null +++ b/lib/std/private/ospaths2.nim @@ -0,0 +1,1030 @@ +include system/inclrtl +import std/private/since + +import std/[strutils, pathnorm] +import std/oserrors + +import oscommon +export ReadDirEffect, WriteDirEffect + +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions, widestrs] + +## .. importdoc:: osappdirs.nim, osdirs.nim, osseps.nim, os.nim + +const weirdTarget = defined(nimscript) or defined(js) + +when weirdTarget: + discard +elif defined(windows): + import std/winlean +elif defined(posix): + import std/posix, system/ansi_c +else: + {.error: "OS module not ported to your operating system!".} + +when weirdTarget: + {.pragma: noWeirdTarget, error: "this proc is not available on the NimScript/js target".} +else: + {.pragma: noWeirdTarget.} + +when defined(nimscript): + # for procs already defined in scriptconfig.nim + template noNimJs(body): untyped = discard +elif defined(js): + {.pragma: noNimJs, error: "this proc is not available on the js target".} +else: + {.pragma: noNimJs.} + + +proc normalizePathAux(path: var string){.inline, raises: [], noSideEffect.} + + +import std/private/osseps +export osseps + +proc absolutePathInternal(path: string): string {.gcsafe.} + +proc normalizePathEnd*(path: var string, trailingSep = false) = + ## Ensures ``path`` has exactly 0 or 1 trailing `DirSep`, depending on + ## ``trailingSep``, and taking care of edge cases: it preservers whether + ## a path is absolute or relative, and makes sure trailing sep is `DirSep`, + ## not `AltSep`. Trailing `/.` are compressed, see examples. + if path.len == 0: return + var i = path.len + while i >= 1: + if path[i-1] in {DirSep, AltSep}: dec(i) + elif path[i-1] == '.' and i >= 2 and path[i-2] in {DirSep, AltSep}: dec(i) + else: break + if trailingSep: + # foo// => foo + path.setLen(i) + # foo => foo/ + path.add DirSep + elif i > 0: + # foo// => foo + path.setLen(i) + else: + # // => / (empty case was already taken care of) + path = $DirSep + +proc normalizePathEnd*(path: string, trailingSep = false): string = + ## outplace overload + runnableExamples: + when defined(posix): + assert normalizePathEnd("/lib//.//", trailingSep = true) == "/lib/" + assert normalizePathEnd("lib/./.", trailingSep = false) == "lib" + assert normalizePathEnd(".//./.", trailingSep = false) == "." + assert normalizePathEnd("", trailingSep = true) == "" # not / ! + assert normalizePathEnd("/", trailingSep = false) == "/" # not "" ! + result = path + result.normalizePathEnd(trailingSep) + +template endsWith(a: string, b: set[char]): bool = + a.len > 0 and a[^1] in b + +proc joinPathImpl(result: var string, state: var int, tail: string) = + let trailingSep = tail.endsWith({DirSep, AltSep}) or tail.len == 0 and result.endsWith({DirSep, AltSep}) + normalizePathEnd(result, trailingSep=false) + addNormalizePath(tail, result, state, DirSep) + normalizePathEnd(result, trailingSep=trailingSep) + +proc joinPath*(head, tail: string): string {. + noSideEffect, rtl, extern: "nos$1".} = + ## Joins two directory names to one. + ## + ## returns normalized path concatenation of `head` and `tail`, preserving + ## whether or not `tail` has a trailing slash (or, if tail if empty, whether + ## head has one). + ## + ## See also: + ## * `joinPath(parts: varargs[string]) proc`_ + ## * `/ proc`_ + ## * `splitPath proc`_ + ## * `uri.combine proc <uri.html#combine,Uri,Uri>`_ + ## * `uri./ proc <uri.html#/,Uri,string>`_ + runnableExamples: + when defined(posix): + assert joinPath("usr", "lib") == "usr/lib" + assert joinPath("usr", "lib/") == "usr/lib/" + assert joinPath("usr", "") == "usr" + assert joinPath("usr/", "") == "usr/" + assert joinPath("", "") == "" + assert joinPath("", "lib") == "lib" + assert joinPath("", "/lib") == "/lib" + assert joinPath("usr/", "/lib") == "usr/lib" + assert joinPath("usr/lib", "../bin") == "usr/bin" + + result = newStringOfCap(head.len + tail.len) + var state = 0 + joinPathImpl(result, state, head) + joinPathImpl(result, state, tail) + when false: + if len(head) == 0: + result = tail + elif head[len(head)-1] in {DirSep, AltSep}: + if tail.len > 0 and tail[0] in {DirSep, AltSep}: + result = head & substr(tail, 1) + else: + result = head & tail + else: + if tail.len > 0 and tail[0] in {DirSep, AltSep}: + result = head & tail + else: + result = head & DirSep & tail + +proc joinPath*(parts: varargs[string]): string {.noSideEffect, + rtl, extern: "nos$1OpenArray".} = + ## The same as `joinPath(head, tail) proc`_, + ## but works with any number of directory parts. + ## + ## You need to pass at least one element or the proc + ## will assert in debug builds and crash on release builds. + ## + ## See also: + ## * `joinPath(head, tail) proc`_ + ## * `/ proc`_ + ## * `/../ proc`_ + ## * `splitPath proc`_ + runnableExamples: + when defined(posix): + assert joinPath("a") == "a" + assert joinPath("a", "b", "c") == "a/b/c" + assert joinPath("usr/lib", "../../var", "log") == "var/log" + + var estimatedLen = 0 + for p in parts: estimatedLen += p.len + result = newStringOfCap(estimatedLen) + var state = 0 + for i in 0..high(parts): + joinPathImpl(result, state, parts[i]) + +proc `/`*(head, tail: string): string {.noSideEffect, inline.} = + ## The same as `joinPath(head, tail) proc`_. + ## + ## See also: + ## * `/../ proc`_ + ## * `joinPath(head, tail) proc`_ + ## * `joinPath(parts: varargs[string]) proc`_ + ## * `splitPath proc`_ + ## * `uri.combine proc <uri.html#combine,Uri,Uri>`_ + ## * `uri./ proc <uri.html#/,Uri,string>`_ + runnableExamples: + when defined(posix): + assert "usr" / "" == "usr" + assert "" / "lib" == "lib" + assert "" / "/lib" == "/lib" + assert "usr/" / "/lib/" == "usr/lib/" + assert "usr" / "lib" / "../bin" == "usr/bin" + + result = joinPath(head, tail) + +when doslikeFileSystem: + import std/private/ntpath + +proc splitPath*(path: string): tuple[head, tail: string] {. + noSideEffect, rtl, extern: "nos$1".} = + ## Splits a directory into `(head, tail)` tuple, so that + ## ``head / tail == path`` (except for edge cases like "/usr"). + ## + ## See also: + ## * `joinPath(head, tail) proc`_ + ## * `joinPath(parts: varargs[string]) proc`_ + ## * `/ proc`_ + ## * `/../ proc`_ + ## * `relativePath proc`_ + runnableExamples: + assert splitPath("usr/local/bin") == ("usr/local", "bin") + assert splitPath("usr/local/bin/") == ("usr/local/bin", "") + assert splitPath("/bin/") == ("/bin", "") + when (NimMajor, NimMinor) <= (1, 0): + assert splitPath("/bin") == ("", "bin") + else: + assert splitPath("/bin") == ("/", "bin") + assert splitPath("bin") == ("", "bin") + assert splitPath("") == ("", "") + + when doslikeFileSystem: + let (drive, splitpath) = splitDrive(path) + let stop = drive.len + else: + const stop = 0 + + var sepPos = -1 + for i in countdown(len(path)-1, stop): + if path[i] in {DirSep, AltSep}: + sepPos = i + break + if sepPos >= 0: + result.head = substr(path, 0, + when (NimMajor, NimMinor) <= (1, 0): + sepPos-1 + else: + if likely(sepPos >= 1): sepPos-1 else: 0 + ) + result.tail = substr(path, sepPos+1) + else: + when doslikeFileSystem: + result.head = drive + result.tail = splitpath + else: + result.head = "" + result.tail = path + +proc isAbsolute*(path: string): bool {.rtl, noSideEffect, extern: "nos$1", raises: [].} = + ## Checks whether a given `path` is absolute. + ## + ## On Windows, network paths are considered absolute too. + runnableExamples: + assert not "".isAbsolute + assert not ".".isAbsolute + when defined(posix): + assert "/".isAbsolute + assert not "a/".isAbsolute + assert "/a/".isAbsolute + + if len(path) == 0: return false + + when doslikeFileSystem: + var len = len(path) + result = (path[0] in {'/', '\\'}) or + (len > 1 and path[0] in {'a'..'z', 'A'..'Z'} and path[1] == ':') + elif defined(macos): + # according to https://perldoc.perl.org/File/Spec/Mac.html `:a` is a relative path + result = path[0] != ':' + elif defined(RISCOS): + result = path[0] == '$' + elif defined(posix): + result = path[0] == '/' + elif defined(nodejs): + {.emit: [result," = require(\"path\").isAbsolute(",path.cstring,");"].} + else: + raiseAssert "unreachable" # if ever hits here, adapt as needed + +when FileSystemCaseSensitive: + template `!=?`(a, b: char): bool = a != b +else: + template `!=?`(a, b: char): bool = toLowerAscii(a) != toLowerAscii(b) + +when doslikeFileSystem: + proc isAbsFromCurrentDrive(path: string): bool {.noSideEffect, raises: [].} = + ## An absolute path from the root of the current drive (e.g. "\foo") + path.len > 0 and + (path[0] == AltSep or + (path[0] == DirSep and + (path.len == 1 or path[1] notin {DirSep, AltSep, ':'}))) + + proc sameRoot(path1, path2: string): bool {.noSideEffect, raises: [].} = + ## Return true if path1 and path2 have a same root. + ## + ## Detail of Windows path formats: + ## https://docs.microsoft.com/en-us/dotnet/standard/io/file-path-formats + + assert(isAbsolute(path1)) + assert(isAbsolute(path2)) + + if isAbsFromCurrentDrive(path1) and isAbsFromCurrentDrive(path2): + result = true + elif cmpIgnoreCase(splitDrive(path1).drive, splitDrive(path2).drive) == 0: + result = true + else: + result = false + +proc relativePath*(path, base: string, sep = DirSep): string {. + rtl, extern: "nos$1".} = + ## Converts `path` to a path relative to `base`. + ## + ## The `sep` (default: DirSep_) is used for the path normalizations, + ## this can be useful to ensure the relative path only contains `'/'` + ## so that it can be used for URL constructions. + ## + ## On Windows, if a root of `path` and a root of `base` are different, + ## returns `path` as is because it is impossible to make a relative path. + ## That means an absolute path can be returned. + ## + ## See also: + ## * `splitPath proc`_ + ## * `parentDir proc`_ + ## * `tailDir proc`_ + runnableExamples: + assert relativePath("/Users/me/bar/z.nim", "/Users/other/bad", '/') == "../../me/bar/z.nim" + assert relativePath("/Users/me/bar/z.nim", "/Users/other", '/') == "../me/bar/z.nim" + when not doslikeFileSystem: # On Windows, UNC-paths start with `//` + assert relativePath("/Users///me/bar//z.nim", "//Users/", '/') == "me/bar/z.nim" + assert relativePath("/Users/me/bar/z.nim", "/Users/me", '/') == "bar/z.nim" + assert relativePath("", "/users/moo", '/') == "" + assert relativePath("foo", ".", '/') == "foo" + assert relativePath("foo", "foo", '/') == "." + + if path.len == 0: return "" + var base = if base == ".": "" else: base + var path = path + path.normalizePathAux + base.normalizePathAux + let a1 = isAbsolute(path) + let a2 = isAbsolute(base) + if a1 and not a2: + base = absolutePathInternal(base) + elif a2 and not a1: + path = absolutePathInternal(path) + + when doslikeFileSystem: + if isAbsolute(path) and isAbsolute(base): + if not sameRoot(path, base): + return path + + var f = default PathIter + var b = default PathIter + var ff = (0, -1) + var bb = (0, -1) # (int, int) + result = newStringOfCap(path.len) + # skip the common prefix: + while f.hasNext(path) and b.hasNext(base): + ff = next(f, path) + bb = next(b, base) + let diff = ff[1] - ff[0] + if diff != bb[1] - bb[0]: break + var same = true + for i in 0..diff: + if path[i + ff[0]] !=? base[i + bb[0]]: + same = false + break + if not same: break + ff = (0, -1) + bb = (0, -1) + # for i in 0..diff: + # result.add base[i + bb[0]] + + # /foo/bar/xxx/ -- base + # /foo/bar/baz -- path path + # ../baz + # every directory that is in 'base', needs to add '..' + while true: + if bb[1] >= bb[0]: + if result.len > 0 and result[^1] != sep: + result.add sep + result.add ".." + if not b.hasNext(base): break + bb = b.next(base) + + # add the rest of 'path': + while true: + if ff[1] >= ff[0]: + if result.len > 0 and result[^1] != sep: + result.add sep + for i in 0..ff[1] - ff[0]: + result.add path[i + ff[0]] + if not f.hasNext(path): break + ff = f.next(path) + + when not defined(nimOldRelativePathBehavior): + if result.len == 0: result.add "." + +proc isRelativeTo*(path: string, base: string): bool {.since: (1, 1).} = + ## Returns true if `path` is relative to `base`. + runnableExamples: + doAssert isRelativeTo("./foo//bar", "foo") + doAssert isRelativeTo("foo/bar", ".") + doAssert isRelativeTo("/foo/bar.nim", "/foo/bar.nim") + doAssert not isRelativeTo("foo/bar.nims", "foo/bar.nim") + let path = path.normalizePath + let base = base.normalizePath + let ret = relativePath(path, base) + result = path.len > 0 and not ret.startsWith ".." + +proc parentDirPos(path: string): int = + var q = 1 + if len(path) >= 1 and path[len(path)-1] in {DirSep, AltSep}: q = 2 + for i in countdown(len(path)-q, 0): + if path[i] in {DirSep, AltSep}: return i + result = -1 + +proc parentDir*(path: string): string {. + noSideEffect, rtl, extern: "nos$1".} = + ## Returns the parent directory of `path`. + ## + ## This is similar to ``splitPath(path).head`` when ``path`` doesn't end + ## in a dir separator, but also takes care of path normalizations. + ## The remainder can be obtained with `lastPathPart(path) proc`_. + ## + ## See also: + ## * `relativePath proc`_ + ## * `splitPath proc`_ + ## * `tailDir proc`_ + ## * `parentDirs iterator`_ + runnableExamples: + assert parentDir("") == "" + when defined(posix): + assert parentDir("/usr/local/bin") == "/usr/local" + assert parentDir("foo/bar//") == "foo" + assert parentDir("//foo//bar//.") == "/foo" + assert parentDir("./foo") == "." + assert parentDir("/./foo//./") == "/" + assert parentDir("a//./") == "." + assert parentDir("a/b/c/..") == "a" + result = pathnorm.normalizePath(path) + when doslikeFileSystem: + let (drive, splitpath) = splitDrive(result) + result = splitpath + var sepPos = parentDirPos(result) + if sepPos >= 0: + result = substr(result, 0, sepPos) + normalizePathEnd(result) + elif result == ".." or result == "." or result.len == 0 or result[^1] in {DirSep, AltSep}: + # `.` => `..` and .. => `../..`(etc) would be a sensible alternative + # `/` => `/` (as done with splitFile) would be a sensible alternative + result = "" + else: + result = "." + when doslikeFileSystem: + if result.len == 0: + discard + elif drive.len > 0 and result.len == 1 and result[0] in {DirSep, AltSep}: + result = drive + else: + result = drive & result + +proc tailDir*(path: string): string {. + noSideEffect, rtl, extern: "nos$1".} = + ## Returns the tail part of `path`. + ## + ## See also: + ## * `relativePath proc`_ + ## * `splitPath proc`_ + ## * `parentDir proc`_ + runnableExamples: + assert tailDir("/bin") == "bin" + assert tailDir("bin") == "" + assert tailDir("bin/") == "" + assert tailDir("/usr/local/bin") == "usr/local/bin" + assert tailDir("//usr//local//bin//") == "usr//local//bin//" + assert tailDir("./usr/local/bin") == "usr/local/bin" + assert tailDir("usr/local/bin") == "local/bin" + + var i = 0 + when doslikeFileSystem: + let (drive, splitpath) = path.splitDrive + if drive != "": + return splitpath.strip(chars = {DirSep, AltSep}, trailing = false) + while i < len(path): + if path[i] in {DirSep, AltSep}: + while i < len(path) and path[i] in {DirSep, AltSep}: inc i + return substr(path, i) + inc i + result = "" + +proc isRootDir*(path: string): bool {. + noSideEffect, rtl, extern: "nos$1".} = + ## Checks whether a given `path` is a root directory. + runnableExamples: + assert isRootDir("") + assert isRootDir(".") + assert isRootDir("/") + assert isRootDir("a") + assert not isRootDir("/a") + assert not isRootDir("a/b/c") + + when doslikeFileSystem: + if splitDrive(path).path == "": + return true + result = parentDirPos(path) < 0 + +iterator parentDirs*(path: string, fromRoot=false, inclusive=true): string = + ## Walks over all parent directories of a given `path`. + ## + ## If `fromRoot` is true (default: false), the traversal will start from + ## the file system root directory. + ## If `inclusive` is true (default), the original argument will be included + ## in the traversal. + ## + ## Relative paths won't be expanded by this iterator. Instead, it will traverse + ## only the directories appearing in the relative path. + ## + ## See also: + ## * `parentDir proc`_ + ## + runnableExamples: + let g = "a/b/c" + + for p in g.parentDirs: + echo p + # a/b/c + # a/b + # a + + for p in g.parentDirs(fromRoot=true): + echo p + # a/ + # a/b/ + # a/b/c + + for p in g.parentDirs(inclusive=false): + echo p + # a/b + # a + + if not fromRoot: + var current = path + if inclusive: yield path + while true: + if current.isRootDir: break + current = current.parentDir + yield current + else: + when doslikeFileSystem: + let start = path.splitDrive.drive.len + else: + const start = 0 + for i in countup(start, path.len - 2): # ignore the last / + # deal with non-normalized paths such as /foo//bar//baz + if path[i] in {DirSep, AltSep} and + (i == 0 or path[i-1] notin {DirSep, AltSep}): + yield path.substr(0, i) + + if inclusive: yield path + +proc `/../`*(head, tail: string): string {.noSideEffect.} = + ## The same as ``parentDir(head) / tail``, unless there is no parent + ## directory. Then ``head / tail`` is performed instead. + ## + ## See also: + ## * `/ proc`_ + ## * `parentDir proc`_ + runnableExamples: + when defined(posix): + assert "a/b/c" /../ "d/e" == "a/b/d/e" + assert "a" /../ "d/e" == "a/d/e" + + when doslikeFileSystem: + let (drive, head) = splitDrive(head) + let sepPos = parentDirPos(head) + if sepPos >= 0: + result = substr(head, 0, sepPos-1) / tail + else: + result = head / tail + when doslikeFileSystem: + result = drive / result + +proc normExt(ext: string): string = + if ext == "" or ext[0] == ExtSep: result = ext # no copy needed here + else: result = ExtSep & ext + +proc searchExtPos*(path: string): int = + ## Returns index of the `'.'` char in `path` if it signifies the beginning + ## of the file extension. Returns -1 otherwise. + ## + ## See also: + ## * `splitFile proc`_ + ## * `extractFilename proc`_ + ## * `lastPathPart proc`_ + ## * `changeFileExt proc`_ + ## * `addFileExt proc`_ + runnableExamples: + assert searchExtPos("a/b/c") == -1 + assert searchExtPos("c.nim") == 1 + assert searchExtPos("a/b/c.nim") == 5 + assert searchExtPos("a.b.c.nim") == 5 + assert searchExtPos(".nim") == -1 + assert searchExtPos("..nim") == -1 + assert searchExtPos("a..nim") == 2 + + # Unless there is any char that is not `ExtSep` before last `ExtSep` in the file name, + # it is not a file extension. + const DirSeps = when doslikeFileSystem: {DirSep, AltSep, ':'} else: {DirSep, AltSep} + result = -1 + var i = path.high + while i >= 1: + if path[i] == ExtSep: + break + elif path[i] in DirSeps: + return -1 # do not skip over path + dec i + + for j in countdown(i - 1, 0): + if path[j] in DirSeps: + return -1 + elif path[j] != ExtSep: + result = i + break + +proc splitFile*(path: string): tuple[dir, name, ext: string] {. + noSideEffect, rtl, extern: "nos$1".} = + ## Splits a filename into `(dir, name, extension)` tuple. + ## + ## `dir` does not end in DirSep_ unless it's `/`. + ## `extension` includes the leading dot. + ## + ## If `path` has no extension, `ext` is the empty string. + ## If `path` has no directory component, `dir` is the empty string. + ## If `path` has no filename component, `name` and `ext` are empty strings. + ## + ## See also: + ## * `searchExtPos proc`_ + ## * `extractFilename proc`_ + ## * `lastPathPart proc`_ + ## * `changeFileExt proc`_ + ## * `addFileExt proc`_ + runnableExamples: + var (dir, name, ext) = splitFile("usr/local/nimc.html") + assert dir == "usr/local" + assert name == "nimc" + assert ext == ".html" + (dir, name, ext) = splitFile("/usr/local/os") + assert dir == "/usr/local" + assert name == "os" + assert ext == "" + (dir, name, ext) = splitFile("/usr/local/") + assert dir == "/usr/local" + assert name == "" + assert ext == "" + (dir, name, ext) = splitFile("/tmp.txt") + assert dir == "/" + assert name == "tmp" + assert ext == ".txt" + + var namePos = 0 + var dotPos = 0 + when doslikeFileSystem: + let (drive, _) = splitDrive(path) + let stop = len(drive) + result.dir = drive + else: + const stop = 0 + for i in countdown(len(path) - 1, stop): + if path[i] in {DirSep, AltSep} or i == 0: + if path[i] in {DirSep, AltSep}: + result.dir = substr(path, 0, if likely(i >= 1): i - 1 else: 0) + namePos = i + 1 + if dotPos > i: + result.name = substr(path, namePos, dotPos - 1) + result.ext = substr(path, dotPos) + else: + result.name = substr(path, namePos) + break + elif path[i] == ExtSep and i > 0 and i < len(path) - 1 and + path[i - 1] notin {DirSep, AltSep} and + path[i + 1] != ExtSep and dotPos == 0: + dotPos = i + +proc extractFilename*(path: string): string {. + noSideEffect, rtl, extern: "nos$1".} = + ## Extracts the filename of a given `path`. + ## + ## This is the same as ``name & ext`` from `splitFile(path) proc`_. + ## + ## See also: + ## * `searchExtPos proc`_ + ## * `splitFile proc`_ + ## * `lastPathPart proc`_ + ## * `changeFileExt proc`_ + ## * `addFileExt proc`_ + runnableExamples: + assert extractFilename("foo/bar/") == "" + assert extractFilename("foo/bar") == "bar" + assert extractFilename("foo/bar.baz") == "bar.baz" + + if path.len == 0 or path[path.len-1] in {DirSep, AltSep}: + result = "" + else: + result = splitPath(path).tail + +proc lastPathPart*(path: string): string {. + noSideEffect, rtl, extern: "nos$1".} = + ## Like `extractFilename proc`_, but ignores + ## trailing dir separator; aka: `baseName`:idx: in some other languages. + ## + ## See also: + ## * `searchExtPos proc`_ + ## * `splitFile proc`_ + ## * `extractFilename proc`_ + ## * `changeFileExt proc`_ + ## * `addFileExt proc`_ + runnableExamples: + assert lastPathPart("foo/bar/") == "bar" + assert lastPathPart("foo/bar") == "bar" + + let path = path.normalizePathEnd(trailingSep = false) + result = extractFilename(path) + +proc changeFileExt*(filename, ext: string): string {. + noSideEffect, rtl, extern: "nos$1".} = + ## Changes the file extension to `ext`. + ## + ## If the `filename` has no extension, `ext` will be added. + ## If `ext` == "" then any extension is removed. + ## + ## `Ext` should be given without the leading `'.'`, because some + ## filesystems may use a different character. (Although I know + ## of none such beast.) + ## + ## See also: + ## * `searchExtPos proc`_ + ## * `splitFile proc`_ + ## * `extractFilename proc`_ + ## * `lastPathPart proc`_ + ## * `addFileExt proc`_ + runnableExamples: + assert changeFileExt("foo.bar", "baz") == "foo.baz" + assert changeFileExt("foo.bar", "") == "foo" + assert changeFileExt("foo", "baz") == "foo.baz" + + var extPos = searchExtPos(filename) + if extPos < 0: result = filename & normExt(ext) + else: result = substr(filename, 0, extPos-1) & normExt(ext) + +proc addFileExt*(filename, ext: string): string {. + noSideEffect, rtl, extern: "nos$1".} = + ## Adds the file extension `ext` to `filename`, unless + ## `filename` already has an extension. + ## + ## `Ext` should be given without the leading `'.'`, because some + ## filesystems may use a different character. + ## (Although I know of none such beast.) + ## + ## See also: + ## * `searchExtPos proc`_ + ## * `splitFile proc`_ + ## * `extractFilename proc`_ + ## * `lastPathPart proc`_ + ## * `changeFileExt proc`_ + runnableExamples: + assert addFileExt("foo.bar", "baz") == "foo.bar" + assert addFileExt("foo.bar", "") == "foo.bar" + assert addFileExt("foo", "baz") == "foo.baz" + + var extPos = searchExtPos(filename) + if extPos < 0: result = filename & normExt(ext) + else: result = filename + +proc cmpPaths*(pathA, pathB: string): int {. + noSideEffect, rtl, extern: "nos$1".} = + ## Compares two paths. + ## + ## On a case-sensitive filesystem this is done + ## case-sensitively otherwise case-insensitively. Returns: + ## + ## | `0` if pathA == pathB + ## | `< 0` if pathA < pathB + ## | `> 0` if pathA > pathB + runnableExamples: + when defined(macosx): + assert cmpPaths("foo", "Foo") == 0 + elif defined(posix): + assert cmpPaths("foo", "Foo") > 0 + + let a = normalizePath(pathA) + let b = normalizePath(pathB) + if FileSystemCaseSensitive: + result = cmp(a, b) + else: + when defined(nimscript): + result = cmpic(a, b) + elif defined(nimdoc): discard + else: + result = cmpIgnoreCase(a, b) + +proc unixToNativePath*(path: string, drive=""): string {. + noSideEffect, rtl, extern: "nos$1".} = + ## Converts an UNIX-like path to a native one. + ## + ## On an UNIX system this does nothing. Else it converts + ## `'/'`, `'.'`, `'..'` to the appropriate things. + ## + ## On systems with a concept of "drives", `drive` is used to determine + ## which drive label to use during absolute path conversion. + ## `drive` defaults to the drive of the current working directory, and is + ## ignored on systems that do not have a concept of "drives". + when defined(unix): + result = path + else: + if path.len == 0: return "" + + var start: int + if path[0] == '/': + # an absolute path + when doslikeFileSystem: + if drive != "": + result = drive & ":" & DirSep + else: + result = $DirSep + elif defined(macos): + result = "" # must not start with ':' + else: + result = $DirSep + start = 1 + elif path[0] == '.' and (path.len == 1 or path[1] == '/'): + # current directory + result = $CurDir + start = when doslikeFileSystem: 1 else: 2 + else: + result = "" + start = 0 + + var i = start + while i < len(path): # ../../../ --> :::: + if i+2 < path.len and path[i] == '.' and path[i+1] == '.' and path[i+2] == '/': + # parent directory + when defined(macos): + if result[high(result)] == ':': + add result, ':' + else: + add result, ParDir + else: + add result, ParDir & DirSep + inc(i, 3) + elif path[i] == '/': + add result, DirSep + inc(i) + else: + add result, path[i] + inc(i) + + +when not defined(nimscript): + proc getCurrentDir*(): string {.rtl, extern: "nos$1", tags: [].} = + ## Returns the `current working directory`:idx: i.e. where the built + ## binary is run. + ## + ## So the path returned by this proc is determined at run time. + ## + ## See also: + ## * `getHomeDir proc`_ + ## * `getConfigDir proc`_ + ## * `getTempDir proc`_ + ## * `setCurrentDir proc`_ + ## * `currentSourcePath template <system.html#currentSourcePath.t>`_ + ## * `getProjectPath proc <macros.html#getProjectPath>`_ + when defined(nodejs): + var ret: cstring + {.emit: "`ret` = process.cwd();".} + return $ret + elif defined(js): + raiseAssert "use -d:nodejs to have `getCurrentDir` defined" + elif defined(windows): + var bufsize = MAX_PATH.int32 + var res = newWideCString(bufsize) + while true: + var L = getCurrentDirectoryW(bufsize, res) + if L == 0'i32: + raiseOSError(osLastError()) + elif L > bufsize: + res = newWideCString(L) + bufsize = L + else: + result = res$L + break + else: + var bufsize = 1024 # should be enough + result = newString(bufsize) + while true: + if getcwd(result.cstring, bufsize) != nil: + setLen(result, c_strlen(result.cstring)) + break + else: + let err = osLastError() + if err.int32 == ERANGE: + bufsize = bufsize shl 1 + doAssert(bufsize >= 0) + result = newString(bufsize) + else: + raiseOSError(osLastError()) + +proc absolutePath*(path: string, root = getCurrentDir()): string = + ## Returns the absolute path of `path`, rooted at `root` (which must be absolute; + ## default: current directory). + ## If `path` is absolute, return it, ignoring `root`. + ## + ## See also: + ## * `normalizedPath proc`_ + ## * `normalizePath proc`_ + runnableExamples: + assert absolutePath("a") == getCurrentDir() / "a" + + if isAbsolute(path): path + else: + if not root.isAbsolute: + raise newException(ValueError, "The specified root is not absolute: " & root) + joinPath(root, path) + +proc absolutePathInternal(path: string): string = + absolutePath(path, getCurrentDir()) + + +proc normalizePath*(path: var string) {.rtl, extern: "nos$1", tags: [].} = + ## Normalize a path. + ## + ## Consecutive directory separators are collapsed, including an initial double slash. + ## + ## On relative paths, double dot (`..`) sequences are collapsed if possible. + ## On absolute paths they are always collapsed. + ## + ## .. warning:: URL-encoded and Unicode attempts at directory traversal are not detected. + ## Triple dot is not handled. + ## + ## See also: + ## * `absolutePath proc`_ + ## * `normalizedPath proc`_ for outplace version + ## * `normalizeExe proc`_ + runnableExamples: + when defined(posix): + var a = "a///b//..//c///d" + a.normalizePath() + assert a == "a/c/d" + + path = pathnorm.normalizePath(path) + when false: + let isAbs = isAbsolute(path) + var stack: seq[string] = @[] + for p in split(path, {DirSep}): + case p + of "", ".": + continue + of "..": + if stack.len == 0: + if isAbs: + discard # collapse all double dots on absoluta paths + else: + stack.add(p) + elif stack[^1] == "..": + stack.add(p) + else: + discard stack.pop() + else: + stack.add(p) + + if isAbs: + path = DirSep & join(stack, $DirSep) + elif stack.len > 0: + path = join(stack, $DirSep) + else: + path = "." + +proc normalizePathAux(path: var string) = normalizePath(path) + +proc normalizedPath*(path: string): string {.rtl, extern: "nos$1", tags: [].} = + ## Returns a normalized path for the current OS. + ## + ## See also: + ## * `absolutePath proc`_ + ## * `normalizePath proc`_ for the in-place version + runnableExamples: + when defined(posix): + assert normalizedPath("a///b//..//c///d") == "a/c/d" + result = pathnorm.normalizePath(path) + +proc normalizeExe*(file: var string) {.since: (1, 3, 5).} = + ## on posix, prepends `./` if `file` doesn't contain `/` and is not `"", ".", ".."`. + runnableExamples: + import std/sugar + when defined(posix): + doAssert "foo".dup(normalizeExe) == "./foo" + doAssert "foo/../bar".dup(normalizeExe) == "foo/../bar" + doAssert "".dup(normalizeExe) == "" + when defined(posix): + if file.len > 0 and DirSep notin file and file != "." and file != "..": + file = "./" & file + +proc sameFile*(path1, path2: string): bool {.rtl, extern: "nos$1", + tags: [ReadDirEffect], noWeirdTarget.} = + ## Returns true if both pathname arguments refer to the same physical + ## file or directory. + ## + ## Raises `OSError` if any of the files does not + ## exist or information about it can not be obtained. + ## + ## This proc will return true if given two alternative hard-linked or + ## sym-linked paths to the same file or directory. + ## + ## See also: + ## * `sameFileContent proc`_ + when defined(windows): + var success = true + var f1 = openHandle(path1) + var f2 = openHandle(path2) + + var lastErr: OSErrorCode + if f1 != INVALID_HANDLE_VALUE and f2 != INVALID_HANDLE_VALUE: + var fi1, fi2: BY_HANDLE_FILE_INFORMATION + + if getFileInformationByHandle(f1, addr(fi1)) != 0 and + getFileInformationByHandle(f2, addr(fi2)) != 0: + result = fi1.dwVolumeSerialNumber == fi2.dwVolumeSerialNumber and + fi1.nFileIndexHigh == fi2.nFileIndexHigh and + fi1.nFileIndexLow == fi2.nFileIndexLow + else: + lastErr = osLastError() + success = false + else: + lastErr = osLastError() + success = false + + discard closeHandle(f1) + discard closeHandle(f2) + + if not success: raiseOSError(lastErr, $(path1, path2)) + else: + var a, b: Stat + if stat(path1, a) < 0'i32 or stat(path2, b) < 0'i32: + raiseOSError(osLastError(), $(path1, path2)) + else: + result = a.st_dev == b.st_dev and a.st_ino == b.st_ino diff --git a/lib/std/private/osseps.nim b/lib/std/private/osseps.nim new file mode 100644 index 000000000..f2d49d886 --- /dev/null +++ b/lib/std/private/osseps.nim @@ -0,0 +1,113 @@ +# Include file that implements 'DirSep' and friends. Do not import this when +# you also import `os.nim`! + +# Improved based on info in 'compiler/platform.nim' + +## .. importdoc:: ospaths2.nim + +const + doslikeFileSystem* = defined(windows) or defined(OS2) or defined(DOS) + +const + CurDir* = + when defined(macos): ':' + elif defined(genode): '/' + else: '.' + ## The constant character used by the operating system to refer to the + ## current directory. + ## + ## For example: `'.'` for POSIX or `':'` for the classic Macintosh. + + ParDir* = + when defined(macos): "::" + else: ".." + ## The constant string used by the operating system to refer to the + ## parent directory. + ## + ## For example: `".."` for POSIX or `"::"` for the classic Macintosh. + + DirSep* = + when defined(macos): ':' + elif doslikeFileSystem or defined(vxworks): '\\' + elif defined(RISCOS): '.' + else: '/' + ## The character used by the operating system to separate pathname + ## components, for example: `'/'` for POSIX, `':'` for the classic + ## Macintosh, and `'\\'` on Windows. + + AltSep* = + when doslikeFileSystem: '/' + else: DirSep + ## An alternative character used by the operating system to separate + ## pathname components, or the same as DirSep_ if only one separator + ## character exists. This is set to `'/'` on Windows systems + ## where DirSep_ is a backslash (`'\\'`). + + PathSep* = + when defined(macos) or defined(RISCOS): ',' + elif doslikeFileSystem or defined(vxworks): ';' + elif defined(PalmOS) or defined(MorphOS): ':' # platform has ':' but osseps has ';' + else: ':' + ## The character conventionally used by the operating system to separate + ## search path components (as in PATH), such as `':'` for POSIX + ## or `';'` for Windows. + + FileSystemCaseSensitive* = + when defined(macos) or defined(macosx) or doslikeFileSystem or defined(vxworks) or + defined(PalmOS) or defined(MorphOS): false + else: true + ## True if the file system is case sensitive, false otherwise. Used by + ## `cmpPaths proc`_ to compare filenames properly. + + ExeExt* = + when doslikeFileSystem: "exe" + elif defined(atari): "tpp" + elif defined(netware): "nlm" + elif defined(vxworks): "vxe" + elif defined(nintendoswitch): "elf" + else: "" + ## The file extension of native executables. For example: + ## `""` for POSIX, `"exe"` on Windows (without a dot). + + ScriptExt* = + when doslikeFileSystem: "bat" + else: "" + ## The file extension of a script file. For example: `""` for POSIX, + ## `"bat"` on Windows. + + DynlibFormat* = + when defined(macos): "$1.dylib" # platform has $1Lib + elif defined(macosx): "lib$1.dylib" + elif doslikeFileSystem or defined(atari): "$1.dll" + elif defined(MorphOS): "$1.prc" + elif defined(PalmOS): "$1.prc" # platform has lib$1.so + elif defined(genode): "$1.lib.so" + elif defined(netware): "$1.nlm" + elif defined(amiga): "$1.Library" + else: "lib$1.so" + ## The format string to turn a filename into a `DLL`:idx: file (also + ## called `shared object`:idx: on some operating systems). + + ExtSep* = '.' + ## The character which separates the base filename from the extension; + ## for example, the `'.'` in `os.nim`. + + # MacOS paths + # =========== + # MacOS directory separator is a colon ":" which is the only character not + # allowed in filenames. + # + # A path containing no colon or which begins with a colon is a partial + # path. + # E.g. ":kalle:petter" ":kalle" "kalle" + # + # All other paths are full (absolute) paths. E.g. "HD:kalle:" "HD:" + # When generating paths, one is safe if one ensures that all partial paths + # begin with a colon, and all full paths end with a colon. + # In full paths the first name (e g HD above) is the name of a mounted + # volume. + # These names are not unique, because, for instance, two diskettes with the + # same names could be inserted. This means that paths on MacOS are not + # waterproof. In case of equal names the first volume found will do. + # Two colons "::" are the relative path to the parent. Three is to the + # grandparent etc. diff --git a/lib/std/private/ossymlinks.nim b/lib/std/private/ossymlinks.nim new file mode 100644 index 000000000..c1760c42e --- /dev/null +++ b/lib/std/private/ossymlinks.nim @@ -0,0 +1,78 @@ +include system/inclrtl +import std/oserrors + +import oscommon +export symlinkExists + +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions, widestrs] + +when weirdTarget: + discard +elif defined(windows): + import std/[winlean, times] +elif defined(posix): + import std/posix +else: + {.error: "OS module not ported to your operating system!".} + + +when weirdTarget: + {.pragma: noWeirdTarget, error: "this proc is not available on the NimScript/js target".} +else: + {.pragma: noWeirdTarget.} + + +when defined(nimscript): + # for procs already defined in scriptconfig.nim + template noNimJs(body): untyped = discard +elif defined(js): + {.pragma: noNimJs, error: "this proc is not available on the js target".} +else: + {.pragma: noNimJs.} + +## .. importdoc:: os.nim + +proc createSymlink*(src, dest: string) {.noWeirdTarget.} = + ## Create a symbolic link at `dest` which points to the item specified + ## by `src`. On most operating systems, will fail if a link already exists. + ## + ## .. warning:: Some OS's (such as Microsoft Windows) restrict the creation + ## of symlinks to root users (administrators) or users with developer mode enabled. + ## + ## See also: + ## * `createHardlink proc`_ + ## * `expandSymlink proc`_ + + when defined(windows): + const SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE = 2 + # allows anyone with developer mode on to create a link + let flag = dirExists(src).int32 or SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE + var wSrc = newWideCString(src) + var wDst = newWideCString(dest) + if createSymbolicLinkW(wDst, wSrc, flag) == 0 or getLastError() != 0: + raiseOSError(osLastError(), $(src, dest)) + else: + if symlink(src, dest) != 0: + raiseOSError(osLastError(), $(src, dest)) + +proc expandSymlink*(symlinkPath: string): string {.noWeirdTarget.} = + ## Returns a string representing the path to which the symbolic link points. + ## + ## On Windows this is a noop, `symlinkPath` is simply returned. + ## + ## See also: + ## * `createSymlink proc`_ + when defined(windows) or defined(nintendoswitch): + result = symlinkPath + else: + var bufLen = 1024 + while true: + result = newString(bufLen) + let len = readlink(symlinkPath.cstring, result.cstring, bufLen) + if len < 0: + raiseOSError(osLastError(), symlinkPath) + if len < bufLen: + result.setLen(len) + break + bufLen = bufLen shl 1 diff --git a/lib/std/private/schubfach.nim b/lib/std/private/schubfach.nim index 872317ebf..b8c85d2bc 100644 --- a/lib/std/private/schubfach.nim +++ b/lib/std/private/schubfach.nim @@ -39,10 +39,10 @@ const exponentMask: BitsType = maxIeeeExponent shl (significandSize - 1) signMask: BitsType = not (not BitsType(0) shr 1) -proc constructSingle(bits: BitsType): Single {.constructor.} = +proc constructSingle(bits: BitsType): Single = result.bits = bits -proc constructSingle(value: ValueType): Single {.constructor.} = +proc constructSingle(value: ValueType): Single = result.bits = cast[typeof(result.bits)](value) proc physicalSignificand(this: Single): BitsType {.noSideEffect.} = @@ -244,12 +244,12 @@ proc toDecimal32(ieeeSignificand: uint32; ieeeExponent: uint32): FloatingDecimal ## ToChars ## ================================================================================================== -proc printDecimalDigitsBackwards(buf: var openArray[char]; pos: int; output: uint32): int32 {.inline.} = +proc printDecimalDigitsBackwards[T: Ordinal](buf: var openArray[char]; pos: T; output: uint32): int {.inline.} = var output = output var pos = pos - var tz: int32 = 0 + var tz = 0 ## number of trailing zeros removed. - var nd: int32 = 0 + var nd = 0 ## number of decimal digits processed. ## At most 9 digits remaining if output >= 10000: @@ -300,7 +300,7 @@ proc printDecimalDigitsBackwards(buf: var openArray[char]; pos: int; output: uin buf[pos] = chr(uint32('0') + q) return tz -proc decimalLength(v: uint32): int32 {.inline.} = +proc decimalLength(v: uint32): int {.inline.} = sf_Assert(v >= 1) sf_Assert(v <= 999999999'u) if v >= 100000000'u: @@ -321,7 +321,7 @@ proc decimalLength(v: uint32): int32 {.inline.} = return 2 return 1 -proc formatDigits(buffer: var openArray[char]; pos: int; digits: uint32; decimalExponent: int32; +proc formatDigits[T: Ordinal](buffer: var openArray[char]; pos: T; digits: uint32; decimalExponent: int; forceTrailingDotZero: bool = false): int {.inline.} = const minFixedDecimalPoint: int32 = -4 @@ -333,8 +333,8 @@ proc formatDigits(buffer: var openArray[char]; pos: int; digits: uint32; decimal sf_Assert(digits <= 999999999'u) sf_Assert(decimalExponent >= -99) sf_Assert(decimalExponent <= 99) - var numDigits: int32 = decimalLength(digits) - let decimalPoint: int32 = numDigits + decimalExponent + var numDigits = decimalLength(digits) + let decimalPoint = numDigits + decimalExponent let useFixed: bool = minFixedDecimalPoint <= decimalPoint and decimalPoint <= maxFixedDecimalPoint ## Prepare the buffer. @@ -342,7 +342,7 @@ proc formatDigits(buffer: var openArray[char]; pos: int; digits: uint32; decimal for i in 0..<32: buffer[pos+i] = '0' assert(minFixedDecimalPoint >= -30, "internal error") assert(maxFixedDecimalPoint <= 32, "internal error") - var decimalDigitsPosition: int32 + var decimalDigitsPosition: int if useFixed: if decimalPoint <= 0: ## 0.[000]digits @@ -355,7 +355,7 @@ proc formatDigits(buffer: var openArray[char]; pos: int; digits: uint32; decimal ## dE+123 or d.igitsE+123 decimalDigitsPosition = 1 var digitsEnd = pos + decimalDigitsPosition + numDigits - let tz: int32 = printDecimalDigitsBackwards(buffer, digitsEnd, digits) + let tz = printDecimalDigitsBackwards(buffer, digitsEnd, digits) dec(digitsEnd, tz) dec(numDigits, tz) ## decimal_exponent += tz; // => decimal_point unchanged. @@ -386,7 +386,7 @@ proc formatDigits(buffer: var openArray[char]; pos: int; digits: uint32; decimal ## d.igitsE+123 buffer[pos+1] = '.' pos = digitsEnd - let scientificExponent: int32 = decimalPoint - 1 + let scientificExponent = decimalPoint - 1 ## SF_ASSERT(scientific_exponent != 0); buffer[pos] = 'e' buffer[pos+1] = if scientificExponent < 0: '-' else: '+' @@ -412,7 +412,7 @@ proc float32ToChars*(buffer: var openArray[char]; v: float32; forceTrailingDotZe if exponent != 0 or significand != 0: ## != 0 let dec: auto = toDecimal32(significand, exponent) - return formatDigits(buffer, pos, dec.digits, dec.exponent, forceTrailingDotZero) + return formatDigits(buffer, pos, dec.digits, dec.exponent.int, forceTrailingDotZero) else: buffer[pos] = '0' buffer[pos+1] = '.' diff --git a/lib/std/private/since.nim b/lib/std/private/since.nim index 5b22b6391..720120f11 100644 --- a/lib/std/private/since.nim +++ b/lib/std/private/since.nim @@ -1,5 +1,5 @@ ##[ -`since` is used to emulate older versions of nim stdlib with `--useVersion`, +`since` is used to emulate older versions of nim stdlib, see `tuse_version.nim`. If a symbol `foo` is added in version `(1,3,5)`, use `{.since: (1.3.5).}`, not @@ -15,19 +15,19 @@ The emulation cannot be 100% faithful and to avoid adding too much complexity, template since*(version: (int, int), body: untyped) {.dirty.} = ## Evaluates `body` if the ``(NimMajor, NimMinor)`` is greater than ## or equal to `version`. Usage: - ## - ## .. code-block:: Nim + ## ```Nim ## proc fun*() {.since: (1, 3).} ## since (1, 3): fun() + ## ``` when (NimMajor, NimMinor) >= version: body template since*(version: (int, int, int), body: untyped) {.dirty.} = ## Evaluates `body` if ``(NimMajor, NimMinor, NimPatch)`` is greater than ## or equal to `version`. Usage: - ## - ## .. code-block:: Nim + ## ```Nim ## proc fun*() {.since: (1, 3, 1).} ## since (1, 3, 1): fun() + ## ``` when (NimMajor, NimMinor, NimPatch) >= version: body diff --git a/lib/std/private/strimpl.nim b/lib/std/private/strimpl.nim index 7d42a7cf8..f8c9236a5 100644 --- a/lib/std/private/strimpl.nim +++ b/lib/std/private/strimpl.nim @@ -74,3 +74,40 @@ template endsWithImpl*[T: string | cstring](s, suffix: T) = func cmpNimIdentifier*[T: string | cstring](a, b: T): int = cmpIgnoreStyleImpl(a, b, true) + +func c_memchr(cstr: pointer, c: char, n: csize_t): pointer {. + importc: "memchr", header: "<string.h>".} +func c_strstr(haystack, needle: cstring): cstring {. + importc: "strstr", header: "<string.h>".} + + +func find*(s: cstring, sub: char, start: Natural = 0, last = 0): int = + ## Searches for `sub` in `s` inside the range `start..last` (both ends included). + ## If `last` is unspecified, it defaults to `s.high` (the last element). + ## + ## Searching is case-sensitive. If `sub` is not in `s`, -1 is returned. + ## Otherwise the index returned is relative to `s[0]`, not `start`. + ## Use `s[start..last].rfind` for a `start`-origin index. + let last = if last == 0: s.high else: last + let L = last-start+1 + if L > 0: + let found = c_memchr(s[start].unsafeAddr, sub, cast[csize_t](L)) + if not found.isNil: + return cast[int](found) -% cast[int](s) + return -1 + +func find*(s, sub: cstring, start: Natural = 0, last = 0): int = + ## Searches for `sub` in `s` inside the range `start..last` (both ends included). + ## If `last` is unspecified, it defaults to `s.high` (the last element). + ## + ## Searching is case-sensitive. If `sub` is not in `s`, -1 is returned. + ## Otherwise the index returned is relative to `s[0]`, not `start`. + ## Use `s[start..last].find` for a `start`-origin index. + if sub.len > s.len - start: return -1 + if sub.len == 1: return find(s, sub[0], start, last) + if last == 0 and s.len > start: + let found = c_strstr(cast[cstring](s[start].unsafeAddr), sub) + if not found.isNil: + result = cast[int](found) -% cast[int](s) + else: + result = -1 diff --git a/lib/std/private/syslocks.nim b/lib/std/private/syslocks.nim new file mode 100644 index 000000000..e19ec2c04 --- /dev/null +++ b/lib/std/private/syslocks.nim @@ -0,0 +1,234 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2012 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# Low level system locks and condition vars. + +{.push stackTrace: off.} + +when defined(windows): + type + Handle = int + + SysLock* {.importc: "CRITICAL_SECTION", + header: "<windows.h>", final, pure, byref.} = object # CRITICAL_SECTION in WinApi + DebugInfo: pointer + LockCount: int32 + RecursionCount: int32 + OwningThread: int + LockSemaphore: int + SpinCount: int + + SysCond* {.importc: "RTL_CONDITION_VARIABLE", header: "<windows.h>", byref.} = object + thePtr {.importc: "Ptr".} : Handle + + proc initSysLock*(L: var SysLock) {.importc: "InitializeCriticalSection", + header: "<windows.h>".} + ## Initializes the lock `L`. + + proc tryAcquireSysAux(L: var SysLock): int32 {.importc: "TryEnterCriticalSection", + header: "<windows.h>".} + ## Tries to acquire the lock `L`. + + proc tryAcquireSys*(L: var SysLock): bool {.inline.} = + result = tryAcquireSysAux(L) != 0'i32 + + proc acquireSys*(L: var SysLock) {.importc: "EnterCriticalSection", + header: "<windows.h>".} + ## Acquires the lock `L`. + + proc releaseSys*(L: var SysLock) {.importc: "LeaveCriticalSection", + header: "<windows.h>".} + ## Releases the lock `L`. + + proc deinitSys*(L: SysLock) {.importc: "DeleteCriticalSection", + header: "<windows.h>".} + + proc initializeConditionVariable( + conditionVariable: var SysCond + ) {.stdcall, noSideEffect, dynlib: "kernel32", importc: "InitializeConditionVariable".} + + proc sleepConditionVariableCS( + conditionVariable: var SysCond, + PCRITICAL_SECTION: var SysLock, + dwMilliseconds: int + ): int32 {.stdcall, noSideEffect, dynlib: "kernel32", importc: "SleepConditionVariableCS".} + + + proc signalSysCond*(hEvent: var SysCond) {.stdcall, noSideEffect, + dynlib: "kernel32", importc: "WakeConditionVariable".} + + proc broadcastSysCond*(hEvent: var SysCond) {.stdcall, noSideEffect, + dynlib: "kernel32", importc: "WakeAllConditionVariable".} + + proc initSysCond*(cond: var SysCond) {.inline.} = + initializeConditionVariable(cond) + proc deinitSysCond*(cond: SysCond) {.inline.} = + discard + proc waitSysCond*(cond: var SysCond, lock: var SysLock) = + discard sleepConditionVariableCS(cond, lock, -1'i32) + +elif defined(genode): + const + Header = "genode_cpp/syslocks.h" + type + SysLock* {.importcpp: "Nim::SysLock", pure, final, + header: Header.} = object + SysCond* {.importcpp: "Nim::SysCond", pure, final, + header: Header.} = object + + proc initSysLock*(L: var SysLock) = discard + proc deinitSys*(L: SysLock) = discard + proc acquireSys*(L: var SysLock) {.noSideEffect, importcpp.} + proc tryAcquireSys*(L: var SysLock): bool {.noSideEffect, importcpp.} + proc releaseSys*(L: var SysLock) {.noSideEffect, importcpp.} + + proc initSysCond*(L: var SysCond) = discard + proc deinitSysCond*(L: SysCond) = discard + proc waitSysCond*(cond: var SysCond, lock: var SysLock) {. + noSideEffect, importcpp.} + proc signalSysCond*(cond: var SysCond) {. + noSideEffect, importcpp.} + proc broadcastSysCond*(cond: var SysCond) {. + noSideEffect, importcpp.} + +else: + type + SysLockObj {.importc: "pthread_mutex_t", pure, final, + header: """#include <sys/types.h> + #include <pthread.h>""", byref.} = object + when defined(linux) and defined(amd64): + abi: array[40 div sizeof(clong), clong] + + SysLockAttr* {.importc: "pthread_mutexattr_t", pure, final + header: """#include <sys/types.h> + #include <pthread.h>""".} = object + when defined(linux) and defined(amd64): + abi: array[4 div sizeof(cint), cint] # actually a cint + + SysCondObj {.importc: "pthread_cond_t", pure, final, + header: """#include <sys/types.h> + #include <pthread.h>""", byref.} = object + when defined(linux) and defined(amd64): + abi: array[48 div sizeof(clonglong), clonglong] + + SysCondAttr {.importc: "pthread_condattr_t", pure, final + header: """#include <sys/types.h> + #include <pthread.h>""".} = object + when defined(linux) and defined(amd64): + abi: array[4 div sizeof(cint), cint] # actually a cint + + SysLockType = distinct cint + + proc initSysLockAux(L: var SysLockObj, attr: ptr SysLockAttr) {. + importc: "pthread_mutex_init", header: "<pthread.h>", noSideEffect.} + proc deinitSysAux(L: SysLockObj) {.noSideEffect, + importc: "pthread_mutex_destroy", header: "<pthread.h>".} + + proc acquireSysAux(L: var SysLockObj) {.noSideEffect, + importc: "pthread_mutex_lock", header: "<pthread.h>".} + proc tryAcquireSysAux(L: var SysLockObj): cint {.noSideEffect, + importc: "pthread_mutex_trylock", header: "<pthread.h>".} + + proc releaseSysAux(L: var SysLockObj) {.noSideEffect, + importc: "pthread_mutex_unlock", header: "<pthread.h>".} + + when defined(ios): + # iOS will behave badly if sync primitives are moved in memory. In order + # to prevent this once and for all, we're doing an extra malloc when + # initializing the primitive. + type + SysLock* = ptr SysLockObj + SysCond* = ptr SysCondObj + + when not declared(c_malloc): + proc c_malloc(size: csize_t): pointer {. + importc: "malloc", header: "<stdlib.h>".} + proc c_free(p: pointer) {. + importc: "free", header: "<stdlib.h>".} + + proc initSysLock*(L: var SysLock, attr: ptr SysLockAttr = nil) = + L = cast[SysLock](c_malloc(csize_t(sizeof(SysLockObj)))) + initSysLockAux(L[], attr) + + proc deinitSys*(L: SysLock) = + deinitSysAux(L[]) + c_free(L) + + template acquireSys*(L: var SysLock) = + acquireSysAux(L[]) + template tryAcquireSys*(L: var SysLock): bool = + tryAcquireSysAux(L[]) == 0'i32 + template releaseSys*(L: var SysLock) = + releaseSysAux(L[]) + else: + type + SysLock* = SysLockObj + SysCond* = SysCondObj + + template initSysLock*(L: var SysLock, attr: ptr SysLockAttr = nil) = + initSysLockAux(L, attr) + template deinitSys*(L: SysLock) = + deinitSysAux(L) + template acquireSys*(L: var SysLock) = + acquireSysAux(L) + template tryAcquireSys*(L: var SysLock): bool = + tryAcquireSysAux(L) == 0'i32 + template releaseSys*(L: var SysLock) = + releaseSysAux(L) + + # rlocks + var SysLockType_Reentrant* {.importc: "PTHREAD_MUTEX_RECURSIVE", + header: "<pthread.h>".}: SysLockType + proc initSysLockAttr*(a: var SysLockAttr) {. + importc: "pthread_mutexattr_init", header: "<pthread.h>", noSideEffect.} + proc setSysLockType*(a: var SysLockAttr, t: SysLockType) {. + importc: "pthread_mutexattr_settype", header: "<pthread.h>", noSideEffect.} + + # locks + proc initSysCondAux(cond: var SysCondObj, cond_attr: ptr SysCondAttr = nil) {. + importc: "pthread_cond_init", header: "<pthread.h>", noSideEffect.} + proc deinitSysCondAux(cond: SysCondObj) {.noSideEffect, + importc: "pthread_cond_destroy", header: "<pthread.h>".} + + proc waitSysCondAux(cond: var SysCondObj, lock: var SysLockObj): cint {. + importc: "pthread_cond_wait", header: "<pthread.h>", noSideEffect.} + proc signalSysCondAux(cond: var SysCondObj) {. + importc: "pthread_cond_signal", header: "<pthread.h>", noSideEffect.} + proc broadcastSysCondAux(cond: var SysCondObj) {. + importc: "pthread_cond_broadcast", header: "<pthread.h>", noSideEffect.} + + when defined(ios): + proc initSysCond*(cond: var SysCond, cond_attr: ptr SysCondAttr = nil) = + cond = cast[SysCond](c_malloc(csize_t(sizeof(SysCondObj)))) + initSysCondAux(cond[], cond_attr) + + proc deinitSysCond*(cond: SysCond) = + deinitSysCondAux(cond[]) + c_free(cond) + + template waitSysCond*(cond: var SysCond, lock: var SysLock) = + discard waitSysCondAux(cond[], lock[]) + template signalSysCond*(cond: var SysCond) = + signalSysCondAux(cond[]) + template broadcastSysCond*(cond: var SysCond) = + broadcastSysCondAux(cond[]) + else: + template initSysCond*(cond: var SysCond, cond_attr: ptr SysCondAttr = nil) = + initSysCondAux(cond, cond_attr) + template deinitSysCond*(cond: SysCond) = + deinitSysCondAux(cond) + + template waitSysCond*(cond: var SysCond, lock: var SysLock) = + discard waitSysCondAux(cond, lock) + template signalSysCond*(cond: var SysCond) = + signalSysCondAux(cond) + template broadcastSysCond*(cond: var SysCond) = + broadcastSysCondAux(cond) + +{.pop.} diff --git a/lib/std/private/threadtypes.nim b/lib/std/private/threadtypes.nim new file mode 100644 index 000000000..a1cdf21dc --- /dev/null +++ b/lib/std/private/threadtypes.nim @@ -0,0 +1,176 @@ +include system/inclrtl + +const hasSharedHeap* = defined(boehmgc) or defined(gogc) # don't share heaps; every thread has its own + +when defined(windows): + type + Handle* = int + SysThread* = Handle + WinThreadProc* = proc (x: pointer): int32 {.stdcall.} + + proc createThread*(lpThreadAttributes: pointer, dwStackSize: int32, + lpStartAddress: WinThreadProc, + lpParameter: pointer, + dwCreationFlags: int32, + lpThreadId: var int32): SysThread {. + stdcall, dynlib: "kernel32", importc: "CreateThread".} + + proc winSuspendThread*(hThread: SysThread): int32 {. + stdcall, dynlib: "kernel32", importc: "SuspendThread".} + + proc winResumeThread*(hThread: SysThread): int32 {. + stdcall, dynlib: "kernel32", importc: "ResumeThread".} + + proc waitForSingleObject*(hHandle: SysThread, dwMilliseconds: int32): int32 {. + stdcall, dynlib: "kernel32", importc: "WaitForSingleObject".} + + proc waitForMultipleObjects*(nCount: int32, + lpHandles: ptr SysThread, + bWaitAll: int32, + dwMilliseconds: int32): int32 {. + stdcall, dynlib: "kernel32", importc: "WaitForMultipleObjects".} + + proc terminateThread*(hThread: SysThread, dwExitCode: int32): int32 {. + stdcall, dynlib: "kernel32", importc: "TerminateThread".} + + proc setThreadAffinityMask*(hThread: SysThread, dwThreadAffinityMask: uint) {. + importc: "SetThreadAffinityMask", stdcall, header: "<windows.h>".} + +elif defined(genode): + const + GenodeHeader* = "genode_cpp/threads.h" + type + SysThread* {.importcpp: "Nim::SysThread", + header: GenodeHeader, final, pure.} = object + GenodeThreadProc* = proc (x: pointer) {.noconv.} + + proc initThread*(s: var SysThread, + env: GenodeEnv, + stackSize: culonglong, + entry: GenodeThreadProc, + arg: pointer, + affinity: cuint) {. + importcpp: "#.initThread(@)".} + + +else: + when not (defined(macosx) or defined(haiku)): + {.passl: "-pthread".} + + when not defined(haiku): + {.passc: "-pthread".} + + const + schedh = "#define _GNU_SOURCE\n#include <sched.h>" + pthreadh* = "#define _GNU_SOURCE\n#include <pthread.h>" + + when not declared(Time): + when defined(linux): + type Time = clong + else: + type Time = int + + when (defined(linux) or defined(nintendoswitch)) and defined(amd64): + type + SysThread* {.importc: "pthread_t", + header: "<sys/types.h>" .} = distinct culong + Pthread_attr* {.importc: "pthread_attr_t", + header: "<sys/types.h>".} = object + abi: array[56 div sizeof(clong), clong] + elif defined(openbsd) and defined(amd64): + type + SysThread* {.importc: "pthread_t", header: "<pthread.h>".} = object + Pthread_attr* {.importc: "pthread_attr_t", + header: "<pthread.h>".} = object + else: + type + SysThread* {.importc: "pthread_t", header: "<sys/types.h>".} = int + Pthread_attr* {.importc: "pthread_attr_t", + header: "<sys/types.h>".} = object + type + Timespec* {.importc: "struct timespec", header: "<time.h>".} = object + tv_sec*: Time + tv_nsec*: clong + + proc pthread_attr_init*(a1: var Pthread_attr): cint {. + importc, header: pthreadh.} + proc pthread_attr_setstack*(a1: ptr Pthread_attr, a2: pointer, a3: int): cint {. + importc, header: pthreadh.} + proc pthread_attr_setstacksize*(a1: var Pthread_attr, a2: int): cint {. + importc, header: pthreadh.} + proc pthread_attr_destroy*(a1: var Pthread_attr): cint {. + importc, header: pthreadh.} + + proc pthread_create*(a1: var SysThread, a2: var Pthread_attr, + a3: proc (x: pointer): pointer {.noconv.}, + a4: pointer): cint {.importc: "pthread_create", + header: pthreadh.} + proc pthread_join*(a1: SysThread, a2: ptr pointer): cint {. + importc, header: pthreadh.} + + proc pthread_cancel*(a1: SysThread): cint {. + importc: "pthread_cancel", header: pthreadh.} + + type CpuSet* {.importc: "cpu_set_t", header: schedh.} = object + when defined(linux) and defined(amd64): + abi: array[1024 div (8 * sizeof(culong)), culong] + + proc cpusetZero*(s: var CpuSet) {.importc: "CPU_ZERO", header: schedh.} + proc cpusetIncl*(cpu: cint; s: var CpuSet) {. + importc: "CPU_SET", header: schedh.} + + when defined(android): + # libc of android doesn't implement pthread_setaffinity_np, + # it exposes pthread_gettid_np though, so we can use that in combination + # with sched_setaffinity to set the thread affinity. + type Pid* {.importc: "pid_t", header: "<sys/types.h>".} = int32 # From posix_other.nim + + proc setAffinityTID*(tid: Pid; setsize: csize_t; s: var CpuSet) {. + importc: "sched_setaffinity", header: schedh.} + + proc pthread_gettid_np*(thread: SysThread): Pid {. + importc: "pthread_gettid_np", header: pthreadh.} + + proc setAffinity*(thread: SysThread; setsize: csize_t; s: var CpuSet) = + setAffinityTID(pthread_gettid_np(thread), setsize, s) + else: + proc setAffinity*(thread: SysThread; setsize: csize_t; s: var CpuSet) {. + importc: "pthread_setaffinity_np", header: pthreadh.} + + +const + emulatedThreadVars* = compileOption("tlsEmulation") +# we preallocate a fixed size for thread local storage, so that no heap +# allocations are needed. Currently less than 16K are used on a 64bit machine. +# We use `float` for proper alignment: +const nimTlsSize {.intdefine.} = 16000 +type + ThreadLocalStorage* = array[0..(nimTlsSize div sizeof(float)), float] + PGcThread* = ptr GcThread + GcThread* {.pure, inheritable.} = object + when emulatedThreadVars: + tls*: ThreadLocalStorage + else: + nil + when hasSharedHeap: + next*, prev*: PGcThread + stackBottom*, stackTop*: pointer + stackSize*: int + else: + nil + +const hasAllocStack* = defined(zephyr) # maybe freertos too? + +type + Thread*[TArg] = object + core*: PGcThread + sys*: SysThread + when TArg is void: + dataFn*: proc () {.nimcall, gcsafe.} + else: + dataFn*: proc (m: TArg) {.nimcall, gcsafe.} + data*: TArg + when hasAllocStack: + rawStack*: pointer + +proc `=copy`*[TArg](x: var Thread[TArg], y: Thread[TArg]) {.error.} diff --git a/lib/std/private/underscored_calls.nim b/lib/std/private/underscored_calls.nim index f0bcbcc74..f853572b5 100644 --- a/lib/std/private/underscored_calls.nim +++ b/lib/std/private/underscored_calls.nim @@ -10,7 +10,9 @@ ## This is an internal helper module. Do not use. -import macros +import std/macros + +proc underscoredCalls*(result, calls, arg0: NimNode) proc underscoredCall(n, arg0: NimNode): NimNode = proc underscorePos(n: NimNode): int = @@ -19,13 +21,19 @@ proc underscoredCall(n, arg0: NimNode): NimNode = return 0 if n.kind in nnkCallKinds: - result = copyNimNode(n) - result.add n[0] + if n[0].kind in {nnkIdent, nnkSym} and n[0].eqIdent("with"): + expectKind n[1], {nnkIdent, nnkSym} - let u = underscorePos(n) - for i in 1..u-1: result.add n[i] - result.add arg0 - for i in u+1..n.len-1: result.add n[i] + result = newStmtList() + underscoredCalls(result, n[2 .. ^1].newStmtList, newDotExpr(arg0, n[1])) + else: + result = copyNimNode(n) + result.add n[0] + + let u = underscorePos(n) + for i in 1..u-1: result.add n[i] + result.add arg0 + for i in u+1..n.len-1: result.add n[i] elif n.kind in {nnkAsgn, nnkExprEqExpr}: var field = n[0] if n[0].kind == nnkDotExpr and n[0][0].eqIdent("_"): diff --git a/lib/std/private/win_setenv.nim b/lib/std/private/win_setenv.nim index 89bb0421f..66e199dfe 100644 --- a/lib/std/private/win_setenv.nim +++ b/lib/std/private/win_setenv.nim @@ -23,6 +23,9 @@ check errno_t vs cint when not defined(windows): discard else: + when defined(nimPreviewSlimSystem): + import std/widestrs + type wchar_t {.importc: "wchar_t".} = int16 proc setEnvironmentVariableW*(lpName, lpValue: WideCString): int32 {. @@ -30,25 +33,25 @@ else: # same as winlean.setEnvironmentVariableA proc c_getenv(varname: cstring): cstring {.importc: "getenv", header: "<stdlib.h>".} - proc c_wputenv(envstring: WideCString): cint {.importc: "_wputenv", header: "<stdlib.h>".} - proc c_wgetenv(varname: WideCString): WideCString {.importc: "_wgetenv", header: "<stdlib.h>".} + proc c_wputenv(envstring: ptr wchar_t): cint {.importc: "_wputenv", header: "<stdlib.h>".} + proc c_wgetenv(varname: ptr wchar_t): ptr wchar_t {.importc: "_wgetenv", header: "<stdlib.h>".} var errno {.importc, header: "<errno.h>".}: cint var genviron {.importc: "_environ".}: ptr ptr char # xxx `ptr UncheckedArray[WideCString]` did not work - proc wcstombs(wcstr: ptr char, mbstr: WideCString, count: csize_t): csize_t {.importc, header: "<stdlib.h>".} + proc wcstombs(wcstr: ptr char, mbstr: ptr wchar_t, count: csize_t): csize_t {.importc, header: "<stdlib.h>".} # xxx cint vs errno_t? proc setEnvImpl*(name: string, value: string, overwrite: cint): cint = const EINVAL = cint(22) - let wideName = newWideCString(name) - if overwrite == 0 and c_wgetenv(wideName) != nil: + let wideName: WideCString = newWideCString(name) + if overwrite == 0 and c_wgetenv(cast[ptr wchar_t](wideName)) != nil: return 0 if value != "": - let envstring = name & "=" & value - let e = c_wputenv(newWideCString(envstring)) + let envstring: WideCString = newWideCString(name & "=" & value) + let e = c_wputenv(cast[ptr wchar_t](envstring)) if e != 0: errno = EINVAL return -1 @@ -59,19 +62,19 @@ else: SetEnvironmentVariableA doesn't update `_environ`, so we have to do these terrible things. ]# - let envstring = name & "= " - if c_wputenv(newWideCString(envstring)) != 0: + let envstring: WideCString = newWideCString(name & "= ") + if c_wputenv(cast[ptr wchar_t](envstring)) != 0: errno = EINVAL return -1 # Here lies the documentation we blatently ignore to make this work. - var s = c_wgetenv(wideName) + var s = cast[WideCString](c_wgetenv(cast[ptr wchar_t](wideName))) s[0] = Utf16Char('\0') #[ This would result in a double null termination, which normally signifies the end of the environment variable list, so we stick a completely empty environment variable into the list instead. ]# - s = c_wgetenv(wideName) + s = cast[WideCString](c_wgetenv(cast[ptr wchar_t](wideName))) s[1] = Utf16Char('=') #[ If genviron is null, the MBCS environment has not been initialized @@ -85,15 +88,15 @@ else: # in the current codepage. Skip updating MBCS environment in this case. # For some reason, second `wcstombs` can find non-convertible characters # that the first `wcstombs` cannot. - let requiredSizeS = wcstombs(nil, wideName, 0) + let requiredSizeS = wcstombs(nil, cast[ptr wchar_t](wideName), 0) if requiredSizeS != high(csize_t): let requiredSize = requiredSizeS.int var buf = newSeq[char](requiredSize + 1) let buf2 = buf[0].addr - if wcstombs(buf2, wideName, csize_t(requiredSize + 1)) != high(csize_t): - var ptrToEnv = c_getenv(buf2) + if wcstombs(buf2, cast[ptr wchar_t](wideName), csize_t(requiredSize + 1)) != high(csize_t): + var ptrToEnv = c_getenv(cast[cstring](buf2)) ptrToEnv[0] = '\0' - ptrToEnv = c_getenv(buf2) + ptrToEnv = c_getenv(cast[cstring](buf2)) ptrToEnv[1] = '=' # And now, we have to update the outer environment to have a proper empty value. diff --git a/lib/std/setutils.nim b/lib/std/setutils.nim index 4664d6dcc..8e7bc6a92 100644 --- a/lib/std/setutils.nim +++ b/lib/std/setutils.nim @@ -14,7 +14,7 @@ ## * `std/packedsets <packedsets.html>`_ ## * `std/sets <sets.html>`_ -import typetraits, macros +import std/[typetraits, macros] #[ type SetElement* = char|byte|bool|int16|uint16|enum|uint8|int8 diff --git a/lib/std/sha1.nim b/lib/std/sha1.nim index 50175024c..213af4229 100644 --- a/lib/std/sha1.nim +++ b/lib/std/sha1.nim @@ -26,8 +26,11 @@ runnableExamples("-r:off"): b = parseSecureHash("10DFAEBF6BFDBC7939957068E2EFACEC4972933C") assert a == b, "files don't match" -import strutils -from endians import bigEndian32, bigEndian64 + +{.deprecated: "use command `nimble install checksums` and import `checksums/sha1` instead".} + +import std/strutils +from std/endians import bigEndian32, bigEndian64 when defined(nimPreviewSlimSystem): import std/syncio @@ -281,4 +284,4 @@ proc `==`*(a, b: SecureHash): bool = proc isValidSha1Hash*(s: string): bool = ## Checks if a string is a valid sha1 hash sum. - s.len == 40 and allCharsInSet(s, HexDigits) + s.len == 40 and allCharsInSet(s, HexDigits) \ No newline at end of file diff --git a/lib/std/socketstreams.nim b/lib/std/socketstreams.nim index 5c882858d..45e906795 100644 --- a/lib/std/socketstreams.nim +++ b/lib/std/socketstreams.nim @@ -31,39 +31,40 @@ ## Examples ## ======== ## -## .. code-block:: Nim -## import std/socketstreams +## ```Nim +## import std/socketstreams ## -## var -## socket = newSocket(AF_INET, SOCK_DGRAM, IPPROTO_UDP) -## stream = newReadSocketStream(socket) -## socket.sendTo("127.0.0.1", Port(12345), "SOME REQUEST") -## echo stream.readLine() # Will call `recv` -## stream.setPosition(0) -## echo stream.readLine() # Will return the read line from the buffer -## stream.resetStream() # Buffer is now empty, position is 0 -## echo stream.readLine() # Will call `recv` again -## stream.close() # Closes the socket +## var +## socket = newSocket(AF_INET, SOCK_DGRAM, IPPROTO_UDP) +## stream = newReadSocketStream(socket) +## socket.sendTo("127.0.0.1", Port(12345), "SOME REQUEST") +## echo stream.readLine() # Will call `recv` +## stream.setPosition(0) +## echo stream.readLine() # Will return the read line from the buffer +## stream.resetStream() # Buffer is now empty, position is 0 +## echo stream.readLine() # Will call `recv` again +## stream.close() # Closes the socket +## ``` ## -## .. code-block:: Nim +## ```Nim +## import std/socketstreams ## -## import std/socketstreams -## -## var socket = newSocket(AF_INET, SOCK_DGRAM, IPPROTO_UDP) -## socket.connect("127.0.0.1", Port(12345)) -## var sendStream = newWriteSocketStream(socket) -## sendStream.write "NOM" -## sendStream.setPosition(1) -## echo sendStream.peekStr(2) # OM -## sendStream.write "I" -## sendStream.setPosition(0) -## echo sendStream.readStr(3) # NIM -## echo sendStream.getPosition() # 3 -## sendStream.flush() # This actually performs the writing to the socket -## sendStream.setPosition(1) -## sendStream.write "I" # Throws an error as we can't write into an already sent buffer - -import net, streams +## var socket = newSocket(AF_INET, SOCK_DGRAM, IPPROTO_UDP) +## socket.connect("127.0.0.1", Port(12345)) +## var sendStream = newWriteSocketStream(socket) +## sendStream.write "NOM" +## sendStream.setPosition(1) +## echo sendStream.peekStr(2) # OM +## sendStream.write "I" +## sendStream.setPosition(0) +## echo sendStream.readStr(3) # NIM +## echo sendStream.getPosition() # 3 +## sendStream.flush() # This actually performs the writing to the socket +## sendStream.setPosition(1) +## sendStream.write "I" # Throws an error as we can't write into an already sent buffer +## ``` + +import std/[net, streams] type ReadSocketStream* = ref ReadSocketStreamObj @@ -146,7 +147,7 @@ proc wsFlush(s: Stream) = s.lastFlush = s.buf.len proc rsClose(s: Stream) = - {.cast(tags: []).}: + {.cast(raises: [IOError, OSError]), cast(tags: []).}: # todo fixme maybe do something? var s = ReadSocketStream(s) s.data.close() diff --git a/lib/std/staticos.nim b/lib/std/staticos.nim new file mode 100644 index 000000000..2617c6913 --- /dev/null +++ b/lib/std/staticos.nim @@ -0,0 +1,13 @@ +## This module implements path handling like os module but works at only compile-time. +## This module works even when cross compiling to OS that is not supported by os module. + +proc staticFileExists*(filename: string): bool {.compileTime.} = + ## Returns true if `filename` exists and is a regular file or symlink. + ## + ## Directories, device files, named pipes and sockets return false. + discard + +proc staticDirExists*(dir: string): bool {.compileTime.} = + ## Returns true if the directory `dir` exists. If `dir` is a file, false + ## is returned. Follows symlinks. + discard diff --git a/lib/std/strbasics.nim b/lib/std/strbasics.nim index be1dd7a58..b2c36a4be 100644 --- a/lib/std/strbasics.nim +++ b/lib/std/strbasics.nim @@ -23,8 +23,8 @@ proc add*(x: var string, y: openArray[char]) = # Use `{.noalias.}` ? let n = x.len x.setLen n + y.len - # pending https://github.com/nim-lang/Nim/issues/14655#issuecomment-643671397 - # use x.setLen(n + y.len, isInit = false) + # pending #19727 + # setLen unnecessarily zeros memory var i = 0 while i < y.len: x[n + i] = y[i] diff --git a/lib/std/sums.nim b/lib/std/sums.nim deleted file mode 100644 index a6ce1b85d..000000000 --- a/lib/std/sums.nim +++ /dev/null @@ -1,80 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2019 b3liever -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. - -## Accurate summation functions. - -{.deprecated: "use the nimble package `sums` instead.".} - -runnableExamples: - import std/math - - template `~=`(x, y: float): bool = abs(x - y) < 1e-4 - - let - n = 1_000_000 - first = 1e10 - small = 0.1 - var data = @[first] - for _ in 1 .. n: - data.add(small) - - let result = first + small * n.float - - doAssert abs(sum(data) - result) > 0.3 - doAssert sumKbn(data) ~= result - doAssert sumPairs(data) ~= result - -## See also -## ======== -## * `math module <math.html>`_ for a standard `sum proc <math.html#sum,openArray[T]>`_ - -func sumKbn*[T](x: openArray[T]): T = - ## Kahan-Babuška-Neumaier summation: O(1) error growth, at the expense - ## of a considerable increase in computational cost. - ## - ## See: - ## * https://en.wikipedia.org/wiki/Kahan_summation_algorithm#Further_enhancements - if len(x) == 0: return - var sum = x[0] - var c = T(0) - for i in 1 ..< len(x): - let xi = x[i] - let t = sum + xi - if abs(sum) >= abs(xi): - c += (sum - t) + xi - else: - c += (xi - t) + sum - sum = t - result = sum + c - -func sumPairwise[T](x: openArray[T], i0, n: int): T = - if n < 128: - result = x[i0] - for i in i0 + 1 ..< i0 + n: - result += x[i] - else: - let n2 = n div 2 - result = sumPairwise(x, i0, n2) + sumPairwise(x, i0 + n2, n - n2) - -func sumPairs*[T](x: openArray[T]): T = - ## Pairwise (cascade) summation of `x[i0:i0+n-1]`, with O(log n) error growth - ## (vs O(n) for a simple loop) with negligible performance cost if - ## the base case is large enough. - ## - ## See, e.g.: - ## * https://en.wikipedia.org/wiki/Pairwise_summation - ## * Higham, Nicholas J. (1993), "The accuracy of floating point - ## summation", SIAM Journal on Scientific Computing 14 (4): 783–799. - ## - ## In fact, the root-mean-square error growth, assuming random roundoff - ## errors, is only O(sqrt(log n)), which is nearly indistinguishable from O(1) - ## in practice. See: - ## * Manfred Tasche and Hansmartin Zeuner, Handbook of - ## Analytic-Computational Methods in Applied Mathematics (2000). - let n = len(x) - if n == 0: T(0) else: sumPairwise(x, 0, n) diff --git a/lib/std/symlinks.nim b/lib/std/symlinks.nim new file mode 100644 index 000000000..dbe908612 --- /dev/null +++ b/lib/std/symlinks.nim @@ -0,0 +1,33 @@ +## This module implements symlink (symbolic link) handling. + +## .. importdoc:: os.nim + +from std/paths import Path, ReadDirEffect + +from std/private/ossymlinks import symlinkExists, createSymlink, expandSymlink + +proc symlinkExists*(link: Path): bool {.inline, tags: [ReadDirEffect], sideEffect.} = + ## Returns true if the symlink `link` exists. Will return true + ## regardless of whether the link points to a directory or file. + result = symlinkExists(link.string) + +proc createSymlink*(src, dest: Path) {.inline.} = + ## Create a symbolic link at `dest` which points to the item specified + ## by `src`. On most operating systems, will fail if a link already exists. + ## + ## .. warning:: Some OS's (such as Microsoft Windows) restrict the creation + ## of symlinks to root users (administrators) or users with developer mode enabled. + ## + ## See also: + ## * `createHardlink proc`_ + ## * `expandSymlink proc`_ + createSymlink(src.string, dest.string) + +proc expandSymlink*(symlinkPath: Path): Path {.inline.} = + ## Returns a string representing the path to which the symbolic link points. + ## + ## On Windows this is a noop, `symlinkPath` is simply returned. + ## + ## See also: + ## * `createSymlink proc`_ + result = Path(expandSymlink(symlinkPath.string)) diff --git a/lib/std/syncio.nim b/lib/std/syncio.nim index 22e981198..c34a025af 100644 --- a/lib/std/syncio.nim +++ b/lib/std/syncio.nim @@ -12,6 +12,8 @@ include system/inclrtl import std/private/since import std/formatfloat +when defined(windows): + import std/widestrs # ----------------- IO Part ------------------------------------------------ type @@ -36,8 +38,15 @@ type ## at the end. If the file does not exist, it ## will be created. - FileHandle* = cint ## type that represents an OS file handle; this is - ## useful for low-level file access + FileHandle* = cint ## The type that represents an OS file handle; this is + ## useful for low-level file access. + + FileSeekPos* = enum ## Position relative to which seek should happen. + # The values are ordered so that they match with stdio + # SEEK_SET, SEEK_CUR and SEEK_END respectively. + fspSet ## Seek to absolute value + fspCur ## Seek relative to current position + fspEnd ## Seek relative to end # text file handling: when not defined(nimscript) and not defined(js): @@ -96,7 +105,7 @@ proc c_feof(f: File): cint {. importc: "feof", header: "<stdio.h>".} when not declared(c_fwrite): - proc c_fwrite(buf: pointer, size, n: csize_t, f: File): cint {. + proc c_fwrite(buf: pointer, size, n: csize_t, f: File): csize_t {. importc: "fwrite", header: "<stdio.h>".} # C routine that is used here: @@ -142,21 +151,11 @@ proc c_fprintf(f: File, frmt: cstring): cint {. proc c_fputc(c: char, f: File): cint {. importc: "fputc", header: "<stdio.h>".} -# When running nim in android app, stdout goes nowhere, so echo gets ignored -# To redirect echo to the android logcat, use -d:androidNDK -when defined(androidNDK): - const ANDROID_LOG_VERBOSE = 2.cint - proc android_log_print(prio: cint, tag: cstring, fmt: cstring): cint - {.importc: "__android_log_print", header: "<android/log.h>", varargs, discardable.} - -template sysFatal(exc, msg) = - raise newException(exc, msg) - proc raiseEIO(msg: string) {.noinline, noreturn.} = - sysFatal(IOError, msg) + raise newException(IOError, msg) proc raiseEOF() {.noinline, noreturn.} = - sysFatal(EOFError, "EOF reached") + raise newException(EOFError, "EOF reached") proc strerror(errnum: cint): cstring {.importc, header: "<string.h>".} @@ -244,7 +243,7 @@ when defined(windows): # machine. We also enable `setConsoleOutputCP(65001)` now by default. # But we cannot call printf directly as the string might contain \0. # So we have to loop over all the sections separated by potential \0s. - var i = c_fprintf(f, "%s", s) + var i = int c_fprintf(f, "%s", s) while i < s.len: if s[i] == '\0': let w = c_fputc('\0', f) @@ -321,7 +320,7 @@ elif defined(windows): const BufSize = 4000 -proc close*(f: File) {.tags: [], gcsafe.} = +proc close*(f: File) {.tags: [], gcsafe, sideEffect.} = ## Closes the file. if not f.isNil: discard c_fclose(f) @@ -357,12 +356,12 @@ proc getOsFileHandle*(f: File): FileHandle = when defined(nimdoc) or (defined(posix) and not defined(nimscript)) or defined(windows): proc setInheritable*(f: FileHandle, inheritable: bool): bool = - ## control whether a file handle can be inherited by child processes. Returns + ## Controls whether a file handle can be inherited by child processes. Returns ## `true` on success. This requires the OS file handle, which can be ## retrieved via `getOsFileHandle <#getOsFileHandle,File>`_. ## ## This procedure is not guaranteed to be available for all platforms. Test for - ## availability with `declared() <system.html#declared,untyped>`. + ## availability with `declared() <system.html#declared,untyped>`_. when SupportIoctlInheritCtl: result = c_ioctl(f, if inheritable: FIONCLEX else: FIOCLEX) != -1 elif defined(freertos) or defined(zephyr): @@ -388,7 +387,7 @@ proc readLine*(f: File, line: var string): bool {.tags: [ReadIOEffect], proc c_memchr(s: pointer, c: cint, n: csize_t): pointer {. importc: "memchr", header: "<string.h>".} - when defined(windows) and not defined(useWinAnsi): + when defined(windows): proc readConsole(hConsoleInput: FileHandle, lpBuffer: pointer, nNumberOfCharsToRead: int32, lpNumberOfCharsRead: ptr int32, @@ -420,7 +419,7 @@ proc readLine*(f: File, line: var string): bool {.tags: [ReadIOEffect], if f.isatty: const numberOfCharsToRead = 2048 var numberOfCharsRead = 0'i32 - var buffer = newWideCString("", numberOfCharsToRead) + var buffer = newWideCString(numberOfCharsToRead) if readConsole(getOsFileHandle(f), addr(buffer[0]), numberOfCharsToRead, addr(numberOfCharsRead), nil) == 0: var error = getLastError() @@ -465,7 +464,7 @@ proc readLine*(f: File, line: var string): bool {.tags: [ReadIOEffect], while true: # fixes #9634; this pattern may need to be abstracted as a template if reused; # likely other io procs need this for correctness. - fgetsSuccess = c_fgets(addr line[pos], sp.cint, f) != nil + fgetsSuccess = c_fgets(cast[cstring](addr line[pos]), sp.cint, f) != nil if fgetsSuccess: break when not defined(nimscript): if errno == EINTR: @@ -478,15 +477,16 @@ proc readLine*(f: File, line: var string): bool {.tags: [ReadIOEffect], let m = c_memchr(addr line[pos], '\L'.ord, cast[csize_t](sp)) if m != nil: # \l found: Could be our own or the one by fgets, in any case, we're done - var last = cast[ByteAddress](m) - cast[ByteAddress](addr line[0]) + var last = cast[int](m) - cast[int](addr line[0]) if last > 0 and line[last-1] == '\c': line.setLen(last-1) return last > 1 or fgetsSuccess - # We have to distinguish between two possible cases: + elif last > 0 and line[last-1] == '\0': + # We have to distinguish among three possible cases: # \0\l\0 => line ending in a null character. # \0\l\l => last line without newline, null was put there by fgets. - elif last > 0 and line[last-1] == '\0': - if last < pos + sp - 1 and line[last+1] != '\0': + # \0\l => last line without newline, null was put there by fgets. + if last >= pos + sp - 1 or line[last+1] != '\0': # bug #21273 dec last line.setLen(last) return last > 0 or fgetsSuccess @@ -571,7 +571,7 @@ proc readAllFile(file: File, len: int64): string = result = newString(len) let bytes = readBuffer(file, addr(result[0]), len) if endOfFile(file): - if bytes < len: + if bytes.int64 < len: result.setLen(bytes) else: # We read all the bytes but did not reach the EOF @@ -609,7 +609,7 @@ proc writeLine*[Ty](f: File, x: varargs[Ty, `$`]) {.inline, # interface to the C procs: -when defined(windows) and not defined(useWinAnsi): +when defined(windows): when defined(cpp): proc wfopen(filename, mode: WideCString): pointer {. importcpp: "_wfopen((const wchar_t*)#, (const wchar_t*)#)", nodecl.} @@ -648,6 +648,9 @@ const "" else: "" + RawFormatOpen: array[FileMode, cstring] = [ + # used for open by FileHandle, which calls `fdopen` + cstring("rb"), "wb", "w+b", "r+b", "ab"] FormatOpen: array[FileMode, cstring] = [ cstring("rb" & NoInheritFlag), "wb" & NoInheritFlag, "w+b" & NoInheritFlag, "r+b" & NoInheritFlag, "ab" & NoInheritFlag @@ -715,7 +718,7 @@ proc open*(f: var File, filename: string, result = true f = cast[File](p) - if bufSize > 0 and bufSize <= high(cint).int: + if bufSize > 0 and bufSize.uint <= high(uint): discard c_setvbuf(f, nil, IOFBF, cast[csize_t](bufSize)) elif bufSize == 0: discard c_setvbuf(f, nil, IONBF, 0) @@ -749,7 +752,7 @@ proc open*(f: var File, filehandle: FileHandle, filehandle) else: filehandle if not setInheritable(oshandle, false): return false - f = c_fdopen(filehandle, FormatOpen[mode]) + f = c_fdopen(filehandle, RawFormatOpen[mode]) result = f != nil proc open*(filename: string, @@ -761,9 +764,9 @@ proc open*(filename: string, ## ## The file handle associated with the resulting `File` is not inheritable. if not open(result, filename, mode, bufSize): - sysFatal(IOError, "cannot open: " & filename) + raise newException(IOError, "cannot open: " & filename) -proc setFilePos*(f: File, pos: int64, relativeTo: FileSeekPos = fspSet) {.benign.} = +proc setFilePos*(f: File, pos: int64, relativeTo: FileSeekPos = fspSet) {.benign, sideEffect.} = ## Sets the position of the file pointer that is used for read/write ## operations. The file's first byte has the index zero. if c_fseek(f, pos, cint(relativeTo)) != 0: @@ -791,52 +794,6 @@ proc setStdIoUnbuffered*() {.tags: [], benign.} = when declared(stdin): discard c_setvbuf(stdin, nil, IONBF, 0) -when declared(stdout): - when defined(windows) and compileOption("threads"): - proc addSysExitProc(quitProc: proc() {.noconv.}) {.importc: "atexit", header: "<stdlib.h>".} - - const insideRLocksModule = false - include "system/syslocks" - - - var echoLock: SysLock - initSysLock echoLock - addSysExitProc(proc() {.noconv.} = deinitSys(echoLock)) - - const stdOutLock = not defined(windows) and - not defined(android) and - not defined(nintendoswitch) and - not defined(freertos) and - not defined(zephyr) and - hostOS != "any" - - proc echoBinSafe(args: openArray[string]) {.compilerproc.} = - when defined(androidNDK): - var s = "" - for arg in args: - s.add arg - android_log_print(ANDROID_LOG_VERBOSE, "nim", s) - else: - # flockfile deadlocks some versions of Android 5.x.x - when stdOutLock: - proc flockfile(f: File) {.importc, nodecl.} - proc funlockfile(f: File) {.importc, nodecl.} - flockfile(stdout) - when defined(windows) and compileOption("threads"): - acquireSys echoLock - for s in args: - when defined(windows): - writeWindows(stdout, s) - else: - discard c_fwrite(s.cstring, cast[csize_t](s.len), 1, stdout) - const linefeed = "\n" - discard c_fwrite(linefeed.cstring, linefeed.len, 1, stdout) - discard c_fflush(stdout) - when stdOutLock: - funlockfile(stdout) - when defined(windows) and compileOption("threads"): - releaseSys echoLock - when defined(windows) and not defined(nimscript) and not defined(js): # work-around C's sucking abstraction: @@ -854,14 +811,33 @@ when defined(windows) and not defined(nimscript) and not defined(js): when defined(windows) and appType == "console" and not defined(nimDontSetUtf8CodePage) and not defined(nimscript): + import std/exitprocs + proc setConsoleOutputCP(codepage: cuint): int32 {.stdcall, dynlib: "kernel32", importc: "SetConsoleOutputCP".} proc setConsoleCP(wCodePageID: cuint): int32 {.stdcall, dynlib: "kernel32", importc: "SetConsoleCP".} + proc getConsoleOutputCP(): cuint {.stdcall, dynlib: "kernel32", + importc: "GetConsoleOutputCP".} + proc getConsoleCP(): cuint {.stdcall, dynlib: "kernel32", + importc: "GetConsoleCP".} + + const Utf8codepage = 65001'u32 + + let + consoleOutputCP = getConsoleOutputCP() + consoleCP = getConsoleCP() - const Utf8codepage = 65001 - discard setConsoleOutputCP(Utf8codepage) - discard setConsoleCP(Utf8codepage) + proc restoreConsoleOutputCP() = discard setConsoleOutputCP(consoleOutputCP) + proc restoreConsoleCP() = discard setConsoleCP(consoleCP) + + if consoleOutputCP != Utf8codepage: + discard setConsoleOutputCP(Utf8codepage) + addExitProc(restoreConsoleOutputCP) + + if consoleCP != Utf8codepage: + discard setConsoleCP(Utf8codepage) + addExitProc(restoreConsoleCP) proc readFile*(filename: string): string {.tags: [ReadIOEffect], benign.} = ## Opens a file named `filename` for reading, calls `readAll @@ -876,7 +852,7 @@ proc readFile*(filename: string): string {.tags: [ReadIOEffect], benign.} = finally: close(f) else: - sysFatal(IOError, "cannot open: " & filename) + raise newException(IOError, "cannot open: " & filename) proc writeFile*(filename, content: string) {.tags: [WriteIOEffect], benign.} = ## Opens a file named `filename` for writing. Then writes the @@ -889,7 +865,7 @@ proc writeFile*(filename, content: string) {.tags: [WriteIOEffect], benign.} = finally: close(f) else: - sysFatal(IOError, "cannot open: " & filename) + raise newException(IOError, "cannot open: " & filename) proc writeFile*(filename: string, content: openArray[byte]) {.since: (1, 1).} = ## Opens a file named `filename` for writing. Then writes the @@ -898,7 +874,7 @@ proc writeFile*(filename: string, content: openArray[byte]) {.since: (1, 1).} = var f: File = nil if open(f, filename, fmWrite): try: - f.writeBuffer(unsafeAddr content[0], content.len) + discard f.writeBuffer(unsafeAddr content[0], content.len) finally: close(f) else: @@ -919,7 +895,7 @@ proc readLines*(filename: string, n: Natural): seq[string] = finally: close(f) else: - sysFatal(IOError, "cannot open: " & filename) + raise newException(IOError, "cannot open: " & filename) template readLines*(filename: string): seq[ string] {.deprecated: "use readLines with two arguments".} = @@ -960,3 +936,7 @@ iterator lines*(f: File): string {.tags: [ReadIOEffect].} = result.lines += 1 var res = newStringOfCap(80) while f.readLine(res): yield res + +template `&=`*(f: File, x: typed) = + ## An alias for `write`. + write(f, x) diff --git a/lib/std/sysatomics.nim b/lib/std/sysatomics.nim new file mode 100644 index 000000000..2f203b3eb --- /dev/null +++ b/lib/std/sysatomics.nim @@ -0,0 +1,376 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2015 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +when defined(nimPreviewSlimSystem): + {.deprecated: "use `std/atomics` instead".} + +# Atomic operations for Nim. +{.push stackTrace:off, profiler:off.} + +const + hasThreadSupport = compileOption("threads") and not defined(nimscript) +const someGcc = defined(gcc) or defined(llvm_gcc) or defined(clang) or defined(nintendoswitch) +const someVcc = defined(vcc) or defined(clang_cl) + +type + AtomType* = SomeNumber|pointer|ptr|char|bool + ## Type Class representing valid types for use with atomic procs + +when someGcc: + type AtomMemModel* = distinct cint + + var ATOMIC_RELAXED* {.importc: "__ATOMIC_RELAXED", nodecl.}: AtomMemModel + ## No barriers or synchronization. + var ATOMIC_CONSUME* {.importc: "__ATOMIC_CONSUME", nodecl.}: AtomMemModel + ## Data dependency only for both barrier and + ## synchronization with another thread. + var ATOMIC_ACQUIRE* {.importc: "__ATOMIC_ACQUIRE", nodecl.}: AtomMemModel + ## Barrier to hoisting of code and synchronizes with + ## release (or stronger) + ## semantic stores from another thread. + var ATOMIC_RELEASE* {.importc: "__ATOMIC_RELEASE", nodecl.}: AtomMemModel + ## Barrier to sinking of code and synchronizes with + ## acquire (or stronger) + ## semantic loads from another thread. + var ATOMIC_ACQ_REL* {.importc: "__ATOMIC_ACQ_REL", nodecl.}: AtomMemModel + ## Full barrier in both directions and synchronizes + ## with acquire loads + ## and release stores in another thread. + var ATOMIC_SEQ_CST* {.importc: "__ATOMIC_SEQ_CST", nodecl.}: AtomMemModel + ## Full barrier in both directions and synchronizes + ## with acquire loads + ## and release stores in all threads. + + proc atomicLoadN*[T: AtomType](p: ptr T, mem: AtomMemModel): T {. + importc: "__atomic_load_n", nodecl.} + ## This proc implements an atomic load operation. It returns the contents at p. + ## ATOMIC_RELAXED, ATOMIC_SEQ_CST, ATOMIC_ACQUIRE, ATOMIC_CONSUME. + + proc atomicLoad*[T: AtomType](p, ret: ptr T, mem: AtomMemModel) {. + importc: "__atomic_load", nodecl.} + ## This is the generic version of an atomic load. It returns the contents at p in ret. + + proc atomicStoreN*[T: AtomType](p: ptr T, val: T, mem: AtomMemModel) {. + importc: "__atomic_store_n", nodecl.} + ## This proc implements an atomic store operation. It writes val at p. + ## ATOMIC_RELAXED, ATOMIC_SEQ_CST, and ATOMIC_RELEASE. + + proc atomicStore*[T: AtomType](p, val: ptr T, mem: AtomMemModel) {. + importc: "__atomic_store", nodecl.} + ## This is the generic version of an atomic store. It stores the value of val at p + + proc atomicExchangeN*[T: AtomType](p: ptr T, val: T, mem: AtomMemModel): T {. + importc: "__atomic_exchange_n", nodecl.} + ## This proc implements an atomic exchange operation. It writes val at p, + ## and returns the previous contents at p. + ## ATOMIC_RELAXED, ATOMIC_SEQ_CST, ATOMIC_ACQUIRE, ATOMIC_RELEASE, ATOMIC_ACQ_REL + + proc atomicExchange*[T: AtomType](p, val, ret: ptr T, mem: AtomMemModel) {. + importc: "__atomic_exchange", nodecl.} + ## This is the generic version of an atomic exchange. It stores the contents at val at p. + ## The original value at p is copied into ret. + + proc atomicCompareExchangeN*[T: AtomType](p, expected: ptr T, desired: T, + weak: bool, success_memmodel: AtomMemModel, failure_memmodel: AtomMemModel): bool {. + importc: "__atomic_compare_exchange_n", nodecl.} + ## This proc implements an atomic compare and exchange operation. This compares the + ## contents at p with the contents at expected and if equal, writes desired at p. + ## If they are not equal, the current contents at p is written into expected. + ## Weak is true for weak compare_exchange, and false for the strong variation. + ## Many targets only offer the strong variation and ignore the parameter. + ## When in doubt, use the strong variation. + ## True is returned if desired is written at p and the execution is considered + ## to conform to the memory model specified by success_memmodel. There are no + ## restrictions on what memory model can be used here. False is returned otherwise, + ## and the execution is considered to conform to failure_memmodel. This memory model + ## cannot be __ATOMIC_RELEASE nor __ATOMIC_ACQ_REL. It also cannot be a stronger model + ## than that specified by success_memmodel. + + proc atomicCompareExchange*[T: AtomType](p, expected, desired: ptr T, + weak: bool, success_memmodel: AtomMemModel, failure_memmodel: AtomMemModel): bool {. + importc: "__atomic_compare_exchange", nodecl.} + ## This proc implements the generic version of atomic_compare_exchange. + ## The proc is virtually identical to atomic_compare_exchange_n, except the desired + ## value is also a pointer. + + ## Perform the operation return the new value, all memory models are valid + proc atomicAddFetch*[T: AtomType](p: ptr T, val: T, mem: AtomMemModel): T {. + importc: "__atomic_add_fetch", nodecl.} + proc atomicSubFetch*[T: AtomType](p: ptr T, val: T, mem: AtomMemModel): T {. + importc: "__atomic_sub_fetch", nodecl.} + proc atomicOrFetch*[T: AtomType](p: ptr T, val: T, mem: AtomMemModel): T {. + importc: "__atomic_or_fetch", nodecl.} + proc atomicAndFetch*[T: AtomType](p: ptr T, val: T, mem: AtomMemModel): T {. + importc: "__atomic_and_fetch", nodecl.} + proc atomicXorFetch*[T: AtomType](p: ptr T, val: T, mem: AtomMemModel): T {. + importc: "__atomic_xor_fetch", nodecl.} + proc atomicNandFetch*[T: AtomType](p: ptr T, val: T, mem: AtomMemModel): T {. + importc: "__atomic_nand_fetch", nodecl.} + + ## Perform the operation return the old value, all memory models are valid + proc atomicFetchAdd*[T: AtomType](p: ptr T, val: T, mem: AtomMemModel): T {. + importc: "__atomic_fetch_add", nodecl.} + proc atomicFetchSub*[T: AtomType](p: ptr T, val: T, mem: AtomMemModel): T {. + importc: "__atomic_fetch_sub", nodecl.} + proc atomicFetchOr*[T: AtomType](p: ptr T, val: T, mem: AtomMemModel): T {. + importc: "__atomic_fetch_or", nodecl.} + proc atomicFetchAnd*[T: AtomType](p: ptr T, val: T, mem: AtomMemModel): T {. + importc: "__atomic_fetch_and", nodecl.} + proc atomicFetchXor*[T: AtomType](p: ptr T, val: T, mem: AtomMemModel): T {. + importc: "__atomic_fetch_xor", nodecl.} + proc atomicFetchNand*[T: AtomType](p: ptr T, val: T, mem: AtomMemModel): T {. + importc: "__atomic_fetch_nand", nodecl.} + + proc atomicTestAndSet*(p: pointer, mem: AtomMemModel): bool {. + importc: "__atomic_test_and_set", nodecl.} + ## This built-in function performs an atomic test-and-set operation on the byte at p. + ## The byte is set to some implementation defined nonzero "set" value and the return + ## value is true if and only if the previous contents were "set". + ## All memory models are valid. + + proc atomicClear*(p: pointer, mem: AtomMemModel) {. + importc: "__atomic_clear", nodecl.} + ## This built-in function performs an atomic clear operation at p. + ## After the operation, at p contains 0. + ## ATOMIC_RELAXED, ATOMIC_SEQ_CST, ATOMIC_RELEASE + + proc atomicThreadFence*(mem: AtomMemModel) {. + importc: "__atomic_thread_fence", nodecl.} + ## This built-in function acts as a synchronization fence between threads based + ## on the specified memory model. All memory orders are valid. + + proc atomicSignalFence*(mem: AtomMemModel) {. + importc: "__atomic_signal_fence", nodecl.} + ## This built-in function acts as a synchronization fence between a thread and + ## signal handlers based in the same thread. All memory orders are valid. + + proc atomicAlwaysLockFree*(size: int, p: pointer): bool {. + importc: "__atomic_always_lock_free", nodecl.} + ## This built-in function returns true if objects of size bytes always generate + ## lock free atomic instructions for the target architecture. size must resolve + ## to a compile-time constant and the result also resolves to a compile-time constant. + ## ptr is an optional pointer to the object that may be used to determine alignment. + ## A value of 0 indicates typical alignment should be used. The compiler may also + ## ignore this parameter. + + proc atomicIsLockFree*(size: int, p: pointer): bool {. + importc: "__atomic_is_lock_free", nodecl.} + ## This built-in function returns true if objects of size bytes always generate + ## lock free atomic instructions for the target architecture. If it is not known + ## to be lock free a call is made to a runtime routine named __atomic_is_lock_free. + ## ptr is an optional pointer to the object that may be used to determine alignment. + ## A value of 0 indicates typical alignment should be used. The compiler may also + ## ignore this parameter. + + template fence*() = atomicThreadFence(ATOMIC_SEQ_CST) +elif someVcc: + type AtomMemModel* = distinct cint + + const + ATOMIC_RELAXED* = 0.AtomMemModel + ATOMIC_CONSUME* = 1.AtomMemModel + ATOMIC_ACQUIRE* = 2.AtomMemModel + ATOMIC_RELEASE* = 3.AtomMemModel + ATOMIC_ACQ_REL* = 4.AtomMemModel + ATOMIC_SEQ_CST* = 5.AtomMemModel + + proc `==`(x1, x2: AtomMemModel): bool {.borrow.} + + proc readBarrier() {.importc: "_ReadBarrier", header: "<intrin.h>".} + proc writeBarrier() {.importc: "_WriteBarrier", header: "<intrin.h>".} + proc fence*() {.importc: "_ReadWriteBarrier", header: "<intrin.h>".} + + when defined(cpp): + proc interlockedCompareExchange64(p: pointer; exchange, comparand: int64): int64 + {.importcpp: "_InterlockedCompareExchange64(static_cast<NI64 volatile *>(#), #, #)", header: "<intrin.h>".} + proc interlockedCompareExchange32(p: pointer; exchange, comparand: int32): int32 + {.importcpp: "_InterlockedCompareExchange(static_cast<long volatile *>(#), #, #)", header: "<intrin.h>".} + proc interlockedCompareExchange8(p: pointer; exchange, comparand: byte): byte + {.importcpp: "_InterlockedCompareExchange8(static_cast<char volatile *>(#), #, #)", header: "<intrin.h>".} + proc interlockedExchange8(location: pointer; desired: int8): int8 {.importcpp: "_InterlockedExchange8(static_cast<NI8 volatile *>(#), #)", header: "<intrin.h>".} + proc interlockedExchange16(location: pointer; desired: int16): int16 {.importcpp: "_InterlockedExchange16(static_cast<NI16 volatile *>(#), #)", header: "<intrin.h>".} + proc interlockedExchange32(location: pointer; desired: int32): int32 {.importcpp: "_InterlockedExchange(static_cast<long volatile *>(#), #)", header: "<intrin.h>".} + proc interlockedExchange64(location: pointer; desired: int64): int64 {.importcpp: "_InterlockedExchange64(static_cast<NI64 volatile *>(#), #)", header: "<intrin.h>".} + else: + proc interlockedCompareExchange64(p: pointer; exchange, comparand: int64): int64 + {.importc: "_InterlockedCompareExchange64", header: "<intrin.h>".} + proc interlockedCompareExchange32(p: pointer; exchange, comparand: int32): int32 + {.importc: "_InterlockedCompareExchange", header: "<intrin.h>".} + proc interlockedCompareExchange8(p: pointer; exchange, comparand: byte): byte + {.importc: "_InterlockedCompareExchange8", header: "<intrin.h>".} + + proc interlockedExchange8(location: pointer; desired: int8): int8 {.importc: "_InterlockedExchange8", header: "<intrin.h>".} + proc interlockedExchange16(location: pointer; desired: int16): int16 {.importc: "_InterlockedExchange16", header: "<intrin.h>".} + proc interlockedExchange32(location: pointer; desired: int32): int32 {.importc: "_InterlockedExchange", header: "<intrin.h>".} + proc interlockedExchange64(location: pointer; desired: int64): int64 {.importc: "_InterlockedExchange64", header: "<intrin.h>".} + + + template barrier(mem: AtomMemModel) = + when mem == ATOMIC_RELAXED: discard + elif mem == ATOMIC_CONSUME: readBarrier() + elif mem == ATOMIC_ACQUIRE: writeBarrier() + elif mem == ATOMIC_RELEASE: fence() + elif mem == ATOMIC_ACQ_REL: fence() + elif mem == ATOMIC_SEQ_CST: fence() + + proc atomicStoreN*[T: AtomType](p: ptr T, val: T, mem: static[AtomMemModel]) = + barrier(mem) + p[] = val + + proc atomicLoadN*[T: AtomType](p: ptr T, mem: static[AtomMemModel]): T = + result = p[] + barrier(mem) + + proc atomicCompareExchangeN*[T: ptr](p, expected: ptr T, desired: T, + weak: bool, success_memmodel: AtomMemModel, failure_memmodel: AtomMemModel): bool = + when sizeof(T) == 8: + interlockedCompareExchange64(p, cast[int64](desired), cast[int64](expected)) == + cast[int64](expected) + elif sizeof(T) == 4: + interlockedCompareExchange32(p, cast[int32](desired), cast[int32](expected)) == + cast[int32](expected) + + proc atomicExchangeN*[T: ptr](p: ptr T, val: T, mem: AtomMemModel): T = + when sizeof(T) == 8: + cast[T](interlockedExchange64(p, cast[int64](val))) + elif sizeof(T) == 4: + cast[T](interlockedExchange32(p, cast[int32](val))) + when defined(cpp): + when sizeof(int) == 8: + proc addAndFetch*(p: ptr int, val: int): int {. + importcpp: "_InterlockedExchangeAdd64(static_cast<NI volatile *>(#), #)", + header: "<intrin.h>".} + else: + proc addAndFetch*(p: ptr int, val: int): int {. + importcpp: "_InterlockedExchangeAdd(reinterpret_cast<long volatile *>(#), static_cast<long>(#))", + header: "<intrin.h>".} + else: + when sizeof(int) == 8: + proc addAndFetch*(p: ptr int, val: int): int {. + importc: "_InterlockedExchangeAdd64", header: "<intrin.h>".} + else: + proc addAndFetch*(p: ptr int, val: int): int {. + importc: "_InterlockedExchangeAdd", header: "<intrin.h>".} + +else: + proc addAndFetch*(p: ptr int, val: int): int {.inline.} = + inc(p[], val) + result = p[] + + +proc atomicInc*(memLoc: var int, x: int = 1): int {.inline, discardable, raises: [], tags: [].} = + ## Atomically increments the integer by some `x`. It returns the new value. + when someGcc and hasThreadSupport: + result = atomicAddFetch(memLoc.addr, x, ATOMIC_SEQ_CST) + elif someVcc and hasThreadSupport: + result = addAndFetch(memLoc.addr, x) + inc(result, x) + else: + inc(memLoc, x) + result = memLoc + +proc atomicDec*(memLoc: var int, x: int = 1): int {.inline, discardable, raises: [], tags: [].} = + ## Atomically decrements the integer by some `x`. It returns the new value. + when someGcc and hasThreadSupport: + when declared(atomicSubFetch): + result = atomicSubFetch(memLoc.addr, x, ATOMIC_SEQ_CST) + else: + result = atomicAddFetch(memLoc.addr, -x, ATOMIC_SEQ_CST) + elif someVcc and hasThreadSupport: + result = addAndFetch(memLoc.addr, -x) + dec(result, x) + else: + dec(memLoc, x) + result = memLoc + +when someVcc: + proc cas*[T: bool|int|ptr](p: ptr T; oldValue, newValue: T): bool = + when sizeof(T) == 8: + interlockedCompareExchange64(p, cast[int64](newValue), cast[int64](oldValue)) == + cast[int64](oldValue) + elif sizeof(T) == 4: + interlockedCompareExchange32(p, cast[int32](newValue), cast[int32](oldValue)) == + cast[int32](oldValue) + elif sizeof(T) == 1: + interlockedCompareExchange8(p, cast[byte](newValue), cast[byte](oldValue)) == + cast[byte](oldValue) + else: + {.error: "invalid CAS instruction".} + +elif defined(tcc): + when defined(amd64): + {.emit:""" +static int __tcc_cas(int *ptr, int oldVal, int newVal) +{ + unsigned char ret; + __asm__ __volatile__ ( + " lock\n" + " cmpxchgq %2,%1\n" + " sete %0\n" + : "=q" (ret), "=m" (*ptr) + : "r" (newVal), "m" (*ptr), "a" (oldVal) + : "memory"); + + return ret; +} +""".} + else: + #assert sizeof(int) == 4 + {.emit:""" +static int __tcc_cas(int *ptr, int oldVal, int newVal) +{ + unsigned char ret; + __asm__ __volatile__ ( + " lock\n" + " cmpxchgl %2,%1\n" + " sete %0\n" + : "=q" (ret), "=m" (*ptr) + : "r" (newVal), "m" (*ptr), "a" (oldVal) + : "memory"); + + return ret; +} +""".} + + proc tcc_cas(p: ptr int; oldValue, newValue: int): bool + {.importc: "__tcc_cas", nodecl.} + proc cas*[T: bool|int|ptr](p: ptr T; oldValue, newValue: T): bool = + tcc_cas(cast[ptr int](p), cast[int](oldValue), cast[int](newValue)) +elif declared(atomicCompareExchangeN): + proc cas*[T: bool|int|ptr](p: ptr T; oldValue, newValue: T): bool = + atomicCompareExchangeN(p, oldValue.unsafeAddr, newValue, false, ATOMIC_SEQ_CST, ATOMIC_SEQ_CST) +else: + # this is valid for GCC and Intel C++ + proc cas*[T: bool|int|ptr](p: ptr T; oldValue, newValue: T): bool + {.importc: "__sync_bool_compare_and_swap", nodecl.} + # XXX is this valid for 'int'? + + +when (defined(x86) or defined(amd64)) and someVcc: + proc cpuRelax* {.importc: "YieldProcessor", header: "<windows.h>".} +elif (defined(x86) or defined(amd64)) and (someGcc or defined(bcc)): + proc cpuRelax* {.inline.} = + {.emit: """asm volatile("pause" ::: "memory");""".} +elif someGcc or defined(tcc): + proc cpuRelax* {.inline.} = + {.emit: """asm volatile("" ::: "memory");""".} +elif defined(icl): + proc cpuRelax* {.importc: "_mm_pause", header: "xmmintrin.h".} +elif false: + from std/os import sleep + + proc cpuRelax* {.inline.} = os.sleep(1) + +when not declared(fence) and hasThreadSupport: + # XXX fixme + proc fence*() {.inline.} = + var dummy: bool + discard cas(addr dummy, false, true) + +{.pop.} diff --git a/lib/std/sysrand.nim b/lib/std/sysrand.nim index ff62c920b..6f2c6b0c1 100644 --- a/lib/std/sysrand.nim +++ b/lib/std/sysrand.nim @@ -20,7 +20,7 @@ ## | :--- | ----: | ## | Windows | `BCryptGenRandom`_ | ## | Linux | `getrandom`_ | -## | MacOSX | `getentropy`_ | +## | MacOSX | `SecRandomCopyBytes`_ | ## | iOS | `SecRandomCopyBytes`_ | ## | OpenBSD | `getentropy openbsd`_ | ## | FreeBSD | `getrandom freebsd`_ | @@ -57,16 +57,16 @@ runnableExamples: when not defined(js): - import os + import std/oserrors when defined(posix): - import posix + import std/posix when defined(nimPreviewSlimSystem): import std/assertions const - batchImplOS = defined(freebsd) or defined(openbsd) or defined(zephyr) or (defined(macosx) and not defined(ios)) + batchImplOS = defined(freebsd) or defined(openbsd) or defined(zephyr) batchSize {.used.} = 256 when batchImplOS: @@ -168,8 +168,10 @@ elif defined(windows): result = randomBytes(addr dest[0], size) elif defined(linux) and not defined(nimNoGetRandom) and not defined(emscripten): - # TODO using let, pending bootstrap >= 1.4.0 - var SYS_getrandom {.importc: "SYS_getrandom", header: "<sys/syscall.h>".}: clong + when (NimMajor, NimMinor) >= (1, 4): + let SYS_getrandom {.importc: "SYS_getrandom", header: "<sys/syscall.h>".}: clong + else: + var SYS_getrandom {.importc: "SYS_getrandom", header: "<sys/syscall.h>".}: clong const syscallHeader = """#include <unistd.h> #include <sys/syscall.h>""" @@ -190,12 +192,11 @@ elif defined(linux) and not defined(nimNoGetRandom) and not defined(emscripten): while result < size: let readBytes = syscall(SYS_getrandom, addr dest[result], cint(size - result), 0).int if readBytes == 0: - doAssert false + raiseAssert "unreachable" elif readBytes > 0: inc(result, readBytes) else: - if osLastError().int in {EINTR, EAGAIN}: - discard + if osLastError().cint in [EINTR, EAGAIN]: discard else: result = -1 break @@ -231,8 +232,8 @@ elif defined(freebsd): proc getRandomImpl(p: pointer, size: int): int {.inline.} = result = getrandom(p, csize_t(size), 0) -elif defined(ios): - {.passL: "-framework Security".} +elif defined(ios) or defined(macosx): + {.passl: "-framework Security".} const errSecSuccess = 0 ## No error. @@ -254,19 +255,6 @@ elif defined(ios): result = secRandomCopyBytes(nil, csize_t(size), addr dest[0]) -elif defined(macosx): - const sysrandomHeader = """#include <Availability.h> -#include <sys/random.h> -""" - - proc getentropy(p: pointer, size: csize_t): cint {.importc: "getentropy", header: sysrandomHeader.} - # getentropy() fills a buffer with random data, which can be used as input - # for process-context pseudorandom generators like arc4random(3). - # The maximum buffer size permitted is 256 bytes. - - proc getRandomImpl(p: pointer, size: int): int {.inline.} = - result = getentropy(p, csize_t(size)).int - else: template urandomImpl(result: var int, dest: var openArray[byte]) = let size = dest.len diff --git a/lib/std/tasks.nim b/lib/std/tasks.nim index ac35e26bf..7e59747f5 100644 --- a/lib/std/tasks.nim +++ b/lib/std/tasks.nim @@ -11,7 +11,6 @@ ## A `Task` should be only owned by a single Thread, it cannot be shared by threads. import std/[macros, isolation, typetraits] -import system/ansi_c when defined(nimPreviewSlimSystem): import std/assertions @@ -62,24 +61,33 @@ when compileOption("threads"): type Task* = object ## `Task` contains the callback and its arguments. - callback: proc (args: pointer) {.nimcall, gcsafe.} + callback: proc (args, res: pointer) {.nimcall, gcsafe.} args: pointer destroy: proc (args: pointer) {.nimcall, gcsafe.} proc `=copy`*(x: var Task, y: Task) {.error.} -proc `=destroy`*(t: var Task) {.inline, gcsafe.} = - ## Frees the resources allocated for a `Task`. - if t.args != nil: - if t.destroy != nil: - t.destroy(t.args) - c_free(t.args) - -proc invoke*(task: Task) {.inline, gcsafe.} = +const arcLike = defined(gcArc) or defined(gcAtomicArc) or defined(gcOrc) +when defined(nimAllowNonVarDestructor) and arcLike: + proc `=destroy`*(t: Task) {.inline, gcsafe.} = + ## Frees the resources allocated for a `Task`. + if t.args != nil: + if t.destroy != nil: + t.destroy(t.args) + deallocShared(t.args) +else: + proc `=destroy`*(t: var Task) {.inline, gcsafe.} = + ## Frees the resources allocated for a `Task`. + if t.args != nil: + if t.destroy != nil: + t.destroy(t.args) + deallocShared(t.args) + +proc invoke*(task: Task; res: pointer = nil) {.inline, gcsafe.} = ## Invokes the `task`. assert task.callback != nil - task.callback(task.args) + task.callback(task.args, res) template checkIsolate(scratchAssignList: seq[NimNode], procParam, scratchDotExpr: NimNode) = # block: @@ -102,21 +110,38 @@ template addAllNode(assignParam: NimNode, procParam: NimNode) = tempAssignList.add newLetStmt(tempNode, newDotExpr(objTemp, formalParams[i][0])) scratchRecList.add newIdentDefs(newIdentNode(formalParams[i][0].strVal), assignParam) +proc analyseRootSym(s: NimNode): NimNode = + result = s + while true: + case result.kind + of nnkBracketExpr, nnkDerefExpr, nnkHiddenDeref, + nnkAddr, nnkHiddenAddr, + nnkObjDownConv, nnkObjUpConv: + result = result[0] + of nnkDotExpr, nnkCheckedFieldExpr, nnkHiddenStdConv, nnkHiddenSubConv: + result = result[1] + else: + break + macro toTask*(e: typed{nkCall | nkInfix | nkPrefix | nkPostfix | nkCommand | nkCallStrLit}): Task = ## Converts the call and its arguments to `Task`. - runnableExamples("--gc:orc"): + runnableExamples: proc hello(a: int) = echo a let b = toTask hello(13) assert b is Task - doAssert getTypeInst(e).typeKind == ntyVoid + let retType = getTypeInst(e) + let returnsVoid = retType.typeKind == ntyVoid + + let rootSym = analyseRootSym(e[0]) + expectKind rootSym, nnkSym when compileOption("threads"): - if not isGcSafe(e[0]): + if not isGcSafe(rootSym): error("'toTask' takes a GC safe call expression", e) - if hasClosure(e[0]): + if hasClosure(rootSym): error("closure call is not allowed", e) if e.len > 1: @@ -165,7 +190,7 @@ macro toTask*(e: typed{nkCall | nkInfix | nkPrefix | nkPostfix | nkCommand | nkC # passing by static parameters # so we pass them directly instead of passing by scratchObj callNode.add nnkExprEqExpr.newTree(formalParams[i][0], e[i]) - of nnkSym, nnkPtrTy: + of nnkSym, nnkPtrTy, nnkProcTy, nnkTupleConstr: addAllNode(param, e[i]) of nnkCharLit..nnkNilLit: callNode.add nnkExprEqExpr.newTree(formalParams[i][0], e[i]) @@ -187,40 +212,43 @@ macro toTask*(e: typed{nkCall | nkInfix | nkPrefix | nkPostfix | nkCommand | nkC let scratchObjPtrType = quote do: - cast[ptr `scratchObjType`](c_calloc(csize_t 1, csize_t sizeof(`scratchObjType`))) - - let scratchLetSection = newLetStmt( - scratchIdent, - scratchObjPtrType - ) + cast[ptr `scratchObjType`](allocShared0(sizeof(`scratchObjType`))) - let scratchCheck = quote do: - if `scratchIdent`.isNil: - raise newException(OutOfMemDefect, "Could not allocate memory") + let scratchLetSection = newLetStmt(scratchIdent, scratchObjPtrType) var stmtList = newStmtList() stmtList.add(scratchObj) stmtList.add(scratchLetSection) - stmtList.add(scratchCheck) stmtList.add(nnkBlockStmt.newTree(newEmptyNode(), newStmtList(scratchAssignList))) var functionStmtList = newStmtList() let funcCall = newCall(e[0], callNode) functionStmtList.add tempAssignList - functionStmtList.add funcCall - let funcName = genSym(nskProc, e[0].strVal) + let funcName = genSym(nskProc, rootSym.strVal) let destroyName = genSym(nskProc, "destroyScratch") let objTemp2 = genSym(ident = "obj") let tempNode = quote("@") do: `=destroy`(@objTemp2[]) + var funcDecl: NimNode + if returnsVoid: + funcDecl = quote do: + proc `funcName`(args, res: pointer) {.gcsafe, nimcall.} = + let `objTemp` = cast[ptr `scratchObjType`](args) + `functionStmtList` + `funcCall` + else: + funcDecl = quote do: + proc `funcName`(args, res: pointer) {.gcsafe, nimcall.} = + let `objTemp` = cast[ptr `scratchObjType`](args) + `functionStmtList` + cast[ptr `retType`](res)[] = `funcCall` + result = quote do: `stmtList` - proc `funcName`(args: pointer) {.gcsafe, nimcall.} = - let `objTemp` = cast[ptr `scratchObjType`](args) - `functionStmtList` + `funcDecl` proc `destroyName`(args: pointer) {.gcsafe, nimcall.} = let `objTemp2` = cast[ptr `scratchObjType`](args) @@ -229,18 +257,26 @@ macro toTask*(e: typed{nkCall | nkInfix | nkPrefix | nkPostfix | nkCommand | nkC Task(callback: `funcName`, args: `scratchIdent`, destroy: `destroyName`) else: let funcCall = newCall(e[0]) - let funcName = genSym(nskProc, e[0].strVal) + let funcName = genSym(nskProc, rootSym.strVal) - result = quote do: - proc `funcName`(args: pointer) {.gcsafe, nimcall.} = - `funcCall` + if returnsVoid: + result = quote do: + proc `funcName`(args, res: pointer) {.gcsafe, nimcall.} = + `funcCall` + + Task(callback: `funcName`, args: nil) + else: + result = quote do: + proc `funcName`(args, res: pointer) {.gcsafe, nimcall.} = + cast[ptr `retType`](res)[] = `funcCall` + + Task(callback: `funcName`, args: nil) - Task(callback: `funcName`, args: nil) when defined(nimTasksDebug): echo result.repr -runnableExamples("--gc:orc"): +runnableExamples: block: var num = 0 proc hello(a: int) = inc num, a diff --git a/lib/std/tempfiles.nim b/lib/std/tempfiles.nim index ee42f8a5c..539305bde 100644 --- a/lib/std/tempfiles.nim +++ b/lib/std/tempfiles.nim @@ -17,7 +17,7 @@ See also: * `mkstemp` (posix), refs https://man7.org/linux/man-pages/man3/mkstemp.3.html ]# -import os, random +import std / [os, random] when defined(nimPreviewSlimSystem): import std/syncio @@ -29,7 +29,9 @@ const when defined(windows): - import winlean + import std/winlean + when defined(nimPreviewSlimSystem): + import std/widestrs var O_RDWR {.importc: "_O_RDWR", header: "<fcntl.h>".}: cint @@ -44,7 +46,7 @@ when defined(windows): proc close_osfandle(fd: cint): cint {. importc: "_close", header: "<io.h>".} else: - import posix + import std/posix proc c_fdopen( filehandle: cint, diff --git a/lib/std/time_t.nim b/lib/std/time_t.nim index 7fb6e6d46..de051b135 100644 --- a/lib/std/time_t.nim +++ b/lib/std/time_t.nim @@ -14,10 +14,10 @@ when defined(nimdoc): ## Wrapper for `time_t`. On posix, this is an alias to `posix.Time`. elif defined(windows): when defined(i386) and defined(gcc): - type Time* {.importc: "time_t", header: "<time.h>".} = distinct int32 + type Time* {.importc: "time_t", header: "<time.h>".} = distinct clong else: # newest version of Visual C++ defines time_t to be of 64 bits type Time* {.importc: "time_t", header: "<time.h>".} = distinct int64 elif defined(posix): - import posix + import std/posix export posix.Time \ No newline at end of file diff --git a/lib/std/typedthreads.nim b/lib/std/typedthreads.nim new file mode 100644 index 000000000..7b0b81968 --- /dev/null +++ b/lib/std/typedthreads.nim @@ -0,0 +1,305 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2012 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +##[ +Thread support for Nim. Threads allow multiple functions to execute concurrently. + +In Nim, threads are a low-level construct and using a library like `malebolgia`, `taskpools` or `weave` is recommended. + +When creating a thread, you can pass arguments to it. As Nim's garbage collector does not use atomic references, sharing +`ref` and other variables managed by the garbage collector between threads is not supported. +Use global variables to do so, or pointers. + +Memory allocated using [`sharedAlloc`](./system.html#allocShared.t%2CNatural) can be used and shared between threads. + +To communicate between threads, consider using [channels](./system.html#Channel) + +Examples +======== + +```Nim +import std/locks + +var + thr: array[0..4, Thread[tuple[a,b: int]]] + L: Lock + +proc threadFunc(interval: tuple[a,b: int]) {.thread.} = + for i in interval.a..interval.b: + acquire(L) # lock stdout + echo i + release(L) + +initLock(L) + +for i in 0..high(thr): + createThread(thr[i], threadFunc, (i*10, i*10+5)) +joinThreads(thr) + +deinitLock(L) +``` + +When using a memory management strategy that supports shared heaps like `arc` or `boehm`, +you can pass pointer to threads and share memory between them, but the memory must outlive the thread. +The default memory management strategy, `orc`, supports this. +The example below is **not valid** for memory management strategies that use local heaps like `refc`! + +```Nim +import locks + +var l: Lock + +proc threadFunc(obj: ptr seq[int]) {.thread.} = + withLock l: + for i in 0..<100: + obj[].add(obj[].len * obj[].len) + +proc threadHandler() = + var thr: array[0..4, Thread[ptr seq[int]]] + var s = newSeq[int]() + + for i in 0..high(thr): + createThread(thr[i], threadFunc, s.addr) + joinThreads(thr) + echo s + +initLock(l) +threadHandler() +deinitLock(l) +``` +]## + + +import std/private/[threadtypes] +export Thread + +import system/ansi_c + +when defined(nimPreviewSlimSystem): + import std/assertions + +when defined(genode): + import genode/env + +when hostOS == "any": + {.error: "Threads not implemented for os:any. Please compile with --threads:off.".} + +when hasAllocStack or defined(zephyr) or defined(freertos) or defined(nuttx) or + defined(cpu16) or defined(cpu8): + const + nimThreadStackSize {.intdefine.} = 8192 + nimThreadStackGuard {.intdefine.} = 128 + + StackGuardSize = nimThreadStackGuard + ThreadStackSize = nimThreadStackSize - nimThreadStackGuard +else: + const + StackGuardSize = 4096 + ThreadStackMask = + when defined(genode): + 1024*64*sizeof(int)-1 + else: + 1024*256*sizeof(int)-1 + + ThreadStackSize = ThreadStackMask+1 - StackGuardSize + + +when defined(gcDestructors): + proc allocThreadStorage(size: int): pointer = + result = c_malloc(csize_t size) + zeroMem(result, size) +else: + template allocThreadStorage(size: untyped): untyped = allocShared0(size) + +#const globalsSlot = ThreadVarSlot(0) +#sysAssert checkSlot.int == globalsSlot.int + +# Zephyr doesn't include this properly without some help +when defined(zephyr): + {.emit: """/*INCLUDESECTION*/ + #include <pthread.h> + """.} + + +# We jump through some hops here to ensure that Nim thread procs can have +# the Nim calling convention. This is needed because thread procs are +# ``stdcall`` on Windows and ``noconv`` on UNIX. Alternative would be to just +# use ``stdcall`` since it is mapped to ``noconv`` on UNIX anyway. + + + +{.push stack_trace:off.} +when defined(windows): + proc threadProcWrapper[TArg](closure: pointer): int32 {.stdcall.} = + nimThreadProcWrapperBody(closure) + # implicitly return 0 +elif defined(genode): + proc threadProcWrapper[TArg](closure: pointer) {.noconv.} = + nimThreadProcWrapperBody(closure) +else: + proc threadProcWrapper[TArg](closure: pointer): pointer {.noconv.} = + nimThreadProcWrapperBody(closure) +{.pop.} + +proc running*[TArg](t: Thread[TArg]): bool {.inline.} = + ## Returns true if `t` is running. + result = t.dataFn != nil + +proc handle*[TArg](t: Thread[TArg]): SysThread {.inline.} = + ## Returns the thread handle of `t`. + result = t.sys + +when hostOS == "windows": + const MAXIMUM_WAIT_OBJECTS = 64 + + proc joinThread*[TArg](t: Thread[TArg]) {.inline.} = + ## Waits for the thread `t` to finish. + discard waitForSingleObject(t.sys, -1'i32) + + proc joinThreads*[TArg](t: varargs[Thread[TArg]]) = + ## Waits for every thread in `t` to finish. + var a: array[MAXIMUM_WAIT_OBJECTS, SysThread] + var k = 0 + while k < len(t): + var count = min(len(t) - k, MAXIMUM_WAIT_OBJECTS) + for i in 0..(count - 1): a[i] = t[i + k].sys + discard waitForMultipleObjects(int32(count), + cast[ptr SysThread](addr(a)), 1, -1) + inc(k, MAXIMUM_WAIT_OBJECTS) + +elif defined(genode): + proc joinThread*[TArg](t: Thread[TArg]) {.importcpp.} + ## Waits for the thread `t` to finish. + + proc joinThreads*[TArg](t: varargs[Thread[TArg]]) = + ## Waits for every thread in `t` to finish. + for i in 0..t.high: joinThread(t[i]) + +else: + proc joinThread*[TArg](t: Thread[TArg]) {.inline.} = + ## Waits for the thread `t` to finish. + discard pthread_join(t.sys, nil) + + proc joinThreads*[TArg](t: varargs[Thread[TArg]]) = + ## Waits for every thread in `t` to finish. + for i in 0..t.high: joinThread(t[i]) + +when false: + # XXX a thread should really release its heap here somehow: + proc destroyThread*[TArg](t: var Thread[TArg]) = + ## Forces the thread `t` to terminate. This is potentially dangerous if + ## you don't have full control over `t` and its acquired resources. + when hostOS == "windows": + discard TerminateThread(t.sys, 1'i32) + else: + discard pthread_cancel(t.sys) + when declared(registerThread): unregisterThread(addr(t)) + t.dataFn = nil + ## if thread `t` already exited, `t.core` will be `null`. + if not isNil(t.core): + deallocThreadStorage(t.core) + t.core = nil + +when hostOS == "windows": + proc createThread*[TArg](t: var Thread[TArg], + tp: proc (arg: TArg) {.thread, nimcall.}, + param: TArg) = + ## Creates a new thread `t` and starts its execution. + ## + ## Entry point is the proc `tp`. + ## `param` is passed to `tp`. `TArg` can be `void` if you + ## don't need to pass any data to the thread. + t.core = cast[PGcThread](allocThreadStorage(sizeof(GcThread))) + + when TArg isnot void: t.data = param + t.dataFn = tp + when hasSharedHeap: t.core.stackSize = ThreadStackSize + var dummyThreadId: int32 + t.sys = createThread(nil, ThreadStackSize, threadProcWrapper[TArg], + addr(t), 0'i32, dummyThreadId) + if t.sys <= 0: + raise newException(ResourceExhaustedError, "cannot create thread") + + proc pinToCpu*[Arg](t: var Thread[Arg]; cpu: Natural) = + ## Pins a thread to a `CPU`:idx:. + ## + ## In other words sets a thread's `affinity`:idx:. + ## If you don't know what this means, you shouldn't use this proc. + setThreadAffinityMask(t.sys, uint(1 shl cpu)) + +elif defined(genode): + var affinityOffset: cuint = 1 + ## CPU affinity offset for next thread, safe to roll-over. + + proc createThread*[TArg](t: var Thread[TArg], + tp: proc (arg: TArg) {.thread, nimcall.}, + param: TArg) = + t.core = cast[PGcThread](allocThreadStorage(sizeof(GcThread))) + + when TArg isnot void: t.data = param + t.dataFn = tp + when hasSharedHeap: t.stackSize = ThreadStackSize + t.sys.initThread( + runtimeEnv, + ThreadStackSize.culonglong, + threadProcWrapper[TArg], addr(t), affinityOffset) + inc affinityOffset + + proc pinToCpu*[Arg](t: var Thread[Arg]; cpu: Natural) = + {.hint: "cannot change Genode thread CPU affinity after initialization".} + discard + +else: + proc createThread*[TArg](t: var Thread[TArg], + tp: proc (arg: TArg) {.thread, nimcall.}, + param: TArg) = + ## Creates a new thread `t` and starts its execution. + ## + ## Entry point is the proc `tp`. `param` is passed to `tp`. + ## `TArg` can be `void` if you + ## don't need to pass any data to the thread. + t.core = cast[PGcThread](allocThreadStorage(sizeof(GcThread))) + + when TArg isnot void: t.data = param + t.dataFn = tp + when hasSharedHeap: t.core.stackSize = ThreadStackSize + var a {.noinit.}: Pthread_attr + doAssert pthread_attr_init(a) == 0 + when hasAllocStack: + var + rawstk = allocThreadStorage(ThreadStackSize + StackGuardSize) + stk = cast[pointer](cast[uint](rawstk) + StackGuardSize) + let setstacksizeResult = pthread_attr_setstack(addr a, stk, ThreadStackSize) + t.rawStack = rawstk + else: + let setstacksizeResult = pthread_attr_setstacksize(a, ThreadStackSize) + + when not defined(ios): + # This fails on iOS + doAssert(setstacksizeResult == 0) + if pthread_create(t.sys, a, threadProcWrapper[TArg], addr(t)) != 0: + raise newException(ResourceExhaustedError, "cannot create thread") + doAssert pthread_attr_destroy(a) == 0 + + proc pinToCpu*[Arg](t: var Thread[Arg]; cpu: Natural) = + ## Pins a thread to a `CPU`:idx:. + ## + ## In other words sets a thread's `affinity`:idx:. + ## If you don't know what this means, you shouldn't use this proc. + when not defined(macosx): + var s {.noinit.}: CpuSet + cpusetZero(s) + cpusetIncl(cpu.cint, s) + setAffinity(t.sys, csize_t(sizeof(s)), s) + +proc createThread*(t: var Thread[void], tp: proc () {.thread, nimcall.}) = + createThread[void](t, tp) + +when not defined(gcOrc): + include system/threadids diff --git a/lib/std/varints.nim b/lib/std/varints.nim index 0d18b9069..32fe2fffb 100644 --- a/lib/std/varints.nim +++ b/lib/std/varints.nim @@ -82,29 +82,29 @@ proc writeVu64*(z: var openArray[byte], x: uint64): int = z[3] = cast[uint8](y) return 4 z[0] = 251 - varintWrite32(toOpenArray(z, 1, z.high-1), y) + varintWrite32(toOpenArray(z, 1, 4), y) return 5 if w <= 255: z[0] = 252 z[1] = cast[uint8](w) - varintWrite32(toOpenArray(z, 2, z.high-2), y) + varintWrite32(toOpenArray(z, 2, 5), y) return 6 if w <= 65535: z[0] = 253 z[1] = cast[uint8](w shr 8) z[2] = cast[uint8](w) - varintWrite32(toOpenArray(z, 3, z.high-3), y) + varintWrite32(toOpenArray(z, 3, 6), y) return 7 if w <= 16777215: z[0] = 254 z[1] = cast[uint8](w shr 16) z[2] = cast[uint8](w shr 8) z[3] = cast[uint8](w) - varintWrite32(toOpenArray(z, 4, z.high-4), y) + varintWrite32(toOpenArray(z, 4, 7), y) return 8 z[0] = 255 - varintWrite32(toOpenArray(z, 1, z.high-1), w) - varintWrite32(toOpenArray(z, 5, z.high-5), y) + varintWrite32(toOpenArray(z, 1, 4), w) + varintWrite32(toOpenArray(z, 5, 8), y) return 9 proc sar(a, b: int64): int64 = diff --git a/lib/std/widestrs.nim b/lib/std/widestrs.nim new file mode 100644 index 000000000..2ddf80d14 --- /dev/null +++ b/lib/std/widestrs.nim @@ -0,0 +1,239 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2012 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Nim support for C/C++'s `wide strings`:idx:. + +#when not declared(ThisIsSystem): +# {.error: "You must not import this module explicitly".} + +type + Utf16Char* = distinct int16 + +when not (defined(cpu16) or defined(cpu8)): + when defined(nimv2): + + type + WideCString* = ptr UncheckedArray[Utf16Char] + + WideCStringObj* = object + bytes: int + data: WideCString + + const arcLike = defined(gcArc) or defined(gcAtomicArc) or defined(gcOrc) + when defined(nimAllowNonVarDestructor) and arcLike: + proc `=destroy`(a: WideCStringObj) = + if a.data != nil: + when compileOption("threads"): + deallocShared(a.data) + else: + dealloc(a.data) + else: + proc `=destroy`(a: var WideCStringObj) = + if a.data != nil: + when compileOption("threads"): + deallocShared(a.data) + else: + dealloc(a.data) + + proc `=copy`(a: var WideCStringObj; b: WideCStringObj) {.error.} + + proc `=sink`(a: var WideCStringObj; b: WideCStringObj) = + a.bytes = b.bytes + a.data = b.data + + proc createWide(a: var WideCStringObj; bytes: int) = + a.bytes = bytes + when compileOption("threads"): + a.data = cast[typeof(a.data)](allocShared0(bytes)) + else: + a.data = cast[typeof(a.data)](alloc0(bytes)) + + template `[]`*(a: WideCStringObj; idx: int): Utf16Char = a.data[idx] + template `[]=`*(a: WideCStringObj; idx: int; val: Utf16Char) = a.data[idx] = val + + template nullWide(): untyped = WideCStringObj(bytes: 0, data: nil) + + converter toWideCString*(x: WideCStringObj): WideCString {.inline.} = + result = x.data + + else: + template nullWide(): untyped = nil + + type + WideCString* = ref UncheckedArray[Utf16Char] + WideCStringObj* = WideCString + + template createWide(a; L) = + unsafeNew(a, L) + + proc ord(arg: Utf16Char): int = int(cast[uint16](arg)) + + proc len*(w: WideCString): int = + ## returns the length of a widestring. This traverses the whole string to + ## find the binary zero end marker! + result = 0 + while int16(w[result]) != 0'i16: inc result + + const + UNI_REPLACEMENT_CHAR = Utf16Char(0xFFFD'i16) + UNI_MAX_BMP = 0x0000FFFF + UNI_MAX_UTF16 = 0x0010FFFF + # UNI_MAX_UTF32 = 0x7FFFFFFF + # UNI_MAX_LEGAL_UTF32 = 0x0010FFFF + + halfShift = 10 + halfBase = 0x0010000 + halfMask = 0x3FF + + UNI_SUR_HIGH_START = 0xD800 + UNI_SUR_HIGH_END = 0xDBFF + UNI_SUR_LOW_START = 0xDC00 + UNI_SUR_LOW_END = 0xDFFF + UNI_REPL = 0xFFFD + + template ones(n: untyped): untyped = ((1 shl n)-1) + + template fastRuneAt(s: cstring, i, L: int, result: untyped, doInc = true) = + ## Returns the unicode character `s[i]` in `result`. If `doInc == true` + ## `i` is incremented by the number of bytes that have been processed. + bind ones + + if ord(s[i]) <= 127: + result = ord(s[i]) + when doInc: inc(i) + elif ord(s[i]) shr 5 == 0b110: + #assert(ord(s[i+1]) shr 6 == 0b10) + if i <= L - 2: + result = (ord(s[i]) and (ones(5))) shl 6 or (ord(s[i+1]) and ones(6)) + when doInc: inc(i, 2) + else: + result = UNI_REPL + when doInc: inc(i) + elif ord(s[i]) shr 4 == 0b1110: + if i <= L - 3: + #assert(ord(s[i+1]) shr 6 == 0b10) + #assert(ord(s[i+2]) shr 6 == 0b10) + result = (ord(s[i]) and ones(4)) shl 12 or + (ord(s[i+1]) and ones(6)) shl 6 or + (ord(s[i+2]) and ones(6)) + when doInc: inc(i, 3) + else: + result = UNI_REPL + when doInc: inc(i) + elif ord(s[i]) shr 3 == 0b11110: + if i <= L - 4: + #assert(ord(s[i+1]) shr 6 == 0b10) + #assert(ord(s[i+2]) shr 6 == 0b10) + #assert(ord(s[i+3]) shr 6 == 0b10) + result = (ord(s[i]) and ones(3)) shl 18 or + (ord(s[i+1]) and ones(6)) shl 12 or + (ord(s[i+2]) and ones(6)) shl 6 or + (ord(s[i+3]) and ones(6)) + when doInc: inc(i, 4) + else: + result = UNI_REPL + when doInc: inc(i) + else: + result = 0xFFFD + when doInc: inc(i) + + iterator runes(s: cstring, L: int): int = + var + i = 0 + result: int + while i < L: + fastRuneAt(s, i, L, result, true) + yield result + + proc newWideCString*(size: int): WideCStringObj = + createWide(result, size * 2 + 2) + + proc newWideCString*(source: cstring, L: int): WideCStringObj = + ## Warning:: `source` needs to be preallocated with the length `L` + createWide(result, L * 2 + 2) + var d = 0 + for ch in runes(source, L): + + if ch <= UNI_MAX_BMP: + if ch >= UNI_SUR_HIGH_START and ch <= UNI_SUR_LOW_END: + result[d] = UNI_REPLACEMENT_CHAR + else: + result[d] = cast[Utf16Char](uint16(ch)) + elif ch > UNI_MAX_UTF16: + result[d] = UNI_REPLACEMENT_CHAR + else: + let ch = ch - halfBase + result[d] = cast[Utf16Char](uint16((ch shr halfShift) + UNI_SUR_HIGH_START)) + inc d + result[d] = cast[Utf16Char](uint16((ch and halfMask) + UNI_SUR_LOW_START)) + inc d + result[d] = Utf16Char(0) + + proc newWideCString*(s: cstring): WideCStringObj = + if s.isNil: return nullWide + + result = newWideCString(s, s.len) + + proc newWideCString*(s: string): WideCStringObj = + result = newWideCString(cstring s, s.len) + + proc `$`*(w: WideCString, estimate: int, replacement: int = 0xFFFD): string = + result = newStringOfCap(estimate + estimate shr 2) + + var i = 0 + while w[i].int16 != 0'i16: + var ch = ord(w[i]) + inc i + if ch >= UNI_SUR_HIGH_START and ch <= UNI_SUR_HIGH_END: + # If the 16 bits following the high surrogate are in the source buffer... + let ch2 = ord(w[i]) + + # If it's a low surrogate, convert to UTF32: + if ch2 >= UNI_SUR_LOW_START and ch2 <= UNI_SUR_LOW_END: + ch = (((ch and halfMask) shl halfShift) + (ch2 and halfMask)) + halfBase + inc i + else: + #invalid UTF-16 + ch = replacement + elif ch >= UNI_SUR_LOW_START and ch <= UNI_SUR_LOW_END: + #invalid UTF-16 + ch = replacement + + if ch < 0x80: + result.add chr(ch) + elif ch < 0x800: + result.add chr((ch shr 6) or 0xc0) + result.add chr((ch and 0x3f) or 0x80) + elif ch < 0x10000: + result.add chr((ch shr 12) or 0xe0) + result.add chr(((ch shr 6) and 0x3f) or 0x80) + result.add chr((ch and 0x3f) or 0x80) + elif ch <= 0x10FFFF: + result.add chr((ch shr 18) or 0xf0) + result.add chr(((ch shr 12) and 0x3f) or 0x80) + result.add chr(((ch shr 6) and 0x3f) or 0x80) + result.add chr((ch and 0x3f) or 0x80) + else: + # replacement char(in case user give very large number): + result.add chr(0xFFFD shr 12 or 0b1110_0000) + result.add chr(0xFFFD shr 6 and ones(6) or 0b10_0000_00) + result.add chr(0xFFFD and ones(6) or 0b10_0000_00) + + proc `$`*(s: WideCString): string = + result = s $ 80 + + when defined(nimv2): + proc `$`*(s: WideCStringObj, estimate: int, replacement: int = 0xFFFD): string = + `$`(s.data, estimate, replacement) + + proc `$`*(s: WideCStringObj): string = + $(s.data) + + proc len*(w: WideCStringObj): int {.inline.} = + len(w.data) diff --git a/lib/std/with.nim b/lib/std/with.nim index c7338b4e4..c2eaa4bef 100644 --- a/lib/std/with.nim +++ b/lib/std/with.nim @@ -14,7 +14,7 @@ ## ## **Since:** version 1.2. -import macros, private / underscored_calls +import std/[macros, private / underscored_calls] macro with*(arg: typed; calls: varargs[untyped]): untyped = ## This macro provides `chaining`:idx: of function calls. @@ -35,5 +35,14 @@ macro with*(arg: typed; calls: varargs[untyped]): untyped = -= 5 doAssert a == 43 + # Nesting works for object types too! + var foo = (bar: 1, qux: (baz: 2)) + with foo: + bar = 2 + with qux: + baz = 3 + doAssert foo.bar == 2 + doAssert foo.qux.baz == 3 + result = newNimNode(nnkStmtList, arg) underscoredCalls(result, calls, arg) diff --git a/lib/std/wordwrap.nim b/lib/std/wordwrap.nim index 7dcfc7f59..9333f880b 100644 --- a/lib/std/wordwrap.nim +++ b/lib/std/wordwrap.nim @@ -9,7 +9,7 @@ ## This module contains an algorithm to wordwrap a Unicode string. -import strutils, unicode +import std/[strutils, unicode] proc olen(s: string; start, lastExclusive: int): int = var i = start diff --git a/lib/std/wrapnils.nim b/lib/std/wrapnils.nim index facba85fa..0b75c270e 100644 --- a/lib/std/wrapnils.nim +++ b/lib/std/wrapnils.nim @@ -13,7 +13,7 @@ consider handling indexing operations, eg: doAssert ?.default(seq[int])[3] == default(int) ]# -import macros +import std/macros runnableExamples: type Foo = ref object @@ -60,7 +60,7 @@ proc finalize(n: NimNode, lhs: NimNode, level: int): NimNode = else: result = quote: (let `lhs` = `n`) -proc process(n: NimNode, lhs: NimNode, level: int): NimNode = +proc process(n: NimNode, lhs: NimNode, label: NimNode, level: int): NimNode = var n = n.copyNimTree var it = n let addr2 = bindSym"addr" @@ -78,7 +78,7 @@ proc process(n: NimNode, lhs: NimNode, level: int): NimNode = let okSet = check[1] let kind1 = check[2] let tmp = genSym(nskLet, "tmpCase") - let body = process(objRef, tmp, level + 1) + let body = process(objRef, tmp, label, level + 1) let tmp3 = nnkDerefExpr.newTree(tmp) it[0][0] = tmp3 let dot2 = nnkDotExpr.newTree(@[tmp, dot[1]]) @@ -87,17 +87,17 @@ proc process(n: NimNode, lhs: NimNode, level: int): NimNode = let assgn = finalize(n, lhs, level) result = quote do: `body` - if `tmp3`.`kind1` notin `okSet`: break + if `tmp3`.`kind1` notin `okSet`: break `label` `assgn` break elif it.kind in {nnkHiddenDeref, nnkDerefExpr}: let tmp = genSym(nskLet, "tmp") - let body = process(it[0], tmp, level + 1) + let body = process(it[0], tmp, label, level + 1) it[0] = tmp let assgn = finalize(n, lhs, level) result = quote do: `body` - if `tmp` == nil: break + if `tmp` == nil: break `label` `assgn` break elif it.kind == nnkCall: # consider extending to `nnkCallKinds` @@ -113,15 +113,16 @@ macro `?.`*(a: typed): auto = ## presence of intermediate nil pointers/references, in which case a default ## value is produced. let lhs = genSym(nskVar, "lhs") - let body = process(a, lhs, 0) + let label = genSym(nskLabel, "label") + let body = process(a, lhs, label, 0) result = quote do: var `lhs`: type(`a`) - block: + block `label`: `body` `lhs` # the code below is not needed for `?.` -from options import Option, isSome, get, option, unsafeGet, UnpackDefect +from std/options import Option, isSome, get, option, unsafeGet, UnpackDefect macro `??.`*(a: typed): Option = ## Same as `?.` but returns an `Option`. @@ -144,10 +145,11 @@ macro `??.`*(a: typed): Option = let lhs = genSym(nskVar, "lhs") let lhs2 = genSym(nskVar, "lhs") - let body = process(a, lhs2, 0) + let label = genSym(nskLabel, "label") + let body = process(a, lhs2, label, 0) result = quote do: var `lhs`: Option[type(`a`)] - block: + block `label`: var `lhs2`: type(`a`) `body` `lhs` = option(`lhs2`) |