add typesafe `std/paths`, `std/files`, `std/dirs`, `std/symlinks` (#20582)

* split std/os; add typesafe std/paths
* add more files, dirs, paths
* add documentation
* add testcase
* remove tryRemoveFile
* clean up
* Delete test.nim
* apply changes
* add `add` and fixes

4 files changed, 522 insertions, 0 deletions

diff --git a/lib/std/dirs.nim b/lib/std/dirs.nim
new file mode 100644
index 000000000..e89bfc668
--- /dev/null
+++ b/lib/std/dirs.nim
@@ -0,0 +1,177 @@
+from paths import Path, ReadDirEffect, WriteDirEffect
+
+from std/private/osdirs import dirExists, createDir, existsOrCreateDir, removeDir,
+                               moveDir, walkPattern, walkFiles, walkDirs, walkDir,
+                               walkDirRec, PathComponent
+
+export PathComponent
+
+proc dirExists*(dir: Path): bool {.inline, tags: [ReadDirEffect].} =
+  ## Returns true if the directory `dir` exists. If `dir` is a file, false
+  ## is returned. Follows symlinks.
+  result = dirExists(dir.string)
+
+proc createDir*(dir: Path) {.inline, tags: [WriteDirEffect, ReadDirEffect].} =
+  ## Creates the `directory`:idx: `dir`.
+  ##
+  ## The directory may contain several subdirectories that do not exist yet.
+  ## The full path is created. If this fails, `OSError` is raised.
+  ##
+  ## It does **not** fail if the directory already exists because for
+  ## most usages this does not indicate an error.
+  ##
+  ## See also:
+  ## * `removeDir proc`_
+  ## * `existsOrCreateDir proc`_
+  ## * `copyDir proc`_
+  ## * `copyDirWithPermissions proc`_
+  ## * `moveDir proc`_
+  createDir(dir.string)
+
+proc existsOrCreateDir*(dir: Path): bool {.inline, tags: [WriteDirEffect, ReadDirEffect].} =
+  ## Checks if a `directory`:idx: `dir` exists, and creates it otherwise.
+  ##
+  ## Does not create parent directories (raises `OSError` if parent directories do not exist).
+  ## Returns `true` if the directory already exists, and `false` otherwise.
+  ##
+  ## See also:
+  ## * `removeDir proc`_
+  ## * `createDir proc`_
+  ## * `copyDir proc`_
+  ## * `copyDirWithPermissions proc`_
+  ## * `moveDir proc`_
+  result = existsOrCreateDir(dir.string)
+
+proc removeDir*(dir: Path, checkDir = false
+                ) {.inline, tags: [WriteDirEffect, ReadDirEffect].} =
+  ## Removes the directory `dir` including all subdirectories and files
+  ## in `dir` (recursively).
+  ##
+  ## If this fails, `OSError` is raised. This does not fail if the directory never
+  ## existed in the first place, unless `checkDir` = true.
+  ##
+  ## See also:
+  ## * `removeFile proc`_
+  ## * `existsOrCreateDir proc`_
+  ## * `createDir proc`_
+  ## * `copyDir proc`_
+  ## * `copyDirWithPermissions proc`_
+  ## * `moveDir proc`_
+  removeDir(dir.string, checkDir)
+
+proc moveDir*(source, dest: Path) {.inline, tags: [ReadIOEffect, WriteIOEffect].} =
+  ## Moves a directory from `source` to `dest`.
+  ##
+  ## Symlinks are not followed: if `source` contains symlinks, they themself are
+  ## moved, not their target.
+  ##
+  ## If this fails, `OSError` is raised.
+  ##
+  ## See also:
+  ## * `moveFile proc`_
+  ## * `copyDir proc`_
+  ## * `copyDirWithPermissions proc`_
+  ## * `removeDir proc`_
+  ## * `existsOrCreateDir proc`_
+  ## * `createDir proc`_
+  moveDir(source.string, dest.string)
+
+iterator walkPattern*(pattern: Path): Path {.tags: [ReadDirEffect].} =
+  ## Iterate over all the files and directories that match the `pattern`.
+  ##
+  ## On POSIX this uses the `glob`:idx: call.
+  ## `pattern` is OS dependent, but at least the `"*.ext"`
+  ## notation is supported.
+  ##
+  ## See also:
+  ## * `walkFiles iterator`_
+  ## * `walkDirs iterator`_
+  ## * `walkDir iterator`_
+  ## * `walkDirRec iterator`_
+  for p in walkPattern(pattern.string):
+    yield Path(p)
+
+iterator walkFiles*(pattern: Path): Path {.tags: [ReadDirEffect].} =
+  ## Iterate over all the files that match the `pattern`.
+  ##
+  ## On POSIX this uses the `glob`:idx: call.
+  ## `pattern` is OS dependent, but at least the `"*.ext"`
+  ## notation is supported.
+  ##
+  ## See also:
+  ## * `walkPattern iterator`_
+  ## * `walkDirs iterator`_
+  ## * `walkDir iterator`_
+  ## * `walkDirRec iterator`_
+  for p in walkFiles(pattern.string):
+    yield Path(p)
+
+iterator walkDirs*(pattern: Path): Path {.tags: [ReadDirEffect].} =
+  ## Iterate over all the directories that match the `pattern`.
+  ##
+  ## On POSIX this uses the `glob`:idx: call.
+  ## `pattern` is OS dependent, but at least the `"*.ext"`
+  ## notation is supported.
+  ##
+  ## See also:
+  ## * `walkPattern iterator`_
+  ## * `walkFiles iterator`_
+  ## * `walkDir iterator`_
+  ## * `walkDirRec iterator`_
+  for p in walkDirs(pattern.string):
+    yield Path(p)
+
+iterator walkDir*(dir: Path; relative = false, checkDir = false):
+    tuple[kind: PathComponent, path: Path] {.tags: [ReadDirEffect].} =
+  ## Walks over the directory `dir` and yields for each directory or file in
+  ## `dir`. The component type and full path for each item are returned.
+  ##
+  ## Walking is not recursive. If ``relative`` is true (default: false)
+  ## the resulting path is shortened to be relative to ``dir``.
+  ##
+  ## If `checkDir` is true, `OSError` is raised when `dir`
+  ## doesn't exist.
+  for (k, p) in walkDir(dir.string, relative, checkDir):
+    yield (k, Path(p))
+
+iterator walkDirRec*(dir: Path,
+                     yieldFilter = {pcFile}, followFilter = {pcDir},
+                     relative = false, checkDir = false): Path {.tags: [ReadDirEffect].} =
+  ## Recursively walks over the directory `dir` and yields for each file
+  ## or directory in `dir`.
+  ##
+  ## If ``relative`` is true (default: false) the resulting path is
+  ## shortened to be relative to ``dir``, otherwise the full path is returned.
+  ##
+  ## If `checkDir` is true, `OSError` is raised when `dir`
+  ## doesn't exist.
+  ##
+  ## .. warning:: Modifying the directory structure while the iterator
+  ##   is traversing may result in undefined behavior!
+  ##
+  ## Walking is recursive. `followFilter` controls the behaviour of the iterator:
+  ##
+  ## =====================   =============================================
+  ## yieldFilter             meaning
+  ## =====================   =============================================
+  ## ``pcFile``              yield real files (default)
+  ## ``pcLinkToFile``        yield symbolic links to files
+  ## ``pcDir``               yield real directories
+  ## ``pcLinkToDir``         yield symbolic links to directories
+  ## =====================   =============================================
+  ##
+  ## =====================   =============================================
+  ## followFilter            meaning
+  ## =====================   =============================================
+  ## ``pcDir``               follow real directories (default)
+  ## ``pcLinkToDir``         follow symbolic links to directories
+  ## =====================   =============================================
+  ##
+  ##
+  ## See also:
+  ## * `walkPattern iterator`_
+  ## * `walkFiles iterator`_
+  ## * `walkDirs iterator`_
+  ## * `walkDir iterator`_
+  for p in walkDirRec(dir.string, yieldFilter, followFilter, relative, checkDir):
+    yield Path(p)
diff --git a/lib/std/files.nim b/lib/std/files.nim
new file mode 100644
index 000000000..f7fbd5acd
--- /dev/null
+++ b/lib/std/files.nim
@@ -0,0 +1,45 @@
+from paths import Path, ReadDirEffect, WriteDirEffect
+
+from std/private/osfiles import fileExists, removeFile,
+                                moveFile
+
+
+proc fileExists*(filename: Path): bool {.inline, tags: [ReadDirEffect].} =
+  ## Returns true if `filename` exists and is a regular file or symlink.
+  ##
+  ## Directories, device files, named pipes and sockets return false.
+  result = fileExists(filename.string)
+
+proc removeFile*(file: Path) {.inline, tags: [WriteDirEffect].} =
+  ## Removes the `file`.
+  ##
+  ## If this fails, `OSError` is raised. This does not fail
+  ## if the file never existed in the first place.
+  ##
+  ## On Windows, ignores the read-only attribute.
+  ##
+  ## See also:
+  ## * `removeDir proc`_
+  ## * `copyFile proc`_
+  ## * `copyFileWithPermissions proc`_
+  ## * `moveFile proc`_
+  removeFile(file.string)
+
+proc moveFile*(source, dest: Path) {.inline,
+    tags: [ReadDirEffect, ReadIOEffect, WriteIOEffect].} =
+  ## Moves a file from `source` to `dest`.
+  ##
+  ## Symlinks are not followed: if `source` is a symlink, it is itself moved,
+  ## not its target.
+  ##
+  ## If this fails, `OSError` is raised.
+  ## If `dest` already exists, it will be overwritten.
+  ##
+  ## Can be used to `rename files`:idx:.
+  ##
+  ## See also:
+  ## * `moveDir proc`_
+  ## * `copyFile proc`_
+  ## * `copyFileWithPermissions proc`_
+  ## * `removeFile proc`_
+  moveFile(source.string, dest.string)
diff --git a/lib/std/paths.nim b/lib/std/paths.nim
new file mode 100644
index 000000000..b3cd7de83
--- /dev/null
+++ b/lib/std/paths.nim
@@ -0,0 +1,270 @@
+import std/private/osseps
+export osseps
+
+import pathnorm
+
+from std/private/ospaths2 import  joinPath, splitPath,
+                                  ReadDirEffect, WriteDirEffect,
+                                  isAbsolute, relativePath,
+                                  normalizePathEnd, isRelativeTo, parentDir,
+                                  tailDir, isRootDir, parentDirs, `/../`,
+                                  extractFilename, lastPathPart,
+                                  changeFileExt, addFileExt, cmpPaths, splitFile,
+                                  unixToNativePath, absolutePath, normalizeExe,
+                                  normalizePath
+export ReadDirEffect, WriteDirEffect
+
+type
+  Path* = distinct string
+
+template endsWith(a: string, b: set[char]): bool =
+  a.len > 0 and a[^1] in b
+
+func add(x: var string, tail: string) =
+  var state = 0
+  let trailingSep = tail.endsWith({DirSep, AltSep}) or tail.len == 0 and x.endsWith({DirSep, AltSep})
+  normalizePathEnd(x, trailingSep=false)
+  addNormalizePath(tail, x, state, DirSep)
+  normalizePathEnd(x, trailingSep=trailingSep)
+
+func add*(x: var Path, y: Path) {.borrow.}
+
+func `/`*(head, tail: Path): Path {.inline.} =
+  ## Joins two directory names to one.
+  ##
+  ## returns normalized path concatenation of `head` and `tail`, preserving
+  ## whether or not `tail` has a trailing slash (or, if tail if empty, whether
+  ## head has one).
+  ##
+  ## See also:
+  ## * `splitPath proc`_
+  ## * `uri.combine proc <uri.html#combine,Uri,Uri>`_
+  ## * `uri./ proc <uri.html#/,Uri,string>`_
+  Path(joinPath(head.string, tail.string))
+
+func splitPath*(path: Path): tuple[head, tail: Path] {.inline.} =
+  ## Splits a directory into `(head, tail)` tuple, so that
+  ## ``head / tail == path`` (except for edge cases like "/usr").
+  ##
+  ## See also:
+  ## * `add proc`_
+  ## * `/ proc`_
+  ## * `/../ proc`_
+  ## * `relativePath proc`_
+  let res = splitPath(path.string)
+  result = (Path(res.head), Path(res.tail))
+
+func splitFile*(path: Path): tuple[dir, name: Path, ext: string] {.inline.} =
+  ## Splits a filename into `(dir, name, extension)` tuple.
+  ##
+  ## `dir` does not end in DirSep_ unless it's `/`.
+  ## `extension` includes the leading dot.
+  ##
+  ## If `path` has no extension, `ext` is the empty string.
+  ## If `path` has no directory component, `dir` is the empty string.
+  ## If `path` has no filename component, `name` and `ext` are empty strings.
+  ##
+  ## See also:
+  ## * `extractFilename proc`_
+  ## * `lastPathPart proc`_
+  ## * `changeFileExt proc`_
+  ## * `addFileExt proc`_
+  let res = splitFile(path.string)
+  result = (Path(res.dir), Path(res.name), res.ext)
+
+func isAbsolute*(path: Path): bool {.inline, raises: [].} =
+  ## Checks whether a given `path` is absolute.
+  ##
+  ## On Windows, network paths are considered absolute too.
+  result = isAbsolute(path.string)
+
+proc relativePath*(path, base: Path, sep = DirSep): Path {.inline.} =
+  ## Converts `path` to a path relative to `base`.
+  ##
+  ## The `sep` (default: DirSep_) is used for the path normalizations,
+  ## this can be useful to ensure the relative path only contains `'/'`
+  ## so that it can be used for URL constructions.
+  ##
+  ## On Windows, if a root of `path` and a root of `base` are different,
+  ## returns `path` as is because it is impossible to make a relative path.
+  ## That means an absolute path can be returned.
+  ##
+  ## See also:
+  ## * `splitPath proc`_
+  ## * `parentDir proc`_
+  ## * `tailDir proc`_
+  result = Path(relativePath(path.string, base.string, sep))
+
+proc isRelativeTo*(path: Path, base: Path): bool {.inline.} =
+  ## Returns true if `path` is relative to `base`.
+  result = isRelativeTo(path.string, base.string)
+
+
+func parentDir*(path: Path): Path {.inline.} =
+  ## Returns the parent directory of `path`.
+  ##
+  ## This is similar to ``splitPath(path).head`` when ``path`` doesn't end
+  ## in a dir separator, but also takes care of path normalizations.
+  ## The remainder can be obtained with `lastPathPart(path) proc`_.
+  ##
+  ## See also:
+  ## * `relativePath proc`_
+  ## * `splitPath proc`_
+  ## * `tailDir proc`_
+  ## * `parentDirs iterator`_
+  result = Path(parentDir(path.string))
+
+func tailDir*(path: Path): Path {.inline.} =
+  ## Returns the tail part of `path`.
+  ##
+  ## See also:
+  ## * `relativePath proc`_
+  ## * `splitPath proc`_
+  ## * `parentDir proc`_
+  result = Path(tailDir(path.string))
+
+func isRootDir*(path: Path): bool {.inline.} =
+  ## Checks whether a given `path` is a root directory.
+  result = isRootDir(path.string)
+
+iterator parentDirs*(path: Path, fromRoot=false, inclusive=true): Path =
+  ## Walks over all parent directories of a given `path`.
+  ##
+  ## If `fromRoot` is true (default: false), the traversal will start from
+  ## the file system root directory.
+  ## If `inclusive` is true (default), the original argument will be included
+  ## in the traversal.
+  ##
+  ## Relative paths won't be expanded by this iterator. Instead, it will traverse
+  ## only the directories appearing in the relative path.
+  ##
+  ## See also:
+  ## * `parentDir proc`_
+  ##
+  for p in parentDirs(path.string, fromRoot, inclusive):
+    yield Path(p)
+
+func `/../`*(head, tail: Path): Path {.inline.} =
+  ## The same as ``parentDir(head) / tail``, unless there is no parent
+  ## directory. Then ``head / tail`` is performed instead.
+  ##
+  ## See also:
+  ## * `/ proc`_
+  ## * `parentDir proc`_
+  Path(`/../`(head.string, tail.string))
+
+func extractFilename*(path: Path): Path {.inline.} =
+  ## Extracts the filename of a given `path`.
+  ##
+  ## This is the same as ``name & ext`` from `splitFile(path) proc`_.
+  ##
+  ## See also:
+  ## * `splitFile proc`_
+  ## * `lastPathPart proc`_
+  ## * `changeFileExt proc`_
+  ## * `addFileExt proc`_
+  result = Path(extractFilename(path.string))
+
+func lastPathPart*(path: Path): Path {.inline.} =
+  ## Like `extractFilename proc`_, but ignores
+  ## trailing dir separator; aka: `baseName`:idx: in some other languages.
+  ##
+  ## See also:
+  ## * `splitFile proc`_
+  ## * `extractFilename proc`_
+  ## * `changeFileExt proc`_
+  ## * `addFileExt proc`_
+  result = Path(lastPathPart(path.string))
+
+func changeFileExt*(filename: Path, ext: string): Path {.inline.} =
+  ## Changes the file extension to `ext`.
+  ##
+  ## If the `filename` has no extension, `ext` will be added.
+  ## If `ext` == "" then any extension is removed.
+  ##
+  ## `Ext` should be given without the leading `'.'`, because some
+  ## filesystems may use a different character. (Although I know
+  ## of none such beast.)
+  ##
+  ## See also:
+  ## * `splitFile proc`_
+  ## * `extractFilename proc`_
+  ## * `lastPathPart proc`_
+  ## * `addFileExt proc`_
+  result = Path(changeFileExt(filename.string, ext))
+
+func addFileExt*(filename: Path, ext: string): Path {.inline.} =
+  ## Adds the file extension `ext` to `filename`, unless
+  ## `filename` already has an extension.
+  ##
+  ## `Ext` should be given without the leading `'.'`, because some
+  ## filesystems may use a different character.
+  ## (Although I know of none such beast.)
+  ##
+  ## See also:
+  ## * `splitFile proc`_
+  ## * `extractFilename proc`_
+  ## * `lastPathPart proc`_
+  ## * `changeFileExt proc`_
+  result = Path(addFileExt(filename.string, ext))
+
+func cmpPaths*(pathA, pathB: Path): int {.inline.} =
+  ## Compares two paths.
+  ##
+  ## On a case-sensitive filesystem this is done
+  ## case-sensitively otherwise case-insensitively. Returns:
+  ##
+  ## | 0 if pathA == pathB
+  ## | < 0 if pathA < pathB
+  ## | > 0 if pathA > pathB
+  result = cmpPaths(pathA.string, pathB.string)
+
+func unixToNativePath*(path: Path, drive=Path("")): Path {.inline.} =
+  ## Converts an UNIX-like path to a native one.
+  ##
+  ## On an UNIX system this does nothing. Else it converts
+  ## `'/'`, `'.'`, `'..'` to the appropriate things.
+  ##
+  ## On systems with a concept of "drives", `drive` is used to determine
+  ## which drive label to use during absolute path conversion.
+  ## `drive` defaults to the drive of the current working directory, and is
+  ## ignored on systems that do not have a concept of "drives".
+  result = Path(unixToNativePath(path.string, drive.string))
+
+proc getCurrentDir*(): Path {.inline, tags: [].} =
+  ## Returns the `current working directory`:idx: i.e. where the built
+  ## binary is run.
+  ##
+  ## So the path returned by this proc is determined at run time.
+  ##
+  ## See also:
+  ## * `getHomeDir proc`_
+  ## * `getConfigDir proc`_
+  ## * `getTempDir proc`_
+  ## * `setCurrentDir proc`_
+  ## * `currentSourcePath template <system.html#currentSourcePath.t>`_
+  ## * `getProjectPath proc <macros.html#getProjectPath>`_
+  result = Path(ospaths2.getCurrentDir())
+
+proc setCurrentDir*(newDir: Path) {.inline, tags: [].} =
+  ## Sets the `current working directory`:idx:; `OSError`
+  ## is raised if `newDir` cannot been set.
+  ##
+  ## See also:
+  ## * `getCurrentDir proc`_
+  ospaths2.setCurrentDir(newDir.string)
+
+proc normalizeExe*(file: var Path) {.borrow.}
+
+proc normalizePath*(path: var Path) {.borrow.}
+
+proc normalizePathEnd*(path: var Path, trailingSep = false) {.borrow.}
+
+proc absolutePath*(path: Path, root = getCurrentDir()): Path =
+  ## Returns the absolute path of `path`, rooted at `root` (which must be absolute;
+  ## default: current directory).
+  ## If `path` is absolute, return it, ignoring `root`.
+  ##
+  ## See also:
+  ## * `normalizePath proc`_
+  result = Path(absolutePath(path.string, root.string))
diff --git a/lib/std/symlinks.nim b/lib/std/symlinks.nim
new file mode 100644
index 000000000..cb469d8c3
--- /dev/null
+++ b/lib/std/symlinks.nim
@@ -0,0 +1,30 @@
+from paths import Path, ReadDirEffect
+
+from std/private/ossymlinks import symlinkExists, createSymlink, expandSymlink
+
+
+proc symlinkExists*(link: Path): bool {.inline, tags: [ReadDirEffect].} =
+  ## Returns true if the symlink `link` exists. Will return true
+  ## regardless of whether the link points to a directory or file.
+  result = symlinkExists(link.string)
+
+proc createSymlink*(src, dest: Path) {.inline.} =
+  ## Create a symbolic link at `dest` which points to the item specified
+  ## by `src`. On most operating systems, will fail if a link already exists.
+  ##
+  ## .. warning:: Some OS's (such as Microsoft Windows) restrict the creation
+  ##   of symlinks to root users (administrators) or users with developer mode enabled.
+  ##
+  ## See also:
+  ## * `createHardlink proc`_
+  ## * `expandSymlink proc`_
+  createSymlink(src.string, dest.string)
+
+proc expandSymlink*(symlinkPath: Path): Path {.inline.} =
+  ## Returns a string representing the path to which the symbolic link points.
+  ##
+  ## On Windows this is a noop, `symlinkPath` is simply returned.
+  ##
+  ## See also:
+  ## * `createSymlink proc`_
+  result = Path(expandSymlink(symlinkPath.string))