move io out of system (#19442)

* move io out of system

* fix tests

* fix tests

* next step

* rename to syncio

* rename

* fix nimscript

* comma

* fix

* fix parts of errors

* good for now

* fix test

1 files changed, 962 insertions, 0 deletions

diff --git a/lib/std/syncio.nim b/lib/std/syncio.nim
new file mode 100644
index 000000000..eab96254a
--- /dev/null
+++ b/lib/std/syncio.nim
@@ -0,0 +1,962 @@
+#
+#
+#            Nim's Runtime Library
+#        (c) Copyright 2022 Nim contributors
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## This module implements various synchronized I/O operations.
+
+include system/inclrtl
+import std/private/since
+import system/formatfloat
+
+# ----------------- IO Part ------------------------------------------------
+type
+  CFile {.importc: "FILE", header: "<stdio.h>",
+          incompleteStruct.} = object
+  File* = ptr CFile ## The type representing a file handle.
+
+  FileMode* = enum       ## The file mode when opening a file.
+    fmRead,              ## Open the file for read access only.
+                         ## If the file does not exist, it will not
+                         ## be created.
+    fmWrite,             ## Open the file for write access only.
+                         ## If the file does not exist, it will be
+                         ## created. Existing files will be cleared!
+    fmReadWrite,         ## Open the file for read and write access.
+                         ## If the file does not exist, it will be
+                         ## created. Existing files will be cleared!
+    fmReadWriteExisting, ## Open the file for read and write access.
+                         ## If the file does not exist, it will not be
+                         ## created. The existing file will not be cleared.
+    fmAppend             ## Open the file for writing only; append data
+                         ## at the end. If the file does not exist, it
+                         ## will be created.
+
+  FileHandle* = cint ## type that represents an OS file handle; this is
+                      ## useful for low-level file access
+
+# text file handling:
+when not defined(nimscript) and not defined(js):
+  # duplicated between io and ansi_c
+  const stdioUsesMacros = (defined(osx) or defined(freebsd) or defined(
+      dragonfly)) and not defined(emscripten)
+  const stderrName = when stdioUsesMacros: "__stderrp" else: "stderr"
+  const stdoutName = when stdioUsesMacros: "__stdoutp" else: "stdout"
+  const stdinName = when stdioUsesMacros: "__stdinp" else: "stdin"
+
+  var
+    stdin* {.importc: stdinName, header: "<stdio.h>".}: File
+      ## The standard input stream.
+    stdout* {.importc: stdoutName, header: "<stdio.h>".}: File
+      ## The standard output stream.
+    stderr* {.importc: stderrName, header: "<stdio.h>".}: File
+      ## The standard error stream.
+
+when defined(useStdoutAsStdmsg):
+  template stdmsg*: File = stdout
+else:
+  template stdmsg*: File = stderr
+    ## Template which expands to either stdout or stderr depending on
+    ## `useStdoutAsStdmsg` compile-time switch.
+
+when defined(windows):
+  proc c_fileno(f: File): cint {.
+    importc: "_fileno", header: "<stdio.h>".}
+else:
+  proc c_fileno(f: File): cint {.
+    importc: "fileno", header: "<fcntl.h>".}
+
+when defined(windows):
+  proc c_fdopen(filehandle: cint, mode: cstring): File {.
+    importc: "_fdopen", header: "<stdio.h>".}
+else:
+  proc c_fdopen(filehandle: cint, mode: cstring): File {.
+    importc: "fdopen", header: "<stdio.h>".}
+proc c_fputs(c: cstring, f: File): cint {.
+  importc: "fputs", header: "<stdio.h>", tags: [WriteIOEffect].}
+proc c_fgets(c: cstring, n: cint, f: File): cstring {.
+  importc: "fgets", header: "<stdio.h>", tags: [ReadIOEffect].}
+proc c_fgetc(stream: File): cint {.
+  importc: "fgetc", header: "<stdio.h>", tags: [].}
+proc c_ungetc(c: cint, f: File): cint {.
+  importc: "ungetc", header: "<stdio.h>", tags: [].}
+proc c_putc(c: cint, stream: File): cint {.
+  importc: "putc", header: "<stdio.h>", tags: [WriteIOEffect].}
+proc c_fflush(f: File): cint {.
+  importc: "fflush", header: "<stdio.h>".}
+proc c_fclose(f: File): cint {.
+  importc: "fclose", header: "<stdio.h>".}
+proc c_clearerr(f: File) {.
+  importc: "clearerr", header: "<stdio.h>".}
+proc c_feof(f: File): cint {.
+  importc: "feof", header: "<stdio.h>".}
+
+when not declared(c_fwrite):
+  proc c_fwrite(buf: pointer, size, n: csize_t, f: File): cint {.
+    importc: "fwrite", header: "<stdio.h>".}
+
+# C routine that is used here:
+proc c_fread(buf: pointer, size, n: csize_t, f: File): csize_t {.
+  importc: "fread", header: "<stdio.h>", tags: [ReadIOEffect].}
+when defined(windows):
+  when not defined(amd64):
+    proc c_fseek(f: File, offset: int64, whence: cint): cint {.
+      importc: "fseek", header: "<stdio.h>", tags: [].}
+    proc c_ftell(f: File): int64 {.
+      importc: "ftell", header: "<stdio.h>", tags: [].}
+  else:
+    proc c_fseek(f: File, offset: int64, whence: cint): cint {.
+      importc: "_fseeki64", header: "<stdio.h>", tags: [].}
+    when defined(tcc):
+      proc c_fsetpos(f: File, pos: var int64): int32 {.
+        importc: "fsetpos", header: "<stdio.h>", tags: [].}
+      proc c_fgetpos(f: File, pos: var int64): int32 {.
+        importc: "fgetpos", header: "<stdio.h>", tags: [].}
+      proc c_telli64(f: cint): int64 {.
+        importc: "_telli64", header: "<io.h>", tags: [].}
+      proc c_ftell(f: File): int64 =
+        # Taken from https://pt.osdn.net/projects/mingw/scm/git/mingw-org-wsl/blobs/5.4-trunk/mingwrt/mingwex/stdio/ftelli64.c
+        result = -1'i64
+        var pos: int64
+        if c_fgetpos(f, pos) == 0 and c_fsetpos(f, pos) == 0:
+          result = c_telli64(c_fileno(f))
+    else:
+      proc c_ftell(f: File): int64 {.
+        importc: "_ftelli64", header: "<stdio.h>", tags: [].}
+else:
+  proc c_fseek(f: File, offset: int64, whence: cint): cint {.
+    importc: "fseeko", header: "<stdio.h>", tags: [].}
+  proc c_ftell(f: File): int64 {.
+    importc: "ftello", header: "<stdio.h>", tags: [].}
+proc c_ferror(f: File): cint {.
+  importc: "ferror", header: "<stdio.h>", tags: [].}
+proc c_setvbuf(f: File, buf: pointer, mode: cint, size: csize_t): cint {.
+  importc: "setvbuf", header: "<stdio.h>", tags: [].}
+
+proc c_fprintf(f: File, frmt: cstring): cint {.
+  importc: "fprintf", header: "<stdio.h>", varargs, discardable.}
+proc c_fputc(c: char, f: File): cint {.
+  importc: "fputc", header: "<stdio.h>".}
+
+# When running nim in android app, stdout goes nowhere, so echo gets ignored
+# To redirect echo to the android logcat, use -d:androidNDK
+when defined(androidNDK):
+  const ANDROID_LOG_VERBOSE = 2.cint
+  proc android_log_print(prio: cint, tag: cstring, fmt: cstring): cint
+    {.importc: "__android_log_print", header: "<android/log.h>", varargs, discardable.}
+
+template sysFatal(exc, msg) =
+  raise newException(exc, msg)
+
+proc raiseEIO(msg: string) {.noinline, noreturn.} =
+  sysFatal(IOError, msg)
+
+proc raiseEOF() {.noinline, noreturn.} =
+  sysFatal(EOFError, "EOF reached")
+
+proc strerror(errnum: cint): cstring {.importc, header: "<string.h>".}
+
+when not defined(nimscript):
+  var
+    errno {.importc, header: "<errno.h>".}: cint ## error variable
+    EINTR {.importc: "EINTR", header: "<errno.h>".}: cint
+
+proc checkErr(f: File) =
+  when not defined(nimscript):
+    if c_ferror(f) != 0:
+      let msg = "errno: " & $errno & " `" & $strerror(errno) & "`"
+      c_clearerr(f)
+      raiseEIO(msg)
+  else:
+    # shouldn't happen
+    quit(1)
+
+{.push stackTrace: off, profiler: off.}
+proc readBuffer*(f: File, buffer: pointer, len: Natural): int {.
+  tags: [ReadIOEffect], benign.} =
+  ## Reads `len` bytes into the buffer pointed to by `buffer`. Returns
+  ## the actual number of bytes that have been read which may be less than
+  ## `len` (if not as many bytes are remaining), but not greater.
+  result = cast[int](c_fread(buffer, 1, cast[csize_t](len), f))
+  if result != len: checkErr(f)
+
+proc readBytes*(f: File, a: var openArray[int8|uint8], start,
+    len: Natural): int {.
+  tags: [ReadIOEffect], benign.} =
+  ## Reads `len` bytes into the buffer `a` starting at `a[start]`. Returns
+  ## the actual number of bytes that have been read which may be less than
+  ## `len` (if not as many bytes are remaining), but not greater.
+  result = readBuffer(f, addr(a[start]), len)
+
+proc readChars*(f: File, a: var openArray[char]): int {.tags: [ReadIOEffect], benign.} =
+  ## Reads up to `a.len` bytes into the buffer `a`. Returns
+  ## the actual number of bytes that have been read which may be less than
+  ## `a.len` (if not as many bytes are remaining), but not greater.
+  result = readBuffer(f, addr(a[0]), a.len)
+
+proc readChars*(f: File, a: var openArray[char], start, len: Natural): int {.
+  tags: [ReadIOEffect], benign, deprecated:
+    "use other `readChars` overload, possibly via: readChars(toOpenArray(buf, start, len-1))".} =
+  ## Reads `len` bytes into the buffer `a` starting at `a[start]`. Returns
+  ## the actual number of bytes that have been read which may be less than
+  ## `len` (if not as many bytes are remaining), but not greater.
+  if (start + len) > len(a):
+    raiseEIO("buffer overflow: (start+len) > length of openarray buffer")
+  result = readBuffer(f, addr(a[start]), len)
+
+proc write*(f: File, c: cstring) {.tags: [WriteIOEffect], benign.} =
+  ## Writes a value to the file `f`. May throw an IO exception.
+  discard c_fputs(c, f)
+  checkErr(f)
+
+proc writeBuffer*(f: File, buffer: pointer, len: Natural): int {.
+  tags: [WriteIOEffect], benign.} =
+  ## Writes the bytes of buffer pointed to by the parameter `buffer` to the
+  ## file `f`. Returns the number of actual written bytes, which may be less
+  ## than `len` in case of an error.
+  result = cast[int](c_fwrite(buffer, 1, cast[csize_t](len), f))
+  checkErr(f)
+
+proc writeBytes*(f: File, a: openArray[int8|uint8], start, len: Natural): int {.
+  tags: [WriteIOEffect], benign.} =
+  ## Writes the bytes of `a[start..start+len-1]` to the file `f`. Returns
+  ## the number of actual written bytes, which may be less than `len` in case
+  ## of an error.
+  var x = cast[ptr UncheckedArray[int8]](a)
+  result = writeBuffer(f, addr(x[int(start)]), len)
+
+proc writeChars*(f: File, a: openArray[char], start, len: Natural): int {.
+  tags: [WriteIOEffect], benign.} =
+  ## Writes the bytes of `a[start..start+len-1]` to the file `f`. Returns
+  ## the number of actual written bytes, which may be less than `len` in case
+  ## of an error.
+  var x = cast[ptr UncheckedArray[int8]](a)
+  result = writeBuffer(f, addr(x[int(start)]), len)
+
+when defined(windows):
+  proc writeWindows(f: File; s: string; doRaise = false) =
+    # Don't ask why but the 'printf' family of function is the only thing
+    # that writes utf-8 strings reliably on Windows. At least on my Win 10
+    # machine. We also enable `setConsoleOutputCP(65001)` now by default.
+    # But we cannot call printf directly as the string might contain \0.
+    # So we have to loop over all the sections separated by potential \0s.
+    var i = c_fprintf(f, "%s", s)
+    while i < s.len:
+      if s[i] == '\0':
+        let w = c_fputc('\0', f)
+        if w != 0:
+          if doRaise: raiseEIO("cannot write string to file")
+          break
+        inc i
+      else:
+        let w = c_fprintf(f, "%s", unsafeAddr s[i])
+        if w <= 0:
+          if doRaise: raiseEIO("cannot write string to file")
+          break
+        inc i, w
+
+proc write*(f: File, s: string) {.tags: [WriteIOEffect], benign.} =
+  when defined(windows):
+    writeWindows(f, s, doRaise = true)
+  else:
+    if writeBuffer(f, cstring(s), s.len) != s.len:
+      raiseEIO("cannot write string to file")
+{.pop.}
+
+when defined(nimscript):
+  when defined(windows):
+    const
+      IOFBF = cint(0)
+      IONBF = cint(4)
+  else:
+    # On all systems I could find, including Linux, Mac OS X, and the BSDs
+    const
+      IOFBF = cint(0)
+      IONBF = cint(2)
+else:
+  var
+    IOFBF {.importc: "_IOFBF", nodecl.}: cint
+    IONBF {.importc: "_IONBF", nodecl.}: cint
+
+const SupportIoctlInheritCtl = (defined(linux) or defined(bsd)) and
+                              not defined(nimscript)
+when SupportIoctlInheritCtl:
+  var
+    FIOCLEX {.importc, header: "<sys/ioctl.h>".}: cint
+    FIONCLEX {.importc, header: "<sys/ioctl.h>".}: cint
+
+  proc c_ioctl(fd: cint, request: cint): cint {.
+    importc: "ioctl", header: "<sys/ioctl.h>", varargs.}
+elif defined(posix) and not defined(lwip) and not defined(nimscript):
+  var
+    F_GETFD {.importc, header: "<fcntl.h>".}: cint
+    F_SETFD {.importc, header: "<fcntl.h>".}: cint
+    FD_CLOEXEC {.importc, header: "<fcntl.h>".}: cint
+
+  proc c_fcntl(fd: cint, cmd: cint): cint {.
+    importc: "fcntl", header: "<fcntl.h>", varargs.}
+elif defined(windows):
+  type
+    WinDWORD = culong
+    WinBOOL = cint
+
+  const HANDLE_FLAG_INHERIT = 1.WinDWORD
+
+  proc getOsfhandle(fd: cint): int {.
+    importc: "_get_osfhandle", header: "<io.h>".}
+
+  type
+    IoHandle = distinct pointer
+      ## Windows' HANDLE type. Defined as an untyped pointer but is **not**
+      ## one. Named like this to avoid collision with other `system` modules.
+
+  proc setHandleInformation(hObject: IoHandle, dwMask, dwFlags: WinDWORD):
+                           WinBOOL {.stdcall, dynlib: "kernel32",
+                                  importc: "SetHandleInformation".}
+
+const
+  BufSize = 4000
+
+proc close*(f: File) {.tags: [], gcsafe.} =
+  ## Closes the file.
+  if not f.isNil:
+    discard c_fclose(f)
+
+proc readChar*(f: File): char {.tags: [ReadIOEffect].} =
+  ## Reads a single character from the stream `f`. Should not be used in
+  ## performance sensitive code.
+  let x = c_fgetc(f)
+  if x < 0:
+    checkErr(f)
+    raiseEOF()
+  result = char(x)
+
+proc flushFile*(f: File) {.tags: [WriteIOEffect].} =
+  ## Flushes `f`'s buffer.
+  discard c_fflush(f)
+
+proc getFileHandle*(f: File): FileHandle =
+  ## Returns the file handle of the file `f`. This is only useful for
+  ## platform specific programming.
+  ## Note that on Windows this doesn't return the Windows-specific handle,
+  ## but the C library's notion of a handle, whatever that means.
+  ## Use `getOsFileHandle` instead.
+  c_fileno(f)
+
+proc getOsFileHandle*(f: File): FileHandle =
+  ## Returns the OS file handle of the file `f`. This is only useful for
+  ## platform specific programming.
+  when defined(windows):
+    result = FileHandle getOsfhandle(cint getFileHandle(f))
+  else:
+    result = c_fileno(f)
+
+when defined(nimdoc) or (defined(posix) and not defined(nimscript)) or defined(windows):
+  proc setInheritable*(f: FileHandle, inheritable: bool): bool =
+    ## control whether a file handle can be inherited by child processes. Returns
+    ## `true` on success. This requires the OS file handle, which can be
+    ## retrieved via `getOsFileHandle <#getOsFileHandle,File>`_.
+    ##
+    ## This procedure is not guaranteed to be available for all platforms. Test for
+    ## availability with `declared() <system.html#declared,untyped>`.
+    when SupportIoctlInheritCtl:
+      result = c_ioctl(f, if inheritable: FIONCLEX else: FIOCLEX) != -1
+    elif defined(freertos) or defined(zephyr):
+      result = true
+    elif defined(posix):
+      var flags = c_fcntl(f, F_GETFD)
+      if flags == -1:
+        return false
+      flags = if inheritable: flags and not FD_CLOEXEC else: flags or FD_CLOEXEC
+      result = c_fcntl(f, F_SETFD, flags) != -1
+    else:
+      result = setHandleInformation(cast[IoHandle](f), HANDLE_FLAG_INHERIT,
+                                    inheritable.WinDWORD) != 0
+
+proc readLine*(f: File, line: var string): bool {.tags: [ReadIOEffect],
+              benign.} =
+  ## Reads a line of text from the file `f` into `line`. May throw an IO
+  ## exception.
+  ## A line of text may be delimited by `LF` or `CRLF`. The newline
+  ## character(s) are not part of the returned string. Returns `false`
+  ## if the end of the file has been reached, `true` otherwise. If
+  ## `false` is returned `line` contains no new data.
+  proc c_memchr(s: pointer, c: cint, n: csize_t): pointer {.
+    importc: "memchr", header: "<string.h>".}
+
+  when defined(windows) and not defined(useWinAnsi):
+    proc readConsole(hConsoleInput: FileHandle, lpBuffer: pointer,
+                     nNumberOfCharsToRead: int32,
+                     lpNumberOfCharsRead: ptr int32,
+                     pInputControl: pointer): int32 {.
+      importc: "ReadConsoleW", stdcall, dynlib: "kernel32".}
+
+    proc getLastError(): int32 {.
+      importc: "GetLastError", stdcall, dynlib: "kernel32", sideEffect.}
+
+    proc formatMessageW(dwFlags: int32, lpSource: pointer,
+                        dwMessageId, dwLanguageId: int32,
+                        lpBuffer: pointer, nSize: int32,
+                        arguments: pointer): int32 {.
+      importc: "FormatMessageW", stdcall, dynlib: "kernel32".}
+
+    proc localFree(p: pointer) {.
+      importc: "LocalFree", stdcall, dynlib: "kernel32".}
+
+    proc isatty(f: File): bool =
+      when defined(posix):
+        proc isatty(fildes: FileHandle): cint {.
+          importc: "isatty", header: "<unistd.h>".}
+      else:
+        proc isatty(fildes: FileHandle): cint {.
+          importc: "_isatty", header: "<io.h>".}
+      result = isatty(getFileHandle(f)) != 0'i32
+
+    # this implies the file is open
+    if f.isatty:
+      const numberOfCharsToRead = 2048
+      var numberOfCharsRead = 0'i32
+      var buffer = newWideCString("", numberOfCharsToRead)
+      if readConsole(getOsFileHandle(f), addr(buffer[0]),
+        numberOfCharsToRead, addr(numberOfCharsRead), nil) == 0:
+        var error = getLastError()
+        var errorMsg: string
+        var msgbuf: WideCString
+        if formatMessageW(0x00000100 or 0x00001000 or 0x00000200,
+                        nil, error, 0, addr(msgbuf), 0, nil) != 0'i32:
+          errorMsg = $msgbuf
+          if msgbuf != nil:
+            localFree(cast[pointer](msgbuf))
+        raiseEIO("error: " & $error & " `" & errorMsg & "`")
+      # input always ends with "\r\n"
+      numberOfCharsRead -= 2
+      # handle Ctrl+Z as EOF
+      for i in 0..<numberOfCharsRead:
+        if buffer[i].uint16 == 26: #Ctrl+Z
+          close(f) #has the same effect as setting EOF
+          if i == 0:
+            line = ""
+            return false
+          numberOfCharsRead = i
+          break
+      buffer[numberOfCharsRead] = 0.Utf16Char
+      when defined(nimv2):
+        line = $toWideCString(buffer)
+      else:
+        line = $buffer
+      return(true)
+
+  var pos = 0
+
+  # Use the currently reserved space for a first try
+  var sp = max(line.len, 80)
+  line.setLen(sp)
+
+  while true:
+    # memset to \L so that we can tell how far fgets wrote, even on EOF, where
+    # fgets doesn't append an \L
+    for i in 0..<sp: line[pos+i] = '\L'
+
+    var fgetsSuccess: bool
+    while true:
+      # fixes #9634; this pattern may need to be abstracted as a template if reused;
+      # likely other io procs need this for correctness.
+      fgetsSuccess = c_fgets(addr line[pos], sp.cint, f) != nil
+      if fgetsSuccess: break
+      when not defined(nimscript):
+        if errno == EINTR:
+          errno = 0
+          c_clearerr(f)
+          continue
+      checkErr(f)
+      break
+
+    let m = c_memchr(addr line[pos], '\L'.ord, cast[csize_t](sp))
+    if m != nil:
+      # \l found: Could be our own or the one by fgets, in any case, we're done
+      var last = cast[ByteAddress](m) - cast[ByteAddress](addr line[0])
+      if last > 0 and line[last-1] == '\c':
+        line.setLen(last-1)
+        return last > 1 or fgetsSuccess
+        # We have to distinguish between two possible cases:
+        # \0\l\0 => line ending in a null character.
+        # \0\l\l => last line without newline, null was put there by fgets.
+      elif last > 0 and line[last-1] == '\0':
+        if last < pos + sp - 1 and line[last+1] != '\0':
+          dec last
+      line.setLen(last)
+      return last > 0 or fgetsSuccess
+    else:
+      # fgets will have inserted a null byte at the end of the string.
+      dec sp
+    # No \l found: Increase buffer and read more
+    inc pos, sp
+    sp = 128 # read in 128 bytes at a time
+    line.setLen(pos+sp)
+
+proc readLine*(f: File): string {.tags: [ReadIOEffect], benign.} =
+  ## Reads a line of text from the file `f`. May throw an IO exception.
+  ## A line of text may be delimited by `LF` or `CRLF`. The newline
+  ## character(s) are not part of the returned string.
+  result = newStringOfCap(80)
+  if not readLine(f, result): raiseEOF()
+
+proc write*(f: File, i: int) {.tags: [WriteIOEffect], benign.} =
+  when sizeof(int) == 8:
+    if c_fprintf(f, "%lld", i) < 0: checkErr(f)
+  else:
+    if c_fprintf(f, "%ld", i) < 0: checkErr(f)
+
+proc write*(f: File, i: BiggestInt) {.tags: [WriteIOEffect], benign.} =
+  when sizeof(BiggestInt) == 8:
+    if c_fprintf(f, "%lld", i) < 0: checkErr(f)
+  else:
+    if c_fprintf(f, "%ld", i) < 0: checkErr(f)
+
+proc write*(f: File, b: bool) {.tags: [WriteIOEffect], benign.} =
+  if b: write(f, "true")
+  else: write(f, "false")
+
+proc write*(f: File, r: float32) {.tags: [WriteIOEffect], benign.} =
+  var buffer {.noinit.}: array[65, char]
+  discard writeFloatToBuffer(buffer, r)
+  if c_fprintf(f, "%s", buffer[0].addr) < 0: checkErr(f)
+
+proc write*(f: File, r: BiggestFloat) {.tags: [WriteIOEffect], benign.} =
+  var buffer {.noinit.}: array[65, char]
+  discard writeFloatToBuffer(buffer, r)
+  if c_fprintf(f, "%s", buffer[0].addr) < 0: checkErr(f)
+
+proc write*(f: File, c: char) {.tags: [WriteIOEffect], benign.} =
+  discard c_putc(cint(c), f)
+
+proc write*(f: File, a: varargs[string, `$`]) {.tags: [WriteIOEffect], benign.} =
+  for x in items(a): write(f, x)
+
+proc readAllBuffer(file: File): string =
+  # This proc is for File we want to read but don't know how many
+  # bytes we need to read before the buffer is empty.
+  result = ""
+  var buffer = newString(BufSize)
+  while true:
+    var bytesRead = readBuffer(file, addr(buffer[0]), BufSize)
+    if bytesRead == BufSize:
+      result.add(buffer)
+    else:
+      buffer.setLen(bytesRead)
+      result.add(buffer)
+      break
+
+proc rawFileSize(file: File): int64 =
+  # this does not raise an error opposed to `getFileSize`
+  var oldPos = c_ftell(file)
+  discard c_fseek(file, 0, 2) # seek the end of the file
+  result = c_ftell(file)
+  discard c_fseek(file, oldPos, 0)
+
+proc endOfFile*(f: File): bool {.tags: [], benign.} =
+  ## Returns true if `f` is at the end.
+  var c = c_fgetc(f)
+  discard c_ungetc(c, f)
+  return c < 0'i32
+  #result = c_feof(f) != 0
+
+proc readAllFile(file: File, len: int64): string =
+  # We acquire the filesize beforehand and hope it doesn't change.
+  # Speeds things up.
+  result = newString(len)
+  let bytes = readBuffer(file, addr(result[0]), len)
+  if endOfFile(file):
+    if bytes < len:
+      result.setLen(bytes)
+  else:
+    # We read all the bytes but did not reach the EOF
+    # Try to read it as a buffer
+    result.add(readAllBuffer(file))
+
+proc readAllFile(file: File): string =
+  var len = rawFileSize(file)
+  result = readAllFile(file, len)
+
+proc readAll*(file: File): string {.tags: [ReadIOEffect], benign.} =
+  ## Reads all data from the stream `file`.
+  ##
+  ## Raises an IO exception in case of an error. It is an error if the
+  ## current file position is not at the beginning of the file.
+
+  # Separate handling needed because we need to buffer when we
+  # don't know the overall length of the File.
+  when declared(stdin):
+    let len = if file != stdin: rawFileSize(file) else: -1
+  else:
+    let len = rawFileSize(file)
+  if len > 0:
+    result = readAllFile(file, len)
+  else:
+    result = readAllBuffer(file)
+
+proc writeLine*[Ty](f: File, x: varargs[Ty, `$`]) {.inline,
+                          tags: [WriteIOEffect], benign.} =
+  ## Writes the values `x` to `f` and then writes "\\n".
+  ## May throw an IO exception.
+  for i in items(x):
+    write(f, i)
+  write(f, "\n")
+
+# interface to the C procs:
+
+when defined(windows) and not defined(useWinAnsi):
+  when defined(cpp):
+    proc wfopen(filename, mode: WideCString): pointer {.
+      importcpp: "_wfopen((const wchar_t*)#, (const wchar_t*)#)", nodecl.}
+    proc wfreopen(filename, mode: WideCString, stream: File): File {.
+      importcpp: "_wfreopen((const wchar_t*)#, (const wchar_t*)#, #)", nodecl.}
+  else:
+    proc wfopen(filename, mode: WideCString): pointer {.
+      importc: "_wfopen", nodecl.}
+    proc wfreopen(filename, mode: WideCString, stream: File): File {.
+      importc: "_wfreopen", nodecl.}
+
+  proc fopen(filename, mode: cstring): pointer =
+    var f = newWideCString(filename)
+    var m = newWideCString(mode)
+    result = wfopen(f, m)
+
+  proc freopen(filename, mode: cstring, stream: File): File =
+    var f = newWideCString(filename)
+    var m = newWideCString(mode)
+    result = wfreopen(f, m, stream)
+
+else:
+  proc fopen(filename, mode: cstring): pointer {.importc: "fopen", nodecl.}
+  proc freopen(filename, mode: cstring, stream: File): File {.
+    importc: "freopen", nodecl.}
+
+const
+  NoInheritFlag =
+    # Platform specific flag for creating a File without inheritance.
+    when not defined(nimInheritHandles):
+      when defined(windows):
+        "N"
+      elif defined(linux) or defined(bsd):
+        "e"
+      else:
+        ""
+    else:
+      ""
+  FormatOpen: array[FileMode, cstring] = [
+    cstring("rb" & NoInheritFlag), "wb" & NoInheritFlag, "w+b" & NoInheritFlag,
+    "r+b" & NoInheritFlag, "ab" & NoInheritFlag
+  ]
+    #"rt", "wt", "w+t", "r+t", "at"
+    # we always use binary here as for Nim the OS line ending
+    # should not be translated.
+
+when defined(posix) and not defined(nimscript):
+  when defined(linux) and defined(amd64):
+    type
+      Mode {.importc: "mode_t", header: "<sys/types.h>".} = cint
+
+      # fillers ensure correct size & offsets
+      Stat {.importc: "struct stat",
+              header: "<sys/stat.h>", final, pure.} = object ## struct stat
+        filler_1: array[24, char]
+        st_mode: Mode ## Mode of file
+        filler_2: array[144 - 24 - 4, char]
+
+    proc modeIsDir(m: Mode): bool =
+      ## Test for a directory.
+      (m and 0o170000) == 0o40000
+
+  else:
+    type
+      Mode {.importc: "mode_t", header: "<sys/types.h>".} = cint
+
+      Stat {.importc: "struct stat",
+               header: "<sys/stat.h>", final, pure.} = object ## struct stat
+        st_mode: Mode ## Mode of file
+
+    proc modeIsDir(m: Mode): bool {.importc: "S_ISDIR", header: "<sys/stat.h>".}
+      ## Test for a directory.
+
+  proc c_fstat(a1: cint, a2: var Stat): cint {.
+    importc: "fstat", header: "<sys/stat.h>".}
+
+
+proc open*(f: var File, filename: string,
+          mode: FileMode = fmRead,
+          bufSize: int = -1): bool {.tags: [], raises: [], benign.} =
+  ## Opens a file named `filename` with given `mode`.
+  ##
+  ## Default mode is readonly. Returns true if the file could be opened.
+  ## This throws no exception if the file could not be opened.
+  ##
+  ## The file handle associated with the resulting `File` is not inheritable.
+  var p = fopen(filename.cstring, FormatOpen[mode])
+  if p != nil:
+    var f2 = cast[File](p)
+    when defined(posix) and not defined(nimscript):
+      # How `fopen` handles opening a directory is not specified in ISO C and
+      # POSIX. We do not want to handle directories as regular files that can
+      # be opened.
+      var res {.noinit.}: Stat
+      if c_fstat(getFileHandle(f2), res) >= 0'i32 and modeIsDir(res.st_mode):
+        close(f2)
+        return false
+    when not defined(nimInheritHandles) and declared(setInheritable) and
+         NoInheritFlag.len == 0:
+      if not setInheritable(getOsFileHandle(f2), false):
+        close(f2)
+        return false
+
+    result = true
+    f = cast[File](p)
+    if bufSize > 0 and bufSize <= high(cint).int:
+      discard c_setvbuf(f, nil, IOFBF, cast[csize_t](bufSize))
+    elif bufSize == 0:
+      discard c_setvbuf(f, nil, IONBF, 0)
+
+proc reopen*(f: File, filename: string, mode: FileMode = fmRead): bool {.
+  tags: [], benign.} =
+  ## Reopens the file `f` with given `filename` and `mode`. This
+  ## is often used to redirect the `stdin`, `stdout` or `stderr`
+  ## file variables.
+  ##
+  ## Default mode is readonly. Returns true if the file could be reopened.
+  ##
+  ## The file handle associated with `f` won't be inheritable.
+  if freopen(filename.cstring, FormatOpen[mode], f) != nil:
+    when not defined(nimInheritHandles) and declared(setInheritable) and
+         NoInheritFlag.len == 0:
+      if not setInheritable(getOsFileHandle(f), false):
+        close(f)
+        return false
+    result = true
+
+proc open*(f: var File, filehandle: FileHandle,
+           mode: FileMode = fmRead): bool {.tags: [], raises: [], benign.} =
+  ## Creates a `File` from a `filehandle` with given `mode`.
+  ##
+  ## Default mode is readonly. Returns true if the file could be opened.
+  ##
+  ## The passed file handle will no longer be inheritable.
+  when not defined(nimInheritHandles) and declared(setInheritable):
+    let oshandle = when defined(windows): FileHandle getOsfhandle(
+        filehandle) else: filehandle
+    if not setInheritable(oshandle, false):
+      return false
+  f = c_fdopen(filehandle, FormatOpen[mode])
+  result = f != nil
+
+proc open*(filename: string,
+            mode: FileMode = fmRead, bufSize: int = -1): File =
+  ## Opens a file named `filename` with given `mode`.
+  ##
+  ## Default mode is readonly. Raises an `IOError` if the file
+  ## could not be opened.
+  ##
+  ## The file handle associated with the resulting `File` is not inheritable.
+  if not open(result, filename, mode, bufSize):
+    sysFatal(IOError, "cannot open: " & filename)
+
+proc setFilePos*(f: File, pos: int64, relativeTo: FileSeekPos = fspSet) {.benign.} =
+  ## Sets the position of the file pointer that is used for read/write
+  ## operations. The file's first byte has the index zero.
+  if c_fseek(f, pos, cint(relativeTo)) != 0:
+    raiseEIO("cannot set file position")
+
+proc getFilePos*(f: File): int64 {.benign.} =
+  ## Retrieves the current position of the file pointer that is used to
+  ## read from the file `f`. The file's first byte has the index zero.
+  result = c_ftell(f)
+  if result < 0: raiseEIO("cannot retrieve file position")
+
+proc getFileSize*(f: File): int64 {.tags: [ReadIOEffect], benign.} =
+  ## Retrieves the file size (in bytes) of `f`.
+  let oldPos = getFilePos(f)
+  discard c_fseek(f, 0, 2) # seek the end of the file
+  result = getFilePos(f)
+  setFilePos(f, oldPos)
+
+proc setStdIoUnbuffered*() {.tags: [], benign.} =
+  ## Configures `stdin`, `stdout` and `stderr` to be unbuffered.
+  when declared(stdout):
+    discard c_setvbuf(stdout, nil, IONBF, 0)
+  when declared(stderr):
+    discard c_setvbuf(stderr, nil, IONBF, 0)
+  when declared(stdin):
+    discard c_setvbuf(stdin, nil, IONBF, 0)
+
+when declared(stdout):
+  when defined(windows) and compileOption("threads"):
+    proc addSysExitProc(quitProc: proc() {.noconv.}) {.importc: "atexit", header: "<stdlib.h>".}
+
+    const insideRLocksModule = false
+    include "system/syslocks"
+
+
+    var echoLock: SysLock
+    initSysLock echoLock
+    addSysExitProc(proc() {.noconv.} = deinitSys(echoLock))
+
+  const stdOutLock = not defined(windows) and
+                     not defined(android) and
+                     not defined(nintendoswitch) and
+                     not defined(freertos) and
+                     not defined(zephyr) and
+                     hostOS != "any"
+
+  proc echoBinSafe(args: openArray[string]) {.compilerproc.} =
+    when defined(androidNDK):
+      var s = ""
+      for arg in args:
+        s.add arg
+      android_log_print(ANDROID_LOG_VERBOSE, "nim", s)
+    else:
+      # flockfile deadlocks some versions of Android 5.x.x
+      when stdOutLock:
+        proc flockfile(f: File) {.importc, nodecl.}
+        proc funlockfile(f: File) {.importc, nodecl.}
+        flockfile(stdout)
+      when defined(windows) and compileOption("threads"):
+        acquireSys echoLock
+      for s in args:
+        when defined(windows):
+          writeWindows(stdout, s)
+        else:
+          discard c_fwrite(s.cstring, cast[csize_t](s.len), 1, stdout)
+      const linefeed = "\n"
+      discard c_fwrite(linefeed.cstring, linefeed.len, 1, stdout)
+      discard c_fflush(stdout)
+      when stdOutLock:
+        funlockfile(stdout)
+      when defined(windows) and compileOption("threads"):
+        releaseSys echoLock
+
+
+when defined(windows) and not defined(nimscript) and not defined(js):
+  # work-around C's sucking abstraction:
+  # BUGFIX: stdin and stdout should be binary files!
+  proc c_setmode(handle, mode: cint) {.
+    importc: when defined(bcc): "setmode" else: "_setmode",
+    header: "<io.h>".}
+  var
+    O_BINARY {.importc: "_O_BINARY", header: "<fcntl.h>".}: cint
+
+  # we use binary mode on Windows:
+  c_setmode(c_fileno(stdin), O_BINARY)
+  c_setmode(c_fileno(stdout), O_BINARY)
+  c_setmode(c_fileno(stderr), O_BINARY)
+
+when defined(windows) and appType == "console" and
+    not defined(nimDontSetUtf8CodePage) and not defined(nimscript):
+  proc setConsoleOutputCP(codepage: cuint): int32 {.stdcall, dynlib: "kernel32",
+    importc: "SetConsoleOutputCP".}
+  proc setConsoleCP(wCodePageID: cuint): int32 {.stdcall, dynlib: "kernel32",
+    importc: "SetConsoleCP".}
+
+  const Utf8codepage = 65001
+  discard setConsoleOutputCP(Utf8codepage)
+  discard setConsoleCP(Utf8codepage)
+
+proc readFile*(filename: string): string {.tags: [ReadIOEffect], benign.} =
+  ## Opens a file named `filename` for reading, calls `readAll
+  ## <#readAll,File>`_ and closes the file afterwards. Returns the string.
+  ## Raises an IO exception in case of an error. If you need to call
+  ## this inside a compile time macro you can use `staticRead
+  ## <system.html#staticRead,string>`_.
+  var f: File = nil
+  if open(f, filename):
+    try:
+      result = readAll(f)
+    finally:
+      close(f)
+  else:
+    sysFatal(IOError, "cannot open: " & filename)
+
+proc writeFile*(filename, content: string) {.tags: [WriteIOEffect], benign.} =
+  ## Opens a file named `filename` for writing. Then writes the
+  ## `content` completely to the file and closes the file afterwards.
+  ## Raises an IO exception in case of an error.
+  var f: File = nil
+  if open(f, filename, fmWrite):
+    try:
+      f.write(content)
+    finally:
+      close(f)
+  else:
+    sysFatal(IOError, "cannot open: " & filename)
+
+proc writeFile*(filename: string, content: openArray[byte]) {.since: (1, 1).} =
+  ## Opens a file named `filename` for writing. Then writes the
+  ## `content` completely to the file and closes the file afterwards.
+  ## Raises an IO exception in case of an error.
+  var f: File = nil
+  if open(f, filename, fmWrite):
+    try:
+      f.writeBuffer(unsafeAddr content[0], content.len)
+    finally:
+      close(f)
+  else:
+    raise newException(IOError, "cannot open: " & filename)
+
+proc readLines*(filename: string, n: Natural): seq[string] =
+  ## Reads `n` lines from the file named `filename`. Raises an IO exception
+  ## in case of an error. Raises EOF if file does not contain at least `n` lines.
+  ## Available at compile time. A line of text may be delimited by `LF` or `CRLF`.
+  ## The newline character(s) are not part of the returned strings.
+  var f: File = nil
+  if open(f, filename):
+    try:
+      result = newSeq[string](n)
+      for i in 0 .. n - 1:
+        if not readLine(f, result[i]):
+          raiseEOF()
+    finally:
+      close(f)
+  else:
+    sysFatal(IOError, "cannot open: " & filename)
+
+template readLines*(filename: string): seq[
+    string] {.deprecated: "use readLines with two arguments".} =
+  readLines(filename, 1)
+
+iterator lines*(filename: string): string {.tags: [ReadIOEffect].} =
+  ## Iterates over any line in the file named `filename`.
+  ##
+  ## If the file does not exist `IOError` is raised. The trailing newline
+  ## character(s) are removed from the iterated lines. Example:
+  ##
+  runnableExamples:
+    import std/strutils
+
+    proc transformLetters(filename: string) =
+      var buffer = ""
+      for line in filename.lines:
+        buffer.add(line.replace("a", "0") & '\n')
+      writeFile(filename, buffer)
+  var f = open(filename, bufSize = 8000)
+  try:
+    var res = newStringOfCap(80)
+    while f.readLine(res): yield res
+  finally:
+    close(f)
+
+iterator lines*(f: File): string {.tags: [ReadIOEffect].} =
+  ## Iterates over any line in the file `f`.
+  ##
+  ## The trailing newline character(s) are removed from the iterated lines.
+  ##
+  runnableExamples:
+    proc countZeros(filename: File): tuple[lines, zeros: int] =
+      for line in filename.lines:
+        for letter in line:
+          if letter == '0':
+            result.zeros += 1
+        result.lines += 1
+  var res = newStringOfCap(80)
+  while f.readLine(res): yield res