Add MemMapFileStream. Fixes in memFiles. (#7944)

* Add MemMapFileStream

* Added tests

* Fixed bug in memfiles (zero index for string)

* Added flush to changelog

* Attempt to fix Win's nuances

* Fix attempt to fix

* Continue...

* And again...

* Reworked tests (all for win on Win)

* Fixes in flush (Win)

* Replace fn vars to consts

* Added the attempts parameter to the flush

* Replace while to for

* Move to memfiles

* Use Natural instead of uint

* Better error messages for append mode. Handle specific cases.

5 files changed, 195 insertions, 23 deletions

diff --git a/changelog.md b/changelog.md
index 8919cf702..4067cb693 100644
--- a/changelog.md
+++ b/changelog.md
@@ -78,6 +78,8 @@
 - Added the proc ``math.prod`` for product of elements in openArray.
 - Added the proc ``parseBinInt`` to parse a binary integer from a string, which returns the value.
 - ``parseOct`` and ``parseBin`` in parseutils now also support the ``maxLen`` argument similar to ``parseHexInt``
+- Added the proc ``flush`` for memory mapped files.
+- Added the ``MemMapFileStream``.
 
 ### Library changes
 
diff --git a/lib/pure/memfiles.nim b/lib/pure/memfiles.nim
index bf6795b0c..c7b8ebbd8 100644
--- a/lib/pure/memfiles.nim
+++ b/lib/pure/memfiles.nim
@@ -22,7 +22,11 @@ elif defined(posix):
 else:
   {.error: "the memfiles module is not supported on your operating system!".}
 
-import os
+import os, streams
+
+proc newEIO(msg: string): ref IOError =
+  new(result)
+  result.msg = msg
 
 type
   MemFile* = object  ## represents a memory mapped file
@@ -44,11 +48,14 @@ proc mapMem*(m: var MemFile, mode: FileMode = fmRead,
   ##
   ## ``mappedSize`` of ``-1`` maps to the whole file, and
   ## ``offset`` must be multiples of the PAGE SIZE of your OS
+  if mode == fmAppend:
+    raise newEIO("The append mode is not supported.")
+
   var readonly = mode == fmRead
   when defined(windows):
     result = mapViewOfFileEx(
       m.mapHandle,
-      if readonly: FILE_MAP_READ else: FILE_MAP_WRITE,
+      if readonly: FILE_MAP_READ else: FILE_MAP_READ or FILE_MAP_WRITE,
       int32(offset shr 32),
       int32(offset and 0xffffffff),
       if mappedSize == -1: 0 else: mappedSize,
@@ -113,6 +120,9 @@ proc open*(filename: string, mode: FileMode = fmRead,
   ##   mm_half = memfiles.open("/tmp/test.mmap", mode = fmReadWrite, mappedSize = 512)
 
   # The file can be resized only when write mode is used:
+  if mode == fmAppend:
+    raise newEIO("The append mode is not supported.")
+
   assert newFileSize == -1 or mode != fmRead
   var readonly = mode == fmRead
 
@@ -121,6 +131,10 @@ proc open*(filename: string, mode: FileMode = fmRead,
     result.size = 0
 
   when defined(windows):
+    let desiredAccess = GENERIC_READ
+    let shareMode = FILE_SHARE_READ
+    let flags = FILE_FLAG_RANDOM_ACCESS
+
     template fail(errCode: OSErrorCode, msg: untyped) =
       rollback()
       if result.fHandle != 0: discard closeHandle(result.fHandle)
@@ -133,11 +147,11 @@ proc open*(filename: string, mode: FileMode = fmRead,
       winApiProc(
         filename,
         # GENERIC_ALL != (GENERIC_READ or GENERIC_WRITE)
-        if readonly: GENERIC_READ else: GENERIC_READ or GENERIC_WRITE,
-        FILE_SHARE_READ,
+        if readonly: desiredAccess else: desiredAccess or GENERIC_WRITE,
+        if readonly: shareMode else: shareMode or FILE_SHARE_WRITE,
         nil,
         if newFileSize != -1: CREATE_ALWAYS else: OPEN_EXISTING,
-        if readonly: FILE_ATTRIBUTE_READONLY else: FILE_ATTRIBUTE_TEMPORARY,
+        if readonly: FILE_ATTRIBUTE_READONLY or flags else: FILE_ATTRIBUTE_NORMAL or flags,
         0)
 
     when useWinUnicode:
@@ -172,7 +186,7 @@ proc open*(filename: string, mode: FileMode = fmRead,
 
     result.mem = mapViewOfFileEx(
       result.mapHandle,
-      if readonly: FILE_MAP_READ else: FILE_MAP_WRITE,
+      if readonly: FILE_MAP_READ else: FILE_MAP_READ or FILE_MAP_WRITE,
       int32(offset shr 32),
       int32(offset and 0xffffffff),
       if mappedSize == -1: 0 else: mappedSize,
@@ -245,6 +259,28 @@ proc open*(filename: string, mode: FileMode = fmRead,
       if close(result.handle) == 0:
         result.handle = -1
 
+proc flush*(f: var MemFile; attempts: Natural = 3) =
+  ## Flushes `f`'s buffer for the number of attempts equal to `attempts`.
+  ## If were errors an exception `OSError` will be raised.
+  var res = false
+  var lastErr: OSErrorCode
+  when defined(windows):
+    for i in 1..attempts:
+      res = flushViewOfFile(f.mem, 0) != 0
+      if res:
+        break
+      lastErr = osLastError()
+      if lastErr != ERROR_LOCK_VIOLATION.OSErrorCode:
+        raiseOSError(lastErr)
+  else:
+    for i in 1..attempts:
+      res = msync(f.mem, f.size, MS_SYNC or MS_INVALIDATE) == 0
+      if res:
+        break
+      lastErr = osLastError()
+      if lastErr != EBUSY.OSErrorCode:
+        raiseOSError(lastErr, "error flushing mapping")
+
 proc close*(f: var MemFile) =
   ## closes the memory mapped file `f`. All changes are written back to the
   ## file system, if `f` was opened with write access.
@@ -362,9 +398,8 @@ iterator lines*(mfile: MemFile, buf: var TaintedString, delim='\l', eat='\r'): T
   ##     echo line
 
   for ms in memSlices(mfile, delim, eat):
-    buf.setLen(ms.size)
-    copyMem(addr(buf[0]), ms.data, ms.size)
-    buf[ms.size] = '\0'
+    setLen(buf.string, ms.size)
+    copyMem(buf.cstring, ms.data, ms.size)
     yield buf
 
 iterator lines*(mfile: MemFile, delim='\l', eat='\r'): TaintedString {.inline.} =
@@ -382,3 +417,68 @@ iterator lines*(mfile: MemFile, delim='\l', eat='\r'): TaintedString {.inline.}
   var buf = TaintedString(newStringOfCap(80))
   for line in lines(mfile, buf, delim, eat):
     yield buf
+
+type
+  MemMapFileStream* = ref MemMapFileStreamObj ## a stream that encapsulates a `MemFile`
+  MemMapFileStreamObj* = object of Stream
+    mf: MemFile
+    mode: FileMode
+    pos: ByteAddress
+
+proc mmsClose(s: Stream) =
+  MemMapFileStream(s).pos = -1
+  close(MemMapFileStream(s).mf)
+
+proc mmsFlush(s: Stream) = flush(MemMapFileStream(s).mf)
+
+proc mmsAtEnd(s: Stream): bool = (MemMapFileStream(s).pos >= MemMapFileStream(s).mf.size) or
+                                 (MemMapFileStream(s).pos < 0)
+
+proc mmsSetPosition(s: Stream, pos: int) =
+  if pos > MemMapFileStream(s).mf.size or pos < 0:
+    raise newEIO("cannot set pos in stream")
+  MemMapFileStream(s).pos = pos
+
+proc mmsGetPosition(s: Stream): int = MemMapFileStream(s).pos
+
+proc mmsPeekData(s: Stream, buffer: pointer, bufLen: int): int =
+  let startAddress = cast[ByteAddress](MemMapFileStream(s).mf.mem)
+  let p = cast[ByteAddress](MemMapFileStream(s).pos)
+  let l = min(bufLen, MemMapFileStream(s).mf.size - p)
+  moveMem(buffer, cast[pointer](startAddress + p), l)
+  result = l
+
+proc mmsReadData(s: Stream, buffer: pointer, bufLen: int): int =
+  result = mmsPeekData(s, buffer, bufLen)
+  inc(MemMapFileStream(s).pos, result)
+
+proc mmsWriteData(s: Stream, buffer: pointer, bufLen: int) =
+  if MemMapFileStream(s).mode == fmRead:
+    raise newEIO("cannot write to read-only stream")
+  let size = MemMapFileStream(s).mf.size
+  if MemMapFileStream(s).pos + bufLen > size:
+    raise newEIO("cannot write to stream")
+  let p = cast[ByteAddress](MemMapFileStream(s).mf.mem) +
+          cast[ByteAddress](MemMapFileStream(s).pos)
+  moveMem(cast[pointer](p), buffer, bufLen)
+  inc(MemMapFileStream(s).pos, bufLen)
+
+proc newMemMapFileStream*(filename: string, mode: FileMode = fmRead, fileSize: int = -1):
+  MemMapFileStream =
+  ## creates a new stream from the file named `filename` with the mode `mode`.
+  ## Raises ## `EOS` if the file cannot be opened. See the `system
+  ## <system.html>`_ module for a list of available FileMode enums.
+  ## ``fileSize`` can only be set if the file does not exist and is opened
+  ## with write access (e.g., with fmReadWrite).
+  var mf: MemFile = open(filename, mode, newFileSize = fileSize)
+  new(result)
+  result.mode = mode
+  result.mf = mf
+  result.closeImpl = mmsClose
+  result.atEndImpl = mmsAtEnd
+  result.setPositionImpl = mmsSetPosition
+  result.getPositionImpl = mmsGetPosition
+  result.readDataImpl = mmsReadData
+  result.peekDataImpl = mmsPeekData
+  result.writeDataImpl = mmsWriteData
+  result.flushImpl = mmsFlush
diff --git a/lib/windows/winlean.nim b/lib/windows/winlean.nim
index 7e22f98c7..3263f1d37 100644
--- a/lib/windows/winlean.nim
+++ b/lib/windows/winlean.nim
@@ -129,7 +129,6 @@ const
   PIPE_ACCESS_OUTBOUND* = 2'i32
   PIPE_NOWAIT* = 0x00000001'i32
   SYNCHRONIZE* = 0x00100000'i32
-  FILE_FLAG_WRITE_THROUGH* = 0x80000000'i32
 
   CREATE_NO_WINDOW* = 0x08000000'i32
 
@@ -281,15 +280,31 @@ else:
     importc:"CreateHardLinkA", dynlib: "kernel32", stdcall.}
 
 const
-  FILE_ATTRIBUTE_ARCHIVE* = 32'i32
-  FILE_ATTRIBUTE_COMPRESSED* = 2048'i32
-  FILE_ATTRIBUTE_NORMAL* = 128'i32
-  FILE_ATTRIBUTE_DIRECTORY* = 16'i32
-  FILE_ATTRIBUTE_HIDDEN* = 2'i32
-  FILE_ATTRIBUTE_READONLY* = 1'i32
-  FILE_ATTRIBUTE_REPARSE_POINT* = 1024'i32
-  FILE_ATTRIBUTE_SYSTEM* = 4'i32
-  FILE_ATTRIBUTE_TEMPORARY* = 256'i32
+  FILE_ATTRIBUTE_READONLY* = 0x00000001'i32
+  FILE_ATTRIBUTE_HIDDEN* = 0x00000002'i32
+  FILE_ATTRIBUTE_SYSTEM* = 0x00000004'i32
+  FILE_ATTRIBUTE_DIRECTORY* = 0x00000010'i32
+  FILE_ATTRIBUTE_ARCHIVE* = 0x00000020'i32
+  FILE_ATTRIBUTE_DEVICE* = 0x00000040'i32
+  FILE_ATTRIBUTE_NORMAL* = 0x00000080'i32
+  FILE_ATTRIBUTE_TEMPORARY* = 0x00000100'i32
+  FILE_ATTRIBUTE_SPARSE_FILE* = 0x00000200'i32
+  FILE_ATTRIBUTE_REPARSE_POINT* = 0x00000400'i32
+  FILE_ATTRIBUTE_COMPRESSED* = 0x00000800'i32
+  FILE_ATTRIBUTE_OFFLINE* = 0x00001000'i32
+  FILE_ATTRIBUTE_NOT_CONTENT_INDEXED* = 0x00002000'i32
+
+  FILE_FLAG_FIRST_PIPE_INSTANCE* = 0x00080000'i32
+  FILE_FLAG_OPEN_NO_RECALL* = 0x00100000'i32
+  FILE_FLAG_OPEN_REPARSE_POINT* = 0x00200000'i32
+  FILE_FLAG_POSIX_SEMANTICS* = 0x01000000'i32
+  FILE_FLAG_BACKUP_SEMANTICS* = 0x02000000'i32
+  FILE_FLAG_DELETE_ON_CLOSE* = 0x04000000'i32
+  FILE_FLAG_SEQUENTIAL_SCAN* = 0x08000000'i32
+  FILE_FLAG_RANDOM_ACCESS* = 0x10000000'i32
+  FILE_FLAG_NO_BUFFERING* = 0x20000000'i32
+  FILE_FLAG_OVERLAPPED* = 0x40000000'i32
+  FILE_FLAG_WRITE_THROUGH* = 0x80000000'i32
 
   MAX_PATH* = 260
 
@@ -683,8 +698,6 @@ const
   FILE_MAP_WRITE* = 2'i32
   INVALID_FILE_SIZE* = -1'i32
 
-  FILE_FLAG_BACKUP_SEMANTICS* = 33554432'i32
-  FILE_FLAG_OPEN_REPARSE_POINT* = 0x00200000'i32
   DUPLICATE_SAME_ACCESS* = 2
   FILE_READ_DATA* = 0x00000001 # file & pipe
   FILE_WRITE_DATA* = 0x00000002 # file & pipe
@@ -695,6 +708,7 @@ const
   ERROR_PATH_NOT_FOUND* = 3
   ERROR_ACCESS_DENIED* = 5
   ERROR_NO_MORE_FILES* = 18
+  ERROR_LOCK_VIOLATION* = 33
   ERROR_HANDLE_EOF* = 38
   ERROR_BAD_ARGUMENTS* = 165
 
@@ -763,6 +777,9 @@ when not useWinUnicode:
 proc unmapViewOfFile*(lpBaseAddress: pointer): WINBOOL {.stdcall,
     dynlib: "kernel32", importc: "UnmapViewOfFile".}
 
+proc flushViewOfFile*(lpBaseAddress: pointer, dwNumberOfBytesToFlush: DWORD): WINBOOL {.
+  stdcall, dynlib: "kernel32", importc: "FlushViewOfFile".}
+
 type
   OVERLAPPED* {.pure, inheritable.} = object
     internal*: PULONG
@@ -785,7 +802,6 @@ type
 
 const
   ERROR_IO_PENDING* = 997 # a.k.a WSA_IO_PENDING
-  FILE_FLAG_OVERLAPPED* = 1073741824
   WSAECONNABORTED* = 10053
   WSAEADDRINUSE* = 10048
   WSAECONNRESET* = 10054
diff --git a/tests/stdlib/tmemfiles2.nim b/tests/stdlib/tmemfiles2.nim
index 7ea94cffc..d6cfa533a 100644
--- a/tests/stdlib/tmemfiles2.nim
+++ b/tests/stdlib/tmemfiles2.nim
@@ -4,9 +4,10 @@ discard """
 Half read size: 10 Data: Hello'''
 """
 import memfiles, os
+const
+  fn = "test.mmap"
 var
   mm, mm_full, mm_half: MemFile
-  fn = "test.mmap"
   p: pointer
 
 if fileExists(fn): removeFile(fn)
diff --git a/tests/stdlib/tmemmapstreams.nim b/tests/stdlib/tmemmapstreams.nim
new file mode 100644
index 000000000..243574f1a
--- /dev/null
+++ b/tests/stdlib/tmemmapstreams.nim
@@ -0,0 +1,53 @@
+discard """
+  file: "tmemmapstreams.nim"
+  output: '''Created size: 10
+Position after writing: 5
+Position after writing one char: 6
+Peeked data: Hello
+Position after peeking: 0
+Readed data: Hello!
+Position after reading line: 7
+Position after setting position: 6
+Readed line: Hello!
+Position after reading line: 7'''
+"""
+import os, streams, memfiles
+const
+  fn = "test.mmapstream"
+var
+  mms: MemMapFileStream
+
+if fileExists(fn): removeFile(fn)
+
+# Create a new memory mapped file, data all zeros
+mms = newMemMapFileStream(fn, mode = fmReadWrite, fileSize = 10)
+mms.close()
+if fileExists(fn): echo "Created size: ", getFileSize(fn)
+
+# write, flush, peek, read
+mms = newMemMapFileStream(fn, mode = fmReadWrite)
+let s = "Hello"
+
+mms.write(s)
+mms.flush
+echo "Position after writing: ", mms.getPosition()
+mms.write('!')
+mms.flush
+echo "Position after writing one char: ", mms.getPosition()
+mms.close()
+
+mms = newMemMapFileStream(fn, mode = fmRead)
+echo "Peeked data: ", mms.peekStr(s.len)
+echo "Position after peeking: ", mms.getPosition()
+echo "Readed data: ", mms.readLine
+echo "Position after reading line: ", mms.getPosition()
+mms.setPosition(mms.getPosition() - 1)
+echo "Position after setting position: ", mms.getPosition()
+
+mms.setPosition(0)
+echo "Readed line: ", mms.readLine
+echo "Position after reading line: ", mms.getPosition()
+
+mms.close()
+
+if fileExists(fn): removeFile(fn)