Add MemMapFileStream. Fixes in memFiles. (#7944)

* Add MemMapFileStream

* Added tests

* Fixed bug in memfiles (zero index for string)

* Added flush to changelog

* Attempt to fix Win's nuances

* Fix attempt to fix

* Continue...

* And again...

* Reworked tests (all for win on Win)

* Fixes in flush (Win)

* Replace fn vars to consts

* Added the attempts parameter to the flush

* Replace while to for

* Move to memfiles

* Use Natural instead of uint

* Better error messages for append mode. Handle specific cases.

1 files changed, 109 insertions, 9 deletions

diff --git a/lib/pure/memfiles.nim b/lib/pure/memfiles.nim
index bf6795b0c..c7b8ebbd8 100644
--- a/lib/pure/memfiles.nim
+++ b/lib/pure/memfiles.nim
@@ -22,7 +22,11 @@ elif defined(posix):
 else:
   {.error: "the memfiles module is not supported on your operating system!".}
 
-import os
+import os, streams
+
+proc newEIO(msg: string): ref IOError =
+  new(result)
+  result.msg = msg
 
 type
   MemFile* = object  ## represents a memory mapped file
@@ -44,11 +48,14 @@ proc mapMem*(m: var MemFile, mode: FileMode = fmRead,
   ##
   ## ``mappedSize`` of ``-1`` maps to the whole file, and
   ## ``offset`` must be multiples of the PAGE SIZE of your OS
+  if mode == fmAppend:
+    raise newEIO("The append mode is not supported.")
+
   var readonly = mode == fmRead
   when defined(windows):
     result = mapViewOfFileEx(
       m.mapHandle,
-      if readonly: FILE_MAP_READ else: FILE_MAP_WRITE,
+      if readonly: FILE_MAP_READ else: FILE_MAP_READ or FILE_MAP_WRITE,
       int32(offset shr 32),
       int32(offset and 0xffffffff),
       if mappedSize == -1: 0 else: mappedSize,
@@ -113,6 +120,9 @@ proc open*(filename: string, mode: FileMode = fmRead,
   ##   mm_half = memfiles.open("/tmp/test.mmap", mode = fmReadWrite, mappedSize = 512)
 
   # The file can be resized only when write mode is used:
+  if mode == fmAppend:
+    raise newEIO("The append mode is not supported.")
+
   assert newFileSize == -1 or mode != fmRead
   var readonly = mode == fmRead
 
@@ -121,6 +131,10 @@ proc open*(filename: string, mode: FileMode = fmRead,
     result.size = 0
 
   when defined(windows):
+    let desiredAccess = GENERIC_READ
+    let shareMode = FILE_SHARE_READ
+    let flags = FILE_FLAG_RANDOM_ACCESS
+
     template fail(errCode: OSErrorCode, msg: untyped) =
       rollback()
       if result.fHandle != 0: discard closeHandle(result.fHandle)
@@ -133,11 +147,11 @@ proc open*(filename: string, mode: FileMode = fmRead,
       winApiProc(
         filename,
         # GENERIC_ALL != (GENERIC_READ or GENERIC_WRITE)
-        if readonly: GENERIC_READ else: GENERIC_READ or GENERIC_WRITE,
-        FILE_SHARE_READ,
+        if readonly: desiredAccess else: desiredAccess or GENERIC_WRITE,
+        if readonly: shareMode else: shareMode or FILE_SHARE_WRITE,
         nil,
         if newFileSize != -1: CREATE_ALWAYS else: OPEN_EXISTING,
-        if readonly: FILE_ATTRIBUTE_READONLY else: FILE_ATTRIBUTE_TEMPORARY,
+        if readonly: FILE_ATTRIBUTE_READONLY or flags else: FILE_ATTRIBUTE_NORMAL or flags,
         0)
 
     when useWinUnicode:
@@ -172,7 +186,7 @@ proc open*(filename: string, mode: FileMode = fmRead,
 
     result.mem = mapViewOfFileEx(
       result.mapHandle,
-      if readonly: FILE_MAP_READ else: FILE_MAP_WRITE,
+      if readonly: FILE_MAP_READ else: FILE_MAP_READ or FILE_MAP_WRITE,
       int32(offset shr 32),
       int32(offset and 0xffffffff),
       if mappedSize == -1: 0 else: mappedSize,
@@ -245,6 +259,28 @@ proc open*(filename: string, mode: FileMode = fmRead,
       if close(result.handle) == 0:
         result.handle = -1
 
+proc flush*(f: var MemFile; attempts: Natural = 3) =
+  ## Flushes `f`'s buffer for the number of attempts equal to `attempts`.
+  ## If were errors an exception `OSError` will be raised.
+  var res = false
+  var lastErr: OSErrorCode
+  when defined(windows):
+    for i in 1..attempts:
+      res = flushViewOfFile(f.mem, 0) != 0
+      if res:
+        break
+      lastErr = osLastError()
+      if lastErr != ERROR_LOCK_VIOLATION.OSErrorCode:
+        raiseOSError(lastErr)
+  else:
+    for i in 1..attempts:
+      res = msync(f.mem, f.size, MS_SYNC or MS_INVALIDATE) == 0
+      if res:
+        break
+      lastErr = osLastError()
+      if lastErr != EBUSY.OSErrorCode:
+        raiseOSError(lastErr, "error flushing mapping")
+
 proc close*(f: var MemFile) =
   ## closes the memory mapped file `f`. All changes are written back to the
   ## file system, if `f` was opened with write access.
@@ -362,9 +398,8 @@ iterator lines*(mfile: MemFile, buf: var TaintedString, delim='\l', eat='\r'): T
   ##     echo line
 
   for ms in memSlices(mfile, delim, eat):
-    buf.setLen(ms.size)
-    copyMem(addr(buf[0]), ms.data, ms.size)
-    buf[ms.size] = '\0'
+    setLen(buf.string, ms.size)
+    copyMem(buf.cstring, ms.data, ms.size)
     yield buf
 
 iterator lines*(mfile: MemFile, delim='\l', eat='\r'): TaintedString {.inline.} =
@@ -382,3 +417,68 @@ iterator lines*(mfile: MemFile, delim='\l', eat='\r'): TaintedString {.inline.}
   var buf = TaintedString(newStringOfCap(80))
   for line in lines(mfile, buf, delim, eat):
     yield buf
+
+type
+  MemMapFileStream* = ref MemMapFileStreamObj ## a stream that encapsulates a `MemFile`
+  MemMapFileStreamObj* = object of Stream
+    mf: MemFile
+    mode: FileMode
+    pos: ByteAddress
+
+proc mmsClose(s: Stream) =
+  MemMapFileStream(s).pos = -1
+  close(MemMapFileStream(s).mf)
+
+proc mmsFlush(s: Stream) = flush(MemMapFileStream(s).mf)
+
+proc mmsAtEnd(s: Stream): bool = (MemMapFileStream(s).pos >= MemMapFileStream(s).mf.size) or
+                                 (MemMapFileStream(s).pos < 0)
+
+proc mmsSetPosition(s: Stream, pos: int) =
+  if pos > MemMapFileStream(s).mf.size or pos < 0:
+    raise newEIO("cannot set pos in stream")
+  MemMapFileStream(s).pos = pos
+
+proc mmsGetPosition(s: Stream): int = MemMapFileStream(s).pos
+
+proc mmsPeekData(s: Stream, buffer: pointer, bufLen: int): int =
+  let startAddress = cast[ByteAddress](MemMapFileStream(s).mf.mem)
+  let p = cast[ByteAddress](MemMapFileStream(s).pos)
+  let l = min(bufLen, MemMapFileStream(s).mf.size - p)
+  moveMem(buffer, cast[pointer](startAddress + p), l)
+  result = l
+
+proc mmsReadData(s: Stream, buffer: pointer, bufLen: int): int =
+  result = mmsPeekData(s, buffer, bufLen)
+  inc(MemMapFileStream(s).pos, result)
+
+proc mmsWriteData(s: Stream, buffer: pointer, bufLen: int) =
+  if MemMapFileStream(s).mode == fmRead:
+    raise newEIO("cannot write to read-only stream")
+  let size = MemMapFileStream(s).mf.size
+  if MemMapFileStream(s).pos + bufLen > size:
+    raise newEIO("cannot write to stream")
+  let p = cast[ByteAddress](MemMapFileStream(s).mf.mem) +
+          cast[ByteAddress](MemMapFileStream(s).pos)
+  moveMem(cast[pointer](p), buffer, bufLen)
+  inc(MemMapFileStream(s).pos, bufLen)
+
+proc newMemMapFileStream*(filename: string, mode: FileMode = fmRead, fileSize: int = -1):
+  MemMapFileStream =
+  ## creates a new stream from the file named `filename` with the mode `mode`.
+  ## Raises ## `EOS` if the file cannot be opened. See the `system
+  ## <system.html>`_ module for a list of available FileMode enums.
+  ## ``fileSize`` can only be set if the file does not exist and is opened
+  ## with write access (e.g., with fmReadWrite).
+  var mf: MemFile = open(filename, mode, newFileSize = fileSize)
+  new(result)
+  result.mode = mode
+  result.mf = mf
+  result.closeImpl = mmsClose
+  result.atEndImpl = mmsAtEnd
+  result.setPositionImpl = mmsSetPosition
+  result.getPositionImpl = mmsGetPosition
+  result.readDataImpl = mmsReadData
+  result.peekDataImpl = mmsPeekData
+  result.writeDataImpl = mmsWriteData
+  result.flushImpl = mmsFlush