From bbb0fd4eb734ed1a0445865077c27b05b46d0cbd Mon Sep 17 00:00:00 2001
From: Araq <rumpf_a@web.de>
Date: Fri, 4 May 2018 14:03:16 +0200
Subject: remove deprecated stuff from the stdlib; introduce better deprecation
 warnings

---
 lib/pure/memfiles.nim | 2 --
 1 file changed, 2 deletions(-)

(limited to 'lib/pure/memfiles.nim')

diff --git a/lib/pure/memfiles.nim b/lib/pure/memfiles.nim
index 5c73381ff..bf6795b0c 100644
--- a/lib/pure/memfiles.nim
+++ b/lib/pure/memfiles.nim
@@ -38,8 +38,6 @@ type
     else:
       handle: cint
 
-{.deprecated: [TMemFile: MemFile].}
-
 proc mapMem*(m: var MemFile, mode: FileMode = fmRead,
              mappedSize = -1, offset = 0): pointer =
   ## returns a pointer to a mapped portion of MemFile `m`
-- 
cgit 1.4.1-2-gfad0


From bf5d619a52da04c857a6f7fb3d68afc12182bc22 Mon Sep 17 00:00:00 2001
From: Dmitry Atamanov <data-man@users.noreply.github.com>
Date: Thu, 14 Jun 2018 19:34:26 +0300
Subject: Add MemMapFileStream. Fixes in memFiles. (#7944)

* Add MemMapFileStream

* Added tests

* Fixed bug in memfiles (zero index for string)

* Added flush to changelog

* Attempt to fix Win's nuances

* Fix attempt to fix

* Continue...

* And again...

* Reworked tests (all for win on Win)

* Fixes in flush (Win)

* Replace fn vars to consts

* Added the attempts parameter to the flush

* Replace while to for

* Move to memfiles

* Use Natural instead of uint

* Better error messages for append mode. Handle specific cases.
---
 changelog.md                    |   2 +
 lib/pure/memfiles.nim           | 118 +++++++++++++++++++++++++++++++++++++---
 lib/windows/winlean.nim         |  42 +++++++++-----
 tests/stdlib/tmemfiles2.nim     |   3 +-
 tests/stdlib/tmemmapstreams.nim |  53 ++++++++++++++++++
 5 files changed, 195 insertions(+), 23 deletions(-)
 create mode 100644 tests/stdlib/tmemmapstreams.nim

(limited to 'lib/pure/memfiles.nim')

diff --git a/changelog.md b/changelog.md
index 8919cf702..4067cb693 100644
--- a/changelog.md
+++ b/changelog.md
@@ -78,6 +78,8 @@
 - Added the proc ``math.prod`` for product of elements in openArray.
 - Added the proc ``parseBinInt`` to parse a binary integer from a string, which returns the value.
 - ``parseOct`` and ``parseBin`` in parseutils now also support the ``maxLen`` argument similar to ``parseHexInt``
+- Added the proc ``flush`` for memory mapped files.
+- Added the ``MemMapFileStream``.
 
 ### Library changes
 
diff --git a/lib/pure/memfiles.nim b/lib/pure/memfiles.nim
index bf6795b0c..c7b8ebbd8 100644
--- a/lib/pure/memfiles.nim
+++ b/lib/pure/memfiles.nim
@@ -22,7 +22,11 @@ elif defined(posix):
 else:
   {.error: "the memfiles module is not supported on your operating system!".}
 
-import os
+import os, streams
+
+proc newEIO(msg: string): ref IOError =
+  new(result)
+  result.msg = msg
 
 type
   MemFile* = object  ## represents a memory mapped file
@@ -44,11 +48,14 @@ proc mapMem*(m: var MemFile, mode: FileMode = fmRead,
   ##
   ## ``mappedSize`` of ``-1`` maps to the whole file, and
   ## ``offset`` must be multiples of the PAGE SIZE of your OS
+  if mode == fmAppend:
+    raise newEIO("The append mode is not supported.")
+
   var readonly = mode == fmRead
   when defined(windows):
     result = mapViewOfFileEx(
       m.mapHandle,
-      if readonly: FILE_MAP_READ else: FILE_MAP_WRITE,
+      if readonly: FILE_MAP_READ else: FILE_MAP_READ or FILE_MAP_WRITE,
       int32(offset shr 32),
       int32(offset and 0xffffffff),
       if mappedSize == -1: 0 else: mappedSize,
@@ -113,6 +120,9 @@ proc open*(filename: string, mode: FileMode = fmRead,
   ##   mm_half = memfiles.open("/tmp/test.mmap", mode = fmReadWrite, mappedSize = 512)
 
   # The file can be resized only when write mode is used:
+  if mode == fmAppend:
+    raise newEIO("The append mode is not supported.")
+
   assert newFileSize == -1 or mode != fmRead
   var readonly = mode == fmRead
 
@@ -121,6 +131,10 @@ proc open*(filename: string, mode: FileMode = fmRead,
     result.size = 0
 
   when defined(windows):
+    let desiredAccess = GENERIC_READ
+    let shareMode = FILE_SHARE_READ
+    let flags = FILE_FLAG_RANDOM_ACCESS
+
     template fail(errCode: OSErrorCode, msg: untyped) =
       rollback()
       if result.fHandle != 0: discard closeHandle(result.fHandle)
@@ -133,11 +147,11 @@ proc open*(filename: string, mode: FileMode = fmRead,
       winApiProc(
         filename,
         # GENERIC_ALL != (GENERIC_READ or GENERIC_WRITE)
-        if readonly: GENERIC_READ else: GENERIC_READ or GENERIC_WRITE,
-        FILE_SHARE_READ,
+        if readonly: desiredAccess else: desiredAccess or GENERIC_WRITE,
+        if readonly: shareMode else: shareMode or FILE_SHARE_WRITE,
         nil,
         if newFileSize != -1: CREATE_ALWAYS else: OPEN_EXISTING,
-        if readonly: FILE_ATTRIBUTE_READONLY else: FILE_ATTRIBUTE_TEMPORARY,
+        if readonly: FILE_ATTRIBUTE_READONLY or flags else: FILE_ATTRIBUTE_NORMAL or flags,
         0)
 
     when useWinUnicode:
@@ -172,7 +186,7 @@ proc open*(filename: string, mode: FileMode = fmRead,
 
     result.mem = mapViewOfFileEx(
       result.mapHandle,
-      if readonly: FILE_MAP_READ else: FILE_MAP_WRITE,
+      if readonly: FILE_MAP_READ else: FILE_MAP_READ or FILE_MAP_WRITE,
       int32(offset shr 32),
       int32(offset and 0xffffffff),
       if mappedSize == -1: 0 else: mappedSize,
@@ -245,6 +259,28 @@ proc open*(filename: string, mode: FileMode = fmRead,
       if close(result.handle) == 0:
         result.handle = -1
 
+proc flush*(f: var MemFile; attempts: Natural = 3) =
+  ## Flushes `f`'s buffer for the number of attempts equal to `attempts`.
+  ## If were errors an exception `OSError` will be raised.
+  var res = false
+  var lastErr: OSErrorCode
+  when defined(windows):
+    for i in 1..attempts:
+      res = flushViewOfFile(f.mem, 0) != 0
+      if res:
+        break
+      lastErr = osLastError()
+      if lastErr != ERROR_LOCK_VIOLATION.OSErrorCode:
+        raiseOSError(lastErr)
+  else:
+    for i in 1..attempts:
+      res = msync(f.mem, f.size, MS_SYNC or MS_INVALIDATE) == 0
+      if res:
+        break
+      lastErr = osLastError()
+      if lastErr != EBUSY.OSErrorCode:
+        raiseOSError(lastErr, "error flushing mapping")
+
 proc close*(f: var MemFile) =
   ## closes the memory mapped file `f`. All changes are written back to the
   ## file system, if `f` was opened with write access.
@@ -362,9 +398,8 @@ iterator lines*(mfile: MemFile, buf: var TaintedString, delim='\l', eat='\r'): T
   ##     echo line
 
   for ms in memSlices(mfile, delim, eat):
-    buf.setLen(ms.size)
-    copyMem(addr(buf[0]), ms.data, ms.size)
-    buf[ms.size] = '\0'
+    setLen(buf.string, ms.size)
+    copyMem(buf.cstring, ms.data, ms.size)
     yield buf
 
 iterator lines*(mfile: MemFile, delim='\l', eat='\r'): TaintedString {.inline.} =
@@ -382,3 +417,68 @@ iterator lines*(mfile: MemFile, delim='\l', eat='\r'): TaintedString {.inline.}
   var buf = TaintedString(newStringOfCap(80))
   for line in lines(mfile, buf, delim, eat):
     yield buf
+
+type
+  MemMapFileStream* = ref MemMapFileStreamObj ## a stream that encapsulates a `MemFile`
+  MemMapFileStreamObj* = object of Stream
+    mf: MemFile
+    mode: FileMode
+    pos: ByteAddress
+
+proc mmsClose(s: Stream) =
+  MemMapFileStream(s).pos = -1
+  close(MemMapFileStream(s).mf)
+
+proc mmsFlush(s: Stream) = flush(MemMapFileStream(s).mf)
+
+proc mmsAtEnd(s: Stream): bool = (MemMapFileStream(s).pos >= MemMapFileStream(s).mf.size) or
+                                 (MemMapFileStream(s).pos < 0)
+
+proc mmsSetPosition(s: Stream, pos: int) =
+  if pos > MemMapFileStream(s).mf.size or pos < 0:
+    raise newEIO("cannot set pos in stream")
+  MemMapFileStream(s).pos = pos
+
+proc mmsGetPosition(s: Stream): int = MemMapFileStream(s).pos
+
+proc mmsPeekData(s: Stream, buffer: pointer, bufLen: int): int =
+  let startAddress = cast[ByteAddress](MemMapFileStream(s).mf.mem)
+  let p = cast[ByteAddress](MemMapFileStream(s).pos)
+  let l = min(bufLen, MemMapFileStream(s).mf.size - p)
+  moveMem(buffer, cast[pointer](startAddress + p), l)
+  result = l
+
+proc mmsReadData(s: Stream, buffer: pointer, bufLen: int): int =
+  result = mmsPeekData(s, buffer, bufLen)
+  inc(MemMapFileStream(s).pos, result)
+
+proc mmsWriteData(s: Stream, buffer: pointer, bufLen: int) =
+  if MemMapFileStream(s).mode == fmRead:
+    raise newEIO("cannot write to read-only stream")
+  let size = MemMapFileStream(s).mf.size
+  if MemMapFileStream(s).pos + bufLen > size:
+    raise newEIO("cannot write to stream")
+  let p = cast[ByteAddress](MemMapFileStream(s).mf.mem) +
+          cast[ByteAddress](MemMapFileStream(s).pos)
+  moveMem(cast[pointer](p), buffer, bufLen)
+  inc(MemMapFileStream(s).pos, bufLen)
+
+proc newMemMapFileStream*(filename: string, mode: FileMode = fmRead, fileSize: int = -1):
+  MemMapFileStream =
+  ## creates a new stream from the file named `filename` with the mode `mode`.
+  ## Raises ## `EOS` if the file cannot be opened. See the `system
+  ## <system.html>`_ module for a list of available FileMode enums.
+  ## ``fileSize`` can only be set if the file does not exist and is opened
+  ## with write access (e.g., with fmReadWrite).
+  var mf: MemFile = open(filename, mode, newFileSize = fileSize)
+  new(result)
+  result.mode = mode
+  result.mf = mf
+  result.closeImpl = mmsClose
+  result.atEndImpl = mmsAtEnd
+  result.setPositionImpl = mmsSetPosition
+  result.getPositionImpl = mmsGetPosition
+  result.readDataImpl = mmsReadData
+  result.peekDataImpl = mmsPeekData
+  result.writeDataImpl = mmsWriteData
+  result.flushImpl = mmsFlush
diff --git a/lib/windows/winlean.nim b/lib/windows/winlean.nim
index 7e22f98c7..3263f1d37 100644
--- a/lib/windows/winlean.nim
+++ b/lib/windows/winlean.nim
@@ -129,7 +129,6 @@ const
   PIPE_ACCESS_OUTBOUND* = 2'i32
   PIPE_NOWAIT* = 0x00000001'i32
   SYNCHRONIZE* = 0x00100000'i32
-  FILE_FLAG_WRITE_THROUGH* = 0x80000000'i32
 
   CREATE_NO_WINDOW* = 0x08000000'i32
 
@@ -281,15 +280,31 @@ else:
     importc:"CreateHardLinkA", dynlib: "kernel32", stdcall.}
 
 const
-  FILE_ATTRIBUTE_ARCHIVE* = 32'i32
-  FILE_ATTRIBUTE_COMPRESSED* = 2048'i32
-  FILE_ATTRIBUTE_NORMAL* = 128'i32
-  FILE_ATTRIBUTE_DIRECTORY* = 16'i32
-  FILE_ATTRIBUTE_HIDDEN* = 2'i32
-  FILE_ATTRIBUTE_READONLY* = 1'i32
-  FILE_ATTRIBUTE_REPARSE_POINT* = 1024'i32
-  FILE_ATTRIBUTE_SYSTEM* = 4'i32
-  FILE_ATTRIBUTE_TEMPORARY* = 256'i32
+  FILE_ATTRIBUTE_READONLY* = 0x00000001'i32
+  FILE_ATTRIBUTE_HIDDEN* = 0x00000002'i32
+  FILE_ATTRIBUTE_SYSTEM* = 0x00000004'i32
+  FILE_ATTRIBUTE_DIRECTORY* = 0x00000010'i32
+  FILE_ATTRIBUTE_ARCHIVE* = 0x00000020'i32
+  FILE_ATTRIBUTE_DEVICE* = 0x00000040'i32
+  FILE_ATTRIBUTE_NORMAL* = 0x00000080'i32
+  FILE_ATTRIBUTE_TEMPORARY* = 0x00000100'i32
+  FILE_ATTRIBUTE_SPARSE_FILE* = 0x00000200'i32
+  FILE_ATTRIBUTE_REPARSE_POINT* = 0x00000400'i32
+  FILE_ATTRIBUTE_COMPRESSED* = 0x00000800'i32
+  FILE_ATTRIBUTE_OFFLINE* = 0x00001000'i32
+  FILE_ATTRIBUTE_NOT_CONTENT_INDEXED* = 0x00002000'i32
+
+  FILE_FLAG_FIRST_PIPE_INSTANCE* = 0x00080000'i32
+  FILE_FLAG_OPEN_NO_RECALL* = 0x00100000'i32
+  FILE_FLAG_OPEN_REPARSE_POINT* = 0x00200000'i32
+  FILE_FLAG_POSIX_SEMANTICS* = 0x01000000'i32
+  FILE_FLAG_BACKUP_SEMANTICS* = 0x02000000'i32
+  FILE_FLAG_DELETE_ON_CLOSE* = 0x04000000'i32
+  FILE_FLAG_SEQUENTIAL_SCAN* = 0x08000000'i32
+  FILE_FLAG_RANDOM_ACCESS* = 0x10000000'i32
+  FILE_FLAG_NO_BUFFERING* = 0x20000000'i32
+  FILE_FLAG_OVERLAPPED* = 0x40000000'i32
+  FILE_FLAG_WRITE_THROUGH* = 0x80000000'i32
 
   MAX_PATH* = 260
 
@@ -683,8 +698,6 @@ const
   FILE_MAP_WRITE* = 2'i32
   INVALID_FILE_SIZE* = -1'i32
 
-  FILE_FLAG_BACKUP_SEMANTICS* = 33554432'i32
-  FILE_FLAG_OPEN_REPARSE_POINT* = 0x00200000'i32
   DUPLICATE_SAME_ACCESS* = 2
   FILE_READ_DATA* = 0x00000001 # file & pipe
   FILE_WRITE_DATA* = 0x00000002 # file & pipe
@@ -695,6 +708,7 @@ const
   ERROR_PATH_NOT_FOUND* = 3
   ERROR_ACCESS_DENIED* = 5
   ERROR_NO_MORE_FILES* = 18
+  ERROR_LOCK_VIOLATION* = 33
   ERROR_HANDLE_EOF* = 38
   ERROR_BAD_ARGUMENTS* = 165
 
@@ -763,6 +777,9 @@ when not useWinUnicode:
 proc unmapViewOfFile*(lpBaseAddress: pointer): WINBOOL {.stdcall,
     dynlib: "kernel32", importc: "UnmapViewOfFile".}
 
+proc flushViewOfFile*(lpBaseAddress: pointer, dwNumberOfBytesToFlush: DWORD): WINBOOL {.
+  stdcall, dynlib: "kernel32", importc: "FlushViewOfFile".}
+
 type
   OVERLAPPED* {.pure, inheritable.} = object
     internal*: PULONG
@@ -785,7 +802,6 @@ type
 
 const
   ERROR_IO_PENDING* = 997 # a.k.a WSA_IO_PENDING
-  FILE_FLAG_OVERLAPPED* = 1073741824
   WSAECONNABORTED* = 10053
   WSAEADDRINUSE* = 10048
   WSAECONNRESET* = 10054
diff --git a/tests/stdlib/tmemfiles2.nim b/tests/stdlib/tmemfiles2.nim
index 7ea94cffc..d6cfa533a 100644
--- a/tests/stdlib/tmemfiles2.nim
+++ b/tests/stdlib/tmemfiles2.nim
@@ -4,9 +4,10 @@ discard """
 Half read size: 10 Data: Hello'''
 """
 import memfiles, os
+const
+  fn = "test.mmap"
 var
   mm, mm_full, mm_half: MemFile
-  fn = "test.mmap"
   p: pointer
 
 if fileExists(fn): removeFile(fn)
diff --git a/tests/stdlib/tmemmapstreams.nim b/tests/stdlib/tmemmapstreams.nim
new file mode 100644
index 000000000..243574f1a
--- /dev/null
+++ b/tests/stdlib/tmemmapstreams.nim
@@ -0,0 +1,53 @@
+discard """
+  file: "tmemmapstreams.nim"
+  output: '''Created size: 10
+Position after writing: 5
+Position after writing one char: 6
+Peeked data: Hello
+Position after peeking: 0
+Readed data: Hello!
+Position after reading line: 7
+Position after setting position: 6
+Readed line: Hello!
+Position after reading line: 7'''
+"""
+import os, streams, memfiles
+const
+  fn = "test.mmapstream"
+var
+  mms: MemMapFileStream
+
+if fileExists(fn): removeFile(fn)
+
+# Create a new memory mapped file, data all zeros
+mms = newMemMapFileStream(fn, mode = fmReadWrite, fileSize = 10)
+mms.close()
+if fileExists(fn): echo "Created size: ", getFileSize(fn)
+
+# write, flush, peek, read
+mms = newMemMapFileStream(fn, mode = fmReadWrite)
+let s = "Hello"
+
+mms.write(s)
+mms.flush
+echo "Position after writing: ", mms.getPosition()
+mms.write('!')
+mms.flush
+echo "Position after writing one char: ", mms.getPosition()
+mms.close()
+
+mms = newMemMapFileStream(fn, mode = fmRead)
+echo "Peeked data: ", mms.peekStr(s.len)
+echo "Position after peeking: ", mms.getPosition()
+echo "Readed data: ", mms.readLine
+echo "Position after reading line: ", mms.getPosition()
+mms.setPosition(mms.getPosition() - 1)
+echo "Position after setting position: ", mms.getPosition()
+
+mms.setPosition(0)
+echo "Readed line: ", mms.readLine
+echo "Position after reading line: ", mms.getPosition()
+
+mms.close()
+
+if fileExists(fn): removeFile(fn)
-- 
cgit 1.4.1-2-gfad0


From 8bcaee1fdf2054f026d6fdb760f67624b6dfe0e6 Mon Sep 17 00:00:00 2001
From: Araq <rumpf_a@web.de>
Date: Fri, 6 Jul 2018 15:53:32 +0200
Subject: make tmemfile2 work again

---
 compiler/ccgexprs.nim       | 1 +
 lib/pure/memfiles.nim       | 3 ++-
 tests/stdlib/tmemfiles2.nim | 5 +++--
 3 files changed, 6 insertions(+), 3 deletions(-)

(limited to 'lib/pure/memfiles.nim')

diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim
index fdafa1927..9b31167e3 100644
--- a/compiler/ccgexprs.nim
+++ b/compiler/ccgexprs.nim
@@ -1769,6 +1769,7 @@ proc convStrToCStr(p: BProc, n: PNode, d: var TLoc) =
   initLocExpr(p, n.sons[0], a)
   putIntoDest(p, d, n,
               ropecg(p.module, "#nimToCStringConv($1)", [rdLoc(a)]),
+#                "($1 ? $1->data : (NCSTRING)\"\")" % [a.rdLoc],
               a.storage)
 
 proc convCStrToStr(p: BProc, n: PNode, d: var TLoc) =
diff --git a/lib/pure/memfiles.nim b/lib/pure/memfiles.nim
index c7b8ebbd8..bda0ecb77 100644
--- a/lib/pure/memfiles.nim
+++ b/lib/pure/memfiles.nim
@@ -399,7 +399,8 @@ iterator lines*(mfile: MemFile, buf: var TaintedString, delim='\l', eat='\r'): T
 
   for ms in memSlices(mfile, delim, eat):
     setLen(buf.string, ms.size)
-    copyMem(buf.cstring, ms.data, ms.size)
+    if ms.size > 0:
+      copyMem(addr buf[0], ms.data, ms.size)
     yield buf
 
 iterator lines*(mfile: MemFile, delim='\l', eat='\r'): TaintedString {.inline.} =
diff --git a/tests/stdlib/tmemfiles2.nim b/tests/stdlib/tmemfiles2.nim
index d6cfa533a..a6f557e85 100644
--- a/tests/stdlib/tmemfiles2.nim
+++ b/tests/stdlib/tmemfiles2.nim
@@ -18,8 +18,9 @@ mm.close()
 
 # read, change
 mm_full = memfiles.open(fn, mode = fmWrite, mappedSize = -1, allowRemap = true)
-echo "Full read size: ",mm_full.size
+let size = mm_full.size
 p = mm_full.mapMem(fmReadWrite, 20, 0)
+echo "Full read size: ", size
 var p2 = cast[cstring](p)
 p2[0] = 'H'
 p2[1] = 'e'
@@ -32,7 +33,7 @@ mm_full.close()
 
 # read half, and verify data change
 mm_half = memfiles.open(fn, mode = fmRead, mappedSize = 10)
-echo "Half read size: ",mm_half.size, " Data: ", cast[cstring](mm_half.mem)
+echo "Half read size: ", mm_half.size, " Data: ", cast[cstring](mm_half.mem)
 mm_half.close()
 
 if fileExists(fn): removeFile(fn)
-- 
cgit 1.4.1-2-gfad0


From dfe3f160227dadd5d93bd6c697106e71899eccce Mon Sep 17 00:00:00 2001
From: Yuriy Glukhov <yglukhov@users.noreply.github.com>
Date: Fri, 13 Jul 2018 18:41:59 +0300
Subject: Don't depend on string.h in codegen (#8299)

---
 compiler/ccgexprs.nim | 34 +++++++++++++---------------------
 compiler/cgen.nim     | 13 +++----------
 lib/pure/memfiles.nim |  3 +--
 lib/system.nim        | 34 +++++++++++++++++-----------------
 lib/system/alloc.nim  |  4 ++--
 lib/system/ansi_c.nim |  2 ++
 lib/system/memory.nim | 47 +++++++++++++++++++++++++++++++++++++++++++++++
 lib/system/sysio.nim  |  2 +-
 lib/system/sysstr.nim |  2 +-
 9 files changed, 87 insertions(+), 54 deletions(-)
 create mode 100644 lib/system/memory.nim

(limited to 'lib/pure/memfiles.nim')

diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim
index 9b31167e3..2e9fb2cc2 100644
--- a/compiler/ccgexprs.nim
+++ b/compiler/ccgexprs.nim
@@ -257,9 +257,8 @@ proc genGenericAsgn(p: BProc, dest, src: TLoc, flags: TAssignmentFlags) =
   if needToCopy notin flags or
       tfShallow in skipTypes(dest.t, abstractVarRange).flags:
     if dest.storage == OnStack or not usesNativeGC(p.config):
-      useStringh(p.module)
       linefmt(p, cpsStmts,
-           "memcpy((void*)$1, (NIM_CONST void*)$2, sizeof($3));$n",
+           "#nimCopyMem((void*)$1, (NIM_CONST void*)$2, sizeof($3));$n",
            addrLoc(p.config, dest), addrLoc(p.config, src), rdLoc(dest))
     else:
       linefmt(p, cpsStmts, "#genericShallowAssign((void*)$1, (void*)$2, $3);$n",
@@ -336,9 +335,8 @@ proc genAssignment(p: BProc, dest, src: TLoc, flags: TAssignmentFlags) =
     if needsComplexAssignment(dest.t):
       genGenericAsgn(p, dest, src, flags)
     else:
-      useStringh(p.module)
       linefmt(p, cpsStmts,
-           "memcpy((void*)$1, (NIM_CONST void*)$2, sizeof($3));$n",
+           "#nimCopyMem((void*)$1, (NIM_CONST void*)$2, sizeof($3));$n",
            rdLoc(dest), rdLoc(src), getTypeDesc(p.module, dest.t))
   of tyOpenArray, tyVarargs:
     # open arrays are always on the stack - really? What if a sequence is
@@ -349,16 +347,14 @@ proc genAssignment(p: BProc, dest, src: TLoc, flags: TAssignmentFlags) =
            addrLoc(p.config, dest), addrLoc(p.config, src),
            genTypeInfo(p.module, dest.t, dest.lode.info))
     else:
-      useStringh(p.module)
       linefmt(p, cpsStmts,
-           # bug #4799, keep the memcpy for a while
-           #"memcpy((void*)$1, (NIM_CONST void*)$2, sizeof($1[0])*$1Len_0);$n",
+           # bug #4799, keep the nimCopyMem for a while
+           #"#nimCopyMem((void*)$1, (NIM_CONST void*)$2, sizeof($1[0])*$1Len_0);$n",
            "$1 = $2;$n",
            rdLoc(dest), rdLoc(src))
   of tySet:
     if mapType(p.config, ty) == ctArray:
-      useStringh(p.module)
-      linefmt(p, cpsStmts, "memcpy((void*)$1, (NIM_CONST void*)$2, $3);$n",
+      linefmt(p, cpsStmts, "#nimCopyMem((void*)$1, (NIM_CONST void*)$2, $3);$n",
               rdLoc(dest), rdLoc(src), rope(getSize(p.config, dest.t)))
     else:
       linefmt(p, cpsStmts, "$1 = $2;$n", rdLoc(dest), rdLoc(src))
@@ -403,8 +399,7 @@ proc genDeepCopy(p: BProc; dest, src: TLoc) =
          genTypeInfo(p.module, dest.t, dest.lode.info))
   of tySet:
     if mapType(p.config, ty) == ctArray:
-      useStringh(p.module)
-      linefmt(p, cpsStmts, "memcpy((void*)$1, (NIM_CONST void*)$2, $3);$n",
+      linefmt(p, cpsStmts, "#nimCopyMem((void*)$1, (NIM_CONST void*)$2, $3);$n",
               rdLoc(dest), rdLoc(src), rope(getSize(p.config, dest.t)))
     else:
       linefmt(p, cpsStmts, "$1 = $2;$n", rdLoc(dest), rdLoc(src))
@@ -1481,9 +1476,8 @@ proc genArrayLen(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
     if op == mHigh: unaryExpr(p, e, d, "($1Len_0-1)")
     else: unaryExpr(p, e, d, "$1Len_0")
   of tyCString:
-    useStringh(p.module)
-    if op == mHigh: unaryExpr(p, e, d, "($1 ? (strlen($1)-1) : -1)")
-    else: unaryExpr(p, e, d, "($1 ? strlen($1) : 0)")
+    if op == mHigh: unaryExpr(p, e, d, "($1 ? (#nimCStrLen($1)-1) : -1)")
+    else: unaryExpr(p, e, d, "($1 ? #nimCStrLen($1) : 0)")
   of tyString:
     if not p.module.compileToCpp:
       if op == mHigh: unaryExpr(p, e, d, "($1 ? ($1->Sup.len-1) : -1)")
@@ -1632,7 +1626,7 @@ proc genSetOp(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
         "  $3 = (($4[$1] & ~ $5[$1]) == 0);$n" &
         "  if (!$3) break;}$n", "for ($1 = 0; $1 < $2; $1++) { $n" &
         "  $3 = (($4[$1] & ~ $5[$1]) == 0);$n" & "  if (!$3) break;}$n" &
-        "if ($3) $3 = (memcmp($4, $5, $2) != 0);$n",
+        "if ($3) $3 = (#nimCmpMem($4, $5, $2) != 0);$n",
       "&", "|", "& ~", "^"]
   var a, b, i: TLoc
   var setType = skipTypes(e.sons[1].typ, abstractVar)
@@ -1671,11 +1665,10 @@ proc genSetOp(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
       initLocExpr(p, e.sons[1], a)
       initLocExpr(p, e.sons[2], b)
       if d.k == locNone: getTemp(p, getSysType(p.module.g.graph, unknownLineInfo(), tyBool), d)
-      lineF(p, cpsStmts, lookupOpr[op],
+      linefmt(p, cpsStmts, lookupOpr[op],
            [rdLoc(i), rope(size), rdLoc(d), rdLoc(a), rdLoc(b)])
     of mEqSet:
-      useStringh(p.module)
-      binaryExprChar(p, e, d, "(memcmp($1, $2, " & $(size) & ")==0)")
+      binaryExprChar(p, e, d, "(#nimCmpMem($1, $2, " & $(size) & ")==0)")
     of mMulSet, mPlusSet, mMinusSet, mSymDiffSet:
       # we inline the simple for loop for better code generation:
       getTemp(p, getSysType(p.module.g.graph, unknownLineInfo(), tyInt), i) # our counter
@@ -1934,7 +1927,7 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
 proc genSetConstr(p: BProc, e: PNode, d: var TLoc) =
   # example: { a..b, c, d, e, f..g }
   # we have to emit an expression of the form:
-  # memset(tmp, 0, sizeof(tmp)); inclRange(tmp, a, b); incl(tmp, c);
+  # nimZeroMem(tmp, sizeof(tmp)); inclRange(tmp, a, b); incl(tmp, c);
   # incl(tmp, d); incl(tmp, e); inclRange(tmp, f, g);
   var
     a, b, idx: TLoc
@@ -1944,8 +1937,7 @@ proc genSetConstr(p: BProc, e: PNode, d: var TLoc) =
     if d.k == locNone: getTemp(p, e.typ, d)
     if getSize(p.config, e.typ) > 8:
       # big set:
-      useStringh(p.module)
-      lineF(p, cpsStmts, "memset($1, 0, sizeof($2));$n",
+      linefmt(p, cpsStmts, "#nimZeroMem($1, sizeof($2));$n",
           [rdLoc(d), getTypeDesc(p.module, e.typ)])
       for it in e.sons:
         if it.kind == nkRange:
diff --git a/compiler/cgen.nim b/compiler/cgen.nim
index 32891c62c..ea82c496f 100644
--- a/compiler/cgen.nim
+++ b/compiler/cgen.nim
@@ -80,11 +80,6 @@ proc isSimpleConst(typ: PType): bool =
       {tyTuple, tyObject, tyArray, tySet, tySequence} and not
       (t.kind == tyProc and t.callConv == ccClosure)
 
-proc useStringh(m: BModule) =
-  if includesStringh notin m.flags:
-    incl m.flags, includesStringh
-    m.includeHeader("<string.h>")
-
 proc useHeader(m: BModule, sym: PSym) =
   if lfHeader in sym.loc.flags:
     assert(sym.annex != nil)
@@ -320,10 +315,9 @@ proc resetLoc(p: BProc, loc: var TLoc) =
       # field, so disabling this should be safe:
       genObjectInit(p, cpsStmts, loc.t, loc, true)
     else:
-      useStringh(p.module)
       # array passed as argument decayed into pointer, bug #7332
       # so we use getTypeDesc here rather than rdLoc(loc)
-      linefmt(p, cpsStmts, "memset((void*)$1, 0, sizeof($2));$n",
+      linefmt(p, cpsStmts, "#nimZeroMem((void*)$1, sizeof($2));$n",
               addrLoc(p.config, loc), getTypeDesc(p.module, loc.t))
       # XXX: We can be extra clever here and call memset only
       # on the bytes following the m_type field?
@@ -336,11 +330,10 @@ proc constructLoc(p: BProc, loc: TLoc, isTemp = false) =
       getTypeDesc(p.module, typ))
   else:
     if not isTemp or containsGarbageCollectedRef(loc.t):
-      # don't use memset for temporary values for performance if we can
+      # don't use nimZeroMem for temporary values for performance if we can
       # avoid it:
       if not isImportedCppType(typ):
-        useStringh(p.module)
-        linefmt(p, cpsStmts, "memset((void*)$1, 0, sizeof($2));$n",
+        linefmt(p, cpsStmts, "#nimZeroMem((void*)$1, sizeof($2));$n",
                 addrLoc(p.config, loc), getTypeDesc(p.module, typ))
     genObjectInit(p, cpsStmts, loc.t, loc, true)
 
diff --git a/lib/pure/memfiles.nim b/lib/pure/memfiles.nim
index bda0ecb77..0249b7413 100644
--- a/lib/pure/memfiles.nim
+++ b/lib/pure/memfiles.nim
@@ -322,8 +322,7 @@ type MemSlice* = object  ## represent slice of a MemFile for iteration over deli
 
 proc `==`*(x, y: MemSlice): bool =
   ## Compare a pair of MemSlice for strict equality.
-  proc memcmp(a, b: pointer, n:int):int {.importc: "memcmp",header: "string.h".}
-  result = (x.size == y.size and memcmp(x.data, y.data, x.size) == 0)
+  result = (x.size == y.size and equalMem(x.data, y.data, x.size))
 
 proc `$`*(ms: MemSlice): string {.inline.} =
   ## Return a Nim string built from a MemSlice.
diff --git a/lib/system.nim b/lib/system.nim
index 10f1df150..fed02f6d0 100644
--- a/lib/system.nim
+++ b/lib/system.nim
@@ -2903,6 +2903,22 @@ when not defined(JS): #and not defined(nimscript):
                        ## useful for low-level file access
 
   include "system/ansi_c"
+  include "system/memory"
+
+  proc zeroMem(p: pointer, size: Natural) =
+    nimZeroMem(p, size)
+    when declared(memTrackerOp):
+      memTrackerOp("zeroMem", p, size)
+  proc copyMem(dest, source: pointer, size: Natural) =
+    nimCopyMem(dest, source, size)
+    when declared(memTrackerOp):
+      memTrackerOp("copyMem", dest, size)
+  proc moveMem(dest, source: pointer, size: Natural) =
+    c_memmove(dest, source, size)
+    when declared(memTrackerOp):
+      memTrackerOp("moveMem", dest, size)
+  proc equalMem(a, b: pointer, size: Natural): bool =
+    nimCmpMem(a, b, size) == 0
 
   proc cmp(x, y: string): int =
     when nimvm:
@@ -2911,7 +2927,7 @@ when not defined(JS): #and not defined(nimscript):
       else: result = 0
     else:
       let minlen = min(x.len, y.len)
-      result = int(c_memcmp(x.cstring, y.cstring, minlen.csize))
+      result = int(nimCmpMem(x.cstring, y.cstring, minlen.csize))
       if result == 0:
         result = x.len - y.len
 
@@ -3200,22 +3216,6 @@ when not defined(JS): #and not defined(nimscript):
     when defined(memtracker):
       include "system/memtracker"
 
-    when not defined(nimscript):
-      proc zeroMem(p: pointer, size: Natural) =
-        c_memset(p, 0, size)
-        when declared(memTrackerOp):
-          memTrackerOp("zeroMem", p, size)
-      proc copyMem(dest, source: pointer, size: Natural) =
-        c_memcpy(dest, source, size)
-        when declared(memTrackerOp):
-          memTrackerOp("copyMem", dest, size)
-      proc moveMem(dest, source: pointer, size: Natural) =
-        c_memmove(dest, source, size)
-        when declared(memTrackerOp):
-          memTrackerOp("moveMem", dest, size)
-      proc equalMem(a, b: pointer, size: Natural): bool =
-        c_memcmp(a, b, size) == 0
-
     when hostOS == "standalone":
       include "system/embedded"
     else:
diff --git a/lib/system/alloc.nim b/lib/system/alloc.nim
index 6aef4f411..95becde22 100644
--- a/lib/system/alloc.nim
+++ b/lib/system/alloc.nim
@@ -830,7 +830,7 @@ proc rawDealloc(a: var MemRegion, p: pointer) =
     c.freeList = f
     when overwriteFree:
       # set to 0xff to check for usage after free bugs:
-      c_memset(cast[pointer](cast[int](p) +% sizeof(FreeCell)), -1'i32,
+      nimSetMem(cast[pointer](cast[int](p) +% sizeof(FreeCell)), -1'i32,
                s -% sizeof(FreeCell))
     # check if it is not in the freeSmallChunks[s] list:
     if c.free < s:
@@ -847,7 +847,7 @@ proc rawDealloc(a: var MemRegion, p: pointer) =
                s == 0, "rawDealloc 2")
   else:
     # set to 0xff to check for usage after free bugs:
-    when overwriteFree: c_memset(p, -1'i32, c.size -% bigChunkOverhead())
+    when overwriteFree: nimSetMem(p, -1'i32, c.size -% bigChunkOverhead())
     # free big chunk
     var c = cast[PBigChunk](c)
     dec a.occ, c.size
diff --git a/lib/system/ansi_c.nim b/lib/system/ansi_c.nim
index 52cb15e39..4d21f8747 100644
--- a/lib/system/ansi_c.nim
+++ b/lib/system/ansi_c.nim
@@ -25,6 +25,8 @@ proc c_memset(p: pointer, value: cint, size: csize): pointer {.
   importc: "memset", header: "<string.h>", discardable.}
 proc c_strcmp(a, b: cstring): cint {.
   importc: "strcmp", header: "<string.h>", noSideEffect.}
+proc c_strlen(a: cstring): csize {.
+  importc: "strlen", header: "<string.h>", noSideEffect.}
 
 when defined(linux) and defined(amd64):
   type
diff --git a/lib/system/memory.nim b/lib/system/memory.nim
new file mode 100644
index 000000000..f86fd4696
--- /dev/null
+++ b/lib/system/memory.nim
@@ -0,0 +1,47 @@
+const useLibC = not defined(nimNoLibc)
+
+proc nimCopyMem(dest, source: pointer, size: Natural) {.compilerproc, inline.} =
+  when useLibC:
+    c_memcpy(dest, source, size)
+  else:
+    let d = cast[ptr UncheckedArray[byte]](dest)
+    let s = cast[ptr UncheckedArray[byte]](source)
+    var i = 0
+    while i < size:
+      d[i] = s[i]
+      inc i
+
+proc nimSetMem(a: pointer, v: cint, size: Natural) {.inline.} =
+  when useLibC:
+    c_memset(a, v, size)
+  else:
+    let a = cast[ptr UncheckedArray[byte]](a)
+    var i = 0
+    let v = cast[byte](v)
+    while i < size:
+      a[i] = v
+      inc i
+
+proc nimZeroMem(p: pointer, size: Natural) {.compilerproc, inline.} =
+  nimSetMem(p, 0, size)
+
+proc nimCmpMem(a, b: pointer, size: Natural): cint {.compilerproc, inline.} =
+  when useLibC:
+    c_memcmp(a, b, size)
+  else:
+    let a = cast[ptr UncheckedArray[byte]](a)
+    let b = cast[ptr UncheckedArray[byte]](b)
+    var i = 0
+    while i < size:
+      let d = a[i].cint - b[i].cint
+      if d != 0: return d
+      inc i
+
+proc nimCStrLen(a: cstring): csize {.compilerproc, inline.} =
+  when useLibC:
+    c_strlen(a)
+  else:
+    var a = cast[ptr byte](a)
+    while a[] != 0:
+      a = cast[ptr byte](cast[uint](a) + 1)
+      inc result
diff --git a/lib/system/sysio.nim b/lib/system/sysio.nim
index f3a576be0..2ece56916 100644
--- a/lib/system/sysio.nim
+++ b/lib/system/sysio.nim
@@ -154,7 +154,7 @@ proc readLine(f: File, line: var TaintedString): bool =
   while true:
     # memset to \L so that we can tell how far fgets wrote, even on EOF, where
     # fgets doesn't append an \L
-    c_memset(addr line.string[pos], '\L'.ord, sp)
+    nimSetMem(addr line.string[pos], '\L'.ord, sp)
     var fgetsSuccess = c_fgets(addr line.string[pos], sp, f) != nil
     if not fgetsSuccess: checkErr(f)
     let m = c_memchr(addr line.string[pos], '\L'.ord, sp)
diff --git a/lib/system/sysstr.nim b/lib/system/sysstr.nim
index 90ae91cf5..8e9e18b05 100644
--- a/lib/system/sysstr.nim
+++ b/lib/system/sysstr.nim
@@ -32,7 +32,7 @@ proc cmpStrings(a, b: NimString): int {.inline, compilerProc.} =
     let blen = b.len
   let minlen = min(alen, blen)
   if minlen > 0:
-    result = c_memcmp(addr a.data, addr b.data, minlen.csize)
+    result = nimCmpMem(addr a.data, addr b.data, minlen.csize)
     if result == 0:
       result = alen - blen
   else:
-- 
cgit 1.4.1-2-gfad0


From cec89d835494bda07d7b27636a24544460574904 Mon Sep 17 00:00:00 2001
From: c-blake <c-blake@users.noreply.github.com>
Date: Tue, 4 Sep 2018 09:26:34 -0400
Subject: This has been unnecessary as long as rawNewObj has called zeroMem,
 (#8867)

and more recently indexing past the Nim-logical end has become
illegal making this line cause a crash.
---
 lib/pure/memfiles.nim | 1 -
 1 file changed, 1 deletion(-)

(limited to 'lib/pure/memfiles.nim')

diff --git a/lib/pure/memfiles.nim b/lib/pure/memfiles.nim
index 0249b7413..9fccd08d4 100644
--- a/lib/pure/memfiles.nim
+++ b/lib/pure/memfiles.nim
@@ -328,7 +328,6 @@ proc `$`*(ms: MemSlice): string {.inline.} =
   ## Return a Nim string built from a MemSlice.
   var buf = newString(ms.size)
   copyMem(addr(buf[0]), ms.data, ms.size)
-  buf[ms.size] = '\0'
   result = buf
 
 iterator memSlices*(mfile: MemFile, delim='\l', eat='\r'): MemSlice {.inline.} =
-- 
cgit 1.4.1-2-gfad0