diff options
| -rw-r--r-- | lib/posix/posix.nim | 11 | ||||
| -rw-r--r-- | lib/pure/memfiles.nim | 123 | ||||
| -rw-r--r-- | tests/stdlib/tmemfiles2.nim | 5 |
3 files changed, 92 insertions, 47 deletions
diff --git a/lib/posix/posix.nim b/lib/posix/posix.nim index 43346608d..aee9308a6 100644 --- a/lib/posix/posix.nim +++ b/lib/posix/posix.nim @@ -195,8 +195,14 @@ proc open*(a1: cstring, a2: cint, mode: Mode | cint = 0.Mode): cint {.inline.} = proc posix_fadvise*(a1: cint, a2, a3: Off, a4: cint): cint {. importc, header: "<fcntl.h>".} -proc posix_fallocate*(a1: cint, a2, a3: Off): cint {. - importc, header: "<fcntl.h>".} + +proc ftruncate*(a1: cint, a2: Off): cint {.importc, header: "<unistd.h>".} +when defined(osx): # 2001 POSIX evidently does not concern Apple + proc posix_fallocate*(a1: cint, a2, a3: Off): cint = + ftruncate(a1, a2 + a3) # Set size to off + len, max offset +else: # TODO: Use fcntl(fd, F_PREALLOCATE, ..) above + proc posix_fallocate*(a1: cint, a2, a3: Off): cint {. + importc, header: "<fcntl.h>".} when not defined(haiku) and not defined(openbsd): proc fmtmsg*(a1: int, a2: cstring, a3: cint, @@ -511,7 +517,6 @@ proc fpathconf*(a1, a2: cint): int {.importc, header: "<unistd.h>".} proc fsync*(a1: cint): cint {.importc, header: "<unistd.h>".} ## synchronize a file's buffer cache to the storage device -proc ftruncate*(a1: cint, a2: Off): cint {.importc, header: "<unistd.h>".} proc getcwd*(a1: cstring, a2: int): cstring {.importc, header: "<unistd.h>", sideEffect.} proc getuid*(): Uid {.importc, header: "<unistd.h>", sideEffect.} ## returns the real user ID of the calling process diff --git a/lib/pure/memfiles.nim b/lib/pure/memfiles.nim index 9ff8bcd74..58c4888f1 100644 --- a/lib/pure/memfiles.nim +++ b/lib/pure/memfiles.nim @@ -35,6 +35,31 @@ proc newEIO(msg: string): ref IOError = new(result) result.msg = msg +proc setFileSize(fh: FileHandle, newFileSize = -1): OSErrorCode = + ## Set the size of open file pointed to by `fh` to `newFileSize` if != -1. + ## Space is only allocated if that is cheaper than writing to the file. This + ## routine returns the last OSErrorCode found rather than raising to support + ## old rollback/clean-up code style. [ Should maybe move to std/osfiles. ] + if newFileSize == -1: + return + when defined(windows): + var sizeHigh = int32(newFileSize shr 32) + let sizeLow = int32(newFileSize and 0xffffffff) + let status = setFilePointer(fh, sizeLow, addr(sizeHigh), FILE_BEGIN) + let lastErr = osLastError() + if (status == INVALID_SET_FILE_POINTER and lastErr.int32 != NO_ERROR) or + setEndOfFile(fh) == 0: + result = lastErr + else: + var e: cint # posix_fallocate truncates up when needed. + when declared(posix_fallocate): + while (e = posix_fallocate(fh, 0, newFileSize); e == EINTR): + discard + if e in [EINVAL, EOPNOTSUPP] and ftruncate(fh, newFileSize) == -1: + result = osLastError() # fallback arguable; Most portable, but allows SEGV + elif e != 0: + result = osLastError() + type MemFile* = object ## represents a memory mapped file mem*: pointer ## a pointer to the memory mapped file. The pointer @@ -182,17 +207,8 @@ proc open*(filename: string, mode: FileMode = fmRead, if result.fHandle == INVALID_HANDLE_VALUE: fail(osLastError(), "error opening file") - if newFileSize != -1: - var - sizeHigh = int32(newFileSize shr 32) - sizeLow = int32(newFileSize and 0xffffffff) - - var status = setFilePointer(result.fHandle, sizeLow, addr(sizeHigh), - FILE_BEGIN) - let lastErr = osLastError() - if (status == INVALID_SET_FILE_POINTER and lastErr.int32 != NO_ERROR) or - (setEndOfFile(result.fHandle) == 0): - fail(lastErr, "error setting file size") + if (let e = setFileSize(result.fHandle.FileHandle, newFileSize); + e != 0.OSErrorCode): fail(e, "error setting file size") # since the strings are always 'nil', we simply always call # CreateFileMappingW which should be slightly faster anyway: @@ -226,7 +242,7 @@ proc open*(filename: string, mode: FileMode = fmRead, result.wasOpened = true if not allowRemap and result.fHandle != INVALID_HANDLE_VALUE: - if closeHandle(result.fHandle) == 0: + if closeHandle(result.fHandle) != 0: result.fHandle = INVALID_HANDLE_VALUE else: @@ -249,9 +265,8 @@ proc open*(filename: string, mode: FileMode = fmRead, # Is there an exception that wraps it? fail(osLastError(), "error opening file") - if newFileSize != -1: - if ftruncate(result.handle, newFileSize) == -1: - fail(osLastError(), "error setting file size") + if (let e = setFileSize(result.handle.FileHandle, newFileSize); + e != 0.OSErrorCode): fail(e, "error setting file size") if mappedSize != -1: result.size = mappedSize @@ -306,34 +321,58 @@ proc flush*(f: var MemFile; attempts: Natural = 3) = if lastErr != EBUSY.OSErrorCode: raiseOSError(lastErr, "error flushing mapping") -when defined(posix) or defined(nimdoc): - proc resize*(f: var MemFile, newFileSize: int) {.raises: [IOError, OSError].} = - ## resize and re-map the file underlying an `allowRemap MemFile`. - ## **Note**: this assumes the entire file is mapped read-write at offset zero. - ## Also, the value of `.mem` will probably change. - ## **Note**: This is not (yet) available on Windows. - when defined(posix): - if f.handle == -1: - raise newException(IOError, - "Cannot resize MemFile opened with allowRemap=false") - if ftruncate(f.handle, newFileSize) == -1: - raiseOSError(osLastError()) - when defined(linux): #Maybe NetBSD, too? - #On Linux this can be over 100 times faster than a munmap,mmap cycle. - proc mremap(old: pointer; oldSize, newSize: csize_t; flags: cint): - pointer {.importc: "mremap", header: "<sys/mman.h>".} - let newAddr = mremap(f.mem, csize_t(f.size), csize_t(newFileSize), cint(1)) - if newAddr == cast[pointer](MAP_FAILED): - raiseOSError(osLastError()) - else: - if munmap(f.mem, f.size) != 0: - raiseOSError(osLastError()) - let newAddr = mmap(nil, newFileSize, PROT_READ or PROT_WRITE, - f.flags, f.handle, 0) - if newAddr == cast[pointer](MAP_FAILED): - raiseOSError(osLastError()) - f.mem = newAddr +proc resize*(f: var MemFile, newFileSize: int) {.raises: [IOError, OSError].} = + ## Resize & re-map the file underlying an `allowRemap MemFile`. If the OS/FS + ## supports it, file space is reserved to ensure room for new virtual pages. + ## Caller should wait often enough for `flush` to finish to limit use of + ## system RAM for write buffering, perhaps just prior to this call. + ## **Note**: this assumes the entire file is mapped read-write at offset 0. + ## Also, the value of `.mem` will probably change. + if newFileSize < 1: # Q: include system/bitmasks & use PageSize ? + raise newException(IOError, "Cannot resize MemFile to < 1 byte") + when defined(windows): + if not f.wasOpened: + raise newException(IOError, "Cannot resize unopened MemFile") + if f.fHandle == INVALID_HANDLE_VALUE: + raise newException(IOError, + "Cannot resize MemFile opened with allowRemap=false") + if unmapViewOfFile(f.mem) == 0 or closeHandle(f.mapHandle) == 0: # Un-do map + raiseOSError(osLastError()) + if newFileSize != f.size: # Seek to size & `setEndOfFile` => allocated. + if (let e = setFileSize(f.fHandle.FileHandle, newFileSize); + e != 0.OSErrorCode): raiseOSError(e) + f.mapHandle = createFileMappingW(f.fHandle, nil, PAGE_READWRITE, 0,0,nil) + if f.mapHandle == 0: # Re-do map + raiseOSError(osLastError()) + if (let m = mapViewOfFileEx(f.mapHandle, FILE_MAP_READ or FILE_MAP_WRITE, + 0, 0, WinSizeT(newFileSize), nil); m != nil): + f.mem = m f.size = newFileSize + else: + raiseOSError(osLastError()) + elif defined(posix): + if f.handle == -1: + raise newException(IOError, + "Cannot resize MemFile opened with allowRemap=false") + if newFileSize != f.size: + if (let e = setFileSize(f.handle.FileHandle, newFileSize); + e != 0.OSErrorCode): raiseOSError(e) + when defined(linux): #Maybe NetBSD, too? + # On Linux this can be over 100 times faster than a munmap,mmap cycle. + proc mremap(old: pointer; oldSize, newSize: csize_t; flags: cint): + pointer {.importc: "mremap", header: "<sys/mman.h>".} + let newAddr = mremap(f.mem, csize_t(f.size), csize_t(newFileSize), 1.cint) + if newAddr == cast[pointer](MAP_FAILED): + raiseOSError(osLastError()) + else: + if munmap(f.mem, f.size) != 0: + raiseOSError(osLastError()) + let newAddr = mmap(nil, newFileSize, PROT_READ or PROT_WRITE, + f.flags, f.handle, 0) + if newAddr == cast[pointer](MAP_FAILED): + raiseOSError(osLastError()) + f.mem = newAddr + f.size = newFileSize proc close*(f: var MemFile) = ## closes the memory mapped file `f`. All changes are written back to the diff --git a/tests/stdlib/tmemfiles2.nim b/tests/stdlib/tmemfiles2.nim index 6fee3c1ae..c79f85ebf 100644 --- a/tests/stdlib/tmemfiles2.nim +++ b/tests/stdlib/tmemfiles2.nim @@ -15,8 +15,9 @@ var if fileExists(fn): removeFile(fn) -# Create a new file, data all zeros -mm = memfiles.open(fn, mode = fmReadWrite, newFileSize = 20) +# Create a new file, data all zeros, starting at size 10 +mm = memfiles.open(fn, mode = fmReadWrite, newFileSize = 10, allowRemap=true) +mm.resize 20 # resize up to 20 mm.close() # read, change |