1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
|
#
#
# Nim's Runtime Library
# (c) Copyright 2015 Nim Contributors
#
# See the file "copying.txt", included in this
# distribution, for details about the copyright.
#
## :Authors: Zahary Karadjov, Andreas Rumpf
##
## This module provides support for `memory mapped files`:idx:
## (Posix's `mmap`:idx:) on the different operating systems.
when defined(windows):
import winlean
elif defined(posix):
import posix
else:
{.error: "the memfiles module is not supported on your operating system!".}
import os
type
MemFile* = object ## represents a memory mapped file
mem*: pointer ## a pointer to the memory mapped file. The pointer
## can be used directly to change the contents of the
## file, if it was opened with write access.
size*: int ## size of the memory mapped file
when defined(windows):
fHandle: int
mapHandle: int
else:
handle: cint
{.deprecated: [TMemFile: MemFile].}
proc mapMem*(m: var MemFile, mode: FileMode = fmRead,
mappedSize = -1, offset = 0): pointer =
var readonly = mode == fmRead
when defined(windows):
result = mapViewOfFileEx(
m.mapHandle,
if readonly: FILE_MAP_READ else: FILE_MAP_WRITE,
int32(offset shr 32),
int32(offset and 0xffffffff),
if mappedSize == -1: 0 else: mappedSize,
nil)
if result == nil:
raiseOSError(osLastError())
else:
assert mappedSize > 0
result = mmap(
nil,
mappedSize,
if readonly: PROT_READ else: PROT_READ or PROT_WRITE,
if readonly: (MAP_PRIVATE or MAP_POPULATE) else: (MAP_SHARED or MAP_POPULATE),
m.handle, offset)
if result == cast[pointer](MAP_FAILED):
raiseOSError(osLastError())
proc unmapMem*(f: var MemFile, p: pointer, size: int) =
## unmaps the memory region ``(p, <p+size)`` of the mapped file `f`.
## All changes are written back to the file system, if `f` was opened
## with write access. ``size`` must be of exactly the size that was requested
## via ``mapMem``.
when defined(windows):
if unmapViewOfFile(p) == 0: raiseOSError(osLastError())
else:
if munmap(p, size) != 0: raiseOSError(osLastError())
proc open*(filename: string, mode: FileMode = fmRead,
mappedSize = -1, offset = 0, newFileSize = -1): MemFile =
## opens a memory mapped file. If this fails, ``EOS`` is raised.
## `newFileSize` can only be set if the file does not exist and is opened
## with write access (e.g., with fmReadWrite). `mappedSize` and `offset`
## can be used to map only a slice of the file. Example:
##
## .. code-block:: nim
## var
## mm, mm_full, mm_half: MemFile
##
## mm = memfiles.open("/tmp/test.mmap", mode = fmWrite, newFileSize = 1024) # Create a new file
## mm.close()
##
## # Read the whole file, would fail if newFileSize was set
## mm_full = memfiles.open("/tmp/test.mmap", mode = fmReadWrite, mappedSize = -1)
##
## # Read the first 512 bytes
## mm_half = memfiles.open("/tmp/test.mmap", mode = fmReadWrite, mappedSize = 512)
# The file can be resized only when write mode is used:
assert newFileSize == -1 or mode != fmRead
var readonly = mode == fmRead
template rollback =
result.mem = nil
result.size = 0
when defined(windows):
template fail(errCode: OSErrorCode, msg: expr) =
rollback()
if result.fHandle != 0: discard closeHandle(result.fHandle)
if result.mapHandle != 0: discard closeHandle(result.mapHandle)
raiseOSError(errCode)
# return false
#raise newException(EIO, msg)
template callCreateFile(winApiProc, filename: expr): expr =
winApiProc(
filename,
if readonly: GENERIC_READ else: GENERIC_ALL,
FILE_SHARE_READ,
nil,
if newFileSize != -1: CREATE_ALWAYS else: OPEN_EXISTING,
if readonly: FILE_ATTRIBUTE_READONLY else: FILE_ATTRIBUTE_TEMPORARY,
0)
when useWinUnicode:
result.fHandle = callCreateFile(createFileW, newWideCString(filename))
else:
result.fHandle = callCreateFile(createFileA, filename)
if result.fHandle == INVALID_HANDLE_VALUE:
fail(osLastError(), "error opening file")
if newFileSize != -1:
var
sizeHigh = int32(newFileSize shr 32)
sizeLow = int32(newFileSize and 0xffffffff)
var status = setFilePointer(result.fHandle, sizeLow, addr(sizeHigh),
FILE_BEGIN)
let lastErr = osLastError()
if (status == INVALID_SET_FILE_POINTER and lastErr.int32 != NO_ERROR) or
(setEndOfFile(result.fHandle) == 0):
fail(lastErr, "error setting file size")
# since the strings are always 'nil', we simply always call
# CreateFileMappingW which should be slightly faster anyway:
result.mapHandle = createFileMappingW(
result.fHandle, nil,
if readonly: PAGE_READONLY else: PAGE_READWRITE,
0, 0, nil)
if result.mapHandle == 0:
fail(osLastError(), "error creating mapping")
result.mem = mapViewOfFileEx(
result.mapHandle,
if readonly: FILE_MAP_READ else: FILE_MAP_WRITE,
int32(offset shr 32),
int32(offset and 0xffffffff),
if mappedSize == -1: 0 else: mappedSize,
nil)
if result.mem == nil:
fail(osLastError(), "error mapping view")
var hi, low: int32
low = getFileSize(result.fHandle, addr(hi))
if low == INVALID_FILE_SIZE:
fail(osLastError(), "error getting file size")
else:
var fileSize = (int64(hi) shr 32) or low
if mappedSize != -1: result.size = min(fileSize, mappedSize).int
else: result.size = fileSize.int
else:
template fail(errCode: OSErrorCode, msg: expr) =
rollback()
if result.handle != 0: discard close(result.handle)
raiseOSError(errCode)
var flags = if readonly: O_RDONLY else: O_RDWR
if newFileSize != -1:
flags = flags or O_CREAT or O_TRUNC
var permissions_mode = S_IRUSR or S_IWUSR
result.handle = open(filename, flags, permissions_mode)
else:
result.handle = open(filename, flags)
if result.handle == -1:
# XXX: errno is supposed to be set here
# Is there an exception that wraps it?
fail(osLastError(), "error opening file")
if newFileSize != -1:
if ftruncate(result.handle, newFileSize) == -1:
fail(osLastError(), "error setting file size")
if mappedSize != -1:
result.size = mappedSize
else:
var stat: TStat
if fstat(result.handle, stat) != -1:
# XXX: Hmm, this could be unsafe
# Why is mmap taking int anyway?
result.size = int(stat.st_size)
else:
fail(osLastError(), "error getting file size")
result.mem = mmap(
nil,
result.size,
if readonly: PROT_READ else: PROT_READ or PROT_WRITE,
if readonly: (MAP_PRIVATE or MAP_POPULATE) else: (MAP_SHARED or MAP_POPULATE),
result.handle,
offset)
if result.mem == cast[pointer](MAP_FAILED):
fail(osLastError(), "file mapping failed")
proc close*(f: var MemFile) =
## closes the memory mapped file `f`. All changes are written back to the
## file system, if `f` was opened with write access.
var error = false
var lastErr: OSErrorCode
when defined(windows):
if f.fHandle != INVALID_HANDLE_VALUE:
error = unmapViewOfFile(f.mem) == 0
lastErr = osLastError()
error = (closeHandle(f.mapHandle) == 0) or error
error = (closeHandle(f.fHandle) == 0) or error
else:
if f.handle != 0:
error = munmap(f.mem, f.size) != 0
lastErr = osLastError()
error = (close(f.handle) != 0) or error
f.size = 0
f.mem = nil
when defined(windows):
f.fHandle = 0
f.mapHandle = 0
else:
f.handle = 0
if error: raiseOSError(lastErr)
|