From 5ede29cd051b818c7c2fc926c4acba86ddb76a0e Mon Sep 17 00:00:00 2001 From: Simon Hafner Date: Thu, 3 Nov 2011 17:05:32 +0100 Subject: implemented readAll() --- lib/system.nim | 9 +++++--- lib/system/sysio.nim | 64 ++++++++++++++++++++++++++++++++++++---------------- 2 files changed, 51 insertions(+), 22 deletions(-) (limited to 'lib') diff --git a/lib/system.nim b/lib/system.nim index 8a99781cc..41100639f 100755 --- a/lib/system.nim +++ b/lib/system.nim @@ -1675,10 +1675,13 @@ when not defined(EcmaScript) and not defined(NimrodVM): proc FlushFile*(f: TFile) {.importc: "fflush", noDecl.} ## Flushes `f`'s buffer. + proc readAll*(file: TFile): TaintedString + ## Reads all data from the stream `file`. Raises an IO exception + ## in case of an Error + proc readFile*(filename: string): TaintedString - ## Opens a file named `filename` for reading. Then reads the - ## file's content completely into a string and - ## closes the file afterwards. Returns the string. + ## Opens a file named `filename` for reading. Then calls readAll + ## and closes the file afterwards. Returns the string. ## Raises an IO exception in case of an error. proc writeFile*(filename, content: string) diff --git a/lib/system/sysio.nim b/lib/system/sysio.nim index 890f526ea..6641d7218 100755 --- a/lib/system/sysio.nim +++ b/lib/system/sysio.nim @@ -24,6 +24,13 @@ proc putc(c: Char, stream: TFile) {.importc: "putc", nodecl.} proc fprintf(f: TFile, frmt: CString) {.importc: "fprintf", nodecl, varargs.} proc strlen(c: cstring): int {.importc: "strlen", nodecl.} + +# C routine that is used here: +proc fread(buf: Pointer, size, n: int, f: TFile): int {. + importc: "fread", noDecl.} +proc fseek(f: TFile, offset: clong, whence: int): int {. + importc: "fseek", noDecl.} +proc ftell(f: TFile): int {.importc: "ftell", noDecl.} proc setvbuf(stream: TFile, buf: pointer, typ, size: cint): cint {. importc, nodecl.} @@ -36,6 +43,9 @@ var IOFBF {.importc: "_IOFBF", nodecl.}: cint IONBF {.importc: "_IONBF", nodecl.}: cint +const + BUF_SIZE = 4000 + proc raiseEIO(msg: string) {.noinline, noreturn.} = raise newException(EIO, msg) @@ -82,21 +92,44 @@ proc write(f: TFile, c: Char) = putc(c, f) proc write(f: TFile, a: openArray[string]) = for x in items(a): write(f, x) +proc readAllBuffer(file: TFile): string = + # This proc is for TFile we want to read but don't know how many + # bytes we need to read before the buffer is empty. + result = "" + var buffer = newString(BUF_SIZE) + var bytesRead = BUF_SIZE + while bytesRead == BUF_SIZE: + bytesRead = readBuffer(file, addr(buffer[0]), BUF_SIZE) + result.add(buffer) + +proc readAllFile(file: TFile): string = + # We aquire the filesize beforehand and hope it doesn't change. + # Speeds things up. + var len = getFileSize(file) + if len >= high(int): + raiseEIO("file too big to fit in memory") + result = newString(int(len)) + if readBuffer(file, addr(result[0]), int(len)) != len: + raiseEIO("error while reading from file") + +proc hasDefinedLength(file: TFile): bool = + var oldPos = getFilePos(file) + discard fseek(file, 0, 2) # seek the end of the file + result = ftell(file) >= 0 + setFilePos(file, oldPos) + +proc readAll(file: TFile): TaintedString = + # Separate handling needed because we need to buffer when we + # don't know the overall length of the TFile. + if hasDefinedLength(file): + result = readAllBuffer(file).TaintedSTring + else: + result = readAllFile(file).TaintedString + proc readFile(filename: string): TaintedString = var f = open(filename) try: - var len = getFileSize(f) - if len < high(int): - when taintMode: - result = newString(int(len)).TaintedString - if readBuffer(f, addr(string(result)[0]), int(len)) != len: - raiseEIO("error while reading from file") - else: - result = newString(int(len)) - if readBuffer(f, addr(result[0]), int(len)) != len: - raiseEIO("error while reading from file") - else: - raiseEIO("file too big to fit in memory") + result = readAllFile(f) finally: close(f) @@ -157,13 +190,6 @@ proc open(f: var TFile, filehandle: TFileHandle, mode: TFileMode): bool = f = fdopen(filehandle, FormatOpen[mode]) result = f != nil -# C routine that is used here: -proc fread(buf: Pointer, size, n: int, f: TFile): int {. - importc: "fread", noDecl.} -proc fseek(f: TFile, offset: clong, whence: int): int {. - importc: "fseek", noDecl.} -proc ftell(f: TFile): int {.importc: "ftell", noDecl.} - proc fwrite(buf: Pointer, size, n: int, f: TFile): int {. importc: "fwrite", noDecl.} -- cgit 1.4.1-2-gfad0 From 24917aaf1ecd1e0d443861c37d3b2e9e7673f856 Mon Sep 17 00:00:00 2001 From: Simon Hafner Date: Fri, 30 Dec 2011 14:15:12 +0100 Subject: fixes for readAll --- lib/system.nim | 4 ++-- lib/system/sysio.nim | 39 ++++++++++++++++++++++----------------- tests/specials.nim | 7 +++++++ tests/system/io.nim | 10 +++++++--- 4 files changed, 38 insertions(+), 22 deletions(-) (limited to 'lib') diff --git a/lib/system.nim b/lib/system.nim index 41100639f..282e03add 100755 --- a/lib/system.nim +++ b/lib/system.nim @@ -1677,10 +1677,10 @@ when not defined(EcmaScript) and not defined(NimrodVM): proc readAll*(file: TFile): TaintedString ## Reads all data from the stream `file`. Raises an IO exception - ## in case of an Error + ## in case of an error proc readFile*(filename: string): TaintedString - ## Opens a file named `filename` for reading. Then calls readAll + ## Opens a file named `filename` for reading. Then calls `readAll` ## and closes the file afterwards. Returns the string. ## Raises an IO exception in case of an error. diff --git a/lib/system/sysio.nim b/lib/system/sysio.nim index 6641d7218..d012110f1 100755 --- a/lib/system/sysio.nim +++ b/lib/system/sysio.nim @@ -44,7 +44,7 @@ var IONBF {.importc: "_IONBF", nodecl.}: cint const - BUF_SIZE = 4000 + buf_size = 4000 proc raiseEIO(msg: string) {.noinline, noreturn.} = raise newException(EIO, msg) @@ -96,40 +96,45 @@ proc readAllBuffer(file: TFile): string = # This proc is for TFile we want to read but don't know how many # bytes we need to read before the buffer is empty. result = "" - var buffer = newString(BUF_SIZE) - var bytesRead = BUF_SIZE - while bytesRead == BUF_SIZE: - bytesRead = readBuffer(file, addr(buffer[0]), BUF_SIZE) + var buffer = newString(buf_size) + var bytesRead = buf_size + while bytesRead == buf_size: + bytesRead = readBuffer(file, addr(buffer[0]), buf_size) result.add(buffer) -proc readAllFile(file: TFile): string = +proc rawFileSize(file: TFile): int = + # this does not raise an error opposed to `getFileSize` + var oldPos = ftell(file) + discard fseek(file, 0, 2) # seek the end of the file + result = ftell(file) + discard fseek(file, clong(oldPos), 0) + +proc readAllFile(file: TFile, len: int): string = # We aquire the filesize beforehand and hope it doesn't change. # Speeds things up. - var len = getFileSize(file) if len >= high(int): raiseEIO("file too big to fit in memory") result = newString(int(len)) if readBuffer(file, addr(result[0]), int(len)) != len: raiseEIO("error while reading from file") -proc hasDefinedLength(file: TFile): bool = - var oldPos = getFilePos(file) - discard fseek(file, 0, 2) # seek the end of the file - result = ftell(file) >= 0 - setFilePos(file, oldPos) - +proc readAllFile(file: TFile): string = + var len = rawFileSize(file) + result = readAllFile(file, len) + proc readAll(file: TFile): TaintedString = # Separate handling needed because we need to buffer when we # don't know the overall length of the TFile. - if hasDefinedLength(file): - result = readAllBuffer(file).TaintedSTring + var len = rawFileSize(file) + if len >= 0: + result = readAllFile(file, len).TaintedSTring else: - result = readAllFile(file).TaintedString + result = readAllBuffer(file).TaintedString proc readFile(filename: string): TaintedString = var f = open(filename) try: - result = readAllFile(f) + result = readAllFile(f).TaintedString finally: close(f) diff --git a/tests/specials.nim b/tests/specials.nim index a1aa1bc5a..3da5b2074 100644 --- a/tests/specials.nim +++ b/tests/specials.nim @@ -128,12 +128,19 @@ proc rejectThreadTests(r: var TResults, options: string) = rejectSingleTest(r, "tests/threads/tthreadanalysis3", options) rejectSingleTest(r, "tests/threads/tthreadheapviolation1", options) +# ------------------------- IO tests ----------------------------------- + +proc runIOTests(r: var TResults, options: string) = + discard callCompiler(r"nimrod cc --hints:on $# $#", "tests/system/helpers/readall_echo", options) + runSingleTest(r, "tests/system/io", options) + # ------------------------- register special tests here ----------------------- proc runSpecialTests(r: var TResults, options: string) = runRodFiles(r, options) runDLLTests(r, options) runGCTests(r, options) runThreadTests(r, options & " --threads:on") + runIOTests(r, options) proc rejectSpecialTests(r: var TResults, options: string) = rejectThreadTests(r, options) diff --git a/tests/system/io.nim b/tests/system/io.nim index 1c3be47f4..4837f7093 100644 --- a/tests/system/io.nim +++ b/tests/system/io.nim @@ -1,11 +1,15 @@ +discard """output: '''[OK] stdin + +[OK] file'''""" + import - unittest, osproc, streams, os -const STRING_DATA = "Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet." + unittest, osproc, streams, os, readall_echo +const STRING_DATA = """Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet.""" const TEST_FILE = "tests/testdata/string" proc echoLoop(str: string): string = result = "" - var process = startProcess("tests/system/helpers/readall_echo") + var process = startProcess(os.addFileExt("tests/system/helpers/readall_echo", ExeExt)) var input = process.inputStream input.write(str) input.close() -- cgit 1.4.1-2-gfad0