From e39f2a9283fc63f529d74acb0d50b0035d513e79 Mon Sep 17 00:00:00 2001 From: jcosborn Date: Sat, 17 Mar 2018 17:59:04 -0500 Subject: fix allocator corruption for large sizes (#7338) * fix allocator corruption for large sizes * allow large chunks to coalesce and added test case * use correct constants in MaxBigChunkSize --- tests/system/alloc.nim | 52 ------------------------------------------- tests/system/io.nim | 48 ---------------------------------------- tests/system/params.nim | 18 --------------- tests/system/talloc.nim | 57 ++++++++++++++++++++++++++++++++++++++++++++++++ tests/system/talloc2.nim | 37 +++++++++++++++++++++++++++++++ tests/system/tio.nim | 48 ++++++++++++++++++++++++++++++++++++++++ tests/system/tparams.nim | 18 +++++++++++++++ 7 files changed, 160 insertions(+), 118 deletions(-) delete mode 100644 tests/system/alloc.nim delete mode 100644 tests/system/io.nim delete mode 100644 tests/system/params.nim create mode 100644 tests/system/talloc.nim create mode 100644 tests/system/talloc2.nim create mode 100644 tests/system/tio.nim create mode 100644 tests/system/tparams.nim (limited to 'tests/system') diff --git a/tests/system/alloc.nim b/tests/system/alloc.nim deleted file mode 100644 index 7abefec2a..000000000 --- a/tests/system/alloc.nim +++ /dev/null @@ -1,52 +0,0 @@ -var x: ptr int - -x = cast[ptr int](alloc(7)) -assert x != nil -x = cast[ptr int](x.realloc(2)) -assert x != nil -x.dealloc() - -x = createU(int, 3) -assert x != nil -x.free() - -x = create(int, 4) -assert cast[ptr array[4, int]](x)[0] == 0 -assert cast[ptr array[4, int]](x)[1] == 0 -assert cast[ptr array[4, int]](x)[2] == 0 -assert cast[ptr array[4, int]](x)[3] == 0 - -x = x.resize(4) -assert x != nil -x.free() - -x = cast[ptr int](allocShared(100)) -assert x != nil -deallocShared(x) - -x = createSharedU(int, 3) -assert x != nil -x.freeShared() - -x = createShared(int, 3) -assert x != nil -assert cast[ptr array[3, int]](x)[0] == 0 -assert cast[ptr array[3, int]](x)[1] == 0 -assert cast[ptr array[3, int]](x)[2] == 0 - -assert x != nil -x = cast[ptr int](x.resizeShared(2)) -assert x != nil -x.freeShared() - -x = create(int, 10) -assert x != nil -x = x.resize(12) -assert x != nil -x.dealloc() - -x = createShared(int, 1) -assert x != nil -x = x.resizeShared(1) -assert x != nil -x.freeShared() diff --git a/tests/system/io.nim b/tests/system/io.nim deleted file mode 100644 index 3d4df806b..000000000 --- a/tests/system/io.nim +++ /dev/null @@ -1,48 +0,0 @@ -import - unittest, osproc, streams, os, strformat -const STRING_DATA = "Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet." -const TEST_FILE = "tests/testdata/string.txt" - -proc echoLoop(str: string): string = - result = "" - var process = startProcess(findExe("tests/system/helpers/readall_echo")) - var input = process.inputStream - input.write(str) - input.close() - var output = process.outputStream - discard process.waitForExit - while not output.atEnd: - result.add(output.readLine) - -suite "io": - suite "readAll": - test "stdin": - check: - echoLoop(STRING_DATA) == STRING_DATA - echoLoop(STRING_DATA[0..3999]) == STRING_DATA[0..3999] - test "file": - check: - readFile(TEST_FILE) == STRING_DATA - - -proc verifyFileSize(sz: int64) = - # issue 7121, large file size (2-4GB and >4Gb) - const fn = "tmpfile112358" - let size_in_mb = sz div 1_000_000 - - when defined(windows): - discard execProcess(&"fsutil file createnew {fn} {sz}" ) - else: - discard execProcess(&"dd if=/dev/zero of={fn} bs=1000000 count={size_in_mb}") - - doAssert os.getFileSize(fn) == sz # Verify OS filesize by string - - var f = open(fn) - doAssert f.getFileSize() == sz # Verify file handle filesize - f.close() - - os.removeFile(fn) - -#disable tests for automatic testers -#for s in [50_000_000'i64, 3_000_000_000, 5_000_000_000]: -# verifyFileSize(s) diff --git a/tests/system/params.nim b/tests/system/params.nim deleted file mode 100644 index 1358212f2..000000000 --- a/tests/system/params.nim +++ /dev/null @@ -1,18 +0,0 @@ -import os -import osproc -import parseopt2 -import sequtils - -let argv = commandLineParams() - -if argv == @[]: - # this won't work with spaces - assert execShellCmd(getAppFilename() & " \"foo bar\" --aa:bar=a --a=c:d --ab -c --a[baz]:doo") == 0 -else: - let f = toSeq(getopt()) - echo f.repr - assert f[0].kind == cmdArgument and f[0].key == "foo bar" and f[0].val == "" - assert f[1].kind == cmdLongOption and f[1].key == "aa" and f[1].val == "bar=a" - assert f[2].kind == cmdLongOption and f[2].key == "a=c" and f[2].val == "d" - assert f[3].kind == cmdLongOption and f[3].key == "ab" and f[3].val == "" - assert f[4].kind == cmdShortOption and f[4].key == "c" and f[4].val == "" diff --git a/tests/system/talloc.nim b/tests/system/talloc.nim new file mode 100644 index 000000000..18396041d --- /dev/null +++ b/tests/system/talloc.nim @@ -0,0 +1,57 @@ +var x: ptr int + +x = cast[ptr int](alloc(7)) +assert x != nil +x = cast[ptr int](x.realloc(2)) +assert x != nil +x.dealloc() + +x = createU(int, 3) +assert x != nil +x.dealloc() + +x = create(int, 4) +assert cast[ptr array[4, int]](x)[0] == 0 +assert cast[ptr array[4, int]](x)[1] == 0 +assert cast[ptr array[4, int]](x)[2] == 0 +assert cast[ptr array[4, int]](x)[3] == 0 + +x = x.resize(4) +assert x != nil +x.dealloc() + +x = cast[ptr int](allocShared(100)) +assert x != nil +deallocShared(x) + +x = createSharedU(int, 3) +assert x != nil +x.deallocShared() + +x = createShared(int, 3) +assert x != nil +assert cast[ptr array[3, int]](x)[0] == 0 +assert cast[ptr array[3, int]](x)[1] == 0 +assert cast[ptr array[3, int]](x)[2] == 0 + +assert x != nil +x = cast[ptr int](x.resizeShared(2)) +assert x != nil +x.deallocShared() + +x = create(int, 10) +assert x != nil +x = x.resize(12) +assert x != nil +x.dealloc() + +x = createShared(int, 1) +assert x != nil +x = x.resizeShared(1) +assert x != nil +x.deallocShared() + +x = cast[ptr int](alloc0(125 shl 23)) +dealloc(x) +x = cast[ptr int](alloc0(126 shl 23)) +dealloc(x) diff --git a/tests/system/talloc2.nim b/tests/system/talloc2.nim new file mode 100644 index 000000000..c8cab78a1 --- /dev/null +++ b/tests/system/talloc2.nim @@ -0,0 +1,37 @@ +const + nmax = 2*1024*1024*1024 + +proc test(n: int) = + var a = alloc0(9999) + var t = cast[ptr UncheckedArray[int8]](alloc(n)) + var b = alloc0(9999) + t[0] = 1 + t[1] = 2 + t[n-2] = 3 + t[n-1] = 4 + dealloc(a) + dealloc(t) + dealloc(b) + +# allocator adds 48 bytes to BigChunk +# BigChunk allocator edges at 2^n * (1 - s) for s = [1..32]/64 +proc test2(n: int) = + let d = n div 256 # cover edges and more + for i in countdown(128,1): + for j in [-4096, -64, -49, -48, -47, -32, 0, 4096]: + let b = n + j - i*d + if b>0 and b<=nmax: + test(b) + #echo b, ": ", getTotalMem(), " ", getOccupiedMem(), " ", getFreeMem() + +proc test3 = + var n = 1 + while n <= nmax: + test2(n) + n *= 2 + n = nmax + while n >= 1: + test2(n) + n = n div 2 + +test3() diff --git a/tests/system/tio.nim b/tests/system/tio.nim new file mode 100644 index 000000000..3d4df806b --- /dev/null +++ b/tests/system/tio.nim @@ -0,0 +1,48 @@ +import + unittest, osproc, streams, os, strformat +const STRING_DATA = "Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet." +const TEST_FILE = "tests/testdata/string.txt" + +proc echoLoop(str: string): string = + result = "" + var process = startProcess(findExe("tests/system/helpers/readall_echo")) + var input = process.inputStream + input.write(str) + input.close() + var output = process.outputStream + discard process.waitForExit + while not output.atEnd: + result.add(output.readLine) + +suite "io": + suite "readAll": + test "stdin": + check: + echoLoop(STRING_DATA) == STRING_DATA + echoLoop(STRING_DATA[0..3999]) == STRING_DATA[0..3999] + test "file": + check: + readFile(TEST_FILE) == STRING_DATA + + +proc verifyFileSize(sz: int64) = + # issue 7121, large file size (2-4GB and >4Gb) + const fn = "tmpfile112358" + let size_in_mb = sz div 1_000_000 + + when defined(windows): + discard execProcess(&"fsutil file createnew {fn} {sz}" ) + else: + discard execProcess(&"dd if=/dev/zero of={fn} bs=1000000 count={size_in_mb}") + + doAssert os.getFileSize(fn) == sz # Verify OS filesize by string + + var f = open(fn) + doAssert f.getFileSize() == sz # Verify file handle filesize + f.close() + + os.removeFile(fn) + +#disable tests for automatic testers +#for s in [50_000_000'i64, 3_000_000_000, 5_000_000_000]: +# verifyFileSize(s) diff --git a/tests/system/tparams.nim b/tests/system/tparams.nim new file mode 100644 index 000000000..1358212f2 --- /dev/null +++ b/tests/system/tparams.nim @@ -0,0 +1,18 @@ +import os +import osproc +import parseopt2 +import sequtils + +let argv = commandLineParams() + +if argv == @[]: + # this won't work with spaces + assert execShellCmd(getAppFilename() & " \"foo bar\" --aa:bar=a --a=c:d --ab -c --a[baz]:doo") == 0 +else: + let f = toSeq(getopt()) + echo f.repr + assert f[0].kind == cmdArgument and f[0].key == "foo bar" and f[0].val == "" + assert f[1].kind == cmdLongOption and f[1].key == "aa" and f[1].val == "bar=a" + assert f[2].kind == cmdLongOption and f[2].key == "a=c" and f[2].val == "d" + assert f[3].kind == cmdLongOption and f[3].key == "ab" and f[3].val == "" + assert f[4].kind == cmdShortOption and f[4].key == "c" and f[4].val == "" -- cgit 1.4.1-2-gfad0