diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/system/alloc.nim | 61 | ||||
| -rw-r--r-- | lib/system/gc.nim | 175 |
2 files changed, 51 insertions, 185 deletions
diff --git a/lib/system/alloc.nim b/lib/system/alloc.nim index e0fd53b7b..00a16e2bb 100644 --- a/lib/system/alloc.nim +++ b/lib/system/alloc.nim @@ -27,15 +27,14 @@ const type PTrunk = ptr Trunk - Trunk {.final.} = object + Trunk = object next: PTrunk # all nodes are connected with this pointer key: int # start address at bit 0 bits: array[0..IntsPerTrunk-1, int] # a bit vector TrunkBuckets = array[0..255, PTrunk] - IntSet {.final.} = object + IntSet = object data: TrunkBuckets -{.deprecated: [TIntSet: IntSet, TTrunk: Trunk, TTrunkBuckets: TrunkBuckets].} type AlignType = BiggestFloat @@ -64,8 +63,6 @@ type next, prev: PBigChunk # chunks of the same (or bigger) size align: int data: AlignType # start of usable memory -{.deprecated: [TAlignType: AlignType, TFreeCell: FreeCell, TBaseChunk: BaseChunk, - TBigChunk: BigChunk, TSmallChunk: SmallChunk].} template smallChunkOverhead(): expr = sizeof(SmallChunk)-sizeof(AlignType) template bigChunkOverhead(): expr = sizeof(BigChunk)-sizeof(AlignType) @@ -79,18 +76,18 @@ template bigChunkOverhead(): expr = sizeof(BigChunk)-sizeof(AlignType) type PLLChunk = ptr LLChunk - LLChunk {.pure.} = object ## *low-level* chunk + LLChunk = object ## *low-level* chunk size: int # remaining size acc: int # accumulator next: PLLChunk # next low-level chunk; only needed for dealloc PAvlNode = ptr AvlNode - AvlNode {.pure, final.} = object + AvlNode = object link: array[0..1, PAvlNode] # Left (0) and right (1) links key, upperBound: int level: int - MemRegion {.final, pure.} = object + MemRegion = object minLargeObj, maxLargeObj: int freeSmallChunks: array[0..SmallChunkSize div MemAlign-1, PSmallChunk] llmem: PLLChunk @@ -99,6 +96,7 @@ type freeChunksList: PBigChunk # XXX make this a datastructure with O(1) access chunkStarts: IntSet root, deleted, last, freeAvlNodes: PAvlNode + locked: bool # if locked, we cannot free pages. {.deprecated: [TLLChunk: LLChunk, TAvlNode: AvlNode, TMemRegion: MemRegion].} # shared: @@ -234,7 +232,8 @@ proc isSmallChunk(c: PChunk): bool {.inline.} = proc chunkUnused(c: PChunk): bool {.inline.} = result = not c.used -iterator allObjects(m: MemRegion): pointer {.inline.} = +iterator allObjects(m: var MemRegion): pointer {.inline.} = + m.locked = true for s in elements(m.chunkStarts): # we need to check here again as it could have been modified: if s in m.chunkStarts: @@ -252,6 +251,7 @@ iterator allObjects(m: MemRegion): pointer {.inline.} = else: let c = cast[PBigChunk](c) yield addr(c.data) + m.locked = false proc iterToProc*(iter: typed, envType: typedesc; procName: untyped) {. magic: "Plugin", compileTime.} @@ -385,7 +385,7 @@ proc freeBigChunk(a: var MemRegion, c: PBigChunk) = excl(a.chunkStarts, pageIndex(c)) c = cast[PBigChunk](le) - if c.size < ChunkOsReturn or doNotUnmap: + if c.size < ChunkOsReturn or doNotUnmap or a.locked: incl(a, a.chunkStarts, pageIndex(c)) updatePrevSize(a, c, c.size) listAdd(a.freeChunksList, c) @@ -442,26 +442,29 @@ proc getSmallChunk(a: var MemRegion): PSmallChunk = # ----------------------------------------------------------------------------- proc isAllocatedPtr(a: MemRegion, p: pointer): bool {.benign.} -proc allocInv(a: MemRegion): bool = - ## checks some (not all yet) invariants of the allocator's data structures. - for s in low(a.freeSmallChunks)..high(a.freeSmallChunks): - var c = a.freeSmallChunks[s] - while not (c == nil): - if c.next == c: - echo "[SYSASSERT] c.next == c" - return false - if not (c.size == s * MemAlign): - echo "[SYSASSERT] c.size != s * MemAlign" - return false - var it = c.freeList - while not (it == nil): - if not (it.zeroField == 0): - echo "[SYSASSERT] it.zeroField != 0" - c_printf("%ld %p\n", it.zeroField, it) +when true: + template allocInv(a: MemRegion): bool = true +else: + proc allocInv(a: MemRegion): bool = + ## checks some (not all yet) invariants of the allocator's data structures. + for s in low(a.freeSmallChunks)..high(a.freeSmallChunks): + var c = a.freeSmallChunks[s] + while not (c == nil): + if c.next == c: + echo "[SYSASSERT] c.next == c" return false - it = it.next - c = c.next - result = true + if not (c.size == s * MemAlign): + echo "[SYSASSERT] c.size != s * MemAlign" + return false + var it = c.freeList + while not (it == nil): + if not (it.zeroField == 0): + echo "[SYSASSERT] it.zeroField != 0" + c_printf("%ld %p\n", it.zeroField, it) + return false + it = it.next + c = c.next + result = true proc rawAlloc(a: var MemRegion, requestedSize: int): pointer = sysAssert(allocInv(a), "rawAlloc: begin") diff --git a/lib/system/gc.nim b/lib/system/gc.nim index 5c2170a17..ed5b2d1e9 100644 --- a/lib/system/gc.nim +++ b/lib/system/gc.nim @@ -1,7 +1,7 @@ # # # Nim's Runtime Library -# (c) Copyright 2015 Andreas Rumpf +# (c) Copyright 2016 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -9,13 +9,8 @@ # Garbage Collector # -# The basic algorithm is *Deferred Reference Counting* with cycle detection. -# This is achieved by combining a Deutsch-Bobrow garbage collector -# together with Christoper's partial mark-sweep garbage collector. -# -# Special care has been taken to avoid recursion as far as possible to avoid -# stack overflows when traversing deep datastructures. It is well-suited -# for soft real time applications (like games). +# Refcounting + Mark&Sweep. Complex algorithms avoided. +# Been there, done that, didn't work. when defined(nimCoroutines): import arch @@ -30,7 +25,7 @@ const # this seems to be a good value withRealTime = defined(useRealtimeGC) useMarkForDebug = defined(gcGenerational) - useBackupGc = false # use a simple M&S GC to collect + useBackupGc = true # use a simple M&S GC to collect # cycles instead of the complex # algorithm @@ -55,8 +50,7 @@ type WalkOp = enum waMarkGlobal, # part of the backup/debug mark&sweep waMarkPrecise, # part of the backup/debug mark&sweep - waZctDecRef, waPush, waCycleDecRef, waMarkGray, waScan, waScanBlack, - waCollectWhite #, waDebug + waZctDecRef, waPush Finalizer {.compilerproc.} = proc (self: pointer) {.nimcall, benign.} # A ref type can have a finalizer that is called before the object's @@ -87,7 +81,6 @@ type idGenerator: int zct: CellSeq # the zero count table decStack: CellSeq # cells in the stack that are to decref again - cycleRoots: CellSet tempStack: CellSeq # temporary stack for recursion elimination recGcLock: int # prevent recursion via finalizers; no thread lock when withRealTime: @@ -136,9 +129,6 @@ proc usrToCell(usr: pointer): PCell {.inline.} = # convert pointer to userdata to object (=pointer to refcount) result = cast[PCell](cast[ByteAddress](usr)-%ByteAddress(sizeof(Cell))) -proc canBeCycleRoot(c: PCell): bool {.inline.} = - result = ntfAcyclic notin c.typ.flags - proc extGetCellType(c: pointer): PNimType {.compilerproc.} = # used for code generation concerning debugging result = usrToCell(c).typ @@ -200,14 +190,16 @@ proc prepareDealloc(cell: PCell) = (cast[Finalizer](cell.typ.finalizer))(cellToUsr(cell)) dec(gch.recGcLock) +template beforeDealloc(gch: var GcHeap; c: PCell; msg: typed) = + when false: + for i in 0..gch.decStack.len-1: + if gch.decStack.d[i] == c: + sysAssert(false, msg) + proc rtlAddCycleRoot(c: PCell) {.rtl, inl.} = # we MUST access gch as a global here, because this crosses DLL boundaries! when hasThreadSupport and hasSharedHeap: acquireSys(HeapLock) - when cycleGC: - if c.color != rcPurple: - c.setColor(rcPurple) - incl(gch.cycleRoots, c) when hasThreadSupport and hasSharedHeap: releaseSys(HeapLock) @@ -224,19 +216,12 @@ proc decRef(c: PCell) {.inline.} = gcAssert(c.refcount >=% rcIncrement, "decRef") if --c.refcount: rtlAddZCT(c) - elif canbeCycleRoot(c): - # unfortunately this is necessary here too, because a cycle might just - # have been broken up and we could recycle it. - rtlAddCycleRoot(c) - #writeCell("decRef", c) proc incRef(c: PCell) {.inline.} = gcAssert(isAllocatedPtr(gch.region, c), "incRef: interiorPtr") c.refcount = c.refcount +% rcIncrement # and not colorMask #writeCell("incRef", c) - if canbeCycleRoot(c): - rtlAddCycleRoot(c) proc nimGCref(p: pointer) {.compilerProc, inline.} = incRef(usrToCell(p)) proc nimGCunref(p: pointer) {.compilerProc, inline.} = decRef(usrToCell(p)) @@ -306,7 +291,6 @@ proc initGC() = # init the rt init(gch.zct) init(gch.tempStack) - init(gch.cycleRoots) init(gch.decStack) when useMarkForDebug or useBackupGc: init(gch.marked) @@ -563,7 +547,7 @@ proc growObj(old: pointer, newsize: int, gch: var GcHeap): pointer = d[j] = res break dec(j) - if canbeCycleRoot(ol): excl(gch.cycleRoots, ol) + beforeDealloc(gch, ol, "growObj stack trash") rawDealloc(gch.region, ol) else: # we split the old refcount in 2 parts. XXX This is still not entirely @@ -597,54 +581,12 @@ proc freeCyclicCell(gch: var GcHeap, c: PCell) = when logGC: writeCell("cycle collector dealloc cell", c) when reallyDealloc: sysAssert(allocInv(gch.region), "free cyclic cell") + beforeDealloc(gch, c, "freeCyclicCell: stack trash") rawDealloc(gch.region, c) else: gcAssert(c.typ != nil, "freeCyclicCell") zeroMem(c, sizeof(Cell)) -proc markGray(s: PCell) = - if s.color != rcGray: - setColor(s, rcGray) - forAllChildren(s, waMarkGray) - -proc scanBlack(s: PCell) = - s.setColor(rcBlack) - forAllChildren(s, waScanBlack) - -proc scan(s: PCell) = - if s.color == rcGray: - if s.refcount >=% rcIncrement: - scanBlack(s) - else: - s.setColor(rcWhite) - forAllChildren(s, waScan) - -proc collectWhite(s: PCell) = - # This is a hacky way to deal with the following problem (bug #1796) - # Consider this content in cycleRoots: - # x -> a; y -> a where 'a' is an acyclic object so not included in - # cycleRoots itself. Then 'collectWhite' used to free 'a' twice. The - # 'isAllocatedPtr' check prevents this. This also means we do not need - # to query 's notin gch.cycleRoots' at all. - if isAllocatedPtr(gch.region, s) and s.color == rcWhite: - s.setColor(rcBlack) - forAllChildren(s, waCollectWhite) - freeCyclicCell(gch, s) - -proc markRoots(gch: var GcHeap) = - var tabSize = 0 - for s in elements(gch.cycleRoots): - #writeCell("markRoot", s) - inc tabSize - if s.color == rcPurple and s.refcount >=% rcIncrement: - markGray(s) - else: - excl(gch.cycleRoots, s) - # (s.color == rcBlack and rc == 0) as 1 condition: - if s.refcount == 0: - freeCyclicCell(gch, s) - gch.stat.cycleTableSize = max(gch.stat.cycleTableSize, tabSize) - when useBackupGc: proc sweep(gch: var GcHeap) = for x in allObjects(gch.region): @@ -667,16 +609,6 @@ when useMarkForDebug or useBackupGc: proc markGlobals(gch: var GcHeap) = for i in 0 .. < globalMarkersLen: globalMarkers[i]() - proc stackMarkS(gch: var GcHeap, p: pointer) {.inline.} = - # the addresses are not as cells on the stack, so turn them to cells: - var cell = usrToCell(p) - var c = cast[ByteAddress](cell) - if c >% PageSize: - # fast check: does it look like a cell? - var objStart = cast[PCell](interiorAllocatedPtr(gch.region, cell)) - if objStart != nil: - markS(gch, objStart) - when logGC: var cycleCheckA: array[100, PCell] @@ -717,19 +649,6 @@ proc doOperation(p: pointer, op: WalkOp) = #if c.refcount <% rcIncrement: addZCT(gch.zct, c) of waPush: add(gch.tempStack, c) - of waCycleDecRef: - gcAssert(c.refcount >=% rcIncrement, "doOperation 3") - c.refcount = c.refcount -% rcIncrement - of waMarkGray: - gcAssert(c.refcount >=% rcIncrement, "waMarkGray") - c.refcount = c.refcount -% rcIncrement - markGray(c) - of waScan: scan(c) - of waScanBlack: - c.refcount = c.refcount +% rcIncrement - if c.color != rcBlack: - scanBlack(c) - of waCollectWhite: collectWhite(c) of waMarkGlobal: when useMarkForDebug or useBackupGc: when hasThreadSupport: @@ -748,14 +667,6 @@ proc nimGCvisit(d: pointer, op: int) {.compilerRtl.} = proc collectZCT(gch: var GcHeap): bool {.benign.} -when useMarkForDebug or useBackupGc: - proc markStackAndRegistersForSweep(gch: var GcHeap) {.noinline, cdecl, - benign.} - -proc collectRoots(gch: var GcHeap) = - for s in elements(gch.cycleRoots): - collectWhite(s) - proc collectCycles(gch: var GcHeap) = when hasThreadSupport: for c in gch.toDispose: @@ -764,33 +675,12 @@ proc collectCycles(gch: var GcHeap) = while gch.zct.len > 0: discard collectZCT(gch) when useBackupGc: cellsetReset(gch.marked) - markStackAndRegistersForSweep(gch) - markGlobals(gch) - sweep(gch) - else: - markRoots(gch) - # scanRoots: - for s in elements(gch.cycleRoots): scan(s) - collectRoots(gch) - - cellsetReset(gch.cycleRoots) - # alive cycles need to be kept in 'cycleRoots' if they are referenced - # from the stack; otherwise the write barrier will add the cycle root again - # anyway: - when false: var d = gch.decStack.d - var cycleRootsLen = 0 for i in 0..gch.decStack.len-1: - var c = d[i] - gcAssert isAllocatedPtr(gch.region, c), "addBackStackRoots" - gcAssert c.refcount >=% rcIncrement, "addBackStackRoots: dead cell" - if canBeCycleRoot(c): - #if c notin gch.cycleRoots: - inc cycleRootsLen - incl(gch.cycleRoots, c) - gcAssert c.typ != nil, "addBackStackRoots 2" - if cycleRootsLen != 0: - cfprintf(cstdout, "cycle roots: %ld\n", cycleRootsLen) + sysAssert isAllocatedPtr(gch.region, d[i]), "collectCycles" + markS(gch, d[i]) + markGlobals(gch) + sweep(gch) proc gcMark(gch: var GcHeap, p: pointer) {.inline.} = # the addresses are not as cells on the stack, so turn them to cells: @@ -812,31 +702,11 @@ proc gcMark(gch: var GcHeap, p: pointer) {.inline.} = add(gch.decStack, cell) sysAssert(allocInv(gch.region), "gcMark end") -proc markThreadStacks(gch: var GcHeap) = - when hasThreadSupport and hasSharedHeap: - {.error: "not fully implemented".} - var it = threadList - while it != nil: - # mark registers: - for i in 0 .. high(it.registers): gcMark(gch, it.registers[i]) - var sp = cast[ByteAddress](it.stackBottom) - var max = cast[ByteAddress](it.stackTop) - # XXX stack direction? - # XXX unroll this loop: - while sp <=% max: - gcMark(gch, cast[ppointer](sp)[]) - sp = sp +% sizeof(pointer) - it = it.next - include gc_common proc markStackAndRegisters(gch: var GcHeap) {.noinline, cdecl.} = forEachStackSlot(gch, gcMark) -when useMarkForDebug or useBackupGc: - proc markStackAndRegistersForSweep(gch: var GcHeap) = - forEachStackSlot(gch, stackMarkS) - proc collectZCT(gch: var GcHeap): bool = # Note: Freeing may add child objects to the ZCT! So essentially we do # deep freeing, which is bad for incremental operation. In order to @@ -866,8 +736,6 @@ proc collectZCT(gch: var GcHeap): bool = # as this might be too slow. # In any case, it should be removed from the ZCT. But not # freed. **KEEP THIS IN MIND WHEN MAKING THIS INCREMENTAL!** - when cycleGC: - if canbeCycleRoot(c): excl(gch.cycleRoots, c) when logGC: writeCell("zct dealloc cell", c) gcTrace(c, csZctFreed) # We are about to free the object, call the finalizer BEFORE its @@ -877,6 +745,7 @@ proc collectZCT(gch: var GcHeap): bool = forAllChildren(c, waZctDecRef) when reallyDealloc: sysAssert(allocInv(gch.region), "collectZCT: rawDealloc") + beforeDealloc(gch, c, "collectZCT: stack trash") rawDealloc(gch.region, c) else: sysAssert(c.typ != nil, "collectZCT 2") @@ -915,7 +784,6 @@ proc collectCTBody(gch: var GcHeap) = sysAssert(gch.decStack.len == 0, "collectCT") prepareForInteriorPointerChecking(gch.region) markStackAndRegisters(gch) - markThreadStacks(gch) gch.stat.maxStackCells = max(gch.stat.maxStackCells, gch.decStack.len) inc(gch.stat.stackScans) if collectZCT(gch): @@ -937,11 +805,6 @@ proc collectCTBody(gch: var GcHeap) = if gch.maxPause > 0 and duration > gch.maxPause: c_fprintf(c_stdout, "[GC] missed deadline: %ld\n", duration) -when useMarkForDebug or useBackupGc: - proc markForDebug(gch: var GcHeap) = - markStackAndRegistersForSweep(gch) - markGlobals(gch) - when defined(nimCoroutines): proc currentStackSizes(): int = for stack in items(gch.stack): @@ -1035,7 +898,7 @@ when not defined(useNimRtl): "[GC] max threshold: " & $gch.stat.maxThreshold & "\n" & "[GC] zct capacity: " & $gch.zct.cap & "\n" & "[GC] max cycle table size: " & $gch.stat.cycleTableSize & "\n" & - "[GC] max pause time [ms]: " & $(gch.stat.maxPause div 1000_000) + "[GC] max pause time [ms]: " & $(gch.stat.maxPause div 1000_000) & "\n" when defined(nimCoroutines): result = result & "[GC] number of stacks: " & $gch.stack.len & "\n" for stack in items(gch.stack): |