diff options
Diffstat (limited to 'lib/system/gc.nim')
| -rw-r--r-- | lib/system/gc.nim | 84 |
1 files changed, 44 insertions, 40 deletions
diff --git a/lib/system/gc.nim b/lib/system/gc.nim index 1f4279c8f..bbf8cbf66 100644 --- a/lib/system/gc.nim +++ b/lib/system/gc.nim @@ -1,7 +1,7 @@ # # # Nim's Runtime Library -# (c) Copyright 2013 Andreas Rumpf +# (c) Copyright 2015 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -48,7 +48,7 @@ type TWalkOp = enum waMarkGlobal, # part of the backup/debug mark&sweep waMarkPrecise, # part of the backup/debug mark&sweep - waZctDecRef, waPush, waCycleDecRef, waMarkGray, waScan, waScanBlack, + waZctDecRef, waPush, waCycleDecRef, waMarkGray, waScan, waScanBlack, waCollectWhite #, waDebug TFinalizer {.compilerproc.} = proc (self: pointer) {.nimcall, benign.} @@ -61,9 +61,9 @@ type maxThreshold: int # max threshold that has been set maxStackSize: int # max stack size maxStackCells: int # max stack cells in ``decStack`` - cycleTableSize: int # max entries in cycle table + cycleTableSize: int # max entries in cycle table maxPause: int64 # max measured GC pause in nanoseconds - + TGcHeap {.final, pure.} = object # this contains the zero count and # non-zero count table stackBottom: pointer @@ -88,11 +88,11 @@ var when not defined(useNimRtl): instantiateForRegion(gch.region) -template acquire(gch: TGcHeap) = +template acquire(gch: TGcHeap) = when hasThreadSupport and hasSharedHeap: acquireSys(HeapLock) -template release(gch: TGcHeap) = +template release(gch: TGcHeap) = when hasThreadSupport and hasSharedHeap: releaseSys(HeapLock) @@ -117,7 +117,7 @@ proc usrToCell(usr: pointer): PCell {.inline.} = # convert pointer to userdata to object (=pointer to refcount) result = cast[PCell](cast[ByteAddress](usr)-%ByteAddress(sizeof(TCell))) -proc canbeCycleRoot(c: PCell): bool {.inline.} = +proc canBeCycleRoot(c: PCell): bool {.inline.} = result = ntfAcyclic notin c.typ.flags proc extGetCellType(c: pointer): PNimType {.compilerproc.} = @@ -163,7 +163,7 @@ when hasThreadSupport and hasSharedHeap: template `--`(x: expr): expr = atomicDec(x, rcIncrement) <% rcIncrement template `++`(x: expr): stmt = discard atomicInc(x, rcIncrement) else: - template `--`(x: expr): expr = + template `--`(x: expr): expr = dec(x, rcIncrement) x <% rcIncrement template `++`(x: expr): stmt = inc(x, rcIncrement) @@ -181,7 +181,7 @@ proc prepareDealloc(cell: PCell) = (cast[TFinalizer](cell.typ.finalizer))(cellToUsr(cell)) dec(gch.recGcLock) -proc rtlAddCycleRoot(c: PCell) {.rtl, inl.} = +proc rtlAddCycleRoot(c: PCell) {.rtl, inl.} = # we MUST access gch as a global here, because this crosses DLL boundaries! when hasThreadSupport and hasSharedHeap: acquireSys(HeapLock) @@ -211,7 +211,7 @@ proc decRef(c: PCell) {.inline.} = rtlAddCycleRoot(c) #writeCell("decRef", c) -proc incRef(c: PCell) {.inline.} = +proc incRef(c: PCell) {.inline.} = gcAssert(isAllocatedPtr(gch.region, c), "incRef: interiorPtr") c.refcount = c.refcount +% rcIncrement # and not colorMask @@ -246,12 +246,12 @@ proc asgnRef(dest: PPointer, src: pointer) {.compilerProc, inline.} = dest[] = src proc asgnRefNoCycle(dest: PPointer, src: pointer) {.compilerProc, inline.} = - # the code generator calls this proc if it is known at compile time that no + # the code generator calls this proc if it is known at compile time that no # cycle is possible. if src != nil: var c = usrToCell(src) ++c.refcount - if dest[] != nil: + if dest[] != nil: var c = usrToCell(dest[]) if --c.refcount: rtlAddZCT(c) @@ -269,7 +269,7 @@ proc unsureAsgnRef(dest: PPointer, src: pointer) {.compilerProc.} = if cast[int](dest[]) >=% PageSize: decRef(usrToCell(dest[])) else: # can't be an interior pointer if it's a stack location! - gcAssert(interiorAllocatedPtr(gch.region, dest) == nil, + gcAssert(interiorAllocatedPtr(gch.region, dest) == nil, "stack loc AND interior pointer") dest[] = src @@ -321,7 +321,7 @@ when useMarkForDebug or useBackupGc: echo "[GC] cannot register global variable; too many global variables" quit 1 -proc cellsetReset(s: var TCellSet) = +proc cellsetReset(s: var TCellSet) = deinit(s) init(s) @@ -336,7 +336,7 @@ proc forAllSlotsAux(dest: pointer, n: ptr TNimNode, op: TWalkOp) {.benign.} = if n.sons[i].typ.kind in {tyRef, tyString, tySequence}: doOperation(cast[PPointer](d +% n.sons[i].offset)[], op) else: - forAllChildrenAux(cast[pointer](d +% n.sons[i].offset), + forAllChildrenAux(cast[pointer](d +% n.sons[i].offset), n.sons[i].typ, op) else: forAllSlotsAux(dest, n.sons[i], op) @@ -384,7 +384,7 @@ proc addNewObjToZCT(res: PCell, gch: var TGcHeap) {.inline.} = # we check the last 8 entries (cache line) for a slot that could be reused. # In 63% of all cases we succeed here! But we have to optimize the heck # out of this small linear search so that ``newObj`` is not slowed down. - # + # # Slots to try cache hit # 1 32% # 4 59% @@ -481,7 +481,7 @@ proc newObjRC1(typ: PNimType, size: int): pointer {.compilerRtl.} = gcAssert(typ.kind in {tyRef, tyString, tySequence}, "newObj: 1") collectCT(gch) sysAssert(allocInv(gch.region), "newObjRC1 after collectCT") - + var res = cast[PCell](rawAlloc(gch.region, size + sizeof(TCell))) sysAssert(allocInv(gch.region), "newObjRC1 after rawAlloc") sysAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "newObj: 2") @@ -510,7 +510,7 @@ proc newSeqRC1(typ: PNimType, len: int): pointer {.compilerRtl.} = cast[PGenericSeq](result).len = len cast[PGenericSeq](result).reserved = len when defined(memProfiler): nimProfile(size) - + proc growObj(old: pointer, newsize: int, gch: var TGcHeap): pointer = acquire(gch) collectCT(gch) @@ -522,7 +522,7 @@ proc growObj(old: pointer, newsize: int, gch: var TGcHeap): pointer = var res = cast[PCell](rawAlloc(gch.region, newsize + sizeof(TCell))) var elemSize = 1 if ol.typ.kind != tyString: elemSize = ol.typ.base.size - + var oldsize = cast[PGenericSeq](old).len*elemSize + GenericSeqSize copyMem(res, ol, oldsize + sizeof(TCell)) zeroMem(cast[pointer](cast[ByteAddress](res)+% oldsize +% sizeof(TCell)), @@ -536,7 +536,7 @@ proc growObj(old: pointer, newsize: int, gch: var TGcHeap): pointer = writeCell("growObj new cell", res) gcTrace(ol, csZctFreed) gcTrace(res, csAllocated) - when reallyDealloc: + when reallyDealloc: sysAssert(allocInv(gch.region), "growObj before dealloc") if ol.refcount shr rcShift <=% 1: # free immediately to save space: @@ -580,7 +580,7 @@ proc freeCyclicCell(gch: var TGcHeap, c: PCell) = prepareDealloc(c) gcTrace(c, csCycFreed) when logGC: writeCell("cycle collector dealloc cell", c) - when reallyDealloc: + when reallyDealloc: sysAssert(allocInv(gch.region), "free cyclic cell") rawDealloc(gch.region, c) else: @@ -767,7 +767,7 @@ proc collectCycles(gch: var TGcHeap) = gcAssert isAllocatedPtr(gch.region, c), "addBackStackRoots" gcAssert c.refcount >=% rcIncrement, "addBackStackRoots: dead cell" if canBeCycleRoot(c): - #if c notin gch.cycleRoots: + #if c notin gch.cycleRoots: inc cycleRootsLen incl(gch.cycleRoots, c) gcAssert c.typ != nil, "addBackStackRoots 2" @@ -794,12 +794,12 @@ proc gcMark(gch: var TGcHeap, p: pointer) {.inline.} = add(gch.decStack, cell) sysAssert(allocInv(gch.region), "gcMark end") -proc markThreadStacks(gch: var TGcHeap) = +proc markThreadStacks(gch: var TGcHeap) = when hasThreadSupport and hasSharedHeap: {.error: "not fully implemented".} var it = threadList while it != nil: - # mark registers: + # mark registers: for i in 0 .. high(it.registers): gcMark(gch, it.registers[i]) var sp = cast[TAddress](it.stackBottom) var max = cast[TAddress](it.stackTop) @@ -933,7 +933,7 @@ else: while sp <=% max: gcMark(gch, cast[PPointer](sp)[]) sp = sp +% sizeof(pointer) - + proc markStackAndRegisters(gch: var TGcHeap) {.noinline, cdecl.} = forEachStackSlot(gch, gcMark) @@ -946,13 +946,13 @@ when useMarkForDebug or useBackupGc: # ---------------------------------------------------------------------------- proc collectZCT(gch: var TGcHeap): bool = - # Note: Freeing may add child objects to the ZCT! So essentially we do - # deep freeing, which is bad for incremental operation. In order to + # Note: Freeing may add child objects to the ZCT! So essentially we do + # deep freeing, which is bad for incremental operation. In order to # avoid a deep stack, we move objects to keep the ZCT small. # This is performance critical! const workPackage = 100 var L = addr(gch.zct.len) - + when withRealTime: var steps = workPackage var t0: TTicks @@ -962,15 +962,15 @@ proc collectZCT(gch: var TGcHeap): bool = sysAssert(isAllocatedPtr(gch.region, c), "CollectZCT: isAllocatedPtr") # remove from ZCT: gcAssert((c.refcount and ZctFlag) == ZctFlag, "collectZCT") - + c.refcount = c.refcount and not ZctFlag gch.zct.d[0] = gch.zct.d[L[] - 1] dec(L[]) when withRealTime: dec steps - if c.refcount <% rcIncrement: + if c.refcount <% rcIncrement: # It may have a RC > 0, if it is in the hardware stack or # it has not been removed yet from the ZCT. This is because - # ``incref`` does not bother to remove the cell from the ZCT + # ``incref`` does not bother to remove the cell from the ZCT # as this might be too slow. # In any case, it should be removed from the ZCT. But not # freed. **KEEP THIS IN MIND WHEN MAKING THIS INCREMENTAL!** @@ -983,7 +983,7 @@ proc collectZCT(gch: var TGcHeap): bool = # access invalid memory. This is done by prepareDealloc(): prepareDealloc(c) forAllChildren(c, waZctDecRef) - when reallyDealloc: + when reallyDealloc: sysAssert(allocInv(gch.region), "collectZCT: rawDealloc") rawDealloc(gch.region, c) else: @@ -994,7 +994,7 @@ proc collectZCT(gch: var TGcHeap): bool = steps = workPackage if gch.maxPause > 0: let duration = getticks() - t0 - # the GC's measuring is not accurate and needs some cleanup actions + # the GC's measuring is not accurate and needs some cleanup actions # (stack unmarking), so subtract some short amount of time in # order to miss deadlines less often: if duration >= gch.maxPause - 50_000: @@ -1017,7 +1017,7 @@ proc collectCTBody(gch: var TGcHeap) = when withRealTime: let t0 = getticks() sysAssert(allocInv(gch.region), "collectCT: begin") - + gch.stat.maxStackSize = max(gch.stat.maxStackSize, stackSize()) sysAssert(gch.decStack.len == 0, "collectCT") prepareForInteriorPointerChecking(gch.region) @@ -1036,7 +1036,7 @@ proc collectCTBody(gch: var TGcHeap) = gch.stat.maxThreshold = max(gch.stat.maxThreshold, gch.cycleThreshold) unmarkStackAndRegisters(gch) sysAssert(allocInv(gch.region), "collectCT: end") - + when withRealTime: let duration = getticks() - t0 gch.stat.maxPause = max(gch.stat.maxPause, duration) @@ -1050,8 +1050,12 @@ when useMarkForDebug or useBackupGc: markGlobals(gch) proc collectCT(gch: var TGcHeap) = - if (gch.zct.len >= ZctThreshold or (cycleGC and - getOccupiedMem(gch.region)>=gch.cycleThreshold) or alwaysGC) and + # stackMarkCosts prevents some pathological behaviour: Stack marking + # becomes more expensive with large stacks and large stacks mean that + # cells with RC=0 are more likely to be kept alive by the stack. + let stackMarkCosts = max(stackSize() div (16*sizeof(int)), ZctThreshold) + if (gch.zct.len >= stackMarkCosts or (cycleGC and + getOccupiedMem(gch.region)>=gch.cycleThreshold) or alwaysGC) and gch.recGcLock == 0: when useMarkForDebug: prepareForInteriorPointerChecking(gch.region) @@ -1070,7 +1074,7 @@ when withRealTime: acquire(gch) gch.maxPause = us.toNano if (gch.zct.len >= ZctThreshold or (cycleGC and - getOccupiedMem(gch.region)>=gch.cycleThreshold) or alwaysGC) or + getOccupiedMem(gch.region)>=gch.cycleThreshold) or alwaysGC) or strongAdvice: collectCTBody(gch) release(gch) @@ -1078,13 +1082,13 @@ when withRealTime: proc GC_step*(us: int, strongAdvice = false) = GC_step(gch, us, strongAdvice) when not defined(useNimRtl): - proc GC_disable() = + proc GC_disable() = when hasThreadSupport and hasSharedHeap: discard atomicInc(gch.recGcLock, 1) else: inc(gch.recGcLock) proc GC_enable() = - if gch.recGcLock > 0: + if gch.recGcLock > 0: when hasThreadSupport and hasSharedHeap: discard atomicDec(gch.recGcLock, 1) else: |