gc: destructors is beginning to work (#10483)

* kochdocs.nim: code cleanup
* docgen: nicer indentation
* parser.nim: code cleanup
* fixes #10458
* make tests green again
* make =destroy mixins
* gc:destructors: produced C code is almost working
* --gc:destructors simple program compiles (but leaks memory)
* gc:destructors make examples compile in C++ mode
* destructors: string implementation bugfixes
* strs.nim: minor code cleanup
* destructors: builtin seqs are beginning to work
* remove debugging helpers

6 files changed, 62 insertions, 46 deletions

diff --git a/lib/core/seqs.nim b/lib/core/seqs.nim
index 977b23b26..1a81b89ea 100644
--- a/lib/core/seqs.nim
+++ b/lib/core/seqs.nim
@@ -15,7 +15,7 @@ proc supportsCopyMem(t: typedesc): bool {.magic: "TypeTrait".}
 
 ## Default seq implementation used by Nim's core.
 type
-  NimSeqPayload {.core.}[T] = object
+  NimSeqPayload[T] = object
     cap: int
     region: Allocator
     data: UncheckedArray[T]
@@ -40,6 +40,7 @@ proc `=destroy`[T](s: var seq[T]) =
   var x = cast[ptr NimSeqV2[T]](addr s)
   var p = x.p
   if p != nil:
+    mixin `=destroy`
     when not supportsCopyMem(T):
       for i in 0..<x.len: `=destroy`(p.data[i])
     p.region.dealloc(p.region, p, payloadSize(p.cap))
@@ -47,11 +48,12 @@ proc `=destroy`[T](s: var seq[T]) =
     x.len = 0
 
 proc `=`[T](x: var seq[T]; y: seq[T]) =
+  mixin `=destroy`
   var a = cast[ptr NimSeqV2[T]](addr x)
   var b = cast[ptr NimSeqV2[T]](unsafeAddr y)
 
   if a.p == b.p: return
-  `=destroy`(a)
+  `=destroy`(x)
   a.len = b.len
   if b.p != nil:
     a.p = cast[type(a.p)](alloc(payloadSize(a.len)))
@@ -63,10 +65,11 @@ proc `=`[T](x: var seq[T]; y: seq[T]) =
         a.p.data[i] = b.p.data[i]
 
 proc `=sink`[T](x: var seq[T]; y: seq[T]) =
+  mixin `=destroy`
   var a = cast[ptr NimSeqV2[T]](addr x)
   var b = cast[ptr NimSeqV2[T]](unsafeAddr y)
   if a.p != nil and a.p != b.p:
-    `=destroy`(a)
+    `=destroy`(x)
   a.len = b.len
   a.p = b.p
 
@@ -109,6 +112,7 @@ proc prepareSeqAdd(len: int; p: pointer; addlen, elemSize: int): pointer {.
       result = q
 
 proc shrink*[T](x: var seq[T]; newLen: Natural) =
+  mixin `=destroy`
   sysAssert newLen <= x.len, "invalid newLen parameter for 'shrink'"
   when not supportsCopyMem(T):
     for i in countdown(x.len - 1, newLen - 1):
diff --git a/lib/core/strs.nim b/lib/core/strs.nim
index 186add52a..ccbde76fe 100644
--- a/lib/core/strs.nim
+++ b/lib/core/strs.nim
@@ -51,15 +51,12 @@ proc `=destroy`(s: var string) =
   a.len = 0
   a.p = nil
 
-template lose(a) =
-  frees(a)
-
 proc `=sink`(x: var string, y: string) =
   var a = cast[ptr NimStringV2](addr x)
   var b = cast[ptr NimStringV2](unsafeAddr y)
   # we hope this is optimized away for not yet alive objects:
   if unlikely(a.p == b.p): return
-  lose(a)
+  frees(a)
   a.len = b.len
   a.p = b.p
 
@@ -67,13 +64,13 @@ proc `=`(x: var string, y: string) =
   var a = cast[ptr NimStringV2](addr x)
   var b = cast[ptr NimStringV2](unsafeAddr y)
   if unlikely(a.p == b.p): return
-  lose(a)
+  frees(a)
   a.len = b.len
   if isLiteral(b):
     # we can shallow copy literals:
     a.p = b.p
   else:
-    let region = if a.p.region != nil: a.p.region else: getLocalAllocator()
+    let region = if a.p != nil and a.p.region != nil: a.p.region else: getLocalAllocator()
     # we have to allocate the 'cap' here, consider
     # 'let y = newStringOfCap(); var x = y'
     # on the other hand... These get turned into moves now.
@@ -136,6 +133,7 @@ proc appendString(dest: var NimStringV2; src: NimStringV2) {.compilerproc, inlin
   if src.len > 0:
     # also copy the \0 terminator:
     copyMem(unsafeAddr dest.p.data[dest.len], unsafeAddr src.p.data[0], src.len+1)
+    inc dest.len, src.len
 
 proc appendChar(dest: var NimStringV2; c: char) {.compilerproc, inline.} =
   dest.p.data[dest.len] = c
@@ -166,7 +164,6 @@ proc mnewString(len: int): NimStringV2 {.compilerProc.} =
 proc setLengthStrV2(s: var NimStringV2, newLen: int) {.compilerRtl.} =
   if newLen > s.len:
     prepareAdd(s, newLen - s.len)
-  else:
-    s.len = newLen
-    # this also only works because the destructor
-    # looks at s.p and not s.len
+  s.len = newLen
+  # this also only works because the destructor
+  # looks at s.p and not s.len
diff --git a/lib/system.nim b/lib/system.nim
index 4951961ca..a7cf251f6 100644
--- a/lib/system.nim
+++ b/lib/system.nim
@@ -3049,6 +3049,19 @@ else:
     if x < 0: -x else: x
 {.pop.}
 
+when defined(nimNewRoof):
+  iterator `..<`*[T](a, b: T): T =
+    var i = T(a)
+    while i < b:
+      yield i
+      inc i
+else:
+  iterator `..<`*[S, T](a: S, b: T): T =
+    var i = T(a)
+    while i < b:
+      yield i
+      inc i
+
 when not defined(JS):
   proc likelyProc(val: bool): bool {.importc: "likely", nodecl, nosideeffect.}
   proc unlikelyProc(val: bool): bool {.importc: "unlikely", nodecl, nosideeffect.}
@@ -3144,7 +3157,7 @@ when not defined(JS): #and not defined(nimscript):
   # ----------------- IO Part ------------------------------------------------
   type
     CFile {.importc: "FILE", header: "<stdio.h>",
-            final, incompletestruct.} = object
+            incompletestruct.} = object
     File* = ptr CFile ## The type representing a file handle.
 
     FileMode* = enum           ## The file mode when opening a file.
@@ -3392,6 +3405,10 @@ when not defined(JS): #and not defined(nimscript):
       ## returns the OS file handle of the file ``f``. This is only useful for
       ## platform specific programming.
 
+  when defined(gcDestructors) and not defined(nimscript):
+    include "core/strs"
+    include "core/seqs"
+
   when declared(newSeq):
     proc cstringArrayToSeq*(a: cstringArray, len: Natural): seq[string] =
       ## converts a ``cstringArray`` to a ``seq[string]``. `a` is supposed to be
@@ -3483,10 +3500,6 @@ when not defined(JS): #and not defined(nimscript):
     when defined(memtracker):
       include "system/memtracker"
 
-    when defined(gcDestructors):
-      include "core/strs"
-      include "core/seqs"
-
     when hostOS == "standalone":
       include "system/embedded"
     else:
@@ -3716,19 +3729,6 @@ template `..<`*(a, b: untyped): untyped =
   ## a shortcut for 'a .. (when b is BackwardsIndex: succ(b) else: pred(b))'.
   a .. (when b is BackwardsIndex: succ(b) else: pred(b))
 
-when defined(nimNewRoof):
-  iterator `..<`*[T](a, b: T): T =
-    var i = T(a)
-    while i < b:
-      yield i
-      inc i
-else:
-  iterator `..<`*[S, T](a: S, b: T): T =
-    var i = T(a)
-    while i < b:
-      yield i
-      inc i
-
 template spliceImpl(s, a, L, b: untyped): untyped =
   # make room for additional elements or cut:
   var shift = b.len - max(0,L)  # ignore negative slice size
diff --git a/lib/system/excpt.nim b/lib/system/excpt.nim
index cc0c1f54b..f2f82c3b8 100644
--- a/lib/system/excpt.nim
+++ b/lib/system/excpt.nim
@@ -220,11 +220,12 @@ proc auxWriteStackTrace(f: PFrame; s: var seq[StackTraceEntry]) =
     inc(i)
     it = it.prev
   var last = i-1
-  if s.len == 0:
-    s = newSeq[StackTraceEntry](i)
-  else:
-    last = s.len + i - 1
-    s.setLen(last+1)
+  when true: # not defined(gcDestructors):
+    if s.len == 0:
+      s = newSeq[StackTraceEntry](i)
+    else:
+      last = s.len + i - 1
+      s.setLen(last+1)
   it = f
   while it != nil:
     s[last] = StackTraceEntry(procname: it.procname,
@@ -440,11 +441,13 @@ proc getStackTrace(e: ref Exception): string =
   else:
     result = ""
 
-when not defined(gcDestructors):
-  proc getStackTraceEntries*(e: ref Exception): seq[StackTraceEntry] =
-    ## Returns the attached stack trace to the exception ``e`` as
-    ## a ``seq``. This is not yet available for the JS backend.
+proc getStackTraceEntries*(e: ref Exception): seq[StackTraceEntry] =
+  ## Returns the attached stack trace to the exception ``e`` as
+  ## a ``seq``. This is not yet available for the JS backend.
+  when not defined(gcDestructors):
     shallowCopy(result, e.trace)
+  else:
+    result = move(e.trace)
 
 const nimCallDepthLimit {.intdefine.} = 2000
 
diff --git a/lib/system/gc_regions.nim b/lib/system/gc_regions.nim
index 59f68918f..797eeeebf 100644
--- a/lib/system/gc_regions.nim
+++ b/lib/system/gc_regions.nim
@@ -195,6 +195,19 @@ proc runFinalizers(c: Chunk) =
       (cast[Finalizer](it.typ.finalizer))(it+!sizeof(ObjHeader))
     it = it.nextFinal
 
+proc runFinalizers(c: Chunk; newbump: pointer) =
+  var it = c.head
+  var prev: ptr ObjHeader = nil
+  while it != nil:
+    let nxt = it.nextFinal
+    if it >= newbump:
+      if it.typ != nil and it.typ.finalizer != nil:
+        (cast[Finalizer](it.typ.finalizer))(it+!sizeof(ObjHeader))
+    elif prev != nil:
+      prev.nextFinal = nil
+    prev = it
+    it = nxt
+
 proc dealloc(r: var MemRegion; p: pointer; size: int) =
   let it = cast[ptr ObjHeader](p-!sizeof(ObjHeader))
   if it.typ != nil and it.typ.finalizer != nil:
@@ -237,16 +250,15 @@ template computeRemaining(r): untyped =
 
 proc setObstackPtr*(r: var MemRegion; sp: StackPtr) =
   # free everything after 'sp':
-  if sp.current.next != nil:
+  if sp.current != nil and sp.current.next != nil:
     deallocAll(r, sp.current.next)
     sp.current.next = nil
     when false:
       # better leak this memory than be sorry:
       for i in 0..high(r.freeLists): r.freeLists[i] = nil
       r.holes = nil
-  #else:
-  #  deallocAll(r, r.head)
-  #  r.head = nil
+  if r.tail != nil: runFinalizers(r.tail, sp.bump)
+
   r.bump = sp.bump
   r.tail = sp.current
   r.remaining = sp.remaining
diff --git a/lib/system/helpers2.nim b/lib/system/helpers2.nim
index c67a2c278..8bd69ad71 100644
--- a/lib/system/helpers2.nim
+++ b/lib/system/helpers2.nim
@@ -1,7 +1,7 @@
 # imported by other modules, unlike helpers.nim which is included
 
 template formatErrorIndexBound*[T](i, a, b: T): string =
-  "index out of bounds: (a:" & $a & ") <= (i:" & $i & ") <= (b:" & $b & ") "
+  "index out of bounds: (a: " & $a & ") <= (i: " & $i & ") <= (b: " & $b & ") "
 
 template formatErrorIndexBound*[T](i, n: T): string =
-  "index out of bounds: (i:" & $i & ") <= (n:" & $n & ") "
+  "index out of bounds: (i: " & $i & ") <= (n: " & $n & ") "