ARC now capable of custom extra alignment. Ref, closure and seq support. (#15697)

6 files changed, 110 insertions, 43 deletions

diff --git a/lib/system/arc.nim b/lib/system/arc.nim
index 46579eaef..0eecadd66 100644
--- a/lib/system/arc.nim
+++ b/lib/system/arc.nim
@@ -75,14 +75,13 @@ when defined(nimArcDebug):
 elif defined(nimArcIds):
   var gRefId: int
 
-proc nimNewObj(size: int): pointer {.compilerRtl.} =
-  let s = size + sizeof(RefHeader)
+proc nimNewObj(size, alignment: int): pointer {.compilerRtl.} =
+  let hdrSize = align(sizeof(RefHeader), alignment)
+  let s = size + hdrSize
   when defined(nimscript):
     discard
-  elif compileOption("threads"):
-    result = allocShared0(s) +! sizeof(RefHeader)
   else:
-    result = alloc0(s) +! sizeof(RefHeader)
+    result = alignedAlloc0(s, alignment) +! hdrSize
   when defined(nimArcDebug) or defined(nimArcIds):
     head(result).refId = gRefId
     atomicInc gRefId
@@ -92,20 +91,18 @@ proc nimNewObj(size: int): pointer {.compilerRtl.} =
   when traceCollector:
     cprintf("[Allocated] %p result: %p\n", result -! sizeof(RefHeader), result)
 
-proc nimNewObjUninit(size: int): pointer {.compilerRtl.} =
+proc nimNewObjUninit(size, alignment: int): pointer {.compilerRtl.} =
   # Same as 'newNewObj' but do not initialize the memory to zero.
   # The codegen proved for us that this is not necessary.
-  let s = size + sizeof(RefHeader)
+  let hdrSize = align(sizeof(RefHeader), alignment)
+  let s = size + hdrSize
   when defined(nimscript):
     discard
-  elif compileOption("threads"):
-    var orig = cast[ptr RefHeader](allocShared(s))
   else:
-    var orig = cast[ptr RefHeader](alloc(s))
-  orig.rc = 0
+    result = cast[ptr RefHeader](alignedAlloc0(s, alignment) +! hdrSize)
+  head(result).rc = 0
   when defined(gcOrc):
-    orig.rootIdx = 0
-  result = orig +! sizeof(RefHeader)
+    head(result).rootIdx = 0
   when defined(nimArcDebug):
     head(result).refId = gRefId
     atomicInc gRefId
@@ -147,7 +144,7 @@ when not defined(nimscript) and defined(nimArcDebug):
     else:
       result = 0
 
-proc nimRawDispose(p: pointer) {.compilerRtl.} =
+proc nimRawDispose(p: pointer, alignment: int) {.compilerRtl.} =
   when not defined(nimscript):
     when traceCollector:
       cprintf("[Freed] %p\n", p -! sizeof(RefHeader))
@@ -155,27 +152,21 @@ proc nimRawDispose(p: pointer) {.compilerRtl.} =
       if head(p).rc >= rcIncrement:
         cstderr.rawWrite "[FATAL] dangling references exist\n"
         quit 1
-
-    when defined(gcOrc) and defined(nimArcDebug):
-      if (head(p).rc and 0b100) != 0:
-        cstderr.rawWrite "[FATAL] cycle root freed\n"
-        quit 1
-
     when defined(nimArcDebug):
       # we do NOT really free the memory here in order to reliably detect use-after-frees
       if freedCells.data == nil: init(freedCells)
       freedCells.incl head(p)
-    elif compileOption("threads"):
-      deallocShared(p -! sizeof(RefHeader))
     else:
-      dealloc(p -! sizeof(RefHeader))
+      let hdrSize = align(sizeof(RefHeader), alignment)
+      alignedDealloc(p -! hdrSize, alignment)
 
-template dispose*[T](x: owned(ref T)) = nimRawDispose(cast[pointer](x))
+template dispose*[T](x: owned(ref T)) = nimRawDispose(cast[pointer](x), T.alignOf)
 #proc dispose*(x: pointer) = nimRawDispose(x)
 
 proc nimDestroyAndDispose(p: pointer) {.compilerRtl, raises: [].} =
-  let d = cast[ptr PNimTypeV2](p)[].destructor
-  if d != nil: cast[DestructorProc](d)(p)
+  let rti = cast[ptr PNimTypeV2](p)
+  if rti.destructor != nil: 
+    cast[DestructorProc](rti.destructor)(p)
   when false:
     cstderr.rawWrite cast[ptr PNimTypeV2](p)[].name
     cstderr.rawWrite "\n"
@@ -183,7 +174,7 @@ proc nimDestroyAndDispose(p: pointer) {.compilerRtl, raises: [].} =
       cstderr.rawWrite "bah, nil\n"
     else:
       cstderr.rawWrite "has destructor!\n"
-  nimRawDispose(p)
+  nimRawDispose(p, rti.align)
 
 when defined(gcOrc):
   when defined(nimThinout):
@@ -216,7 +207,7 @@ proc GC_unref*[T](x: ref T) =
   if nimDecRefIsLast(cast[pointer](x)):
     # XXX this does NOT work for virtual destructors!
     `=destroy`(x[])
-    nimRawDispose(cast[pointer](x))
+    nimRawDispose(cast[pointer](x), T.alignOf)
 
 proc GC_ref*[T](x: ref T) =
   ## New runtime only supports this operation for 'ref T'.
diff --git a/lib/system/bitmasks.nim b/lib/system/bitmasks.nim
index 922ad5fb7..d7c55a4d9 100644
--- a/lib/system/bitmasks.nim
+++ b/lib/system/bitmasks.nim
@@ -15,7 +15,11 @@ const
   PageSize = 1 shl PageShift
   PageMask = PageSize-1
 
-  MemAlign = 16 # also minimal allocatable memory block
+  MemAlign = # also minimal allocatable memory block
+    when defined(useMalloc):
+      when defined(amd64): 16 
+      else: 8
+    else: 16
 
   BitsPerPage = PageSize div MemAlign
   UnitsPerPage = BitsPerPage div (sizeof(int)*8)
diff --git a/lib/system/deepcopy.nim b/lib/system/deepcopy.nim
index b9dc594fa..1f30b8427 100644
--- a/lib/system/deepcopy.nim
+++ b/lib/system/deepcopy.nim
@@ -163,7 +163,7 @@ proc genericDeepCopyAux(dest, src: pointer, mt: PNimType; tab: var PtrTable) =
           when defined(nimSeqsV2):
             let typ = if mt.base.kind == tyObject: cast[PNimType](cast[ptr PNimTypeV2](s2)[].typeInfoV1)
                       else: mt.base
-            let z = nimNewObj(typ.size)
+            let z = nimNewObj(typ.size, typ.align)
             cast[PPointer](dest)[] = z
           else:
             # this version should work for any other GC:
diff --git a/lib/system/memalloc.nim b/lib/system/memalloc.nim
index 142762fe7..5f4304502 100644
--- a/lib/system/memalloc.nim
+++ b/lib/system/memalloc.nim
@@ -304,6 +304,87 @@ when hasAlloc and not defined(js):
     ## or other memory may be corrupted.
     deallocShared(p)
 
+  include bitmasks  
+
+  template `+!`(p: pointer, s: SomeInteger): pointer =
+    cast[pointer](cast[int](p) +% int(s))
+
+  template `-!`(p: pointer, s: SomeInteger): pointer =
+    cast[pointer](cast[int](p) -% int(s))
+
+  proc alignedAlloc(size, align: Natural): pointer =
+    if align <= MemAlign:
+      when compileOption("threads"):
+        result = allocShared(size)
+      else:
+        result = alloc(size)
+    else: 
+      # allocate (size + align - 1) necessary for alignment,
+      # plus 2 bytes to store offset
+      when compileOption("threads"):
+        let base = allocShared(size + align - 1 + sizeof(uint16))
+      else:
+        let base = alloc(size + align - 1 + sizeof(uint16))
+      # memory layout: padding + offset (2 bytes) + user_data
+      # in order to deallocate: read offset at user_data - 2 bytes, 
+      # then deallocate user_data - offset
+      let offset = align - (cast[int](base) and (align - 1))
+      cast[ptr uint16](base +! (offset - sizeof(uint16)))[] = uint16(offset)
+      result = base +! offset
+
+  proc alignedAlloc0(size, align: Natural): pointer =
+    if align <= MemAlign:
+      when compileOption("threads"):
+        result = allocShared0(size)
+      else:
+        result = alloc0(size)
+    else: 
+      # see comments for alignedAlloc
+      when compileOption("threads"):
+        let base = allocShared0(size + align - 1 + sizeof(uint16))
+      else:
+        let base = alloc0(size + align - 1 + sizeof(uint16))
+      let offset = align - (cast[int](base) and (align - 1))
+      cast[ptr uint16](base +! (offset - sizeof(uint16)))[] = uint16(offset)
+      result = base +! offset
+
+  proc alignedDealloc(p: pointer, align: int) {.compilerproc.} = 
+    if align <= MemAlign:
+      when compileOption("threads"):
+        deallocShared(p)
+      else:
+        dealloc(p)
+    else:      
+      # read offset at p - 2 bytes, then deallocate (p - offset) pointer
+      let offset = cast[ptr uint16](p -! sizeof(uint16))[]
+      when compileOption("threads"):
+        deallocShared(p -! offset)
+      else:
+        dealloc(p -! offset)
+
+  proc alignedRealloc(p: pointer, oldSize, newSize, align: Natural): pointer =
+    if align <= MemAlign:
+      when compileOption("threads"):
+        result = reallocShared(p, newSize)
+      else:
+        result = realloc(p, newSize)
+    else:
+      result = alignedAlloc(newSize, align)
+      copyMem(result, p, oldSize)
+      alignedDealloc(p, align)
+
+  proc alignedRealloc0(p: pointer, oldSize, newSize, align: Natural): pointer =
+    if align <= MemAlign:
+      when compileOption("threads"):
+        result = reallocShared0(p, oldSize, newSize)
+      else:
+        result = realloc0(p, oldSize, newSize)
+    else:
+      result = alignedAlloc(newSize, align)
+      copyMem(result, p, oldSize)
+      zeroMem(result +! oldSize, newSize - oldSize)
+      alignedDealloc(p, align)
+
   {.pop.}
 
 # GC interface:
diff --git a/lib/system/orc.nim b/lib/system/orc.nim
index 3c2327fd5..28f8e5808 100644
--- a/lib/system/orc.nim
+++ b/lib/system/orc.nim
@@ -88,7 +88,7 @@ proc free(s: Cell; desc: PNimTypeV2) {.inline.} =
     else:
       cstderr.rawWrite "has dispose!\n"
 
-  nimRawDispose(p)
+  nimRawDispose(p, desc.align)
 
 proc nimTraceRef(q: pointer; desc: PNimTypeV2; env: pointer) {.compilerRtl, inline.} =
   let p = cast[ptr pointer](q)
diff --git a/lib/system/seqs_v2.nim b/lib/system/seqs_v2.nim
index d83a0009a..b7f24ecd5 100644
--- a/lib/system/seqs_v2.nim
+++ b/lib/system/seqs_v2.nim
@@ -35,10 +35,7 @@ proc newSeqPayload(cap, elemSize, elemAlign: int): pointer {.compilerRtl, raises
   # we have to use type erasure here as Nim does not support generic
   # compilerProcs. Oh well, this will all be inlined anyway.
   if cap > 0:
-    when compileOption("threads"):
-      var p = cast[ptr NimSeqPayloadBase](allocShared0(align(sizeof(NimSeqPayloadBase), elemAlign) + cap * elemSize))
-    else:
-      var p = cast[ptr NimSeqPayloadBase](alloc0(align(sizeof(NimSeqPayloadBase), elemAlign) + cap * elemSize))
+    var p = cast[ptr NimSeqPayloadBase](alignedAlloc0(align(sizeof(NimSeqPayloadBase), elemAlign) + cap * elemSize, elemAlign))
     p.cap = cap
     result = p
   else:
@@ -65,20 +62,14 @@ proc prepareSeqAdd(len: int; p: pointer; addlen, elemSize, elemAlign: int): poin
       let oldCap = p.cap and not strlitFlag
       let newCap = max(resize(oldCap), len+addlen)
       if (p.cap and strlitFlag) == strlitFlag:
-        when compileOption("threads"):
-          var q = cast[ptr NimSeqPayloadBase](allocShared0(headerSize + elemSize * newCap))
-        else:
-          var q = cast[ptr NimSeqPayloadBase](alloc0(headerSize + elemSize * newCap))
+        var q = cast[ptr NimSeqPayloadBase](alignedAlloc0(headerSize + elemSize * newCap, elemAlign))
         copyMem(q +! headerSize, p +! headerSize, len * elemSize)
         q.cap = newCap
         result = q
       else:
         let oldSize = headerSize + elemSize * oldCap
         let newSize = headerSize + elemSize * newCap
-        when compileOption("threads"):
-          var q = cast[ptr NimSeqPayloadBase](reallocShared0(p, oldSize, newSize))
-        else:
-          var q = cast[ptr NimSeqPayloadBase](realloc0(p, oldSize, newSize))
+        var q = cast[ptr NimSeqPayloadBase](alignedRealloc0(p, oldSize, newSize, elemAlign))
         q.cap = newCap
         result = q