Merge pull request #1363 from Araq/devel

Merge devel into master

12 files changed, 176 insertions, 70 deletions

diff --git a/lib/system/alloc.nim b/lib/system/alloc.nim
index eaef6cd95..602e5c7fa 100644
--- a/lib/system/alloc.nim
+++ b/lib/system/alloc.nim
@@ -835,4 +835,20 @@ template instantiateForRegion(allocator: expr) =
     else:
       result = realloc(p, newsize)
 
+  when hasThreadSupport:
+
+    template sharedMemStatsShared(v: int) {.immediate.} =
+      acquireSys(heapLock)
+      result = v
+      releaseSys(heapLock)
+
+    proc getFreeSharedMem(): int =
+      sharedMemStatsShared(sharedHeap.freeMem)
+
+    proc getTotalSharedMem(): int =
+      sharedMemStatsShared(sharedHeap.currMem)
+
+    proc getOccupiedSharedMem(): int =
+      sharedMemStatsShared(sharedHeap.currMem - sharedHeap.freeMem)
+
 {.pop.}
diff --git a/lib/system/ansi_c.nim b/lib/system/ansi_c.nim
index 2d33965e3..5111bc3cf 100644
--- a/lib/system/ansi_c.nim
+++ b/lib/system/ansi_c.nim
@@ -57,6 +57,7 @@ when not defined(SIGINT):
         SIGINT = cint(2)
         SIGSEGV = cint(11)
         SIGTERM = cint(15)
+        SIGPIPE = cint(13)
     else:
       {.error: "SIGABRT not ported to your platform".}
   else:
@@ -66,6 +67,8 @@ when not defined(SIGINT):
       SIGABRT {.importc: "SIGABRT", nodecl.}: cint
       SIGFPE {.importc: "SIGFPE", nodecl.}: cint
       SIGILL {.importc: "SIGILL", nodecl.}: cint
+    when defined(macosx) or defined(linux):
+      var SIGPIPE {.importc: "SIGPIPE", nodecl.}: cint
 
 when defined(macosx):
   when NoFakeVars:
diff --git a/lib/system/assign.nim b/lib/system/assign.nim
index 75c749633..2ae945fb1 100644
--- a/lib/system/assign.nim
+++ b/lib/system/assign.nim
@@ -179,7 +179,8 @@ when not defined(nimmixin):
     # internal proc used for destroying sequences and arrays
     for i in countup(0, r.len - 1): destroy(r[i])
 else:
-  # XXX Why is this exported and no compilerproc?
+  # XXX Why is this exported and no compilerproc? -> compilerprocs cannot be
+  # generic for now
   proc nimDestroyRange*[T](r: T) =
     # internal proc used for destroying sequences and arrays
     mixin destroy
diff --git a/lib/system/atomics.nim b/lib/system/atomics.nim
index b1a96b209..43b3f0438 100644
--- a/lib/system/atomics.nim
+++ b/lib/system/atomics.nim
@@ -1,15 +1,18 @@
 #
 #
 #            Nimrod's Runtime Library
-#        (c) Copyright 2012 Andreas Rumpf
+#        (c) Copyright 2014 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
 #
 
 ## Atomic operations for Nimrod.
+{.push stackTrace:off.}
 
-when (defined(gcc) or defined(llvm_gcc)) and hasThreadSupport:
+const someGcc = defined(gcc) or defined(llvm_gcc) or defined(clang)
+
+when someGcc and hasThreadSupport:
   type 
     AtomMemModel* = enum
       ATOMIC_RELAXED,  ## No barriers or synchronization. 
@@ -152,41 +155,16 @@ when (defined(gcc) or defined(llvm_gcc)) and hasThreadSupport:
     ## A value of 0 indicates typical alignment should be used. The compiler may also 
     ## ignore this parameter.
 
+  template fence*() = atomicThreadFence(ATOMIC_SEQ_CST)
 elif defined(vcc) and hasThreadSupport:
   proc addAndFetch*(p: ptr int, val: int): int {.
     importc: "NimXadd", nodecl.}
+
 else:
   proc addAndFetch*(p: ptr int, val: int): int {.inline.} =
     inc(p[], val)
     result = p[]
 
-# atomic compare and swap (CAS) funcitons to implement lock-free algorithms  
-      
-#if defined(windows) and not defined(gcc) and hasThreadSupport:
-#    proc InterlockedCompareExchangePointer(mem: ptr pointer,
-#      newValue: pointer, comparand: pointer) : pointer {.nodecl, 
-#        importc: "InterlockedCompareExchangePointer", header:"windows.h".}
-
-#    proc compareAndSwap*[T](mem: ptr T, 
-#      expected: T, newValue: T): bool {.inline.}=
-#      ## Returns true if successfully set value at mem to newValue when value
-#      ## at mem == expected
-#      return InterlockedCompareExchangePointer(addr(mem), 
-#        addr(newValue), addr(expected))[] == expected
-    
-#elif not hasThreadSupport:
-#  proc compareAndSwap*[T](mem: ptr T, 
-#                          expected: T, newValue: T): bool {.inline.} =
-#      ## Returns true if successfully set value at mem to newValue when value
-#      ## at mem == expected
-#      var oldval = mem[]
-#      if oldval == expected:
-#        mem[] = newValue
-#        return true
-#      return false
-
-
-# Some convenient functions 
 proc atomicInc*(memLoc: var int, x: int = 1): int =
   when defined(gcc) and hasThreadSupport:
     result = atomic_add_fetch(memLoc.addr, x, ATOMIC_RELAXED)
@@ -203,3 +181,37 @@ proc atomicDec*(memLoc: var int, x: int = 1): int =
   else:
     dec(memLoc, x)
     result = memLoc
+
+when defined(windows) and not someGcc:
+  proc interlockedCompareExchange(p: pointer; exchange, comparand: int32): int32
+    {.importc: "InterlockedCompareExchange", header: "<windows.h>", cdecl.}
+
+  proc cas*[T: bool|int|ptr](p: ptr T; oldValue, newValue: T): bool =
+    interlockedCompareExchange(p, newValue.int32, oldValue.int32) != 0
+  # XXX fix for 64 bit build
+else:
+  # this is valid for GCC and Intel C++
+  proc cas*[T: bool|int|ptr](p: ptr T; oldValue, newValue: T): bool
+    {.importc: "__sync_bool_compare_and_swap", nodecl.}
+  # XXX is this valid for 'int'?
+
+
+when (defined(x86) or defined(amd64)) and (defined(gcc) or defined(llvm_gcc)):
+  proc cpuRelax {.inline.} =
+    {.emit: """asm volatile("pause" ::: "memory");""".}
+elif (defined(x86) or defined(amd64)) and defined(vcc):
+  proc cpuRelax {.importc: "YieldProcessor", header: "<windows.h>".}
+elif defined(intelc):
+  proc cpuRelax {.importc: "_mm_pause", header: "xmmintrin.h".}
+elif false:
+  from os import sleep
+
+  proc cpuRelax {.inline.} = os.sleep(1)
+
+when not defined(fence) and hasThreadSupport:
+  # XXX fixme
+  proc fence*() {.inline.} =
+    var dummy: bool
+    discard cas(addr dummy, false, true)
+
+{.pop.}
diff --git a/lib/system/chcks.nim b/lib/system/chcks.nim
index f29e222e8..387b54ef1 100644
--- a/lib/system/chcks.nim
+++ b/lib/system/chcks.nim
@@ -67,6 +67,28 @@ proc chckObjAsgn(a, b: PNimType) {.compilerproc, inline.} =
   if a != b:
     sysFatal(EInvalidObjectAssignment, "invalid object assignment")
 
+type ObjCheckCache = array[0..1, PNimType]
+
+proc isObjSlowPath(obj, subclass: PNimType;
+                   cache: var ObjCheckCache): bool {.noinline.} =
+  # checks if obj is of type subclass:
+  var x = obj.base
+  while x != subclass:
+    if x == nil:
+      cache[0] = obj
+      return false
+    x = x.base
+  cache[1] = obj
+  return true
+
+proc isObjWithCache(obj, subclass: PNimType;
+                    cache: var ObjCheckCache): bool {.compilerProc, inline.} =
+  if obj == subclass: return true
+  if obj.base == subclass: return true
+  if cache[0] == obj: return false
+  if cache[1] == obj: return true
+  return isObjSlowPath(obj, subclass, cache)
+
 proc isObj(obj, subclass: PNimType): bool {.compilerproc.} =
   # checks if obj is of type subclass:
   var x = obj
diff --git a/lib/system/excpt.nim b/lib/system/excpt.nim
index 2f7c5ed51..63a61183f 100644
--- a/lib/system/excpt.nim
+++ b/lib/system/excpt.nim
@@ -77,7 +77,7 @@ proc popCurrentException {.compilerRtl, inl.} =
 
 # some platforms have native support for stack traces:
 const
-  nativeStackTraceSupported = (defined(macosx) or defined(linux)) and 
+  nativeStackTraceSupported* = (defined(macosx) or defined(linux)) and
                               not nimrodStackTrace
   hasSomeStackTrace = nimrodStackTrace or 
     defined(nativeStackTrace) and nativeStackTraceSupported
@@ -298,7 +298,13 @@ when not defined(noSignalHandler):
       elif s == SIGILL: action("SIGILL: Illegal operation.\n")
       elif s == SIGBUS: 
         action("SIGBUS: Illegal storage access. (Attempt to read from nil?)\n")
-      else: action("unknown signal\n")
+      else:
+        block platformSpecificSignal:
+          when defined(SIGPIPE):
+            if s == SIGPIPE:
+              action("SIGPIPE: Pipe closed.\n")
+              break platformSpecificSignal
+          action("unknown signal\n")
 
     # print stack trace and quit
     when hasSomeStackTrace:
@@ -323,6 +329,8 @@ when not defined(noSignalHandler):
     c_signal(SIGFPE, signalHandler)
     c_signal(SIGILL, signalHandler)
     c_signal(SIGBUS, signalHandler)
+    when defined(SIGPIPE):
+      c_signal(SIGPIPE, signalHandler)
 
   registerSignalHandler() # call it in initialization section
 
diff --git a/lib/system/gc_ms.nim b/lib/system/gc_ms.nim
index 3c99a57e1..05bcdcc82 100644
--- a/lib/system/gc_ms.nim
+++ b/lib/system/gc_ms.nim
@@ -48,12 +48,15 @@ type
                              # non-zero count table
     stackBottom: pointer
     cycleThreshold: int
+    when useCellIds:
+      idGenerator: int
     when withBitvectors:
       allocated, marked: TCellSet
     tempStack: TCellSeq      # temporary stack for recursion elimination
     recGcLock: int           # prevent recursion via finalizers; no thread lock
     region: TMemRegion       # garbage collected region
     stat: TGcStat
+    additionalRoots: TCellSeq # dummy roots for GC_ref/unref
 
 var
   gch {.rtlThreadVar.}: TGcHeap
@@ -131,13 +134,27 @@ proc prepareDealloc(cell: PCell) =
     (cast[TFinalizer](cell.typ.finalizer))(cellToUsr(cell))
     dec(gch.recGcLock)
 
-proc nimGCref(p: pointer) {.compilerProc, inline.} = 
+proc nimGCref(p: pointer) {.compilerProc.} = 
   # we keep it from being collected by pretending it's not even allocated:
-  when withBitvectors: excl(gch.allocated, usrToCell(p))
-  else: usrToCell(p).refcount = rcBlack
-proc nimGCunref(p: pointer) {.compilerProc, inline.} = 
-  when withBitvectors: incl(gch.allocated, usrToCell(p))
-  else: usrToCell(p).refcount = rcWhite
+  when false:
+    when withBitvectors: excl(gch.allocated, usrToCell(p))
+    else: usrToCell(p).refcount = rcBlack
+  add(gch.additionalRoots, usrToCell(p))
+
+proc nimGCunref(p: pointer) {.compilerProc.} =
+  let cell = usrToCell(p)
+  var L = gch.additionalRoots.len-1
+  var i = L
+  let d = gch.additionalRoots.d
+  while i >= 0:
+    if d[i] == cell:
+      d[i] = d[L]
+      dec gch.additionalRoots.len
+      break
+    dec(i)
+  when false:
+    when withBitvectors: incl(gch.allocated, usrToCell(p))
+    else: usrToCell(p).refcount = rcWhite
 
 proc initGC() =
   when not defined(useNimRtl):
@@ -146,6 +163,7 @@ proc initGC() =
     gch.stat.maxThreshold = 0
     gch.stat.maxStackSize = 0
     init(gch.tempStack)
+    init(gch.additionalRoots)
     when withBitvectors:
       Init(gch.allocated)
       init(gch.marked)
@@ -212,8 +230,16 @@ proc rawNewObj(typ: PNimType, size: int, gch: var TGcHeap): pointer =
   res.refcount = 0
   release(gch)
   when withBitvectors: incl(gch.allocated, res)
+  when useCellIds:
+    inc gch.idGenerator
+    res.id = gch.idGenerator
   result = cellToUsr(res)
 
+when useCellIds:
+  proc getCellId*[T](x: ref T): int =
+    let p = usrToCell(cast[pointer](x))
+    result = p.id
+
 {.pop.}
 
 proc newObj(typ: PNimType, size: int): pointer {.compilerRtl.} =
@@ -262,6 +288,9 @@ proc growObj(old: pointer, newsize: int, gch: var TGcHeap): pointer =
   else:
     zeroMem(ol, sizeof(TCell))
   when withBitvectors: incl(gch.allocated, res)
+  when useCellIds:
+    inc gch.idGenerator
+    res.id = gch.idGenerator
   release(gch)
   result = cellToUsr(res)
   when defined(memProfiler): nimProfile(newsize-oldsize)
@@ -284,6 +313,7 @@ proc mark(gch: var TGcHeap, c: PCell) =
       if not containsOrIncl(gch.marked, d):
         forAllChildren(d, waMarkPrecise)
   else:
+    # XXX no 'if c.refCount != rcBlack' here?
     c.refCount = rcBlack
     gcAssert gch.tempStack.len == 0, "stack not empty!"
     forAllChildren(c, waMarkPrecise)
@@ -332,8 +362,19 @@ proc sweep(gch: var TGcHeap) =
         if c.refcount == rcBlack: c.refcount = rcWhite
         else: freeCyclicCell(gch, c)
 
+when false:
+  proc newGcInvariant*() =
+    for x in allObjects(gch.region):
+      if isCell(x):
+        var c = cast[PCell](x)
+        if c.typ == nil:
+          writeStackTrace()
+          quit 1
+
 proc markGlobals(gch: var TGcHeap) =
   for i in 0 .. < globalMarkersLen: globalMarkers[i]()
+  let d = gch.additionalRoots.d
+  for i in 0 .. < gch.additionalRoots.len: mark(gch, d[i])
 
 proc gcMark(gch: var TGcHeap, p: pointer) {.inline.} =
   # the addresses are not as cells on the stack, so turn them to cells:
diff --git a/lib/system/inclrtl.nim b/lib/system/inclrtl.nim
index 12eb90162..5c82db4da 100644
--- a/lib/system/inclrtl.nim
+++ b/lib/system/inclrtl.nim
@@ -1,7 +1,7 @@
 #
 #
 #            Nimrod's Runtime Library
-#        (c) Copyright 2013 Andreas Rumpf
+#        (c) Copyright 2014 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
diff --git a/lib/system/sets.nim b/lib/system/sets.nim
index 043d37533..794c65cb8 100644
--- a/lib/system/sets.nim
+++ b/lib/system/sets.nim
@@ -10,7 +10,7 @@
 # set handling
 
 type
-  TNimSet = array [0..4*2048-1, int8]
+  TNimSet = array [0..4*2048-1, uint8]
 
 proc countBits32(n: int32): int {.compilerproc.} =
   var v = n
@@ -25,4 +25,4 @@ proc countBits64(n: int64): int {.compilerproc.} =
 proc cardSet(s: TNimSet, len: int): int {.compilerproc.} =
   result = 0
   for i in countup(0, len-1):
-    inc(result, countBits32(int32(ze(s[i]))))
+    inc(result, countBits32(int32(s[i])))
diff --git a/lib/system/sysio.nim b/lib/system/sysio.nim
index 02c17b92b..32d4c3e91 100644
--- a/lib/system/sysio.nim
+++ b/lib/system/sysio.nim
@@ -115,10 +115,14 @@ proc readAllBuffer(file: TFile): string =
   # bytes we need to read before the buffer is empty.
   result = ""
   var buffer = newString(BufSize)
-  var bytesRead = BufSize
-  while bytesRead == BufSize:
-    bytesRead = readBuffer(file, addr(buffer[0]), BufSize)
-    result.add(buffer)
+  while true:
+    var bytesRead = readBuffer(file, addr(buffer[0]), BufSize)
+    if bytesRead == BufSize:
+      result.add(buffer)
+    else:
+      buffer.setLen(bytesRead)
+      result.add(buffer)
+      break
   
 proc rawFileSize(file: TFile): int = 
   # this does not raise an error opposed to `getFileSize`
diff --git a/lib/system/sysspawn.nim b/lib/system/sysspawn.nim
index dabf35a3e..95cdba65d 100644
--- a/lib/system/sysspawn.nim
+++ b/lib/system/sysspawn.nim
@@ -14,30 +14,6 @@ when not defined(NimString):
 
 {.push stackTrace:off.}
 
-when (defined(x86) or defined(amd64)) and defined(gcc):
-  proc cpuRelax {.inline.} =
-    {.emit: """asm volatile("pause" ::: "memory");""".}
-elif (defined(x86) or defined(amd64)) and defined(vcc):
-  proc cpuRelax {.importc: "YieldProcessor", header: "<windows.h>".}
-elif defined(intelc):
-  proc cpuRelax {.importc: "_mm_pause", header: "xmmintrin.h".}
-elif false:
-  from os import sleep
-
-  proc cpuRelax {.inline.} = os.sleep(1)
-
-when defined(windows) and not defined(gcc):
-  proc interlockedCompareExchange(p: pointer; exchange, comparand: int32): int32
-    {.importc: "InterlockedCompareExchange", header: "<windows.h>", cdecl.}
-
-  proc cas(p: ptr bool; oldValue, newValue: bool): bool =
-    interlockedCompareExchange(p, newValue.int32, oldValue.int32) != 0
-
-else:
-  # this is valid for GCC and Intel C++
-  proc cas(p: ptr bool; oldValue, newValue: bool): bool
-    {.importc: "__sync_bool_compare_and_swap", nodecl.}
-
 # We declare our own condition variables here to get rid of the dummy lock
 # on Windows:
 
@@ -54,6 +30,9 @@ proc createCondVar(): CondVar =
     initSysLock(result.stupidLock)
     #acquireSys(result.stupidLock)
 
+proc destroyCondVar(c: var CondVar) {.inline.} =
+  deinitSysCond(c.c)
+
 proc await(cv: var CondVar) =
   when defined(posix):
     acquireSys(cv.stupidLock)
@@ -100,6 +79,26 @@ proc signal(cv: var FastCondVar) =
   #if cas(addr cv.slowPath, true, false):
   signal(cv.slow)
 
+type
+  Barrier* {.compilerProc.} = object
+    counter: int
+    cv: CondVar
+
+proc barrierEnter*(b: ptr Barrier) {.compilerProc.} =
+  atomicInc b.counter
+
+proc barrierLeave*(b: ptr Barrier) {.compilerProc.} =
+  atomicDec b.counter
+  if b.counter <= 0: signal(b.cv)
+
+proc openBarrier*(b: ptr Barrier) {.compilerProc.} =
+  b.counter = 0
+  b.cv = createCondVar()
+
+proc closeBarrier*(b: ptr Barrier) {.compilerProc.} =
+  await(b.cv)
+  destroyCondVar(b.cv)
+
 {.pop.}
 
 # ----------------------------------------------------------------------------
diff --git a/lib/system/threads.nim b/lib/system/threads.nim
index 0d52e4d09..d3b3aa457 100644
--- a/lib/system/threads.nim
+++ b/lib/system/threads.nim
@@ -243,7 +243,7 @@ when not defined(useNimRtl):
       
     # on UNIX, the GC uses ``SIGFREEZE`` to tell every thread to stop so that
     # the GC can examine the stacks?
-    proc stopTheWord() = nil
+    proc stopTheWord() = discard
     
 # We jump through some hops here to ensure that Nimrod thread procs can have
 # the Nimrod calling convention. This is needed because thread procs are