Merged

12 files changed, 125 insertions, 51 deletions

diff --git a/lib/pure/algorithm.nim b/lib/pure/algorithm.nim
index eee4fab22..b83daf245 100644
--- a/lib/pure/algorithm.nim
+++ b/lib/pure/algorithm.nim
@@ -231,7 +231,7 @@ template sortedByIt*(seq1, op: untyped): untyped =
   ##
   ##   echo people.sortedByIt((it.age, it.name))
   ##
-  var result {.gensym.} = sorted(seq1, proc(x, y: type(seq1[0])): int =
+  var result = sorted(seq1, proc(x, y: type(seq1[0])): int =
     var it {.inject.} = x
     let a = op
     it = y
diff --git a/lib/pure/collections/sequtils.nim b/lib/pure/collections/sequtils.nim
index e277ee9e8..f458b7636 100644
--- a/lib/pure/collections/sequtils.nim
+++ b/lib/pure/collections/sequtils.nim
@@ -371,7 +371,7 @@ template filterIt*(seq1, pred: untyped): untyped =
   ##      notAcceptable = filterIt(temperatures, it > 50 or it < -10)
   ##    assert acceptable == @[-2.0, 24.5, 44.31]
   ##    assert notAcceptable == @[-272.15, 99.9, -113.44]
-  var result {.gensym.} = newSeq[type(seq1[0])]()
+  var result = newSeq[type(seq1[0])]()
   for it {.inject.} in items(seq1):
     if pred: result.add(it)
   result
@@ -420,7 +420,7 @@ template allIt*(seq1, pred: untyped): bool =
   ##   let numbers = @[1, 4, 5, 8, 9, 7, 4]
   ##   assert allIt(numbers, it < 10) == true
   ##   assert allIt(numbers, it < 9) == false
-  var result {.gensym.} = true
+  var result = true
   for it {.inject.} in items(seq1):
     if not pred:
       result = false
@@ -451,7 +451,7 @@ template anyIt*(seq1, pred: untyped): bool =
   ##   let numbers = @[1, 4, 5, 8, 9, 7, 4]
   ##   assert anyIt(numbers, it > 8) == true
   ##   assert anyIt(numbers, it > 9) == false
-  var result {.gensym.} = false
+  var result = false
   for it {.inject.} in items(seq1):
     if pred:
       result = true
@@ -512,7 +512,7 @@ template foldl*(sequence, operation: untyped): untyped =
   ##   assert concatenation == "nimiscool"
   let s = sequence
   assert s.len > 0, "Can't fold empty sequences"
-  var result {.gensym.}: type(s[0])
+  var result: type(s[0])
   result = s[0]
   for i in 1..<s.len:
     let
@@ -537,7 +537,7 @@ template foldl*(sequence, operation, first): untyped =
   ##     numbers = @[0, 8, 1, 5]
   ##     digits = foldl(numbers, a & (chr(b + ord('0'))), "")
   ##   assert digits == "0815"
-  var result {.gensym.}: type(first)
+  var result: type(first)
   result = first
   for x in items(sequence):
     let
@@ -574,7 +574,7 @@ template foldr*(sequence, operation: untyped): untyped =
   ##   assert concatenation == "nimiscool"
   let s = sequence
   assert s.len > 0, "Can't fold empty sequences"
-  var result {.gensym.}: type(s[0])
+  var result: type(s[0])
   result = sequence[s.len - 1]
   for i in countdown(s.len - 2, 0):
     let
@@ -598,7 +598,7 @@ template mapIt*(seq1, typ, op: untyped): untyped =
   ##   assert strings == @["4", "8", "12", "16"]
   ## **Deprecated since version 0.12.0:** Use the ``mapIt(seq1, op)``
   ##   template instead.
-  var result {.gensym.}: seq[typ] = @[]
+  var result: seq[typ] = @[]
   for it {.inject.} in items(seq1):
     result.add(op)
   result
@@ -662,7 +662,7 @@ template newSeqWith*(len: int, init: untyped): untyped =
   ##   import random
   ##   var seqRand = newSeqWith(20, random(10))
   ##   echo seqRand
-  var result {.gensym.} = newSeq[type(init)](len)
+  var result = newSeq[type(init)](len)
   for i in 0 .. <len:
     result[i] = init
   result
diff --git a/lib/pure/collections/tableimpl.nim b/lib/pure/collections/tableimpl.nim
index be3507137..a3dfd43a1 100644
--- a/lib/pure/collections/tableimpl.nim
+++ b/lib/pure/collections/tableimpl.nim
@@ -142,7 +142,8 @@ template delImpl() {.dirty.} =
 
 template clearImpl() {.dirty.} =
   for i in 0 .. <t.data.len:
-    t.data[i].hcode = 0
+    when compiles(t.data[i].hcode): # CountTable records don't contain a hcode
+      t.data[i].hcode = 0
     t.data[i].key = default(type(t.data[i].key))
     t.data[i].val = default(type(t.data[i].val))
   t.counter = 0
diff --git a/lib/pure/collections/tables.nim b/lib/pure/collections/tables.nim
index 9308095aa..3f06762ae 100644
--- a/lib/pure/collections/tables.nim
+++ b/lib/pure/collections/tables.nim
@@ -118,7 +118,7 @@ template dataLen(t): untyped = len(t.data)
 
 include tableimpl
 
-proc clear*[A, B](t: Table[A, B] | TableRef[A, B]) =
+proc clear*[A, B](t: var Table[A, B] | TableRef[A, B]) =
   ## Resets the table so that it is empty.
   clearImpl()
 
@@ -457,7 +457,7 @@ proc len*[A, B](t: OrderedTable[A, B]): int {.inline.} =
   ## returns the number of keys in `t`.
   result = t.counter
 
-proc clear*[A, B](t: OrderedTable[A, B] | OrderedTableRef[A, B]) =
+proc clear*[A, B](t: var OrderedTable[A, B] | OrderedTableRef[A, B]) =
   ## Resets the table so that it is empty.
   clearImpl()
   t.first = -1
@@ -786,7 +786,7 @@ proc len*[A](t: CountTable[A]): int =
   ## returns the number of keys in `t`.
   result = t.counter
 
-proc clear*[A](t: CountTable[A] | CountTableRef[A]) =
+proc clear*[A](t: var CountTable[A] | CountTableRef[A]) =
   ## Resets the table so that it is empty.
   clearImpl()
   t.counter = 0
diff --git a/lib/pure/concurrency/threadpool.nim b/lib/pure/concurrency/threadpool.nim
index 9490dbbd5..8cdb83e19 100644
--- a/lib/pure/concurrency/threadpool.nim
+++ b/lib/pure/concurrency/threadpool.nim
@@ -297,9 +297,20 @@ var
   gSomeReady : Semaphore
   readyWorker: ptr Worker
 
+# A workaround for recursion deadlock issue
+# https://github.com/nim-lang/Nim/issues/4597
+var
+  numSlavesLock: Lock
+  numSlavesRunning {.guard: numSlavesLock}: int
+  numSlavesWaiting {.guard: numSlavesLock}: int
+  isSlave {.threadvar.}: bool
+
+numSlavesLock.initLock
+
 gSomeReady.initSemaphore()
 
 proc slave(w: ptr Worker) {.thread.} =
+  isSlave = true
   while true:
     when declared(atomicStoreN):
       atomicStoreN(addr(w.ready), true, ATOMIC_SEQ_CST)
@@ -311,7 +322,15 @@ proc slave(w: ptr Worker) {.thread.} =
     # XXX Somebody needs to look into this (why does this assertion fail
     # in Visual Studio?)
     when not defined(vcc): assert(not w.ready)
+
+    withLock numSlavesLock:
+      inc numSlavesRunning
+
     w.f(w, w.data)
+
+    withLock numSlavesLock:
+      dec numSlavesRunning
+
     if w.q.len != 0: w.cleanFlowVars
     if w.shutdown:
       w.shutdown = false
@@ -464,10 +483,34 @@ proc nimSpawn3(fn: WorkerProc; data: pointer) {.compilerProc.} =
         fn(self, data)
         await(self.taskStarted)
         return
-      else:
-        await(gSomeReady)
-    else:
-      await(gSomeReady)
+
+    if isSlave:
+      # Run under lock until `numSlavesWaiting` increment to avoid a
+      # race (otherwise two last threads might start waiting together)
+      withLock numSlavesLock:
+        if numSlavesRunning <= numSlavesWaiting + 1:
+          # All the other slaves are waiting
+          # If we wait now, we-re deadlocked until
+          # an external spawn happens !
+          if currentPoolSize < maxPoolSize:
+            if not workersData[currentPoolSize].initialized:
+              activateWorkerThread(currentPoolSize)
+            let w = addr(workersData[currentPoolSize])
+            atomicInc currentPoolSize
+            if selectWorker(w, fn, data):
+              return
+          else:
+            # There is no place in the pool. We're deadlocked.
+            # echo "Deadlock!"
+            discard
+
+        inc numSlavesWaiting
+
+    await(gSomeReady)
+
+    if isSlave:
+      withLock numSlavesLock:
+        dec numSlavesWaiting
 
 var
   distinguishedLock: Lock
diff --git a/lib/pure/ioselects/ioselectors_kqueue.nim b/lib/pure/ioselects/ioselectors_kqueue.nim
index 29a201863..3e86f19aa 100644
--- a/lib/pure/ioselects/ioselectors_kqueue.nim
+++ b/lib/pure/ioselects/ioselectors_kqueue.nim
@@ -16,6 +16,10 @@ const
   MAX_KQUEUE_CHANGE_EVENTS = 64
   # Maximum number of events that can be returned.
   MAX_KQUEUE_RESULT_EVENTS = 64
+  # SIG_IGN and SIG_DFL declared in posix.nim as variables, but we need them
+  # to be constants and GC-safe.
+  SIG_DFL = cast[proc(x: cint) {.noconv,gcsafe.}](0)
+  SIG_IGN = cast[proc(x: cint) {.noconv,gcsafe.}](1)
 
 when defined(macosx) or defined(freebsd):
   when defined(macosx):
diff --git a/lib/pure/json.nim b/lib/pure/json.nim
index 19947fbc2..c76bbc3b5 100644
--- a/lib/pure/json.nim
+++ b/lib/pure/json.nim
@@ -580,7 +580,7 @@ type
     of JNull:
       nil
     of JObject:
-      fields*: Table[string, JsonNode]
+      fields*: OrderedTable[string, JsonNode]
     of JArray:
       elems*: seq[JsonNode]
 
@@ -630,7 +630,7 @@ proc newJObject*(): JsonNode =
   ## Creates a new `JObject JsonNode`
   new(result)
   result.kind = JObject
-  result.fields = initTable[string, JsonNode](4)
+  result.fields = initOrderedTable[string, JsonNode](4)
 
 proc newJArray*(): JsonNode =
   ## Creates a new `JArray JsonNode`
@@ -670,8 +670,8 @@ proc getBVal*(n: JsonNode, default: bool = false): bool =
   else: return n.bval
 
 proc getFields*(n: JsonNode,
-    default = initTable[string, JsonNode](4)):
-        Table[string, JsonNode] =
+    default = initOrderedTable[string, JsonNode](4)):
+        OrderedTable[string, JsonNode] =
   ## Retrieves the key, value pairs of a `JObject JsonNode`.
   ##
   ## Returns ``default`` if ``n`` is not a ``JObject``, or if ``n`` is nil.
@@ -760,12 +760,12 @@ proc toJson(x: NimNode): NimNode {.compiletime.} =
 
   result = prefix(result, "%")
 
-macro `%*`*(x: expr): expr =
+macro `%*`*(x: untyped): untyped =
   ## Convert an expression to a JsonNode directly, without having to specify
   ## `%` for every element.
   result = toJson(x)
 
-proc `==`* (a,b: JsonNode): bool =
+proc `==`* (a, b: JsonNode): bool =
   ## Check two nodes for equality
   if a.isNil:
     if b.isNil: return true
@@ -773,23 +773,29 @@ proc `==`* (a,b: JsonNode): bool =
   elif b.isNil or a.kind != b.kind:
     return false
   else:
-    return case a.kind
+    case a.kind
     of JString:
-      a.str == b.str
+      result = a.str == b.str
     of JInt:
-      a.num == b.num
+      result = a.num == b.num
     of JFloat:
-      a.fnum == b.fnum
+      result = a.fnum == b.fnum
     of JBool:
-      a.bval == b.bval
+      result = a.bval == b.bval
     of JNull:
-      true
+      result = true
     of JArray:
-      a.elems == b.elems
+      result = a.elems == b.elems
     of JObject:
-      a.fields == b.fields
+     # we cannot use OrderedTable's equality here as
+     # the order does not matter for equality here.
+     if a.fields.len != b.fields.len: return false
+     for key, val in a.fields:
+       if not b.fields.hasKey(key): return false
+       if b.fields[key] != val: return false
+     result = true
 
-proc hash*(n: Table[string, JsonNode]): Hash {.noSideEffect.}
+proc hash*(n: OrderedTable[string, JsonNode]): Hash {.noSideEffect.}
 
 proc hash*(n: JsonNode): Hash =
   ## Compute the hash for a JSON node
@@ -807,9 +813,9 @@ proc hash*(n: JsonNode): Hash =
   of JString:
     result = hash(n.str)
   of JNull:
-    result = hash(0)
+    result = Hash(0)
 
-proc hash*(n: Table[string, JsonNode]): Hash =
+proc hash*(n: OrderedTable[string, JsonNode]): Hash =
   for key, val in n:
     result = result xor (hash(key) !& hash(val))
   result = !$result
diff --git a/lib/pure/net.nim b/lib/pure/net.nim
index 9501f6dc7..bd208761b 100644
--- a/lib/pure/net.nim
+++ b/lib/pure/net.nim
@@ -205,6 +205,10 @@ proc newSocket*(fd: SocketHandle, domain: Domain = AF_INET,
   if buffered:
     result.currPos = 0
 
+  # Set SO_NOSIGPIPE on OS X.
+  when defined(macosx):
+    setSockOptInt(fd, SOL_SOCKET, SO_NOSIGPIPE, 1)
+
 proc newSocket*(domain, sockType, protocol: cint, buffered = true): Socket =
   ## Creates a new socket.
   ##
@@ -342,8 +346,6 @@ when defineSsl:
     result = SSLContext(context: newCTX, extraInternalIndex: 0,
         referencedData: initSet[int]())
     result.extraInternalIndex = getExtraDataIndex(result)
-    # The PSK callback functions assume the internal index is 0.
-    assert result.extraInternalIndex == 0
 
     let extraInternal = new(SslContextExtraInternal)
     result.setExtraData(result.extraInternalIndex, extraInternal)
@@ -392,6 +394,8 @@ when defineSsl:
     ##
     ## Only used in PSK ciphersuites.
     ctx.getExtraInternal().clientGetPskFunc = fun
+    assert ctx.extraInternalIndex == 0,
+          "The pskClientCallback assumes the extraInternalIndex is 0"
     ctx.context.SSL_CTX_set_psk_client_callback(
         if fun == nil: nil else: pskClientCallback)
 
diff --git a/lib/pure/oids.nim b/lib/pure/oids.nim
index fca10dab6..e4c97b260 100644
--- a/lib/pure/oids.nim
+++ b/lib/pure/oids.nim
@@ -74,8 +74,7 @@ proc genOid*(): Oid =
 
   var t = gettime(nil)
 
-  var i = int32(incr)
-  atomicInc(incr)
+  var i = int32(atomicInc(incr))
 
   if fuzz == 0:
     # racy, but fine semantically:
diff --git a/lib/pure/times.nim b/lib/pure/times.nim
index d6eb29e1c..b78a2b966 100644
--- a/lib/pure/times.nim
+++ b/lib/pure/times.nim
@@ -1248,8 +1248,11 @@ proc parse*(value, layout: string): TimeInfo =
       else:
         parseToken(info, token, value, j)
         token = ""
-  # Reset weekday as it might not have been provided and the default may be wrong
-  info.weekday = getLocalTime(toTime(info)).weekday
+  # Reset weekday (might not have been provided and the default may be wrong)
+  # and yearday (is never provided directly and therefore probably wrong)
+  let processed = getLocalTime(toTime(info))
+  info.weekday = processed.weekday
+  info.yearday = processed.yearday
   return info
 
 # Leap year calculations are adapted from:
diff --git a/lib/pure/unicode.nim b/lib/pure/unicode.nim
index 0cbe8de7a..6ba966816 100644
--- a/lib/pure/unicode.nim
+++ b/lib/pure/unicode.nim
@@ -24,7 +24,7 @@ proc `<=%`*(a, b: Rune): bool = return int(a) <=% int(b)
 proc `<%`*(a, b: Rune): bool = return int(a) <% int(b)
 proc `==`*(a, b: Rune): bool = return int(a) == int(b)
 
-template ones(n: expr): expr = ((1 shl n)-1)
+template ones(n: untyped): untyped = ((1 shl n)-1)
 
 proc runeLen*(s: string): int {.rtl, extern: "nuc$1".} =
   ## Returns the number of Unicode characters of the string ``s``
@@ -49,7 +49,7 @@ proc runeLenAt*(s: string, i: Natural): int =
   elif ord(s[i]) shr 1 == 0b1111110: result = 6
   else: result = 1
 
-template fastRuneAt*(s: string, i: int, result: expr, doInc = true) =
+template fastRuneAt*(s: string, i: int, result: untyped, doInc = true) =
   ## Returns the Unicode character ``s[i]`` in ``result``. If ``doInc == true``
   ## ``i`` is incremented by the number of bytes that have been processed.
   bind ones
@@ -1628,7 +1628,7 @@ proc reversed*(s: string): string =
     blockPos = 0
     r: Rune
 
-  template reverseUntil(pos): stmt =
+  template reverseUntil(pos) =
     var j = pos - 1
     while j > blockPos:
       result[newPos] = s[j]
diff --git a/lib/pure/unittest.nim b/lib/pure/unittest.nim
index 92ddc3e75..0fc2e441e 100644
--- a/lib/pure/unittest.nim
+++ b/lib/pure/unittest.nim
@@ -77,6 +77,15 @@ checkpoints = @[]
 proc shouldRun(testName: string): bool =
   result = true
 
+proc startSuite(name: string) =
+  template rawPrint() = echo("\n[Suite] ", name) 
+  when not defined(ECMAScript):
+    if colorOutput:
+      styledEcho styleBright, fgBlue, "\n[Suite] ", fgWhite, name
+    else: rawPrint()
+  else: rawPrint()
+
+
 template suite*(name, body) {.dirty.} =
   ## Declare a test suite identified by `name` with optional ``setup``
   ## and/or ``teardown`` section.
@@ -103,9 +112,11 @@ template suite*(name, body) {.dirty.} =
   ##
   ## .. code-block::
   ##
-  ##  [OK] 2 + 2 = 4
-  ##  [OK] (2 + -2) != 4
+  ##  [Suite] test suite for addition
+  ##    [OK] 2 + 2 = 4
+  ##    [OK] (2 + -2) != 4
   block:
+    bind startSuite
     template setup(setupBody: untyped) {.dirty.} =
       var testSetupIMPLFlag = true
       template testSetupIMPL: untyped {.dirty.} = setupBody
@@ -114,14 +125,16 @@ template suite*(name, body) {.dirty.} =
       var testTeardownIMPLFlag = true
       template testTeardownIMPL: untyped {.dirty.} = teardownBody
 
+    let testInSuiteImplFlag = true
+    startSuite name
     body
 
-proc testDone(name: string, s: TestStatus) =
+proc testDone(name: string, s: TestStatus, indent: bool) =
   if s == FAILED:
     programResult += 1
-
+  let prefix = if indent: "  " else: ""
   if outputLevel != PRINT_NONE and (outputLevel == PRINT_ALL or s == FAILED):
-    template rawPrint() = echo("[", $s, "] ", name)
+    template rawPrint() = echo(prefix, "[", $s, "] ", name)
     when not defined(ECMAScript):
       if colorOutput and not defined(ECMAScript):
         var color = case s
@@ -129,7 +142,7 @@ proc testDone(name: string, s: TestStatus) =
                     of FAILED: fgRed
                     of SKIPPED: fgYellow
                     else: fgWhite
-        styledEcho styleBright, color, "[", $s, "] ", fgWhite, name
+        styledEcho styleBright, color, prefix, "[", $s, "] ", fgWhite, name
       else:
         rawPrint()
     else:
@@ -168,7 +181,7 @@ template test*(name, body) {.dirty.} =
       fail()
 
     finally:
-      testDone name, testStatusIMPL
+      testDone name, testStatusIMPL, declared(testInSuiteImplFlag)
 
 proc checkpoint*(msg: string) =
   ## Set a checkpoint identified by `msg`. Upon test failure all
@@ -198,8 +211,9 @@ template fail* =
   ##
   ## outputs "Checkpoint A" before quitting.
   bind checkpoints
+  let prefix = if declared(testInSuiteImplFlag): "    " else: ""
   for msg in items(checkpoints):
-    echo msg
+    echo prefix, msg
 
   when not defined(ECMAScript):
     if abortOnError: quit(1)