Merge branch 'devel' into sighashes

15 files changed, 571 insertions, 133 deletions

diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim
index 8c4a0e41d..01088c2e7 100644
--- a/lib/pure/asyncdispatch.nim
+++ b/lib/pure/asyncdispatch.nim
@@ -11,7 +11,7 @@ include "system/inclrtl"
 
 import os, oids, tables, strutils, times, heapqueue
 
-import nativesockets, net, queues
+import nativesockets, net, deques
 
 export Port, SocketFlag
 
@@ -164,7 +164,7 @@ include includes/asyncfutures
 type
   PDispatcherBase = ref object of RootRef
     timers: HeapQueue[tuple[finishAt: float, fut: Future[void]]]
-    callbacks: Queue[proc ()]
+    callbacks: Deque[proc ()]
 
 proc processTimers(p: PDispatcherBase) {.inline.} =
   while p.timers.len > 0 and epochTime() >= p.timers[0].finishAt:
@@ -172,7 +172,7 @@ proc processTimers(p: PDispatcherBase) {.inline.} =
 
 proc processPendingCallbacks(p: PDispatcherBase) =
   while p.callbacks.len > 0:
-    var cb = p.callbacks.dequeue()
+    var cb = p.callbacks.popFirst()
     cb()
 
 proc adjustedTimeout(p: PDispatcherBase, timeout: int): int {.inline.} =
@@ -230,7 +230,7 @@ when defined(windows) or defined(nimdoc):
     result.ioPort = createIoCompletionPort(INVALID_HANDLE_VALUE, 0, 0, 1)
     result.handles = initSet[AsyncFD]()
     result.timers.newHeapQueue()
-    result.callbacks = initQueue[proc ()](64)
+    result.callbacks = initDeque[proc ()](64)
 
   var gDisp{.threadvar.}: PDispatcher ## Global dispatcher
   proc getGlobalDispatcher*(): PDispatcher =
@@ -987,7 +987,7 @@ else:
     new result
     result.selector = newSelector()
     result.timers.newHeapQueue()
-    result.callbacks = initQueue[proc ()](64)
+    result.callbacks = initDeque[proc ()](64)
 
   var gDisp{.threadvar.}: PDispatcher ## Global dispatcher
   proc getGlobalDispatcher*(): PDispatcher =
@@ -1043,6 +1043,26 @@ else:
     p.selector[fd.SocketHandle].data.PData.writeCBs.add(cb)
     update(fd, p.selector[fd.SocketHandle].events + {EvWrite})
 
+  template processCallbacks(callbacks: expr) =
+    # Callback may add items to ``callbacks`` which causes issues if
+    # we are iterating over it at the same time. We therefore
+    # make a copy to iterate over.
+    let currentCBs = callbacks
+    callbacks = @[]
+    # Using another sequence because callbacks themselves can add
+    # other callbacks.
+    var newCBs: seq[Callback] = @[]
+    for cb in currentCBs:
+      if newCBs.len > 0:
+        # A callback has already returned with EAGAIN, don't call any
+        # others until next `poll`.
+        newCBs.add(cb)
+      else:
+        if not cb(data.fd):
+          # Callback wants to be called again.
+          newCBs.add(cb)
+    callbacks = newCBs & callbacks
+
   proc poll*(timeout = 500) =
     let p = getGlobalDispatcher()
 
@@ -1056,23 +1076,10 @@ else:
         # `recv(...)` routines.
 
         if EvRead in info.events or info.events == {EvError}:
-          # Callback may add items to ``data.readCBs`` which causes issues if
-          # we are iterating over ``data.readCBs`` at the same time. We therefore
-          # make a copy to iterate over.
-          let currentCBs = data.readCBs
-          data.readCBs = @[]
-          for cb in currentCBs:
-            if not cb(data.fd):
-              # Callback wants to be called again.
-              data.readCBs.add(cb)
+          processCallbacks(data.readCBs)
 
         if EvWrite in info.events or info.events == {EvError}:
-          let currentCBs = data.writeCBs
-          data.writeCBs = @[]
-          for cb in currentCBs:
-            if not cb(data.fd):
-              # Callback wants to be called again.
-              data.writeCBs.add(cb)
+          processCallbacks(data.writeCBs)
 
         if info.key in p.selector:
           var newEvents: set[Event]
@@ -1410,7 +1417,7 @@ proc recvLine*(socket: AsyncFD): Future[string] {.async, deprecated.} =
 proc callSoon*(cbproc: proc ()) =
   ## Schedule `cbproc` to be called as soon as possible.
   ## The callback is called when control returns to the event loop.
-  getGlobalDispatcher().callbacks.enqueue(cbproc)
+  getGlobalDispatcher().callbacks.addLast(cbproc)
 
 proc runForever*() =
   ## Begins a never ending global dispatcher poll loop.
diff --git a/lib/pure/asyncfile.nim b/lib/pure/asyncfile.nim
index ffe6a391e..0241e4796 100644
--- a/lib/pure/asyncfile.nim
+++ b/lib/pure/asyncfile.nim
@@ -118,8 +118,8 @@ proc readBuffer*(f: AsyncFile, buf: pointer, size: int): Future[int] =
   ## Read ``size`` bytes from the specified file asynchronously starting at
   ## the current position of the file pointer.
   ##
-  ## If the file pointer is past the end of the file then an empty string is
-  ## returned.
+  ## If the file pointer is past the end of the file then zero is returned
+  ## and no bytes are read into ``buf``
   var retFuture = newFuture[int]("asyncfile.readBuffer")
 
   when defined(windows) or defined(nimdoc):
@@ -149,7 +149,11 @@ proc readBuffer*(f: AsyncFile, buf: pointer, size: int): Future[int] =
       let err = osLastError()
       if err.int32 != ERROR_IO_PENDING:
         GC_unref(ol)
-        retFuture.fail(newException(OSError, osErrorMsg(err)))
+        if err.int32 == ERROR_HANDLE_EOF:
+          # This happens in Windows Server 2003
+          retFuture.complete(0)
+        else:
+          retFuture.fail(newException(OSError, osErrorMsg(err)))
     else:
       # Request completed immediately.
       var bytesRead: DWord
@@ -233,7 +237,12 @@ proc read*(f: AsyncFile, size: int): Future[string] =
           dealloc buffer
           buffer = nil
         GC_unref(ol)
-        retFuture.fail(newException(OSError, osErrorMsg(err)))
+
+        if err.int32 == ERROR_HANDLE_EOF:
+          # This happens in Windows Server 2003
+          retFuture.complete("")
+        else:
+          retFuture.fail(newException(OSError, osErrorMsg(err)))
     else:
       # Request completed immediately.
       var bytesRead: DWord
diff --git a/lib/pure/collections/deques.nim b/lib/pure/collections/deques.nim
new file mode 100644
index 000000000..c25429778
--- /dev/null
+++ b/lib/pure/collections/deques.nim
@@ -0,0 +1,266 @@
+#
+#
+#            Nim's Runtime Library
+#        (c) Copyright 2012 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## Implementation of a `deque`:idx: (double-ended queue).
+## The underlying implementation uses a ``seq``.
+##
+## None of the procs that get an individual value from the deque can be used
+## on an empty deque.
+## If compiled with `boundChecks` option, those procs will raise an `IndexError`
+## on such access. This should not be relied upon, as `-d:release` will
+## disable those checks and may return garbage or crash the program.
+##
+## As such, a check to see if the deque is empty is needed before any
+## access, unless your program logic guarantees it indirectly.
+##
+## .. code-block:: Nim
+##   proc foo(a, b: Positive) =  # assume random positive values for `a` and `b`
+##     var deq = initDeque[int]()  # initializes the object
+##     for i in 1 ..< a: deq.addLast i  # populates the deque
+##
+##     if b < deq.len:  # checking before indexed access
+##       echo "The element at index position ", b, " is ", deq[b]
+##
+##     # The following two lines don't need any checking on access due to the
+##     # logic of the program, but that would not be the case if `a` could be 0.
+##     assert deq.peekFirst == 1
+##     assert deq.peekLast == a
+##
+##     while deq.len > 0:  # checking if the deque is empty
+##       echo deq.removeLast()
+##
+## Note: For inter thread communication use
+## a `Channel <channels.html>`_ instead.
+
+import math
+
+type
+  Deque*[T] = object
+    ## A double-ended queue backed with a ringed seq buffer.
+    data: seq[T]
+    head, tail, count, mask: int
+
+proc initDeque*[T](initialSize: int = 4): Deque[T] =
+  ## Create a new deque.
+  ## Optionally, the initial capacity can be reserved via `initialSize` as a
+  ## performance optimization. The length of a newly created deque will still
+  ## be 0.
+  ##
+  ## `initialSize` needs to be a power of two. If you need to accept runtime
+  ## values for this you could use the ``nextPowerOfTwo`` proc from the
+  ## `math <math.html>`_ module.
+  assert isPowerOfTwo(initialSize)
+  result.mask = initialSize-1
+  newSeq(result.data, initialSize)
+
+proc len*[T](deq: Deque[T]): int {.inline.} =
+  ## Return the number of elements of `deq`.
+  result = deq.count
+
+template emptyCheck(deq) =
+  # Bounds check for the regular deque access.
+  when compileOption("boundChecks"):
+    if unlikely(deq.count < 1):
+      raise newException(IndexError, "Empty deque.")
+
+template xBoundsCheck(deq, i) =
+  # Bounds check for the array like accesses.
+  when compileOption("boundChecks"):  # d:release should disable this.
+    if unlikely(i >= deq.count):  # x < deq.low is taken care by the Natural parameter
+      raise newException(IndexError,
+                         "Out of bounds: " & $i & " > " & $(deq.count - 1))
+
+proc `[]`*[T](deq: Deque[T], i: Natural) : T {.inline.} =
+  ## Access the i-th element of `deq` by order from first to last.
+  ## deq[0] is the first, deq[^1] is the last.
+  xBoundsCheck(deq, i)
+  return deq.data[(deq.first + i) and deq.mask]
+
+proc `[]`*[T](deq: var Deque[T], i: Natural): var T {.inline.} =
+  ## Access the i-th element of `deq` and returns a mutable
+  ## reference to it.
+  xBoundsCheck(deq, i)
+  return deq.data[(deq.head + i) and deq.mask]
+
+proc `[]=`* [T] (deq: var Deque[T], i: Natural, val : T) {.inline.} =
+  ## Change the i-th element of `deq`.
+  xBoundsCheck(deq, i)
+  deq.data[(deq.head + i) and deq.mask] = val
+
+iterator items*[T](deq: Deque[T]): T =
+  ## Yield every element of `deq`.
+  var i = deq.head
+  for c in 0 ..< deq.count:
+    yield deq.data[i]
+    i = (i + 1) and deq.mask
+
+iterator mitems*[T](deq: var Deque[T]): var T =
+  ## Yield every element of `deq`.
+  var i = deq.head
+  for c in 0 ..< deq.count:
+    yield deq.data[i]
+    i = (i + 1) and deq.mask
+
+iterator pairs*[T](deq: Deque[T]): tuple[key: int, val: T] =
+  ## Yield every (position, value) of `deq`.
+  var i = deq.head
+  for c in 0 ..< deq.count:
+    yield (c, deq.data[i])
+    i = (i + 1) and deq.mask
+
+proc contains*[T](deq: Deque[T], item: T): bool {.inline.} =
+  ## Return true if `item` is in `deq` or false if not found. Usually used
+  ## via the ``in`` operator. It is the equivalent of ``deq.find(item) >= 0``.
+  ##
+  ## .. code-block:: Nim
+  ##   if x in q:
+  ##     assert q.contains x
+  for e in deq:
+    if e == item: return true
+  return false
+
+proc expandIfNeeded[T](deq: var Deque[T]) =
+  var cap = deq.mask + 1
+  if unlikely(deq.count >= cap):
+    var n = newSeq[T](cap * 2)
+    for i, x in deq:  # don't use copyMem because the GC and because it's slower.
+      shallowCopy(n[i], x)
+    shallowCopy(deq.data, n)
+    deq.mask = cap * 2 - 1
+    deq.tail = deq.count
+    deq.head = 0
+
+proc addFirst*[T](deq: var Deque[T], item: T) =
+  ## Add an `item` to the beginning of the `deq`.
+  expandIfNeeded(deq)
+  inc deq.count
+  deq.head = (deq.head - 1) and deq.mask
+  deq.data[deq.head] = item
+
+proc addLast*[T](deq: var Deque[T], item: T) =
+  ## Add an `item` to the end of the `deq`.
+  expandIfNeeded(deq)
+  inc deq.count
+  deq.data[deq.tail] = item
+  deq.tail = (deq.tail + 1) and deq.mask
+
+proc peekFirst*[T](deq: Deque[T]): T {.inline.}=
+  ## Returns the first element of `deq`, but does not remove it from the deque.
+  emptyCheck(deq)
+  result = deq.data[deq.head]
+
+proc peekLast*[T](deq: Deque[T]): T {.inline.} =
+  ## Returns the last element of `deq`, but does not remove it from the deque.
+  emptyCheck(deq)
+  result = deq.data[(deq.tail - 1) and deq.mask]
+
+proc default[T](t: typedesc[T]): T {.inline.} = discard
+proc popFirst*[T](deq: var Deque[T]): T {.inline, discardable.} =
+  ## Remove and returns the first element of the `deq`.
+  emptyCheck(deq)
+  dec deq.count
+  result = deq.data[deq.head]
+  deq.data[deq.head] = default(type(result))
+  deq.head = (deq.head + 1) and deq.mask
+
+proc popLast*[T](deq: var Deque[T]): T {.inline, discardable.} =
+  ## Remove and returns the last element of the `deq`.
+  emptyCheck(deq)
+  dec deq.count
+  deq.tail = (deq.tail - 1) and deq.mask
+  result = deq.data[deq.tail]
+  deq.data[deq.tail] = default(type(result))
+
+proc `$`*[T](deq: Deque[T]): string =
+  ## Turn a deque into its string representation.
+  result = "["
+  for x in deq:
+    if result.len > 1: result.add(", ")
+    result.add($x)
+  result.add("]")
+
+when isMainModule:
+  var deq = initDeque[int](1)
+  deq.addLast(4)
+  deq.addFirst(9)
+  deq.addFirst(123)
+  var first = deq.popFirst()
+  deq.addLast(56)
+  assert(deq.peekLast() == 56)
+  deq.addLast(6)
+  assert(deq.peekLast() == 6)
+  var second = deq.popFirst()
+  deq.addLast(789)
+  assert(deq.peekLast() == 789)
+
+  assert first == 123
+  assert second == 9
+  assert($deq == "[4, 56, 6, 789]")
+
+  assert deq[0] == deq.peekFirst and deq.peekFirst == 4
+  assert deq[^1] == deq.peekLast and deq.peekLast == 789
+  deq[0] = 42
+  deq[^1] = 7
+
+  assert 6 in deq and 789 notin deq
+  assert deq.find(6) >= 0
+  assert deq.find(789) < 0
+
+  for i in -2 .. 10:
+    if i in deq:
+      assert deq.contains(i) and deq.find(i) >= 0
+    else:
+      assert(not deq.contains(i) and deq.find(i) < 0)
+
+  when compileOption("boundChecks"):
+    try:
+      echo deq[99]
+      assert false
+    except IndexError:
+      discard
+
+    try:
+      assert deq.len == 4
+      for i in 0 ..< 5: deq.popFirst()
+      assert false
+    except IndexError:
+      discard
+
+  # grabs some types of resize error.
+  deq = initDeque[int]()
+  for i in 1 .. 4: deq.addLast i
+  deq.popFirst()
+  deq.popLast()
+  for i in 5 .. 8: deq.addFirst i
+  assert $deq == "[8, 7, 6, 5, 2, 3]"
+
+  # Similar to proc from the documentation example
+  proc foo(a, b: Positive) = # assume random positive values for `a` and `b`.
+    var deq = initDeque[int]()
+    assert deq.len == 0
+    for i in 1 .. a: deq.addLast i
+
+    if b < deq.len: # checking before indexed access.
+      assert deq[b] == b + 1
+
+    # The following two lines don't need any checking on access due to the logic
+    # of the program, but that would not be the case if `a` could be 0.
+    assert deq.peekFirst == 1
+    assert deq.peekLast == a
+
+    while deq.len > 0: # checking if the deque is empty
+      assert deq.popFirst() > 0
+
+  #foo(0,0)
+  foo(8,5)
+  foo(10,9)
+  foo(1,1)
+  foo(2,1)
+  foo(1,5)
+  foo(3,2)
\ No newline at end of file
diff --git a/lib/pure/collections/queues.nim b/lib/pure/collections/queues.nim
index 399e4d413..e4d7eeef1 100644
--- a/lib/pure/collections/queues.nim
+++ b/lib/pure/collections/queues.nim
@@ -39,8 +39,10 @@
 
 import math
 
+{.warning: "`queues` module is deprecated - use `deques` instead".}
+
 type
-  Queue*[T] = object ## A queue.
+  Queue* {.deprecated.} [T] = object ## A queue.
     data: seq[T]
     rd, wr, count, mask: int
 
diff --git a/lib/pure/collections/tableimpl.nim b/lib/pure/collections/tableimpl.nim
index a3dfd43a1..674fdddd2 100644
--- a/lib/pure/collections/tableimpl.nim
+++ b/lib/pure/collections/tableimpl.nim
@@ -39,16 +39,22 @@ template rawGetKnownHCImpl() {.dirty.} =
     h = nextTry(h, maxHash(t))
   result = -1 - h                   # < 0 => MISSING; insert idx = -1 - result
 
-template rawGetImpl() {.dirty.} =
+template genHashImpl(key, hc: typed) =
   hc = hash(key)
   if hc == 0:       # This almost never taken branch should be very predictable.
     hc = 314159265  # Value doesn't matter; Any non-zero favorite is fine.
+
+template genHash(key: typed): Hash =
+  var res: Hash
+  genHashImpl(key, res)
+  res
+
+template rawGetImpl() {.dirty.} =
+  genHashImpl(key, hc)
   rawGetKnownHCImpl()
 
 template rawGetDeepImpl() {.dirty.} =   # Search algo for unconditional add
-  hc = hash(key)
-  if hc == 0:
-    hc = 314159265
+  genHashImpl(key, hc)
   var h: Hash = hc and maxHash(t)
   while isFilled(t.data[h].hcode):
     h = nextTry(h, maxHash(t))
diff --git a/lib/pure/collections/tables.nim b/lib/pure/collections/tables.nim
index bee0a41b2..e6e72d9ed 100644
--- a/lib/pure/collections/tables.nim
+++ b/lib/pure/collections/tables.nim
@@ -224,7 +224,7 @@ template withValue*[A, B](t: var Table[A, B], key: A,
 
 iterator allValues*[A, B](t: Table[A, B]; key: A): B =
   ## iterates over any value in the table `t` that belongs to the given `key`.
-  var h: Hash = hash(key) and high(t.data)
+  var h: Hash = genHash(key) and high(t.data)
   while isFilled(t.data[h].hcode):
     if t.data[h].key == key:
       yield t.data[h].val
@@ -479,7 +479,7 @@ proc clear*[A, B](t: var OrderedTableRef[A, B]) =
   ## Resets the table so that is is empty.
   clear(t[])
 
-template forAllOrderedPairs(yieldStmt: untyped) {.oldimmediate, dirty.} =
+template forAllOrderedPairs(yieldStmt: untyped): typed {.dirty.} =
   var h = t.first
   while h >= 0:
     var nxt = t.data[h].next
@@ -674,13 +674,6 @@ proc len*[A, B](t: OrderedTableRef[A, B]): int {.inline.} =
   ## returns the number of keys in `t`.
   result = t.counter
 
-template forAllOrderedPairs(yieldStmt: untyped) {.oldimmediate, dirty.} =
-  var h = t.first
-  while h >= 0:
-    var nxt = t.data[h].next
-    if isFilled(t.data[h].hcode): yieldStmt
-    h = nxt
-
 iterator pairs*[A, B](t: OrderedTableRef[A, B]): (A, B) =
   ## iterates over any (key, value) pair in the table `t` in insertion
   ## order.
@@ -785,20 +778,22 @@ proc sort*[A, B](t: OrderedTableRef[A, B],
 
 proc del*[A, B](t: var OrderedTable[A, B], key: A) =
   ## deletes `key` from ordered hash table `t`. O(n) comlexity.
-  var prev = -1
-  let hc = hash(key)
-  forAllOrderedPairs:
-    if t.data[h].hcode == hc:
-      if t.first == h:
-        t.first = t.data[h].next
+  var n: OrderedKeyValuePairSeq[A, B]
+  newSeq(n, len(t.data))
+  var h = t.first
+  t.first = -1
+  t.last = -1
+  swap(t.data, n)
+  let hc = genHash(key)
+  while h >= 0:
+    var nxt = n[h].next
+    if isFilled(n[h].hcode):
+      if n[h].hcode == hc and n[h].key == key:
+        dec t.counter
       else:
-        t.data[prev].next = t.data[h].next
-      var zeroValue : type(t.data[h])
-      t.data[h] = zeroValue
-      dec t.counter
-      break
-    else:
-      prev = h
+        var j = -1 - rawGetKnownHC(t, n[h].key, n[h].hcode)
+        rawInsert(t, t.data, n[h].key, n[h].val, n[h].hcode, j)
+    h = nxt
 
 proc del*[A, B](t: var OrderedTableRef[A, B], key: A) =
   ## deletes `key` from ordered hash table `t`. O(n) comlexity.
@@ -1164,6 +1159,20 @@ when isMainModule:
       doAssert(prev < i)
       prev = i
 
+  block: # Deletion from OrderedTable should account for collision groups. See issue #5057.
+    # The bug is reproducible only with exact keys
+    const key1 = "boy_jackpot.inGamma"
+    const key2 = "boy_jackpot.outBlack"
+
+    var t = {
+        key1: 0,
+        key2: 0
+    }.toOrderedTable()
+
+    t.del(key1)
+    assert(t.len == 1)
+    assert(key2 in t)
+
   var
     t1 = initCountTable[string]()
     t2 = initCountTable[string]()
diff --git a/lib/pure/includes/asyncfutures.nim b/lib/pure/includes/asyncfutures.nim
index dfcfa37a0..029c5f157 100644
--- a/lib/pure/includes/asyncfutures.nim
+++ b/lib/pure/includes/asyncfutures.nim
@@ -263,13 +263,13 @@ proc all*[T](futs: varargs[Future[T]]): auto =
 
     for fut in futs:
       fut.callback = proc(f: Future[T]) =
-        if f.failed:
-          retFuture.fail(f.error)
-        elif not retFuture.finished:
-          inc(completedFutures)
-
-          if completedFutures == totalFutures:
-            retFuture.complete()
+        inc(completedFutures)
+        if not retFuture.finished:
+          if f.failed:
+            retFuture.fail(f.error)
+          else:
+            if completedFutures == totalFutures:
+              retFuture.complete()
 
     if totalFutures == 0:
       retFuture.complete()
@@ -285,14 +285,15 @@ proc all*[T](futs: varargs[Future[T]]): auto =
     for i, fut in futs:
       proc setCallback(i: int) =
         fut.callback = proc(f: Future[T]) =
-          if f.failed:
-            retFuture.fail(f.error)
-          elif not retFuture.finished:
-            retValues[i] = f.read()
-            inc(completedFutures)
-
-            if completedFutures == len(retValues):
-              retFuture.complete(retValues)
+          inc(completedFutures)
+          if not retFuture.finished:
+            if f.failed:
+              retFuture.fail(f.error)
+            else:
+              retValues[i] = f.read()
+
+              if completedFutures == len(retValues):
+                retFuture.complete(retValues)
 
       setCallback(i)
 
diff --git a/lib/pure/json.nim b/lib/pure/json.nim
index 0b7908c02..5fff7352f 100644
--- a/lib/pure/json.nim
+++ b/lib/pure/json.nim
@@ -954,9 +954,11 @@ proc newIndent(curr, indent: int, ml: bool): int =
 proc nl(s: var string, ml: bool) =
   if ml: s.add("\n")
 
-proc escapeJson*(s: string): string =
+proc escapeJson*(s: string; result: var string) =
   ## Converts a string `s` to its JSON representation.
-  result = newStringOfCap(s.len + s.len shr 3)
+  ## Appends to ``result``.
+  const
+    HexChars = "0123456789ABCDEF"
   result.add("\"")
   for x in runes(s):
     var r = int(x)
@@ -967,10 +969,19 @@ proc escapeJson*(s: string): string =
       of '\\': result.add("\\\\")
       else: result.add(c)
     else:
-      result.add("\\u")
-      result.add(toHex(r, 4))
+      # toHex inlined for more speed (saves stupid string allocations):
+      result.add("\\u0000")
+      let start = result.len - 4
+      for j in countdown(3, 0):
+        result[j+start] = HexChars[r and 0xF]
+        r = r shr 4
   result.add("\"")
 
+proc escapeJson*(s: string): string =
+  ## Converts a string `s` to its JSON representation.
+  result = newStringOfCap(s.len + s.len shr 3)
+  escapeJson(s, result)
+
 proc toPretty(result: var string, node: JsonNode, indent = 2, ml = true,
               lstArr = false, currIndent = 0) =
   case node.kind
@@ -988,7 +999,7 @@ proc toPretty(result: var string, node: JsonNode, indent = 2, ml = true,
         inc i
         # Need to indent more than {
         result.indent(newIndent(currIndent, indent, ml))
-        result.add(escapeJson(key))
+        escapeJson(key, result)
         result.add(": ")
         toPretty(result, val, indent, ml, false,
                  newIndent(currIndent, indent, ml))
@@ -999,16 +1010,19 @@ proc toPretty(result: var string, node: JsonNode, indent = 2, ml = true,
       result.add("{}")
   of JString:
     if lstArr: result.indent(currIndent)
-    result.add(escapeJson(node.str))
+    escapeJson(node.str, result)
   of JInt:
     if lstArr: result.indent(currIndent)
-    result.add($node.num)
+    when defined(js): result.add($node.num)
+    else: result.add(node.num)
   of JFloat:
     if lstArr: result.indent(currIndent)
-    result.add($node.fnum)
+    # Fixme: implement new system.add ops for the JS target
+    when defined(js): result.add($node.fnum)
+    else: result.add(node.fnum)
   of JBool:
     if lstArr: result.indent(currIndent)
-    result.add($node.bval)
+    result.add(if node.bval: "true" else: "false")
   of JArray:
     if lstArr: result.indent(currIndent)
     if len(node.elems) != 0:
@@ -1057,16 +1071,18 @@ proc toUgly*(result: var string, node: JsonNode) =
     for key, value in pairs(node.fields):
       if comma: result.add ","
       else:     comma = true
-      result.add key.escapeJson()
+      key.escapeJson(result)
       result.add ":"
       result.toUgly value
     result.add "}"
   of JString:
-    result.add node.str.escapeJson()
+    node.str.escapeJson(result)
   of JInt:
-    result.add($node.num)
+    when defined(js): result.add($node.num)
+    else: result.add(node.num)
   of JFloat:
-    result.add($node.fnum)
+    when defined(js): result.add($node.fnum)
+    else: result.add(node.fnum)
   of JBool:
     result.add(if node.bval: "true" else: "false")
   of JNull:
@@ -1394,4 +1410,6 @@ when isMainModule:
     var parsed2 = parseFile("tests/testdata/jsontest2.json")
     doAssert(parsed2{"repository", "description"}.str=="IRC Library for Haskell", "Couldn't fetch via multiply nested key using {}")
 
+  doAssert escapeJson("\10FoobarÄ") == "\"\\u000AFoobar\\u00C4\""
+
   echo("Tests succeeded!")
diff --git a/lib/pure/logging.nim b/lib/pure/logging.nim
index b23b1e5bb..5544a4b3f 100644
--- a/lib/pure/logging.nim
+++ b/lib/pure/logging.nim
@@ -172,18 +172,26 @@ when not defined(js):
     var (path, name, _) = splitFile(getAppFilename())
     result = changeFileExt(path / name, "log")
 
+  proc newFileLogger*(file: File,
+                      levelThreshold = lvlAll,
+                      fmtStr = defaultFmtStr): FileLogger =
+    ## Creates a new file logger. This logger logs to ``file``.
+    new(result)
+    result.file = file
+    result.levelThreshold = levelThreshold
+    result.fmtStr = fmtStr
+
   proc newFileLogger*(filename = defaultFilename(),
                       mode: FileMode = fmAppend,
                       levelThreshold = lvlAll,
                       fmtStr = defaultFmtStr,
                       bufSize: int = -1): FileLogger =
-    ## Creates a new file logger. This logger logs to a file.
+    ## Creates a new file logger. This logger logs to a file, specified
+    ## by ``fileName``.
     ## Use ``bufSize`` as size of the output buffer when writing the file
     ## (-1: use system defaults, 0: unbuffered, >0: fixed buffer size).
-    new(result)
-    result.levelThreshold = levelThreshold
-    result.file = open(filename, mode, bufSize = bufSize)
-    result.fmtStr = fmtStr
+    let file = open(filename, mode, bufSize = bufSize)
+    newFileLogger(file, levelThreshold, fmtStr)
 
   # ------
 
diff --git a/lib/pure/marshal.nim b/lib/pure/marshal.nim
index 36e6cf52f..c4c731acf 100644
--- a/lib/pure/marshal.nim
+++ b/lib/pure/marshal.nim
@@ -9,6 +9,7 @@
 
 ## This module contains procs for `serialization`:idx: and `deseralization`:idx:
 ## of arbitrary Nim data structures. The serialization format uses `JSON`:idx:.
+## Warning: The serialization format could change in future!
 ##
 ## **Restriction**: For objects their type is **not** serialized. This means
 ## essentially that it does not work if the object has some other runtime
@@ -29,6 +30,12 @@
 ##   a = b
 ##   echo($$a[]) # produces "{}", not "{f: 0}"
 ##
+##   # unmarshal
+##   let c = to[B]("""{"f": 2}""")
+##
+##   # marshal
+##   let s = $$c
+
 ## **Note**: The ``to`` and ``$$`` operations are available at compile-time!
 
 import streams, typeinfo, json, intsets, tables, unicode
diff --git a/lib/pure/net.nim b/lib/pure/net.nim
index 58f5e5777..863a8a6f4 100644
--- a/lib/pure/net.nim
+++ b/lib/pure/net.nim
@@ -1250,7 +1250,7 @@ proc IPv6_loopback*(): IpAddress =
     address_v6: [0'u8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1])
 
 proc `==`*(lhs, rhs: IpAddress): bool =
-  ## Compares two IpAddresses for Equality. Returns two if the addresses are equal
+  ## Compares two IpAddresses for Equality. Returns true if the addresses are equal
   if lhs.family != rhs.family: return false
   if lhs.family == IpAddressFamily.IPv4:
     for i in low(lhs.address_v4) .. high(lhs.address_v4):
diff --git a/lib/pure/nimtracker.nim b/lib/pure/nimtracker.nim
new file mode 100644
index 000000000..52fa9da77
--- /dev/null
+++ b/lib/pure/nimtracker.nim
@@ -0,0 +1,80 @@
+#
+#
+#            Nim's Runtime Library
+#        (c) Copyright 2016 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## Memory tracking support for Nim.
+
+when not defined(memTracker) and not isMainModule:
+  {.error: "Memory tracking support is turned off!".}
+
+{.push memtracker: off.}
+# we import the low level wrapper and are careful not to use Nim's
+# memory manager for anything here.
+import sqlite3
+
+var
+  dbHandle: PSqlite3
+  insertStmt: Pstmt
+
+template sbind(x: int; value) =
+  when value is cstring:
+    let ret = insertStmt.bindText(x, value, value.len.int32, SQLITE_TRANSIENT)
+    if ret != SQLITE_OK:
+      quit "could not bind value"
+  else:
+    let ret = insertStmt.bindInt64(x, value)
+    if ret != SQLITE_OK:
+      quit "could not bind value"
+
+when defined(memTracker):
+  proc logEntries(log: TrackLog) {.nimcall, locks: 0, tags: [].} =
+    for i in 0..log.count-1:
+      var success = false
+      let e = log.data[i]
+      discard sqlite3.reset(insertStmt)
+      discard clearBindings(insertStmt)
+      sbind 1, e.op
+      sbind(2, cast[int](e.address))
+      sbind 3, e.size
+      sbind 4, e.file
+      sbind 5, e.line
+      if step(insertStmt) == SQLITE_DONE:
+        success = true
+      if not success:
+        quit "could not write to database!"
+
+proc execQuery(q: string) =
+  var s: Pstmt
+  if prepare_v2(dbHandle, q, q.len.int32, s, nil) == SQLITE_OK:
+    discard step(s)
+    if finalize(s) != SQLITE_OK:
+      quit "could not finalize " & $sqlite3.errmsg(dbHandle)
+  else:
+    quit "could not prepare statement " & $sqlite3.errmsg(dbHandle)
+
+proc setupDb() =
+  execQuery """create table if not exists tracking(
+       id integer primary key,
+       op varchar not null,
+       address integer not null,
+       size integer not null,
+       file varchar not null,
+       line integer not null
+     )"""
+  execQuery "delete from tracking"
+
+if sqlite3.open("memtrack.db", dbHandle) == SQLITE_OK:
+  setupDb()
+  const query = "INSERT INTO tracking(op, address, size, file, line) values (?, ?, ?, ?, ?)"
+  if prepare_v2(dbHandle, query,
+      query.len, insertStmt, nil) == SQLITE_OK:
+    when defined(memTracker):
+      setTrackLogger logEntries
+  else:
+    quit "could not prepare statement B " & $sqlite3.errmsg(dbHandle)
+{.pop.}
diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim
index 129869373..14877eb4d 100644
--- a/lib/pure/strutils.nim
+++ b/lib/pure/strutils.nim
@@ -1309,10 +1309,12 @@ proc join*[T: not string](a: openArray[T], sep: string = ""): string {.
 type
   SkipTable = array[char, int]
 
+{.push profiler: off.}
 proc preprocessSub(sub: string, a: var SkipTable) =
   var m = len(sub)
   for i in 0..0xff: a[chr(i)] = m+1
   for i in 0..m-1: a[sub[i]] = m-i
+{.pop.}
 
 proc findAux(s, sub: string, start: int, a: SkipTable): int =
   # Fast "quick search" algorithm:
diff --git a/lib/pure/times.nim b/lib/pure/times.nim
index 1e869d301..1767a37be 100644
--- a/lib/pure/times.nim
+++ b/lib/pure/times.nim
@@ -144,8 +144,9 @@ type
     yearday*: range[0..365]   ## The number of days since January 1,
                               ## in the range 0 to 365.
                               ## Always 0 if the target is JS.
-    isDST*: bool              ## Determines whether DST is in effect. Always
-                              ## ``False`` if time is UTC.
+    isDST*: bool              ## Determines whether DST is in effect.
+                              ## Semantically, this adds another negative hour
+                              ## offset to the time in addition to the timezone.
     timezone*: int            ## The offset of the (non-DST) timezone in seconds
                               ## west of UTC. Note that the sign of this number
                               ## is the opposite of the one in a formatted
@@ -278,16 +279,20 @@ proc `+`*(ti1, ti2: TimeInterval): TimeInterval =
   carryO = `div`(ti1.months + ti2.months, 12)
   result.years = carryO + ti1.years + ti2.years
 
+proc `-`*(ti: TimeInterval): TimeInterval =
+  result = TimeInterval(
+    milliseconds: -ti.milliseconds,
+    seconds: -ti.seconds,
+    minutes: -ti.minutes,
+    hours: -ti.hours,
+    days: -ti.days,
+    months: -ti.months,
+    years: -ti.years
+  )
+
 proc `-`*(ti1, ti2: TimeInterval): TimeInterval =
   ## Subtracts TimeInterval ``ti1`` from ``ti2``.
-  result = ti1
-  result.milliseconds -= ti2.milliseconds
-  result.seconds -= ti2.seconds
-  result.minutes -= ti2.minutes
-  result.hours -= ti2.hours
-  result.days -= ti2.days
-  result.months -= ti2.months
-  result.years -= ti2.years
+  result = ti1 + (-ti2)
 
 proc isLeapYear*(year: int): bool =
   ## returns true if ``year`` is a leap year
@@ -363,16 +368,9 @@ proc `-`*(a: TimeInfo, interval: TimeInterval): TimeInfo =
   ##
   ## **Note:** This has been only briefly tested, it is inaccurate especially
   ## when you subtract so much that you reach the Julian calendar.
-  let t = toSeconds(toTime(a))
-  var intval: TimeInterval
-  intval.milliseconds = - interval.milliseconds
-  intval.seconds = - interval.seconds
-  intval.minutes = - interval.minutes
-  intval.hours = - interval.hours
-  intval.days = - interval.days
-  intval.months = - interval.months
-  intval.years = - interval.years
-  let secs = toSeconds(a, intval)
+  let
+    t = toSeconds(toTime(a))
+    secs = toSeconds(a, -interval)
   if a.timezone == 0:
     result = getGMTime(fromSeconds(t + secs))
   else:
@@ -732,6 +730,7 @@ const
   secondsInMin = 60
   secondsInHour = 60*60
   secondsInDay = 60*60*24
+  minutesInHour = 60
   epochStartYear = 1970
 
 proc formatToken(info: TimeInfo, token: string, buf: var string) =
@@ -823,28 +822,32 @@ proc formatToken(info: TimeInfo, token: string, buf: var string) =
     if fyear.len != 5: fyear = repeat('0', 5-fyear.len()) & fyear
     buf.add(fyear)
   of "z":
-    let hours = abs(info.timezone) div secondsInHour
-    if info.timezone < 0: buf.add('-')
-    else: buf.add('+')
+    let
+      nonDstTz = info.timezone - int(info.isDst) * secondsInHour
+      hours = abs(nonDstTz) div secondsInHour
+    if nonDstTz <= 0: buf.add('+')
+    else: buf.add('-')
     buf.add($hours)
   of "zz":
-    let hours = abs(info.timezone) div secondsInHour
-    if info.timezone < 0: buf.add('-')
-    else: buf.add('+')
+    let
+      nonDstTz = info.timezone - int(info.isDst) * secondsInHour
+      hours = abs(nonDstTz) div secondsInHour
+    if nonDstTz <= 0: buf.add('+')
+    else: buf.add('-')
     if hours < 10: buf.add('0')
     buf.add($hours)
   of "zzz":
     let
-      hours = abs(info.timezone) div secondsInHour
-      minutes = abs(info.timezone) mod 60
-    if info.timezone < 0: buf.add('-')
-    else: buf.add('+')
+      nonDstTz = info.timezone - int(info.isDst) * secondsInHour
+      hours = abs(nonDstTz) div secondsInHour
+      minutes = (abs(nonDstTz) div secondsInMin) mod minutesInHour
+    if nonDstTz <= 0: buf.add('+')
+    else: buf.add('-')
     if hours < 10: buf.add('0')
     buf.add($hours)
     buf.add(':')
     if minutes < 10: buf.add('0')
     buf.add($minutes)
-
   of "":
     discard
   else:
@@ -999,7 +1002,6 @@ proc parseToken(info: var TimeInfo; token, value: string; j: var int) =
   of "M":
     var pd = parseInt(value[j..j+1], sv)
     info.month = Month(sv-1)
-    info.monthday = sv
     j += pd
   of "MM":
     var month = value[j..j+1].parseInt()
@@ -1088,27 +1090,42 @@ proc parseToken(info: var TimeInfo; token, value: string; j: var int) =
     info.year = value[j..j+3].parseInt()
     j += 4
   of "z":
+    info.isDST = false
     if value[j] == '+':
       info.timezone = 0 - parseInt($value[j+1]) * secondsInHour
     elif value[j] == '-':
       info.timezone = parseInt($value[j+1]) * secondsInHour
+    elif value[j] == 'Z':
+      info.timezone = 0
+      j += 1
+      return
     else:
       raise newException(ValueError,
         "Couldn't parse timezone offset (z), got: " & value[j])
     j += 2
   of "zz":
+    info.isDST = false
     if value[j] == '+':
       info.timezone = 0 - value[j+1..j+2].parseInt() * secondsInHour
     elif value[j] == '-':
       info.timezone = value[j+1..j+2].parseInt() * secondsInHour
+    elif value[j] == 'Z':
+      info.timezone = 0
+      j += 1
+      return
     else:
       raise newException(ValueError,
         "Couldn't parse timezone offset (zz), got: " & value[j])
     j += 3
   of "zzz":
+    info.isDST = false
     var factor = 0
     if value[j] == '+': factor = -1
     elif value[j] == '-': factor = 1
+    elif value[j] == 'Z':
+      info.timezone = 0
+      j += 1
+      return
     else:
       raise newException(ValueError,
         "Couldn't parse timezone offset (zzz), got: " & value[j])
@@ -1121,8 +1138,11 @@ proc parseToken(info: var TimeInfo; token, value: string; j: var int) =
     j += token.len
 
 proc parse*(value, layout: string): TimeInfo =
-  ## This function parses a date/time string using the standard format identifiers (below)
-  ## The function defaults information not provided in the format string from the running program (timezone, month, year, etc)
+  ## This function parses a date/time string using the standard format
+  ## identifiers as listed below. The function defaults information not provided
+  ## in the format string from the running program (timezone, month, year, etc).
+  ## Daylight saving time is only set if no timezone is given and the given date
+  ## lies within the DST period of the current locale.
   ##
   ## ==========  =================================================================================  ================================================
   ## Specifier   Description                                                                        Example
@@ -1147,7 +1167,7 @@ proc parse*(value, layout: string): TimeInfo =
   ##    tt       Same as above, but ``AM`` and ``PM`` instead of ``A`` and ``P`` respectively.
   ##    yy       Displays the year to two digits.                                                   ``2012 -> 12``
   ##    yyyy     Displays the year to four digits.                                                  ``2012 -> 2012``
-  ##    z        Displays the timezone offset from UTC.                                             ``GMT+7 -> +7``, ``GMT-5 -> -5``
+  ##    z        Displays the timezone offset from UTC. ``Z`` is parsed as ``+0``                   ``GMT+7 -> +7``, ``GMT-5 -> -5``
   ##    zz       Same as above but with leading 0.                                                  ``GMT+7 -> +07``, ``GMT-5 -> -05``
   ##    zzz      Same as above but with ``:mm`` where *mm* represents minutes.                      ``GMT+7 -> +07:00``, ``GMT-5 -> -05:00``
   ## ==========  =================================================================================  ================================================
@@ -1165,6 +1185,8 @@ proc parse*(value, layout: string): TimeInfo =
   info.hour = 0
   info.minute = 0
   info.second = 0
+  info.isDST = true # using this is flag for checking whether a timezone has \
+      # been read (because DST is always false when a tz is parsed)
   while true:
     case layout[i]
     of ' ', '-', '/', ':', '\'', '\0', '(', ')', '[', ']', ',':
@@ -1194,16 +1216,17 @@ proc parse*(value, layout: string): TimeInfo =
         parseToken(info, token, value, j)
         token = ""
 
-  # We are going to process the date to find out if we are in DST, because the
-  # default based on the current time may be wrong. Calling getLocalTime will
-  # set this correctly, but the actual time may be offset from when we called
-  # toTime with a possibly incorrect DST setting, so we are only going to take
-  # the isDST from this result.
-  let correctDST = getLocalTime(toTime(info))
-  info.isDST = correctDST.isDST
-
-  # Now we process it again with the correct isDST to correct things like
-  # weekday and yearday.
+  if info.isDST:
+    # means that no timezone has been parsed. In this case, we need to check
+    # whether the date is within DST of the local time.
+    let tmp = getLocalTime(toTime(info))
+    # correctly set isDST so that the following step works on the correct time
+    info.isDST = tmp.isDST
+
+  # Correct weekday and yearday; transform timestamp to local time.
+  # There currently is no way of returning this with the original (parsed)
+  # timezone while also setting weekday and yearday (we are depending on stdlib
+  # to provide this calculation).
   return getLocalTime(toTime(info))
 
 # Leap year calculations are adapted from:
diff --git a/lib/pure/unittest.nim b/lib/pure/unittest.nim
index 12553e3da..cdca02ed7 100644
--- a/lib/pure/unittest.nim
+++ b/lib/pure/unittest.nim
@@ -98,7 +98,7 @@ proc startSuite(name: string) =
   template rawPrint() = echo("\n[Suite] ", name) 
   when not defined(ECMAScript):
     if colorOutput:
-      styledEcho styleBright, fgBlue, "\n[Suite] ", fgWhite, name
+      styledEcho styleBright, fgBlue, "\n[Suite] ", resetStyle, name
     else: rawPrint()
   else: rawPrint()
 
@@ -159,7 +159,7 @@ proc testDone(name: string, s: TestStatus, indent: bool) =
                     of FAILED: fgRed
                     of SKIPPED: fgYellow
                     else: fgWhite
-        styledEcho styleBright, color, prefix, "[", $s, "] ", fgWhite, name
+        styledEcho styleBright, color, prefix, "[", $s, "] ", resetStyle, name
       else:
         rawPrint()
     else: