Merge branch 'devel' into OpenBSDFix

5 files changed, 296 insertions, 23 deletions

diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim
index 85c31fdd0..fcf947831 100644
--- a/lib/pure/asyncdispatch.nim
+++ b/lib/pure/asyncdispatch.nim
@@ -811,7 +811,7 @@ template createVar(futSymName: string, asyncProc: PNimrodNode,
   result.add generateExceptionCheck(futSym, exceptBranch, rootReceiver)
 
 proc processBody(node, retFutureSym: PNimrodNode,
-                 subtypeName: string,
+                 subTypeIsVoid: bool,
                  exceptBranch: PNimrodNode): PNimrodNode {.compileTime.} =
   #echo(node.treeRepr)
   result = node
@@ -819,14 +819,14 @@ proc processBody(node, retFutureSym: PNimrodNode,
   of nnkReturnStmt:
     result = newNimNode(nnkStmtList)
     if node[0].kind == nnkEmpty:
-      if subtypeName != "void":
+      if not subtypeIsVoid:
         result.add newCall(newIdentNode("complete"), retFutureSym,
             newIdentNode("result"))
       else:
         result.add newCall(newIdentNode("complete"), retFutureSym)
     else:
       result.add newCall(newIdentNode("complete"), retFutureSym,
-        node[0].processBody(retFutureSym, subtypeName, exceptBranch))
+        node[0].processBody(retFutureSym, subtypeIsVoid, exceptBranch))
 
     result.add newNimNode(nnkReturnStmt).add(newNilLit())
     return # Don't process the children of this return stmt
@@ -869,7 +869,8 @@ proc processBody(node, retFutureSym: PNimrodNode,
     else: discard
   of nnkDiscardStmt:
     # discard await x
-    if node[0][0].kind == nnkIdent and node[0][0].ident == !"await":
+    if node[0].kind != nnkEmpty and node[0][0].kind == nnkIdent and
+          node[0][0].ident == !"await":
       var newDiscard = node
       createVar("futureDiscard_" & $toStrLit(node[0][1]), node[0][1],
                 newDiscard[0], newDiscard)
@@ -880,7 +881,7 @@ proc processBody(node, retFutureSym: PNimrodNode,
                        res: PNimrodNode): bool {.compileTime.} =
       result = false
       while i < n[0].len:
-        var processed = processBody(n[0][i], retFutureSym, subtypeName, n[1])
+        var processed = processBody(n[0][i], retFutureSym, subtypeIsVoid, n[1])
         if processed.kind != n[0][i].kind or processed.len != n[0][i].len:
           expectKind(processed, nnkStmtList)
           expectKind(processed[2][1], nnkElse)
@@ -900,7 +901,7 @@ proc processBody(node, retFutureSym: PNimrodNode,
   else: discard
 
   for i in 0 .. <result.len:
-    result[i] = processBody(result[i], retFutureSym, subtypeName, exceptBranch)
+    result[i] = processBody(result[i], retFutureSym, subtypeIsVoid, exceptBranch)
 
 proc getName(node: PNimrodNode): string {.compileTime.} =
   case node.kind
@@ -920,35 +921,36 @@ macro async*(prc: stmt): stmt {.immediate.} =
   hint("Processing " & prc[0].getName & " as an async proc.")
 
   let returnType = prc[3][0]
-  var subtypeName = ""
   # Verify that the return type is a PFuture[T]
   if returnType.kind == nnkIdent:
     error("Expected return type of 'PFuture' got '" & $returnType & "'")
   elif returnType.kind == nnkBracketExpr:
     if $returnType[0] != "PFuture":
       error("Expected return type of 'PFuture' got '" & $returnType[0] & "'")
-    subtypeName = $returnType[1].ident
-  elif returnType.kind == nnkEmpty:
-    subtypeName = "void"
+
+  let subtypeIsVoid = returnType.kind == nnkEmpty
 
   var outerProcBody = newNimNode(nnkStmtList)
 
   # -> var retFuture = newFuture[T]()
   var retFutureSym = genSym(nskVar, "retFuture")
+  var subRetType =
+    if returnType.kind == nnkEmpty: newIdentNode("void")
+    else: returnType[1]
   outerProcBody.add(
     newVarStmt(retFutureSym, 
       newCall(
         newNimNode(nnkBracketExpr).add(
           newIdentNode(!"newFuture"), # TODO: Strange bug here? Remove the `!`.
-          newIdentNode(subtypeName))))) # Get type from return type of this proc
+          subRetType)))) # Get type from return type of this proc
   
   # -> iterator nameIter(): PFutureBase {.closure.} = 
   # ->   var result: T
   # ->   <proc_body>
   # ->   complete(retFuture, result)
   var iteratorNameSym = genSym(nskIterator, $prc[0].getName & "Iter")
-  var procBody = prc[6].processBody(retFutureSym, subtypeName, nil)
-  if subtypeName != "void":
+  var procBody = prc[6].processBody(retFutureSym, subtypeIsVoid, nil)
+  if not subtypeIsVoid:
     procBody.insert(0, newNimNode(nnkVarSection).add(
       newIdentDefs(newIdentNode("result"), returnType[1]))) # -> var result: T
     procBody.add(
@@ -977,7 +979,7 @@ macro async*(prc: stmt): stmt {.immediate.} =
   for i in 0 .. <result[4].len:
     if result[4][i].ident == !"async":
       result[4].del(i)
-  if subtypeName == "void":
+  if subtypeIsVoid:
     # Add discardable pragma.
     result[4].add(newIdentNode("discardable"))
     if returnType.kind == nnkEmpty:
diff --git a/lib/pure/collections/tables.nim b/lib/pure/collections/tables.nim
index 33e558aee..848f4b8ba 100644
--- a/lib/pure/collections/tables.nim
+++ b/lib/pure/collections/tables.nim
@@ -66,6 +66,7 @@ type
   TTable* {.final, myShallow.}[A, B] = object ## generic hash table
     data: TKeyValuePairSeq[A, B]
     counter: int
+  PTable*[A,B] = ref TTable[A, B]
 
 when not defined(nimhygiene):
   {.pragma: dirty.}
@@ -231,7 +232,7 @@ proc `$`*[A, B](t: TTable[A, B]): string =
   ## The `$` operator for hash tables.
   dollarImpl()
   
-proc `==`*[A, B](s, t: TTable[A, B]): bool =
+template equalsImpl() =
   if s.counter == t.counter:
     # different insertion orders mean different 'data' seqs, so we have
     # to use the slow route here:
@@ -240,6 +241,9 @@ proc `==`*[A, B](s, t: TTable[A, B]): bool =
       if t[key] != val: return false
     return true
   
+proc `==`*[A, B](s, t: TTable[A, B]): bool =
+  equalsImpl()
+  
 proc indexBy*[A, B, C](collection: A, index: proc(x: B): C): TTable[C, B] =
   ## Index the collection with the proc provided.
   # TODO: As soon as supported, change collection: A to collection: A[B]
@@ -247,6 +251,88 @@ proc indexBy*[A, B, C](collection: A, index: proc(x: B): C): TTable[C, B] =
   for item in collection:
     result[index(item)] = item
 
+proc len*[A, B](t: PTable[A, B]): int =
+  ## returns the number of keys in `t`.
+  result = t.counter
+
+iterator pairs*[A, B](t: PTable[A, B]): tuple[key: A, val: B] =
+  ## iterates over any (key, value) pair in the table `t`.
+  for h in 0..high(t.data):
+    if t.data[h].slot == seFilled: yield (t.data[h].key, t.data[h].val)
+
+iterator mpairs*[A, B](t: PTable[A, B]): tuple[key: A, val: var B] =
+  ## iterates over any (key, value) pair in the table `t`. The values
+  ## can be modified.
+  for h in 0..high(t.data):
+    if t.data[h].slot == seFilled: yield (t.data[h].key, t.data[h].val)
+
+iterator keys*[A, B](t: PTable[A, B]): A =
+  ## iterates over any key in the table `t`.
+  for h in 0..high(t.data):
+    if t.data[h].slot == seFilled: yield t.data[h].key
+
+iterator values*[A, B](t: PTable[A, B]): B =
+  ## iterates over any value in the table `t`.
+  for h in 0..high(t.data):
+    if t.data[h].slot == seFilled: yield t.data[h].val
+
+iterator mvalues*[A, B](t: PTable[A, B]): var B =
+  ## iterates over any value in the table `t`. The values can be modified.
+  for h in 0..high(t.data):
+    if t.data[h].slot == seFilled: yield t.data[h].val
+
+proc `[]`*[A, B](t: PTable[A, B], key: A): B =
+  ## retrieves the value at ``t[key]``. If `key` is not in `t`,
+  ## default empty value for the type `B` is returned
+  ## and no exception is raised. One can check with ``hasKey`` whether the key
+  ## exists.
+  result = t[][key]
+
+proc mget*[A, B](t: PTable[A, B], key: A): var B =
+  ## retrieves the value at ``t[key]``. The value can be modified.
+  ## If `key` is not in `t`, the ``EInvalidKey`` exception is raised.
+  t[].mget(key)
+
+proc hasKey*[A, B](t: PTable[A, B], key: A): bool =
+  ## returns true iff `key` is in the table `t`.
+  result = t[].hasKey(key)
+
+proc `[]=`*[A, B](t: PTable[A, B], key: A, val: B) =
+  ## puts a (key, value)-pair into `t`.
+  t[][key] = val
+
+proc add*[A, B](t: PTable[A, B], key: A, val: B) =
+  ## puts a new (key, value)-pair into `t` even if ``t[key]`` already exists.
+  t[].add(key, val)
+  
+proc del*[A, B](t: PTable[A, B], key: A) =
+  ## deletes `key` from hash table `t`.
+  t[].del(key)
+
+proc newTable*[A, B](initialSize=64): PTable[A, B] =
+  new(result)
+  result[] = initTable[A, B](initialSize)
+
+proc newTable*[A, B](pairs: openArray[tuple[key: A, 
+                    val: B]]): PTable[A, B] =
+  ## creates a new hash table that contains the given `pairs`.
+  new(result)
+  result[] = toTable[A, B](pairs)
+
+proc `$`*[A, B](t: PTable[A, B]): string =
+  ## The `$` operator for hash tables.
+  dollarImpl()
+
+proc `==`*[A, B](s, t: PTable[A, B]): bool =
+  equalsImpl()
+
+proc newTableFrom*[A, B, C](collection: A, index: proc(x: B): C): PTable[C, B] =
+  ## Index the collection with the proc provided.
+  # TODO: As soon as supported, change collection: A to collection: A[B]
+  result = newTable[C, B]()
+  for item in collection:
+    result[index(item)] = item
+
 # ------------------------------ ordered table ------------------------------
 
 type
@@ -257,6 +343,7 @@ type
       final, myShallow.}[A, B] = object ## table that remembers insertion order
     data: TOrderedKeyValuePairSeq[A, B]
     counter, first, last: int
+  POrderedTable*[A, B] = ref TOrderedTable[A, B]
 
 proc len*[A, B](t: TOrderedTable[A, B]): int {.inline.} =
   ## returns the number of keys in `t`.
@@ -417,6 +504,96 @@ proc sort*[A, B](t: var TOrderedTable[A, B],
   t.first = list
   t.last = tail
 
+proc len*[A, B](t: POrderedTable[A, B]): int {.inline.} =
+  ## returns the number of keys in `t`.
+  result = t.counter
+
+template forAllOrderedPairs(yieldStmt: stmt) {.dirty, immediate.} =
+  var h = t.first
+  while h >= 0:
+    var nxt = t.data[h].next
+    if t.data[h].slot == seFilled: yieldStmt
+    h = nxt
+
+iterator pairs*[A, B](t: POrderedTable[A, B]): tuple[key: A, val: B] =
+  ## iterates over any (key, value) pair in the table `t` in insertion
+  ## order.
+  forAllOrderedPairs:
+    yield (t.data[h].key, t.data[h].val)
+
+iterator mpairs*[A, B](t: POrderedTable[A, B]): tuple[key: A, val: var B] =
+  ## iterates over any (key, value) pair in the table `t` in insertion
+  ## order. The values can be modified.
+  forAllOrderedPairs:
+    yield (t.data[h].key, t.data[h].val)
+
+iterator keys*[A, B](t: POrderedTable[A, B]): A =
+  ## iterates over any key in the table `t` in insertion order.
+  forAllOrderedPairs:
+    yield t.data[h].key
+
+iterator values*[A, B](t: POrderedTable[A, B]): B =
+  ## iterates over any value in the table `t` in insertion order.
+  forAllOrderedPairs:
+    yield t.data[h].val
+
+iterator mvalues*[A, B](t: POrderedTable[A, B]): var B =
+  ## iterates over any value in the table `t` in insertion order. The values
+  ## can be modified.
+  forAllOrderedPairs:
+    yield t.data[h].val
+
+proc `[]`*[A, B](t: POrderedTable[A, B], key: A): B =
+  ## retrieves the value at ``t[key]``. If `key` is not in `t`,
+  ## default empty value for the type `B` is returned
+  ## and no exception is raised. One can check with ``hasKey`` whether the key
+  ## exists.
+  result = t[][key]
+
+proc mget*[A, B](t: POrderedTable[A, B], key: A): var B =
+  ## retrieves the value at ``t[key]``. The value can be modified.
+  ## If `key` is not in `t`, the ``EInvalidKey`` exception is raised.
+  result = t[].mget(key)
+
+proc hasKey*[A, B](t: POrderedTable[A, B], key: A): bool =
+  ## returns true iff `key` is in the table `t`.
+  result = t[].hasKey(key)
+
+proc `[]=`*[A, B](t: POrderedTable[A, B], key: A, val: B) =
+  ## puts a (key, value)-pair into `t`.
+  t[][key] = val
+
+proc add*[A, B](t: POrderedTable[A, B], key: A, val: B) =
+  ## puts a new (key, value)-pair into `t` even if ``t[key]`` already exists.
+  t[].add(key, val)
+
+proc newOrderedTable*[A, B](initialSize=64): POrderedTable[A, B] =
+  ## creates a new ordered hash table that is empty.
+  ##
+  ## `initialSize` needs to be a power of two. If you need to accept runtime
+  ## values for this you could use the ``nextPowerOfTwo`` proc from the
+  ## `math <math.html>`_ module.
+  new(result)
+  result[] = initOrderedTable[A, B]()
+
+proc newOrderedTable*[A, B](pairs: openArray[tuple[key: A, 
+                           val: B]]): POrderedTable[A, B] =
+  ## creates a new ordered hash table that contains the given `pairs`.
+  result = newOrderedTable[A, B](nextPowerOfTwo(pairs.len+10))
+  for key, val in items(pairs): result[key] = val
+
+proc `$`*[A, B](t: POrderedTable[A, B]): string =
+  ## The `$` operator for ordered hash tables.
+  dollarImpl()
+
+proc sort*[A, B](t: POrderedTable[A, B], 
+                 cmp: proc (x,y: tuple[key: A, val: B]): int) =
+  ## sorts `t` according to `cmp`. This modifies the internal list
+  ## that kept the insertion order, so insertion order is lost after this
+  ## call but key lookup and insertions remain possible after `sort` (in
+  ## contrast to the `sort` for count tables).
+  t[].sort(cmp)
+
 # ------------------------------ count tables -------------------------------
 
 type
@@ -424,6 +601,7 @@ type
       A] = object ## table that counts the number of each key
     data: seq[tuple[key: A, val: int]]
     counter: int
+  PCountTable*[A] = ref TCountTable[A]
 
 proc len*[A](t: TCountTable[A]): int =
   ## returns the number of keys in `t`.
@@ -567,6 +745,93 @@ proc sort*[A](t: var TCountTable[A]) =
         if j < h: break
     if h == 1: break
 
+proc len*[A](t: PCountTable[A]): int =
+  ## returns the number of keys in `t`.
+  result = t.counter
+
+iterator pairs*[A](t: PCountTable[A]): tuple[key: A, val: int] =
+  ## iterates over any (key, value) pair in the table `t`.
+  for h in 0..high(t.data):
+    if t.data[h].val != 0: yield (t.data[h].key, t.data[h].val)
+
+iterator mpairs*[A](t: PCountTable[A]): tuple[key: A, val: var int] =
+  ## iterates over any (key, value) pair in the table `t`. The values can
+  ## be modified.
+  for h in 0..high(t.data):
+    if t.data[h].val != 0: yield (t.data[h].key, t.data[h].val)
+
+iterator keys*[A](t: PCountTable[A]): A =
+  ## iterates over any key in the table `t`.
+  for h in 0..high(t.data):
+    if t.data[h].val != 0: yield t.data[h].key
+
+iterator values*[A](t: PCountTable[A]): int =
+  ## iterates over any value in the table `t`.
+  for h in 0..high(t.data):
+    if t.data[h].val != 0: yield t.data[h].val
+
+iterator mvalues*[A](t: PCountTable[A]): var int =
+  ## iterates over any value in the table `t`. The values can be modified.
+  for h in 0..high(t.data):
+    if t.data[h].val != 0: yield t.data[h].val
+
+proc `[]`*[A](t: PCountTable[A], key: A): int =
+  ## retrieves the value at ``t[key]``. If `key` is not in `t`,
+  ## 0 is returned. One can check with ``hasKey`` whether the key
+  ## exists.
+  result = t[][key]
+
+proc mget*[A](t: PCountTable[A], key: A): var int =
+  ## retrieves the value at ``t[key]``. The value can be modified.
+  ## If `key` is not in `t`, the ``EInvalidKey`` exception is raised.
+  result = t[].mget(key)
+
+proc hasKey*[A](t: PCountTable[A], key: A): bool =
+  ## returns true iff `key` is in the table `t`.
+  result = t[].hasKey(key)
+
+proc `[]=`*[A](t: PCountTable[A], key: A, val: int) =
+  ## puts a (key, value)-pair into `t`. `val` has to be positive.
+  assert val > 0
+  t[][key] = val
+
+proc newCountTable*[A](initialSize=64): PCountTable[A] =
+  ## creates a new count table that is empty.
+  ##
+  ## `initialSize` needs to be a power of two. If you need to accept runtime
+  ## values for this you could use the ``nextPowerOfTwo`` proc from the
+  ## `math <math.html>`_ module.
+  new(result)
+  result[] = initCountTable[A](initialSize)
+
+proc newCountTable*[A](keys: openArray[A]): PCountTable[A] =
+  ## creates a new count table with every key in `keys` having a count of 1.
+  result = newCountTable[A](nextPowerOfTwo(keys.len+10))
+  for key in items(keys): result[key] = 1
+
+proc `$`*[A](t: PCountTable[A]): string =
+  ## The `$` operator for count tables.
+  dollarImpl()
+
+proc inc*[A](t: PCountTable[A], key: A, val = 1) = 
+  ## increments `t[key]` by `val`.
+  t[].inc(key, val)
+
+proc smallest*[A](t: PCountTable[A]): tuple[key: A, val: int] =
+  ## returns the largest (key,val)-pair. Efficiency: O(n)
+  t[].smallest
+
+proc largest*[A](t: PCountTable[A]): tuple[key: A, val: int] =
+  ## returns the (key,val)-pair with the largest `val`. Efficiency: O(n)
+  t[].largest
+
+proc sort*[A](t: PCountTable[A]) =
+  ## sorts the count table so that the entry with the highest counter comes
+  ## first. This is destructive! You must not modify `t` afterwards!
+  ## You can use the iterators `pairs`,  `keys`, and `values` to iterate over
+  ## `t` in the sorted order.
+  t[].sort
+
 when isMainModule:
   type
     Person = object
diff --git a/lib/system.nim b/lib/system.nim
index 6263f7b24..ad98540a7 100644
--- a/lib/system.nim
+++ b/lib/system.nim
@@ -2620,7 +2620,7 @@ proc `[]=`*[Idx, T](a: var array[Idx, T], x: TSlice[int], b: openArray[T]) =
   if L == b.len:
     for i in 0 .. <L: a[i+x.a] = b[i]
   else:
-    sysFatal(EOutOfRange, "differing lengths for slice assignment")
+    sysFatal(EOutOfRange, "different lengths for slice assignment")
 
 proc `[]`*[Idx, T](a: array[Idx, T], x: TSlice[Idx]): seq[T] =
   ## slice operation for arrays. Negative indexes are **not** supported
@@ -2642,7 +2642,7 @@ proc `[]=`*[Idx, T](a: var array[Idx, T], x: TSlice[Idx], b: openArray[T]) =
       a[j] = b[i]
       inc(j)
   else:
-    sysFatal(EOutOfRange, "differing lengths for slice assignment")
+    sysFatal(EOutOfRange, "different lengths for slice assignment")
 
 proc `[]`*[T](s: seq[T], x: TSlice[int]): seq[T] = 
   ## slice operation for sequences. Negative indexes are supported.
@@ -2719,9 +2719,6 @@ proc `/=`*[T: float|float32|float64] (x: var T, y: T) {.inline, noSideEffect.} =
 
 proc `&=`* (x: var string, y: string) {.magic: "AppendStrStr", noSideEffect.}
 
-proc rand*(max: int): int {.magic: "Rand", sideEffect.}
-  ## compile-time `random` function. Useful for debugging.
-
 proc astToStr*[T](x: T): string {.magic: "AstToStr", noSideEffect.}
   ## converts the AST of `x` into a string representation. This is very useful
   ## for debugging.
diff --git a/lib/system/gc_ms.nim b/lib/system/gc_ms.nim
index 410243528..05bcdcc82 100644
--- a/lib/system/gc_ms.nim
+++ b/lib/system/gc_ms.nim
@@ -313,6 +313,7 @@ proc mark(gch: var TGcHeap, c: PCell) =
       if not containsOrIncl(gch.marked, d):
         forAllChildren(d, waMarkPrecise)
   else:
+    # XXX no 'if c.refCount != rcBlack' here?
     c.refCount = rcBlack
     gcAssert gch.tempStack.len == 0, "stack not empty!"
     forAllChildren(c, waMarkPrecise)
diff --git a/lib/windows/winlean.nim b/lib/windows/winlean.nim
index a868f3025..dcae6ffaf 100644
--- a/lib/windows/winlean.nim
+++ b/lib/windows/winlean.nim
@@ -640,8 +640,8 @@ proc unmapViewOfFile*(lpBaseAddress: pointer): WINBOOL {.stdcall,
 
 type
   TOVERLAPPED* {.pure, inheritable.} = object
-    Internal*: DWORD
-    InternalHigh*: DWORD
+    Internal*: PULONG
+    InternalHigh*: PULONG
     Offset*: DWORD
     OffsetHigh*: DWORD
     hEvent*: THANDLE
@@ -718,4 +718,12 @@ proc WSASend*(s: TSocketHandle, buf: ptr TWSABuf, bufCount: DWORD,
   stdcall, importc: "WSASend", dynlib: "Ws2_32.dll".}
 
 proc get_osfhandle*(fd:TFileHandle): THandle {.
-  importc:"_get_osfhandle", header:"<io.h>".}
+  importc: "_get_osfhandle", header:"<io.h>".}
+
+proc getSystemTimes*(lpIdleTime, lpKernelTime, 
+                     lpUserTime: var TFILETIME): WINBOOL {.stdcall,
+  dynlib: "kernel32", importc: "GetSystemTimes".}
+
+proc getProcessTimes*(hProcess: THandle; lpCreationTime, lpExitTime,
+  lpKernelTime, lpUserTime: var TFILETIME): WINBOOL {.stdcall,
+  dynlib: "kernel32", importc: "GetProcessTimes".}