code gen bugfixes; marshal.nim implemented

5 files changed, 339 insertions, 48 deletions

diff --git a/lib/pure/json.nim b/lib/pure/json.nim
index 7b2707784..7c4961b61 100755
--- a/lib/pure/json.nim
+++ b/lib/pure/json.nim
@@ -22,7 +22,7 @@ type
     jsonError,           ## an error ocurred during parsing
     jsonEof,             ## end of file reached
     jsonString,          ## a string literal
-    jsonInt,             ## a integer literal
+    jsonInt,             ## an integer literal
     jsonFloat,           ## a float literal
     jsonTrue,            ## the value ``true``
     jsonFalse,           ## the value ``false``
@@ -121,7 +121,7 @@ proc str*(my: TJsonParser): string {.inline.} =
 proc getInt*(my: TJsonParser): biggestInt {.inline.} = 
   ## returns the number for the event: ``jsonInt``
   assert(my.kind == jsonInt)
-  return parseInt(my.a)
+  return parseBiggestInt(my.a)
 
 proc getFloat*(my: TJsonParser): float {.inline.} = 
   ## returns the number for the event: ``jsonFloat``
@@ -512,9 +512,10 @@ type
     of JArray:
       elems*: seq[PJsonNode]
 
-  EJsonParsingError* = object of EInvalidValue
+  EJsonParsingError* = object of EInvalidValue ## is raised for a JSON error
 
-proc raiseParseErr(p: TJsonParser, msg: string) =
+proc raiseParseErr*(p: TJsonParser, msg: string) {.noinline, noreturn.} =
+  ## raises an `EJsonParsingError` exception.
   raise newException(EJsonParsingError, errorMsgExpected(p, msg))
 
 proc newJString*(s: String): PJsonNode =
@@ -607,6 +608,7 @@ proc `[]=`*(obj: PJsonNode, key: String, val: PJsonNode) =
   obj.fields.add((key, val))
 
 proc delete*(obj: PJsonNode, key: string) =
+  ## Deletes ``obj[key]`` preserving the order of the other (key, value)-pairs.
   assert(obj.kind == JObject)
   for i in 0..obj.fields.len-1:
     if obj.fields[i].key == key:
@@ -615,6 +617,7 @@ proc delete*(obj: PJsonNode, key: string) =
   raise newException(EInvalidIndex, "key not in object")
 
 proc copy*(p: PJsonNode): PJsonNode =
+  ## Performs a deep copy of `a`.
   case p.kind
   of JString:
     result = newJString(p.str)
diff --git a/lib/pure/marshal.nim b/lib/pure/marshal.nim
new file mode 100755
index 000000000..f96d177ae
--- /dev/null
+++ b/lib/pure/marshal.nim
@@ -0,0 +1,288 @@
+#

+#

+#            Nimrod's Runtime Library

+#        (c) Copyright 2011 Andreas Rumpf

+#

+#    See the file "copying.txt", included in this

+#    distribution, for details about the copyright.

+#

+

+## This module contains procs for serialization and deseralization of 

+## arbitrary Nimrod data structures. The serialization format uses JSON.

+

+import streams, typeinfo, json, intsets, tables

+

+proc ptrToInt(x: pointer): int {.inline.} =

+  result = cast[int](x) # don't skip alignment

+

+proc storeAny(s: PStream, a: TAny, stored: var TIntSet) =

+  case a.kind

+  of akNone: assert false

+  of akBool: s.write($getBool(a))

+  of akChar: s.write(escapeJson($getChar(a)))

+  of akArray, akSequence:

+    if a.kind == akSequence and isNil(a): s.write("null")

+    else:

+      s.write("[")

+      for i in 0 .. a.len-1:

+        if i > 0: s.write(", ")

+        storeAny(s, a[i], stored)

+      s.write("]")

+  of akObject, akPureObject, akTuple:

+    s.write("{")

+    var i = 0

+    for key, val in fields(a):

+      if i > 0: s.write(", ")

+      s.write(escapeJson(key))

+      s.write(": ")

+      storeAny(s, val, stored)

+      inc(i)

+    s.write("}")

+  of akSet:

+    s.write("[")

+    var i = 0

+    for e in elements(a):

+      if i > 0: s.write(", ")

+      s.write($e)

+      inc(i)

+    s.write("]")

+  of akRange: storeAny(s, skipRange(a), stored)

+  of akEnum: s.write(getEnumField(a).escapeJson)

+  of akPtr, akRef:

+    var x = a.getPointer

+    if isNil(x): s.write("null")

+    elif stored.containsOrIncl(x.ptrToInt):

+      # already stored, so we simply write out the pointer as an int:

+      s.write($x.ptrToInt)

+    else:

+      # else as a [value, key] pair:

+      # (reversed order for convenient x[0] access!)

+      s.write("[")

+      s.write($x.ptrToInt)

+      s.write(", ")

+      storeAny(s, a[], stored)

+      s.write("]")

+  of akProc, akPointer, akCString: s.write($a.getPointer.ptrToInt)

+  of akString:

+    var x = getString(a)

+    if IsNil(x): s.write("null")

+    else: s.write(escapeJson(x))

+  of akInt..akInt64: s.write($getBiggestInt(a))

+  of akFloat..akFloat128: s.write($getBiggestFloat(a))

+

+proc loadAny(p: var TJsonParser, a: TAny, t: var TTable[biggestInt, pointer]) =

+  case a.kind

+  of akNone: assert false

+  of akBool: 

+    case p.kind

+    of jsonFalse: setBiggestInt(a, 0)

+    of jsonTrue: setBiggestInt(a, 1)

+    else: raiseParseErr(p, "'true' or 'false' expected for a bool")

+    next(p)

+  of akChar:

+    if p.kind == jsonString:

+      var x = p.str

+      if x.len == 1:

+        setBiggestInt(a, ord(x[0]))

+        next(p)

+        return

+    raiseParseErr(p, "string of length 1 expected for a char")

+  of akEnum: 

+    if p.kind == jsonString:

+      setBiggestInt(a, getEnumOrdinal(a, p.str))

+      next(p)

+      return

+    raiseParseErr(p, "string expected for an enum")

+  of akArray:

+    if p.kind != jsonArrayStart: raiseParseErr(p, "'[' expected for an array")

+    next(p)

+    var i = 0

+    while p.kind != jsonArrayEnd and p.kind != jsonEof:

+      loadAny(p, a[i], t)

+      inc(i)

+    if p.kind == jsonArrayEnd: next(p)

+    else: raiseParseErr(p, "']' end of array expected")

+  of akSequence:

+    case p.kind 

+    of jsonNull:

+      setPointer(a, nil)

+      next(p)

+    of jsonArrayStart:

+      next(p)

+      invokeNewSeq(a, 0)
+      var i = 0
+      while p.kind != jsonArrayEnd and p.kind != jsonEof:

+        extendSeq(a)
+        loadAny(p, a[i], t)

+        inc(i)

+      if p.kind == jsonArrayEnd: next(p)

+      else: raiseParseErr(p, "")

+    else:
+      raiseParseErr(p, "'[' expected for a seq")

+  of akObject, akPureObject, akTuple:

+    if a.kind == akObject: setObjectRuntimeType(a)
+    if p.kind != jsonObjectStart: raiseParseErr(p, "'{' expected for an object")

+    next(p)

+    while p.kind != jsonObjectEnd and p.kind != jsonEof:

+      if p.kind != jsonString: 

+        raiseParseErr(p, "string expected for a field name")

+      var fieldName = p.str

+      next(p)

+      loadAny(p, a[fieldName], t)

+    if p.kind == jsonObjectEnd: next(p)

+    else: raiseParseErr(p, "'}' end of object expected")

+  of akSet:

+    if p.kind != jsonArrayStart: raiseParseErr(p, "'[' expected for a set")

+    next(p)

+    while p.kind != jsonArrayEnd and p.kind != jsonEof:

+      if p.kind != jsonInt: raiseParseErr(p, "int expected for a set")

+      inclSetElement(a, p.getInt.int)

+      next(p)

+    if p.kind == jsonArrayEnd: next(p)

+    else: raiseParseErr(p, "']' end of array expected")

+  of akPtr, akRef:

+    case p.kind 

+    of jsonNull:

+      setPointer(a, nil)

+      next(p)

+    of jsonInt:

+      setPointer(a, t[p.getInt])

+      next(p)

+    of jsonArrayStart:

+      next(p)

+      if a.kind == akRef: invokeNew(a)
+      else: setPointer(a, alloc0(a.baseTypeSize))      
+      if p.kind == jsonInt:
+        t[p.getInt] = getPointer(a)
+        next(p)
+      else: raiseParseErr(p, "index for ref type expected")
+      loadAny(p, a[], t)
+      if p.kind == jsonArrayEnd: next(p)

+      else: raiseParseErr(p, "']' end of ref-address pair expected")

+    else: raiseParseErr(p, "int for pointer type expected")

+  of akProc, akPointer, akCString: 

+    case p.kind 

+    of jsonNull:

+      setPointer(a, nil)

+      next(p)

+    of jsonInt:

+      setPointer(a, cast[pointer](p.getInt.int))

+      next(p)

+    else: raiseParseErr(p, "int for pointer type expected")

+  of akString:

+    case p.kind 

+    of jsonNull:

+      setPointer(a, nil)

+      next(p)

+    of jsonString:

+      setString(a, p.str)

+      next(p)

+    else: raiseParseErr(p, "string expected")

+  of akInt..akInt64: 

+    if p.kind == jsonInt:

+      setBiggestInt(a, getInt(p))

+      next(p)

+      return

+    raiseParseErr(p, "int expected")

+  of akFloat..akFloat128:

+    if p.kind == jsonFloat:

+      setBiggestFloat(a, getFloat(p))

+      next(p)

+      return

+    raiseParseErr(p, "float expected")
+  of akRange: loadAny(p, a.skipRange, t)

+

+proc loadAny(s: PStream, a: TAny, t: var TTable[biggestInt, pointer]) =

+  var p: TJsonParser

+  open(p, s, "unknown file")

+  next(p)

+  loadAny(p, a, t)

+  close(p)

+

+proc load*[T](s: PStream, data: var T) =

+  ## loads `data` from the stream `s`. Raises `EIO` in case of an error.

+  var tab = initTable[biggestInt, pointer]()
+  loadAny(s, toAny(data), tab)

+

+proc store*[T](s: PStream, data: T) =

+  ## stores `data` into the stream `s`. Raises `EIO` in case of an error.

+  var stored = initIntSet()

+  var d: T

+  shallowCopy(d, data)

+  storeAny(s, toAny(d), stored)

+
+proc `$$`*[T](x: T): string =
+  ## returns a string representation of `x`.
+  var stored = initIntSet()

+  var d: T

+  shallowCopy(d, x)

+  var s = newStringStream()
+  storeAny(s, toAny(d), stored)

+  result = s.data
+
+proc to*[T](data: string): T =
+  ## reads data and transforms it to a ``T``.
+  var tab = initTable[biggestInt, pointer]()
+  loadAny(newStringStream(data), toAny(result), tab)

+  

+when isMainModule:
+  template testit(x: expr) = echo($$to[type(x)]($$x))
+
+  var x: array[0..4, array[0..4, string]] = [

+    ["test", "1", "2", "3", "4"], ["test", "1", "2", "3", "4"], 

+    ["test", "1", "2", "3", "4"], ["test", "1", "2", "3", "4"], 

+    ["test", "1", "2", "3", "4"]]

+  testit(x)

+  var test2: tuple[name: string, s: int] = ("tuple test", 56)
+  testit(test2)
+  
+  type
+    TE = enum
+      blah, blah2
+  
+    TestObj = object
+      test, asd: int
+      case test2: TE
+      of blah:
+        help: string
+      else:
+        nil
+        
+    PNode = ref TNode
+    TNode = object
+      next, prev: PNode
+      data: string
+
+  proc buildList(): PNode =
+    new(result)
+    new(result.next)
+    new(result.prev)
+    result.data = "middle"
+    result.next.data = "next"
+    result.prev.data = "prev"
+    result.next.next = result.prev
+    result.next.prev = result
+    result.prev.next = result
+    result.prev.prev = result.next
+
+  var test3: TestObj
+  test3.test = 42
+  test3.test2 = blah
+  testit(test3)
+
+  var test4: ref tuple[a, b: string]
+  new(test4)
+  test4.a = "ref string test: A"
+  test4.b = "ref string test: B"
+  testit(test4)
+  
+  var test5 = @[(0,1),(2,3),(4,5)]
+  testit(test5)
+
+  var test6: set[char] = {'A'..'Z', '_'}
+  testit(test6)
+
+  var test7 = buildList()
+  echo($$test7)
+  testit(test7)
+
diff --git a/lib/pure/streams.nim b/lib/pure/streams.nim
index d0e6ecec7..242e40d83 100755
--- a/lib/pure/streams.nim
+++ b/lib/pure/streams.nim
@@ -33,8 +33,9 @@ proc write*[T](s: PStream, x: T) =
   ## .. code-block:: Nimrod
   ##
   ##     s.writeData(s, addr(x), sizeof(x))
-  var x = x
-  s.writeData(s, addr(x), sizeof(x))
+  var y: T
+  shallowCopy(y, x)
+  s.writeData(s, addr(y), sizeof(y))
 
 proc write*(s: PStream, x: string) = 
   ## writes the string `x` to the the stream `s`. No length field or 
diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim
index 8659c7d29..58e1e5fed 100755
--- a/lib/pure/strutils.nim
+++ b/lib/pure/strutils.nim
@@ -89,15 +89,15 @@ proc normalize*(s: string): string {.noSideEffect, procvar,
   rtl, extern: "nsuNormalize".} =

   ## Normalizes the string `s`. That means to convert it to lower case and

   ## remove any '_'. This is needed for Nimrod identifiers for example.

-  result = newString(s.len)
+  result = newString(s.len)

   var j = 0

   for i in 0..len(s) - 1:

     if s[i] in {'A'..'Z'}:

-      result[j] = Chr(Ord(s[i]) + (Ord('a') - Ord('A')))
+      result[j] = Chr(Ord(s[i]) + (Ord('a') - Ord('A')))

       inc j

     elif s[i] != '_':

-      result[j] = s[i]
-      inc j
+      result[j] = s[i]

+      inc j

   if j != s.len: setLen(result, j)

 

 proc cmpIgnoreCase*(a, b: string): int {.noSideEffect,

@@ -461,7 +461,7 @@ proc repeatChar*(count: int, c: Char = ' '): string {.noSideEffect,
   ## the character `c`.

   result = newString(count)

   for i in 0..count-1: result[i] = c

-
+

 proc repeatStr*(count: int, s: string): string {.noSideEffect,

   rtl, extern: "nsuRepeatStr".} =

   ## Returns `s` concatenated `count` times.

@@ -480,40 +480,40 @@ proc align*(s: string, count: int): string {.
     for i in spaces..count-1: result[i] = s[i-spaces]

   else:

     result = s

-

-iterator tokenize*(s: string, seps: set[char] = Whitespace): tuple[

-  token: string, isSep: bool] =

-  ## Tokenizes the string `s` into substrings.

-  ##

-  ## Substrings are separated by a substring containing only `seps`.

-  ## Examples:

-  ##

-  ## .. code-block:: nimrod

-  ##   for word in tokenize("  this is an  example  "):

-  ##     writeln(stdout, word)

-  ##

-  ## Results in:

-  ##

-  ## .. code-block:: nimrod

-  ##   ("  ", true)

-  ##   ("this", false)

-  ##   (" ", true)

-  ##   ("is", false)

-  ##   (" ", true)

-  ##   ("an", false)

-  ##   ("  ", true)

-  ##   ("example", false)

-  ##   ("  ", true)

-  var i = 0

-  while true:

-    var j = i

-    var isSep = s[j] in seps

-    while j < s.len and (s[j] in seps) == isSep: inc(j)

-    if j > i:

-      yield (substr(s, i, j-1), isSep)

-    else:

-      break

-    i = j

+
+iterator tokenize*(s: string, seps: set[char] = Whitespace): tuple[
+  token: string, isSep: bool] =
+  ## Tokenizes the string `s` into substrings.
+  ##
+  ## Substrings are separated by a substring containing only `seps`.
+  ## Examples:
+  ##
+  ## .. code-block:: nimrod
+  ##   for word in tokenize("  this is an  example  "):
+  ##     writeln(stdout, word)
+  ##
+  ## Results in:
+  ##
+  ## .. code-block:: nimrod
+  ##   ("  ", true)
+  ##   ("this", false)
+  ##   (" ", true)
+  ##   ("is", false)
+  ##   (" ", true)
+  ##   ("an", false)
+  ##   ("  ", true)
+  ##   ("example", false)
+  ##   ("  ", true)
+  var i = 0
+  while true:
+    var j = i
+    var isSep = s[j] in seps
+    while j < s.len and (s[j] in seps) == isSep: inc(j)
+    if j > i:
+      yield (substr(s, i, j-1), isSep)
+    else:
+      break
+    i = j
 

 proc wordWrap*(s: string, maxLineWidth = 80, 

                splitLongWords = true,

@@ -815,7 +815,7 @@ proc escape*(s: string, prefix = "\"", suffix = "\""): string {.noSideEffect,
   ## The procedure has been designed so that its output is usable for many

   ## different common syntaxes. The resulting string is prefixed with

   ## `prefix` and suffixed with `suffix`. Both may be empty strings.

-  result = newStringOfCap(s.len + s.len shr 2)
+  result = newStringOfCap(s.len + s.len shr 2)

   result.add(prefix)

   for c in items(s):

     case c

diff --git a/lib/pure/xmltree.nim b/lib/pure/xmltree.nim
index 41765b87a..7f6d5cff8 100755
--- a/lib/pure/xmltree.nim
+++ b/lib/pure/xmltree.nim
@@ -145,8 +145,7 @@ proc escape*(s: string): string =
   ##  ``&``          ``&amp;``
   ##  ``"``          ``&quot;``
   ## ------------    -------------------
-  result = newString(s.len)
-  setLen(result, 0)
+  result = newStringOfCap(s.len)
   addEscaped(result, s)
   
 proc addIndent(result: var string, indent: int) =