path: root/lib/std/jsonutils.nim

##[
This module implements a hookable (de)serialization for arbitrary types.
Design goal: avoid importing modules where a custom serialization is needed;
see strtabs.fromJsonHook,toJsonHook for an example.
]##

runnableExamples:
  import std/[strtabs,json]
  type Foo = ref object
    t: bool
    z1: int8
  let a = (1.5'f32, (b: "b2", a: "a2"), 'x', @[Foo(t: true, z1: -3), nil], [{"name": "John"}.newStringTable])
  let j = a.toJson
  assert j.jsonTo(typeof(a)).toJson == j
  assert $[NaN, Inf, -Inf, 0.0, -0.0, 1.0, 1e-2].toJson == """["nan","inf","-inf",0.0,-0.0,1.0,0.01]"""
  assert 0.0.toJson.kind == JFloat
  assert Inf.toJson.kind == JString

import json, strutils, tables, sets, strtabs, options, strformat

#[
Future directions:
add a way to customize serialization, for e.g.:
* field renaming
* allow serializing `enum` and `char` as `string` instead of `int`
  (enum is more compact/efficient, and robust to enum renamings, but string
  is more human readable)
* handle cyclic references, using a cache of already visited addresses
* implement support for serialization and de-serialization of nested variant
  objects.
]#

import macros
from enumutils import symbolName
from typetraits import OrdinalEnum, tupleLen

when defined(nimPreviewSlimSystem):
  import std/assertions

when not defined(nimFixedForwardGeneric):
  # xxx remove pending csources_v1 update >= 1.2.0
  proc to[T](node: JsonNode, t: typedesc[T]): T =
    when T is string: node.getStr
    elif T is bool: node.getBool
    else: static: doAssert false, $T # support as needed (only needed during bootstrap)
  proc isNamedTuple(T: typedesc): bool = # old implementation
    when T isnot tuple: result = false
    else:
      var t: T
      for name, _ in t.fieldPairs:
        when name == "Field0": return compiles(t.Field0)
        else: return true
      return false
else:
  proc isNamedTuple(T: typedesc): bool {.magic: "TypeTrait".}

type
  Joptions* = object # xxx rename FromJsonOptions
    ## Options controlling the behavior of `fromJson`.
    allowExtraKeys*: bool
      ## If `true` Nim's object to which the JSON is parsed is not required to
      ## have a field for every JSON key.
    allowMissingKeys*: bool
      ## If `true` Nim's object to which JSON is parsed is allowed to have
      ## fields without corresponding JSON keys.
    # in future work: a key rename could be added
  EnumMode* = enum
    joptEnumOrd
    joptEnumSymbol
    joptEnumString
  JsonNodeMode* = enum ## controls `toJson` for JsonNode types
    joptJsonNodeAsRef ## returns the ref as is
    joptJsonNodeAsCopy ## returns a deep copy of the JsonNode
    joptJsonNodeAsObject ## treats JsonNode as a regular ref object
  ToJsonOptions* = object
    enumMode*: EnumMode
    jsonNodeMode*: JsonNodeMode
    # xxx charMode, etc

proc initToJsonOptions*(): ToJsonOptions =
  ## initializes `ToJsonOptions` with sane options.
  ToJsonOptions(enumMode: joptEnumOrd, jsonNodeMode: joptJsonNodeAsRef)

proc distinctBase(T: typedesc, recursive: static bool = true): typedesc {.magic: "TypeTrait".}
template distinctBase[T](a: T, recursive: static bool = true): untyped = distinctBase(typeof(a), recursive)(a)

macro getDiscriminants(a: typedesc): seq[string] =
  ## return the discriminant keys
  # candidate for std/typetraits
  var a = a.getTypeImpl
  doAssert a.kind == nnkBracketExpr
  let sym = a[1]
  let t = sym.getTypeImpl
  let t2 = t[2]
  doAssert t2.kind == nnkRecList
  result = newTree(nnkBracket)
  for ti in t2:
    if ti.kind == nnkRecCase:
      let key = ti[0][0]
      let typ = ti[0][1]
      result.add newLit key.strVal
  if result.len > 0:
    result = quote do:
      @`result`
  else:
    result = quote do:
      seq[string].default

macro initCaseObject(T: typedesc, fun: untyped): untyped =
  ## does the minimum to construct a valid case object, only initializing
  ## the discriminant fields; see also `getDiscriminants`
  # maybe candidate for std/typetraits
  var a = T.getTypeImpl
  doAssert a.kind == nnkBracketExpr
  let sym = a[1]
  let t = sym.getTypeImpl
  var t2: NimNode
  case t.kind
  of nnkObjectTy: t2 = t[2]
  of nnkRefTy: t2 = t[0].getTypeImpl[2]
  else: doAssert false, $t.kind # xxx `nnkPtrTy` could be handled too
  doAssert t2.kind == nnkRecList
  result = newTree(nnkObjConstr)
  result.add sym
  for ti in t2:
    if ti.kind == nnkRecCase:
      let key = ti[0][0]
      let typ = ti[0][1]
      let key2 = key.strVal
      let val = quote do:
        `fun`(`key2`, typedesc[`typ`])
      result.add newTree(nnkExprColonExpr, key, val)

proc raiseJsonException(condStr: string, msg: string) {.noinline.} =
  # just pick 1 exception type for simplicity; other choices would be:
  # JsonError, JsonParser, JsonKindError
  raise newException(ValueError, condStr & " failed: " & msg)

template checkJson(cond: untyped, msg = "") =
  if not cond:
    raiseJsonException(astToStr(cond), msg)

proc hasField[T](obj: T, field: string): bool =
  for k, _ in fieldPairs(obj):
    if k == field:
      return true
  return false

macro accessField(obj: typed, name: static string): untyped =
  newDotExpr(obj, ident(name))

template fromJsonFields(newObj, oldObj, json, discKeys, opt) =
  type T = typeof(newObj)
  # we could customize whether to allow JNull
  checkJson json.kind == JObject, $json.kind
  var num, numMatched = 0
  for key, val in fieldPairs(newObj):
    num.inc
    when key notin discKeys:
      if json.hasKey key:
        numMatched.inc
        fromJson(val, json[key], opt)
      elif opt.allowMissingKeys:
        # if there are no discriminant keys the `oldObj` must always have the
        # same keys as the new one. Otherwise we must check, because they could
        # be set to different branches.
        when typeof(oldObj) isnot typeof(nil):
          if discKeys.len == 0 or hasField(oldObj, key):
            val = accessField(oldObj, key)
      else:
        checkJson false, "key '$1' for $2 not in $3" % [key, $T, json.pretty()]
    else:
      if json.hasKey key:
        numMatched.inc

  let ok =
    if opt.allowExtraKeys and opt.allowMissingKeys:
      true
    elif opt.allowExtraKeys:
      # This check is redundant because if here missing keys are not allowed,
      # and if `num != numMatched` it will fail in the loop above but it is left
      # for clarity.
      assert num == numMatched
      num == numMatched
    elif opt.allowMissingKeys:
      json.len == numMatched
    else:
      json.len == num and num == numMatched

  checkJson ok, "There were $1 keys (expecting $2) for $3 with $4" % [$json.len, $num, $T, json.pretty()]

proc fromJson*[T](a: var T, b: JsonNode, opt = Joptions())

proc discKeyMatch[T](obj: T, json: JsonNode, key: static string): bool =
  if not json.hasKey key:
    return true
  let field = accessField(obj, key)
  var jsonVal: typeof(field)
  fromJson(jsonVal, json[key])
  if jsonVal != field:
    return false
  return true

macro discKeysMatchBodyGen(obj: typed, json: JsonNode,
                           keys: static seq[string]): untyped =
  result = newStmtList()
  let r = ident("result")
  for key in keys:
    let keyLit = newLit key
    result.add quote do:
      `r` = `r` and discKeyMatch(`obj`, `json`, `keyLit`)

proc discKeysMatch[T](obj: T, json: JsonNode, keys: static seq[string]): bool =
  result = true
  discKeysMatchBodyGen(obj, json, keys)

proc fromJson*[T](a: var T, b: JsonNode, opt = Joptions()) =
  ## inplace version of `jsonTo`
  #[
  adding "json path" leading to `b` can be added in future work.
  ]#
  checkJson b != nil, $($T, b)
  when compiles(fromJsonHook(a, b, opt)): fromJsonHook(a, b, opt)
  elif compiles(fromJsonHook(a, b)): fromJsonHook(a, b)
  elif T is bool: a = to(b,T)
  elif T is enum:
    case b.kind
    of JInt: a = T(b.getBiggestInt())
    of JString: a = parseEnum[T](b.getStr())
    else: checkJson false, fmt"Expecting int/string for {$T} got {b.pretty()}"
  elif T is uint|uint64: a = T(to(b, uint64))
  elif T is Ordinal: a = cast[T](to(b, int))
  elif T is pointer: a = cast[pointer](to(b, int))
  elif T is distinct:
    when nimvm:
      # bug, potentially related to https://github.com/nim-lang/Nim/issues/12282
      a = T(jsonTo(b, distinctBase(T)))
    else:
      a.distinctBase.fromJson(b)
  elif T is string|SomeNumber: a = to(b,T)
  elif T is cstring:
    case b.kind
    of JNull: a = nil
    of JString: a = b.str
    else: checkJson false, fmt"Expecting null/string for {$T} got {b.pretty()}"
  elif T is JsonNode: a = b
  elif T is ref | ptr:
    if b.kind == JNull: a = nil
    else:
      a = T()
      fromJson(a[], b, opt)
  elif T is array:
    checkJson a.len == b.len, fmt"Json array size doesn't match for {$T}"
    var i = 0
    for ai in mitems(a):
      fromJson(ai, b[i], opt)
      i.inc
  elif T is set:
    type E = typeof(for ai in a: ai)
    for val in b.getElems:
      incl a, jsonTo(val, E)
  elif T is seq:
    a.setLen b.len
    for i, val in b.getElems:
      fromJson(a[i], val, opt)
  elif T is object:
    template fun(key, typ): untyped {.used.} =
      if b.hasKey key:
        jsonTo(b[key], typ)
      elif hasField(a, key):
        accessField(a, key)
      else:
        default(typ)
    const keys = getDiscriminants(T)
    when keys.len == 0:
      fromJsonFields(a, nil, b, keys, opt)
    else:
      if discKeysMatch(a, b, keys):
        fromJsonFields(a, nil, b, keys, opt)
      else:
        var newObj = initCaseObject(T, fun)
        fromJsonFields(newObj, a, b, keys, opt)
        a = newObj
  elif T is tuple:
    when isNamedTuple(T):
      fromJsonFields(a, nil, b, seq[string].default, opt)
    else:
      checkJson b.kind == JArray, $(b.kind) # we could customize whether to allow JNull

      when compiles(tupleLen(T)):
        let tupleSize = tupleLen(T)
      else:
        # Tuple len isn't in csources_v1 so using tupleLen would fail.
        # Else branch basically never runs (tupleLen added in 1.1 and jsonutils in 1.4), but here for consistency
        var tupleSize = 0
        for val in fields(a):
          tupleSize.inc

      checkJson b.len == tupleSize, fmt"Json doesn't match expected length of {tupleSize}, got {b.pretty()}"
      var i = 0
      for val in fields(a):
        fromJson(val, b[i], opt)
        i.inc
  else:
    # checkJson not appropriate here
    static: doAssert false, "not yet implemented: " & $T

proc jsonTo*(b: JsonNode, T: typedesc, opt = Joptions()): T =
  ## reverse of `toJson`
  fromJson(result, b, opt)

proc toJson*[T](a: T, opt = initToJsonOptions()): JsonNode =
  ## serializes `a` to json; uses `toJsonHook(a: T)` if it's in scope to
  ## customize serialization, see strtabs.toJsonHook for an example.
  ##
  ## .. note:: With `-d:nimPreviewJsonutilsHoleyEnum`, `toJson` now can 
  ##    serialize/deserialize holey enums as regular enums (via `ord`) instead of as strings.
  ##    It is expected that this behavior becomes the new default in upcoming versions.
  when compiles(toJsonHook(a, opt)): result = toJsonHook(a, opt)
  elif compiles(toJsonHook(a)): result = toJsonHook(a)
  elif T is object | tuple:
    when T is object or isNamedTuple(T):
      result = newJObject()
      for k, v in a.fieldPairs: result[k] = toJson(v, opt)
    else:
      result = newJArray()
      for v in a.fields: result.add toJson(v, opt)
  elif T is ref | ptr:
    template impl =
      if system.`==`(a, nil): result = newJNull()
      else: result = toJson(a[], opt)
    when T is JsonNode:
      case opt.jsonNodeMode
      of joptJsonNodeAsRef: result = a
      of joptJsonNodeAsCopy: result = copy(a)
      of joptJsonNodeAsObject: impl()
    else: impl()
  elif T is array | seq | set:
    result = newJArray()
    for ai in a: result.add toJson(ai, opt)
  elif T is pointer: result = toJson(cast[int](a), opt)
    # edge case: `a == nil` could've also led to `newJNull()`, but this results
    # in simpler code for `toJson` and `fromJson`.
  elif T is distinct: result = toJson(a.distinctBase, opt)
  elif T is bool: result = %(a)
  elif T is SomeInteger: result = %a
  elif T is enum:
    case opt.enumMode
    of joptEnumOrd:
      when T is Ordinal or defined(nimPreviewJsonutilsHoleyEnum): %(a.ord)
      else: toJson($a, opt)
    of joptEnumSymbol:
      when T is OrdinalEnum:
        toJson(symbolName(a), opt)
      else:
        toJson($a, opt)
    of joptEnumString: toJson($a, opt)
  elif T is Ordinal: result = %(a.ord)
  elif T is cstring: (if a == nil: result = newJNull() else: result = % $a)
  else: result = %a

proc fromJsonHook*[K: string|cstring, V](t: var (Table[K, V] | OrderedTable[K, V]),
                         jsonNode: JsonNode, opt = Joptions()) =
  ## Enables `fromJson` for `Table` and `OrderedTable` types.
  ##
  ## See also:
  ## * `toJsonHook proc<#toJsonHook>`_
  runnableExamples:
    import std/[tables, json]
    var foo: tuple[t: Table[string, int], ot: OrderedTable[string, int]]
    fromJson(foo, parseJson("""
      {"t":{"two":2,"one":1},"ot":{"one":1,"three":3}}"""))
    assert foo.t == [("one", 1), ("two", 2)].toTable
    assert foo.ot == [("one", 1), ("three", 3)].toOrderedTable

  assert jsonNode.kind == JObject,
          "The kind of the `jsonNode` must be `JObject`, but its actual " &
          "type is `" & $jsonNode.kind & "`."
  clear(t)
  for k, v in jsonNode:
    t[k] = jsonTo(v, V, opt)

proc toJsonHook*[K: string|cstring, V](t: (Table[K, V] | OrderedTable[K, V]), opt = initToJsonOptions()): JsonNode =
  ## Enables `toJson` for `Table` and `OrderedTable` types.
  ##
  ## See also:
  ## * `fromJsonHook proc<#fromJsonHook,,JsonNode>`_
  # pending PR #9217 use: toSeq(a) instead of `collect` in `runnableExamples`.
  runnableExamples:
    import std/[tables, json, sugar]
    let foo = (
      t: [("two", 2)].toTable,
      ot: [("one", 1), ("three", 3)].toOrderedTable)
    assert $toJson(foo) == """{"t":{"two":2},"ot":{"one":1,"three":3}}"""
    # if keys are not string|cstring, you can use this:
    let a = {10: "foo", 11: "bar"}.newOrderedTable
    let a2 = collect: (for k,v in a: (k,v))
    assert $toJson(a2) == """[[10,"foo"],[11,"bar"]]"""

  result = newJObject()
  for k, v in pairs(t):
    # not sure if $k has overhead for string
    result[(when K is string: k else: $k)] = toJson(v, opt)

proc fromJsonHook*[A](s: var SomeSet[A], jsonNode: JsonNode, opt = Joptions()) =
  ## Enables `fromJson` for `HashSet` and `OrderedSet` types.
  ##
  ## See also:
  ## * `toJsonHook proc<#toJsonHook,SomeSet[A]>`_
  runnableExamples:
    import std/[sets, json]
    var foo: tuple[hs: HashSet[string], os: OrderedSet[string]]
    fromJson(foo, parseJson("""
      {"hs": ["hash", "set"], "os": ["ordered", "set"]}"""))
    assert foo.hs == ["hash", "set"].toHashSet
    assert foo.os == ["ordered", "set"].toOrderedSet

  assert jsonNode.kind == JArray,
          "The kind of the `jsonNode` must be `JArray`, but its actual " &
          "type is `" & $jsonNode.kind & "`."
  clear(s)
  for v in jsonNode:
    incl(s, jsonTo(v, A, opt))

proc toJsonHook*[A](s: SomeSet[A], opt = initToJsonOptions()): JsonNode =
  ## Enables `toJson` for `HashSet` and `OrderedSet` types.
  ##
  ## See also:
  ## * `fromJsonHook proc<#fromJsonHook,SomeSet[A],JsonNode>`_
  runnableExamples:
    import std/[sets, json]
    let foo = (hs: ["hash"].toHashSet, os: ["ordered", "set"].toOrderedSet)
    assert $toJson(foo) == """{"hs":["hash"],"os":["ordered","set"]}"""

  result = newJArray()
  for k in s:
    add(result, toJson(k, opt))

proc fromJsonHook*[T](self: var Option[T], jsonNode: JsonNode, opt = Joptions()) =
  ## Enables `fromJson` for `Option` types.
  ##
  ## See also:
  ## * `toJsonHook proc<#toJsonHook,Option[T]>`_
  runnableExamples:
    import std/[options, json]
    var opt: Option[string]
    fromJsonHook(opt, parseJson("\"test\""))
    assert get(opt) == "test"
    fromJson(opt, parseJson("null"))
    assert isNone(opt)

  if jsonNode.kind != JNull:
    self = some(jsonTo(jsonNode, T, opt))
  else:
    self = none[T]()

proc toJsonHook*[T](self: Option[T], opt = initToJsonOptions()): JsonNode =
  ## Enables `toJson` for `Option` types.
  ##
  ## See also:
  ## * `fromJsonHook proc<#fromJsonHook,Option[T],JsonNode>`_
  runnableExamples:
    import std/[options, json]
    let optSome = some("test")
    assert $toJson(optSome) == "\"test\""
    let optNone = none[string]()
    assert $toJson(optNone) == "null"

  if isSome(self):
    toJson(get(self), opt)
  else:
    newJNull()

proc fromJsonHook*(a: var StringTableRef, b: JsonNode) =
  ## Enables `fromJson` for `StringTableRef` type.
  ##
  ## See also:
  ## * `toJsonHook proc<#toJsonHook,StringTableRef>`_
  runnableExamples:
    import std/[strtabs, json]
    var t = newStringTable(modeCaseSensitive)
    let jsonStr = """{"mode": 0, "table": {"name": "John", "surname": "Doe"}}"""
    fromJsonHook(t, parseJson(jsonStr))
    assert t[] == newStringTable("name", "John", "surname", "Doe",
                                 modeCaseSensitive)[]

  var mode = jsonTo(b["mode"], StringTableMode)
  a = newStringTable(mode)
  let b2 = b["table"]
  for k,v in b2: a[k] = jsonTo(v, string)

proc toJsonHook*(a: StringTableRef): JsonNode =
  ## Enables `toJson` for `StringTableRef` type.
  ##
  ## See also:
  ## * `fromJsonHook proc<#fromJsonHook,StringTableRef,JsonNode>`_
  runnableExamples:
    import std/[strtabs, json]
    let t = newStringTable("name", "John", "surname", "Doe", modeCaseSensitive)
    let jsonStr = """{"mode": "modeCaseSensitive",
                      "table": {"name": "John", "surname": "Doe"}}"""
    assert toJson(t) == parseJson(jsonStr)

  result = newJObject()
  result["mode"] = toJson($a.mode)
  let t = newJObject()
  for k,v in a: t[k] = toJson(v)
  result["table"] = t