`toJson`, `jsonTo`, json (de)serialization for custom types; remove dependency on strtabs thanks to a hooking mechanism (#14563)

* json custom serialization; application for strtabs
* serialize using nesting
* make toJson more feature complete
* add since
* Revert "Improve JSON serialisation of strtabs (#14549)"

This reverts commit 7cb4ef26addb3bb5ce2405d8396df6fd41664dae.

* better approach via mixin
* toJson, jsonTo
* fix test
* address comments
* move to jsonutils
* doc
* cleanups
* also test for js
* also test for vm

3 files changed, 157 insertions, 31 deletions

diff --git a/lib/pure/json.nim b/lib/pure/json.nim
index 58a8c1dce..041816c7d 100644
--- a/lib/pure/json.nim
+++ b/lib/pure/json.nim
@@ -140,6 +140,9 @@
 ##   var j2 = %* {"name": "Isaac", "books": ["Robot Dreams"]}
 ##   j2["details"] = %* {"age":35, "pi":3.1415}
 ##   echo j2
+##
+## See also: std/jsonutils for hookable json serialization/deserialization
+## of arbitrary types.
 
 runnableExamples:
   ## Note: for JObject, key ordering is preserved, unlike in some languages,
@@ -149,8 +152,10 @@ runnableExamples:
   doAssert $(%* Foo()) == """{"a1":0,"a2":0,"a0":0,"a3":0,"a4":0}"""
 
 import
-  hashes, tables, strtabs, strutils, lexbase, streams, macros, parsejson,
-  options
+  hashes, tables, strutils, lexbase, streams, macros, parsejson
+
+import options # xxx remove this dependency using same approach as https://github.com/nim-lang/Nim/pull/14563
+import std/private/since
 
 export
   tables.`$`
@@ -353,14 +358,6 @@ proc `[]=`*(obj: JsonNode, key: string, val: JsonNode) {.inline.} =
   assert(obj.kind == JObject)
   obj.fields[key] = val
 
-proc `%`*(table: StringTableRef): JsonNode =
-  ## Generic constructor for JSON data. Creates a new ``JObject JsonNode``.
-  result = newJObject()
-  result["mode"] = %($table.mode)
-  var data = newJObject()
-  for k, v in table: data[k] = %v
-  result["data"] = data
-
 proc `%`*[T: object](o: T): JsonNode =
   ## Construct JsonNode from tuples and objects.
   result = newJObject()
@@ -378,20 +375,20 @@ proc `%`*(o: enum): JsonNode =
   ## string. Creates a new ``JString JsonNode``.
   result = %($o)
 
-proc toJson(x: NimNode): NimNode {.compileTime.} =
+proc toJsonImpl(x: NimNode): NimNode {.compileTime.} =
   case x.kind
   of nnkBracket: # array
     if x.len == 0: return newCall(bindSym"newJArray")
     result = newNimNode(nnkBracket)
     for i in 0 ..< x.len:
-      result.add(toJson(x[i]))
+      result.add(toJsonImpl(x[i]))
     result = newCall(bindSym("%", brOpen), result)
   of nnkTableConstr: # object
     if x.len == 0: return newCall(bindSym"newJObject")
     result = newNimNode(nnkTableConstr)
     for i in 0 ..< x.len:
       x[i].expectKind nnkExprColonExpr
-      result.add newTree(nnkExprColonExpr, x[i][0], toJson(x[i][1]))
+      result.add newTree(nnkExprColonExpr, x[i][0], toJsonImpl(x[i][1]))
     result = newCall(bindSym("%", brOpen), result)
   of nnkCurly: # empty object
     x.expectLen(0)
@@ -399,7 +396,7 @@ proc toJson(x: NimNode): NimNode {.compileTime.} =
   of nnkNilLit:
     result = newCall(bindSym"newJNull")
   of nnkPar:
-    if x.len == 1: result = toJson(x[0])
+    if x.len == 1: result = toJsonImpl(x[0])
     else: result = newCall(bindSym("%", brOpen), x)
   else:
     result = newCall(bindSym("%", brOpen), x)
@@ -407,7 +404,7 @@ proc toJson(x: NimNode): NimNode {.compileTime.} =
 macro `%*`*(x: untyped): untyped =
   ## Convert an expression to a JsonNode directly, without having to specify
   ## `%` for every element.
-  result = toJson(x)
+  result = toJsonImpl(x)
 
 proc `==`*(a, b: JsonNode): bool =
   ## Check two nodes for equality
@@ -992,7 +989,6 @@ when defined(nimFixedForwardGeneric):
   proc initFromJson[S,T](dst: var array[S,T]; jsonNode: JsonNode; jsonPath: var string)
   proc initFromJson[T](dst: var Table[string,T]; jsonNode: JsonNode; jsonPath: var string)
   proc initFromJson[T](dst: var OrderedTable[string,T]; jsonNode: JsonNode; jsonPath: var string)
-  proc initFromJson(dst: var StringTableRef; jsonNode: JsonNode; jsonPath: var string)
   proc initFromJson[T](dst: var ref T; jsonNode: JsonNode; jsonPath: var string)
   proc initFromJson[T](dst: var Option[T]; jsonNode: JsonNode; jsonPath: var string)
   proc initFromJson[T: distinct](dst: var T; jsonNode: JsonNode; jsonPath: var string)
@@ -1073,20 +1069,6 @@ when defined(nimFixedForwardGeneric):
       initFromJson(mgetOrPut(dst, key, default(T)), jsonNode[key], jsonPath)
       jsonPath.setLen originalJsonPathLen
 
-  proc mgetOrPut(tab: var StringTableRef, key: string): var string =
-    if not tab.hasKey(key): tab[key] = ""
-    result = tab[key]
-
-  proc initFromJson(dst: var StringTableRef; jsonNode: JsonNode; jsonPath: var string) =
-    dst = newStringTable(parseEnum[StringTableMode](jsonNode["mode"].getStr))
-    verifyJsonKind(jsonNode, {JObject}, jsonPath)
-    let originalJsonPathLen = jsonPath.len
-    for key in keys(jsonNode["data"].fields):
-      jsonPath.add '.'
-      jsonPath.add key
-      initFromJson(mgetOrPut(dst, key), jsonNode[key], jsonPath)
-      jsonPath.setLen originalJsonPathLen
-
   proc initFromJson[T](dst: var ref T; jsonNode: JsonNode; jsonPath: var string) =
     verifyJsonKind(jsonNode, {JObject, JNull}, jsonPath)
     if jsonNode.kind == JNull:
diff --git a/lib/pure/strtabs.nim b/lib/pure/strtabs.nim
index cfafdf018..81ff7fbde 100644
--- a/lib/pure/strtabs.nim
+++ b/lib/pure/strtabs.nim
@@ -71,7 +71,7 @@ type
   StringTableObj* = object of RootObj
     counter: int
     data: KeyValuePairSeq
-    mode*: StringTableMode
+    mode: StringTableMode
 
   StringTableRef* = ref StringTableObj
 
@@ -419,6 +419,24 @@ proc `%`*(f: string, t: StringTableRef, flags: set[FormatFlag] = {}): string {.
       add(result, f[i])
       inc(i)
 
+since (1,3,5):
+  proc fromJsonHook*[T](a: var StringTableRef, b: T) =
+    ## for json.fromJson
+    mixin jsonTo
+    var mode = jsonTo(b["mode"], StringTableMode)
+    a = newStringTable(mode)
+    let b2 = b["table"]
+    for k,v in b2: a[k] = jsonTo(v, string)
+
+  proc toJsonHook*[](a: StringTableRef): auto =
+    ## for json.toJson
+    mixin newJObject
+    mixin toJson
+    result = newJObject()
+    result["mode"] = toJson($a.mode)
+    let t = newJObject()
+    for k,v in a: t[k] = toJson(v)
+    result["table"] = t
 
 when isMainModule:
   var x = {"k": "v", "11": "22", "565": "67"}.newStringTable
diff --git a/lib/std/jsonutils.nim b/lib/std/jsonutils.nim
new file mode 100644
index 000000000..bfa600fa9
--- /dev/null
+++ b/lib/std/jsonutils.nim
@@ -0,0 +1,126 @@
+##[
+This module implements a hookable (de)serialization for arbitrary types.
+Design goal: avoid importing modules where a custom serialization is needed;
+see strtabs.fromJsonHook,toJsonHook for an example.
+
+]##
+
+import std/[json,tables,strutils]
+
+#[
+xxx
+use toJsonHook,fromJsonHook for Table|OrderedTable
+add Options support also using toJsonHook,fromJsonHook and remove `json=>options` dependency
+
+future direction:
+add a way to customize serialization, for eg allowing missing
+or extra fields in JsonNode, field renaming, and a way to handle cyclic references
+using a cache of already visited addresses.
+]#
+
+proc isNamedTuple(T: typedesc): bool {.magic: "TypeTrait".}
+proc distinctBase(T: typedesc): typedesc {.magic: "TypeTrait".}
+template distinctBase[T](a: T): untyped = distinctBase(type(a))(a)
+
+proc checkJsonImpl(cond: bool, condStr: string, msg = "") =
+  if not cond:
+    # just pick 1 exception type for simplicity; other choices would be:
+    # JsonError, JsonParser, JsonKindError
+    raise newException(ValueError, msg)
+
+template checkJson(cond: untyped, msg = "") =
+  checkJsonImpl(cond, astToStr(cond), msg)
+
+proc fromJson*[T](a: var T, b: JsonNode) =
+  ## inplace version of `jsonTo`
+  #[
+  adding "json path" leading to `b` can be added in future work.
+  ]#
+  checkJson b != nil, $($T, b)
+  when compiles(fromJsonHook(a, b)): fromJsonHook(a, b)
+  elif T is bool: a = to(b,T)
+  elif T is Table | OrderedTable:
+    a.clear
+    for k,v in b:
+      a[k] = jsonTo(v, typeof(a[k]))
+  elif T is enum:
+    case b.kind
+    of JInt: a = T(b.getBiggestInt())
+    of JString: a = parseEnum[T](b.getStr())
+    else: checkJson false, $($T, " ", b)
+  elif T is Ordinal: a = T(to(b, int))
+  elif T is pointer: a = cast[pointer](to(b, int))
+  elif T is distinct:
+    when nimvm:
+      # bug, potentially related to https://github.com/nim-lang/Nim/issues/12282
+      a = T(jsonTo(b, distinctBase(T)))
+    else:
+      a.distinctBase.fromJson(b)
+  elif T is string|SomeNumber: a = to(b,T)
+  elif T is JsonNode: a = b
+  elif T is ref | ptr:
+    if b.kind == JNull: a = nil
+    else:
+      a = T()
+      fromJson(a[], b)
+  elif T is array:
+    checkJson a.len == b.len, $(a.len, b.len, $T)
+    for i, val in b.getElems:
+      fromJson(a[i], val)
+  elif T is seq:
+    a.setLen b.len
+    for i, val in b.getElems:
+      fromJson(a[i], val)
+  elif T is object | tuple:
+    const isNamed = T is object or isNamedTuple(T)
+    when isNamed:
+      checkJson b.kind == JObject, $(b.kind) # we could customize whether to allow JNull
+      var num = 0
+      for key, val in fieldPairs(a):
+        num.inc
+        if b.hasKey key:
+          fromJson(val, b[key])
+        else:
+          # we could customize to allow this
+          checkJson false, $($T, key, b)
+      checkJson b.len == num, $(b.len, num, $T, b) # could customize
+    else:
+      checkJson b.kind == JArray, $(b.kind) # we could customize whether to allow JNull
+      var i = 0
+      for val in fields(a):
+        fromJson(val, b[i])
+        i.inc
+  else:
+    # checkJson not appropriate here
+    static: doAssert false, "not yet implemented: " & $T
+
+proc jsonTo*(b: JsonNode, T: typedesc): T =
+  ## reverse of `toJson`
+  fromJson(result, b)
+
+proc toJson*[T](a: T): JsonNode =
+  ## serializes `a` to json; uses `toJsonHook(a: T)` if it's in scope to
+  ## customize serialization, see strtabs.toJsonHook for an example.
+  when compiles(toJsonHook(a)): result = toJsonHook(a)
+  elif T is Table | OrderedTable:
+    result = newJObject()
+    for k, v in pairs(a): result[k] = toJson(v)
+  elif T is object | tuple:
+    const isNamed = T is object or isNamedTuple(T)
+    when isNamed:
+      result = newJObject()
+      for k, v in a.fieldPairs: result[k] = toJson(v)
+    else:
+      result = newJArray()
+      for v in a.fields: result.add toJson(v)
+  elif T is ref | ptr:
+    if a == nil: result = newJNull()
+    else: result = toJson(a[])
+  elif T is array | seq:
+    result = newJArray()
+    for ai in a: result.add toJson(ai)
+  elif T is pointer: result = toJson(cast[int](a))
+  elif T is distinct: result = toJson(a.distinctBase)
+  elif T is bool: result = %(a)
+  elif T is Ordinal: result = %(a.ord)
+  else: result = %a