From c7a1a7b8bf38ab5c15decdf913dfc272ad922c21 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Mon, 8 Jun 2020 01:35:23 -0700 Subject: `toJson`, `jsonTo`, json (de)serialization for custom types; remove dependency on strtabs thanks to a hooking mechanism (#14563) * json custom serialization; application for strtabs * serialize using nesting * make toJson more feature complete * add since * Revert "Improve JSON serialisation of strtabs (#14549)" This reverts commit 7cb4ef26addb3bb5ce2405d8396df6fd41664dae. * better approach via mixin * toJson, jsonTo * fix test * address comments * move to jsonutils * doc * cleanups * also test for js * also test for vm --- lib/pure/json.nim | 42 +++++------------ lib/pure/strtabs.nim | 20 +++++++- lib/std/jsonutils.nim | 126 ++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 157 insertions(+), 31 deletions(-) create mode 100644 lib/std/jsonutils.nim (limited to 'lib') diff --git a/lib/pure/json.nim b/lib/pure/json.nim index 58a8c1dce..041816c7d 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -140,6 +140,9 @@ ## var j2 = %* {"name": "Isaac", "books": ["Robot Dreams"]} ## j2["details"] = %* {"age":35, "pi":3.1415} ## echo j2 +## +## See also: std/jsonutils for hookable json serialization/deserialization +## of arbitrary types. runnableExamples: ## Note: for JObject, key ordering is preserved, unlike in some languages, @@ -149,8 +152,10 @@ runnableExamples: doAssert $(%* Foo()) == """{"a1":0,"a2":0,"a0":0,"a3":0,"a4":0}""" import - hashes, tables, strtabs, strutils, lexbase, streams, macros, parsejson, - options + hashes, tables, strutils, lexbase, streams, macros, parsejson + +import options # xxx remove this dependency using same approach as https://github.com/nim-lang/Nim/pull/14563 +import std/private/since export tables.`$` @@ -353,14 +358,6 @@ proc `[]=`*(obj: JsonNode, key: string, val: JsonNode) {.inline.} = assert(obj.kind == JObject) obj.fields[key] = val -proc `%`*(table: StringTableRef): JsonNode = - ## Generic constructor for JSON data. Creates a new ``JObject JsonNode``. - result = newJObject() - result["mode"] = %($table.mode) - var data = newJObject() - for k, v in table: data[k] = %v - result["data"] = data - proc `%`*[T: object](o: T): JsonNode = ## Construct JsonNode from tuples and objects. result = newJObject() @@ -378,20 +375,20 @@ proc `%`*(o: enum): JsonNode = ## string. Creates a new ``JString JsonNode``. result = %($o) -proc toJson(x: NimNode): NimNode {.compileTime.} = +proc toJsonImpl(x: NimNode): NimNode {.compileTime.} = case x.kind of nnkBracket: # array if x.len == 0: return newCall(bindSym"newJArray") result = newNimNode(nnkBracket) for i in 0 ..< x.len: - result.add(toJson(x[i])) + result.add(toJsonImpl(x[i])) result = newCall(bindSym("%", brOpen), result) of nnkTableConstr: # object if x.len == 0: return newCall(bindSym"newJObject") result = newNimNode(nnkTableConstr) for i in 0 ..< x.len: x[i].expectKind nnkExprColonExpr - result.add newTree(nnkExprColonExpr, x[i][0], toJson(x[i][1])) + result.add newTree(nnkExprColonExpr, x[i][0], toJsonImpl(x[i][1])) result = newCall(bindSym("%", brOpen), result) of nnkCurly: # empty object x.expectLen(0) @@ -399,7 +396,7 @@ proc toJson(x: NimNode): NimNode {.compileTime.} = of nnkNilLit: result = newCall(bindSym"newJNull") of nnkPar: - if x.len == 1: result = toJson(x[0]) + if x.len == 1: result = toJsonImpl(x[0]) else: result = newCall(bindSym("%", brOpen), x) else: result = newCall(bindSym("%", brOpen), x) @@ -407,7 +404,7 @@ proc toJson(x: NimNode): NimNode {.compileTime.} = macro `%*`*(x: untyped): untyped = ## Convert an expression to a JsonNode directly, without having to specify ## `%` for every element. - result = toJson(x) + result = toJsonImpl(x) proc `==`*(a, b: JsonNode): bool = ## Check two nodes for equality @@ -992,7 +989,6 @@ when defined(nimFixedForwardGeneric): proc initFromJson[S,T](dst: var array[S,T]; jsonNode: JsonNode; jsonPath: var string) proc initFromJson[T](dst: var Table[string,T]; jsonNode: JsonNode; jsonPath: var string) proc initFromJson[T](dst: var OrderedTable[string,T]; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson(dst: var StringTableRef; jsonNode: JsonNode; jsonPath: var string) proc initFromJson[T](dst: var ref T; jsonNode: JsonNode; jsonPath: var string) proc initFromJson[T](dst: var Option[T]; jsonNode: JsonNode; jsonPath: var string) proc initFromJson[T: distinct](dst: var T; jsonNode: JsonNode; jsonPath: var string) @@ -1073,20 +1069,6 @@ when defined(nimFixedForwardGeneric): initFromJson(mgetOrPut(dst, key, default(T)), jsonNode[key], jsonPath) jsonPath.setLen originalJsonPathLen - proc mgetOrPut(tab: var StringTableRef, key: string): var string = - if not tab.hasKey(key): tab[key] = "" - result = tab[key] - - proc initFromJson(dst: var StringTableRef; jsonNode: JsonNode; jsonPath: var string) = - dst = newStringTable(parseEnum[StringTableMode](jsonNode["mode"].getStr)) - verifyJsonKind(jsonNode, {JObject}, jsonPath) - let originalJsonPathLen = jsonPath.len - for key in keys(jsonNode["data"].fields): - jsonPath.add '.' - jsonPath.add key - initFromJson(mgetOrPut(dst, key), jsonNode[key], jsonPath) - jsonPath.setLen originalJsonPathLen - proc initFromJson[T](dst: var ref T; jsonNode: JsonNode; jsonPath: var string) = verifyJsonKind(jsonNode, {JObject, JNull}, jsonPath) if jsonNode.kind == JNull: diff --git a/lib/pure/strtabs.nim b/lib/pure/strtabs.nim index cfafdf018..81ff7fbde 100644 --- a/lib/pure/strtabs.nim +++ b/lib/pure/strtabs.nim @@ -71,7 +71,7 @@ type StringTableObj* = object of RootObj counter: int data: KeyValuePairSeq - mode*: StringTableMode + mode: StringTableMode StringTableRef* = ref StringTableObj @@ -419,6 +419,24 @@ proc `%`*(f: string, t: StringTableRef, flags: set[FormatFlag] = {}): string {. add(result, f[i]) inc(i) +since (1,3,5): + proc fromJsonHook*[T](a: var StringTableRef, b: T) = + ## for json.fromJson + mixin jsonTo + var mode = jsonTo(b["mode"], StringTableMode) + a = newStringTable(mode) + let b2 = b["table"] + for k,v in b2: a[k] = jsonTo(v, string) + + proc toJsonHook*[](a: StringTableRef): auto = + ## for json.toJson + mixin newJObject + mixin toJson + result = newJObject() + result["mode"] = toJson($a.mode) + let t = newJObject() + for k,v in a: t[k] = toJson(v) + result["table"] = t when isMainModule: var x = {"k": "v", "11": "22", "565": "67"}.newStringTable diff --git a/lib/std/jsonutils.nim b/lib/std/jsonutils.nim new file mode 100644 index 000000000..bfa600fa9 --- /dev/null +++ b/lib/std/jsonutils.nim @@ -0,0 +1,126 @@ +##[ +This module implements a hookable (de)serialization for arbitrary types. +Design goal: avoid importing modules where a custom serialization is needed; +see strtabs.fromJsonHook,toJsonHook for an example. + +]## + +import std/[json,tables,strutils] + +#[ +xxx +use toJsonHook,fromJsonHook for Table|OrderedTable +add Options support also using toJsonHook,fromJsonHook and remove `json=>options` dependency + +future direction: +add a way to customize serialization, for eg allowing missing +or extra fields in JsonNode, field renaming, and a way to handle cyclic references +using a cache of already visited addresses. +]# + +proc isNamedTuple(T: typedesc): bool {.magic: "TypeTrait".} +proc distinctBase(T: typedesc): typedesc {.magic: "TypeTrait".} +template distinctBase[T](a: T): untyped = distinctBase(type(a))(a) + +proc checkJsonImpl(cond: bool, condStr: string, msg = "") = + if not cond: + # just pick 1 exception type for simplicity; other choices would be: + # JsonError, JsonParser, JsonKindError + raise newException(ValueError, msg) + +template checkJson(cond: untyped, msg = "") = + checkJsonImpl(cond, astToStr(cond), msg) + +proc fromJson*[T](a: var T, b: JsonNode) = + ## inplace version of `jsonTo` + #[ + adding "json path" leading to `b` can be added in future work. + ]# + checkJson b != nil, $($T, b) + when compiles(fromJsonHook(a, b)): fromJsonHook(a, b) + elif T is bool: a = to(b,T) + elif T is Table | OrderedTable: + a.clear + for k,v in b: + a[k] = jsonTo(v, typeof(a[k])) + elif T is enum: + case b.kind + of JInt: a = T(b.getBiggestInt()) + of JString: a = parseEnum[T](b.getStr()) + else: checkJson false, $($T, " ", b) + elif T is Ordinal: a = T(to(b, int)) + elif T is pointer: a = cast[pointer](to(b, int)) + elif T is distinct: + when nimvm: + # bug, potentially related to https://github.com/nim-lang/Nim/issues/12282 + a = T(jsonTo(b, distinctBase(T))) + else: + a.distinctBase.fromJson(b) + elif T is string|SomeNumber: a = to(b,T) + elif T is JsonNode: a = b + elif T is ref | ptr: + if b.kind == JNull: a = nil + else: + a = T() + fromJson(a[], b) + elif T is array: + checkJson a.len == b.len, $(a.len, b.len, $T) + for i, val in b.getElems: + fromJson(a[i], val) + elif T is seq: + a.setLen b.len + for i, val in b.getElems: + fromJson(a[i], val) + elif T is object | tuple: + const isNamed = T is object or isNamedTuple(T) + when isNamed: + checkJson b.kind == JObject, $(b.kind) # we could customize whether to allow JNull + var num = 0 + for key, val in fieldPairs(a): + num.inc + if b.hasKey key: + fromJson(val, b[key]) + else: + # we could customize to allow this + checkJson false, $($T, key, b) + checkJson b.len == num, $(b.len, num, $T, b) # could customize + else: + checkJson b.kind == JArray, $(b.kind) # we could customize whether to allow JNull + var i = 0 + for val in fields(a): + fromJson(val, b[i]) + i.inc + else: + # checkJson not appropriate here + static: doAssert false, "not yet implemented: " & $T + +proc jsonTo*(b: JsonNode, T: typedesc): T = + ## reverse of `toJson` + fromJson(result, b) + +proc toJson*[T](a: T): JsonNode = + ## serializes `a` to json; uses `toJsonHook(a: T)` if it's in scope to + ## customize serialization, see strtabs.toJsonHook for an example. + when compiles(toJsonHook(a)): result = toJsonHook(a) + elif T is Table | OrderedTable: + result = newJObject() + for k, v in pairs(a): result[k] = toJson(v) + elif T is object | tuple: + const isNamed = T is object or isNamedTuple(T) + when isNamed: + result = newJObject() + for k, v in a.fieldPairs: result[k] = toJson(v) + else: + result = newJArray() + for v in a.fields: result.add toJson(v) + elif T is ref | ptr: + if a == nil: result = newJNull() + else: result = toJson(a[]) + elif T is array | seq: + result = newJArray() + for ai in a: result.add toJson(ai) + elif T is pointer: result = toJson(cast[int](a)) + elif T is distinct: result = toJson(a.distinctBase) + elif T is bool: result = %(a) + elif T is Ordinal: result = %(a.ord) + else: result = %a -- cgit 1.4.1-2-gfad0