diff options
Diffstat (limited to 'lib')
161 files changed, 8816 insertions, 6429 deletions
diff --git a/lib/core/allocators.nim b/lib/core/allocators.nim index d6608a203..62f5e9756 100644 --- a/lib/core/allocators.nim +++ b/lib/core/allocators.nim @@ -8,8 +8,8 @@ # type - Allocator* {.inheritable.} = ptr object - alloc*: proc (a: Allocator; size: int; alignment = 8): pointer {.nimcall.} + Allocator* = ptr object {.inheritable.} + alloc*: proc (a: Allocator; size: int; alignment: int = 8): pointer {.nimcall.} dealloc*: proc (a: Allocator; p: pointer; size: int) {.nimcall.} realloc*: proc (a: Allocator; p: pointer; oldSize, newSize: int): pointer {.nimcall.} @@ -22,14 +22,14 @@ proc getCurrentAllocator*(): Allocator = proc setCurrentAllocator*(a: Allocator) = currentAllocator = a -proc alloc*(size: int): pointer = +proc alloc*(size: int; alignment: int = 8): pointer = let a = getCurrentAllocator() - result = a.alloc(a, size) + result = a.alloc(a, size, alignment) proc dealloc*(p: pointer; size: int) = let a = getCurrentAllocator() - a.dealloc(a, size) + a.dealloc(a, p, size) proc realloc*(p: pointer; oldSize, newSize: int): pointer = let a = getCurrentAllocator() - result = a.realloc(a, oldSize, newSize) + result = a.realloc(a, p, oldSize, newSize) diff --git a/lib/core/macros.nim b/lib/core/macros.nim index b08a2198e..fa5cd67df 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -14,6 +14,9 @@ include "system/inclrtl" ## .. include:: ../../doc/astspec.txt +# If you look for the implementation of the magic symbol +# ``{.magic: "Foo".}``, search for `mFoo` and `opcFoo`. + type NimNodeKind* = enum nnkNone, nnkEmpty, nnkIdent, nnkSym, @@ -76,7 +79,8 @@ type nnkGotoState, nnkState, nnkBreakState, - nnkFuncDef + nnkFuncDef, + nnkTupleConstr NimNodeKinds* = set[NimNodeKind] NimTypeKind* = enum # some types are no longer used, see ast.nim @@ -118,7 +122,7 @@ type ## use ``ident"abc"``. NimSymObj = object # hidden - NimSym* = ref NimSymObj + NimSym* {.deprecated.} = ref NimSymObj ## represents a Nim *symbol* in the compiler; a *symbol* is a looked-up ## *ident*. @@ -130,28 +134,27 @@ const nnkLiterals* = {nnkCharLit..nnkNilLit} nnkCallKinds* = {nnkCall, nnkInfix, nnkPrefix, nnkPostfix, nnkCommand, nnkCallStrLit} + nnkPragmaCallKinds = {nnkExprColonExpr, nnkCall, nnkCallStrLit} proc `!`*(s: string): NimIdent {.magic: "StrToIdent", noSideEffect, deprecated.} ## constructs an identifier from the string `s` - ## **Deprecated since version 0.18.0**: Use ``toNimIdent`` instead. + ## **Deprecated since version 0.18.0**: Use ``ident`` or ``newIdentNode`` instead. -proc toNimIdent*(s: string): NimIdent {.magic: "StrToIdent", noSideEffect.} +proc toNimIdent*(s: string): NimIdent {.magic: "StrToIdent", noSideEffect, deprecated.} ## constructs an identifier from the string `s` + ## **Deprecated since version 0.18.1**; Use ``ident`` or ``newIdentNode`` instead. -proc `$`*(i: NimIdent): string {.magic: "IdentToStr", noSideEffect.} - ## converts a Nim identifier to a string - -proc `$`*(s: NimSym): string {.magic: "IdentToStr", noSideEffect.} - ## converts a Nim symbol to a string - -proc `==`*(a, b: NimIdent): bool {.magic: "EqIdent", noSideEffect.} +proc `==`*(a, b: NimIdent): bool {.magic: "EqIdent", noSideEffect, deprecated.} ## compares two Nim identifiers + ## **Deprecated since version 0.18.1**; Use ``==`` on ``NimNode`` instead. proc `==`*(a, b: NimNode): bool {.magic: "EqNimrodNode", noSideEffect.} ## compares two Nim nodes -proc `==`*(a, b: NimSym): bool {.magic: "EqNimrodNode", noSideEffect.} +proc `==`*(a, b: NimSym): bool {.magic: "EqNimrodNode", noSideEffect, deprecated.} ## compares two Nim symbols + ## **Deprecated since version 0.18.1**; Use ```==`(NimNode,NimNode)`` instead. + proc sameType*(a, b: NimNode): bool {.magic: "SameNodeType", noSideEffect.} = ## compares two Nim nodes' types. Return true if the types are the same, @@ -194,8 +197,47 @@ proc kind*(n: NimNode): NimNodeKind {.magic: "NKind", noSideEffect.} proc intVal*(n: NimNode): BiggestInt {.magic: "NIntVal", noSideEffect.} proc floatVal*(n: NimNode): BiggestFloat {.magic: "NFloatVal", noSideEffect.} -proc symbol*(n: NimNode): NimSym {.magic: "NSymbol", noSideEffect.} -proc ident*(n: NimNode): NimIdent {.magic: "NIdent", noSideEffect.} + +proc ident*(n: NimNode): NimIdent {.magic: "NIdent", noSideEffect, deprecated.} = + ## **Deprecated since version 0.18.1**; All functionality is defined on ``NimNode``. + +proc symbol*(n: NimNode): NimSym {.magic: "NSymbol", noSideEffect, deprecated.} + ## **Deprecated since version 0.18.1**; All functionality is defined on ``NimNode``. + +proc getImpl*(s: NimSym): NimNode {.magic: "GetImpl", noSideEffect, deprecated: "use `getImpl: NimNode -> NimNode` instead".} + +when defined(nimSymKind): + proc symKind*(symbol: NimNode): NimSymKind {.magic: "NSymKind", noSideEffect.} + proc getImpl*(symbol: NimNode): NimNode {.magic: "GetImpl", noSideEffect.} + proc strVal*(n: NimNode): string {.magic: "NStrVal", noSideEffect.} + ## retrieve the implementation of `symbol`. `symbol` can be a + ## routine or a const. + + proc `$`*(i: NimIdent): string {.magic: "NStrVal", noSideEffect, deprecated.} + ## converts a Nim identifier to a string + ## **Deprecated since version 0.18.1**; Use ``strVal`` instead. + + proc `$`*(s: NimSym): string {.magic: "NStrVal", noSideEffect, deprecated.} + ## converts a Nim symbol to a string + ## **Deprecated since version 0.18.1**; Use ``strVal`` instead. + +else: # bootstrapping substitute + proc getImpl*(symbol: NimNode): NimNode = + symbol.symbol.getImpl + + proc strValOld(n: NimNode): string {.magic: "NStrVal", noSideEffect.} + + proc `$`*(s: NimSym): string {.magic: "IdentToStr", noSideEffect.} + + proc `$`*(i: NimIdent): string {.magic: "IdentToStr", noSideEffect.} + + proc strVal*(n: NimNode): string = + if n.kind == nnkIdent: + $n.ident + elif n.kind == nnkSym: + $n.symbol + else: + n.strValOld proc getType*(n: NimNode): NimNode {.magic: "NGetType", noSideEffect.} ## with 'getType' you can access the node's `type`:idx:. A Nim type is @@ -213,26 +255,65 @@ proc getType*(n: typedesc): NimNode {.magic: "NGetType", noSideEffect.} proc typeKind*(n: NimNode): NimTypeKind {.magic: "NGetType", noSideEffect.} ## Returns the type kind of the node 'n' that should represent a type, that - ## means the node should have been obtained via `getType`. - -proc getTypeInst*(n: NimNode): NimNode {.magic: "NGetType", noSideEffect.} - ## Like getType except it includes generic parameters for a specific instance + ## means the node should have been obtained via ``getType``. + +proc getTypeInst*(n: NimNode): NimNode {.magic: "NGetType", noSideEffect.} = + ## Returns the `type`:idx: of a node in a form matching the way the + ## type instance was declared in the code. + runnableExamples: + type + Vec[N: static[int], T] = object + arr: array[N, T] + Vec4[T] = Vec[4, T] + Vec4f = Vec4[float32] + var a: Vec4f + var b: Vec4[float32] + var c: Vec[4, float32] + macro dumpTypeInst(x: typed): untyped = + newLit(x.getTypeInst.repr) + doAssert(dumpTypeInst(a) == "Vec4f") + doAssert(dumpTypeInst(b) == "Vec4[float32]") + doAssert(dumpTypeInst(c) == "Vec[4, float32]") proc getTypeInst*(n: typedesc): NimNode {.magic: "NGetType", noSideEffect.} - ## Like getType except it includes generic parameters for a specific instance - -proc getTypeImpl*(n: NimNode): NimNode {.magic: "NGetType", noSideEffect.} - ## Like getType except it includes generic parameters for the implementation + ## Version of ``getTypeInst`` which takes a ``typedesc``. + +proc getTypeImpl*(n: NimNode): NimNode {.magic: "NGetType", noSideEffect.} = + ## Returns the `type`:idx: of a node in a form matching the implementation + ## of the type. Any intermediate aliases are expanded to arrive at the final + ## type implementation. You can instead use ``getImpl`` on a symbol if you + ## want to find the intermediate aliases. + runnableExamples: + type + Vec[N: static[int], T] = object + arr: array[N, T] + Vec4[T] = Vec[4, T] + Vec4f = Vec4[float32] + var a: Vec4f + var b: Vec4[float32] + var c: Vec[4, float32] + macro dumpTypeImpl(x: typed): untyped = + newLit(x.getTypeImpl.repr) + let t = """ +object + arr: array[0 .. 3, float32] +""" + doAssert(dumpTypeImpl(a) == t) + doAssert(dumpTypeImpl(b) == t) + doAssert(dumpTypeImpl(c) == t) proc getTypeImpl*(n: typedesc): NimNode {.magic: "NGetType", noSideEffect.} - ## Like getType except it includes generic parameters for the implementation - -proc strVal*(n: NimNode): string {.magic: "NStrVal", noSideEffect.} + ## Version of ``getTypeImpl`` which takes a ``typedesc``. proc `intVal=`*(n: NimNode, val: BiggestInt) {.magic: "NSetIntVal", noSideEffect.} proc `floatVal=`*(n: NimNode, val: BiggestFloat) {.magic: "NSetFloatVal", noSideEffect.} -proc `symbol=`*(n: NimNode, val: NimSym) {.magic: "NSetSymbol", noSideEffect.} -proc `ident=`*(n: NimNode, val: NimIdent) {.magic: "NSetIdent", noSideEffect.} + +proc `symbol=`*(n: NimNode, val: NimSym) {.magic: "NSetSymbol", noSideEffect, deprecated.} + ## **Deprecated since version 0.18.1**; Generate a new ``NimNode`` with ``genSym`` instead. + +proc `ident=`*(n: NimNode, val: NimIdent) {.magic: "NSetIdent", noSideEffect, deprecated.} + ## **Deprecated since version 0.18.1**; Generate a new ``NimNode`` with ``ident(string)`` instead. + #proc `typ=`*(n: NimNode, typ: typedesc) {.magic: "NSetType".} # this is not sound! Unfortunately forbidding 'typ=' is not enough, as you # can easily do: @@ -254,11 +335,6 @@ proc newNimNode*(kind: NimNodeKind, proc copyNimNode*(n: NimNode): NimNode {.magic: "NCopyNimNode", noSideEffect.} proc copyNimTree*(n: NimNode): NimNode {.magic: "NCopyNimTree", noSideEffect.} -proc getImpl*(s: NimSym): NimNode {.magic: "GetImpl", noSideEffect.} = - ## retrieve the implementation of a symbol `s`. `s` can be a routine or a - ## const. - discard - proc error*(msg: string, n: NimNode = nil) {.magic: "NError", benign.} ## writes an error message at compile time @@ -293,11 +369,9 @@ proc newIdentNode*(i: NimIdent): NimNode {.compileTime.} = result = newNimNode(nnkIdent) result.ident = i -proc newIdentNode*(i: string): NimNode {.compileTime.} = - ## creates an identifier node from `i` - result = newNimNode(nnkIdent) - result.ident = toNimIdent i - +proc newIdentNode*(i: string): NimNode {.magic: "StrToIdent", noSideEffect.} + ## creates an identifier node from `i`. It is simply an alias for + ## ``ident(string)``. Use that, it's shorter. type BindSymRule* = enum ## specifies how ``bindSym`` behaves @@ -311,7 +385,7 @@ type {.deprecated: [TBindSymRule: BindSymRule].} -proc bindSym*(ident: string, rule: BindSymRule = brClosed): NimNode {. +proc bindSym*(ident: static[string], rule: BindSymRule = brClosed): NimNode {. magic: "NBindSym", noSideEffect.} ## creates a node that binds `ident` to a symbol node. The bound symbol ## may be an overloaded symbol. @@ -327,8 +401,10 @@ proc genSym*(kind: NimSymKind = nskLet; ident = ""): NimNode {. ## generates a fresh symbol that is guaranteed to be unique. The symbol ## needs to occur in a declaration context. -proc callsite*(): NimNode {.magic: "NCallSite", benign.} +proc callsite*(): NimNode {.magic: "NCallSite", benign, + deprecated: "use varargs[untyped] in the macro prototype instead".} ## returns the AST of the invocation expression that invoked this macro. + ## **Deprecated since version 0.18.1**. proc toStrLit*(n: NimNode): NimNode {.compileTime.} = ## converts the AST `n` to the concrete Nim code and wraps that @@ -463,9 +539,11 @@ proc newCall*(theProc: NimNode, result.add(args) proc newCall*(theProc: NimIdent, - args: varargs[NimNode]): NimNode {.compileTime.} = + args: varargs[NimNode]): NimNode {.compileTime, deprecated.} = ## produces a new call node. `theProc` is the proc that is called with ## the arguments ``args[0..]``. + ## **Deprecated since version 0.18.1**; Use ``newCall(string, ...)``, + ## or ``newCall(NimNode, ...)`` instead. result = newNimNode(nnkCall) result.add(newIdentNode(theProc)) result.add(args) @@ -593,17 +671,30 @@ proc newLit*(s: string): NimNode {.compileTime.} = result = newNimNode(nnkStrLit) result.strVal = s -proc nestList*(theProc: NimIdent, - x: NimNode): NimNode {.compileTime.} = - ## nests the list `x` into a tree of call expressions: - ## ``[a, b, c]`` is transformed into ``theProc(a, theProc(c, d))``. +proc nestList*(op: NimNode; pack: NimNode): NimNode {.compileTime.} = + ## nests the list `pack` into a tree of call expressions: + ## ``[a, b, c]`` is transformed into ``op(a, op(c, d))``. + ## This is also known as fold expression. + if pack.len < 1: + error("`nestList` expects a node with at least 1 child") + result = pack[^1] + for i in countdown(pack.len - 2, 0): + result = newCall(op, pack[i], result) + +proc nestList*(op: NimNode; pack: NimNode; init: NimNode): NimNode {.compileTime.} = + ## nests the list `pack` into a tree of call expressions: + ## ``[a, b, c]`` is transformed into ``op(a, op(c, d))``. + ## This is also known as fold expression. + result = init + for i in countdown(pack.len - 1, 0): + result = newCall(op, pack[i], result) + +proc nestList*(theProc: NimIdent, x: NimNode): NimNode {.compileTime, deprecated.} = + ## **Deprecated since version 0.18.1**; Use one of ``nestList(NimNode, ...)`` instead. var L = x.len result = newCall(theProc, x[L-2], x[L-1]) for i in countdown(L-3, 0): - # XXX the 'copyNimTree' here is necessary due to a bug in the evaluation - # engine that would otherwise create an endless loop here. :-( - # This could easily user code and so should be fixed in evals.nim somehow. - result = newCall(theProc, x[i], copyNimTree(result)) + result = newCall(theProc, x[i], result) proc treeRepr*(n: NimNode): string {.compileTime, benign.} = ## Convert the AST `n` to a human-readable tree-like string. @@ -615,13 +706,11 @@ proc treeRepr*(n: NimNode): string {.compileTime, benign.} = res.add(($n.kind).substr(3)) case n.kind - of nnkEmpty: discard # same as nil node in this representation - of nnkNilLit: res.add(" nil") + of nnkEmpty, nnkNilLit: discard # same as nil node in this representation of nnkCharLit..nnkInt64Lit: res.add(" " & $n.intVal) of nnkFloatLit..nnkFloat64Lit: res.add(" " & $n.floatVal) - of nnkStrLit..nnkTripleStrLit: res.add(" " & $n.strVal) - of nnkIdent: res.add(" ident\"" & $n.ident & '"') - of nnkSym: res.add(" \"" & $n.symbol & '"') + of nnkStrLit..nnkTripleStrLit, nnkIdent, nnkSym: + res.add(" " & $n.strVal.newLit.repr) of nnkNone: assert false else: for j in 0..n.len-1: @@ -640,13 +729,11 @@ proc lispRepr*(n: NimNode): string {.compileTime, benign.} = add(result, "(") case n.kind - of nnkEmpty: discard # same as nil node in this representation - of nnkNilLit: add(result, "nil") + of nnkEmpty, nnkNilLit: discard # same as nil node in this representation of nnkCharLit..nnkInt64Lit: add(result, $n.intVal) of nnkFloatLit..nnkFloat64Lit: add(result, $n.floatVal) - of nnkStrLit..nnkTripleStrLit: add(result, $n.strVal) - of nnkIdent: add(result, "ident\"" & $n.ident & '"') - of nnkSym: add(result, $n.symbol) + of nnkStrLit..nnkTripleStrLit, nnkCommentStmt, nnkident, nnkSym: + add(result, n.strVal.newLit.repr) of nnkNone: assert false else: if n.len > 0: @@ -677,54 +764,27 @@ proc astGenRepr*(n: NimNode): string {.compileTime, benign.} = ## See also `repr`, `treeRepr`, and `lispRepr`. const - NodeKinds = {nnkEmpty, nnkNilLit, nnkIdent, nnkSym, nnkNone} + NodeKinds = {nnkEmpty, nnkIdent, nnkSym, nnkNone, nnkCommentStmt} LitKinds = {nnkCharLit..nnkInt64Lit, nnkFloatLit..nnkFloat64Lit, nnkStrLit..nnkTripleStrLit} - proc escape(s: string, prefix = "\"", suffix = "\""): string {.noSideEffect.} = - ## Functions copied from strutils - proc toHex(x: BiggestInt, len: Positive): string {.noSideEffect, rtl.} = - const - HexChars = "0123456789ABCDEF" - var - t = x - result = newString(len) - for j in countdown(len-1, 0): - result[j] = HexChars[int(t and 0xF)] - t = t shr 4 - # handle negative overflow - if t == 0 and x < 0: t = -1 - - result = newStringOfCap(s.len + s.len shr 2) - result.add(prefix) - for c in items(s): - case c - of '\0'..'\31', '\128'..'\255': - add(result, "\\x") - add(result, toHex(ord(c), 2)) - of '\\': add(result, "\\\\") - of '\'': add(result, "\\'") - of '\"': add(result, "\\\"") - else: add(result, c) - add(result, suffix) - proc traverse(res: var string, level: int, n: NimNode) {.benign.} = for i in 0..level-1: res.add " " if n.kind in NodeKinds: res.add("new" & ($n.kind).substr(3) & "Node(") elif n.kind in LitKinds: res.add("newLit(") + elif n.kind == nnkNilLit: + res.add("newNilLit()") else: res.add($n.kind) case n.kind - of nnkEmpty: discard - of nnkNilLit: res.add("nil") + of nnkEmpty, nnkNilLit: discard of nnkCharLit: res.add("'" & $chr(n.intVal) & "'") of nnkIntLit..nnkInt64Lit: res.add($n.intVal) of nnkFloatLit..nnkFloat64Lit: res.add($n.floatVal) - of nnkStrLit..nnkTripleStrLit: res.add($n.strVal.escape()) - of nnkIdent: res.add(($n.ident).escape()) - of nnkSym: res.add(($n.symbol).escape()) + of nnkStrLit..nnkTripleStrLit, nnkCommentStmt, nnkIdent, nnkSym: + res.add(n.strVal.newLit.repr) of nnkNone: assert false else: res.add(".newTree(") @@ -768,11 +828,10 @@ macro dumpAstGen*(s: untyped): untyped = echo s.astGenRepr ## See `dumpTree`. macro dumpTreeImm*(s: untyped): untyped {.deprecated.} = echo s.treeRepr - ## Deprecated. + ## Deprecated. Use `dumpTree` instead. macro dumpLispImm*(s: untyped): untyped {.deprecated.} = echo s.lispRepr - ## Deprecated. - + ## Deprecated. Use `dumpLisp` instead. proc newEmptyNode*(): NimNode {.compileTime, noSideEffect.} = ## Create a new empty node @@ -1018,28 +1077,21 @@ proc `body=`*(someProc: NimNode, val: NimNode) {.compileTime.} = proc basename*(a: NimNode): NimNode {.compiletime, benign.} - proc `$`*(node: NimNode): string {.compileTime.} = ## Get the string of an identifier node case node.kind - of nnkIdent: - result = $node.ident of nnkPostfix: - result = $node.basename.ident & "*" - of nnkStrLit..nnkTripleStrLit: + result = node.basename.strVal & "*" + of nnkStrLit..nnkTripleStrLit, nnkCommentStmt, nnkSym, nnkIdent: result = node.strVal - of nnkSym: - result = $node.symbol of nnkOpenSymChoice, nnkClosedSymChoice: result = $node[0] of nnkAccQuoted: result = $node[0] - of nnkCommentStmt: - result = node.strVal else: badNodeKind node.kind, "$" -proc ident*(name: string): NimNode {.compileTime,inline.} = newIdentNode(name) +proc ident*(name: string): NimNode {.magic: "StrToIdent", noSideEffect.} ## Create a new ident node from a string iterator items*(n: NimNode): NimNode {.inline.} = @@ -1130,40 +1182,57 @@ proc copy*(node: NimNode): NimNode {.compileTime.} = ## An alias for copyNimTree(). return node.copyNimTree() -proc cmpIgnoreStyle(a, b: cstring): int {.noSideEffect.} = - proc toLower(c: char): char {.inline.} = - if c in {'A'..'Z'}: result = chr(ord(c) + (ord('a') - ord('A'))) - else: result = c - var i = 0 - var j = 0 - # first char is case sensitive - if a[0] != b[0]: return 1 - while true: - while a[i] == '_': inc(i) - while b[j] == '_': inc(j) # BUGFIX: typo - var aa = toLower(a[i]) - var bb = toLower(b[j]) - result = ord(aa) - ord(bb) - if result != 0 or aa == '\0': break - inc(i) - inc(j) - -proc eqIdent*(a, b: string): bool = cmpIgnoreStyle(a, b) == 0 - ## Check if two idents are identical. - -proc eqIdent*(node: NimNode; s: string): bool {.compileTime.} = - ## Check if node is some identifier node (``nnkIdent``, ``nnkSym``, etc.) - ## is the same as ``s``. Note that this is the preferred way to check! Most - ## other ways like ``node.ident`` are much more error-prone, unfortunately. - case node.kind - of nnkIdent: - result = node.ident == toNimIdent s - of nnkSym: - result = eqIdent($node.symbol, s) - of nnkOpenSymChoice, nnkClosedSymChoice: - result = eqIdent($node[0], s) - else: - result = false +when defined(nimVmEqIdent): + proc eqIdent*(a: string; b: string): bool {.magic: "EqIdent", noSideEffect.} + ## Style insensitive comparison. + + proc eqIdent*(a: NimNode; b: string): bool {.magic: "EqIdent", noSideEffect.} + ## Style insensitive comparison. + ## ``a`` can be an identifier or a symbol. + + proc eqIdent*(a: string; b: NimNode): bool {.magic: "EqIdent", noSideEffect.} + ## Style insensitive comparison. + ## ``b`` can be an identifier or a symbol. + + proc eqIdent*(a: NimNode; b: NimNode): bool {.magic: "EqIdent", noSideEffect.} + ## Style insensitive comparison. + ## ``a`` and ``b`` can be an identifier or a symbol. + +else: + # this procedure is optimized for native code, it should not be compiled to nimVM bytecode. + proc cmpIgnoreStyle(a, b: cstring): int {.noSideEffect.} = + proc toLower(c: char): char {.inline.} = + if c in {'A'..'Z'}: result = chr(ord(c) + (ord('a') - ord('A'))) + else: result = c + var i = 0 + var j = 0 + # first char is case sensitive + if a[0] != b[0]: return 1 + while true: + while a[i] == '_': inc(i) + while b[j] == '_': inc(j) # BUGFIX: typo + var aa = toLower(a[i]) + var bb = toLower(b[j]) + result = ord(aa) - ord(bb) + if result != 0 or aa == '\0': break + inc(i) + inc(j) + + + proc eqIdent*(a, b: string): bool = cmpIgnoreStyle(a, b) == 0 + ## Check if two idents are identical. + + proc eqIdent*(node: NimNode; s: string): bool {.compileTime.} = + ## Check if node is some identifier node (``nnkIdent``, ``nnkSym``, etc.) + ## is the same as ``s``. Note that this is the preferred way to check! Most + ## other ways like ``node.ident`` are much more error-prone, unfortunately. + case node.kind + of nnkSym, nnkIdent: + result = eqIdent(node.strVal, s) + of nnkOpenSymChoice, nnkClosedSymChoice: + result = eqIdent($node[0], s) + else: + result = false proc hasArgOfName*(params: NimNode; name: string): bool {.compiletime.}= ## Search nnkFormalParams for an argument. @@ -1213,6 +1282,108 @@ macro expandMacros*(body: typed): untyped = result = getAst(inner(body)) echo result.toStrLit +proc customPragmaNode(n: NimNode): NimNode = + expectKind(n, {nnkSym, nnkDotExpr, nnkBracketExpr, nnkTypeOfExpr, nnkCheckedFieldExpr}) + let + typ = n.getTypeInst() + + if typ.typeKind == ntyTypeDesc: + let impl = typ[1].getImpl() + if impl[0].kind == nnkPragmaExpr: + return impl[0][1] + else: + return impl[0] # handle types which don't have macro at all + + if n.kind == nnkSym: # either an variable or a proc + let impl = n.getImpl() + if impl.kind in RoutineNodes: + return impl.pragma + else: + return typ.getImpl()[0][1] + + if n.kind in {nnkDotExpr, nnkCheckedFieldExpr}: + let name = (if n.kind == nnkCheckedFieldExpr: n[0][1] else: n[1]) + var typDef = getImpl(getTypeInst(if n.kind == nnkCheckedFieldExpr or n[0].kind == nnkHiddenDeref: n[0][0] else: n[0])) + while typDef != nil: + typDef.expectKind(nnkTypeDef) + typDef[2].expectKind({nnkRefTy, nnkPtrTy, nnkObjectTy}) + let isRef = typDef[2].kind in {nnkRefTy, nnkPtrTy} + if isRef and typDef[2][0].kind in {nnkSym, nnkBracketExpr}: # defines ref type for another object(e.g. X = ref X) + typDef = getImpl(typDef[2][0]) + else: # object definition, maybe an object directly defined as a ref type + let + obj = (if isRef: typDef[2][0] else: typDef[2]) + var identDefsStack = newSeq[NimNode](obj[2].len) + for i in 0..<identDefsStack.len: identDefsStack[i] = obj[2][i] + while identDefsStack.len > 0: + var identDefs = identDefsStack.pop() + if identDefs.kind == nnkRecCase: + identDefsStack.add(identDefs[0]) + for i in 1..<identDefs.len: + if identDefs[i][1].kind == nnkIdentDefs: + identDefsStack.add(identDefs[i][1]) + else: # nnkRecList + for j in 0..<identDefs[i][1].len: + identDefsStack.add(identDefs[i][1][j]) + + else: + for i in 0 .. identDefs.len - 3: + if identDefs[i].kind == nnkPragmaExpr and + identDefs[i][0].kind == nnkIdent and $identDefs[i][0] == $name: + return identDefs[i][1] + + if obj[1].kind == nnkOfInherit: # explore the parent object + typDef = getImpl(obj[1][0]) + else: + typDef = nil + +macro hasCustomPragma*(n: typed, cp: typed{nkSym}): untyped = + ## Expands to `true` if expression `n` which is expected to be `nnkDotExpr` + ## (if checking a field), a proc or a type has custom pragma `cp`. + ## + ## See also `getCustomPragmaVal`. + ## + ## .. code-block:: nim + ## template myAttr() {.pragma.} + ## type + ## MyObj = object + ## myField {.myAttr.}: int + ## + ## proc myProc() {.myAttr.} = discard + ## + ## var o: MyObj + ## assert(o.myField.hasCustomPragma(myAttr)) + ## assert(myProc.hasCustomPragma(myAttr)) + let pragmaNode = customPragmaNode(n) + for p in pragmaNode: + if (p.kind == nnkSym and p == cp) or + (p.kind in nnkPragmaCallKinds and p.len > 0 and p[0].kind == nnkSym and p[0] == cp): + return newLit(true) + return newLit(false) + +macro getCustomPragmaVal*(n: typed, cp: typed{nkSym}): untyped = + ## Expands to value of custom pragma `cp` of expression `n` which is expected + ## to be `nnkDotExpr`, a proc or a type. + ## + ## See also `hasCustomPragma` + ## + ## .. code-block:: nim + ## template serializationKey(key: string) {.pragma.} + ## type + ## MyObj {.serializationKey: "mo".} = object + ## myField {.serializationKey: "mf".}: int + ## var o: MyObj + ## assert(o.myField.getCustomPragmaVal(serializationKey) == "mf") + ## assert(o.getCustomPragmaVal(serializationKey) == "mo") + ## assert(MyObj.getCustomPragmaVal(serializationKey) == "mo") + let pragmaNode = customPragmaNode(n) + for p in pragmaNode: + if p.kind in nnkPragmaCallKinds and p.len > 0 and p[0].kind == nnkSym and p[0] == cp: + return p[1] + + error(n.repr & " doesn't have a pragma named " & cp.repr()) # returning an empty node results in most cases in a cryptic error, + + when not defined(booting): template emit*(e: static[string]): untyped {.deprecated.} = ## accepts a single string argument and treats it as nim code @@ -1231,3 +1402,7 @@ macro unpackVarargs*(callee: untyped; args: varargs[untyped]): untyped = result = newCall(callee) for i in 0 ..< args.len: result.add args[i] + +proc getProjectPath*(): string = discard + ## Returns the path to the currently compiling project + diff --git a/lib/core/seqs.nim b/lib/core/seqs.nim index 6be95a3bc..02c192851 100644 --- a/lib/core/seqs.nim +++ b/lib/core/seqs.nim @@ -7,7 +7,7 @@ # distribution, for details about the copyright. # -import allocators +import allocators, typetraits ## Default seq implementation used by Nim's core. type @@ -15,11 +15,11 @@ type len, cap: int data: ptr UncheckedArray[T] +const nimSeqVersion {.core.} = 2 + template frees(s) = dealloc(s.data, s.cap * sizeof(T)) # XXX make code memory safe for overflows in '*' -proc nimSeqLiteral[T](x: openArray[T]): seq[T] {.core.} = - seq[T](len: x.len, cap: x.len, data: x) when defined(nimHasTrace): proc `=trace`[T](s: seq[T]; a: Allocator) = @@ -115,3 +115,25 @@ proc `@`*[T](elems: openArray[T]): seq[T] = result.data[i] = elems[i] proc len*[T](x: seq[T]): int {.inline.} = x.len + +proc `$`*[T](x: seq[T]): string = + result = "@[" + var firstElement = true + for i in 0..<x.len: + let + value = x.data[i] + if firstElement: + firstElement = false + else: + result.add(", ") + + when compiles(value.isNil): + # this branch should not be necessary + if value.isNil: + result.add "nil" + else: + result.addQuoted(value) + else: + result.addQuoted(value) + + result.add("]") diff --git a/lib/core/strs.nim b/lib/core/strs.nim index 1958f4974..ff38aef1d 100644 --- a/lib/core/strs.nim +++ b/lib/core/strs.nim @@ -12,12 +12,11 @@ import allocators type - string {.core.} = object + string {.core, exportc: "NimStringV2".} = object len, cap: int data: ptr UncheckedArray[char] -proc nimStringLiteral(x: cstring; len: int): string {.core.} = - string(len: len, cap: len, data: x) +const nimStrVersion {.core.} = 2 template frees(s) = dealloc(s.data, s.cap + 1) @@ -80,7 +79,7 @@ proc newString*(len: int): string = if len > 0: result.data = alloc0(len+1) -converter toCString(x: string): cstring {.core.} = +converter toCString(x: string): cstring {.core, inline.} = if x.len == 0: cstring"" else: cast[cstring](x.data) proc newStringOfCap*(cap: int): string = diff --git a/lib/deprecated/pure/asyncio.nim b/lib/deprecated/pure/asyncio.nim index 5fd45b215..34cabefb0 100644 --- a/lib/deprecated/pure/asyncio.nim +++ b/lib/deprecated/pure/asyncio.nim @@ -101,8 +101,8 @@ when defined(windows): from winlean import TimeVal, SocketHandle, FD_SET, FD_ZERO, TFdSet, FD_ISSET, select else: - from posix import TimeVal, SocketHandle, FD_SET, FD_ZERO, TFdSet, - FD_ISSET, select + from posix import TimeVal, Time, Suseconds, SocketHandle, FD_SET, FD_ZERO, + TFdSet, FD_ISSET, select type DelegateObj* = object @@ -556,8 +556,12 @@ proc send*(sock: AsyncSocket, data: string) = proc timeValFromMilliseconds(timeout = 500): Timeval = if timeout != -1: var seconds = timeout div 1000 - result.tv_sec = seconds.int32 - result.tv_usec = ((timeout - seconds * 1000) * 1000).int32 + when defined(posix): + result.tv_sec = seconds.Time + result.tv_usec = ((timeout - seconds * 1000) * 1000).Suseconds + else: + result.tv_sec = seconds.int32 + result.tv_usec = ((timeout - seconds * 1000) * 1000).int32 proc createFdSet(fd: var TFdSet, s: seq[Delegate], m: var int) = FD_ZERO(fd) diff --git a/lib/deprecated/pure/ftpclient.nim b/lib/deprecated/pure/ftpclient.nim index ed2f14450..7645258b6 100644 --- a/lib/deprecated/pure/ftpclient.nim +++ b/lib/deprecated/pure/ftpclient.nim @@ -12,7 +12,7 @@ import sockets, strutils, parseutils, times, os, asyncio from asyncnet import nil from nativesockets import nil -from asyncdispatch import PFuture +from asyncdispatch import Future ## **Note**: This module is deprecated since version 0.11.3. ## You should use the async version of this module ## `asyncftpclient <asyncftpclient.html>`_. diff --git a/lib/deprecated/pure/sockets.nim b/lib/deprecated/pure/sockets.nim index f068c7d56..05aebef76 100644 --- a/lib/deprecated/pure/sockets.nim +++ b/lib/deprecated/pure/sockets.nim @@ -32,7 +32,7 @@ include "system/inclrtl" -{.deadCodeElim: on.} +{.deadCodeElim: on.} # dce option deprecated when hostOS == "solaris": {.passl: "-lsocket -lnsl".} @@ -953,8 +953,12 @@ when defined(ssl): proc timeValFromMilliseconds(timeout = 500): Timeval = if timeout != -1: var seconds = timeout div 1000 - result.tv_sec = seconds.int32 - result.tv_usec = ((timeout - seconds * 1000) * 1000).int32 + when defined(posix): + result.tv_sec = seconds.Time + result.tv_usec = ((timeout - seconds * 1000) * 1000).Suseconds + else: + result.tv_sec = seconds.int32 + result.tv_usec = ((timeout - seconds * 1000) * 1000).int32 proc createFdSet(fd: var TFdSet, s: seq[Socket], m: var int) = FD_ZERO(fd) diff --git a/lib/impure/db_mysql.nim b/lib/impure/db_mysql.nim index 1b79b3543..ca0e29d11 100644 --- a/lib/impure/db_mysql.nim +++ b/lib/impure/db_mysql.nim @@ -89,7 +89,7 @@ import db_common export db_common type - DbConn* = PMySQL ## encapsulates a database connection + DbConn* = distinct PMySQL ## encapsulates a database connection Row* = seq[string] ## a row of a dataset. NULL database values will be ## converted to nil. InstantRow* = object ## a handle that can be used to get a row's @@ -102,7 +102,7 @@ proc dbError*(db: DbConn) {.noreturn.} = ## raises a DbError exception. var e: ref DbError new(e) - e.msg = $mysql.error(db) + e.msg = $mysql.error(PMySQL db) raise e when false: @@ -128,7 +128,7 @@ proc dbFormat(formatstr: SqlQuery, args: varargs[string]): string = var a = 0 for c in items(string(formatstr)): if c == '?': - if args[a] == nil: + if args[a].isNil: add(result, "NULL") else: add(result, dbQuote(args[a])) @@ -140,17 +140,17 @@ proc tryExec*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): bool {. tags: [ReadDbEffect, WriteDbEffect].} = ## tries to execute the query and returns true if successful, false otherwise. var q = dbFormat(query, args) - return mysql.realQuery(db, q, q.len) == 0'i32 + return mysql.realQuery(PMySQL db, q, q.len) == 0'i32 proc rawExec(db: DbConn, query: SqlQuery, args: varargs[string, `$`]) = var q = dbFormat(query, args) - if mysql.realQuery(db, q, q.len) != 0'i32: dbError(db) + if mysql.realQuery(PMySQL db, q, q.len) != 0'i32: dbError(db) proc exec*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]) {. tags: [ReadDbEffect, WriteDbEffect].} = ## executes the query and raises EDB if not successful. var q = dbFormat(query, args) - if mysql.realQuery(db, q, q.len) != 0'i32: dbError(db) + if mysql.realQuery(PMySQL db, q, q.len) != 0'i32: dbError(db) proc newRow(L: int): Row = newSeq(result, L) @@ -171,7 +171,7 @@ iterator fastRows*(db: DbConn, query: SqlQuery, ## Breaking the fastRows() iterator during a loop will cause the next ## database query to raise an [EDb] exception ``Commands out of sync``. rawExec(db, query, args) - var sqlres = mysql.useResult(db) + var sqlres = mysql.useResult(PMySQL db) if sqlres != nil: var L = int(mysql.numFields(sqlres)) @@ -210,7 +210,7 @@ iterator instantRows*(db: DbConn, query: SqlQuery, ## Same as fastRows but returns a handle that can be used to get column text ## on demand using []. Returned handle is valid only within the iterator body. rawExec(db, query, args) - var sqlres = mysql.useResult(db) + var sqlres = mysql.useResult(PMySQL db) if sqlres != nil: let L = int(mysql.numFields(sqlres)) var row: cstringArray @@ -290,7 +290,7 @@ iterator instantRows*(db: DbConn; columns: var DbColumns; query: SqlQuery; ## Same as fastRows but returns a handle that can be used to get column text ## on demand using []. Returned handle is valid only within the iterator body. rawExec(db, query, args) - var sqlres = mysql.useResult(db) + var sqlres = mysql.useResult(PMySQL db) if sqlres != nil: let L = int(mysql.numFields(sqlres)) setColumnInfo(columns, sqlres, L) @@ -315,7 +315,7 @@ proc getRow*(db: DbConn, query: SqlQuery, ## Retrieves a single row. If the query doesn't return any rows, this proc ## will return a Row with empty strings for each column. rawExec(db, query, args) - var sqlres = mysql.useResult(db) + var sqlres = mysql.useResult(PMySQL db) if sqlres != nil: var L = int(mysql.numFields(sqlres)) result = newRow(L) @@ -334,7 +334,7 @@ proc getAllRows*(db: DbConn, query: SqlQuery, ## executes the query and returns the whole result dataset. result = @[] rawExec(db, query, args) - var sqlres = mysql.useResult(db) + var sqlres = mysql.useResult(PMySQL db) if sqlres != nil: var L = int(mysql.numFields(sqlres)) var row: cstringArray @@ -369,10 +369,10 @@ proc tryInsertId*(db: DbConn, query: SqlQuery, ## executes the query (typically "INSERT") and returns the ## generated ID for the row or -1 in case of an error. var q = dbFormat(query, args) - if mysql.realQuery(db, q, q.len) != 0'i32: + if mysql.realQuery(PMySQL db, q, q.len) != 0'i32: result = -1'i64 else: - result = mysql.insertId(db) + result = mysql.insertId(PMySQL db) proc insertId*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): int64 {.tags: [WriteDbEffect].} = @@ -387,32 +387,33 @@ proc execAffectedRows*(db: DbConn, query: SqlQuery, ## runs the query (typically "UPDATE") and returns the ## number of affected rows rawExec(db, query, args) - result = mysql.affectedRows(db) + result = mysql.affectedRows(PMySQL db) proc close*(db: DbConn) {.tags: [DbEffect].} = ## closes the database connection. - if db != nil: mysql.close(db) + if PMySQL(db) != nil: mysql.close(PMySQL db) proc open*(connection, user, password, database: string): DbConn {. tags: [DbEffect].} = ## opens a database connection. Raises `EDb` if the connection could not ## be established. - result = mysql.init(nil) - if result == nil: dbError("could not open database connection") + var res = mysql.init(nil) + if res == nil: dbError("could not open database connection") let colonPos = connection.find(':') host = if colonPos < 0: connection else: substr(connection, 0, colonPos-1) port: int32 = if colonPos < 0: 0'i32 else: substr(connection, colonPos+1).parseInt.int32 - if mysql.realConnect(result, host, user, password, database, + if mysql.realConnect(res, host, user, password, database, port, nil, 0) == nil: - var errmsg = $mysql.error(result) - db_mysql.close(result) + var errmsg = $mysql.error(res) + mysql.close(res) dbError(errmsg) + result = DbConn(res) proc setEncoding*(connection: DbConn, encoding: string): bool {. tags: [DbEffect].} = ## sets the encoding of a database connection, returns true for ## success, false for failure. - result = mysql.set_character_set(connection, encoding) == 0 + result = mysql.set_character_set(PMySQL connection, encoding) == 0 diff --git a/lib/impure/db_sqlite.nim b/lib/impure/db_sqlite.nim index 21049571f..fd25b2b94 100644 --- a/lib/impure/db_sqlite.nim +++ b/lib/impure/db_sqlite.nim @@ -31,7 +31,7 @@ ## ## .. code-block:: Nim ## import db_sqlite -## let db = open("localhost", "user", "password", "dbname") +## let db = open("mytest.db", nil, nil, nil) # user, password, database name can be nil ## db.close() ## ## Creating a table @@ -57,7 +57,7 @@ ## ## import db_sqlite, math ## -## let theDb = open("mytest.db", nil, nil, nil) +## let theDb = open("mytest.db", "", "", "") ## ## theDb.exec(sql"Drop table if exists myTestTbl") ## theDb.exec(sql("""create table myTestTbl ( @@ -81,7 +81,7 @@ ## ## theDb.close() -{.deadCodeElim:on.} +{.deadCodeElim: on.} # dce option deprecated import strutils, sqlite3 @@ -126,6 +126,7 @@ proc tryExec*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): bool {. tags: [ReadDbEffect, WriteDbEffect].} = ## tries to execute the query and returns true if successful, false otherwise. + assert(not db.isNil, "Database not connected.") var q = dbFormat(query, args) var stmt: sqlite3.Pstmt if prepare_v2(db, q, q.len.cint, stmt, nil) == SQLITE_OK: @@ -144,6 +145,7 @@ proc newRow(L: int): Row = proc setupQuery(db: DbConn, query: SqlQuery, args: varargs[string]): Pstmt = + assert(not db.isNil, "Database not connected.") var q = dbFormat(query, args) if prepare_v2(db, q, q.len.cint, result, nil) != SQLITE_OK: dbError(db) @@ -267,6 +269,7 @@ proc tryInsertID*(db: DbConn, query: SqlQuery, {.tags: [WriteDbEffect], raises: [].} = ## executes the query (typically "INSERT") and returns the ## generated ID for the row or -1 in case of an error. + assert(not db.isNil, "Database not connected.") var q = dbFormat(query, args) var stmt: sqlite3.Pstmt result = -1 diff --git a/lib/impure/nre.nim b/lib/impure/nre.nim index 3d4afc0ae..6058128dd 100644 --- a/lib/impure/nre.nim +++ b/lib/impure/nre.nim @@ -10,7 +10,7 @@ from pcre import nil import nre.private.util import tables -from strutils import toLower, `%` +from strutils import `%` from math import ceil import options from unicode import runeLenAt @@ -326,15 +326,15 @@ proc `$`*(pattern: RegexMatch): string = proc `==`*(a, b: Regex): bool = if not a.isNil and not b.isNil: - return a.pattern == b.pattern and - a.pcreObj == b.pcreObj and + return a.pattern == b.pattern and + a.pcreObj == b.pcreObj and a.pcreExtra == b.pcreExtra else: return system.`==`(a, b) proc `==`*(a, b: RegexMatch): bool = return a.pattern == b.pattern and - a.str == b.str + a.str == b.str # }}} # Creation & Destruction {{{ @@ -645,7 +645,6 @@ template replaceImpl(str: string, pattern: Regex, let bounds = match.matchBounds result.add(str.substr(lastIdx, bounds.a - 1)) let nextVal = replacement - assert(nextVal != nil) result.add(nextVal) lastIdx = bounds.b + 1 diff --git a/lib/impure/rdstdin.nim b/lib/impure/rdstdin.nim index b06e8de6c..54bab82f0 100644 --- a/lib/impure/rdstdin.nim +++ b/lib/impure/rdstdin.nim @@ -13,7 +13,7 @@ ## is used. This suffices because Windows' console already provides the ## wanted functionality. -{.deadCodeElim: on.} +{.deadCodeElim: on.} # dce option deprecated when defined(Windows): proc readLineFromStdin*(prompt: string): TaintedString {. @@ -73,32 +73,6 @@ when defined(Windows): discard readConsoleInputW(hStdin, irInputRecord, 1, dwEventsRead) return result - from unicode import toUTF8, Rune, runeLenAt - - proc readPasswordFromStdin*(prompt: string, password: var TaintedString): - bool {.tags: [ReadIOEffect, WriteIOEffect].} = - ## Reads a `password` from stdin without printing it. `password` must not - ## be ``nil``! Returns ``false`` if the end of the file has been reached, - ## ``true`` otherwise. - password.setLen(0) - stdout.write(prompt) - while true: - let c = getch() - case c.char - of '\r', chr(0xA): - break - of '\b': - # ensure we delete the whole UTF-8 character: - var i = 0 - var x = 1 - while i < password.len: - x = runeLenAt(password, i) - inc i, x - password.setLen(max(password.len - x, 0)) - else: - password.add(toUTF8(c.Rune)) - stdout.write "\n" - else: import linenoise, termios @@ -124,21 +98,3 @@ else: linenoise.free(buffer) result = true - proc readPasswordFromStdin*(prompt: string, password: var TaintedString): - bool {.tags: [ReadIOEffect, WriteIOEffect].} = - password.setLen(0) - let fd = stdin.getFileHandle() - var cur, old: Termios - discard fd.tcgetattr(cur.addr) - old = cur - cur.c_lflag = cur.c_lflag and not Cflag(ECHO) - discard fd.tcsetattr(TCSADRAIN, cur.addr) - stdout.write prompt - result = stdin.readLine(password) - stdout.write "\n" - discard fd.tcsetattr(TCSADRAIN, old.addr) - -proc readPasswordFromStdin*(prompt: string): TaintedString = - ## Reads a password from stdin without printing it. - result = TaintedString("") - discard readPasswordFromStdin(prompt, result) diff --git a/lib/impure/re.nim b/lib/impure/re.nim index c7f8f336b..201c490f3 100644 --- a/lib/impure/re.nim +++ b/lib/impure/re.nim @@ -7,18 +7,14 @@ # distribution, for details about the copyright. # -## Regular expression support for Nim. This module still has some -## obscure bugs and limitations, -## consider using the ``nre`` or ``pegs`` modules instead. -## We had to de-deprecate this module since too much code relies on it -## and many people prefer its API over ``nre``'s. +## Regular expression support for Nim. ## ## This module is implemented by providing a wrapper around the -## `PRCE (Perl-Compatible Regular Expressions) <http://www.pcre.org>`_ -## C library. This means that your application will depend on the PRCE +## `PCRE (Perl-Compatible Regular Expressions) <http://www.pcre.org>`_ +## C library. This means that your application will depend on the PCRE ## library's licence when using this module, which should not be a problem ## though. -## PRCE's licence follows: +## PCRE's licence follows: ## ## .. include:: ../../doc/regexprs.txt ## @@ -502,7 +498,7 @@ proc transformFile*(infile, outfile: string, var x = readFile(infile).string writeFile(outfile, x.multiReplace(subs)) -iterator split*(s: string, sep: Regex): string = +iterator split*(s: string, sep: Regex; maxsplit = -1): string = ## Splits the string ``s`` into substrings. ## ## Substrings are separated by the regular expression ``sep`` @@ -524,22 +520,28 @@ iterator split*(s: string, sep: Regex): string = ## "example" ## "" ## - var - first = -1 - last = -1 - while last < len(s): - var x = matchLen(s, sep, last) - if x > 0: inc(last, x) - first = last - if x == 0: inc(last) + var last = 0 + var splits = maxsplit + var x: int + while last <= len(s): + var first = last + var sepLen = 1 while last < len(s): x = matchLen(s, sep, last) - if x >= 0: break + if x >= 0: + sepLen = x + break inc(last) - if first <= last: - yield substr(s, first, last-1) - -proc split*(s: string, sep: Regex): seq[string] {.inline.} = + if x == 0: + if last >= len(s): break + inc last + if splits == 0: last = len(s) + yield substr(s, first, last-1) + if splits == 0: break + dec(splits) + inc(last, sepLen) + +proc split*(s: string, sep: Regex, maxsplit = -1): seq[string] {.inline.} = ## Splits the string ``s`` into a seq of substrings. ## ## The portion matched by ``sep`` is not returned. @@ -636,6 +638,14 @@ when isMainModule: accum.add(word) doAssert(accum == @["AAA", "", "BBB"]) + doAssert(split("abc", re"") == @["a", "b", "c"]) + doAssert(split("", re"") == @[]) + + doAssert(split("a;b;c", re";") == @["a", "b", "c"]) + doAssert(split(";a;b;c", re";") == @["", "a", "b", "c"]) + doAssert(split(";a;b;c;", re";") == @["", "a", "b", "c", ""]) + doAssert(split("a;b;c;", re";") == @["a", "b", "c", ""]) + for x in findAll("abcdef", re"^{.}", 3): doAssert x == "d" accum = @[] diff --git a/lib/js/asyncjs.nim b/lib/js/asyncjs.nim index ec410ee39..894102ca0 100644 --- a/lib/js/asyncjs.nim +++ b/lib/js/asyncjs.nim @@ -71,14 +71,17 @@ type PromiseJs* {.importcpp: "Promise".} = ref object ## A JavaScript Promise + proc replaceReturn(node: var NimNode) = var z = 0 for s in node: var son = node[z] + let jsResolve = ident("jsResolve") if son.kind == nnkReturnStmt: - node[z] = nnkReturnStmt.newTree(nnkCall.newTree(ident("jsResolve"), son[0])) + let value = if son[0].kind != nnkEmpty: nnkCall.newTree(jsResolve, son[0]) else: jsResolve + node[z] = nnkReturnStmt.newTree(value) elif son.kind == nnkAsgn and son[0].kind == nnkIdent and $son[0] == "result": - node[z] = nnkAsgn.newTree(son[0], nnkCall.newTree(ident("jsResolve"), son[1])) + node[z] = nnkAsgn.newTree(son[0], nnkCall.newTree(jsResolve, son[1])) else: replaceReturn(son) inc z @@ -92,8 +95,7 @@ proc generateJsasync(arg: NimNode): NimNode = assert arg.kind == nnkProcDef result = arg var isVoid = false - var jsResolveNode = ident("jsResolve") - + let jsResolve = ident("jsResolve") if arg.params[0].kind == nnkEmpty: result.params[0] = nnkBracketExpr.newTree(ident("Future"), ident("void")) isVoid = true @@ -112,7 +114,7 @@ proc generateJsasync(arg: NimNode): NimNode = var resolve: NimNode if isVoid: resolve = quote: - var `jsResolveNode` {.importcpp: "undefined".}: Future[void] + var `jsResolve` {.importcpp: "undefined".}: Future[void] else: resolve = quote: proc jsResolve[T](a: T): Future[T] {.importcpp: "#".} @@ -124,12 +126,13 @@ proc generateJsasync(arg: NimNode): NimNode = if len(code) > 0 and isVoid: var voidFix = quote: - return `jsResolveNode` + return `jsResolve` result.body.add(voidFix) - result.pragma = quote: + let asyncPragma = quote: {.codegenDecl: "async function $2($3)".} + result.addPragma(asyncPragma[0]) macro async*(arg: untyped): untyped = ## Macro which converts normal procedures into @@ -139,3 +142,7 @@ macro async*(arg: untyped): untyped = proc newPromise*[T](handler: proc(resolve: proc(response: T))): Future[T] {.importcpp: "(new Promise(#))".} ## A helper for wrapping callback-based functions ## into promises and async procedures + +proc newPromise*(handler: proc(resolve: proc())): Future[void] {.importcpp: "(new Promise(#))".} + ## A helper for wrapping callback-based functions + ## into promises and async procedures diff --git a/lib/js/dom.nim b/lib/js/dom.nim index aa7f5d839..fd81fdf3f 100644 --- a/lib/js/dom.nim +++ b/lib/js/dom.nim @@ -45,6 +45,7 @@ type location*: Location closed*: bool defaultStatus*: cstring + devicePixelRatio*: float innerHeight*, innerWidth*: int locationbar*: ref TLocationBar menubar*: ref TMenuBar @@ -53,11 +54,15 @@ type pageXOffset*, pageYOffset*: int personalbar*: ref TPersonalBar scrollbars*: ref TScrollBars + scrollX*: float + scrollY*: float statusbar*: ref TStatusBar status*: cstring toolbar*: ref TToolBar frames*: seq[TFrame] screen*: Screen + performance*: Performance + onpopstate*: proc (event: Event) Frame* = ref FrameObj FrameObj {.importc.} = object of WindowObj @@ -171,6 +176,12 @@ type text*: cstring value*: cstring + TextAreaElement* = ref object of ElementObj + value*: cstring + selectionStart*, selectionEnd*: int + selectionDirection*: cstring + rows*, cols*: int + FormElement* = ref FormObj FormObj {.importc.} = object of ElementObj action*: cstring @@ -253,6 +264,8 @@ type minHeight*: cstring minWidth*: cstring overflow*: cstring + overflowX*: cstring + overflowY*: cstring padding*: cstring paddingBottom*: cstring paddingLeft*: cstring @@ -400,12 +413,47 @@ type once*: bool passive*: bool + BoundingRect* {.importc.} = ref object + top*, bottom*, left*, right*, x*, y*, width*, height*: float + + PerformanceMemory* {.importc.} = ref object + jsHeapSizeLimit*: float + totalJSHeapSize*: float + usedJSHeapSize*: float + + PerformanceTiming* {.importc.} = ref object + connectStart*: float + domComplete*: float + domContentLoadedEventEnd*: float + domContentLoadedEventStart*: float + domInteractive*: float + domLoading*: float + domainLookupEnd*: float + domainLookupStart*: float + fetchStart*: float + loadEventEnd*: float + loadEventStart*: float + navigationStart*: float + redirectEnd*: float + redirectStart*: float + requestStart*: float + responseEnd*: float + responseStart*: float + secureConnectionStart*: float + unloadEventEnd*: float + unloadEventStart*: float + + Performance* {.importc.} = ref object + memory*: PerformanceMemory + timing*: PerformanceTiming + {.push importcpp.} # EventTarget "methods" proc addEventListener*(et: EventTarget, ev: cstring, cb: proc(ev: Event), useCapture: bool = false) proc addEventListener*(et: EventTarget, ev: cstring, cb: proc(ev: Event), options: AddEventListenerOptions) - +proc removeEventListener*(et: EventTarget, ev: cstring, cb: proc(ev: Event), useCapture: bool = false) +proc dispatchEvent*(et: EventTarget, ev: Event) # Window "methods" proc alert*(w: Window, msg: cstring) @@ -451,6 +499,7 @@ proc cloneNode*(n: Node, copyContent: bool): Node proc deleteData*(n: Node, start, len: int) proc getAttribute*(n: Node, attr: cstring): cstring proc getAttributeNode*(n: Node, attr: cstring): Node +proc getBoundingClientRect*(e: Node): BoundingRect proc hasChildNodes*(n: Node): bool proc insertBefore*(n, newNode, before: Node) proc insertData*(n: Node, position: int, data: cstring) @@ -459,7 +508,7 @@ proc removeAttributeNode*(n, attr: Node) proc removeChild*(n, child: Node) proc replaceChild*(n, newNode, oldNode: Node) proc replaceData*(n: Node, start, len: int, text: cstring) -proc scrollIntoView*(n: Node) +proc scrollIntoView*(n: Node, alignToTop: bool=true) proc setAttribute*(n: Node, name, value: cstring) proc setAttributeNode*(n: Node, attr: Node) @@ -507,6 +556,7 @@ proc replace*(loc: Location, s: cstring) proc back*(h: History) proc forward*(h: History) proc go*(h: History, pagesToJump: int) +proc pushState*[T](h: History, stateObject: T, title, url: cstring) # Navigator "methods" proc javaEnabled*(h: Navigator): bool @@ -529,6 +579,9 @@ proc preventDefault*(ev: Event) proc identifiedTouch*(list: TouchList): Touch proc item*(list: TouchList, i: int): Touch +# Performance "methods" +proc now*(p: Performance): float + {.pop.} var @@ -551,6 +604,7 @@ proc parseFloat*(s: cstring): BiggestFloat {.importc, nodecl.} proc parseInt*(s: cstring): int {.importc, nodecl.} proc parseInt*(s: cstring, radix: int):int {.importc, nodecl.} +proc newEvent*(name: cstring): Event {.importcpp: "new Event(@)", constructor.} type TEventHandlers* {.deprecated.} = EventTargetObj diff --git a/lib/js/jscore.nim b/lib/js/jscore.nim new file mode 100644 index 000000000..91f3ff8bb --- /dev/null +++ b/lib/js/jscore.nim @@ -0,0 +1,91 @@ +## This module wraps core JavaScript functions. +## +## Unless your application has very +## specific requirements and solely targets JavaScript, you should be using +## the relevant functions in the ``math``, ``json``, and ``times`` stdlib +## modules instead. + +when not defined(js) and not defined(Nimdoc): + {.error: "This module only works on the JavaScript platform".} + +type + MathLib* = ref object + JsonLib* = ref object + DateLib* = ref object + DateTime* = ref object + +var + Math* {.importc, nodecl.}: MathLib + Date* {.importc, nodecl.}: DateLib + JSON* {.importc, nodecl.}: JsonLib + +{.push importcpp.} + +# Math library +proc abs*(m: MathLib, a: SomeNumber): SomeNumber +proc acos*(m: MathLib, a: SomeNumber): float +proc acosh*(m: MathLib, a: SomeNumber): float +proc asin*(m: MathLib, a: SomeNumber): float +proc asinh*(m: MathLib, a: SomeNumber): float +proc atan*(m: MathLib, a: SomeNumber): float +proc atan2*(m: MathLib, a: SomeNumber): float +proc atanh*(m: MathLib, a: SomeNumber): float +proc cbrt*(m: MathLib, f: SomeFloat): SomeFloat +proc ceil*(m: MathLib, f: SomeFloat): SomeFloat +proc clz32*(m: MathLib, f: SomeInteger): int +proc cos*(m: MathLib, a: SomeNumber): float +proc cosh*(m: MathLib, a: SomeNumber): float +proc exp*(m: MathLib, a: SomeNumber): float +proc expm1*(m: MathLib, a: SomeNumber): float +proc floor*(m: MathLib, f: SomeFloat): int +proc fround*(m: MathLib, f: SomeFloat): float32 +proc hypot*(m: MathLib, args: varargs[distinct SomeNumber]): float +proc imul*(m: MathLib, a, b: int32): int32 +proc log*(m: MathLib, a: SomeNumber): float +proc log10*(m: MathLib, a: SomeNumber): float +proc log1p*(m: MathLib, a: SomeNumber): float +proc log2*(m: MathLib, a: SomeNumber): float +proc max*(m: MathLib, a, b: SomeNumber): SomeNumber +proc min*[T: SomeNumber | JsRoot](m: MathLib, a, b: T): T +proc pow*(m: MathLib, a, b: distinct SomeNumber): float +proc random*(m: MathLib): float +proc round*(m: MathLib, f: SomeFloat): int +proc sign*(m: MathLib, f: SomeNumber): int +proc sin*(m: MathLib, a: SomeNumber): float +proc sinh*(m: MathLib, a: SomeNumber): float +proc sqrt*(m: MathLib, f: SomeFloat): SomeFloat +proc tan*(m: MathLib, a: SomeNumber): float +proc tanh*(m: MathLib, a: SomeNumber): float +proc trunc*(m: MathLib, f: SomeFloat): int + +# Date library +proc now*(d: DateLib): int +proc UTC*(d: DateLib): int +proc parse*(d: DateLib, s: cstring): int + +proc newDate*(): DateTime {. + importcpp: "new Date()".} + +proc newDate*(date: int|string): DateTime {. + importcpp: "new Date(#)".} + +proc newDate*(year, month, day, hours, minutes, + seconds, milliseconds: int): DateTime {. + importcpp: "new Date(#,#,#,#,#,#,#)".} + +proc getDay*(d: DateTime): int +proc getFullYear*(d: DateTime): int +proc getHours*(d: DateTime): int +proc getMilliseconds*(d: DateTime): int +proc getMinutes*(d: DateTime): int +proc getMonth*(d: DateTime): int +proc getSeconds*(d: DateTime): int +proc getYear*(d: DateTime): int +proc getTime*(d: DateTime): int +proc toString*(d: DateTime): cstring + +#JSON library +proc stringify*(l: JsonLib, s: JsRoot): cstring +proc parse*(l: JsonLib, s: cstring): JsRoot + +{.pop.} diff --git a/lib/js/jsffi.nim b/lib/js/jsffi.nim index f34efe9a2..6e48db6c7 100644 --- a/lib/js/jsffi.nim +++ b/lib/js/jsffi.nim @@ -70,22 +70,31 @@ template mangleJsName(name: cstring): cstring = "mangledName" & $nameCounter type - JsRoot* = ref object of RootObj - ## Root type of both JsObject and JsAssoc JsObject* = ref object of JsRoot ## Dynamically typed wrapper around a JavaScript object. JsAssoc*[K, V] = ref object of JsRoot ## Statically typed wrapper around a JavaScript object. + NotString = concept c c isnot string js* = JsObject -var jsarguments* {.importc: "arguments", nodecl}: JsObject - ## JavaScript's arguments pseudo-variable +var + jsArguments* {.importc: "arguments", nodecl}: JsObject + ## JavaScript's arguments pseudo-variable + jsNull* {.importc: "null", nodecl.}: JsObject + ## JavaScript's null literal + jsUndefined* {.importc: "undefined", nodecl.}: JsObject + ## JavaScript's undefined literal + jsDirname* {.importc: "__dirname", nodecl.}: cstring + ## JavaScript's __dirname pseudo-variable + jsFilename* {.importc: "__filename", nodecl.}: cstring + ## JavaScript's __filename pseudo-variable # New proc newJsObject*: JsObject {. importcpp: "{@}" .} ## Creates a new empty JsObject + proc newJsAssoc*[K, V]: JsAssoc[K, V] {. importcpp: "{@}" .} ## Creates a new empty JsAssoc with key type `K` and value type `V`. @@ -97,13 +106,16 @@ proc hasOwnProperty*(x: JsObject, prop: cstring): bool proc jsTypeOf*(x: JsObject): cstring {. importcpp: "typeof(#)" .} ## Returns the name of the JsObject's JavaScript type as a cstring. -proc jsnew*(x: auto): JsObject {.importcpp: "(new #)".} +proc jsNew*(x: auto): JsObject {.importcpp: "(new #)".} ## Turns a regular function call into an invocation of the ## JavaScript's `new` operator -proc jsdelete*(x: auto): JsObject {.importcpp: "(delete #)".} +proc jsDelete*(x: auto): JsObject {.importcpp: "(delete #)".} ## JavaScript's `delete` operator +proc require*(module: cstring): JsObject {.importc.} + ## JavaScript's `require` function + # Conversion to and from JsObject proc to*(x: JsObject, T: typedesc): T {. importcpp: "(#)" .} ## Converts a JsObject `x` to type `T`. diff --git a/lib/nimbase.h b/lib/nimbase.h index 31075bbd2..20ac9979b 100644 --- a/lib/nimbase.h +++ b/lib/nimbase.h @@ -70,7 +70,7 @@ __clang__ #if defined(_MSC_VER) # pragma warning(disable: 4005 4100 4101 4189 4191 4200 4244 4293 4296 4309) # pragma warning(disable: 4310 4365 4456 4477 4514 4574 4611 4668 4702 4706) -# pragma warning(disable: 4710 4711 4774 4800 4820 4996 4090 4297) +# pragma warning(disable: 4710 4711 4774 4800 4809 4820 4996 4090 4297) #endif /* ------------------------------------------------------------------------- */ @@ -264,6 +264,11 @@ __clang__ # define HAVE_STDINT_H #endif +/* wrap all Nim typedefs into namespace Nim */ +#ifdef USE_NIM_NAMESPACE +namespace Nim { +#endif + /* bool types (C++ has it): */ #ifdef __cplusplus # ifndef NIM_TRUE @@ -274,13 +279,6 @@ __clang__ # endif # define NIM_BOOL bool # define NIM_NIL 0 -struct NimException -{ - NimException(struct Exception* exp, const char* msg): exp(exp), msg(msg) {} - - struct Exception* exp; - const char* msg; -}; #else # ifdef bool # define NIM_BOOL bool @@ -420,8 +418,8 @@ typedef struct TStringDesc* string; # endif #endif -typedef struct TFrame TFrame; -struct TFrame { +typedef struct TFrame_ TFrame; +struct TFrame_ { TFrame* prev; NCSTRING procname; NI line; @@ -483,6 +481,10 @@ static inline void GCGuard (void *ptr) { asm volatile ("" :: "X" (ptr)); } "error: 'Nim_and_C_compiler_disagree_on_target_architecture' declared as an array with a negative size" */ typedef int Nim_and_C_compiler_disagree_on_target_architecture[sizeof(NI) == sizeof(void*) && NIM_INTBITS == sizeof(NI)*8 ? 1 : -1]; +#ifdef USE_NIM_NAMESPACE +} +#endif + #ifdef __cplusplus # define NIM_EXTERNC extern "C" #else diff --git a/lib/packages/docutils/docutils.babel b/lib/packages/docutils/docutils.babel deleted file mode 100644 index 1ed86ca05..000000000 --- a/lib/packages/docutils/docutils.babel +++ /dev/null @@ -1,6 +0,0 @@ -[Package] -name = "docutils" -version = "0.9.0" -author = "Andreas Rumpf" -description = "Nimrod's reStructuredText processor." -license = "MIT" diff --git a/lib/packages/docutils/docutils.nimble b/lib/packages/docutils/docutils.nimble new file mode 100644 index 000000000..e32cc6bdb --- /dev/null +++ b/lib/packages/docutils/docutils.nimble @@ -0,0 +1,5 @@ +name = "docutils" +version = "0.10.0" +author = "Andreas Rumpf" +description = "Nim's reStructuredText processor." +license = "MIT" diff --git a/lib/packages/docutils/highlite.nim b/lib/packages/docutils/highlite.nim index 2a58854a6..4f1264c9e 100644 --- a/lib/packages/docutils/highlite.nim +++ b/lib/packages/docutils/highlite.nim @@ -13,6 +13,7 @@ import strutils +from algorithm import binarySearch type TokenClass* = enum @@ -365,32 +366,10 @@ proc generalStrLit(g: var GeneralTokenizer, position: int): int = result = pos proc isKeyword(x: openArray[string], y: string): int = - var a = 0 - var b = len(x) - 1 - while a <= b: - var mid = (a + b) div 2 - var c = cmp(x[mid], y) - if c < 0: - a = mid + 1 - elif c > 0: - b = mid - 1 - else: - return mid - result = - 1 + binarySearch(x, y) proc isKeywordIgnoreCase(x: openArray[string], y: string): int = - var a = 0 - var b = len(x) - 1 - while a <= b: - var mid = (a + b) div 2 - var c = cmpIgnoreCase(x[mid], y) - if c < 0: - a = mid + 1 - elif c > 0: - b = mid - 1 - else: - return mid - result = - 1 + binarySearch(x, y, cmpIgnoreCase) type TokenizerFlag = enum diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index 223fc836a..adac16777 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -43,8 +43,8 @@ type mwUnsupportedField MsgHandler* = proc (filename: string, line, col: int, msgKind: MsgKind, - arg: string) {.nimcall.} ## what to do in case of an error - FindFileHandler* = proc (filename: string): string {.nimcall.} + arg: string) {.closure.} ## what to do in case of an error + FindFileHandler* = proc (filename: string): string {.closure.} const messages: array[MsgKind, string] = [ @@ -853,7 +853,6 @@ type DirKind = enum # must be ordered alphabetically! dkNone, dkAuthor, dkAuthors, dkCode, dkCodeBlock, dkContainer, dkContents, dkFigure, dkImage, dkInclude, dkIndex, dkRaw, dkTitle -{.deprecated: [TDirKind: DirKind].} const DirIds: array[0..12, string] = ["", "author", "authors", "code", @@ -1114,7 +1113,6 @@ proc parseHeadline(p: var RstParser): PRstNode = type IntSeq = seq[int] -{.deprecated: [TIntSeq: IntSeq].} proc tokEnd(p: RstParser): int = result = p.tok[p.idx].col + len(p.tok[p.idx].symbol) - 1 @@ -1408,8 +1406,6 @@ type hasArg, hasOptions, argIsFile, argIsWord DirFlags = set[DirFlag] SectionParser = proc (p: var RstParser): PRstNode {.nimcall.} -{.deprecated: [TDirFlag: DirFlag, TDirFlags: DirFlags, - TSectionParser: SectionParser].} proc parseDirective(p: var RstParser, flags: DirFlags): PRstNode = ## Parses arguments and options for a directive block. diff --git a/lib/packages/docutils/rstast.nim b/lib/packages/docutils/rstast.nim index 7be4470c1..f3596b571 100644 --- a/lib/packages/docutils/rstast.nim +++ b/lib/packages/docutils/rstast.nim @@ -70,8 +70,6 @@ type ## the document or the section level*: int ## valid for some node kinds sons*: RstNodeSeq ## the node's sons -{.deprecated: [TRstNodeKind: RstNodeKind, TRstNodeSeq: RstNodeSeq, - TRstNode: RstNode].} proc len*(n: PRstNode): int = result = len(n.sons) @@ -99,7 +97,6 @@ type RenderContext {.pure.} = object indent: int verbatim: int -{.deprecated: [TRenderContext: RenderContext].} proc renderRstToRst(d: var RenderContext, n: PRstNode, result: var string) {.gcsafe.} diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim index e6c95b59e..03a27017a 100644 --- a/lib/packages/docutils/rstgen.nim +++ b/lib/packages/docutils/rstgen.nim @@ -208,6 +208,7 @@ proc nextSplitPoint*(s: string, start: int): int = dec(result) # last valid index proc esc*(target: OutputTarget, s: string, splitAfter = -1): string = + ## Escapes the HTML. result = "" if splitAfter >= 0: var partLen = 0 @@ -769,43 +770,45 @@ proc renderTocEntries*(d: var RstGenerator, j: var int, lvl: int, result.add(tmp) proc renderImage(d: PDoc, n: PRstNode, result: var string) = - template valid(s): bool = - s.len > 0 and allCharsInSet(s, {'.','/',':','%','_','\\','\128'..'\xFF'} + - Digits + Letters + WhiteSpace) let arg = getArgument(n) - isObject = arg.toLower().endsWith(".svg") var options = "" - content = "" - var s = getFieldValue(n, "scale") - if s.valid: dispA(d.target, options, if isObject: "" else: " scale=\"$1\"", - " scale=$1", [strip(s)]) - s = getFieldValue(n, "height") - if s.valid: dispA(d.target, options, " height=\"$1\"", " height=$1", [strip(s)]) + var s = esc(d.target, getFieldValue(n, "scale").strip()) + if s.len > 0: + dispA(d.target, options, " scale=\"$1\"", " scale=$1", [s]) - s = getFieldValue(n, "width") - if s.valid: dispA(d.target, options, " width=\"$1\"", " width=$1", [strip(s)]) + s = esc(d.target, getFieldValue(n, "height").strip()) + if s.len > 0: + dispA(d.target, options, " height=\"$1\"", " height=$1", [s]) - s = getFieldValue(n, "alt") - if s.valid: - # <object> displays its content if it cannot render the image - if isObject: dispA(d.target, content, "$1", "", [strip(s)]) - else: dispA(d.target, options, " alt=\"$1\"", "", [strip(s)]) + s = esc(d.target, getFieldValue(n, "width").strip()) + if s.len > 0: + dispA(d.target, options, " width=\"$1\"", " width=$1", [s]) - s = getFieldValue(n, "align") - if s.valid: dispA(d.target, options, " align=\"$1\"", "", [strip(s)]) + s = esc(d.target, getFieldValue(n, "alt").strip()) + if s.len > 0: + dispA(d.target, options, " alt=\"$1\"", "", [s]) + + s = esc(d.target, getFieldValue(n, "align").strip()) + if s.len > 0: + dispA(d.target, options, " align=\"$1\"", "", [s]) if options.len > 0: options = dispF(d.target, "$1", "[$1]", [options]) - if arg.valid: - let htmlOut = if isObject: - "<object data=\"$1\" type=\"image/svg+xml\"$2 >" & content & "</object>" - else: - "<img src=\"$1\"$2 />" - dispA(d.target, result, htmlOut, "\\includegraphics$2{$1}", - [arg, options]) + var htmlOut = "" + if arg.endsWith(".mp4") or arg.endsWith(".ogg") or + arg.endsWith(".webm"): + htmlOut = """ + <video src="$1"$2 autoPlay='true' loop='true' muted='true'> + Sorry, your browser doesn't support embedded videos + </video> + """ + else: + htmlOut = "<img src=\"$1\"$2/>" + dispA(d.target, result, htmlOut, "\\includegraphics$2{$1}", + [esc(d.target, arg), options]) if len(n) >= 3: renderRstToOut(d, n.sons[2], result) proc renderSmiley(d: PDoc, n: PRstNode, result: var string) = @@ -820,7 +823,7 @@ proc parseCodeBlockField(d: PDoc, n: PRstNode, params: var CodeBlockParams) = ## ## This supports the special ``default-language`` internal string generated ## by the ``rst`` module to communicate a specific default language. - case n.getArgument.toLower + case n.getArgument.toLowerAscii of "number-lines": params.numberLines = true # See if the field has a parameter specifying a different line than 1. @@ -836,8 +839,11 @@ proc parseCodeBlockField(d: PDoc, n: PRstNode, params: var CodeBlockParams) = params.filename = n.getFieldValue.strip of "test": params.testCmd = n.getFieldValue.strip - if params.testCmd.len == 0: params.testCmd = "nim c -r $1" - of "status": + if params.testCmd.len == 0: + params.testCmd = "nim c -r $1" + else: + params.testCmd = unescape(params.testCmd) + of "status", "exitcode": var status: int if parseInt(n.getFieldValue, status) > 0: params.status = status @@ -878,7 +884,8 @@ proc buildLinesHTMLTable(d: PDoc; params: CodeBlockParams, code: string): inc d.listingCounter let id = $d.listingCounter if not params.numberLines: - result = (d.config.getOrDefault"doc.listing_start" % [id, $params.lang], + result = (d.config.getOrDefault"doc.listing_start" % + [id, sourceLanguageToStr[params.lang]], d.config.getOrDefault"doc.listing_end" % id) return @@ -891,7 +898,8 @@ proc buildLinesHTMLTable(d: PDoc; params: CodeBlockParams, code: string): line.inc codeLines.dec result.beginTable.add("</pre></td><td>" & ( - d.config.getOrDefault"doc.listing_start" % [id, $params.lang])) + d.config.getOrDefault"doc.listing_start" % + [id, sourceLanguageToStr[params.lang]])) result.endTable = (d.config.getOrDefault"doc.listing_end" % id) & "</td></tr></tbody></table>" & ( d.config.getOrDefault"doc.listing_button" % id) @@ -941,7 +949,7 @@ proc renderCodeBlock(d: PDoc, n: PRstNode, result: var string) = proc renderContainer(d: PDoc, n: PRstNode, result: var string) = var tmp = "" renderRstToOut(d, n.sons[2], tmp) - var arg = strip(getArgument(n)) + var arg = esc(d.target, strip(getArgument(n))) if arg == "": dispA(d.target, result, "<div>$1</div>", "$1", [tmp]) else: diff --git a/lib/posix/epoll.nim b/lib/posix/epoll.nim index c5ed1a873..2d38137bb 100644 --- a/lib/posix/epoll.nim +++ b/lib/posix/epoll.nim @@ -7,7 +7,7 @@ # distribution, for details about the copyright. # -{.deadCodeElim:on.} +{.deadCodeElim: on.} # dce option deprecated from posix import SocketHandle diff --git a/lib/posix/inotify.nim b/lib/posix/inotify.nim index a206f8067..359e88617 100644 --- a/lib/posix/inotify.nim +++ b/lib/posix/inotify.nim @@ -7,7 +7,7 @@ # distribution, for details about the copyright. # -{.deadCodeElim:on.} +{.deadCodeElim: on.} # dce option deprecated # Get the platform-dependent flags. # Structure describing an inotify event. diff --git a/lib/posix/kqueue.nim b/lib/posix/kqueue.nim index 730491a53..18b47f5d5 100644 --- a/lib/posix/kqueue.nim +++ b/lib/posix/kqueue.nim @@ -7,8 +7,6 @@ # distribution, for details about the copyright. # -{.deadCodeElim:on.} - from posix import Timespec when defined(macosx) or defined(freebsd) or defined(openbsd) or @@ -61,7 +59,7 @@ const EV_CLEAR* = 0x0020 ## Clear event state after reporting. EV_RECEIPT* = 0x0040 ## Force EV_ERROR on success, data == 0 EV_DISPATCH* = 0x0080 ## Disable event after reporting. - + EV_SYSFLAGS* = 0xF000 ## Reserved by system EV_DROP* = 0x1000 ## Not should be dropped EV_FLAG1* = 0x2000 ## Filter-specific flag @@ -87,10 +85,10 @@ when defined(macosx) or defined(freebsd) or defined(dragonfly): NOTE_FFAND* = 0x40000000'u32 ## AND fflags NOTE_FFOR* = 0x80000000'u32 ## OR fflags NOTE_FFCOPY* = 0xc0000000'u32 ## copy fflags - NOTE_FFCTRLMASK* = 0xc0000000'u32 ## masks for operations + NOTE_FFCTRLMASK* = 0xc0000000'u32 ## masks for operations NOTE_FFLAGSMASK* = 0x00ffffff'u32 - NOTE_TRIGGER* = 0x01000000'u32 ## Cause the event to be triggered + NOTE_TRIGGER* = 0x01000000'u32 ## Cause the event to be triggered ## for output. # data/hint flags for EVFILT_{READ|WRITE}, shared with userspace diff --git a/lib/posix/linux.nim b/lib/posix/linux.nim index 8786ab535..25c7c7979 100644 --- a/lib/posix/linux.nim +++ b/lib/posix/linux.nim @@ -1,4 +1,4 @@ -{.deadCodeElim:on.} +{.deadCodeElim: on.} # dce option deprecated import posix @@ -27,4 +27,4 @@ proc clone*(fn: pointer; child_stack: pointer; flags: cint; arg: pointer; ptid: ptr Pid; tls: pointer; ctid: ptr Pid): cint {.importc, header: "<sched.h>".} -proc pipe2*(a: array[0..1, cint], flags: cint): cint {.importc, header: "<unistd.h>".} \ No newline at end of file +proc pipe2*(a: array[0..1, cint], flags: cint): cint {.importc, header: "<unistd.h>".} diff --git a/lib/posix/posix.nim b/lib/posix/posix.nim index fba35868c..db5f575af 100644 --- a/lib/posix/posix.nim +++ b/lib/posix/posix.nim @@ -27,10 +27,10 @@ ## resulting C code will just ``#include <XYZ.h>`` and *not* define the ## symbols declared here. -# This ensures that we don't accidentally generate #includes for files that -# might not exist on a specific platform! The user will get an error only -# if they actualy try to use the missing declaration -{.deadCodeElim: on.} +# Dead code elimination ensures that we don't accidentally generate #includes +# for files that might not exist on a specific platform! The user will get an +# error only if they actualy try to use the missing declaration +{.deadCodeElim: on.} # dce option deprecated # TODO these constants don't seem to be fetched from a header file for unknown # platforms - where do they come from and why are they here? @@ -82,6 +82,14 @@ const # Special types type Sighandler = proc (a: cint) {.noconv.} +const StatHasNanoseconds* = defined(linux) or defined(freebsd) or + defined(openbsd) or defined(dragonfly) ## \ + ## Boolean flag that indicates if the system supports nanosecond time + ## resolution in the fields of ``Stat``. Note that the nanosecond based fields + ## (``Stat.st_atim``, ``Stat.st_mtim`` and ``Stat.st_ctim``) can be accessed + ## without checking this flag, because this module defines fallback procs + ## when they are not available. + # Platform specific stuff when defined(linux) and defined(amd64): @@ -92,9 +100,9 @@ else: # There used to be this name in posix.nim a long time ago, not sure why! {.deprecated: [cSIG_HOLD: SIG_HOLD].} -when not defined(macosx) and not defined(android): +when StatHasNanoseconds: proc st_atime*(s: Stat): Time {.inline.} = - ## Second-granularity time of last access + ## Second-granularity time of last access. result = s.st_atim.tv_sec proc st_mtime*(s: Stat): Time {.inline.} = ## Second-granularity time of last data modification. @@ -102,6 +110,16 @@ when not defined(macosx) and not defined(android): proc st_ctime*(s: Stat): Time {.inline.} = ## Second-granularity time of last status change. result = s.st_ctim.tv_sec +else: + proc st_atim*(s: Stat): TimeSpec {.inline.} = + ## Nanosecond-granularity time of last access. + result.tv_sec = s.st_atime + proc st_mtim*(s: Stat): TimeSpec {.inline.} = + ## Nanosecond-granularity time of last data modification. + result.tv_sec = s.st_mtime + proc st_ctim*(s: Stat): TimeSpec {.inline.} = + ## Nanosecond-granularity time of last data modification. + result.tv_sec = s.st_ctime when hasAioH: proc aio_cancel*(a1: cint, a2: ptr Taiocb): cint {.importc, header: "<aio.h>".} @@ -973,3 +991,19 @@ template onSignal*(signals: varargs[cint], body: untyped) = proc (sig: cint) {.noconv.} = body ) + +type + RLimit* {.importc: "struct rlimit", + header: "<sys/resource.h>", pure, final.} = object + rlim_cur*: int + rlim_max*: int + ## The getrlimit() and setrlimit() system calls get and set resource limits respectively. + ## Each resource has an associated soft and hard limit, as defined by the RLimit structure + +proc setrlimit*(resource: cint, rlp: var RLimit): cint + {.importc: "setrlimit",header: "<sys/resource.h>".} + ## The setrlimit() system calls sets resource limits. + +proc getrlimit*(resource: cint, rlp: var RLimit): cint + {.importc: "getrlimit",header: "<sys/resource.h>".} + ## The getrlimit() system call gets resource limits. diff --git a/lib/posix/posix_linux_amd64.nim b/lib/posix/posix_linux_amd64.nim index 9e6211b63..4f114d394 100644 --- a/lib/posix/posix_linux_amd64.nim +++ b/lib/posix/posix_linux_amd64.nim @@ -351,8 +351,8 @@ type Timeval* {.importc: "struct timeval", header: "<sys/select.h>", final, pure.} = object ## struct timeval - tv_sec*: clong ## Seconds. - tv_usec*: clong ## Microseconds. + tv_sec*: Time ## Seconds. + tv_usec*: Suseconds ## Microseconds. TFdSet* {.importc: "fd_set", header: "<sys/select.h>", final, pure.} = object abi: array[1024 div (8 * sizeof(clong)), clong] diff --git a/lib/posix/posix_linux_amd64_consts.nim b/lib/posix/posix_linux_amd64_consts.nim index 4b693960e..ee4fac1e8 100644 --- a/lib/posix/posix_linux_amd64_consts.nim +++ b/lib/posix/posix_linux_amd64_consts.nim @@ -433,6 +433,9 @@ const POSIX_MADV_WILLNEED* = cint(3) const POSIX_MADV_DONTNEED* = cint(4) const MAP_POPULATE* = cint(32768) +# <sys/resource.h> +const RLIMIT_NOFILE* = cint(7) + # <sys/select.h> const FD_SETSIZE* = cint(1024) diff --git a/lib/posix/posix_other.nim b/lib/posix/posix_other.nim index 01bc1c1e5..b7570bd15 100644 --- a/lib/posix/posix_other.nim +++ b/lib/posix/posix_other.nim @@ -7,7 +7,7 @@ # distribution, for details about the copyright. # -{.deadCodeElim:on.} +{.deadCodeElim: on.} # dce option deprecated const hasSpawnH = not defined(haiku) # should exist for every Posix system nowadays @@ -215,14 +215,14 @@ type ## For a typed memory object, the length in bytes. ## For other file types, the use of this field is ## unspecified. - when defined(macosx) or defined(android): - st_atime*: Time ## Time of last access. - st_mtime*: Time ## Time of last data modification. - st_ctime*: Time ## Time of last status change. - else: + when StatHasNanoseconds: st_atim*: Timespec ## Time of last access. st_mtim*: Timespec ## Time of last data modification. st_ctim*: Timespec ## Time of last status change. + else: + st_atime*: Time ## Time of last access. + st_mtime*: Time ## Time of last data modification. + st_ctime*: Time ## Time of last status change. st_blksize*: Blksize ## A file system-specific preferred I/O block size ## for this object. In some file system types, this ## may vary from file to file. @@ -335,8 +335,8 @@ type Timeval* {.importc: "struct timeval", header: "<sys/select.h>", final, pure.} = object ## struct timeval - tv_sec*: int ## Seconds. - tv_usec*: int ## Microseconds. + tv_sec*: Time ## Seconds. + tv_usec*: Suseconds ## Microseconds. TFdSet* {.importc: "fd_set", header: "<sys/select.h>", final, pure.} = object Mcontext* {.importc: "mcontext_t", header: "<ucontext.h>", diff --git a/lib/posix/posix_other_consts.nim b/lib/posix/posix_other_consts.nim index 003414a6a..1b27fc5f6 100644 --- a/lib/posix/posix_other_consts.nim +++ b/lib/posix/posix_other_consts.nim @@ -451,6 +451,9 @@ var POSIX_TYPED_MEM_ALLOCATE* {.importc: "POSIX_TYPED_MEM_ALLOCATE", header: "<s var POSIX_TYPED_MEM_ALLOCATE_CONTIG* {.importc: "POSIX_TYPED_MEM_ALLOCATE_CONTIG", header: "<sys/mman.h>".}: cint var POSIX_TYPED_MEM_MAP_ALLOCATABLE* {.importc: "POSIX_TYPED_MEM_MAP_ALLOCATABLE", header: "<sys/mman.h>".}: cint +# <sys/resource.h> +var RLIMIT_NOFILE* {.importc: "RLIMIT_NOFILE", header: "<sys/resource.h>".}: cint + # <sys/select.h> var FD_SETSIZE* {.importc: "FD_SETSIZE", header: "<sys/select.h>".}: cint diff --git a/lib/posix/termios.nim b/lib/posix/termios.nim index f86e408fb..60d540107 100644 --- a/lib/posix/termios.nim +++ b/lib/posix/termios.nim @@ -7,7 +7,7 @@ # distribution, for details about the copyright. # -{.deadCodeElim: on.} +{.deadCodeElim: on.} # dce option deprecated import posix type diff --git a/lib/pure/algorithm.nim b/lib/pure/algorithm.nim index fdf2d7cbb..81badfae6 100644 --- a/lib/pure/algorithm.nim +++ b/lib/pure/algorithm.nim @@ -13,9 +13,6 @@ type SortOrder* = enum ## sort order Descending, Ascending -{.deprecated: [TSortOrder: SortOrder].} - - proc `*`*(x: int, order: SortOrder): int {.inline.} = ## flips `x` if ``order == Descending``; ## if ``order == Ascending`` then `x` is returned. @@ -24,16 +21,20 @@ proc `*`*(x: int, order: SortOrder): int {.inline.} = var y = order.ord - 1 result = (x xor y) - y -proc fill*[T](a: var openArray[T], first, last: Natural, value: T) = - ## fills the array ``a[first..last]`` with `value`. +template fillImpl[T](a: var openArray[T], first, last: int, value: T) = var x = first while x <= last: a[x] = value inc(x) +proc fill*[T](a: var openArray[T], first, last: Natural, value: T) = + ## fills the array ``a[first..last]`` with `value`. + fillImpl(a, first, last, value) + proc fill*[T](a: var openArray[T], value: T) = ## fills the array `a` with `value`. - fill(a, 0, a.high, value) + fillImpl(a, 0, a.high, value) + proc reverse*[T](a: var openArray[T], first, last: Natural) = ## reverses the array ``a[first..last]``. @@ -63,36 +64,72 @@ proc reversed*[T](a: openArray[T]): seq[T] = ## returns the reverse of the array `a`. reversed(a, 0, a.high) +proc binarySearch*[T, K](a: openArray[T], key: K, + cmp: proc (x: T, y: K): int {.closure.}): int = + ## binary search for `key` in `a`. Returns -1 if not found. + ## + ## `cmp` is the comparator function to use, the expected return values are + ## the same as that of system.cmp. + if a.len == 0: + return -1 + + let len = a.len + + if len == 1: + if cmp(a[0], key) == 0: + return 0 + else: + return -1 + + if (len and (len - 1)) == 0: + # when `len` is a power of 2, a faster shr can be used. + var step = len shr 1 + var cmpRes: int + while step > 0: + let i = result or step + cmpRes = cmp(a[i], key) + if cmpRes == 0: + return i + + if cmpRes < 1: + result = i + step = step shr 1 + if cmp(a[result], key) != 0: result = -1 + else: + var b = len + var cmpRes: int + while result < b: + var mid = (result + b) shr 1 + cmpRes = cmp(a[mid], key) + if cmpRes == 0: + return mid + + if cmpRes < 0: + result = mid + 1 + else: + b = mid + if result >= len or cmp(a[result], key) != 0: result = -1 + proc binarySearch*[T](a: openArray[T], key: T): int = ## binary search for `key` in `a`. Returns -1 if not found. - var b = len(a) - while result < b: - var mid = (result + b) div 2 - if a[mid] < key: result = mid + 1 - else: b = mid - if result >= len(a) or a[result] != key: result = -1 - -proc smartBinarySearch*[T](a: openArray[T], key: T): int = - ## ``a.len`` must be a power of 2 for this to work. - var step = a.len div 2 - while step > 0: - if a[result or step] <= key: - result = result or step - step = step shr 1 - if a[result] != key: result = -1 + binarySearch(a, key, cmp[T]) + +proc smartBinarySearch*[T](a: openArray[T], key: T): int {.deprecated.} = + ## **Deprecated since version 0.18.1**; Use ``binarySearch`` instead. + binarySearch(a, key, cmp[T]) const onlySafeCode = true -proc lowerBound*[T](a: openArray[T], key: T, cmp: proc(x,y: T): int {.closure.}): int = - ## same as binarySearch except that if key is not in `a` then this - ## returns the location where `key` would be if it were. In other - ## words if you have a sorted sequence and you call +proc lowerBound*[T, K](a: openArray[T], key: K, cmp: proc(x: T, k: K): int {.closure.}): int = + ## Returns a position to the first element in the `a` that is greater than `key`, or last + ## if no such element is found. In other words if you have a sorted sequence and you call ## insert(thing, elm, lowerBound(thing, elm)) ## the sequence will still be sorted. ## - ## `cmp` is the comparator function to use, the expected return values are + ## The first version uses `cmp` to compare the elements. The expected return values are ## the same as that of system.cmp. + ## The second version uses the default comparison function `cmp`. ## ## example:: ## @@ -103,7 +140,7 @@ proc lowerBound*[T](a: openArray[T], key: T, cmp: proc(x,y: T): int {.closure.}) var count = a.high - a.low + 1 var step, pos: int while count != 0: - step = count div 2 + step = count shr 1 pos = result + step if cmp(a[pos], key) < 0: result = pos + 1 @@ -113,6 +150,36 @@ proc lowerBound*[T](a: openArray[T], key: T, cmp: proc(x,y: T): int {.closure.}) proc lowerBound*[T](a: openArray[T], key: T): int = lowerBound(a, key, cmp[T]) +proc upperBound*[T, K](a: openArray[T], key: K, cmp: proc(x: T, k: K): int {.closure.}): int = + ## Returns a position to the first element in the `a` that is not less + ## (i.e. greater or equal to) than `key`, or last if no such element is found. + ## In other words if you have a sorted sequence and you call + ## insert(thing, elm, upperBound(thing, elm)) + ## the sequence will still be sorted. + ## + ## The first version uses `cmp` to compare the elements. The expected return values are + ## the same as that of system.cmp. + ## The second version uses the default comparison function `cmp`. + ## + ## example:: + ## + ## var arr = @[1,2,3,4,6,7,8,9] + ## arr.insert(5, arr.upperBound(4)) + ## # after running the above arr is `[1,2,3,4,5,6,7,8,9]` + result = a.low + var count = a.high - a.low + 1 + var step, pos: int + while count != 0: + step = count shr 1 + pos = result + step + if cmp(a[pos], key) <= 0: + result = pos + 1 + count -= step + 1 + else: + count = step + +proc upperBound*[T](a: openArray[T], key: T): int = upperBound(a, key, cmp[T]) + template `<-` (a, b) = when false: a = b @@ -359,10 +426,11 @@ when isMainModule: var srt1 = [1,2,3,4,4,4,4,5] var srt2 = ["iello","hello"] var srt3 = [1.0,1.0,1.0] - var srt4: seq[int] = @[] + var srt4: seq[int] assert srt1.isSorted(cmp) == true assert srt2.isSorted(cmp) == false assert srt3.isSorted(cmp) == true + assert srt4.isSorted(cmp) == true var srtseq = newSeq[int]() assert srtseq.isSorted(cmp) == true # Tests for reversed @@ -391,7 +459,7 @@ proc rotateInternal[T](arg: var openarray[T]; first, middle, last: int): int = swap(arg[mFirst], arg[next]) mFirst += 1 - next += 1 + next += 1 if mFirst == mMiddle: mMiddle = next @@ -443,7 +511,7 @@ proc rotateLeft*[T](arg: var openarray[T]; slice: HSlice[int, int]; dist: int): ## the distance in amount of elements that the data should be rotated. Can be negative, can be any number. ## ## .. code-block:: nim - ## var list = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + ## var list = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] ## list.rotateLeft(1 .. 8, 3) ## doAssert list == [0, 4, 5, 6, 7, 8, 1, 2, 3, 9, 10] let sliceLen = slice.b + 1 - slice.a @@ -472,12 +540,12 @@ proc rotatedLeft*[T](arg: openarray[T]; dist: int): seq[T] = arg.rotatedInternal(0, distLeft, arg.len) when isMainModule: - var list = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] - let list2 = list.rotatedLeft(1 ..< 9, 3) + var list = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + let list2 = list.rotatedLeft(1 ..< 9, 3) let expected = [0, 4, 5, 6, 7, 8, 1, 2, 3, 9, 10] doAssert list.rotateLeft(1 ..< 9, 3) == 6 - doAssert list == expected + doAssert list == expected doAssert list2 == @expected var s0,s1,s2,s3,s4,s5 = "xxxabcdefgxxx" @@ -494,3 +562,45 @@ when isMainModule: doAssert s4 == "xxxefgabcdxxx" doAssert s5.rotateLeft(3 ..< 10, 11) == 6 doAssert s5 == "xxxefgabcdxxx" + + block product: + doAssert product(newSeq[seq[int]]()) == newSeq[seq[int]](), "empty input" + doAssert product(@[newSeq[int](), @[], @[]]) == newSeq[seq[int]](), "bit more empty input" + doAssert product(@[@[1,2]]) == @[@[1,2]], "a simple case of one element" + doAssert product(@[@[1,2], @[3,4]]) == @[@[2,4],@[1,4],@[2,3],@[1,3]], "two elements" + doAssert product(@[@[1,2], @[3,4], @[5,6]]) == @[@[2,4,6],@[1,4,6],@[2,3,6],@[1,3,6], @[2,4,5],@[1,4,5],@[2,3,5],@[1,3,5]], "three elements" + doAssert product(@[@[1,2], @[]]) == newSeq[seq[int]](), "two elements, but one empty" + + block lowerBound: + doAssert lowerBound([1,2,4], 3, system.cmp[int]) == 2 + doAssert lowerBound([1,2,2,3], 4, system.cmp[int]) == 4 + doAssert lowerBound([1,2,3,10], 11) == 4 + + block upperBound: + doAssert upperBound([1,2,4], 3, system.cmp[int]) == 2 + doAssert upperBound([1,2,2,3], 3, system.cmp[int]) == 4 + doAssert upperBound([1,2,3,5], 3) == 3 + + block fillEmptySeq: + var s = newSeq[int]() + s.fill(0) + + block testBinarySearch: + var noData: seq[int] + doAssert binarySearch(noData, 7) == -1 + let oneData = @[1] + doAssert binarySearch(oneData, 1) == 0 + doAssert binarySearch(onedata, 7) == -1 + let someData = @[1,3,4,7] + doAssert binarySearch(someData, 1) == 0 + doAssert binarySearch(somedata, 7) == 3 + doAssert binarySearch(someData, -1) == -1 + doAssert binarySearch(someData, 5) == -1 + doAssert binarySearch(someData, 13) == -1 + let moreData = @[1,3,5,7,4711] + doAssert binarySearch(moreData, -1) == -1 + doAssert binarySearch(moreData, 1) == 0 + doAssert binarySearch(moreData, 5) == 2 + doAssert binarySearch(moreData, 6) == -1 + doAssert binarySearch(moreData, 4711) == 4 + doAssert binarySearch(moreData, 4712) == -1 diff --git a/lib/pure/async.nim b/lib/pure/async.nim new file mode 100644 index 000000000..97b29f81d --- /dev/null +++ b/lib/pure/async.nim @@ -0,0 +1,6 @@ +when defined(js): + import asyncjs + export asyncjs +else: + import asyncmacro, asyncfutures + export asyncmacro, asyncfutures diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index 42ffa236c..dfc7201b8 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -265,9 +265,15 @@ when defined(windows) or defined(nimdoc): setGlobalDispatcher(newDispatcher()) result = gDisp + proc getIoHandler*(disp: PDispatcher): Handle = + ## Returns the underlying IO Completion Port handle (Windows) or selector + ## (Unix) for the specified dispatcher. + return disp.ioPort + proc register*(fd: AsyncFD) = ## Registers ``fd`` with the dispatcher. let p = getGlobalDispatcher() + if createIoCompletionPort(fd.Handle, p.ioPort, cast[CompletionKey](fd), 1) == 0: raiseOSError(osLastError()) @@ -757,6 +763,9 @@ when defined(windows) or defined(nimdoc): ## Unregisters ``fd``. getGlobalDispatcher().handles.excl(fd) + proc contains*(disp: PDispatcher, fd: AsyncFD): bool = + return fd in disp.handles + {.push stackTrace:off.} proc waitableCallback(param: pointer, timerOrWaitFired: WINBOOL): void {.stdcall.} = @@ -977,7 +986,7 @@ when defined(windows) or defined(nimdoc): proc newAsyncEvent*(): AsyncEvent = ## Creates a new thread-safe ``AsyncEvent`` object. ## - ## New ``AsyncEvent`` object is not automatically registered with # TODO: Why? -- DP + ## New ``AsyncEvent`` object is not automatically registered with ## dispatcher like ``AsyncSocket``. var sa = SECURITY_ATTRIBUTES( nLength: sizeof(SECURITY_ATTRIBUTES).cint, @@ -1095,6 +1104,9 @@ else: setGlobalDispatcher(newDispatcher()) result = gDisp + proc getIoHandler*(disp: PDispatcher): Selector[AsyncData] = + return disp.selector + proc register*(fd: AsyncFD) = let p = getGlobalDispatcher() var data = newAsyncData() @@ -1110,6 +1122,9 @@ else: proc unregister*(ev: AsyncEvent) = getGlobalDispatcher().selector.unregister(SelectEvent(ev)) + + proc contains*(disp: PDispatcher, fd: AsyncFd): bool = + return fd.SocketHandle in disp.selector proc addRead*(fd: AsyncFD, cb: Callback) = let p = getGlobalDispatcher() @@ -1498,7 +1513,7 @@ proc poll*(timeout = 500) = # Common procedures between current and upcoming asyncdispatch include includes.asynccommon -proc sleepAsync*(ms: int): Future[void] = +proc sleepAsync*(ms: int | float): Future[void] = ## Suspends the execution of the current async procedure for the next ## ``ms`` milliseconds. var retFuture = newFuture[void]("sleepAsync") @@ -1633,4 +1648,8 @@ proc waitFor*[T](fut: Future[T]): T = fut.read -{.deprecated: [setEvent: trigger].} +proc setEvent*(ev: AsyncEvent) {.deprecated.} = + ## Set event ``ev`` to signaled state. + ## + ## **Deprecated since v0.18.0:** Use ``trigger`` instead. + ev.trigger() \ No newline at end of file diff --git a/lib/pure/asyncfile.nim b/lib/pure/asyncfile.nim index 9f4da16a3..1df7c3fc0 100644 --- a/lib/pure/asyncfile.nim +++ b/lib/pure/asyncfile.nim @@ -50,22 +50,21 @@ when defined(windows) or defined(nimdoc): case mode of fmRead, fmReadWriteExisting: OPEN_EXISTING - of fmAppend, fmReadWrite, fmWrite: - if fileExists(filename): - OPEN_EXISTING - else: - CREATE_NEW + of fmReadWrite, fmWrite: + CREATE_ALWAYS + of fmAppend: + OPEN_ALWAYS else: proc getPosixFlags(mode: FileMode): cint = case mode of fmRead: result = O_RDONLY of fmWrite: - result = O_WRONLY or O_CREAT + result = O_WRONLY or O_CREAT or O_TRUNC of fmAppend: result = O_WRONLY or O_CREAT or O_APPEND of fmReadWrite: - result = O_RDWR or O_CREAT + result = O_RDWR or O_CREAT or O_TRUNC of fmReadWriteExisting: result = O_RDWR result = result or O_NONBLOCK @@ -79,13 +78,16 @@ proc getFileSize*(f: AsyncFile): int64 = raiseOSError(osLastError()) result = (high shl 32) or low else: + let curPos = lseek(f.fd.cint, 0, SEEK_CUR) result = lseek(f.fd.cint, 0, SEEK_END) + f.offset = lseek(f.fd.cint, curPos, SEEK_SET) + assert(f.offset == curPos) proc newAsyncFile*(fd: AsyncFd): AsyncFile = ## Creates `AsyncFile` with a previously opened file descriptor `fd`. new result result.fd = fd - register(result.fd) + register(fd) proc openAsync*(filename: string, mode = fmRead): AsyncFile = ## Opens a file specified by the path in ``filename`` using @@ -97,16 +99,16 @@ proc openAsync*(filename: string, mode = fmRead): AsyncFile = when useWinUnicode: let fd = createFileW(newWideCString(filename), desiredAccess, FILE_SHARE_READ, - nil, creationDisposition, flags, 0).AsyncFd + nil, creationDisposition, flags, 0) else: let fd = createFileA(filename, desiredAccess, FILE_SHARE_READ, - nil, creationDisposition, flags, 0).AsyncFd + nil, creationDisposition, flags, 0) - if fd.Handle == INVALID_HANDLE_VALUE: + if fd == INVALID_HANDLE_VALUE: raiseOSError(osLastError()) - result = newAsyncFile(fd) + result = newAsyncFile(fd.AsyncFd) if mode == fmAppend: result.offset = getFileSize(result) @@ -115,11 +117,11 @@ proc openAsync*(filename: string, mode = fmRead): AsyncFile = let flags = getPosixFlags(mode) # RW (Owner), RW (Group), R (Other) let perm = S_IRUSR or S_IWUSR or S_IRGRP or S_IWGRP or S_IROTH - let fd = open(filename, flags, perm).AsyncFD - if fd.cint == -1: + let fd = open(filename, flags, perm) + if fd == -1: raiseOSError(osLastError()) - result = newAsyncFile(fd) + result = newAsyncFile(fd.AsyncFd) proc readBuffer*(f: AsyncFile, buf: pointer, size: int): Future[int] = ## Read ``size`` bytes from the specified file asynchronously starting at @@ -282,6 +284,7 @@ proc read*(f: AsyncFile, size: int): Future[string] = result = false # We still want this callback to be called. elif res == 0: # EOF + f.offset = lseek(fd.cint, 0, SEEK_CUR) retFuture.complete("") else: readBuffer.setLen(res) diff --git a/lib/pure/asyncfutures.nim b/lib/pure/asyncfutures.nim index bcc3ab613..863a6843b 100644 --- a/lib/pure/asyncfutures.nim +++ b/lib/pure/asyncfutures.nim @@ -27,8 +27,6 @@ type FutureError* = object of Exception cause*: FutureBase -{.deprecated: [PFutureBase: FutureBase, PFuture: Future].} - when not defined(release): var currentID = 0 @@ -177,7 +175,7 @@ proc fail*[T](future: Future[T], error: ref Exception) = if getStackTrace(error) == "": getStackTrace() else: getStackTrace(error) future.callbacks.call() -proc clearCallbacks(future: FutureBase) = +proc clearCallbacks*(future: FutureBase) = future.callbacks.function = nil future.callbacks.next = nil @@ -324,12 +322,12 @@ proc mget*[T](future: FutureVar[T]): var T = ## Future has not been finished. result = Future[T](future).value -proc finished*[T](future: Future[T] | FutureVar[T]): bool = +proc finished*(future: FutureBase | FutureVar): bool = ## Determines whether ``future`` has completed. ## ## ``True`` may indicate an error or a value. Use ``failed`` to distinguish. - when future is FutureVar[T]: - result = (Future[T](future)).finished + when future is FutureVar: + result = (FutureBase(future)).finished else: result = future.finished @@ -342,6 +340,7 @@ proc asyncCheck*[T](future: Future[T]) = ## finished with an error. ## ## This should be used instead of ``discard`` to discard void futures. + assert(not future.isNil, "Future is nil") future.callback = proc () = if future.failed: diff --git a/lib/pure/asynchttpserver.nim b/lib/pure/asynchttpserver.nim index ba1615651..fe5a835d7 100644 --- a/lib/pure/asynchttpserver.nim +++ b/lib/pure/asynchttpserver.nim @@ -60,9 +60,6 @@ type reusePort: bool maxBody: int ## The maximum content-length that will be read for the body. -{.deprecated: [TRequest: Request, PAsyncHttpServer: AsyncHttpServer, - THttpCode: HttpCode, THttpVersion: HttpVersion].} - proc newAsyncHttpServer*(reuseAddr = true, reusePort = false, maxBody = 8388608): AsyncHttpServer = ## Creates a new ``AsyncHttpServer`` instance. diff --git a/lib/pure/asyncmacro.nim b/lib/pure/asyncmacro.nim index 8c679929d..96a6fa158 100644 --- a/lib/pure/asyncmacro.nim +++ b/lib/pure/asyncmacro.nim @@ -11,7 +11,7 @@ ## ************* ## `asyncdispatch` module depends on the `asyncmacro` module to work properly. -import macros, strutils +import macros, strutils, asyncfutures proc skipUntilStmtList(node: NimNode): NimNode {.compileTime.} = # Skips a nest of StmtList's. @@ -26,12 +26,20 @@ proc skipStmtList(node: NimNode): NimNode {.compileTime.} = template createCb(retFutureSym, iteratorNameSym, strName, identName, futureVarCompletions: untyped) = + bind finished + var nameIterVar = iteratorNameSym #{.push stackTrace: off.} proc identName {.closure.} = try: if not nameIterVar.finished: var next = nameIterVar() + # Continue while the yielded future is already finished. + while (not next.isNil) and next.finished: + next = nameIterVar() + if nameIterVar.finished: + break + if next == nil: if not retFutureSym.finished: let msg = "Async procedure ($1) yielded `nil`, are you await'ing a " & @@ -173,7 +181,7 @@ proc processBody(node, retFutureSym: NimNode, result.add newNimNode(nnkReturnStmt, node).add(newNilLit()) return # Don't process the children of this return stmt of nnkCommand, nnkCall: - if node[0].kind == nnkIdent and node[0].ident == !"await": + if node[0].kind == nnkIdent and node[0].eqIdent("await"): case node[1].kind of nnkIdent, nnkInfix, nnkDotExpr, nnkCall, nnkCommand: # await x @@ -186,7 +194,7 @@ proc processBody(node, retFutureSym: NimNode, else: error("Invalid node kind in 'await', got: " & $node[1].kind) elif node.len > 1 and node[1].kind == nnkCommand and - node[1][0].kind == nnkIdent and node[1][0].ident == !"await": + node[1][0].kind == nnkIdent and node[1][0].eqIdent("await"): # foo await x var newCommand = node result.createVar("future" & $node[0].toStrLit, node[1][1], newCommand[1], @@ -195,16 +203,16 @@ proc processBody(node, retFutureSym: NimNode, of nnkVarSection, nnkLetSection: case node[0][2].kind of nnkCommand: - if node[0][2][0].kind == nnkIdent and node[0][2][0].ident == !"await": + if node[0][2][0].kind == nnkIdent and node[0][2][0].eqIdent("await"): # var x = await y var newVarSection = node # TODO: Should this use copyNimNode? - result.createVar("future" & $node[0][0].ident, node[0][2][1], + result.createVar("future" & node[0][0].strVal, node[0][2][1], newVarSection[0][2], newVarSection, node) else: discard of nnkAsgn: case node[1].kind of nnkCommand: - if node[1][0].ident == !"await": + if node[1][0].eqIdent("await"): # x = await y var newAsgn = node result.createVar("future" & $node[0].toStrLit, node[1][1], newAsgn[1], newAsgn, node) @@ -212,7 +220,7 @@ proc processBody(node, retFutureSym: NimNode, of nnkDiscardStmt: # discard await x if node[0].kind == nnkCommand and node[0][0].kind == nnkIdent and - node[0][0].ident == !"await": + node[0][0].eqIdent("await"): var newDiscard = node result.createVar("futureDiscard_" & $toStrLit(node[0][1]), node[0][1], newDiscard[0], newDiscard, node) @@ -277,9 +285,9 @@ proc processBody(node, retFutureSym: NimNode, proc getName(node: NimNode): string {.compileTime.} = case node.kind of nnkPostfix: - return $node[1].ident + return node[1].strVal of nnkIdent: - return $node.ident + return node.strVal of nnkEmpty: return "anonymous" else: @@ -290,7 +298,7 @@ proc getFutureVarIdents(params: NimNode): seq[NimNode] {.compileTime.} = for i in 1 ..< len(params): expectKind(params[i], nnkIdentDefs) if params[i][1].kind == nnkBracketExpr and - ($params[i][1][0].ident).normalize == "futurevar": + params[i][1][0].eqIdent("futurevar"): result.add(params[i][0]) proc isInvalidReturnType(typeName: string): bool = @@ -317,7 +325,7 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = let fut = repr(returnType[0]) verifyReturnType(fut) baseType = returnType[1] - elif returnType.kind in nnkCallKinds and $returnType[0] == "[]": + elif returnType.kind in nnkCallKinds and returnType[0].eqIdent("[]"): let fut = repr(returnType[1]) verifyReturnType(fut) baseType = returnType[2] @@ -327,7 +335,7 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = verifyReturnType(repr(returnType)) let subtypeIsVoid = returnType.kind == nnkEmpty or - (baseType.kind == nnkIdent and returnType[1].ident == !"void") + (baseType.kind == nnkIdent and returnType[1].eqIdent("void")) let futureVarIdents = getFutureVarIdents(prc.params) @@ -342,7 +350,7 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = newVarStmt(retFutureSym, newCall( newNimNode(nnkBracketExpr, prc.body).add( - newIdentNode(!"newFuture"), # TODO: Strange bug here? Remove the `!`. + newIdentNode("newFuture"), subRetType), newLit(prcName)))) # Get type from return type of this proc @@ -442,30 +450,30 @@ proc stripAwait(node: NimNode): NimNode = case node.kind of nnkCommand, nnkCall: - if node[0].kind == nnkIdent and node[0].ident == !"await": + if node[0].kind == nnkIdent and node[0].eqIdent("await"): node[0] = emptyNoopSym elif node.len > 1 and node[1].kind == nnkCommand and - node[1][0].kind == nnkIdent and node[1][0].ident == !"await": + node[1][0].kind == nnkIdent and node[1][0].eqIdent("await"): # foo await x node[1][0] = emptyNoopSym of nnkVarSection, nnkLetSection: case node[0][2].kind of nnkCommand: - if node[0][2][0].kind == nnkIdent and node[0][2][0].ident == !"await": + if node[0][2][0].kind == nnkIdent and node[0][2][0].eqIdent("await"): # var x = await y node[0][2][0] = emptyNoopSym else: discard of nnkAsgn: case node[1].kind of nnkCommand: - if node[1][0].ident == !"await": + if node[1][0].eqIdent("await"): # x = await y node[1][0] = emptyNoopSym else: discard of nnkDiscardStmt: # discard await x if node[0].kind == nnkCommand and node[0][0].kind == nnkIdent and - node[0][0].ident == !"await": + node[0][0].eqIdent("await"): node[0][0] = emptyNoopSym else: discard @@ -474,9 +482,9 @@ proc stripAwait(node: NimNode): NimNode = proc splitParamType(paramType: NimNode, async: bool): NimNode = result = paramType - if paramType.kind == nnkInfix and $paramType[0].ident in ["|", "or"]: - let firstAsync = "async" in ($paramType[1].ident).normalize - let secondAsync = "async" in ($paramType[2].ident).normalize + if paramType.kind == nnkInfix and paramType[0].strVal in ["|", "or"]: + let firstAsync = "async" in paramType[1].strVal.normalize + let secondAsync = "async" in paramType[2].strVal.normalize if firstAsync: result = paramType[if async: 1 else: 2] diff --git a/lib/pure/asyncnet.nim b/lib/pure/asyncnet.nim index 5be457d2a..e7552e3e3 100644 --- a/lib/pure/asyncnet.nim +++ b/lib/pure/asyncnet.nim @@ -134,15 +134,20 @@ type protocol: Protocol AsyncSocket* = ref AsyncSocketDesc -{.deprecated: [PAsyncSocket: AsyncSocket].} - proc newAsyncSocket*(fd: AsyncFD, domain: Domain = AF_INET, sockType: SockType = SOCK_STREAM, protocol: Protocol = IPPROTO_TCP, buffered = true): AsyncSocket = ## Creates a new ``AsyncSocket`` based on the supplied params. + ## + ## The supplied ``fd``'s non-blocking state will be enabled implicitly. + ## + ## **Note**: This procedure will **NOT** register ``fd`` with the global + ## async dispatcher. You need to do this manually. If you have used + ## ``newAsyncNativeSocket`` to create ``fd`` then it's already registered. assert fd != osInvalidSocket.AsyncFD new(result) result.fd = fd.SocketHandle + fd.SocketHandle.setBlocking(false) result.isBuffered = buffered result.domain = domain result.sockType = sockType @@ -156,8 +161,10 @@ proc newAsyncSocket*(domain: Domain = AF_INET, sockType: SockType = SOCK_STREAM, ## ## This procedure will also create a brand new file descriptor for ## this socket. - result = newAsyncSocket(newAsyncNativeSocket(domain, sockType, protocol), - domain, sockType, protocol, buffered) + let fd = createAsyncNativeSocket(domain, sockType, protocol) + if fd.SocketHandle == osInvalidSocket: + raiseOSError(osLastError()) + result = newAsyncSocket(fd, domain, sockType, protocol, buffered) proc newAsyncSocket*(domain, sockType, protocol: cint, buffered = true): AsyncSocket = @@ -165,8 +172,10 @@ proc newAsyncSocket*(domain, sockType, protocol: cint, ## ## This procedure will also create a brand new file descriptor for ## this socket. - result = newAsyncSocket(newAsyncNativeSocket(domain, sockType, protocol), - Domain(domain), SockType(sockType), + let fd = createAsyncNativeSocket(domain, sockType, protocol) + if fd.SocketHandle == osInvalidSocket: + raiseOSError(osLastError()) + result = newAsyncSocket(fd, Domain(domain), SockType(sockType), Protocol(protocol), buffered) when defineSsl: @@ -190,7 +199,7 @@ when defineSsl: flags: set[SocketFlag]) {.async.} = let len = bioCtrlPending(socket.bioOut) if len > 0: - var data = newStringOfCap(len) + var data = newString(len) let read = bioRead(socket.bioOut, addr data[0], len) assert read != 0 if read < 0: @@ -277,6 +286,7 @@ template readInto(buf: pointer, size: int, socket: AsyncSocket, flags: set[SocketFlag]): int = ## Reads **up to** ``size`` bytes from ``socket`` into ``buf``. Note that ## this is a template and not a proc. + assert(not socket.closed, "Cannot `recv` on a closed socket") var res = 0 if socket.isSsl: when defineSsl: @@ -403,6 +413,7 @@ proc send*(socket: AsyncSocket, buf: pointer, size: int, ## Sends ``size`` bytes from ``buf`` to ``socket``. The returned future will complete once all ## data has been sent. assert socket != nil + assert(not socket.closed, "Cannot `send` on a closed socket") if socket.isSsl: when defineSsl: sslLoop(socket, flags, diff --git a/lib/pure/base64.nim b/lib/pure/base64.nim index 4b0d08292..bfb8a1666 100644 --- a/lib/pure/base64.nim +++ b/lib/pure/base64.nim @@ -52,7 +52,7 @@ template encodeInternal(s: typed, lineLen: int, newLine: string): untyped = if numLines > 0: inc(total, (numLines - 1) * newLine.len) result = newString(total) - var + var i = 0 r = 0 currLine = 0 @@ -76,7 +76,7 @@ template encodeInternal(s: typed, lineLen: int, newLine: string): untyped = currLine = 0 if i < s.len-1: - let + let a = ord(s[i]) b = ord(s[i+1]) result[r] = cb64[a shr 2] @@ -130,11 +130,11 @@ proc decode*(s: string): string = # total is an upper bound, as we will skip arbitrary whitespace: result = newString(total) - var + var i = 0 r = 0 while true: - while s[i] in Whitespace: inc(i) + while i < s.len and s[i] in Whitespace: inc(i) if i < s.len-3: let a = s[i].decodeByte diff --git a/lib/pure/cgi.nim b/lib/pure/cgi.nim index 5de6aa487..e0cdc2ec0 100644 --- a/lib/pure/cgi.nim +++ b/lib/pure/cgi.nim @@ -64,8 +64,6 @@ type methodPost, ## query uses the POST method methodGet ## query uses the GET method -{.deprecated: [TRequestMethod: RequestMethod, ECgi: CgiError].} - proc cgiError*(msg: string) {.noreturn.} = ## raises an ECgi exception with message `msg`. var e: ref CgiError @@ -97,11 +95,10 @@ iterator decodeData*(data: string): tuple[key, value: TaintedString] = var name = "" var value = "" # decode everything in one pass: - while data[i] != '\0': + while i < data.len: setLen(name, 0) # reuse memory - while true: + while i < data.len: case data[i] - of '\0': break of '%': var x = 0 handleHexChar(data[i+1], x) @@ -112,15 +109,16 @@ iterator decodeData*(data: string): tuple[key, value: TaintedString] = of '=', '&': break else: add(name, data[i]) inc(i) - if data[i] != '=': cgiError("'=' expected") + if i >= data.len or data[i] != '=': cgiError("'=' expected") inc(i) # skip '=' setLen(value, 0) # reuse memory - while true: + while i < data.len: case data[i] of '%': var x = 0 - handleHexChar(data[i+1], x) - handleHexChar(data[i+2], x) + if i+2 < data.len: + handleHexChar(data[i+1], x) + handleHexChar(data[i+2], x) inc(i, 2) add(value, chr(x)) of '+': add(value, ' ') @@ -128,9 +126,9 @@ iterator decodeData*(data: string): tuple[key, value: TaintedString] = else: add(value, data[i]) inc(i) yield (name.TaintedString, value.TaintedString) - if data[i] == '&': inc(i) - elif data[i] == '\0': break - else: cgiError("'&' expected") + if i < data.len: + if data[i] == '&': inc(i) + else: cgiError("'&' expected") iterator decodeData*(allowedMethods: set[RequestMethod] = {methodNone, methodPost, methodGet}): tuple[key, value: TaintedString] = diff --git a/lib/pure/collections/critbits.nim b/lib/pure/collections/critbits.nim index 34f5c5470..5ae5e26b2 100644 --- a/lib/pure/collections/critbits.nim +++ b/lib/pure/collections/critbits.nim @@ -74,18 +74,19 @@ proc rawInsert[T](c: var CritBitTree[T], key: string): Node[T] = var newByte = 0 block blockX: while newbyte < key.len: - if it.key[newbyte] != key[newbyte]: - newotherbits = it.key[newbyte].ord xor key[newbyte].ord + let ch = if newbyte < it.key.len: it.key[newbyte] else: '\0' + if ch != key[newbyte]: + newotherbits = ch.ord xor key[newbyte].ord break blockX inc newbyte - if it.key[newbyte] != '\0': + if newbyte < it.key.len: newotherbits = it.key[newbyte].ord else: return it while (newOtherBits and (newOtherBits-1)) != 0: newOtherBits = newOtherBits and (newOtherBits-1) newOtherBits = newOtherBits xor 255 - let ch = it.key[newByte] + let ch = if newByte < it.key.len: it.key[newByte] else: '\0' let dir = (1 + (ord(ch) or newOtherBits)) shr 8 var inner: Node[T] @@ -162,13 +163,13 @@ proc containsOrIncl*(c: var CritBitTree[void], key: string): bool = var n = rawInsert(c, key) result = c.count == oldCount -proc inc*(c: var CritBitTree[int]; key: string) = - ## counts the 'key'. +proc inc*(c: var CritBitTree[int]; key: string, val: int = 1) = + ## increments `c[key]` by `val`. let oldCount = c.count var n = rawInsert(c, key) - if c.count == oldCount: + if c.count == oldCount or oldCount == 0: # not a new key: - inc n.val + inc n.val, val proc incl*(c: var CritBitTree[void], key: string) = ## includes `key` in `c`. @@ -351,3 +352,13 @@ when isMainModule: assert toSeq(r.items) == @["abc", "definition", "prefix", "xyz"] assert toSeq(r.itemsWithPrefix("de")) == @["definition"] + var c = CritBitTree[int]() + + c.inc("a") + assert c["a"] == 1 + + c.inc("a", 4) + assert c["a"] == 5 + + c.inc("a", -5) + assert c["a"] == 0 diff --git a/lib/pure/collections/intsets.nim b/lib/pure/collections/intsets.nim index 085232564..bfecfe447 100644 --- a/lib/pure/collections/intsets.nim +++ b/lib/pure/collections/intsets.nim @@ -108,6 +108,28 @@ proc contains*(s: IntSet, key: int): bool = else: result = false +iterator items*(s: IntSet): int {.inline.} = + ## iterates over any included element of `s`. + if s.elems <= s.a.len: + for i in 0..<s.elems: + yield s.a[i] + else: + var r = s.head + while r != nil: + var i = 0 + while i <= high(r.bits): + var w = r.bits[i] + # taking a copy of r.bits[i] here is correct, because + # modifying operations are not allowed during traversation + var j = 0 + while w != 0: # test all remaining bits for zero + if (w and 1) != 0: # the bit is set! + yield (r.key shl TrunkShift) or (i shl IntShift +% j) + inc(j) + w = w shr 1 + inc(i) + r = r.next + proc bitincl(s: var IntSet, key: int) {.inline.} = var t = intSetPut(s, `shr`(key, TrunkShift)) var u = key and TrunkMask @@ -131,6 +153,10 @@ proc incl*(s: var IntSet, key: int) = # fall through: bitincl(s, key) +proc incl*(s: var IntSet, other: IntSet) = + ## Includes all elements from `other` into `s`. + for item in other: incl(s, item) + proc exclImpl(s: var IntSet, key: int) = if s.elems <= s.a.len: for i in 0..<s.elems: @@ -149,6 +175,10 @@ proc excl*(s: var IntSet, key: int) = ## excludes `key` from the set `s`. exclImpl(s, key) +proc excl*(s: var IntSet, other: IntSet) = + ## Excludes all elements from `other` from `s`. + for item in other: excl(s, item) + proc missingOrExcl*(s: var IntSet, key: int) : bool = ## returns true if `s` does not contain `key`, otherwise ## `key` is removed from `s` and false is returned. @@ -232,27 +262,77 @@ proc assign*(dest: var IntSet, src: IntSet) = it = it.next -iterator items*(s: IntSet): int {.inline.} = - ## iterates over any included element of `s`. - if s.elems <= s.a.len: - for i in 0..<s.elems: - yield s.a[i] +proc union*(s1, s2: IntSet): IntSet = + ## Returns the union of the sets `s1` and `s2`. + result.assign(s1) + incl(result, s2) + +proc intersection*(s1, s2: IntSet): IntSet = + ## Returns the intersection of the sets `s1` and `s2`. + result = initIntSet() + for item in s1: + if contains(s2, item): + incl(result, item) + +proc difference*(s1, s2: IntSet): IntSet = + ## Returns the difference of the sets `s1` and `s2`. + result = initIntSet() + for item in s1: + if not contains(s2, item): + incl(result, item) + +proc symmetricDifference*(s1, s2: IntSet): IntSet = + ## Returns the symmetric difference of the sets `s1` and `s2`. + result.assign(s1) + for item in s2: + if containsOrIncl(result, item): excl(result, item) + +proc `+`*(s1, s2: IntSet): IntSet {.inline.} = + ## Alias for `union(s1, s2) <#union>`_. + result = union(s1, s2) + +proc `*`*(s1, s2: IntSet): IntSet {.inline.} = + ## Alias for `intersection(s1, s2) <#intersection>`_. + result = intersection(s1, s2) + +proc `-`*(s1, s2: IntSet): IntSet {.inline.} = + ## Alias for `difference(s1, s2) <#difference>`_. + result = difference(s1, s2) + +proc disjoint*(s1, s2: IntSet): bool = + ## Returns true iff the sets `s1` and `s2` have no items in common. + for item in s1: + if contains(s2, item): + return false + return true + +proc len*(s: IntSet): int {.inline.} = + ## Returns the number of keys in `s`. + if s.elems < s.a.len: + result = s.elems else: - var r = s.head - while r != nil: - var i = 0 - while i <= high(r.bits): - var w = r.bits[i] - # taking a copy of r.bits[i] here is correct, because - # modifying operations are not allowed during traversation - var j = 0 - while w != 0: # test all remaining bits for zero - if (w and 1) != 0: # the bit is set! - yield (r.key shl TrunkShift) or (i shl IntShift +% j) - inc(j) - w = w shr 1 - inc(i) - r = r.next + result = 0 + for _ in s: + inc(result) + +proc card*(s: IntSet): int {.inline.} = + ## alias for `len() <#len>` _. + result = s.len() + +proc `<=`*(s1, s2: IntSet): bool = + ## Returns true iff `s1` is subset of `s2`. + for item in s1: + if not s2.contains(item): + return false + return true + +proc `<`*(s1, s2: IntSet): bool = + ## Returns true iff `s1` is proper subset of `s2`. + return s1 <= s2 and not (s2 <= s1) + +proc `==`*(s1, s2: IntSet): bool = + ## Returns true if both `s` and `t` have the same members and set size. + return s1 <= s2 and s2 <= s1 template dollarImpl(): untyped = result = "{" @@ -301,9 +381,64 @@ when isMainModule: ys.sort(cmp[int]) assert ys == @[1, 2, 7, 1056] + assert x == y + var z: IntSet for i in 0..1000: incl z, i + assert z.len() == i+1 for i in 0..1000: - assert i in z + assert z.contains(i) + + var w = initIntSet() + w.incl(1) + w.incl(4) + w.incl(50) + w.incl(1001) + w.incl(1056) + + var xuw = x.union(w) + var xuws = toSeq(items(xuw)) + xuws.sort(cmp[int]) + assert xuws == @[1, 2, 4, 7, 50, 1001, 1056] + + var xiw = x.intersection(w) + var xiws = toSeq(items(xiw)) + xiws.sort(cmp[int]) + assert xiws == @[1, 1056] + + var xdw = x.difference(w) + var xdws = toSeq(items(xdw)) + xdws.sort(cmp[int]) + assert xdws == @[2, 7] + + var xsw = x.symmetricDifference(w) + var xsws = toSeq(items(xsw)) + xsws.sort(cmp[int]) + assert xsws == @[2, 4, 7, 50, 1001] + + x.incl(w) + xs = toSeq(items(x)) + xs.sort(cmp[int]) + assert xs == @[1, 2, 4, 7, 50, 1001, 1056] + + assert w <= x + + assert w < x + + assert(not disjoint(w, x)) + var u = initIntSet() + u.incl(3) + u.incl(5) + u.incl(500) + assert disjoint(u, x) + + var v = initIntSet() + v.incl(2) + v.incl(50) + + x.excl(v) + xs = toSeq(items(x)) + xs.sort(cmp[int]) + assert xs == @[1, 4, 7, 1001, 1056] diff --git a/lib/pure/collections/sequtils.nim b/lib/pure/collections/sequtils.nim index 06e96ca36..44c59c627 100644 --- a/lib/pure/collections/sequtils.nim +++ b/lib/pure/collections/sequtils.nim @@ -657,7 +657,7 @@ template mapIt*(s, op: untyped): untyped = when compiles(s.len): let t = s var i = 0 - result = newSeq[outType](s.len) + result = newSeq[outType](t.len) for it {.inject.} in t: result[i] = op i += 1 @@ -711,7 +711,7 @@ proc mapLitsImpl(constructor: NimNode; op: NimNode; nested: bool; result.add op result.add constructor else: - result = newNimNode(constructor.kind, lineInfoFrom=constructor) + result = copyNimNode(constructor) for v in constructor: if nested or v.kind in filter: result.add mapLitsImpl(v, op, nested, filter) diff --git a/lib/pure/collections/sets.nim b/lib/pure/collections/sets.nim index 9e9152fc8..59c90bc2b 100644 --- a/lib/pure/collections/sets.nim +++ b/lib/pure/collections/sets.nim @@ -11,7 +11,7 @@ ## ordered hash set. ## ## Hash sets are different from the `built in set type -## <manual.html#set-type>`_. Sets allow you to store any value that can be +## <manual.html#types-set-type>`_. Sets allow you to store any value that can be ## `hashed <hashes.html>`_ and they don't contain duplicate entries. ## ## **Note**: The data types declared here have *value semantics*: This means @@ -120,6 +120,13 @@ iterator items*[A](s: HashSet[A]): A = for h in 0..high(s.data): if isFilled(s.data[h].hcode): yield s.data[h].key +proc hash*[A](s: HashSet[A]): Hash = + ## hashing of HashSet + assert s.isValid, "The set needs to be initialized." + for h in 0..high(s.data): + result = result xor s.data[h].hcode + result = !$result + const growthFactor = 2 @@ -690,6 +697,13 @@ iterator items*[A](s: OrderedSet[A]): A = forAllOrderedPairs: yield s.data[h].key +proc hash*[A](s: OrderedSet[A]): Hash = + ## hashing of OrderedSet + assert s.isValid, "The set needs to be initialized." + forAllOrderedPairs: + result = result !& s.data[h].hcode + result = !$result + iterator pairs*[A](s: OrderedSet[A]): tuple[a: int, b: A] = assert s.isValid, "The set needs to be initialized" forAllOrderedPairs: diff --git a/lib/pure/collections/sharedtables.nim b/lib/pure/collections/sharedtables.nim index 4f311af87..0292a27a2 100644 --- a/lib/pure/collections/sharedtables.nim +++ b/lib/pure/collections/sharedtables.nim @@ -136,11 +136,13 @@ proc withKey*[A, B](t: var SharedTable[A, B], key: A, ## procedure. ## ## The ``mapper`` takes 3 arguments: - ## #. ``key`` - the current key, if it exists, or the key passed to - ## ``withKey`` otherwise; - ## #. ``val`` - the current value, if the key exists, or default value - ## of the type otherwise; - ## #. ``pairExists`` - ``true`` if the key exists, ``false`` otherwise. + ## + ## 1. ``key`` - the current key, if it exists, or the key passed to + ## ``withKey`` otherwise; + ## 2. ``val`` - the current value, if the key exists, or default value + ## of the type otherwise; + ## 3. ``pairExists`` - ``true`` if the key exists, ``false`` otherwise. + ## ## The ``mapper`` can can modify ``val`` and ``pairExists`` values to change ## the mapping of the key or delete it from the table. ## When adding a value, make sure to set ``pairExists`` to ``true`` along diff --git a/lib/pure/collections/tables.nim b/lib/pure/collections/tables.nim index 777beabc3..7b508b294 100644 --- a/lib/pure/collections/tables.nim +++ b/lib/pure/collections/tables.nim @@ -158,6 +158,12 @@ template getOrDefaultImpl(t, key): untyped = var index = rawGet(t, key, hc) if index >= 0: result = t.data[index].val +template getOrDefaultImpl(t, key, default: untyped): untyped = + mixin rawGet + var hc: Hash + var index = rawGet(t, key, hc) + result = if index >= 0: t.data[index].val else: default + proc `[]`*[A, B](t: Table[A, B], key: A): B {.deprecatedGet.} = ## retrieves the value at ``t[key]``. If `key` is not in `t`, the ## ``KeyError`` exception is raised. One can check with ``hasKey`` whether @@ -175,10 +181,18 @@ proc mget*[A, B](t: var Table[A, B], key: A): var B {.deprecated.} = ## instead. get(t, key) -proc getOrDefault*[A, B](t: Table[A, B], key: A): B = getOrDefaultImpl(t, key) +proc getOrDefault*[A, B](t: Table[A, B], key: A): B = + ## retrieves the value at ``t[key]`` iff `key` is in `t`. Otherwise, the + ## default initialization value for type `B` is returned (e.g. 0 for any + ## integer type). + getOrDefaultImpl(t, key) -template withValue*[A, B](t: var Table[A, B], key: A, - value, body: untyped) = +proc getOrDefault*[A, B](t: Table[A, B], key: A, default: B): B = + ## retrieves the value at ``t[key]`` iff `key` is in `t`. Otherwise, `default` + ## is returned. + getOrDefaultImpl(t, key, default) + +template withValue*[A, B](t: var Table[A, B], key: A, value, body: untyped) = ## retrieves the value at ``t[key]``. ## `value` can be modified in the scope of the ``withValue`` call. ## @@ -325,8 +339,7 @@ proc initTable*[A, B](initialSize=64): Table[A, B] = result.counter = 0 newSeq(result.data, initialSize) -proc toTable*[A, B](pairs: openArray[(A, - B)]): Table[A, B] = +proc toTable*[A, B](pairs: openArray[(A, B)]): Table[A, B] = ## creates a new hash table that contains the given `pairs`. result = initTable[A, B](rightSize(pairs.len)) for key, val in items(pairs): result[key] = val @@ -410,7 +423,16 @@ proc mget*[A, B](t: TableRef[A, B], key: A): var B {.deprecated.} = ## Use ```[]``` instead. t[][key] -proc getOrDefault*[A, B](t: TableRef[A, B], key: A): B = getOrDefault(t[], key) +proc getOrDefault*[A, B](t: TableRef[A, B], key: A): B = + ## retrieves the value at ``t[key]`` iff `key` is in `t`. Otherwise, the + ## default initialization value for type `B` is returned (e.g. 0 for any + ## integer type). + getOrDefault(t[], key) + +proc getOrDefault*[A, B](t: TableRef[A, B], key: A, default: B): B = + ## retrieves the value at ``t[key]`` iff `key` is in `t`. Otherwise, `default` + ## is returned. + getOrDefault(t[], key, default) proc mgetOrPut*[A, B](t: TableRef[A, B], key: A, val: B): var B = ## retrieves value at ``t[key]`` or puts ``val`` if not present, either way @@ -562,8 +584,15 @@ proc mget*[A, B](t: var OrderedTable[A, B], key: A): var B {.deprecated.} = get(t, key) proc getOrDefault*[A, B](t: OrderedTable[A, B], key: A): B = + ## retrieves the value at ``t[key]`` iff `key` is in `t`. Otherwise, the + ## default initialization value for type `B` is returned (e.g. 0 for any + ## integer type). getOrDefaultImpl(t, key) +proc getOrDefault*[A, B](t: OrderedTable[A, B], key: A, default: B): B = + ## retrieves the value at ``t[key]`` iff `key` is in `t`. Otherwise, `default` + ## is returned. + getOrDefaultImpl(t, key, default) proc hasKey*[A, B](t: OrderedTable[A, B], key: A): bool = ## returns true iff `key` is in the table `t`. @@ -630,8 +659,7 @@ proc initOrderedTable*[A, B](initialSize=64): OrderedTable[A, B] = result.last = -1 newSeq(result.data, initialSize) -proc toOrderedTable*[A, B](pairs: openArray[(A, - B)]): OrderedTable[A, B] = +proc toOrderedTable*[A, B](pairs: openArray[(A, B)]): OrderedTable[A, B] = ## creates a new ordered hash table that contains the given `pairs`. result = initOrderedTable[A, B](rightSize(pairs.len)) for key, val in items(pairs): result[key] = val @@ -657,8 +685,7 @@ proc `==`*[A, B](s, t: OrderedTable[A, B]): bool = hs = nxts return true -proc sort*[A, B](t: var OrderedTable[A, B], - cmp: proc (x,y: (A, B)): int) = +proc sort*[A, B](t: var OrderedTable[A, B], cmp: proc (x,y: (A, B)): int) = ## sorts `t` according to `cmp`. This modifies the internal list ## that kept the insertion order, so insertion order is lost after this ## call but key lookup and insertions remain possible after `sort` (in @@ -748,8 +775,16 @@ proc mget*[A, B](t: OrderedTableRef[A, B], key: A): var B {.deprecated.} = result = t[][key] proc getOrDefault*[A, B](t: OrderedTableRef[A, B], key: A): B = + ## retrieves the value at ``t[key]`` iff `key` is in `t`. Otherwise, the + ## default initialization value for type `B` is returned (e.g. 0 for any + ## integer type). getOrDefault(t[], key) +proc getOrDefault*[A, B](t: OrderedTableRef[A, B], key: A, default: B): B = + ## retrieves the value at ``t[key]`` iff `key` is in `t`. Otherwise, `default` + ## is returned. + getOrDefault(t[], key, default) + proc mgetOrPut*[A, B](t: OrderedTableRef[A, B], key: A, val: B): var B = ## retrieves value at ``t[key]`` or puts ``val`` if not present, either way ## returning a value which can be modified. @@ -802,8 +837,7 @@ proc `==`*[A, B](s, t: OrderedTableRef[A, B]): bool = elif isNil(t): result = false else: result = s[] == t[] -proc sort*[A, B](t: OrderedTableRef[A, B], - cmp: proc (x,y: (A, B)): int) = +proc sort*[A, B](t: OrderedTableRef[A, B], cmp: proc (x,y: (A, B)): int) = ## sorts `t` according to `cmp`. This modifies the internal list ## that kept the insertion order, so insertion order is lost after this ## call but key lookup and insertions remain possible after `sort` (in @@ -916,9 +950,17 @@ proc mget*[A](t: var CountTable[A], key: A): var int {.deprecated.} = ctget(t, key) proc getOrDefault*[A](t: CountTable[A], key: A): int = + ## retrieves the value at ``t[key]`` iff `key` is in `t`. Otherwise, 0 (the + ## default initialization value of `int`), is returned. var index = rawGet(t, key) if index >= 0: result = t.data[index].val +proc getOrDefault*[A](t: CountTable[A], key: A, default: int): int = + ## retrieves the value at ``t[key]`` iff `key` is in `t`. Otherwise, the + ## integer value of `default` is returned. + var index = rawGet(t, key) + result = if index >= 0: t.data[index].val else: default + proc hasKey*[A](t: CountTable[A], key: A): bool = ## returns true iff `key` is in the table `t`. result = rawGet(t, key) >= 0 @@ -955,6 +997,17 @@ proc `[]=`*[A](t: var CountTable[A], key: A, val: int) = #t.data[h].key = key #t.data[h].val = val +proc inc*[A](t: var CountTable[A], key: A, val = 1) = + ## increments `t[key]` by `val`. + var index = rawGet(t, key) + if index >= 0: + inc(t.data[index].val, val) + if t.data[index].val == 0: dec(t.counter) + else: + if mustRehash(len(t.data), t.counter): enlarge(t) + rawInsert(t, t.data, key, val) + inc(t.counter) + proc initCountTable*[A](initialSize=64): CountTable[A] = ## creates a new count table that is empty. ## @@ -969,7 +1022,7 @@ proc toCountTable*[A](keys: openArray[A]): CountTable[A] = ## creates a new count table with every key in `keys` having a count ## of how many times it occurs in `keys`. result = initCountTable[A](rightSize(keys.len)) - for key in items(keys): result.inc key + for key in items(keys): result.inc(key) proc `$`*[A](t: CountTable[A]): string = ## The `$` operator for count tables. @@ -980,17 +1033,6 @@ proc `==`*[A](s, t: CountTable[A]): bool = ## contain the same keys with the same count. Insert order does not matter. equalsImpl(s, t) -proc inc*[A](t: var CountTable[A], key: A, val = 1) = - ## increments `t[key]` by `val`. - var index = rawGet(t, key) - if index >= 0: - inc(t.data[index].val, val) - if t.data[index].val == 0: dec(t.counter) - else: - if mustRehash(len(t.data), t.counter): enlarge(t) - rawInsert(t, t.data, key, val) - inc(t.counter) - proc smallest*[A](t: CountTable[A]): tuple[key: A, val: int] = ## returns the (key,val)-pair with the smallest `val`. Efficiency: O(n) assert t.len > 0 @@ -1073,8 +1115,15 @@ proc mget*[A](t: CountTableRef[A], key: A): var int {.deprecated.} = result = t[][key] proc getOrDefault*[A](t: CountTableRef[A], key: A): int = + ## retrieves the value at ``t[key]`` iff `key` is in `t`. Otherwise, 0 (the + ## default initialization value of `int`), is returned. result = t[].getOrDefault(key) +proc getOrDefault*[A](t: CountTableRef[A], key: A, default: int): int = + ## retrieves the value at ``t[key]`` iff `key` is in `t`. Otherwise, the + ## integer value of `default` is returned. + result = t[].getOrDefault(key, default) + proc hasKey*[A](t: CountTableRef[A], key: A): bool = ## returns true iff `key` is in the table `t`. result = t[].hasKey(key) @@ -1088,6 +1137,10 @@ proc `[]=`*[A](t: CountTableRef[A], key: A, val: int) = assert val > 0 t[][key] = val +proc inc*[A](t: CountTableRef[A], key: A, val = 1) = + ## increments `t[key]` by `val`. + t[].inc(key, val) + proc newCountTable*[A](initialSize=64): CountTableRef[A] = ## creates a new count table that is empty. ## @@ -1098,9 +1151,10 @@ proc newCountTable*[A](initialSize=64): CountTableRef[A] = result[] = initCountTable[A](initialSize) proc newCountTable*[A](keys: openArray[A]): CountTableRef[A] = - ## creates a new count table with every key in `keys` having a count of 1. + ## creates a new count table with every key in `keys` having a count + ## of how many times it occurs in `keys`. result = newCountTable[A](rightSize(keys.len)) - for key in items(keys): result[key] = 1 + for key in items(keys): result.inc(key) proc `$`*[A](t: CountTableRef[A]): string = ## The `$` operator for count tables. @@ -1114,10 +1168,6 @@ proc `==`*[A](s, t: CountTableRef[A]): bool = elif isNil(t): result = false else: result = s[] == t[] -proc inc*[A](t: CountTableRef[A], key: A, val = 1) = - ## increments `t[key]` by `val`. - t[].inc(key, val) - proc smallest*[A](t: CountTableRef[A]): (A, int) = ## returns the (key,val)-pair with the smallest `val`. Efficiency: O(n) t[].smallest @@ -1266,7 +1316,7 @@ when isMainModule: #lib/pure/collections/tables.nim(117, 21) template/generic instantiation from here #lib/pure/collections/tableimpl.nim(32, 27) Error: undeclared field: 'hcode doAssert 0 == t.getOrDefault(testKey) - t.inc(testKey,3) + t.inc(testKey, 3) doAssert 3 == t.getOrDefault(testKey) block: @@ -1316,7 +1366,6 @@ when isMainModule: assert a == b assert a == c - block: #6250 let a = {3: 1}.toOrderedTable @@ -1332,6 +1381,23 @@ when isMainModule: assert((b == a) == true) block: # CountTable.smallest - var t = initCountTable[int]() - for v in items([0, 0, 5, 5, 5]): t.inc(v) + let t = toCountTable([0, 0, 5, 5, 5]) doAssert t.smallest == (0, 2) + + block: + var tp: Table[string, string] = initTable[string, string]() + doAssert "test1" == tp.getOrDefault("test1", "test1") + tp["test2"] = "test2" + doAssert "test2" == tp.getOrDefault("test2", "test1") + var tr: TableRef[string, string] = newTable[string, string]() + doAssert "test1" == tr.getOrDefault("test1", "test1") + tr["test2"] = "test2" + doAssert "test2" == tr.getOrDefault("test2", "test1") + var op: OrderedTable[string, string] = initOrderedTable[string, string]() + doAssert "test1" == op.getOrDefault("test1", "test1") + op["test2"] = "test2" + doAssert "test2" == op.getOrDefault("test2", "test1") + var orf: OrderedTableRef[string, string] = newOrderedTable[string, string]() + doAssert "test1" == orf.getOrDefault("test1", "test1") + orf["test2"] = "test2" + doAssert "test2" == orf.getOrDefault("test2", "test1") diff --git a/lib/pure/colors.nim b/lib/pure/colors.nim index 4ec76dee0..843f29a63 100644 --- a/lib/pure/colors.nim +++ b/lib/pure/colors.nim @@ -10,12 +10,11 @@ ## the ``graphics`` module. import strutils +from algorithm import binarySearch type Color* = distinct int ## a color stored as RGB -{.deprecated: [TColor: Color].} - proc `==` *(a, b: Color): bool {.borrow.} ## compares two colors. @@ -373,37 +372,28 @@ proc `$`*(c: Color): string = ## converts a color into its textual representation. Example: ``#00FF00``. result = '#' & toHex(int(c), 6) -proc binaryStrSearch(x: openArray[tuple[name: string, col: Color]], - y: string): int = - var a = 0 - var b = len(x) - 1 - while a <= b: - var mid = (a + b) div 2 - var c = cmp(x[mid].name, y) - if c < 0: a = mid + 1 - elif c > 0: b = mid - 1 - else: return mid - result = - 1 +proc colorNameCmp(x: tuple[name: string, col: Color], y: string): int = + result = cmpIgnoreCase(x.name, y) proc parseColor*(name: string): Color = ## parses `name` to a color value. If no valid color could be - ## parsed ``EInvalidValue`` is raised. + ## parsed ``EInvalidValue`` is raised. Case insensitive. if name[0] == '#': result = Color(parseHexInt(name)) else: - var idx = binaryStrSearch(colorNames, name) + var idx = binarySearch(colorNames, name, colorNameCmp) if idx < 0: raise newException(ValueError, "unknown color: " & name) result = colorNames[idx][1] proc isColor*(name: string): bool = ## returns true if `name` is a known color name or a hexadecimal color - ## prefixed with ``#``. + ## prefixed with ``#``. Case insensitive. if name[0] == '#': for i in 1 .. name.len-1: if name[i] notin {'0'..'9', 'a'..'f', 'A'..'F'}: return false result = true else: - result = binaryStrSearch(colorNames, name) >= 0 + result = binarySearch(colorNames, name, colorNameCmp) >= 0 proc rgb*(r, g, b: range[0..255]): Color = ## constructs a color from RGB values. diff --git a/lib/pure/complex.nim b/lib/pure/complex.nim index ccde5ee0a..ba5c571ce 100644 --- a/lib/pure/complex.nim +++ b/lib/pure/complex.nim @@ -25,7 +25,9 @@ type Complex* = tuple[re, im: float] ## a complex number, consisting of a real and an imaginary part -{.deprecated: [TComplex: Complex].} +const + im*: Complex = (re: 0.0, im: 1.0) + ## The imaginary unit. √-1. proc toComplex*(x: SomeInteger): Complex = ## Convert some integer ``x`` to a complex number. diff --git a/lib/pure/concurrency/cpuload.nim b/lib/pure/concurrency/cpuload.nim index db5f47407..1ec739485 100644 --- a/lib/pure/concurrency/cpuload.nim +++ b/lib/pure/concurrency/cpuload.nim @@ -60,17 +60,20 @@ proc advice*(s: var ThreadPoolState): ThreadPoolAdvice = proc fscanf(c: File, frmt: cstring) {.varargs, importc, header: "<stdio.h>".} - var f = open("/proc/loadavg") - var b: float - var busy, total: int - fscanf(f,"%lf %lf %lf %ld/%ld", - addr b, addr b, addr b, addr busy, addr total) - f.close() - let cpus = countProcessors() - if busy-1 < cpus: - result = doCreateThread - elif busy-1 >= cpus*2: - result = doShutdownThread + var f: File + if f.open("/proc/loadavg"): + var b: float + var busy, total: int + fscanf(f,"%lf %lf %lf %ld/%ld", + addr b, addr b, addr b, addr busy, addr total) + f.close() + let cpus = countProcessors() + if busy-1 < cpus: + result = doCreateThread + elif busy-1 >= cpus*2: + result = doShutdownThread + else: + result = doNothing else: result = doNothing else: diff --git a/lib/pure/concurrency/threadpool.nim b/lib/pure/concurrency/threadpool.nim index a5eaec86e..6ec71e912 100644 --- a/lib/pure/concurrency/threadpool.nim +++ b/lib/pure/concurrency/threadpool.nim @@ -168,6 +168,15 @@ proc wakeupWorkerToProcessQueue(w: ptr Worker) = signal(w.q.empty) signal(w.taskArrived) +proc attach(fv: FlowVarBase; i: int): bool = + acquire(fv.cv.L) + if fv.cv.counter <= 0: + fv.idx = i + result = true + else: + result = false + release(fv.cv.L) + proc finished(fv: FlowVarBase) = doAssert fv.ai.isNil, "flowVar is still attached to an 'awaitAny'" # we have to protect against the rare cases where the owner of the flowVar @@ -245,26 +254,27 @@ proc `^`*[T](fv: FlowVar[T]): T = proc awaitAny*(flowVars: openArray[FlowVarBase]): int = ## awaits any of the given flowVars. Returns the index of one flowVar for ## which a value arrived. A flowVar only supports one call to 'awaitAny' at - ## the same time. That means if you await([a,b]) and await([b,c]) the second + ## the same time. That means if you awaitAny([a,b]) and awaitAny([b,c]) the second ## call will only await 'c'. If there is no flowVar left to be able to wait ## on, -1 is returned. - ## **Note**: This results in non-deterministic behaviour and so should be - ## avoided. + ## **Note**: This results in non-deterministic behaviour and should be avoided. var ai: AwaitInfo ai.cv.initSemaphore() var conflicts = 0 + result = -1 for i in 0 .. flowVars.high: if cas(addr flowVars[i].ai, nil, addr ai): - flowVars[i].idx = i + if not attach(flowVars[i], i): + result = i + break else: inc conflicts if conflicts < flowVars.len: - await(ai.cv) - result = ai.idx + if result < 0: + await(ai.cv) + result = ai.idx for i in 0 .. flowVars.high: discard cas(addr flowVars[i].ai, addr ai, nil) - else: - result = -1 destroySemaphore(ai.cv) proc isReady*(fv: FlowVarBase): bool = @@ -321,7 +331,7 @@ proc slave(w: ptr Worker) {.thread.} = await(w.taskArrived) # XXX Somebody needs to look into this (why does this assertion fail # in Visual Studio?) - when not defined(vcc): assert(not w.ready) + when not defined(vcc) and not defined(tcc): assert(not w.ready) withLock numSlavesLock: inc numSlavesRunning diff --git a/lib/pure/cookies.nim b/lib/pure/cookies.nim index 8f16717ac..eaff86ae6 100644 --- a/lib/pure/cookies.nim +++ b/lib/pure/cookies.nim @@ -25,16 +25,16 @@ proc parseCookies*(s: string): StringTableRef = result = newStringTable(modeCaseInsensitive) var i = 0 while true: - while s[i] == ' ' or s[i] == '\t': inc(i) + while i < s.len and (s[i] == ' ' or s[i] == '\t'): inc(i) var keystart = i - while s[i] != '=' and s[i] != '\0': inc(i) + while i < s.len and s[i] != '=': inc(i) var keyend = i-1 - if s[i] == '\0': break + if i >= s.len: break inc(i) # skip '=' var valstart = i - while s[i] != ';' and s[i] != '\0': inc(i) + while i < s.len and s[i] != ';': inc(i) result[substr(s, keystart, keyend)] = substr(s, valstart, i-1) - if s[i] == '\0': break + if i >= s.len: break inc(i) # skip ';' proc setCookie*(key, value: string, domain = "", path = "", @@ -51,26 +51,26 @@ proc setCookie*(key, value: string, domain = "", path = "", if secure: result.add("; Secure") if httpOnly: result.add("; HttpOnly") -proc setCookie*(key, value: string, expires: DateTime, +proc setCookie*(key, value: string, expires: DateTime|Time, domain = "", path = "", noName = false, secure = false, httpOnly = false): string = ## Creates a command in the format of ## ``Set-Cookie: key=value; Domain=...; ...`` - ## - ## **Note:** UTC is assumed as the timezone for ``expires``. return setCookie(key, value, domain, path, - format(expires, "ddd',' dd MMM yyyy HH:mm:ss 'GMT'"), + format(expires.utc, "ddd',' dd MMM yyyy HH:mm:ss 'GMT'"), noname, secure, httpOnly) when isMainModule: - var tim = fromUnix(getTime().toUnix + 76 * (60 * 60 * 24)) + let expire = fromUnix(0) + 1.seconds - let cookie = setCookie("test", "value", tim.utc) - when not defined(testing): - echo cookie - let start = "Set-Cookie: test=value; Expires=" - assert cookie[0..start.high] == start + let cookies = [ + setCookie("test", "value", expire), + setCookie("test", "value", expire.local), + setCookie("test", "value", expire.utc) + ] + let expected = "Set-Cookie: test=value; Expires=Thu, 01 Jan 1970 00:00:01 GMT" + doAssert cookies == [expected, expected, expected] let table = parseCookies("uid=1; kp=2") - assert table["uid"] == "1" - assert table["kp"] == "2" + doAssert table["uid"] == "1" + doAssert table["kp"] == "2" diff --git a/lib/pure/cstrutils.nim b/lib/pure/cstrutils.nim index 437140892..fe9ceb68b 100644 --- a/lib/pure/cstrutils.nim +++ b/lib/pure/cstrutils.nim @@ -45,8 +45,10 @@ proc endsWith*(s, suffix: cstring): bool {.noSideEffect, proc cmpIgnoreStyle*(a, b: cstring): int {.noSideEffect, rtl, extern: "csuCmpIgnoreStyle".} = - ## Compares two strings normalized (i.e. case and - ## underscores do not matter). Returns: + ## Semantically the same as ``cmp(normalize($a), normalize($b))``. It + ## is just optimized to not allocate temporary strings. This should + ## NOT be used to compare Nim identifier names. use `macros.eqIdent` + ## for that. Returns: ## ## | 0 iff a == b ## | < 0 iff a < b diff --git a/lib/pure/db_common.nim b/lib/pure/db_common.nim index 957389605..f8689024b 100644 --- a/lib/pure/db_common.nim +++ b/lib/pure/db_common.nim @@ -83,9 +83,6 @@ type foreignKey*: bool ## is this a foreign key? DbColumns* = seq[DbColumn] -{.deprecated: [EDb: DbError, TSqlQuery: SqlQuery, FDb: DbEffect, - FReadDb: ReadDbEffect, FWriteDb: WriteDbEffect].} - template sql*(query: string): SqlQuery = ## constructs a SqlQuery from the string `query`. This is supposed to be ## used as a raw-string-literal modifier: diff --git a/lib/pure/dynlib.nim b/lib/pure/dynlib.nim index fda41dadb..5bd06f6fb 100644 --- a/lib/pure/dynlib.nim +++ b/lib/pure/dynlib.nim @@ -10,14 +10,55 @@ ## This module implements the ability to access symbols from shared ## libraries. On POSIX this uses the ``dlsym`` mechanism, on ## Windows ``LoadLibrary``. +## +## Examples +## -------- +## +## Loading a simple C function +## ^^^^^^^^^^^^^^^^^^^^^^^^^^^ +## +## The following example demonstrates loading a function called 'greet' +## from a library that is determined at runtime based upon a language choice. +## If the library fails to load or the function 'greet' is not found, +## it quits with a failure error code. +## +## .. code-block::nim +## +## import dynlib +## +## type +## greetFunction = proc(): cstring {.gcsafe, stdcall.} +## +## let lang = stdin.readLine() +## +## let lib = case lang +## of "french": +## loadLib("french.dll") +## else: +## loadLib("english.dll") +## +## if lib == nil: +## echo "Error loading library" +## quit(QuitFailure) +## +## let greet = cast[greetFunction](lib.symAddr("greet")) +## +## if greet == nil: +## echo "Error loading 'greet' function from library" +## quit(QuitFailure) +## +## let greeting = greet() +## +## echo greeting +## +## unloadLib(lib) +## import strutils type LibHandle* = pointer ## a handle to a dynamically loaded library -{.deprecated: [TLibHandle: LibHandle].} - proc loadLib*(path: string, global_symbols=false): LibHandle {.gcsafe.} ## loads a library from `path`. Returns nil if the library could not ## be loaded. diff --git a/lib/pure/encodings.nim b/lib/pure/encodings.nim index 5840d443d..c67cd7579 100644 --- a/lib/pure/encodings.nim +++ b/lib/pure/encodings.nim @@ -27,8 +27,6 @@ type EncodingError* = object of ValueError ## exception that is raised ## for encoding errors -{.deprecated: [EInvalidEncoding: EncodingError, PConverter: EncodingConverter].} - when defined(windows): proc eqEncodingNames(a, b: string): bool = var i = 0 @@ -36,7 +34,7 @@ when defined(windows): while i < a.len and j < b.len: if a[i] in {'-', '_'}: inc i if b[j] in {'-', '_'}: inc j - if a[i].toLower != b[j].toLower: return false + if i < a.len and j < b.len and a[i].toLowerAscii != b[j].toLowerAscii: return false inc i inc j result = i == a.len and j == b.len diff --git a/lib/pure/events.nim b/lib/pure/events.nim index 23a8a2c58..a39dbe3bf 100644 --- a/lib/pure/events.nim +++ b/lib/pure/events.nim @@ -43,9 +43,6 @@ type s: seq[EventHandler] EventError* = object of ValueError -{.deprecated: [TEventArgs: EventArgs, TEventHandler: EventHandler, - TEventEmitter: EventEmitter, EInvalidEvent: EventError].} - proc initEventHandler*(name: string): EventHandler = ## Initializes an EventHandler with the specified name and returns it. result.handlers = @[] diff --git a/lib/pure/fenv.nim b/lib/pure/fenv.nim index f8f115ecc..c946c4261 100644 --- a/lib/pure/fenv.nim +++ b/lib/pure/fenv.nim @@ -10,7 +10,7 @@ ## Floating-point environment. Handling of floating-point rounding and ## exceptions (overflow, division by zero, etc.). -{.deadCodeElim:on.} +{.deadCodeElim: on.} # dce option deprecated when defined(Posix) and not defined(haiku): {.passl: "-lm".} @@ -102,32 +102,33 @@ proc feupdateenv*(envp: ptr Tfenv): cint {.importc, header: "<fenv.h>".} ## represented by object pointed to by `envp` and raise exceptions ## according to saved exceptions. -var FP_RADIX_INTERNAL {. importc: "FLT_RADIX" header: "<float.h>" .} : int - -template fpRadix* : int = FP_RADIX_INTERNAL +const + FLT_RADIX = 2 ## the radix of the exponent representation + + FLT_MANT_DIG = 24 ## the number of base FLT_RADIX digits in the mantissa part of a float + FLT_DIG = 6 ## the number of digits of precision of a float + FLT_MIN_EXP = -125 # the minimum value of base FLT_RADIX in the exponent part of a float + FLT_MAX_EXP = 128 # the maximum value of base FLT_RADIX in the exponent part of a float + FLT_MIN_10_EXP = -37 ## the minimum value in base 10 of the exponent part of a float + FLT_MAX_10_EXP = 38 ## the maximum value in base 10 of the exponent part of a float + FLT_MIN = 1.17549435e-38'f32 ## the minimum value of a float + FLT_MAX = 3.40282347e+38'f32 ## the maximum value of a float + FLT_EPSILON = 1.19209290e-07'f32 ## the difference between 1 and the least value greater than 1 of a float + + DBL_MANT_DIG = 53 ## the number of base FLT_RADIX digits in the mantissa part of a double + DBL_DIG = 15 ## the number of digits of precision of a double + DBL_MIN_EXP = -1021 ## the minimum value of base FLT_RADIX in the exponent part of a double + DBL_MAX_EXP = 1024 ## the maximum value of base FLT_RADIX in the exponent part of a double + DBL_MIN_10_EXP = -307 ## the minimum value in base 10 of the exponent part of a double + DBL_MAX_10_EXP = 308 ## the maximum value in base 10 of the exponent part of a double + DBL_MIN = 2.2250738585072014E-308 ## the minimal value of a double + DBL_MAX = 1.7976931348623157E+308 ## the minimal value of a double + DBL_EPSILON = 2.2204460492503131E-16 ## the difference between 1 and the least value greater than 1 of a double + +template fpRadix* : int = FLT_RADIX ## The (integer) value of the radix used to represent any floating ## point type on the architecture used to build the program. -var FLT_MANT_DIG {. importc: "FLT_MANT_DIG" header: "<float.h>" .} : int -var FLT_DIG {. importc: "FLT_DIG" header: "<float.h>" .} : int -var FLT_MIN_EXP {. importc: "FLT_MIN_EXP" header: "<float.h>" .} : int -var FLT_MAX_EXP {. importc: "FLT_MAX_EXP" header: "<float.h>" .} : int -var FLT_MIN_10_EXP {. importc: "FLT_MIN_10_EXP" header: "<float.h>" .} : int -var FLT_MAX_10_EXP {. importc: "FLT_MAX_10_EXP" header: "<float.h>" .} : int -var FLT_MIN {. importc: "FLT_MIN" header: "<float.h>" .} : cfloat -var FLT_MAX {. importc: "FLT_MAX" header: "<float.h>" .} : cfloat -var FLT_EPSILON {. importc: "FLT_EPSILON" header: "<float.h>" .} : cfloat - -var DBL_MANT_DIG {. importc: "DBL_MANT_DIG" header: "<float.h>" .} : int -var DBL_DIG {. importc: "DBL_DIG" header: "<float.h>" .} : int -var DBL_MIN_EXP {. importc: "DBL_MIN_EXP" header: "<float.h>" .} : int -var DBL_MAX_EXP {. importc: "DBL_MAX_EXP" header: "<float.h>" .} : int -var DBL_MIN_10_EXP {. importc: "DBL_MIN_10_EXP" header: "<float.h>" .} : int -var DBL_MAX_10_EXP {. importc: "DBL_MAX_10_EXP" header: "<float.h>" .} : int -var DBL_MIN {. importc: "DBL_MIN" header: "<float.h>" .} : cdouble -var DBL_MAX {. importc: "DBL_MAX" header: "<float.h>" .} : cdouble -var DBL_EPSILON {. importc: "DBL_EPSILON" header: "<float.h>" .} : cdouble - template mantissaDigits*(T : typedesc[float32]) : int = FLT_MANT_DIG ## Number of digits (in base ``floatingPointRadix``) in the mantissa ## of 32-bit floating-point numbers. @@ -179,3 +180,11 @@ template maximumPositiveValue*(T : typedesc[float64]) : float64 = DBL_MAX template epsilon*(T : typedesc[float64]): float64 = DBL_EPSILON ## The difference between 1.0 and the smallest number greater than ## 1.0 that can be represented in a 64-bit floating-point type. + + +when isMainModule: + func is_significant(x: float): bool = + if x > minimumPositiveValue(float) and x < maximumPositiveValue(float): true + else: false + + doAssert is_significant(10.0) diff --git a/lib/pure/future.nim b/lib/pure/future.nim index 1a3ab688d..40dba7846 100644 --- a/lib/pure/future.nim +++ b/lib/pure/future.nim @@ -1,200 +1,4 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Dominik Picheta -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# -## This module implements experimental features which may soon be moved to -## the system module (or other more appropriate modules). +{.deprecated: "Use the new 'sugar' module instead".} -import macros - -proc createProcType(p, b: NimNode): NimNode {.compileTime.} = - #echo treeRepr(p) - #echo treeRepr(b) - result = newNimNode(nnkProcTy) - var formalParams = newNimNode(nnkFormalParams) - - formalParams.add b - - case p.kind - of nnkPar: - for i in 0 ..< p.len: - let ident = p[i] - var identDefs = newNimNode(nnkIdentDefs) - case ident.kind - of nnkExprColonExpr: - identDefs.add ident[0] - identDefs.add ident[1] - else: - identDefs.add newIdentNode("i" & $i) - identDefs.add(ident) - identDefs.add newEmptyNode() - formalParams.add identDefs - else: - var identDefs = newNimNode(nnkIdentDefs) - identDefs.add newIdentNode("i0") - identDefs.add(p) - identDefs.add newEmptyNode() - formalParams.add identDefs - - result.add formalParams - result.add newEmptyNode() - #echo(treeRepr(result)) - #echo(result.toStrLit()) - -macro `=>`*(p, b: untyped): untyped = - ## Syntax sugar for anonymous procedures. - ## - ## .. code-block:: nim - ## - ## proc passTwoAndTwo(f: (int, int) -> int): int = - ## f(2, 2) - ## - ## passTwoAndTwo((x, y) => x + y) # 4 - - #echo treeRepr(p) - #echo(treeRepr(b)) - var params: seq[NimNode] = @[newIdentNode("auto")] - - case p.kind - of nnkPar: - for c in children(p): - var identDefs = newNimNode(nnkIdentDefs) - case c.kind - of nnkExprColonExpr: - identDefs.add(c[0]) - identDefs.add(c[1]) - identDefs.add(newEmptyNode()) - of nnkIdent: - identDefs.add(c) - identDefs.add(newIdentNode("auto")) - identDefs.add(newEmptyNode()) - of nnkInfix: - if c[0].kind == nnkIdent and c[0].ident == !"->": - var procTy = createProcType(c[1], c[2]) - params[0] = procTy[0][0] - for i in 1 ..< procTy[0].len: - params.add(procTy[0][i]) - else: - error("Expected proc type (->) got (" & $c[0].ident & ").") - break - else: - echo treeRepr c - error("Incorrect procedure parameter list.") - params.add(identDefs) - of nnkIdent: - var identDefs = newNimNode(nnkIdentDefs) - identDefs.add(p) - identDefs.add(newIdentNode("auto")) - identDefs.add(newEmptyNode()) - params.add(identDefs) - of nnkInfix: - if p[0].kind == nnkIdent and p[0].ident == !"->": - var procTy = createProcType(p[1], p[2]) - params[0] = procTy[0][0] - for i in 1 ..< procTy[0].len: - params.add(procTy[0][i]) - else: - error("Expected proc type (->) got (" & $p[0].ident & ").") - else: - error("Incorrect procedure parameter list.") - result = newProc(params = params, body = b, procType = nnkLambda) - #echo(result.treeRepr) - #echo(result.toStrLit()) - #return result # TODO: Bug? - -macro `->`*(p, b: untyped): untyped = - ## Syntax sugar for procedure types. - ## - ## .. code-block:: nim - ## - ## proc pass2(f: (float, float) -> float): float = - ## f(2, 2) - ## - ## # is the same as: - ## - ## proc pass2(f: proc (x, y: float): float): float = - ## f(2, 2) - - result = createProcType(p, b) - -type ListComprehension = object -var lc*: ListComprehension - -macro `[]`*(lc: ListComprehension, comp, typ: untyped): untyped = - ## List comprehension, returns a sequence. `comp` is the actual list - ## comprehension, for example ``x | (x <- 1..10, x mod 2 == 0)``. `typ` is - ## the type that will be stored inside the result seq. - ## - ## .. code-block:: nim - ## - ## echo lc[x | (x <- 1..10, x mod 2 == 0), int] - ## - ## const n = 20 - ## echo lc[(x,y,z) | (x <- 1..n, y <- x..n, z <- y..n, x*x + y*y == z*z), - ## tuple[a,b,c: int]] - - expectLen(comp, 3) - expectKind(comp, nnkInfix) - expectKind(comp[0], nnkIdent) - assert($comp[0].ident == "|") - - result = newCall( - newDotExpr( - newIdentNode("result"), - newIdentNode("add")), - comp[1]) - - for i in countdown(comp[2].len-1, 0): - let x = comp[2][i] - expectMinLen(x, 1) - if x[0].kind == nnkIdent and $x[0].ident == "<-": - expectLen(x, 3) - result = newNimNode(nnkForStmt).add(x[1], x[2], result) - else: - result = newIfStmt((x, result)) - - result = newNimNode(nnkCall).add( - newNimNode(nnkPar).add( - newNimNode(nnkLambda).add( - newEmptyNode(), - newEmptyNode(), - newEmptyNode(), - newNimNode(nnkFormalParams).add( - newNimNode(nnkBracketExpr).add( - newIdentNode("seq"), - typ)), - newEmptyNode(), - newEmptyNode(), - newStmtList( - newAssignment( - newIdentNode("result"), - newNimNode(nnkPrefix).add( - newIdentNode("@"), - newNimNode(nnkBracket))), - result)))) - - -macro dump*(x: typed): untyped = - ## Dumps the content of an expression, useful for debugging. - ## It accepts any expression and prints a textual representation - ## of the tree representing the expression - as it would appear in - ## source code - together with the value of the expression. - ## - ## As an example, - ## - ## .. code-block:: nim - ## let - ## x = 10 - ## y = 20 - ## dump(x + y) - ## - ## will print ``x + y = 30``. - let s = x.toStrLit - let r = quote do: - debugEcho `s`, " = ", `x` - return r +include sugar diff --git a/lib/pure/hashes.nim b/lib/pure/hashes.nim index b015ed311..6535bb0d3 100644 --- a/lib/pure/hashes.nim +++ b/lib/pure/hashes.nim @@ -45,7 +45,6 @@ type Hash* = int ## a hash value; hash tables using these values should ## always have a size of a power of two and can use the ``and`` ## operator instead of ``mod`` for truncation of the hash value. -{.deprecated: [THash: Hash].} proc `!&`*(h: Hash, val: int): Hash {.inline.} = ## mixes a hash value `h` with `val` to produce a new hash value. This is diff --git a/lib/pure/htmlgen.nim b/lib/pure/htmlgen.nim index c0934a45b..55b02ea60 100644 --- a/lib/pure/htmlgen.nim +++ b/lib/pure/htmlgen.nim @@ -38,7 +38,8 @@ const proc getIdent(e: NimNode): string {.compileTime.} = case e.kind - of nnkIdent: result = normalize($e.ident) + of nnkIdent: + result = e.strVal.normalize of nnkAccQuoted: result = getIdent(e[0]) for i in 1 .. e.len-1: @@ -92,8 +93,10 @@ proc xmlCheckedTag*(e: NimNode, tag: string, optAttr = "", reqAttr = "", result.add(newStrLitNode("</")) result.add(newStrLitNode(tag)) result.add(newStrLitNode(">")) - result = nestList(!"&", result) - + when compiles(nestList(ident"&", result)): + result = nestList(ident"&", result) + else: + result = nestList(!"&", result) macro a*(e: varargs[untyped]): untyped = ## generates the HTML ``a`` element. diff --git a/lib/pure/htmlparser.nim b/lib/pure/htmlparser.nim index 00c622d74..9d4b57dc0 100644 --- a/lib/pure/htmlparser.nim +++ b/lib/pure/htmlparser.nim @@ -178,7 +178,6 @@ type tagVar, ## the HTML ``var`` element tagVideo, ## the HTML ``video`` element tagWbr ## the HTML ``wbr`` element -{.deprecated: [THtmlTag: HtmlTag].} const tagToStr* = [ @@ -218,79 +217,6 @@ const tagBr, tagCol, tagFrame, tagHr, tagImg, tagIsindex, tagLink, tagMeta, tagParam, tagWbr} - Entities = [ - ("nbsp", 0x00A0), ("iexcl", 0x00A1), ("cent", 0x00A2), ("pound", 0x00A3), - ("curren", 0x00A4), ("yen", 0x00A5), ("brvbar", 0x00A6), ("sect", 0x00A7), - ("uml", 0x00A8), ("copy", 0x00A9), ("ordf", 0x00AA), ("laquo", 0x00AB), - ("not", 0x00AC), ("shy", 0x00AD), ("reg", 0x00AE), ("macr", 0x00AF), - ("deg", 0x00B0), ("plusmn", 0x00B1), ("sup2", 0x00B2), ("sup3", 0x00B3), - ("acute", 0x00B4), ("micro", 0x00B5), ("para", 0x00B6), ("middot", 0x00B7), - ("cedil", 0x00B8), ("sup1", 0x00B9), ("ordm", 0x00BA), ("raquo", 0x00BB), - ("frac14", 0x00BC), ("frac12", 0x00BD), ("frac34", 0x00BE), - ("iquest", 0x00BF), ("Agrave", 0x00C0), ("Aacute", 0x00C1), - ("Acirc", 0x00C2), ("Atilde", 0x00C3), ("Auml", 0x00C4), ("Aring", 0x00C5), - ("AElig", 0x00C6), ("Ccedil", 0x00C7), ("Egrave", 0x00C8), - ("Eacute", 0x00C9), ("Ecirc", 0x00CA), ("Euml", 0x00CB), ("Igrave", 0x00CC), - ("Iacute", 0x00CD), ("Icirc", 0x00CE), ("Iuml", 0x00CF), ("ETH", 0x00D0), - ("Ntilde", 0x00D1), ("Ograve", 0x00D2), ("Oacute", 0x00D3), - ("Ocirc", 0x00D4), ("Otilde", 0x00D5), ("Ouml", 0x00D6), ("times", 0x00D7), - ("Oslash", 0x00D8), ("Ugrave", 0x00D9), ("Uacute", 0x00DA), - ("Ucirc", 0x00DB), ("Uuml", 0x00DC), ("Yacute", 0x00DD), ("THORN", 0x00DE), - ("szlig", 0x00DF), ("agrave", 0x00E0), ("aacute", 0x00E1), - ("acirc", 0x00E2), ("atilde", 0x00E3), ("auml", 0x00E4), ("aring", 0x00E5), - ("aelig", 0x00E6), ("ccedil", 0x00E7), ("egrave", 0x00E8), - ("eacute", 0x00E9), ("ecirc", 0x00EA), ("euml", 0x00EB), ("igrave", 0x00EC), - ("iacute", 0x00ED), ("icirc", 0x00EE), ("iuml", 0x00EF), ("eth", 0x00F0), - ("ntilde", 0x00F1), ("ograve", 0x00F2), ("oacute", 0x00F3), - ("ocirc", 0x00F4), ("otilde", 0x00F5), ("ouml", 0x00F6), ("divide", 0x00F7), - ("oslash", 0x00F8), ("ugrave", 0x00F9), ("uacute", 0x00FA), - ("ucirc", 0x00FB), ("uuml", 0x00FC), ("yacute", 0x00FD), ("thorn", 0x00FE), - ("yuml", 0x00FF), ("OElig", 0x0152), ("oelig", 0x0153), ("Scaron", 0x0160), - ("scaron", 0x0161), ("Yuml", 0x0178), ("fnof", 0x0192), ("circ", 0x02C6), - ("tilde", 0x02DC), ("Alpha", 0x0391), ("Beta", 0x0392), ("Gamma", 0x0393), - ("Delta", 0x0394), ("Epsilon", 0x0395), ("Zeta", 0x0396), ("Eta", 0x0397), - ("Theta", 0x0398), ("Iota", 0x0399), ("Kappa", 0x039A), ("Lambda", 0x039B), - ("Mu", 0x039C), ("Nu", 0x039D), ("Xi", 0x039E), ("Omicron", 0x039F), - ("Pi", 0x03A0), ("Rho", 0x03A1), ("Sigma", 0x03A3), ("Tau", 0x03A4), - ("Upsilon", 0x03A5), ("Phi", 0x03A6), ("Chi", 0x03A7), ("Psi", 0x03A8), - ("Omega", 0x03A9), ("alpha", 0x03B1), ("beta", 0x03B2), ("gamma", 0x03B3), - ("delta", 0x03B4), ("epsilon", 0x03B5), ("zeta", 0x03B6), ("eta", 0x03B7), - ("theta", 0x03B8), ("iota", 0x03B9), ("kappa", 0x03BA), ("lambda", 0x03BB), - ("mu", 0x03BC), ("nu", 0x03BD), ("xi", 0x03BE), ("omicron", 0x03BF), - ("pi", 0x03C0), ("rho", 0x03C1), ("sigmaf", 0x03C2), ("sigma", 0x03C3), - ("tau", 0x03C4), ("upsilon", 0x03C5), ("phi", 0x03C6), ("chi", 0x03C7), - ("psi", 0x03C8), ("omega", 0x03C9), ("thetasym", 0x03D1), ("upsih", 0x03D2), - ("piv", 0x03D6), ("ensp", 0x2002), ("emsp", 0x2003), ("thinsp", 0x2009), - ("zwnj", 0x200C), ("zwj", 0x200D), ("lrm", 0x200E), ("rlm", 0x200F), - ("ndash", 0x2013), ("mdash", 0x2014), ("lsquo", 0x2018), ("rsquo", 0x2019), - ("sbquo", 0x201A), ("ldquo", 0x201C), ("rdquo", 0x201D), ("bdquo", 0x201E), - ("dagger", 0x2020), ("Dagger", 0x2021), ("bull", 0x2022), - ("hellip", 0x2026), ("permil", 0x2030), ("prime", 0x2032), - ("Prime", 0x2033), ("lsaquo", 0x2039), ("rsaquo", 0x203A), - ("oline", 0x203E), ("frasl", 0x2044), ("euro", 0x20AC), - ("image", 0x2111), ("weierp", 0x2118), ("real", 0x211C), - ("trade", 0x2122), ("alefsym", 0x2135), ("larr", 0x2190), - ("uarr", 0x2191), ("rarr", 0x2192), ("darr", 0x2193), - ("harr", 0x2194), ("crarr", 0x21B5), ("lArr", 0x21D0), - ("uArr", 0x21D1), ("rArr", 0x21D2), ("dArr", 0x21D3), - ("hArr", 0x21D4), ("forall", 0x2200), ("part", 0x2202), - ("exist", 0x2203), ("empty", 0x2205), ("nabla", 0x2207), - ("isin", 0x2208), ("notin", 0x2209), ("ni", 0x220B), - ("prod", 0x220F), ("sum", 0x2211), ("minus", 0x2212), - ("lowast", 0x2217), ("radic", 0x221A), ("prop", 0x221D), - ("infin", 0x221E), ("ang", 0x2220), ("and", 0x2227), - ("or", 0x2228), ("cap", 0x2229), ("cup", 0x222A), - ("int", 0x222B), ("there4", 0x2234), ("sim", 0x223C), - ("cong", 0x2245), ("asymp", 0x2248), ("ne", 0x2260), - ("equiv", 0x2261), ("le", 0x2264), ("ge", 0x2265), - ("sub", 0x2282), ("sup", 0x2283), ("nsub", 0x2284), - ("sube", 0x2286), ("supe", 0x2287), ("oplus", 0x2295), - ("otimes", 0x2297), ("perp", 0x22A5), ("sdot", 0x22C5), - ("lceil", 0x2308), ("rceil", 0x2309), ("lfloor", 0x230A), - ("rfloor", 0x230B), ("lang", 0x2329), ("rang", 0x232A), - ("loz", 0x25CA), ("spades", 0x2660), ("clubs", 0x2663), - ("hearts", 0x2665), ("diams", 0x2666)] - proc allLower(s: string): bool = for c in s: if c < 'a' or c > 'z': return false @@ -425,24 +351,1541 @@ proc toHtmlTag(s: string): HtmlTag = proc htmlTag*(n: XmlNode): HtmlTag = - ## gets `n`'s tag as a ``HtmlTag``. + ## Gets `n`'s tag as a ``HtmlTag``. if n.clientData == 0: n.clientData = toHtmlTag(n.tag).ord result = HtmlTag(n.clientData) proc htmlTag*(s: string): HtmlTag = - ## converts `s` to a ``HtmlTag``. If `s` is no HTML tag, ``tagUnknown`` is + ## Converts `s` to a ``HtmlTag``. If `s` is no HTML tag, ``tagUnknown`` is ## returned. let s = if allLower(s): s else: toLowerAscii(s) result = toHtmlTag(s) +proc runeToEntity*(rune: Rune): string = + ## converts a Rune to its numeric HTML entity equivalent. + runnableExamples: + import unicode + doAssert runeToEntity(Rune(0)) == "" + doAssert runeToEntity(Rune(-1)) == "" + doAssert runeToEntity("Ü".runeAt(0)) == "#220" + doAssert runeToEntity("∈".runeAt(0)) == "#8712" + if rune.ord <= 0: result = "" + else: result = '#' & $rune.ord + +proc entityToRune*(entity: string): Rune = + ## Converts an HTML entity name like ``Ü`` or values like ``Ü`` + ## or ``Ü`` to its UTF-8 equivalent. + ## Rune(0) is returned if the entity name is unknown. + runnableExamples: + import unicode + doAssert entityToRune("") == Rune(0) + doAssert entityToRune("a") == Rune(0) + doAssert entityToRune("gt") == ">".runeAt(0) + doAssert entityToRune("Uuml") == "Ü".runeAt(0) + doAssert entityToRune("quest") == "?".runeAt(0) + doAssert entityToRune("#x0003F") == "?".runeAt(0) + if entity.len < 2: return # smallest entity has length 2 + if entity[0] == '#': + case entity[1] + of '0'..'9': + try: return Rune(parseInt(entity[1..^1])) + except: return + of 'x', 'X': # not case sensitive here + try: return Rune(parseHexInt(entity[2..^1])) + except: return + else: return # other entities are not defined with prefix ``#`` + case entity # entity names are case sensitive + of "Tab": result = Rune(0x00009) + of "NewLine": result = Rune(0x0000A) + of "excl": result = Rune(0x00021) + of "quot", "QUOT": result = Rune(0x00022) + of "num": result = Rune(0x00023) + of "dollar": result = Rune(0x00024) + of "percnt": result = Rune(0x00025) + of "amp", "AMP": result = Rune(0x00026) + of "apos": result = Rune(0x00027) + of "lpar": result = Rune(0x00028) + of "rpar": result = Rune(0x00029) + of "ast", "midast": result = Rune(0x0002A) + of "plus": result = Rune(0x0002B) + of "comma": result = Rune(0x0002C) + of "period": result = Rune(0x0002E) + of "sol": result = Rune(0x0002F) + of "colon": result = Rune(0x0003A) + of "semi": result = Rune(0x0003B) + of "lt", "LT": result = Rune(0x0003C) + of "equals": result = Rune(0x0003D) + of "gt", "GT": result = Rune(0x0003E) + of "quest": result = Rune(0x0003F) + of "commat": result = Rune(0x00040) + of "lsqb", "lbrack": result = Rune(0x0005B) + of "bsol": result = Rune(0x0005C) + of "rsqb", "rbrack": result = Rune(0x0005D) + of "Hat": result = Rune(0x0005E) + of "lowbar": result = Rune(0x0005F) + of "grave", "DiacriticalGrave": result = Rune(0x00060) + of "lcub", "lbrace": result = Rune(0x0007B) + of "verbar", "vert", "VerticalLine": result = Rune(0x0007C) + of "rcub", "rbrace": result = Rune(0x0007D) + of "nbsp", "NonBreakingSpace": result = Rune(0x000A0) + of "iexcl": result = Rune(0x000A1) + of "cent": result = Rune(0x000A2) + of "pound": result = Rune(0x000A3) + of "curren": result = Rune(0x000A4) + of "yen": result = Rune(0x000A5) + of "brvbar": result = Rune(0x000A6) + of "sect": result = Rune(0x000A7) + of "Dot", "die", "DoubleDot", "uml": result = Rune(0x000A8) + of "copy", "COPY": result = Rune(0x000A9) + of "ordf": result = Rune(0x000AA) + of "laquo": result = Rune(0x000AB) + of "not": result = Rune(0x000AC) + of "shy": result = Rune(0x000AD) + of "reg", "circledR", "REG": result = Rune(0x000AE) + of "macr", "OverBar", "strns": result = Rune(0x000AF) + of "deg": result = Rune(0x000B0) + of "plusmn", "pm", "PlusMinus": result = Rune(0x000B1) + of "sup2": result = Rune(0x000B2) + of "sup3": result = Rune(0x000B3) + of "acute", "DiacriticalAcute": result = Rune(0x000B4) + of "micro": result = Rune(0x000B5) + of "para": result = Rune(0x000B6) + of "middot", "centerdot", "CenterDot": result = Rune(0x000B7) + of "cedil", "Cedilla": result = Rune(0x000B8) + of "sup1": result = Rune(0x000B9) + of "ordm": result = Rune(0x000BA) + of "raquo": result = Rune(0x000BB) + of "frac14": result = Rune(0x000BC) + of "frac12", "half": result = Rune(0x000BD) + of "frac34": result = Rune(0x000BE) + of "iquest": result = Rune(0x000BF) + of "Agrave": result = Rune(0x000C0) + of "Aacute": result = Rune(0x000C1) + of "Acirc": result = Rune(0x000C2) + of "Atilde": result = Rune(0x000C3) + of "Auml": result = Rune(0x000C4) + of "Aring": result = Rune(0x000C5) + of "AElig": result = Rune(0x000C6) + of "Ccedil": result = Rune(0x000C7) + of "Egrave": result = Rune(0x000C8) + of "Eacute": result = Rune(0x000C9) + of "Ecirc": result = Rune(0x000CA) + of "Euml": result = Rune(0x000CB) + of "Igrave": result = Rune(0x000CC) + of "Iacute": result = Rune(0x000CD) + of "Icirc": result = Rune(0x000CE) + of "Iuml": result = Rune(0x000CF) + of "ETH": result = Rune(0x000D0) + of "Ntilde": result = Rune(0x000D1) + of "Ograve": result = Rune(0x000D2) + of "Oacute": result = Rune(0x000D3) + of "Ocirc": result = Rune(0x000D4) + of "Otilde": result = Rune(0x000D5) + of "Ouml": result = Rune(0x000D6) + of "times": result = Rune(0x000D7) + of "Oslash": result = Rune(0x000D8) + of "Ugrave": result = Rune(0x000D9) + of "Uacute": result = Rune(0x000DA) + of "Ucirc": result = Rune(0x000DB) + of "Uuml": result = Rune(0x000DC) + of "Yacute": result = Rune(0x000DD) + of "THORN": result = Rune(0x000DE) + of "szlig": result = Rune(0x000DF) + of "agrave": result = Rune(0x000E0) + of "aacute": result = Rune(0x000E1) + of "acirc": result = Rune(0x000E2) + of "atilde": result = Rune(0x000E3) + of "auml": result = Rune(0x000E4) + of "aring": result = Rune(0x000E5) + of "aelig": result = Rune(0x000E6) + of "ccedil": result = Rune(0x000E7) + of "egrave": result = Rune(0x000E8) + of "eacute": result = Rune(0x000E9) + of "ecirc": result = Rune(0x000EA) + of "euml": result = Rune(0x000EB) + of "igrave": result = Rune(0x000EC) + of "iacute": result = Rune(0x000ED) + of "icirc": result = Rune(0x000EE) + of "iuml": result = Rune(0x000EF) + of "eth": result = Rune(0x000F0) + of "ntilde": result = Rune(0x000F1) + of "ograve": result = Rune(0x000F2) + of "oacute": result = Rune(0x000F3) + of "ocirc": result = Rune(0x000F4) + of "otilde": result = Rune(0x000F5) + of "ouml": result = Rune(0x000F6) + of "divide", "div": result = Rune(0x000F7) + of "oslash": result = Rune(0x000F8) + of "ugrave": result = Rune(0x000F9) + of "uacute": result = Rune(0x000FA) + of "ucirc": result = Rune(0x000FB) + of "uuml": result = Rune(0x000FC) + of "yacute": result = Rune(0x000FD) + of "thorn": result = Rune(0x000FE) + of "yuml": result = Rune(0x000FF) + of "Amacr": result = Rune(0x00100) + of "amacr": result = Rune(0x00101) + of "Abreve": result = Rune(0x00102) + of "abreve": result = Rune(0x00103) + of "Aogon": result = Rune(0x00104) + of "aogon": result = Rune(0x00105) + of "Cacute": result = Rune(0x00106) + of "cacute": result = Rune(0x00107) + of "Ccirc": result = Rune(0x00108) + of "ccirc": result = Rune(0x00109) + of "Cdot": result = Rune(0x0010A) + of "cdot": result = Rune(0x0010B) + of "Ccaron": result = Rune(0x0010C) + of "ccaron": result = Rune(0x0010D) + of "Dcaron": result = Rune(0x0010E) + of "dcaron": result = Rune(0x0010F) + of "Dstrok": result = Rune(0x00110) + of "dstrok": result = Rune(0x00111) + of "Emacr": result = Rune(0x00112) + of "emacr": result = Rune(0x00113) + of "Edot": result = Rune(0x00116) + of "edot": result = Rune(0x00117) + of "Eogon": result = Rune(0x00118) + of "eogon": result = Rune(0x00119) + of "Ecaron": result = Rune(0x0011A) + of "ecaron": result = Rune(0x0011B) + of "Gcirc": result = Rune(0x0011C) + of "gcirc": result = Rune(0x0011D) + of "Gbreve": result = Rune(0x0011E) + of "gbreve": result = Rune(0x0011F) + of "Gdot": result = Rune(0x00120) + of "gdot": result = Rune(0x00121) + of "Gcedil": result = Rune(0x00122) + of "Hcirc": result = Rune(0x00124) + of "hcirc": result = Rune(0x00125) + of "Hstrok": result = Rune(0x00126) + of "hstrok": result = Rune(0x00127) + of "Itilde": result = Rune(0x00128) + of "itilde": result = Rune(0x00129) + of "Imacr": result = Rune(0x0012A) + of "imacr": result = Rune(0x0012B) + of "Iogon": result = Rune(0x0012E) + of "iogon": result = Rune(0x0012F) + of "Idot": result = Rune(0x00130) + of "imath", "inodot": result = Rune(0x00131) + of "IJlig": result = Rune(0x00132) + of "ijlig": result = Rune(0x00133) + of "Jcirc": result = Rune(0x00134) + of "jcirc": result = Rune(0x00135) + of "Kcedil": result = Rune(0x00136) + of "kcedil": result = Rune(0x00137) + of "kgreen": result = Rune(0x00138) + of "Lacute": result = Rune(0x00139) + of "lacute": result = Rune(0x0013A) + of "Lcedil": result = Rune(0x0013B) + of "lcedil": result = Rune(0x0013C) + of "Lcaron": result = Rune(0x0013D) + of "lcaron": result = Rune(0x0013E) + of "Lmidot": result = Rune(0x0013F) + of "lmidot": result = Rune(0x00140) + of "Lstrok": result = Rune(0x00141) + of "lstrok": result = Rune(0x00142) + of "Nacute": result = Rune(0x00143) + of "nacute": result = Rune(0x00144) + of "Ncedil": result = Rune(0x00145) + of "ncedil": result = Rune(0x00146) + of "Ncaron": result = Rune(0x00147) + of "ncaron": result = Rune(0x00148) + of "napos": result = Rune(0x00149) + of "ENG": result = Rune(0x0014A) + of "eng": result = Rune(0x0014B) + of "Omacr": result = Rune(0x0014C) + of "omacr": result = Rune(0x0014D) + of "Odblac": result = Rune(0x00150) + of "odblac": result = Rune(0x00151) + of "OElig": result = Rune(0x00152) + of "oelig": result = Rune(0x00153) + of "Racute": result = Rune(0x00154) + of "racute": result = Rune(0x00155) + of "Rcedil": result = Rune(0x00156) + of "rcedil": result = Rune(0x00157) + of "Rcaron": result = Rune(0x00158) + of "rcaron": result = Rune(0x00159) + of "Sacute": result = Rune(0x0015A) + of "sacute": result = Rune(0x0015B) + of "Scirc": result = Rune(0x0015C) + of "scirc": result = Rune(0x0015D) + of "Scedil": result = Rune(0x0015E) + of "scedil": result = Rune(0x0015F) + of "Scaron": result = Rune(0x00160) + of "scaron": result = Rune(0x00161) + of "Tcedil": result = Rune(0x00162) + of "tcedil": result = Rune(0x00163) + of "Tcaron": result = Rune(0x00164) + of "tcaron": result = Rune(0x00165) + of "Tstrok": result = Rune(0x00166) + of "tstrok": result = Rune(0x00167) + of "Utilde": result = Rune(0x00168) + of "utilde": result = Rune(0x00169) + of "Umacr": result = Rune(0x0016A) + of "umacr": result = Rune(0x0016B) + of "Ubreve": result = Rune(0x0016C) + of "ubreve": result = Rune(0x0016D) + of "Uring": result = Rune(0x0016E) + of "uring": result = Rune(0x0016F) + of "Udblac": result = Rune(0x00170) + of "udblac": result = Rune(0x00171) + of "Uogon": result = Rune(0x00172) + of "uogon": result = Rune(0x00173) + of "Wcirc": result = Rune(0x00174) + of "wcirc": result = Rune(0x00175) + of "Ycirc": result = Rune(0x00176) + of "ycirc": result = Rune(0x00177) + of "Yuml": result = Rune(0x00178) + of "Zacute": result = Rune(0x00179) + of "zacute": result = Rune(0x0017A) + of "Zdot": result = Rune(0x0017B) + of "zdot": result = Rune(0x0017C) + of "Zcaron": result = Rune(0x0017D) + of "zcaron": result = Rune(0x0017E) + of "fnof": result = Rune(0x00192) + of "imped": result = Rune(0x001B5) + of "gacute": result = Rune(0x001F5) + of "jmath": result = Rune(0x00237) + of "circ": result = Rune(0x002C6) + of "caron", "Hacek": result = Rune(0x002C7) + of "breve", "Breve": result = Rune(0x002D8) + of "dot", "DiacriticalDot": result = Rune(0x002D9) + of "ring": result = Rune(0x002DA) + of "ogon": result = Rune(0x002DB) + of "tilde", "DiacriticalTilde": result = Rune(0x002DC) + of "dblac", "DiacriticalDoubleAcute": result = Rune(0x002DD) + of "DownBreve": result = Rune(0x00311) + of "UnderBar": result = Rune(0x00332) + of "Alpha": result = Rune(0x00391) + of "Beta": result = Rune(0x00392) + of "Gamma": result = Rune(0x00393) + of "Delta": result = Rune(0x00394) + of "Epsilon": result = Rune(0x00395) + of "Zeta": result = Rune(0x00396) + of "Eta": result = Rune(0x00397) + of "Theta": result = Rune(0x00398) + of "Iota": result = Rune(0x00399) + of "Kappa": result = Rune(0x0039A) + of "Lambda": result = Rune(0x0039B) + of "Mu": result = Rune(0x0039C) + of "Nu": result = Rune(0x0039D) + of "Xi": result = Rune(0x0039E) + of "Omicron": result = Rune(0x0039F) + of "Pi": result = Rune(0x003A0) + of "Rho": result = Rune(0x003A1) + of "Sigma": result = Rune(0x003A3) + of "Tau": result = Rune(0x003A4) + of "Upsilon": result = Rune(0x003A5) + of "Phi": result = Rune(0x003A6) + of "Chi": result = Rune(0x003A7) + of "Psi": result = Rune(0x003A8) + of "Omega": result = Rune(0x003A9) + of "alpha": result = Rune(0x003B1) + of "beta": result = Rune(0x003B2) + of "gamma": result = Rune(0x003B3) + of "delta": result = Rune(0x003B4) + of "epsiv", "varepsilon", "epsilon": result = Rune(0x003B5) + of "zeta": result = Rune(0x003B6) + of "eta": result = Rune(0x003B7) + of "theta": result = Rune(0x003B8) + of "iota": result = Rune(0x003B9) + of "kappa": result = Rune(0x003BA) + of "lambda": result = Rune(0x003BB) + of "mu": result = Rune(0x003BC) + of "nu": result = Rune(0x003BD) + of "xi": result = Rune(0x003BE) + of "omicron": result = Rune(0x003BF) + of "pi": result = Rune(0x003C0) + of "rho": result = Rune(0x003C1) + of "sigmav", "varsigma", "sigmaf": result = Rune(0x003C2) + of "sigma": result = Rune(0x003C3) + of "tau": result = Rune(0x003C4) + of "upsi", "upsilon": result = Rune(0x003C5) + of "phi", "phiv", "varphi": result = Rune(0x003C6) + of "chi": result = Rune(0x003C7) + of "psi": result = Rune(0x003C8) + of "omega": result = Rune(0x003C9) + of "thetav", "vartheta", "thetasym": result = Rune(0x003D1) + of "Upsi", "upsih": result = Rune(0x003D2) + of "straightphi": result = Rune(0x003D5) + of "piv", "varpi": result = Rune(0x003D6) + of "Gammad": result = Rune(0x003DC) + of "gammad", "digamma": result = Rune(0x003DD) + of "kappav", "varkappa": result = Rune(0x003F0) + of "rhov", "varrho": result = Rune(0x003F1) + of "epsi", "straightepsilon": result = Rune(0x003F5) + of "bepsi", "backepsilon": result = Rune(0x003F6) + of "IOcy": result = Rune(0x00401) + of "DJcy": result = Rune(0x00402) + of "GJcy": result = Rune(0x00403) + of "Jukcy": result = Rune(0x00404) + of "DScy": result = Rune(0x00405) + of "Iukcy": result = Rune(0x00406) + of "YIcy": result = Rune(0x00407) + of "Jsercy": result = Rune(0x00408) + of "LJcy": result = Rune(0x00409) + of "NJcy": result = Rune(0x0040A) + of "TSHcy": result = Rune(0x0040B) + of "KJcy": result = Rune(0x0040C) + of "Ubrcy": result = Rune(0x0040E) + of "DZcy": result = Rune(0x0040F) + of "Acy": result = Rune(0x00410) + of "Bcy": result = Rune(0x00411) + of "Vcy": result = Rune(0x00412) + of "Gcy": result = Rune(0x00413) + of "Dcy": result = Rune(0x00414) + of "IEcy": result = Rune(0x00415) + of "ZHcy": result = Rune(0x00416) + of "Zcy": result = Rune(0x00417) + of "Icy": result = Rune(0x00418) + of "Jcy": result = Rune(0x00419) + of "Kcy": result = Rune(0x0041A) + of "Lcy": result = Rune(0x0041B) + of "Mcy": result = Rune(0x0041C) + of "Ncy": result = Rune(0x0041D) + of "Ocy": result = Rune(0x0041E) + of "Pcy": result = Rune(0x0041F) + of "Rcy": result = Rune(0x00420) + of "Scy": result = Rune(0x00421) + of "Tcy": result = Rune(0x00422) + of "Ucy": result = Rune(0x00423) + of "Fcy": result = Rune(0x00424) + of "KHcy": result = Rune(0x00425) + of "TScy": result = Rune(0x00426) + of "CHcy": result = Rune(0x00427) + of "SHcy": result = Rune(0x00428) + of "SHCHcy": result = Rune(0x00429) + of "HARDcy": result = Rune(0x0042A) + of "Ycy": result = Rune(0x0042B) + of "SOFTcy": result = Rune(0x0042C) + of "Ecy": result = Rune(0x0042D) + of "YUcy": result = Rune(0x0042E) + of "YAcy": result = Rune(0x0042F) + of "acy": result = Rune(0x00430) + of "bcy": result = Rune(0x00431) + of "vcy": result = Rune(0x00432) + of "gcy": result = Rune(0x00433) + of "dcy": result = Rune(0x00434) + of "iecy": result = Rune(0x00435) + of "zhcy": result = Rune(0x00436) + of "zcy": result = Rune(0x00437) + of "icy": result = Rune(0x00438) + of "jcy": result = Rune(0x00439) + of "kcy": result = Rune(0x0043A) + of "lcy": result = Rune(0x0043B) + of "mcy": result = Rune(0x0043C) + of "ncy": result = Rune(0x0043D) + of "ocy": result = Rune(0x0043E) + of "pcy": result = Rune(0x0043F) + of "rcy": result = Rune(0x00440) + of "scy": result = Rune(0x00441) + of "tcy": result = Rune(0x00442) + of "ucy": result = Rune(0x00443) + of "fcy": result = Rune(0x00444) + of "khcy": result = Rune(0x00445) + of "tscy": result = Rune(0x00446) + of "chcy": result = Rune(0x00447) + of "shcy": result = Rune(0x00448) + of "shchcy": result = Rune(0x00449) + of "hardcy": result = Rune(0x0044A) + of "ycy": result = Rune(0x0044B) + of "softcy": result = Rune(0x0044C) + of "ecy": result = Rune(0x0044D) + of "yucy": result = Rune(0x0044E) + of "yacy": result = Rune(0x0044F) + of "iocy": result = Rune(0x00451) + of "djcy": result = Rune(0x00452) + of "gjcy": result = Rune(0x00453) + of "jukcy": result = Rune(0x00454) + of "dscy": result = Rune(0x00455) + of "iukcy": result = Rune(0x00456) + of "yicy": result = Rune(0x00457) + of "jsercy": result = Rune(0x00458) + of "ljcy": result = Rune(0x00459) + of "njcy": result = Rune(0x0045A) + of "tshcy": result = Rune(0x0045B) + of "kjcy": result = Rune(0x0045C) + of "ubrcy": result = Rune(0x0045E) + of "dzcy": result = Rune(0x0045F) + of "ensp": result = Rune(0x02002) + of "emsp": result = Rune(0x02003) + of "emsp13": result = Rune(0x02004) + of "emsp14": result = Rune(0x02005) + of "numsp": result = Rune(0x02007) + of "puncsp": result = Rune(0x02008) + of "thinsp", "ThinSpace": result = Rune(0x02009) + of "hairsp", "VeryThinSpace": result = Rune(0x0200A) + of "ZeroWidthSpace", "NegativeVeryThinSpace", "NegativeThinSpace", + "NegativeMediumSpace", "NegativeThickSpace": result = Rune(0x0200B) + of "zwnj": result = Rune(0x0200C) + of "zwj": result = Rune(0x0200D) + of "lrm": result = Rune(0x0200E) + of "rlm": result = Rune(0x0200F) + of "hyphen", "dash": result = Rune(0x02010) + of "ndash": result = Rune(0x02013) + of "mdash": result = Rune(0x02014) + of "horbar": result = Rune(0x02015) + of "Verbar", "Vert": result = Rune(0x02016) + of "lsquo", "OpenCurlyQuote": result = Rune(0x02018) + of "rsquo", "rsquor", "CloseCurlyQuote": result = Rune(0x02019) + of "lsquor", "sbquo": result = Rune(0x0201A) + of "ldquo", "OpenCurlyDoubleQuote": result = Rune(0x0201C) + of "rdquo", "rdquor", "CloseCurlyDoubleQuote": result = Rune(0x0201D) + of "ldquor", "bdquo": result = Rune(0x0201E) + of "dagger": result = Rune(0x02020) + of "Dagger", "ddagger": result = Rune(0x02021) + of "bull", "bullet": result = Rune(0x02022) + of "nldr": result = Rune(0x02025) + of "hellip", "mldr": result = Rune(0x02026) + of "permil": result = Rune(0x02030) + of "pertenk": result = Rune(0x02031) + of "prime": result = Rune(0x02032) + of "Prime": result = Rune(0x02033) + of "tprime": result = Rune(0x02034) + of "bprime", "backprime": result = Rune(0x02035) + of "lsaquo": result = Rune(0x02039) + of "rsaquo": result = Rune(0x0203A) + of "oline": result = Rune(0x0203E) + of "caret": result = Rune(0x02041) + of "hybull": result = Rune(0x02043) + of "frasl": result = Rune(0x02044) + of "bsemi": result = Rune(0x0204F) + of "qprime": result = Rune(0x02057) + of "MediumSpace": result = Rune(0x0205F) + of "NoBreak": result = Rune(0x02060) + of "ApplyFunction", "af": result = Rune(0x02061) + of "InvisibleTimes", "it": result = Rune(0x02062) + of "InvisibleComma", "ic": result = Rune(0x02063) + of "euro": result = Rune(0x020AC) + of "tdot", "TripleDot": result = Rune(0x020DB) + of "DotDot": result = Rune(0x020DC) + of "Copf", "complexes": result = Rune(0x02102) + of "incare": result = Rune(0x02105) + of "gscr": result = Rune(0x0210A) + of "hamilt", "HilbertSpace", "Hscr": result = Rune(0x0210B) + of "Hfr", "Poincareplane": result = Rune(0x0210C) + of "quaternions", "Hopf": result = Rune(0x0210D) + of "planckh": result = Rune(0x0210E) + of "planck", "hbar", "plankv", "hslash": result = Rune(0x0210F) + of "Iscr", "imagline": result = Rune(0x02110) + of "image", "Im", "imagpart", "Ifr": result = Rune(0x02111) + of "Lscr", "lagran", "Laplacetrf": result = Rune(0x02112) + of "ell": result = Rune(0x02113) + of "Nopf", "naturals": result = Rune(0x02115) + of "numero": result = Rune(0x02116) + of "copysr": result = Rune(0x02117) + of "weierp", "wp": result = Rune(0x02118) + of "Popf", "primes": result = Rune(0x02119) + of "rationals", "Qopf": result = Rune(0x0211A) + of "Rscr", "realine": result = Rune(0x0211B) + of "real", "Re", "realpart", "Rfr": result = Rune(0x0211C) + of "reals", "Ropf": result = Rune(0x0211D) + of "rx": result = Rune(0x0211E) + of "trade", "TRADE": result = Rune(0x02122) + of "integers", "Zopf": result = Rune(0x02124) + of "ohm": result = Rune(0x02126) + of "mho": result = Rune(0x02127) + of "Zfr", "zeetrf": result = Rune(0x02128) + of "iiota": result = Rune(0x02129) + of "angst": result = Rune(0x0212B) + of "bernou", "Bernoullis", "Bscr": result = Rune(0x0212C) + of "Cfr", "Cayleys": result = Rune(0x0212D) + of "escr": result = Rune(0x0212F) + of "Escr", "expectation": result = Rune(0x02130) + of "Fscr", "Fouriertrf": result = Rune(0x02131) + of "phmmat", "Mellintrf", "Mscr": result = Rune(0x02133) + of "order", "orderof", "oscr": result = Rune(0x02134) + of "alefsym", "aleph": result = Rune(0x02135) + of "beth": result = Rune(0x02136) + of "gimel": result = Rune(0x02137) + of "daleth": result = Rune(0x02138) + of "CapitalDifferentialD", "DD": result = Rune(0x02145) + of "DifferentialD", "dd": result = Rune(0x02146) + of "ExponentialE", "exponentiale", "ee": result = Rune(0x02147) + of "ImaginaryI", "ii": result = Rune(0x02148) + of "frac13": result = Rune(0x02153) + of "frac23": result = Rune(0x02154) + of "frac15": result = Rune(0x02155) + of "frac25": result = Rune(0x02156) + of "frac35": result = Rune(0x02157) + of "frac45": result = Rune(0x02158) + of "frac16": result = Rune(0x02159) + of "frac56": result = Rune(0x0215A) + of "frac18": result = Rune(0x0215B) + of "frac38": result = Rune(0x0215C) + of "frac58": result = Rune(0x0215D) + of "frac78": result = Rune(0x0215E) + of "larr", "leftarrow", "LeftArrow", "slarr", + "ShortLeftArrow": result = Rune(0x02190) + of "uarr", "uparrow", "UpArrow", "ShortUpArrow": result = Rune(0x02191) + of "rarr", "rightarrow", "RightArrow", "srarr", + "ShortRightArrow": result = Rune(0x02192) + of "darr", "downarrow", "DownArrow", + "ShortDownArrow": result = Rune(0x02193) + of "harr", "leftrightarrow", "LeftRightArrow": result = Rune(0x02194) + of "varr", "updownarrow", "UpDownArrow": result = Rune(0x02195) + of "nwarr", "UpperLeftArrow", "nwarrow": result = Rune(0x02196) + of "nearr", "UpperRightArrow", "nearrow": result = Rune(0x02197) + of "searr", "searrow", "LowerRightArrow": result = Rune(0x02198) + of "swarr", "swarrow", "LowerLeftArrow": result = Rune(0x02199) + of "nlarr", "nleftarrow": result = Rune(0x0219A) + of "nrarr", "nrightarrow": result = Rune(0x0219B) + of "rarrw", "rightsquigarrow": result = Rune(0x0219D) + of "Larr", "twoheadleftarrow": result = Rune(0x0219E) + of "Uarr": result = Rune(0x0219F) + of "Rarr", "twoheadrightarrow": result = Rune(0x021A0) + of "Darr": result = Rune(0x021A1) + of "larrtl", "leftarrowtail": result = Rune(0x021A2) + of "rarrtl", "rightarrowtail": result = Rune(0x021A3) + of "LeftTeeArrow", "mapstoleft": result = Rune(0x021A4) + of "UpTeeArrow", "mapstoup": result = Rune(0x021A5) + of "map", "RightTeeArrow", "mapsto": result = Rune(0x021A6) + of "DownTeeArrow", "mapstodown": result = Rune(0x021A7) + of "larrhk", "hookleftarrow": result = Rune(0x021A9) + of "rarrhk", "hookrightarrow": result = Rune(0x021AA) + of "larrlp", "looparrowleft": result = Rune(0x021AB) + of "rarrlp", "looparrowright": result = Rune(0x021AC) + of "harrw", "leftrightsquigarrow": result = Rune(0x021AD) + of "nharr", "nleftrightarrow": result = Rune(0x021AE) + of "lsh", "Lsh": result = Rune(0x021B0) + of "rsh", "Rsh": result = Rune(0x021B1) + of "ldsh": result = Rune(0x021B2) + of "rdsh": result = Rune(0x021B3) + of "crarr": result = Rune(0x021B5) + of "cularr", "curvearrowleft": result = Rune(0x021B6) + of "curarr", "curvearrowright": result = Rune(0x021B7) + of "olarr", "circlearrowleft": result = Rune(0x021BA) + of "orarr", "circlearrowright": result = Rune(0x021BB) + of "lharu", "LeftVector", "leftharpoonup": result = Rune(0x021BC) + of "lhard", "leftharpoondown", "DownLeftVector": result = Rune(0x021BD) + of "uharr", "upharpoonright", "RightUpVector": result = Rune(0x021BE) + of "uharl", "upharpoonleft", "LeftUpVector": result = Rune(0x021BF) + of "rharu", "RightVector", "rightharpoonup": result = Rune(0x021C0) + of "rhard", "rightharpoondown", "DownRightVector": result = Rune(0x021C1) + of "dharr", "RightDownVector", "downharpoonright": result = Rune(0x021C2) + of "dharl", "LeftDownVector", "downharpoonleft": result = Rune(0x021C3) + of "rlarr", "rightleftarrows", "RightArrowLeftArrow": result = Rune(0x021C4) + of "udarr", "UpArrowDownArrow": result = Rune(0x021C5) + of "lrarr", "leftrightarrows", "LeftArrowRightArrow": result = Rune(0x021C6) + of "llarr", "leftleftarrows": result = Rune(0x021C7) + of "uuarr", "upuparrows": result = Rune(0x021C8) + of "rrarr", "rightrightarrows": result = Rune(0x021C9) + of "ddarr", "downdownarrows": result = Rune(0x021CA) + of "lrhar", "ReverseEquilibrium", + "leftrightharpoons": result = Rune(0x021CB) + of "rlhar", "rightleftharpoons", "Equilibrium": result = Rune(0x021CC) + of "nlArr", "nLeftarrow": result = Rune(0x021CD) + of "nhArr", "nLeftrightarrow": result = Rune(0x021CE) + of "nrArr", "nRightarrow": result = Rune(0x021CF) + of "lArr", "Leftarrow", "DoubleLeftArrow": result = Rune(0x021D0) + of "uArr", "Uparrow", "DoubleUpArrow": result = Rune(0x021D1) + of "rArr", "Rightarrow", "Implies", + "DoubleRightArrow": result = Rune(0x021D2) + of "dArr", "Downarrow", "DoubleDownArrow": result = Rune(0x021D3) + of "hArr", "Leftrightarrow", "DoubleLeftRightArrow", + "iff": result = Rune(0x021D4) + of "vArr", "Updownarrow", "DoubleUpDownArrow": result = Rune(0x021D5) + of "nwArr": result = Rune(0x021D6) + of "neArr": result = Rune(0x021D7) + of "seArr": result = Rune(0x021D8) + of "swArr": result = Rune(0x021D9) + of "lAarr", "Lleftarrow": result = Rune(0x021DA) + of "rAarr", "Rrightarrow": result = Rune(0x021DB) + of "zigrarr": result = Rune(0x021DD) + of "larrb", "LeftArrowBar": result = Rune(0x021E4) + of "rarrb", "RightArrowBar": result = Rune(0x021E5) + of "duarr", "DownArrowUpArrow": result = Rune(0x021F5) + of "loarr": result = Rune(0x021FD) + of "roarr": result = Rune(0x021FE) + of "hoarr": result = Rune(0x021FF) + of "forall", "ForAll": result = Rune(0x02200) + of "comp", "complement": result = Rune(0x02201) + of "part", "PartialD": result = Rune(0x02202) + of "exist", "Exists": result = Rune(0x02203) + of "nexist", "NotExists", "nexists": result = Rune(0x02204) + of "empty", "emptyset", "emptyv", "varnothing": result = Rune(0x02205) + of "nabla", "Del": result = Rune(0x02207) + of "isin", "isinv", "Element", "in": result = Rune(0x02208) + of "notin", "NotElement", "notinva": result = Rune(0x02209) + of "niv", "ReverseElement", "ni", "SuchThat": result = Rune(0x0220B) + of "notni", "notniva", "NotReverseElement": result = Rune(0x0220C) + of "prod", "Product": result = Rune(0x0220F) + of "coprod", "Coproduct": result = Rune(0x02210) + of "sum", "Sum": result = Rune(0x02211) + of "minus": result = Rune(0x02212) + of "mnplus", "mp", "MinusPlus": result = Rune(0x02213) + of "plusdo", "dotplus": result = Rune(0x02214) + of "setmn", "setminus", "Backslash", "ssetmn", + "smallsetminus": result = Rune(0x02216) + of "lowast": result = Rune(0x02217) + of "compfn", "SmallCircle": result = Rune(0x02218) + of "radic", "Sqrt": result = Rune(0x0221A) + of "prop", "propto", "Proportional", "vprop", + "varpropto": result = Rune(0x0221D) + of "infin": result = Rune(0x0221E) + of "angrt": result = Rune(0x0221F) + of "ang", "angle": result = Rune(0x02220) + of "angmsd", "measuredangle": result = Rune(0x02221) + of "angsph": result = Rune(0x02222) + of "mid", "VerticalBar", "smid", "shortmid": result = Rune(0x02223) + of "nmid", "NotVerticalBar", "nsmid", "nshortmid": result = Rune(0x02224) + of "par", "parallel", "DoubleVerticalBar", "spar", + "shortparallel": result = Rune(0x02225) + of "npar", "nparallel", "NotDoubleVerticalBar", "nspar", + "nshortparallel": result = Rune(0x02226) + of "and", "wedge": result = Rune(0x02227) + of "or", "vee": result = Rune(0x02228) + of "cap": result = Rune(0x02229) + of "cup": result = Rune(0x0222A) + of "int", "Integral": result = Rune(0x0222B) + of "Int": result = Rune(0x0222C) + of "tint", "iiint": result = Rune(0x0222D) + of "conint", "oint", "ContourIntegral": result = Rune(0x0222E) + of "Conint", "DoubleContourIntegral": result = Rune(0x0222F) + of "Cconint": result = Rune(0x02230) + of "cwint": result = Rune(0x02231) + of "cwconint", "ClockwiseContourIntegral": result = Rune(0x02232) + of "awconint", "CounterClockwiseContourIntegral": result = Rune(0x02233) + of "there4", "therefore", "Therefore": result = Rune(0x02234) + of "becaus", "because", "Because": result = Rune(0x02235) + of "ratio": result = Rune(0x02236) + of "Colon", "Proportion": result = Rune(0x02237) + of "minusd", "dotminus": result = Rune(0x02238) + of "mDDot": result = Rune(0x0223A) + of "homtht": result = Rune(0x0223B) + of "sim", "Tilde", "thksim", "thicksim": result = Rune(0x0223C) + of "bsim", "backsim": result = Rune(0x0223D) + of "ac", "mstpos": result = Rune(0x0223E) + of "acd": result = Rune(0x0223F) + of "wreath", "VerticalTilde", "wr": result = Rune(0x02240) + of "nsim", "NotTilde": result = Rune(0x02241) + of "esim", "EqualTilde", "eqsim": result = Rune(0x02242) + of "sime", "TildeEqual", "simeq": result = Rune(0x02243) + of "nsime", "nsimeq", "NotTildeEqual": result = Rune(0x02244) + of "cong", "TildeFullEqual": result = Rune(0x02245) + of "simne": result = Rune(0x02246) + of "ncong", "NotTildeFullEqual": result = Rune(0x02247) + of "asymp", "ap", "TildeTilde", "approx", "thkap", + "thickapprox": result = Rune(0x02248) + of "nap", "NotTildeTilde", "napprox": result = Rune(0x02249) + of "ape", "approxeq": result = Rune(0x0224A) + of "apid": result = Rune(0x0224B) + of "bcong", "backcong": result = Rune(0x0224C) + of "asympeq", "CupCap": result = Rune(0x0224D) + of "bump", "HumpDownHump", "Bumpeq": result = Rune(0x0224E) + of "bumpe", "HumpEqual", "bumpeq": result = Rune(0x0224F) + of "esdot", "DotEqual", "doteq": result = Rune(0x02250) + of "eDot", "doteqdot": result = Rune(0x02251) + of "efDot", "fallingdotseq": result = Rune(0x02252) + of "erDot", "risingdotseq": result = Rune(0x02253) + of "colone", "coloneq", "Assign": result = Rune(0x02254) + of "ecolon", "eqcolon": result = Rune(0x02255) + of "ecir", "eqcirc": result = Rune(0x02256) + of "cire", "circeq": result = Rune(0x02257) + of "wedgeq": result = Rune(0x02259) + of "veeeq": result = Rune(0x0225A) + of "trie", "triangleq": result = Rune(0x0225C) + of "equest", "questeq": result = Rune(0x0225F) + of "ne", "NotEqual": result = Rune(0x02260) + of "equiv", "Congruent": result = Rune(0x02261) + of "nequiv", "NotCongruent": result = Rune(0x02262) + of "le", "leq": result = Rune(0x02264) + of "ge", "GreaterEqual", "geq": result = Rune(0x02265) + of "lE", "LessFullEqual", "leqq": result = Rune(0x02266) + of "gE", "GreaterFullEqual", "geqq": result = Rune(0x02267) + of "lnE", "lneqq": result = Rune(0x02268) + of "gnE", "gneqq": result = Rune(0x02269) + of "Lt", "NestedLessLess", "ll": result = Rune(0x0226A) + of "Gt", "NestedGreaterGreater", "gg": result = Rune(0x0226B) + of "twixt", "between": result = Rune(0x0226C) + of "NotCupCap": result = Rune(0x0226D) + of "nlt", "NotLess", "nless": result = Rune(0x0226E) + of "ngt", "NotGreater", "ngtr": result = Rune(0x0226F) + of "nle", "NotLessEqual", "nleq": result = Rune(0x02270) + of "nge", "NotGreaterEqual", "ngeq": result = Rune(0x02271) + of "lsim", "LessTilde", "lesssim": result = Rune(0x02272) + of "gsim", "gtrsim", "GreaterTilde": result = Rune(0x02273) + of "nlsim", "NotLessTilde": result = Rune(0x02274) + of "ngsim", "NotGreaterTilde": result = Rune(0x02275) + of "lg", "lessgtr", "LessGreater": result = Rune(0x02276) + of "gl", "gtrless", "GreaterLess": result = Rune(0x02277) + of "ntlg", "NotLessGreater": result = Rune(0x02278) + of "ntgl", "NotGreaterLess": result = Rune(0x02279) + of "pr", "Precedes", "prec": result = Rune(0x0227A) + of "sc", "Succeeds", "succ": result = Rune(0x0227B) + of "prcue", "PrecedesSlantEqual", "preccurlyeq": result = Rune(0x0227C) + of "sccue", "SucceedsSlantEqual", "succcurlyeq": result = Rune(0x0227D) + of "prsim", "precsim", "PrecedesTilde": result = Rune(0x0227E) + of "scsim", "succsim", "SucceedsTilde": result = Rune(0x0227F) + of "npr", "nprec", "NotPrecedes": result = Rune(0x02280) + of "nsc", "nsucc", "NotSucceeds": result = Rune(0x02281) + of "sub", "subset": result = Rune(0x02282) + of "sup", "supset", "Superset": result = Rune(0x02283) + of "nsub": result = Rune(0x02284) + of "nsup": result = Rune(0x02285) + of "sube", "SubsetEqual", "subseteq": result = Rune(0x02286) + of "supe", "supseteq", "SupersetEqual": result = Rune(0x02287) + of "nsube", "nsubseteq", "NotSubsetEqual": result = Rune(0x02288) + of "nsupe", "nsupseteq", "NotSupersetEqual": result = Rune(0x02289) + of "subne", "subsetneq": result = Rune(0x0228A) + of "supne", "supsetneq": result = Rune(0x0228B) + of "cupdot": result = Rune(0x0228D) + of "uplus", "UnionPlus": result = Rune(0x0228E) + of "sqsub", "SquareSubset", "sqsubset": result = Rune(0x0228F) + of "sqsup", "SquareSuperset", "sqsupset": result = Rune(0x02290) + of "sqsube", "SquareSubsetEqual", "sqsubseteq": result = Rune(0x02291) + of "sqsupe", "SquareSupersetEqual", "sqsupseteq": result = Rune(0x02292) + of "sqcap", "SquareIntersection": result = Rune(0x02293) + of "sqcup", "SquareUnion": result = Rune(0x02294) + of "oplus", "CirclePlus": result = Rune(0x02295) + of "ominus", "CircleMinus": result = Rune(0x02296) + of "otimes", "CircleTimes": result = Rune(0x02297) + of "osol": result = Rune(0x02298) + of "odot", "CircleDot": result = Rune(0x02299) + of "ocir", "circledcirc": result = Rune(0x0229A) + of "oast", "circledast": result = Rune(0x0229B) + of "odash", "circleddash": result = Rune(0x0229D) + of "plusb", "boxplus": result = Rune(0x0229E) + of "minusb", "boxminus": result = Rune(0x0229F) + of "timesb", "boxtimes": result = Rune(0x022A0) + of "sdotb", "dotsquare": result = Rune(0x022A1) + of "vdash", "RightTee": result = Rune(0x022A2) + of "dashv", "LeftTee": result = Rune(0x022A3) + of "top", "DownTee": result = Rune(0x022A4) + of "bottom", "bot", "perp", "UpTee": result = Rune(0x022A5) + of "models": result = Rune(0x022A7) + of "vDash", "DoubleRightTee": result = Rune(0x022A8) + of "Vdash": result = Rune(0x022A9) + of "Vvdash": result = Rune(0x022AA) + of "VDash": result = Rune(0x022AB) + of "nvdash": result = Rune(0x022AC) + of "nvDash": result = Rune(0x022AD) + of "nVdash": result = Rune(0x022AE) + of "nVDash": result = Rune(0x022AF) + of "prurel": result = Rune(0x022B0) + of "vltri", "vartriangleleft", "LeftTriangle": result = Rune(0x022B2) + of "vrtri", "vartriangleright", "RightTriangle": result = Rune(0x022B3) + of "ltrie", "trianglelefteq", "LeftTriangleEqual": result = Rune(0x022B4) + of "rtrie", "trianglerighteq", "RightTriangleEqual": result = Rune(0x022B5) + of "origof": result = Rune(0x022B6) + of "imof": result = Rune(0x022B7) + of "mumap", "multimap": result = Rune(0x022B8) + of "hercon": result = Rune(0x022B9) + of "intcal", "intercal": result = Rune(0x022BA) + of "veebar": result = Rune(0x022BB) + of "barvee": result = Rune(0x022BD) + of "angrtvb": result = Rune(0x022BE) + of "lrtri": result = Rune(0x022BF) + of "xwedge", "Wedge", "bigwedge": result = Rune(0x022C0) + of "xvee", "Vee", "bigvee": result = Rune(0x022C1) + of "xcap", "Intersection", "bigcap": result = Rune(0x022C2) + of "xcup", "Union", "bigcup": result = Rune(0x022C3) + of "diam", "diamond", "Diamond": result = Rune(0x022C4) + of "sdot": result = Rune(0x022C5) + of "sstarf", "Star": result = Rune(0x022C6) + of "divonx", "divideontimes": result = Rune(0x022C7) + of "bowtie": result = Rune(0x022C8) + of "ltimes": result = Rune(0x022C9) + of "rtimes": result = Rune(0x022CA) + of "lthree", "leftthreetimes": result = Rune(0x022CB) + of "rthree", "rightthreetimes": result = Rune(0x022CC) + of "bsime", "backsimeq": result = Rune(0x022CD) + of "cuvee", "curlyvee": result = Rune(0x022CE) + of "cuwed", "curlywedge": result = Rune(0x022CF) + of "Sub", "Subset": result = Rune(0x022D0) + of "Sup", "Supset": result = Rune(0x022D1) + of "Cap": result = Rune(0x022D2) + of "Cup": result = Rune(0x022D3) + of "fork", "pitchfork": result = Rune(0x022D4) + of "epar": result = Rune(0x022D5) + of "ltdot", "lessdot": result = Rune(0x022D6) + of "gtdot", "gtrdot": result = Rune(0x022D7) + of "Ll": result = Rune(0x022D8) + of "Gg", "ggg": result = Rune(0x022D9) + of "leg", "LessEqualGreater", "lesseqgtr": result = Rune(0x022DA) + of "gel", "gtreqless", "GreaterEqualLess": result = Rune(0x022DB) + of "cuepr", "curlyeqprec": result = Rune(0x022DE) + of "cuesc", "curlyeqsucc": result = Rune(0x022DF) + of "nprcue", "NotPrecedesSlantEqual": result = Rune(0x022E0) + of "nsccue", "NotSucceedsSlantEqual": result = Rune(0x022E1) + of "nsqsube", "NotSquareSubsetEqual": result = Rune(0x022E2) + of "nsqsupe", "NotSquareSupersetEqual": result = Rune(0x022E3) + of "lnsim": result = Rune(0x022E6) + of "gnsim": result = Rune(0x022E7) + of "prnsim", "precnsim": result = Rune(0x022E8) + of "scnsim", "succnsim": result = Rune(0x022E9) + of "nltri", "ntriangleleft", "NotLeftTriangle": result = Rune(0x022EA) + of "nrtri", "ntriangleright", "NotRightTriangle": result = Rune(0x022EB) + of "nltrie", "ntrianglelefteq", + "NotLeftTriangleEqual": result = Rune(0x022EC) + of "nrtrie", "ntrianglerighteq", + "NotRightTriangleEqual": result = Rune(0x022ED) + of "vellip": result = Rune(0x022EE) + of "ctdot": result = Rune(0x022EF) + of "utdot": result = Rune(0x022F0) + of "dtdot": result = Rune(0x022F1) + of "disin": result = Rune(0x022F2) + of "isinsv": result = Rune(0x022F3) + of "isins": result = Rune(0x022F4) + of "isindot": result = Rune(0x022F5) + of "notinvc": result = Rune(0x022F6) + of "notinvb": result = Rune(0x022F7) + of "isinE": result = Rune(0x022F9) + of "nisd": result = Rune(0x022FA) + of "xnis": result = Rune(0x022FB) + of "nis": result = Rune(0x022FC) + of "notnivc": result = Rune(0x022FD) + of "notnivb": result = Rune(0x022FE) + of "barwed", "barwedge": result = Rune(0x02305) + of "Barwed", "doublebarwedge": result = Rune(0x02306) + of "lceil", "LeftCeiling": result = Rune(0x02308) + of "rceil", "RightCeiling": result = Rune(0x02309) + of "lfloor", "LeftFloor": result = Rune(0x0230A) + of "rfloor", "RightFloor": result = Rune(0x0230B) + of "drcrop": result = Rune(0x0230C) + of "dlcrop": result = Rune(0x0230D) + of "urcrop": result = Rune(0x0230E) + of "ulcrop": result = Rune(0x0230F) + of "bnot": result = Rune(0x02310) + of "profline": result = Rune(0x02312) + of "profsurf": result = Rune(0x02313) + of "telrec": result = Rune(0x02315) + of "target": result = Rune(0x02316) + of "ulcorn", "ulcorner": result = Rune(0x0231C) + of "urcorn", "urcorner": result = Rune(0x0231D) + of "dlcorn", "llcorner": result = Rune(0x0231E) + of "drcorn", "lrcorner": result = Rune(0x0231F) + of "frown", "sfrown": result = Rune(0x02322) + of "smile", "ssmile": result = Rune(0x02323) + of "cylcty": result = Rune(0x0232D) + of "profalar": result = Rune(0x0232E) + of "topbot": result = Rune(0x02336) + of "ovbar": result = Rune(0x0233D) + of "solbar": result = Rune(0x0233F) + of "angzarr": result = Rune(0x0237C) + of "lmoust", "lmoustache": result = Rune(0x023B0) + of "rmoust", "rmoustache": result = Rune(0x023B1) + of "tbrk", "OverBracket": result = Rune(0x023B4) + of "bbrk", "UnderBracket": result = Rune(0x023B5) + of "bbrktbrk": result = Rune(0x023B6) + of "OverParenthesis": result = Rune(0x023DC) + of "UnderParenthesis": result = Rune(0x023DD) + of "OverBrace": result = Rune(0x023DE) + of "UnderBrace": result = Rune(0x023DF) + of "trpezium": result = Rune(0x023E2) + of "elinters": result = Rune(0x023E7) + of "blank": result = Rune(0x02423) + of "oS", "circledS": result = Rune(0x024C8) + of "boxh", "HorizontalLine": result = Rune(0x02500) + of "boxv": result = Rune(0x02502) + of "boxdr": result = Rune(0x0250C) + of "boxdl": result = Rune(0x02510) + of "boxur": result = Rune(0x02514) + of "boxul": result = Rune(0x02518) + of "boxvr": result = Rune(0x0251C) + of "boxvl": result = Rune(0x02524) + of "boxhd": result = Rune(0x0252C) + of "boxhu": result = Rune(0x02534) + of "boxvh": result = Rune(0x0253C) + of "boxH": result = Rune(0x02550) + of "boxV": result = Rune(0x02551) + of "boxdR": result = Rune(0x02552) + of "boxDr": result = Rune(0x02553) + of "boxDR": result = Rune(0x02554) + of "boxdL": result = Rune(0x02555) + of "boxDl": result = Rune(0x02556) + of "boxDL": result = Rune(0x02557) + of "boxuR": result = Rune(0x02558) + of "boxUr": result = Rune(0x02559) + of "boxUR": result = Rune(0x0255A) + of "boxuL": result = Rune(0x0255B) + of "boxUl": result = Rune(0x0255C) + of "boxUL": result = Rune(0x0255D) + of "boxvR": result = Rune(0x0255E) + of "boxVr": result = Rune(0x0255F) + of "boxVR": result = Rune(0x02560) + of "boxvL": result = Rune(0x02561) + of "boxVl": result = Rune(0x02562) + of "boxVL": result = Rune(0x02563) + of "boxHd": result = Rune(0x02564) + of "boxhD": result = Rune(0x02565) + of "boxHD": result = Rune(0x02566) + of "boxHu": result = Rune(0x02567) + of "boxhU": result = Rune(0x02568) + of "boxHU": result = Rune(0x02569) + of "boxvH": result = Rune(0x0256A) + of "boxVh": result = Rune(0x0256B) + of "boxVH": result = Rune(0x0256C) + of "uhblk": result = Rune(0x02580) + of "lhblk": result = Rune(0x02584) + of "block": result = Rune(0x02588) + of "blk14": result = Rune(0x02591) + of "blk12": result = Rune(0x02592) + of "blk34": result = Rune(0x02593) + of "squ", "square", "Square": result = Rune(0x025A1) + of "squf", "squarf", "blacksquare", + "FilledVerySmallSquare": result = Rune(0x025AA) + of "EmptyVerySmallSquare": result = Rune(0x025AB) + of "rect": result = Rune(0x025AD) + of "marker": result = Rune(0x025AE) + of "fltns": result = Rune(0x025B1) + of "xutri", "bigtriangleup": result = Rune(0x025B3) + of "utrif", "blacktriangle": result = Rune(0x025B4) + of "utri", "triangle": result = Rune(0x025B5) + of "rtrif", "blacktriangleright": result = Rune(0x025B8) + of "rtri", "triangleright": result = Rune(0x025B9) + of "xdtri", "bigtriangledown": result = Rune(0x025BD) + of "dtrif", "blacktriangledown": result = Rune(0x025BE) + of "dtri", "triangledown": result = Rune(0x025BF) + of "ltrif", "blacktriangleleft": result = Rune(0x025C2) + of "ltri", "triangleleft": result = Rune(0x025C3) + of "loz", "lozenge": result = Rune(0x025CA) + of "cir": result = Rune(0x025CB) + of "tridot": result = Rune(0x025EC) + of "xcirc", "bigcirc": result = Rune(0x025EF) + of "ultri": result = Rune(0x025F8) + of "urtri": result = Rune(0x025F9) + of "lltri": result = Rune(0x025FA) + of "EmptySmallSquare": result = Rune(0x025FB) + of "FilledSmallSquare": result = Rune(0x025FC) + of "starf", "bigstar": result = Rune(0x02605) + of "star": result = Rune(0x02606) + of "phone": result = Rune(0x0260E) + of "female": result = Rune(0x02640) + of "male": result = Rune(0x02642) + of "spades", "spadesuit": result = Rune(0x02660) + of "clubs", "clubsuit": result = Rune(0x02663) + of "hearts", "heartsuit": result = Rune(0x02665) + of "diams", "diamondsuit": result = Rune(0x02666) + of "sung": result = Rune(0x0266A) + of "flat": result = Rune(0x0266D) + of "natur", "natural": result = Rune(0x0266E) + of "sharp": result = Rune(0x0266F) + of "check", "checkmark": result = Rune(0x02713) + of "cross": result = Rune(0x02717) + of "malt", "maltese": result = Rune(0x02720) + of "sext": result = Rune(0x02736) + of "VerticalSeparator": result = Rune(0x02758) + of "lbbrk": result = Rune(0x02772) + of "rbbrk": result = Rune(0x02773) + of "lobrk", "LeftDoubleBracket": result = Rune(0x027E6) + of "robrk", "RightDoubleBracket": result = Rune(0x027E7) + of "lang", "LeftAngleBracket", "langle": result = Rune(0x027E8) + of "rang", "RightAngleBracket", "rangle": result = Rune(0x027E9) + of "Lang": result = Rune(0x027EA) + of "Rang": result = Rune(0x027EB) + of "loang": result = Rune(0x027EC) + of "roang": result = Rune(0x027ED) + of "xlarr", "longleftarrow", "LongLeftArrow": result = Rune(0x027F5) + of "xrarr", "longrightarrow", "LongRightArrow": result = Rune(0x027F6) + of "xharr", "longleftrightarrow", + "LongLeftRightArrow": result = Rune(0x027F7) + of "xlArr", "Longleftarrow", "DoubleLongLeftArrow": result = Rune(0x027F8) + of "xrArr", "Longrightarrow", "DoubleLongRightArrow": result = Rune(0x027F9) + of "xhArr", "Longleftrightarrow", + "DoubleLongLeftRightArrow": result = Rune(0x027FA) + of "xmap", "longmapsto": result = Rune(0x027FC) + of "dzigrarr": result = Rune(0x027FF) + of "nvlArr": result = Rune(0x02902) + of "nvrArr": result = Rune(0x02903) + of "nvHarr": result = Rune(0x02904) + of "Map": result = Rune(0x02905) + of "lbarr": result = Rune(0x0290C) + of "rbarr", "bkarow": result = Rune(0x0290D) + of "lBarr": result = Rune(0x0290E) + of "rBarr", "dbkarow": result = Rune(0x0290F) + of "RBarr", "drbkarow": result = Rune(0x02910) + of "DDotrahd": result = Rune(0x02911) + of "UpArrowBar": result = Rune(0x02912) + of "DownArrowBar": result = Rune(0x02913) + of "Rarrtl": result = Rune(0x02916) + of "latail": result = Rune(0x02919) + of "ratail": result = Rune(0x0291A) + of "lAtail": result = Rune(0x0291B) + of "rAtail": result = Rune(0x0291C) + of "larrfs": result = Rune(0x0291D) + of "rarrfs": result = Rune(0x0291E) + of "larrbfs": result = Rune(0x0291F) + of "rarrbfs": result = Rune(0x02920) + of "nwarhk": result = Rune(0x02923) + of "nearhk": result = Rune(0x02924) + of "searhk", "hksearow": result = Rune(0x02925) + of "swarhk", "hkswarow": result = Rune(0x02926) + of "nwnear": result = Rune(0x02927) + of "nesear", "toea": result = Rune(0x02928) + of "seswar", "tosa": result = Rune(0x02929) + of "swnwar": result = Rune(0x0292A) + of "rarrc": result = Rune(0x02933) + of "cudarrr": result = Rune(0x02935) + of "ldca": result = Rune(0x02936) + of "rdca": result = Rune(0x02937) + of "cudarrl": result = Rune(0x02938) + of "larrpl": result = Rune(0x02939) + of "curarrm": result = Rune(0x0293C) + of "cularrp": result = Rune(0x0293D) + of "rarrpl": result = Rune(0x02945) + of "harrcir": result = Rune(0x02948) + of "Uarrocir": result = Rune(0x02949) + of "lurdshar": result = Rune(0x0294A) + of "ldrushar": result = Rune(0x0294B) + of "LeftRightVector": result = Rune(0x0294E) + of "RightUpDownVector": result = Rune(0x0294F) + of "DownLeftRightVector": result = Rune(0x02950) + of "LeftUpDownVector": result = Rune(0x02951) + of "LeftVectorBar": result = Rune(0x02952) + of "RightVectorBar": result = Rune(0x02953) + of "RightUpVectorBar": result = Rune(0x02954) + of "RightDownVectorBar": result = Rune(0x02955) + of "DownLeftVectorBar": result = Rune(0x02956) + of "DownRightVectorBar": result = Rune(0x02957) + of "LeftUpVectorBar": result = Rune(0x02958) + of "LeftDownVectorBar": result = Rune(0x02959) + of "LeftTeeVector": result = Rune(0x0295A) + of "RightTeeVector": result = Rune(0x0295B) + of "RightUpTeeVector": result = Rune(0x0295C) + of "RightDownTeeVector": result = Rune(0x0295D) + of "DownLeftTeeVector": result = Rune(0x0295E) + of "DownRightTeeVector": result = Rune(0x0295F) + of "LeftUpTeeVector": result = Rune(0x02960) + of "LeftDownTeeVector": result = Rune(0x02961) + of "lHar": result = Rune(0x02962) + of "uHar": result = Rune(0x02963) + of "rHar": result = Rune(0x02964) + of "dHar": result = Rune(0x02965) + of "luruhar": result = Rune(0x02966) + of "ldrdhar": result = Rune(0x02967) + of "ruluhar": result = Rune(0x02968) + of "rdldhar": result = Rune(0x02969) + of "lharul": result = Rune(0x0296A) + of "llhard": result = Rune(0x0296B) + of "rharul": result = Rune(0x0296C) + of "lrhard": result = Rune(0x0296D) + of "udhar", "UpEquilibrium": result = Rune(0x0296E) + of "duhar", "ReverseUpEquilibrium": result = Rune(0x0296F) + of "RoundImplies": result = Rune(0x02970) + of "erarr": result = Rune(0x02971) + of "simrarr": result = Rune(0x02972) + of "larrsim": result = Rune(0x02973) + of "rarrsim": result = Rune(0x02974) + of "rarrap": result = Rune(0x02975) + of "ltlarr": result = Rune(0x02976) + of "gtrarr": result = Rune(0x02978) + of "subrarr": result = Rune(0x02979) + of "suplarr": result = Rune(0x0297B) + of "lfisht": result = Rune(0x0297C) + of "rfisht": result = Rune(0x0297D) + of "ufisht": result = Rune(0x0297E) + of "dfisht": result = Rune(0x0297F) + of "lopar": result = Rune(0x02985) + of "ropar": result = Rune(0x02986) + of "lbrke": result = Rune(0x0298B) + of "rbrke": result = Rune(0x0298C) + of "lbrkslu": result = Rune(0x0298D) + of "rbrksld": result = Rune(0x0298E) + of "lbrksld": result = Rune(0x0298F) + of "rbrkslu": result = Rune(0x02990) + of "langd": result = Rune(0x02991) + of "rangd": result = Rune(0x02992) + of "lparlt": result = Rune(0x02993) + of "rpargt": result = Rune(0x02994) + of "gtlPar": result = Rune(0x02995) + of "ltrPar": result = Rune(0x02996) + of "vzigzag": result = Rune(0x0299A) + of "vangrt": result = Rune(0x0299C) + of "angrtvbd": result = Rune(0x0299D) + of "ange": result = Rune(0x029A4) + of "range": result = Rune(0x029A5) + of "dwangle": result = Rune(0x029A6) + of "uwangle": result = Rune(0x029A7) + of "angmsdaa": result = Rune(0x029A8) + of "angmsdab": result = Rune(0x029A9) + of "angmsdac": result = Rune(0x029AA) + of "angmsdad": result = Rune(0x029AB) + of "angmsdae": result = Rune(0x029AC) + of "angmsdaf": result = Rune(0x029AD) + of "angmsdag": result = Rune(0x029AE) + of "angmsdah": result = Rune(0x029AF) + of "bemptyv": result = Rune(0x029B0) + of "demptyv": result = Rune(0x029B1) + of "cemptyv": result = Rune(0x029B2) + of "raemptyv": result = Rune(0x029B3) + of "laemptyv": result = Rune(0x029B4) + of "ohbar": result = Rune(0x029B5) + of "omid": result = Rune(0x029B6) + of "opar": result = Rune(0x029B7) + of "operp": result = Rune(0x029B9) + of "olcross": result = Rune(0x029BB) + of "odsold": result = Rune(0x029BC) + of "olcir": result = Rune(0x029BE) + of "ofcir": result = Rune(0x029BF) + of "olt": result = Rune(0x029C0) + of "ogt": result = Rune(0x029C1) + of "cirscir": result = Rune(0x029C2) + of "cirE": result = Rune(0x029C3) + of "solb": result = Rune(0x029C4) + of "bsolb": result = Rune(0x029C5) + of "boxbox": result = Rune(0x029C9) + of "trisb": result = Rune(0x029CD) + of "rtriltri": result = Rune(0x029CE) + of "LeftTriangleBar": result = Rune(0x029CF) + of "RightTriangleBar": result = Rune(0x029D0) + of "race": result = Rune(0x029DA) + of "iinfin": result = Rune(0x029DC) + of "infintie": result = Rune(0x029DD) + of "nvinfin": result = Rune(0x029DE) + of "eparsl": result = Rune(0x029E3) + of "smeparsl": result = Rune(0x029E4) + of "eqvparsl": result = Rune(0x029E5) + of "lozf", "blacklozenge": result = Rune(0x029EB) + of "RuleDelayed": result = Rune(0x029F4) + of "dsol": result = Rune(0x029F6) + of "xodot", "bigodot": result = Rune(0x02A00) + of "xoplus", "bigoplus": result = Rune(0x02A01) + of "xotime", "bigotimes": result = Rune(0x02A02) + of "xuplus", "biguplus": result = Rune(0x02A04) + of "xsqcup", "bigsqcup": result = Rune(0x02A06) + of "qint", "iiiint": result = Rune(0x02A0C) + of "fpartint": result = Rune(0x02A0D) + of "cirfnint": result = Rune(0x02A10) + of "awint": result = Rune(0x02A11) + of "rppolint": result = Rune(0x02A12) + of "scpolint": result = Rune(0x02A13) + of "npolint": result = Rune(0x02A14) + of "pointint": result = Rune(0x02A15) + of "quatint": result = Rune(0x02A16) + of "intlarhk": result = Rune(0x02A17) + of "pluscir": result = Rune(0x02A22) + of "plusacir": result = Rune(0x02A23) + of "simplus": result = Rune(0x02A24) + of "plusdu": result = Rune(0x02A25) + of "plussim": result = Rune(0x02A26) + of "plustwo": result = Rune(0x02A27) + of "mcomma": result = Rune(0x02A29) + of "minusdu": result = Rune(0x02A2A) + of "loplus": result = Rune(0x02A2D) + of "roplus": result = Rune(0x02A2E) + of "Cross": result = Rune(0x02A2F) + of "timesd": result = Rune(0x02A30) + of "timesbar": result = Rune(0x02A31) + of "smashp": result = Rune(0x02A33) + of "lotimes": result = Rune(0x02A34) + of "rotimes": result = Rune(0x02A35) + of "otimesas": result = Rune(0x02A36) + of "Otimes": result = Rune(0x02A37) + of "odiv": result = Rune(0x02A38) + of "triplus": result = Rune(0x02A39) + of "triminus": result = Rune(0x02A3A) + of "tritime": result = Rune(0x02A3B) + of "iprod", "intprod": result = Rune(0x02A3C) + of "amalg": result = Rune(0x02A3F) + of "capdot": result = Rune(0x02A40) + of "ncup": result = Rune(0x02A42) + of "ncap": result = Rune(0x02A43) + of "capand": result = Rune(0x02A44) + of "cupor": result = Rune(0x02A45) + of "cupcap": result = Rune(0x02A46) + of "capcup": result = Rune(0x02A47) + of "cupbrcap": result = Rune(0x02A48) + of "capbrcup": result = Rune(0x02A49) + of "cupcup": result = Rune(0x02A4A) + of "capcap": result = Rune(0x02A4B) + of "ccups": result = Rune(0x02A4C) + of "ccaps": result = Rune(0x02A4D) + of "ccupssm": result = Rune(0x02A50) + of "And": result = Rune(0x02A53) + of "Or": result = Rune(0x02A54) + of "andand": result = Rune(0x02A55) + of "oror": result = Rune(0x02A56) + of "orslope": result = Rune(0x02A57) + of "andslope": result = Rune(0x02A58) + of "andv": result = Rune(0x02A5A) + of "orv": result = Rune(0x02A5B) + of "andd": result = Rune(0x02A5C) + of "ord": result = Rune(0x02A5D) + of "wedbar": result = Rune(0x02A5F) + of "sdote": result = Rune(0x02A66) + of "simdot": result = Rune(0x02A6A) + of "congdot": result = Rune(0x02A6D) + of "easter": result = Rune(0x02A6E) + of "apacir": result = Rune(0x02A6F) + of "apE": result = Rune(0x02A70) + of "eplus": result = Rune(0x02A71) + of "pluse": result = Rune(0x02A72) + of "Esim": result = Rune(0x02A73) + of "Colone": result = Rune(0x02A74) + of "Equal": result = Rune(0x02A75) + of "eDDot", "ddotseq": result = Rune(0x02A77) + of "equivDD": result = Rune(0x02A78) + of "ltcir": result = Rune(0x02A79) + of "gtcir": result = Rune(0x02A7A) + of "ltquest": result = Rune(0x02A7B) + of "gtquest": result = Rune(0x02A7C) + of "les", "LessSlantEqual", "leqslant": result = Rune(0x02A7D) + of "ges", "GreaterSlantEqual", "geqslant": result = Rune(0x02A7E) + of "lesdot": result = Rune(0x02A7F) + of "gesdot": result = Rune(0x02A80) + of "lesdoto": result = Rune(0x02A81) + of "gesdoto": result = Rune(0x02A82) + of "lesdotor": result = Rune(0x02A83) + of "gesdotol": result = Rune(0x02A84) + of "lap", "lessapprox": result = Rune(0x02A85) + of "gap", "gtrapprox": result = Rune(0x02A86) + of "lne", "lneq": result = Rune(0x02A87) + of "gne", "gneq": result = Rune(0x02A88) + of "lnap", "lnapprox": result = Rune(0x02A89) + of "gnap", "gnapprox": result = Rune(0x02A8A) + of "lEg", "lesseqqgtr": result = Rune(0x02A8B) + of "gEl", "gtreqqless": result = Rune(0x02A8C) + of "lsime": result = Rune(0x02A8D) + of "gsime": result = Rune(0x02A8E) + of "lsimg": result = Rune(0x02A8F) + of "gsiml": result = Rune(0x02A90) + of "lgE": result = Rune(0x02A91) + of "glE": result = Rune(0x02A92) + of "lesges": result = Rune(0x02A93) + of "gesles": result = Rune(0x02A94) + of "els", "eqslantless": result = Rune(0x02A95) + of "egs", "eqslantgtr": result = Rune(0x02A96) + of "elsdot": result = Rune(0x02A97) + of "egsdot": result = Rune(0x02A98) + of "el": result = Rune(0x02A99) + of "eg": result = Rune(0x02A9A) + of "siml": result = Rune(0x02A9D) + of "simg": result = Rune(0x02A9E) + of "simlE": result = Rune(0x02A9F) + of "simgE": result = Rune(0x02AA0) + of "LessLess": result = Rune(0x02AA1) + of "GreaterGreater": result = Rune(0x02AA2) + of "glj": result = Rune(0x02AA4) + of "gla": result = Rune(0x02AA5) + of "ltcc": result = Rune(0x02AA6) + of "gtcc": result = Rune(0x02AA7) + of "lescc": result = Rune(0x02AA8) + of "gescc": result = Rune(0x02AA9) + of "smt": result = Rune(0x02AAA) + of "lat": result = Rune(0x02AAB) + of "smte": result = Rune(0x02AAC) + of "late": result = Rune(0x02AAD) + of "bumpE": result = Rune(0x02AAE) + of "pre", "preceq", "PrecedesEqual": result = Rune(0x02AAF) + of "sce", "succeq", "SucceedsEqual": result = Rune(0x02AB0) + of "prE": result = Rune(0x02AB3) + of "scE": result = Rune(0x02AB4) + of "prnE", "precneqq": result = Rune(0x02AB5) + of "scnE", "succneqq": result = Rune(0x02AB6) + of "prap", "precapprox": result = Rune(0x02AB7) + of "scap", "succapprox": result = Rune(0x02AB8) + of "prnap", "precnapprox": result = Rune(0x02AB9) + of "scnap", "succnapprox": result = Rune(0x02ABA) + of "Pr": result = Rune(0x02ABB) + of "Sc": result = Rune(0x02ABC) + of "subdot": result = Rune(0x02ABD) + of "supdot": result = Rune(0x02ABE) + of "subplus": result = Rune(0x02ABF) + of "supplus": result = Rune(0x02AC0) + of "submult": result = Rune(0x02AC1) + of "supmult": result = Rune(0x02AC2) + of "subedot": result = Rune(0x02AC3) + of "supedot": result = Rune(0x02AC4) + of "subE", "subseteqq": result = Rune(0x02AC5) + of "supE", "supseteqq": result = Rune(0x02AC6) + of "subsim": result = Rune(0x02AC7) + of "supsim": result = Rune(0x02AC8) + of "subnE", "subsetneqq": result = Rune(0x02ACB) + of "supnE", "supsetneqq": result = Rune(0x02ACC) + of "csub": result = Rune(0x02ACF) + of "csup": result = Rune(0x02AD0) + of "csube": result = Rune(0x02AD1) + of "csupe": result = Rune(0x02AD2) + of "subsup": result = Rune(0x02AD3) + of "supsub": result = Rune(0x02AD4) + of "subsub": result = Rune(0x02AD5) + of "supsup": result = Rune(0x02AD6) + of "suphsub": result = Rune(0x02AD7) + of "supdsub": result = Rune(0x02AD8) + of "forkv": result = Rune(0x02AD9) + of "topfork": result = Rune(0x02ADA) + of "mlcp": result = Rune(0x02ADB) + of "Dashv", "DoubleLeftTee": result = Rune(0x02AE4) + of "Vdashl": result = Rune(0x02AE6) + of "Barv": result = Rune(0x02AE7) + of "vBar": result = Rune(0x02AE8) + of "vBarv": result = Rune(0x02AE9) + of "Vbar": result = Rune(0x02AEB) + of "Not": result = Rune(0x02AEC) + of "bNot": result = Rune(0x02AED) + of "rnmid": result = Rune(0x02AEE) + of "cirmid": result = Rune(0x02AEF) + of "midcir": result = Rune(0x02AF0) + of "topcir": result = Rune(0x02AF1) + of "nhpar": result = Rune(0x02AF2) + of "parsim": result = Rune(0x02AF3) + of "parsl": result = Rune(0x02AFD) + of "fflig": result = Rune(0x0FB00) + of "filig": result = Rune(0x0FB01) + of "fllig": result = Rune(0x0FB02) + of "ffilig": result = Rune(0x0FB03) + of "ffllig": result = Rune(0x0FB04) + of "Ascr": result = Rune(0x1D49C) + of "Cscr": result = Rune(0x1D49E) + of "Dscr": result = Rune(0x1D49F) + of "Gscr": result = Rune(0x1D4A2) + of "Jscr": result = Rune(0x1D4A5) + of "Kscr": result = Rune(0x1D4A6) + of "Nscr": result = Rune(0x1D4A9) + of "Oscr": result = Rune(0x1D4AA) + of "Pscr": result = Rune(0x1D4AB) + of "Qscr": result = Rune(0x1D4AC) + of "Sscr": result = Rune(0x1D4AE) + of "Tscr": result = Rune(0x1D4AF) + of "Uscr": result = Rune(0x1D4B0) + of "Vscr": result = Rune(0x1D4B1) + of "Wscr": result = Rune(0x1D4B2) + of "Xscr": result = Rune(0x1D4B3) + of "Yscr": result = Rune(0x1D4B4) + of "Zscr": result = Rune(0x1D4B5) + of "ascr": result = Rune(0x1D4B6) + of "bscr": result = Rune(0x1D4B7) + of "cscr": result = Rune(0x1D4B8) + of "dscr": result = Rune(0x1D4B9) + of "fscr": result = Rune(0x1D4BB) + of "hscr": result = Rune(0x1D4BD) + of "iscr": result = Rune(0x1D4BE) + of "jscr": result = Rune(0x1D4BF) + of "kscr": result = Rune(0x1D4C0) + of "lscr": result = Rune(0x1D4C1) + of "mscr": result = Rune(0x1D4C2) + of "nscr": result = Rune(0x1D4C3) + of "pscr": result = Rune(0x1D4C5) + of "qscr": result = Rune(0x1D4C6) + of "rscr": result = Rune(0x1D4C7) + of "sscr": result = Rune(0x1D4C8) + of "tscr": result = Rune(0x1D4C9) + of "uscr": result = Rune(0x1D4CA) + of "vscr": result = Rune(0x1D4CB) + of "wscr": result = Rune(0x1D4CC) + of "xscr": result = Rune(0x1D4CD) + of "yscr": result = Rune(0x1D4CE) + of "zscr": result = Rune(0x1D4CF) + of "Afr": result = Rune(0x1D504) + of "Bfr": result = Rune(0x1D505) + of "Dfr": result = Rune(0x1D507) + of "Efr": result = Rune(0x1D508) + of "Ffr": result = Rune(0x1D509) + of "Gfr": result = Rune(0x1D50A) + of "Jfr": result = Rune(0x1D50D) + of "Kfr": result = Rune(0x1D50E) + of "Lfr": result = Rune(0x1D50F) + of "Mfr": result = Rune(0x1D510) + of "Nfr": result = Rune(0x1D511) + of "Ofr": result = Rune(0x1D512) + of "Pfr": result = Rune(0x1D513) + of "Qfr": result = Rune(0x1D514) + of "Sfr": result = Rune(0x1D516) + of "Tfr": result = Rune(0x1D517) + of "Ufr": result = Rune(0x1D518) + of "Vfr": result = Rune(0x1D519) + of "Wfr": result = Rune(0x1D51A) + of "Xfr": result = Rune(0x1D51B) + of "Yfr": result = Rune(0x1D51C) + of "afr": result = Rune(0x1D51E) + of "bfr": result = Rune(0x1D51F) + of "cfr": result = Rune(0x1D520) + of "dfr": result = Rune(0x1D521) + of "efr": result = Rune(0x1D522) + of "ffr": result = Rune(0x1D523) + of "gfr": result = Rune(0x1D524) + of "hfr": result = Rune(0x1D525) + of "ifr": result = Rune(0x1D526) + of "jfr": result = Rune(0x1D527) + of "kfr": result = Rune(0x1D528) + of "lfr": result = Rune(0x1D529) + of "mfr": result = Rune(0x1D52A) + of "nfr": result = Rune(0x1D52B) + of "ofr": result = Rune(0x1D52C) + of "pfr": result = Rune(0x1D52D) + of "qfr": result = Rune(0x1D52E) + of "rfr": result = Rune(0x1D52F) + of "sfr": result = Rune(0x1D530) + of "tfr": result = Rune(0x1D531) + of "ufr": result = Rune(0x1D532) + of "vfr": result = Rune(0x1D533) + of "wfr": result = Rune(0x1D534) + of "xfr": result = Rune(0x1D535) + of "yfr": result = Rune(0x1D536) + of "zfr": result = Rune(0x1D537) + of "Aopf": result = Rune(0x1D538) + of "Bopf": result = Rune(0x1D539) + of "Dopf": result = Rune(0x1D53B) + of "Eopf": result = Rune(0x1D53C) + of "Fopf": result = Rune(0x1D53D) + of "Gopf": result = Rune(0x1D53E) + of "Iopf": result = Rune(0x1D540) + of "Jopf": result = Rune(0x1D541) + of "Kopf": result = Rune(0x1D542) + of "Lopf": result = Rune(0x1D543) + of "Mopf": result = Rune(0x1D544) + of "Oopf": result = Rune(0x1D546) + of "Sopf": result = Rune(0x1D54A) + of "Topf": result = Rune(0x1D54B) + of "Uopf": result = Rune(0x1D54C) + of "Vopf": result = Rune(0x1D54D) + of "Wopf": result = Rune(0x1D54E) + of "Xopf": result = Rune(0x1D54F) + of "Yopf": result = Rune(0x1D550) + of "aopf": result = Rune(0x1D552) + of "bopf": result = Rune(0x1D553) + of "copf": result = Rune(0x1D554) + of "dopf": result = Rune(0x1D555) + of "eopf": result = Rune(0x1D556) + of "fopf": result = Rune(0x1D557) + of "gopf": result = Rune(0x1D558) + of "hopf": result = Rune(0x1D559) + of "iopf": result = Rune(0x1D55A) + of "jopf": result = Rune(0x1D55B) + of "kopf": result = Rune(0x1D55C) + of "lopf": result = Rune(0x1D55D) + of "mopf": result = Rune(0x1D55E) + of "nopf": result = Rune(0x1D55F) + of "oopf": result = Rune(0x1D560) + of "popf": result = Rune(0x1D561) + of "qopf": result = Rune(0x1D562) + of "ropf": result = Rune(0x1D563) + of "sopf": result = Rune(0x1D564) + of "topf": result = Rune(0x1D565) + of "uopf": result = Rune(0x1D566) + of "vopf": result = Rune(0x1D567) + of "wopf": result = Rune(0x1D568) + of "xopf": result = Rune(0x1D569) + of "yopf": result = Rune(0x1D56A) + of "zopf": result = Rune(0x1D56B) + else: discard + proc entityToUtf8*(entity: string): string = - ## converts an HTML entity name like ``Ü`` to its UTF-8 equivalent. + ## Converts an HTML entity name like ``Ü`` or values like ``Ü`` + ## or ``Ü`` to its UTF-8 equivalent. ## "" is returned if the entity name is unknown. The HTML parser ## already converts entities to UTF-8. - for name, val in items(Entities): - if name == entity: return toUTF8(Rune(val)) - result = "" + runnableExamples: + doAssert entityToUtf8(nil) == "" + doAssert entityToUtf8("") == "" + doAssert entityToUtf8("a") == "" + doAssert entityToUtf8("gt") == ">" + doAssert entityToUtf8("Uuml") == "Ü" + doAssert entityToUtf8("quest") == "?" + doAssert entityToUtf8("#63") == "?" + doAssert entityToUtf8("Sigma") == "Σ" + doAssert entityToUtf8("#931") == "Σ" + doAssert entityToUtf8("#0931") == "Σ" + doAssert entityToUtf8("#x3A3") == "Σ" + doAssert entityToUtf8("#x03A3") == "Σ" + doAssert entityToUtf8("#x3a3") == "Σ" + doAssert entityToUtf8("#X3a3") == "Σ" + let rune = entityToRune(entity) + if rune.ord <= 0: result = "" + else: result = toUTF8(rune) proc addNode(father, son: XmlNode) = if son != nil: add(father, son) @@ -565,7 +2008,7 @@ proc parse(x: var XmlParser, errors: var seq[string]): XmlNode = proc parseHtml*(s: Stream, filename: string, errors: var seq[string]): XmlNode = - ## parses the XML from stream `s` and returns a ``PXmlNode``. Every + ## Parses the XML from stream `s` and returns a ``XmlNode``. Every ## occurred parsing error is added to the `errors` sequence. var x: XmlParser open(x, s, filename, {reportComments, reportWhitespace}) @@ -588,14 +2031,19 @@ proc parseHtml*(s: Stream, filename: string, result = result[0] proc parseHtml*(s: Stream): XmlNode = - ## parses the XTML from stream `s` and returns a ``PXmlNode``. All parsing + ## Parses the HTML from stream `s` and returns a ``XmlNode``. All parsing ## errors are ignored. var errors: seq[string] = @[] result = parseHtml(s, "unknown_html_doc", errors) +proc parseHtml*(html: string): XmlNode = + ## Parses the HTML from string ``html`` and returns a ``XmlNode``. All parsing + ## errors are ignored. + parseHtml(newStringStream(html)) + proc loadHtml*(path: string, errors: var seq[string]): XmlNode = ## Loads and parses HTML from file specified by ``path``, and returns - ## a ``PXmlNode``. Every occurred parsing error is added to + ## a ``XmlNode``. Every occurred parsing error is added to ## the `errors` sequence. var s = newFileStream(path, fmRead) if s == nil: raise newException(IOError, "Unable to read file: " & path) @@ -603,7 +2051,7 @@ proc loadHtml*(path: string, errors: var seq[string]): XmlNode = proc loadHtml*(path: string): XmlNode = ## Loads and parses HTML from file specified by ``path``, and returns - ## a ``PXmlNode``. All parsing errors are ignored. + ## a ``XmlNode``. All parsing errors are ignored. var errors: seq[string] = @[] result = loadHtml(path, errors) diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index 54a8498fa..8530e4c42 100644 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -73,12 +73,18 @@ ## progress of the HTTP request. ## ## .. code-block:: Nim -## var client = newAsyncHttpClient() +## import asyncdispatch, httpclient +## ## proc onProgressChanged(total, progress, speed: BiggestInt) {.async.} = ## echo("Downloaded ", progress, " of ", total) ## echo("Current rate: ", speed div 1000, "kb/s") -## client.onProgressChanged = onProgressChanged -## discard await client.getContent("http://speedtest-ams2.digitalocean.com/100mb.test") +## +## proc asyncProc() {.async.} = +## var client = newAsyncHttpClient() +## client.onProgressChanged = onProgressChanged +## discard await client.getContent("http://speedtest-ams2.digitalocean.com/100mb.test") +## +## waitFor asyncProc() ## ## If you would like to remove the callback simply set it to ``nil``. ## @@ -150,6 +156,31 @@ proc code*(response: Response | AsyncResponse): HttpCode ## corresponding ``HttpCode``. return response.status[0 .. 2].parseInt.HttpCode +proc contentType*(response: Response | AsyncResponse): string = + ## Retrieves the specified response's content type. + ## + ## This is effectively the value of the "Content-Type" header. + response.headers.getOrDefault("content-type") + +proc contentLength*(response: Response | AsyncResponse): int = + ## Retrieves the specified response's content length. + ## + ## This is effectively the value of the "Content-Length" header. + ## + ## A ``ValueError`` exception will be raised if the value is not an integer. + var contentLengthHeader = response.headers.getOrDefault("Content-Length") + return contentLengthHeader.parseInt() + +proc lastModified*(response: Response | AsyncResponse): DateTime = + ## Retrieves the specified response's last modified time. + ## + ## This is effectively the value of the "Last-Modified" header. + ## + ## Raises a ``ValueError`` if the parsing fails or the value is not a correctly + ## formatted time. + var lastModifiedHeader = response.headers.getOrDefault("last-modified") + result = parse(lastModifiedHeader, "dd, dd MMM yyyy HH:mm:ss Z") + proc body*(response: Response): string = ## Retrieves the specified response's body. ## @@ -188,10 +219,6 @@ type ## and ``postContent`` proc, ## when the server returns an error -{.deprecated: [TResponse: Response, PProxy: Proxy, - EInvalidProtocol: ProtocolError, EHttpRequestErr: HttpRequestError -].} - const defUserAgent* = "Nim httpclient/" & NimVersion proc httpError(msg: string) = @@ -216,7 +243,7 @@ proc parseChunks(s: Socket, timeout: int): string = var i = 0 if chunkSizeStr == "": httpError("Server terminated connection prematurely") - while true: + while i < chunkSizeStr.len: case chunkSizeStr[i] of '0'..'9': chunkSize = chunkSize shl 4 or (ord(chunkSizeStr[i]) - ord('0')) @@ -224,8 +251,6 @@ proc parseChunks(s: Socket, timeout: int): string = chunkSize = chunkSize shl 4 or (ord(chunkSizeStr[i]) - ord('a') + 10) of 'A'..'F': chunkSize = chunkSize shl 4 or (ord(chunkSizeStr[i]) - ord('A') + 10) - of '\0': - break of ';': # http://tools.ietf.org/html/rfc2616#section-3.6.1 # We don't care about chunk-extensions. @@ -333,21 +358,17 @@ proc parseResponse(s: Socket, getBody: bool, timeout: int): Response = else: result.body = "" -{.deprecated: [THttpMethod: HttpMethod].} - when not defined(ssl): type SSLContext = ref object var defaultSSLContext {.threadvar.}: SSLContext -when defined(ssl): - defaultSSLContext = newContext(verifyMode = CVerifyNone) - template contextOrDefault(ctx: SSLContext): SSLContext = - var result = ctx - if ctx == nil: - if defaultSSLContext == nil: - defaultSSLContext = newContext(verifyMode = CVerifyNone) +proc getDefaultSSL(): SSLContext = + result = defaultSslContext + when defined(ssl): + if result == nil: + defaultSSLContext = newContext(verifyMode = CVerifyNone) result = defaultSSLContext - result + doAssert result != nil, "failure to initialize the SSL context" proc newProxy*(url: string, auth = ""): Proxy = ## Constructs a new ``TProxy`` object. @@ -457,9 +478,9 @@ proc format(p: MultipartData): tuple[contentType, body: string] = result.body.add("--" & bound & "--\c\L") proc request*(url: string, httpMethod: string, extraHeaders = "", - body = "", sslContext = defaultSSLContext, timeout = -1, + body = "", sslContext = getDefaultSSL(), timeout = -1, userAgent = defUserAgent, proxy: Proxy = nil): Response - {.deprecated.} = + {.deprecated: "use HttpClient.request instead".} = ## | Requests ``url`` with the custom method string specified by the ## | ``httpMethod`` parameter. ## | Extra headers can be specified and must be separated by ``\c\L`` @@ -482,7 +503,7 @@ proc request*(url: string, httpMethod: string, extraHeaders = "", port = net.Port(443) else: raise newException(HttpRequestError, - "SSL support is not available. Cannot connect over SSL.") + "SSL support is not available. Cannot connect over SSL. Compile with -d:ssl to enable.") if r.port != "": port = net.Port(r.port.parseInt) @@ -515,7 +536,8 @@ proc request*(url: string, httpMethod: string, extraHeaders = "", "so a secure connection could not be established.") sslContext.wrapConnectedSocket(s, handshakeAsClient, hostUrl.hostname) else: - raise newException(HttpRequestError, "SSL support not available. Cannot connect via proxy over SSL") + raise newException(HttpRequestError, "SSL support not available. Cannot " & + "connect via proxy over SSL. Compile with -d:ssl to enable.") else: if timeout == -1: s.connect(r.hostname, port) @@ -555,8 +577,8 @@ proc request*(url: string, httpMethod: string, extraHeaders = "", result = parseResponse(s, httpMethod != "HEAD", timeout) -proc request*(url: string, httpMethod = httpGET, extraHeaders = "", - body = "", sslContext = defaultSSLContext, timeout = -1, +proc request*(url: string, httpMethod = HttpGET, extraHeaders = "", + body = "", sslContext = getDefaultSSL(), timeout = -1, userAgent = defUserAgent, proxy: Proxy = nil): Response {.deprecated.} = ## | Requests ``url`` with the specified ``httpMethod``. @@ -587,7 +609,7 @@ proc getNewLocation(lastURL: string, headers: HttpHeaders): string = result = $parsed proc get*(url: string, extraHeaders = "", maxRedirects = 5, - sslContext: SSLContext = defaultSSLContext, + sslContext: SSLContext = getDefaultSSL(), timeout = -1, userAgent = defUserAgent, proxy: Proxy = nil): Response {.deprecated.} = ## | GETs the ``url`` and returns a ``Response`` object @@ -596,19 +618,19 @@ proc get*(url: string, extraHeaders = "", maxRedirects = 5, ## | An optional timeout can be specified in milliseconds, if reading from the ## server takes longer than specified an ETimeout exception will be raised. ## - ## ## **Deprecated since version 0.15.0**: use ``HttpClient.get`` instead. - result = request(url, httpGET, extraHeaders, "", sslContext, timeout, + ## **Deprecated since version 0.15.0**: use ``HttpClient.get`` instead. + result = request(url, HttpGET, extraHeaders, "", sslContext, timeout, userAgent, proxy) var lastURL = url for i in 1..maxRedirects: if result.status.redirection(): let redirectTo = getNewLocation(lastURL, result.headers) - result = request(redirectTo, httpGET, extraHeaders, "", sslContext, + result = request(redirectTo, HttpGET, extraHeaders, "", sslContext, timeout, userAgent, proxy) lastURL = redirectTo proc getContent*(url: string, extraHeaders = "", maxRedirects = 5, - sslContext: SSLContext = defaultSSLContext, + sslContext: SSLContext = getDefaultSSL(), timeout = -1, userAgent = defUserAgent, proxy: Proxy = nil): string {.deprecated.} = ## | GETs the body and returns it as a string. @@ -627,7 +649,7 @@ proc getContent*(url: string, extraHeaders = "", maxRedirects = 5, proc post*(url: string, extraHeaders = "", body = "", maxRedirects = 5, - sslContext: SSLContext = defaultSSLContext, + sslContext: SSLContext = getDefaultSSL(), timeout = -1, userAgent = defUserAgent, proxy: Proxy = nil, multipart: MultipartData = nil): Response {.deprecated.} = @@ -657,20 +679,20 @@ proc post*(url: string, extraHeaders = "", body = "", if not multipart.isNil: xh.add(withNewLine("Content-Type: " & mpContentType)) - result = request(url, httpPOST, xh, xb, sslContext, timeout, userAgent, + result = request(url, HttpPOST, xh, xb, sslContext, timeout, userAgent, proxy) var lastURL = url for i in 1..maxRedirects: if result.status.redirection(): let redirectTo = getNewLocation(lastURL, result.headers) - var meth = if result.status != "307": httpGet else: httpPost + var meth = if result.status != "307": HttpGet else: HttpPost result = request(redirectTo, meth, xh, xb, sslContext, timeout, userAgent, proxy) lastURL = redirectTo proc postContent*(url: string, extraHeaders = "", body = "", maxRedirects = 5, - sslContext: SSLContext = defaultSSLContext, + sslContext: SSLContext = getDefaultSSL(), timeout = -1, userAgent = defUserAgent, proxy: Proxy = nil, multipart: MultipartData = nil): string @@ -693,7 +715,7 @@ proc postContent*(url: string, extraHeaders = "", body = "", return r.body proc downloadFile*(url: string, outputFilename: string, - sslContext: SSLContext = defaultSSLContext, + sslContext: SSLContext = getDefaultSSL(), timeout = -1, userAgent = defUserAgent, proxy: Proxy = nil) {.deprecated.} = ## | Downloads ``url`` and saves it to ``outputFilename`` @@ -713,12 +735,13 @@ proc downloadFile*(url: string, outputFilename: string, proc generateHeaders(requestUrl: Uri, httpMethod: string, headers: HttpHeaders, body: string, proxy: Proxy): string = # GET - result = httpMethod.toUpperAscii() + let upperMethod = httpMethod.toUpperAscii() + result = upperMethod result.add ' ' - if proxy.isNil or (not proxy.isNil and requestUrl.scheme == "https"): + if proxy.isNil or requestUrl.scheme == "https": # /path?query - if requestUrl.path[0] != '/': result.add '/' + if not requestUrl.path.startsWith("/"): result.add '/' result.add(requestUrl.path) if requestUrl.query.len > 0: result.add("?" & requestUrl.query) @@ -742,7 +765,9 @@ proc generateHeaders(requestUrl: Uri, httpMethod: string, add(result, "Connection: Keep-Alive\c\L") # Content length header. - if body.len > 0 and not headers.hasKey("Content-Length"): + const requiresBody = ["POST", "PUT", "PATCH"] + let needsContentLength = body.len > 0 or upperMethod in requiresBody + if needsContentLength and not headers.hasKey("Content-Length"): add(result, "Content-Length: " & $body.len & "\c\L") # Proxy auth header. @@ -790,7 +815,7 @@ type HttpClient* = HttpClientBase[Socket] proc newHttpClient*(userAgent = defUserAgent, - maxRedirects = 5, sslContext = defaultSslContext, proxy: Proxy = nil, + maxRedirects = 5, sslContext = getDefaultSSL(), proxy: Proxy = nil, timeout = -1): HttpClient = ## Creates a new HttpClient instance. ## @@ -817,7 +842,7 @@ proc newHttpClient*(userAgent = defUserAgent, result.bodyStream = newStringStream() result.getBody = true when defined(ssl): - result.sslContext = contextOrDefault(sslContext) + result.sslContext = sslContext type AsyncHttpClient* = HttpClientBase[AsyncSocket] @@ -825,7 +850,7 @@ type {.deprecated: [PAsyncHttpClient: AsyncHttpClient].} proc newAsyncHttpClient*(userAgent = defUserAgent, - maxRedirects = 5, sslContext = defaultSslContext, + maxRedirects = 5, sslContext = getDefaultSSL(), proxy: Proxy = nil): AsyncHttpClient = ## Creates a new AsyncHttpClient instance. ## @@ -849,7 +874,7 @@ proc newAsyncHttpClient*(userAgent = defUserAgent, result.bodyStream = newFutureStream[string]("newAsyncHttpClient") result.getBody = true when defined(ssl): - result.sslContext = contextOrDefault(sslContext) + result.sslContext = sslContext proc close*(client: HttpClient | AsyncHttpClient) = ## Closes any connections held by the HTTP client. @@ -903,7 +928,7 @@ proc parseChunks(client: HttpClient | AsyncHttpClient): Future[void] var i = 0 if chunkSizeStr == "": httpError("Server terminated connection prematurely") - while true: + while i < chunkSizeStr.len: case chunkSizeStr[i] of '0'..'9': chunkSize = chunkSize shl 4 or (ord(chunkSizeStr[i]) - ord('0')) @@ -911,8 +936,6 @@ proc parseChunks(client: HttpClient | AsyncHttpClient): Future[void] chunkSize = chunkSize shl 4 or (ord(chunkSizeStr[i]) - ord('a') + 10) of 'A'..'F': chunkSize = chunkSize shl 4 or (ord(chunkSizeStr[i]) - ord('A') + 10) - of '\0': - break of ';': # http://tools.ietf.org/html/rfc2616#section-3.6.1 # We don't care about chunk-extensions. @@ -923,8 +946,14 @@ proc parseChunks(client: HttpClient | AsyncHttpClient): Future[void] if chunkSize <= 0: discard await recvFull(client, 2, client.timeout, false) # Skip \c\L break - discard await recvFull(client, chunkSize, client.timeout, true) - discard await recvFull(client, 2, client.timeout, false) # Skip \c\L + var bytesRead = await recvFull(client, chunkSize, client.timeout, true) + if bytesRead != chunkSize: + httpError("Server terminated connection prematurely") + + bytesRead = await recvFull(client, 2, client.timeout, false) # Skip \c\L + if bytesRead != 2: + httpError("Server terminated connection prematurely") + # Trailer headers will only be sent if the request specifies that we want # them: http://tools.ietf.org/html/rfc2616#section-3.6.1 @@ -965,7 +994,7 @@ proc parseBody(client: HttpClient | AsyncHttpClient, if headers.getOrDefault"Connection" == "close" or httpVersion == "1.0": while true: let recvLen = await client.recvFull(4000, client.timeout, true) - if recvLen == 0: + if recvLen != 4000: client.close() break @@ -1056,7 +1085,7 @@ proc newConnection(client: HttpClient | AsyncHttpClient, if isSsl and not defined(ssl): raise newException(HttpRequestError, - "SSL support is not available. Cannot connect over SSL.") + "SSL support is not available. Cannot connect over SSL. Compile with -d:ssl to enable.") if client.connected: client.close() @@ -1106,7 +1135,7 @@ proc newConnection(client: HttpClient | AsyncHttpClient, client.socket, handshakeAsClient, url.hostname) else: raise newException(HttpRequestError, - "SSL support is not available. Cannot connect over SSL.") + "SSL support is not available. Cannot connect over SSL. Compile with -d:ssl to enable.") # May be connected through proxy but remember actual URL being accessed client.currentURL = url @@ -1167,7 +1196,7 @@ proc request*(client: HttpClient | AsyncHttpClient, url: string, for i in 1..client.maxRedirects: if result.status.redirection(): let redirectTo = getNewLocation(lastURL, result.headers) - result = await client.request(redirectTo, httpMethod, body, headers) + result = await client.requestAux(redirectTo, httpMethod, body, headers) lastURL = redirectTo @@ -1257,21 +1286,30 @@ proc postContent*(client: HttpClient | AsyncHttpClient, url: string, else: return await resp.bodyStream.readAll() -proc downloadFile*(client: HttpClient | AsyncHttpClient, - url: string, filename: string): Future[void] {.multisync.} = +proc downloadFile*(client: HttpClient, url: string, filename: string) = ## Downloads ``url`` and saves it to ``filename``. client.getBody = false defer: client.getBody = true - let resp = await client.get(url) - - when client is HttpClient: - client.bodyStream = newFileStream(filename, fmWrite) - if client.bodyStream.isNil: - fileError("Unable to open file") - parseBody(client, resp.headers, resp.version) - client.bodyStream.close() - else: + let resp = client.get(url) + + client.bodyStream = newFileStream(filename, fmWrite) + if client.bodyStream.isNil: + fileError("Unable to open file") + parseBody(client, resp.headers, resp.version) + client.bodyStream.close() + + if resp.code.is4xx or resp.code.is5xx: + raise newException(HttpRequestError, resp.status) + +proc downloadFile*(client: AsyncHttpClient, url: string, + filename: string): Future[void] = + proc downloadFileEx(client: AsyncHttpClient, + url, filename: string): Future[void] {.async.} = + ## Downloads ``url`` and saves it to ``filename``. + client.getBody = false + let resp = await client.get(url) + client.bodyStream = newFutureStream[string]("downloadFile") var file = openAsync(filename, fmWrite) # Let `parseBody` write response data into client.bodyStream in the @@ -1282,5 +1320,15 @@ proc downloadFile*(client: HttpClient | AsyncHttpClient, await file.writeFromStream(client.bodyStream) file.close() - if resp.code.is4xx or resp.code.is5xx: - raise newException(HttpRequestError, resp.status) + if resp.code.is4xx or resp.code.is5xx: + raise newException(HttpRequestError, resp.status) + + result = newFuture[void]("downloadFile") + try: + result = downloadFileEx(client, url, filename) + except Exception as exc: + result.fail(exc) + finally: + result.addCallback( + proc () = client.getBody = true + ) diff --git a/lib/pure/httpcore.nim b/lib/pure/httpcore.nim index f150fa1c1..f85375111 100644 --- a/lib/pure/httpcore.nim +++ b/lib/pure/httpcore.nim @@ -45,10 +45,6 @@ type ## TCP/IP tunnel, usually used for proxies. HttpPatch ## Applies partial modifications to a resource. -{.deprecated: [httpGet: HttpGet, httpHead: HttpHead, httpPost: HttpPost, - httpPut: HttpPut, httpDelete: HttpDelete, httpTrace: HttpTrace, - httpOptions: HttpOptions, httpConnect: HttpConnect].} - const Http100* = HttpCode(100) @@ -190,11 +186,11 @@ proc len*(headers: HttpHeaders): int = return headers.table.len proc parseList(line: string, list: var seq[string], start: int): int = var i = 0 var current = "" - while line[start + i] notin {'\c', '\l', '\0'}: + while start+i < line.len and line[start + i] notin {'\c', '\l'}: i += line.skipWhitespace(start + i) i += line.parseUntil(current, {'\c', '\l', ','}, start + i) list.add(current) - if line[start + i] == ',': + if start+i < line.len and line[start + i] == ',': i.inc # Skip , current.setLen(0) diff --git a/lib/pure/httpserver.nim b/lib/pure/httpserver.nim index 632eb198a..a81e8c0a8 100644 --- a/lib/pure/httpserver.nim +++ b/lib/pure/httpserver.nim @@ -110,7 +110,6 @@ when false: # TODO: Fix this, or get rid of it. type RequestMethod = enum reqGet, reqPost - {.deprecated: [TRequestMethod: RequestMethod].} proc executeCgi(client: Socket, path, query: string, meth: RequestMethod) = var env = newStringTable(modeCaseInsensitive) @@ -126,7 +125,7 @@ when false: var dataAvail = false while dataAvail: dataAvail = recvLine(client, buf) # TODO: This is incorrect. - var L = toLower(buf.string) + var L = toLowerAscii(buf.string) if L.startsWith("content-length:"): var i = len("content-length:") while L[i] in Whitespace: inc(i) @@ -199,7 +198,7 @@ when false: notFound(client) else: when defined(Windows): - var ext = splitFile(path).ext.toLower + var ext = splitFile(path).ext.toLowerAscii if ext == ".exe" or ext == ".cgi": # XXX: extract interpreter information here? cgi = true @@ -225,7 +224,6 @@ type PAsyncHTTPServer* = ref AsyncHTTPServer AsyncHTTPServer = object of Server asyncSocket: AsyncSocket -{.deprecated: [TAsyncHTTPServer: AsyncHTTPServer, TServer: Server].} proc open*(s: var Server, port = Port(80), reuseAddr = false) = ## creates a new server at port `port`. If ``port == 0`` a free port is @@ -303,7 +301,7 @@ proc next*(s: var Server) = if s.reqMethod == "POST": # Check for Expect header if s.headers.hasKey("Expect"): - if s.headers["Expect"].toLower == "100-continue": + if s.headers["Expect"].toLowerAscii == "100-continue": s.client.sendStatus("100 Continue") else: s.client.sendStatus("417 Expectation Failed") @@ -427,7 +425,7 @@ proc nextAsync(s: PAsyncHTTPServer) = if s.reqMethod == "POST": # Check for Expect header if s.headers.hasKey("Expect"): - if s.headers["Expect"].toLower == "100-continue": + if s.headers["Expect"].toLowerAscii == "100-continue": s.client.sendStatus("100 Continue") else: s.client.sendStatus("417 Expectation Failed") diff --git a/lib/pure/includes/asynccommon.nim b/lib/pure/includes/asynccommon.nim index 8b760c66a..06f4958c6 100644 --- a/lib/pure/includes/asynccommon.nim +++ b/lib/pure/includes/asynccommon.nim @@ -1,21 +1,31 @@ -template newAsyncNativeSocketImpl(domain, sockType, protocol) = +template createAsyncNativeSocketImpl(domain, sockType, protocol) = let handle = newNativeSocket(domain, sockType, protocol) if handle == osInvalidSocket: - raiseOSError(osLastError()) + return osInvalidSocket.AsyncFD handle.setBlocking(false) when defined(macosx) and not defined(nimdoc): handle.setSockOptInt(SOL_SOCKET, SO_NOSIGPIPE, 1) result = handle.AsyncFD register(result) -proc newAsyncNativeSocket*(domain: cint, sockType: cint, +proc createAsyncNativeSocket*(domain: cint, sockType: cint, protocol: cint): AsyncFD = - newAsyncNativeSocketImpl(domain, sockType, protocol) + createAsyncNativeSocketImpl(domain, sockType, protocol) -proc newAsyncNativeSocket*(domain: Domain = Domain.AF_INET, +proc createAsyncNativeSocket*(domain: Domain = Domain.AF_INET, sockType: SockType = SOCK_STREAM, protocol: Protocol = IPPROTO_TCP): AsyncFD = - newAsyncNativeSocketImpl(domain, sockType, protocol) + createAsyncNativeSocketImpl(domain, sockType, protocol) + +proc newAsyncNativeSocket*(domain: cint, sockType: cint, + protocol: cint): AsyncFD {.deprecated: "use createAsyncNativeSocket instead".} = + createAsyncNativeSocketImpl(domain, sockType, protocol) + +proc newAsyncNativeSocket*(domain: Domain = Domain.AF_INET, + sockType: SockType = SOCK_STREAM, + protocol: Protocol = IPPROTO_TCP): AsyncFD + {.deprecated: "use createAsyncNativeSocket instead".} = + createAsyncNativeSocketImpl(domain, sockType, protocol) when defined(windows) or defined(nimdoc): proc bindToDomain(handle: SocketHandle, domain: Domain) = diff --git a/lib/pure/includes/oserr.nim b/lib/pure/includes/oserr.nim index 0889d7383..bfb6118f2 100644 --- a/lib/pure/includes/oserr.nim +++ b/lib/pure/includes/oserr.nim @@ -12,50 +12,6 @@ when not defined(nimscript): when defined(windows): import winlean -proc osErrorMsg*(): string {.rtl, extern: "nos$1", deprecated.} = - ## Retrieves the operating system's error flag, ``errno``. - ## On Windows ``GetLastError`` is checked before ``errno``. - ## Returns "" if no error occurred. - ## - ## **Deprecated since version 0.9.4**: use the other ``osErrorMsg`` proc. - - result = "" - when defined(Windows) and not defined(nimscript): - var err = getLastError() - if err != 0'i32: - when useWinUnicode: - var msgbuf: WideCString - if formatMessageW(0x00000100 or 0x00001000 or 0x00000200 or 0x000000FF, - nil, err, 0, addr(msgbuf), 0, nil) != 0'i32: - result = $msgbuf - if msgbuf != nil: localFree(cast[pointer](msgbuf)) - else: - var msgbuf: cstring - if formatMessageA(0x00000100 or 0x00001000 or 0x00000200 or 0x000000FF, - nil, err, 0, addr(msgbuf), 0, nil) != 0'i32: - result = $msgbuf - if msgbuf != nil: localFree(msgbuf) - when not defined(nimscript): - if errno != 0'i32: - result = $c_strerror(errno) - -{.push warning[deprecated]: off.} -proc raiseOSError*(msg: string = "") {.noinline, rtl, extern: "nos$1", - deprecated.} = - ## raises an OSError exception with the given message ``msg``. - ## If ``msg == ""``, the operating system's error flag - ## (``errno``) is converted to a readable error message. On Windows - ## ``GetLastError`` is checked before ``errno``. - ## If no error flag is set, the message ``unknown OS error`` is used. - ## - ## **Deprecated since version 0.9.4**: use the other ``raiseOSError`` proc. - if len(msg) == 0: - var m = osErrorMsg() - raise newException(OSError, if m.len > 0: m else: "unknown OS error") - else: - raise newException(OSError, msg) -{.pop.} - proc `==`*(err1, err2: OSErrorCode): bool {.borrow.} proc `$`*(err: OSErrorCode): string {.borrow.} diff --git a/lib/pure/ioselects/ioselectors_epoll.nim b/lib/pure/ioselects/ioselectors_epoll.nim index 8827f239f..36145abc7 100644 --- a/lib/pure/ioselects/ioselectors_epoll.nim +++ b/lib/pure/ioselects/ioselectors_epoll.nim @@ -48,16 +48,6 @@ when not defined(android): proc signalfd(fd: cint, mask: var Sigset, flags: cint): cint {.cdecl, importc: "signalfd", header: "<sys/signalfd.h>".} -var RLIMIT_NOFILE {.importc: "RLIMIT_NOFILE", - header: "<sys/resource.h>".}: cint -type - RLimit {.importc: "struct rlimit", - header: "<sys/resource.h>", pure, final.} = object - rlim_cur: int - rlim_max: int -proc getrlimit(resource: cint, rlp: var RLimit): cint - {.importc: "getrlimit",header: "<sys/resource.h>".} - when hasThreadSupport: type SelectorImpl[T] = object @@ -82,7 +72,7 @@ type proc newSelector*[T](): Selector[T] = # Retrieve the maximum fd count (for current OS) via getrlimit() var a = RLimit() - if getrlimit(RLIMIT_NOFILE, a) != 0: + if getrlimit(posix.RLIMIT_NOFILE, a) != 0: raiseOsError(osLastError()) var maxFD = int(a.rlim_max) doAssert(maxFD > 0) @@ -141,7 +131,7 @@ template checkFd(s, f) = if f >= s.maxFD: raiseIOSelectorsError("Maximum number of descriptors is exhausted!") -proc registerHandle*[T](s: Selector[T], fd: SocketHandle, +proc registerHandle*[T](s: Selector[T], fd: int | SocketHandle, events: set[Event], data: T) = let fdi = int(fd) s.checkFd(fdi) @@ -156,7 +146,7 @@ proc registerHandle*[T](s: Selector[T], fd: SocketHandle, raiseIOSelectorsError(osLastError()) inc(s.count) -proc updateHandle*[T](s: Selector[T], fd: SocketHandle, events: set[Event]) = +proc updateHandle*[T](s: Selector[T], fd: int | SocketHandle, events: set[Event]) = let maskEvents = {Event.Timer, Event.Signal, Event.Process, Event.Vnode, Event.User, Event.Oneshot, Event.Error} let fdi = int(fd) @@ -392,9 +382,19 @@ proc selectInto*[T](s: Selector[T], timeout: int, let pevents = resTable[i].events var pkey = addr(s.fds[fdi]) doAssert(pkey.ident != 0) - var rkey = ReadyKey(fd: int(fdi), events: {}) + var rkey = ReadyKey(fd: fdi, events: {}) if (pevents and EPOLLERR) != 0 or (pevents and EPOLLHUP) != 0: + if (pevents and EPOLLHUP) != 0: + rkey.errorCode = ECONNRESET.OSErrorCode + else: + # Try reading SO_ERROR from fd. + var error: cint + var size = sizeof(error).SockLen + if getsockopt(fdi.SocketHandle, SOL_SOCKET, SO_ERROR, addr(error), + addr(size)) == 0'i32: + rkey.errorCode = error.OSErrorCode + rkey.events.incl(Event.Error) if (pevents and EPOLLOUT) != 0: rkey.events.incl(Event.Write) @@ -482,7 +482,7 @@ template isEmpty*[T](s: Selector[T]): bool = (s.count == 0) proc contains*[T](s: Selector[T], fd: SocketHandle|int): bool {.inline.} = - return s.fds[fd].ident != 0 + return s.fds[fd.int].ident != 0 proc getData*[T](s: Selector[T], fd: SocketHandle|int): var T = let fdi = int(fd) @@ -516,3 +516,6 @@ template withData*[T](s: Selector[T], fd: SocketHandle|int, value, body1, body1 else: body2 + +proc getFd*[T](s: Selector[T]): int = + return s.epollFd.int diff --git a/lib/pure/ioselects/ioselectors_kqueue.nim b/lib/pure/ioselects/ioselectors_kqueue.nim index af5aa15df..10e23c072 100644 --- a/lib/pure/ioselects/ioselectors_kqueue.nim +++ b/lib/pure/ioselects/ioselectors_kqueue.nim @@ -217,7 +217,7 @@ else: raiseIOSelectorsError(osLastError()) s.changes.setLen(0) -proc registerHandle*[T](s: Selector[T], fd: SocketHandle, +proc registerHandle*[T](s: Selector[T], fd: int | SocketHandle, events: set[Event], data: T) = let fdi = int(fd) s.checkFd(fdi) @@ -235,7 +235,7 @@ proc registerHandle*[T](s: Selector[T], fd: SocketHandle, when not declared(CACHE_EVENTS): flushKQueue(s) -proc updateHandle*[T](s: Selector[T], fd: SocketHandle, +proc updateHandle*[T](s: Selector[T], fd: int | SocketHandle, events: set[Event]) = let maskEvents = {Event.Timer, Event.Signal, Event.Process, Event.Vnode, Event.User, Event.Oneshot, Event.Error} @@ -503,6 +503,7 @@ proc selectInto*[T](s: Selector[T], timeout: int, if (kevent.flags and EV_ERROR) != 0: rkey.events = {Event.Error} + rkey.errorCode = kevent.data.OSErrorCode case kevent.filter: of EVFILT_READ: @@ -569,6 +570,13 @@ proc selectInto*[T](s: Selector[T], timeout: int, doAssert(true, "Unsupported kqueue filter in the queue!") if (kevent.flags and EV_EOF) != 0: + if kevent.fflags != 0: + rkey.errorCode = kevent.fflags.OSErrorCode + else: + # This assumes we are dealing with sockets. + # TODO: For future-proofing it might be a good idea to give the + # user access to the raw `kevent`. + rkey.errorCode = ECONNRESET.OSErrorCode rkey.events.incl(Event.Error) results[k] = rkey @@ -585,7 +593,7 @@ template isEmpty*[T](s: Selector[T]): bool = (s.count == 0) proc contains*[T](s: Selector[T], fd: SocketHandle|int): bool {.inline.} = - return s.fds[fd].ident != 0 + return s.fds[fd.int].ident != 0 proc getData*[T](s: Selector[T], fd: SocketHandle|int): var T = let fdi = int(fd) @@ -619,3 +627,7 @@ template withData*[T](s: Selector[T], fd: SocketHandle|int, value, body1, body1 else: body2 + + +proc getFd*[T](s: Selector[T]): int = + return s.kqFD.int \ No newline at end of file diff --git a/lib/pure/ioselects/ioselectors_poll.nim b/lib/pure/ioselects/ioselectors_poll.nim index cc06aa592..c36750c8d 100644 --- a/lib/pure/ioselects/ioselectors_poll.nim +++ b/lib/pure/ioselects/ioselectors_poll.nim @@ -40,16 +40,6 @@ type wfd: cint SelectEvent* = ptr SelectEventImpl -var RLIMIT_NOFILE {.importc: "RLIMIT_NOFILE", - header: "<sys/resource.h>".}: cint -type - rlimit {.importc: "struct rlimit", - header: "<sys/resource.h>", pure, final.} = object - rlim_cur: int - rlim_max: int -proc getrlimit(resource: cint, rlp: var rlimit): cint - {.importc: "getrlimit",header: "<sys/resource.h>".} - when hasThreadSupport: template withPollLock[T](s: Selector[T], body: untyped) = acquire(s.lock) @@ -63,8 +53,8 @@ else: body proc newSelector*[T](): Selector[T] = - var a = rlimit() - if getrlimit(RLIMIT_NOFILE, a) != 0: + var a = RLimit() + if getrlimit(posix.RLIMIT_NOFILE, a) != 0: raiseIOSelectorsError(osLastError()) var maxFD = int(a.rlim_max) @@ -141,7 +131,7 @@ template checkFd(s, f) = if f >= s.maxFD: raiseIOSelectorsError("Maximum number of descriptors is exhausted!") -proc registerHandle*[T](s: Selector[T], fd: SocketHandle, +proc registerHandle*[T](s: Selector[T], fd: int | SocketHandle, events: set[Event], data: T) = var fdi = int(fd) s.checkFd(fdi) @@ -149,7 +139,7 @@ proc registerHandle*[T](s: Selector[T], fd: SocketHandle, setKey(s, fdi, events, 0, data) if events != {}: s.pollAdd(fdi.cint, events) -proc updateHandle*[T](s: Selector[T], fd: SocketHandle, +proc updateHandle*[T](s: Selector[T], fd: int | SocketHandle, events: set[Event]) = let maskEvents = {Event.Timer, Event.Signal, Event.Process, Event.Vnode, Event.User, Event.Oneshot, Event.Error} @@ -280,7 +270,7 @@ template isEmpty*[T](s: Selector[T]): bool = (s.count == 0) proc contains*[T](s: Selector[T], fd: SocketHandle|int): bool {.inline.} = - return s.fds[fd].ident != 0 + return s.fds[fd.int].ident != 0 proc getData*[T](s: Selector[T], fd: SocketHandle|int): var T = let fdi = int(fd) @@ -314,3 +304,7 @@ template withData*[T](s: Selector[T], fd: SocketHandle|int, value, body1, body1 else: body2 + + +proc getFd*[T](s: Selector[T]): int = + return -1 diff --git a/lib/pure/ioselects/ioselectors_select.nim b/lib/pure/ioselects/ioselectors_select.nim index c787f0070..7ed250307 100644 --- a/lib/pure/ioselects/ioselectors_select.nim +++ b/lib/pure/ioselects/ioselectors_select.nim @@ -229,7 +229,7 @@ proc delKey[T](s: Selector[T], fd: SocketHandle) = doAssert(i < FD_SETSIZE, "Descriptor [" & $int(fd) & "] is not registered in the queue!") -proc registerHandle*[T](s: Selector[T], fd: SocketHandle, +proc registerHandle*[T](s: Selector[T], fd: int | SocketHandle, events: set[Event], data: T) = when not defined(windows): let fdi = int(fd) @@ -255,7 +255,7 @@ proc registerEvent*[T](s: Selector[T], ev: SelectEvent, data: T) = IOFD_SET(ev.rsock, addr s.rSet) inc(s.count) -proc updateHandle*[T](s: Selector[T], fd: SocketHandle, +proc updateHandle*[T](s: Selector[T], fd: int | SocketHandle, events: set[Event]) = let maskEvents = {Event.Timer, Event.Signal, Event.Process, Event.Vnode, Event.User, Event.Oneshot, Event.Error} @@ -453,3 +453,6 @@ template withData*[T](s: Selector[T], fd: SocketHandle|int, value, else: body2 + +proc getFd*[T](s: Selector[T]): int = + return -1 \ No newline at end of file diff --git a/lib/pure/json.nim b/lib/pure/json.nim index b5b84863a..e7ad5bd5a 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -89,510 +89,22 @@ ## echo j2 import - hashes, tables, strutils, lexbase, streams, unicode, macros + hashes, tables, strutils, lexbase, streams, unicode, macros, parsejson export tables.`$` +export + parsejson.JsonEventKind, parsejson.JsonError, JsonParser, JsonKindError, + open, close, str, getInt, getFloat, kind, getColumn, getLine, getFilename, + errorMsg, errorMsgExpected, next, JsonParsingError, raiseParseErr + when defined(nimJsonGet): {.pragma: deprecatedGet, deprecated.} else: {.pragma: deprecatedGet.} type - JsonEventKind* = enum ## enumeration of all events that may occur when parsing - jsonError, ## an error occurred during parsing - jsonEof, ## end of file reached - jsonString, ## a string literal - jsonInt, ## an integer literal - jsonFloat, ## a float literal - jsonTrue, ## the value ``true`` - jsonFalse, ## the value ``false`` - jsonNull, ## the value ``null`` - jsonObjectStart, ## start of an object: the ``{`` token - jsonObjectEnd, ## end of an object: the ``}`` token - jsonArrayStart, ## start of an array: the ``[`` token - jsonArrayEnd ## start of an array: the ``]`` token - - TokKind = enum # must be synchronized with TJsonEventKind! - tkError, - tkEof, - tkString, - tkInt, - tkFloat, - tkTrue, - tkFalse, - tkNull, - tkCurlyLe, - tkCurlyRi, - tkBracketLe, - tkBracketRi, - tkColon, - tkComma - - JsonError* = enum ## enumeration that lists all errors that can occur - errNone, ## no error - errInvalidToken, ## invalid token - errStringExpected, ## string expected - errColonExpected, ## ``:`` expected - errCommaExpected, ## ``,`` expected - errBracketRiExpected, ## ``]`` expected - errCurlyRiExpected, ## ``}`` expected - errQuoteExpected, ## ``"`` or ``'`` expected - errEOC_Expected, ## ``*/`` expected - errEofExpected, ## EOF expected - errExprExpected ## expr expected - - ParserState = enum - stateEof, stateStart, stateObject, stateArray, stateExpectArrayComma, - stateExpectObjectComma, stateExpectColon, stateExpectValue - - JsonParser* = object of BaseLexer ## the parser object. - a: string - tok: TokKind - kind: JsonEventKind - err: JsonError - state: seq[ParserState] - filename: string - - JsonKindError* = object of ValueError ## raised by the ``to`` macro if the - ## JSON kind is incorrect. - -{.deprecated: [TJsonEventKind: JsonEventKind, TJsonError: JsonError, - TJsonParser: JsonParser, TTokKind: TokKind].} - -const - errorMessages: array[JsonError, string] = [ - "no error", - "invalid token", - "string expected", - "':' expected", - "',' expected", - "']' expected", - "'}' expected", - "'\"' or \"'\" expected", - "'*/' expected", - "EOF expected", - "expression expected" - ] - tokToStr: array[TokKind, string] = [ - "invalid token", - "EOF", - "string literal", - "int literal", - "float literal", - "true", - "false", - "null", - "{", "}", "[", "]", ":", "," - ] - -proc open*(my: var JsonParser, input: Stream, filename: string) = - ## initializes the parser with an input stream. `Filename` is only used - ## for nice error messages. - lexbase.open(my, input) - my.filename = filename - my.state = @[stateStart] - my.kind = jsonError - my.a = "" - -proc close*(my: var JsonParser) {.inline.} = - ## closes the parser `my` and its associated input stream. - lexbase.close(my) - -proc str*(my: JsonParser): string {.inline.} = - ## returns the character data for the events: ``jsonInt``, ``jsonFloat``, - ## ``jsonString`` - assert(my.kind in {jsonInt, jsonFloat, jsonString}) - return my.a - -proc getInt*(my: JsonParser): BiggestInt {.inline.} = - ## returns the number for the event: ``jsonInt`` - assert(my.kind == jsonInt) - return parseBiggestInt(my.a) - -proc getFloat*(my: JsonParser): float {.inline.} = - ## returns the number for the event: ``jsonFloat`` - assert(my.kind == jsonFloat) - return parseFloat(my.a) - -proc kind*(my: JsonParser): JsonEventKind {.inline.} = - ## returns the current event type for the JSON parser - return my.kind - -proc getColumn*(my: JsonParser): int {.inline.} = - ## get the current column the parser has arrived at. - result = getColNumber(my, my.bufpos) - -proc getLine*(my: JsonParser): int {.inline.} = - ## get the current line the parser has arrived at. - result = my.lineNumber - -proc getFilename*(my: JsonParser): string {.inline.} = - ## get the filename of the file that the parser processes. - result = my.filename - -proc errorMsg*(my: JsonParser): string = - ## returns a helpful error message for the event ``jsonError`` - assert(my.kind == jsonError) - result = "$1($2, $3) Error: $4" % [ - my.filename, $getLine(my), $getColumn(my), errorMessages[my.err]] - -proc errorMsgExpected*(my: JsonParser, e: string): string = - ## returns an error message "`e` expected" in the same format as the - ## other error messages - result = "$1($2, $3) Error: $4" % [ - my.filename, $getLine(my), $getColumn(my), e & " expected"] - -proc handleHexChar(c: char, x: var int): bool = - result = true # Success - case c - of '0'..'9': x = (x shl 4) or (ord(c) - ord('0')) - of 'a'..'f': x = (x shl 4) or (ord(c) - ord('a') + 10) - of 'A'..'F': x = (x shl 4) or (ord(c) - ord('A') + 10) - else: result = false # error - -proc parseEscapedUTF16(buf: cstring, pos: var int): int = - result = 0 - #UTF-16 escape is always 4 bytes. - for _ in 0..3: - if handleHexChar(buf[pos], result): - inc(pos) - else: - return -1 - -proc parseString(my: var JsonParser): TokKind = - result = tkString - var pos = my.bufpos + 1 - var buf = my.buf - while true: - case buf[pos] - of '\0': - my.err = errQuoteExpected - result = tkError - break - of '"': - inc(pos) - break - of '\\': - case buf[pos+1] - of '\\', '"', '\'', '/': - add(my.a, buf[pos+1]) - inc(pos, 2) - of 'b': - add(my.a, '\b') - inc(pos, 2) - of 'f': - add(my.a, '\f') - inc(pos, 2) - of 'n': - add(my.a, '\L') - inc(pos, 2) - of 'r': - add(my.a, '\C') - inc(pos, 2) - of 't': - add(my.a, '\t') - inc(pos, 2) - of 'u': - inc(pos, 2) - var r = parseEscapedUTF16(buf, pos) - if r < 0: - my.err = errInvalidToken - break - # Deal with surrogates - if (r and 0xfc00) == 0xd800: - if buf[pos] & buf[pos+1] != "\\u": - my.err = errInvalidToken - break - inc(pos, 2) - var s = parseEscapedUTF16(buf, pos) - if (s and 0xfc00) == 0xdc00 and s > 0: - r = 0x10000 + (((r - 0xd800) shl 10) or (s - 0xdc00)) - else: - my.err = errInvalidToken - break - add(my.a, toUTF8(Rune(r))) - else: - # don't bother with the error - add(my.a, buf[pos]) - inc(pos) - of '\c': - pos = lexbase.handleCR(my, pos) - buf = my.buf - add(my.a, '\c') - of '\L': - pos = lexbase.handleLF(my, pos) - buf = my.buf - add(my.a, '\L') - else: - add(my.a, buf[pos]) - inc(pos) - my.bufpos = pos # store back - -proc skip(my: var JsonParser) = - var pos = my.bufpos - var buf = my.buf - while true: - case buf[pos] - of '/': - if buf[pos+1] == '/': - # skip line comment: - inc(pos, 2) - while true: - case buf[pos] - of '\0': - break - of '\c': - pos = lexbase.handleCR(my, pos) - buf = my.buf - break - of '\L': - pos = lexbase.handleLF(my, pos) - buf = my.buf - break - else: - inc(pos) - elif buf[pos+1] == '*': - # skip long comment: - inc(pos, 2) - while true: - case buf[pos] - of '\0': - my.err = errEOC_Expected - break - of '\c': - pos = lexbase.handleCR(my, pos) - buf = my.buf - of '\L': - pos = lexbase.handleLF(my, pos) - buf = my.buf - of '*': - inc(pos) - if buf[pos] == '/': - inc(pos) - break - else: - inc(pos) - else: - break - of ' ', '\t': - inc(pos) - of '\c': - pos = lexbase.handleCR(my, pos) - buf = my.buf - of '\L': - pos = lexbase.handleLF(my, pos) - buf = my.buf - else: - break - my.bufpos = pos - -proc parseNumber(my: var JsonParser) = - var pos = my.bufpos - var buf = my.buf - if buf[pos] == '-': - add(my.a, '-') - inc(pos) - if buf[pos] == '.': - add(my.a, "0.") - inc(pos) - else: - while buf[pos] in Digits: - add(my.a, buf[pos]) - inc(pos) - if buf[pos] == '.': - add(my.a, '.') - inc(pos) - # digits after the dot: - while buf[pos] in Digits: - add(my.a, buf[pos]) - inc(pos) - if buf[pos] in {'E', 'e'}: - add(my.a, buf[pos]) - inc(pos) - if buf[pos] in {'+', '-'}: - add(my.a, buf[pos]) - inc(pos) - while buf[pos] in Digits: - add(my.a, buf[pos]) - inc(pos) - my.bufpos = pos - -proc parseName(my: var JsonParser) = - var pos = my.bufpos - var buf = my.buf - if buf[pos] in IdentStartChars: - while buf[pos] in IdentChars: - add(my.a, buf[pos]) - inc(pos) - my.bufpos = pos - -proc getTok(my: var JsonParser): TokKind = - setLen(my.a, 0) - skip(my) # skip whitespace, comments - case my.buf[my.bufpos] - of '-', '.', '0'..'9': - parseNumber(my) - if {'.', 'e', 'E'} in my.a: - result = tkFloat - else: - result = tkInt - of '"': - result = parseString(my) - of '[': - inc(my.bufpos) - result = tkBracketLe - of '{': - inc(my.bufpos) - result = tkCurlyLe - of ']': - inc(my.bufpos) - result = tkBracketRi - of '}': - inc(my.bufpos) - result = tkCurlyRi - of ',': - inc(my.bufpos) - result = tkComma - of ':': - inc(my.bufpos) - result = tkColon - of '\0': - result = tkEof - of 'a'..'z', 'A'..'Z', '_': - parseName(my) - case my.a - of "null": result = tkNull - of "true": result = tkTrue - of "false": result = tkFalse - else: result = tkError - else: - inc(my.bufpos) - result = tkError - my.tok = result - -proc next*(my: var JsonParser) = - ## retrieves the first/next event. This controls the parser. - var tk = getTok(my) - var i = my.state.len-1 - # the following code is a state machine. If we had proper coroutines, - # the code could be much simpler. - case my.state[i] - of stateEof: - if tk == tkEof: - my.kind = jsonEof - else: - my.kind = jsonError - my.err = errEofExpected - of stateStart: - # tokens allowed? - case tk - of tkString, tkInt, tkFloat, tkTrue, tkFalse, tkNull: - my.state[i] = stateEof # expect EOF next! - my.kind = JsonEventKind(ord(tk)) - of tkBracketLe: - my.state.add(stateArray) # we expect any - my.kind = jsonArrayStart - of tkCurlyLe: - my.state.add(stateObject) - my.kind = jsonObjectStart - of tkEof: - my.kind = jsonEof - else: - my.kind = jsonError - my.err = errEofExpected - of stateObject: - case tk - of tkString, tkInt, tkFloat, tkTrue, tkFalse, tkNull: - my.state.add(stateExpectColon) - my.kind = JsonEventKind(ord(tk)) - of tkBracketLe: - my.state.add(stateExpectColon) - my.state.add(stateArray) - my.kind = jsonArrayStart - of tkCurlyLe: - my.state.add(stateExpectColon) - my.state.add(stateObject) - my.kind = jsonObjectStart - of tkCurlyRi: - my.kind = jsonObjectEnd - discard my.state.pop() - else: - my.kind = jsonError - my.err = errCurlyRiExpected - of stateArray: - case tk - of tkString, tkInt, tkFloat, tkTrue, tkFalse, tkNull: - my.state.add(stateExpectArrayComma) # expect value next! - my.kind = JsonEventKind(ord(tk)) - of tkBracketLe: - my.state.add(stateExpectArrayComma) - my.state.add(stateArray) - my.kind = jsonArrayStart - of tkCurlyLe: - my.state.add(stateExpectArrayComma) - my.state.add(stateObject) - my.kind = jsonObjectStart - of tkBracketRi: - my.kind = jsonArrayEnd - discard my.state.pop() - else: - my.kind = jsonError - my.err = errBracketRiExpected - of stateExpectArrayComma: - case tk - of tkComma: - discard my.state.pop() - next(my) - of tkBracketRi: - my.kind = jsonArrayEnd - discard my.state.pop() # pop stateExpectArrayComma - discard my.state.pop() # pop stateArray - else: - my.kind = jsonError - my.err = errBracketRiExpected - of stateExpectObjectComma: - case tk - of tkComma: - discard my.state.pop() - next(my) - of tkCurlyRi: - my.kind = jsonObjectEnd - discard my.state.pop() # pop stateExpectObjectComma - discard my.state.pop() # pop stateObject - else: - my.kind = jsonError - my.err = errCurlyRiExpected - of stateExpectColon: - case tk - of tkColon: - my.state[i] = stateExpectValue - next(my) - else: - my.kind = jsonError - my.err = errColonExpected - of stateExpectValue: - case tk - of tkString, tkInt, tkFloat, tkTrue, tkFalse, tkNull: - my.state[i] = stateExpectObjectComma - my.kind = JsonEventKind(ord(tk)) - of tkBracketLe: - my.state[i] = stateExpectObjectComma - my.state.add(stateArray) - my.kind = jsonArrayStart - of tkCurlyLe: - my.state[i] = stateExpectObjectComma - my.state.add(stateObject) - my.kind = jsonObjectStart - else: - my.kind = jsonError - my.err = errExprExpected - - -# ------------- higher level interface --------------------------------------- - -type JsonNodeKind* = enum ## possible JSON node types JNull, JBool, @@ -620,15 +132,6 @@ type of JArray: elems*: seq[JsonNode] - JsonParsingError* = object of ValueError ## is raised for a JSON error - -{.deprecated: [EJsonParsingError: JsonParsingError, TJsonNode: JsonNodeObj, - PJsonNode: JsonNode, TJsonNodeKind: JsonNodeKind].} - -proc raiseParseErr*(p: JsonParser, msg: string) {.noinline, noreturn.} = - ## raises an `EJsonParsingError` exception. - raise newException(JsonParsingError, errorMsgExpected(p, msg)) - proc newJString*(s: string): JsonNode = ## Creates a new `JString JsonNode`. new(result) @@ -695,8 +198,8 @@ proc getBiggestInt*(n: JsonNode, default: BiggestInt = 0): BiggestInt = if n.isNil or n.kind != JInt: return default else: return n.num -proc getNum*(n: JsonNode, default: BiggestInt = 0): BiggestInt {.deprecated.} = - ## Deprecated - use getInt or getBiggestInt instead +proc getNum*(n: JsonNode, default: BiggestInt = 0): BiggestInt {.deprecated: "use getInt or getBiggestInt instead".} = + ## **Deprecated since v0.18.2:** use ``getInt`` or ``getBiggestInt`` instead. getBiggestInt(n, default) proc getFloat*(n: JsonNode, default: float = 0.0): float = @@ -709,8 +212,8 @@ proc getFloat*(n: JsonNode, default: float = 0.0): float = of JInt: return float(n.num) else: return default -proc getFNum*(n: JsonNode, default: float = 0.0): float {.deprecated.} = - ## Deprecated - use getFloat instead +proc getFNum*(n: JsonNode, default: float = 0.0): float {.deprecated: "use getFloat instead".} = + ## **Deprecated since v0.18.2:** use ``getFloat`` instead. getFloat(n, default) proc getBool*(n: JsonNode, default: bool = false): bool = @@ -720,8 +223,8 @@ proc getBool*(n: JsonNode, default: bool = false): bool = if n.isNil or n.kind != JBool: return default else: return n.bval -proc getBVal*(n: JsonNode, default: bool = false): bool {.deprecated.} = - ## Deprecated - use getBVal instead +proc getBVal*(n: JsonNode, default: bool = false): bool {.deprecated: "use getBool instead".} = + ## **Deprecated since v0.18.2:** use ``getBool`` instead. getBool(n, default) proc getFields*(n: JsonNode, @@ -734,7 +237,7 @@ proc getFields*(n: JsonNode, else: return n.fields proc getElems*(n: JsonNode, default: seq[JsonNode] = @[]): seq[JsonNode] = - ## Retrieves the int value of a `JArray JsonNode`. + ## Retrieves the array of a `JArray JsonNode`. ## ## Returns ``default`` if ``n`` is not a ``JArray``, or if ``n`` is nil. if n.isNil or n.kind != JArray: return default @@ -838,6 +341,9 @@ proc toJson(x: NimNode): NimNode {.compiletime.} = result = newCall(bindSym"newJObject") of nnkNilLit: result = newCall(bindSym"newJNull") + of nnkPar: + if x.len == 1: result = toJson(x[0]) + else: result = newCall(bindSym"%", x) else: result = newCall(bindSym"%", x) @@ -946,8 +452,8 @@ proc contains*(node: JsonNode, val: JsonNode): bool = assert(node.kind == JArray) find(node.elems, val) >= 0 -proc existsKey*(node: JsonNode, key: string): bool {.deprecated.} = node.hasKey(key) - ## Deprecated for `hasKey` +proc existsKey*(node: JsonNode, key: string): bool {.deprecated: "use hasKey instead".} = node.hasKey(key) + ## **Deprecated:** use `hasKey` instead. proc `[]=`*(obj: JsonNode, key: string, val: JsonNode) {.inline.} = ## Sets a field from a `JObject`. @@ -958,12 +464,29 @@ proc `{}`*(node: JsonNode, keys: varargs[string]): JsonNode = ## Traverses the node and gets the given value. If any of the ## keys do not exist, returns ``nil``. Also returns ``nil`` if one of the ## intermediate data structures is not an object. + ## + ## This proc can be used to create tree structures on the + ## fly (sometimes called `autovivification`:idx:): + ## + ## .. code-block:: nim + ## myjson{"parent", "child", "grandchild"} = newJInt(1) + ## result = node for key in keys: if isNil(result) or result.kind != JObject: return nil result = result.fields.getOrDefault(key) +proc `{}`*(node: JsonNode, index: varargs[int]): JsonNode = + ## Traverses the node and gets the given value. If any of the + ## indexes do not exist, returns ``nil``. Also returns ``nil`` if one of the + ## intermediate data structures is not an array. + result = node + for i in index: + if isNil(result) or result.kind != JArray or i >= node.len: + return nil + result = result.elems[i] + proc getOrDefault*(node: JsonNode, key: string): JsonNode = ## Gets a field from a `node`. If `node` is nil or not an object or ## value at `key` does not exist, returns nil @@ -986,7 +509,7 @@ proc delete*(obj: JsonNode, key: string) = ## Deletes ``obj[key]``. assert(obj.kind == JObject) if not obj.fields.hasKey(key): - raise newException(IndexError, "key not in object") + raise newException(KeyError, "key not in object") obj.fields.del(key) proc copy*(p: JsonNode): JsonNode = @@ -1035,6 +558,8 @@ proc escapeJson*(s: string; result: var string) = of '\t': result.add("\\t") of '\r': result.add("\\r") of '"': result.add("\\\"") + of '\0'..'\7': result.add("\\u000" & $ord(c)) + of '\14'..'\31': result.add("\\u00" & $ord(c)) of '\\': result.add("\\\\") else: result.add(c) result.add("\"") @@ -1180,10 +705,6 @@ iterator mpairs*(node: var JsonNode): tuple[key: string, val: var JsonNode] = for key, val in mpairs(node.fields): yield (key, val) -proc eat(p: var JsonParser, tok: TokKind) = - if p.tok == tok: discard getTok(p) - else: raiseParseErr(p, tokToStr[tok]) - proc parseJson(p: var JsonParser): JsonNode = ## Parses JSON from a JSON Parser `p`. case p.tok @@ -1239,10 +760,12 @@ when not defined(js): ## If `s` contains extra data, it will raise `JsonParsingError`. var p: JsonParser p.open(s, filename) - defer: p.close() - discard getTok(p) # read first token - result = p.parseJson() - eat(p, tkEof) # check if there is no extra data + try: + discard getTok(p) # read first token + result = p.parseJson() + eat(p, tkEof) # check if there is no extra data + finally: + p.close() proc parseJson*(buffer: string): JsonNode = ## Parses JSON from `buffer`. @@ -1260,7 +783,6 @@ else: from math import `mod` type JSObject = object - {.deprecated: [TJSObject: JSObject].} proc parseNativeJson(x: cstring): JSObject {.importc: "JSON.parse".} @@ -1687,7 +1209,7 @@ proc createConstructor(typeSym, jsonNode: NimNode): NimNode = result = quote do: ( - if `lenientJsonNode`.isNil: `workaround`[`optionGeneric`]() else: some[`optionGeneric`](`value`) + if `lenientJsonNode`.isNil or `jsonNode`.kind == JNull: `workaround`[`optionGeneric`]() else: some[`optionGeneric`](`value`) ) of "table", "orderedtable": let tableKeyType = typeSym[1] @@ -1769,7 +1291,7 @@ proc createConstructor(typeSym, jsonNode: NimNode): NimNode = result = processType(typeSym, obj, jsonNode, false) of nnkTupleTy: result = processType(typeSym, typeSym, jsonNode, false) - of nnkPar: + of nnkPar, nnkTupleConstr: # TODO: The fact that `jsonNode` here works to give a good line number # is weird. Specifying typeSym should work but doesn't. error("Use a named tuple instead of: " & $toStrLit(typeSym), jsonNode) @@ -1896,7 +1418,7 @@ macro to*(node: JsonNode, T: typedesc): untyped = ## doAssert data.person.age == 21 ## doAssert data.list == @[1, 2, 3, 4] - let typeNode = getType(T) + let typeNode = getTypeInst(T) expectKind(typeNode, nnkBracketExpr) doAssert(($typeNode[0]).normalize == "typedesc") @@ -1976,18 +1498,18 @@ when isMainModule: # Bounds checking try: let a = testJson["a"][9] - doAssert(false, "EInvalidIndex not thrown") + doAssert(false, "IndexError not thrown") except IndexError: discard try: let a = testJson["a"][-1] - doAssert(false, "EInvalidIndex not thrown") + doAssert(false, "IndexError not thrown") except IndexError: discard try: doAssert(testJson["a"][0].num == 1, "Index doesn't correspond to its value") except: - doAssert(false, "EInvalidIndex thrown for valid index") + doAssert(false, "IndexError thrown for valid index") doAssert(testJson{"b"}.str=="asd", "Couldn't fetch a singly nested key with {}") doAssert(isNil(testJson{"nonexistent"}), "Non-existent keys should return nil") @@ -2061,6 +1583,7 @@ when isMainModule: doAssert(parsed2{"repository", "description"}.str=="IRC Library for Haskell", "Couldn't fetch via multiply nested key using {}") doAssert escapeJson("\10Foo🎃barÄ") == "\"\\nFoo🎃barÄ\"" + doAssert escapeJson("\0\7\20") == "\"\\u0000\\u0007\\u0020\"" # for #7887 # Test with extra data when not defined(js): diff --git a/lib/pure/lenientops.nim b/lib/pure/lenientops.nim index b124a9512..cc7784c19 100644 --- a/lib/pure/lenientops.nim +++ b/lib/pure/lenientops.nim @@ -26,33 +26,33 @@ import typetraits -proc `+`*[I: SomeInteger, F: SomeReal](i: I, f: F): F {.noSideEffect, inline.} = +proc `+`*[I: SomeInteger, F: SomeFloat](i: I, f: F): F {.noSideEffect, inline.} = F(i) + f -proc `+`*[I: SomeInteger, F: SomeReal](f: F, i: I): F {.noSideEffect, inline.} = +proc `+`*[I: SomeInteger, F: SomeFloat](f: F, i: I): F {.noSideEffect, inline.} = f + F(i) -proc `-`*[I: SomeInteger, F: SomeReal](i: I, f: F): F {.noSideEffect, inline.} = +proc `-`*[I: SomeInteger, F: SomeFloat](i: I, f: F): F {.noSideEffect, inline.} = F(i) - f -proc `-`*[I: SomeInteger, F: SomeReal](f: F, i: I): F {.noSideEffect, inline.} = +proc `-`*[I: SomeInteger, F: SomeFloat](f: F, i: I): F {.noSideEffect, inline.} = f - F(i) -proc `*`*[I: SomeInteger, F: SomeReal](i: I, f: F): F {.noSideEffect, inline.} = +proc `*`*[I: SomeInteger, F: SomeFloat](i: I, f: F): F {.noSideEffect, inline.} = F(i) * f -proc `*`*[I: SomeInteger, F: SomeReal](f: F, i: I): F {.noSideEffect, inline.} = +proc `*`*[I: SomeInteger, F: SomeFloat](f: F, i: I): F {.noSideEffect, inline.} = f * F(i) -proc `/`*[I: SomeInteger, F: SomeReal](i: I, f: F): F {.noSideEffect, inline.} = +proc `/`*[I: SomeInteger, F: SomeFloat](i: I, f: F): F {.noSideEffect, inline.} = F(i) / f -proc `/`*[I: SomeInteger, F: SomeReal](f: F, i: I): F {.noSideEffect, inline.} = +proc `/`*[I: SomeInteger, F: SomeFloat](f: F, i: I): F {.noSideEffect, inline.} = f / F(i) -proc `<`*[I: SomeInteger, F: SomeReal](i: I, f: F): bool {.noSideEffect, inline.} = +proc `<`*[I: SomeInteger, F: SomeFloat](i: I, f: F): bool {.noSideEffect, inline.} = F(i) < f -proc `<`*[I: SomeInteger, F: SomeReal](f: F, i: I): bool {.noSideEffect, inline.} = +proc `<`*[I: SomeInteger, F: SomeFloat](f: F, i: I): bool {.noSideEffect, inline.} = f < F(i) -proc `<=`*[I: SomeInteger, F: SomeReal](i: I, f: F): bool {.noSideEffect, inline.} = +proc `<=`*[I: SomeInteger, F: SomeFloat](i: I, f: F): bool {.noSideEffect, inline.} = F(i) <= f -proc `<=`*[I: SomeInteger, F: SomeReal](f: F, i: I): bool {.noSideEffect, inline.} = +proc `<=`*[I: SomeInteger, F: SomeFloat](f: F, i: I): bool {.noSideEffect, inline.} = f <= F(i) # Note that we must not defined `>=` and `>`, because system.nim already has a diff --git a/lib/pure/lexbase.nim b/lib/pure/lexbase.nim index 15a390f0b..e38acd5ef 100644 --- a/lib/pure/lexbase.nim +++ b/lib/pure/lexbase.nim @@ -40,8 +40,6 @@ type offsetBase*: int # use ``offsetBase + bufpos`` to get the offset refillChars: set[char] -{.deprecated: [TBaseLexer: BaseLexer].} - const chrSize = sizeof(char) diff --git a/lib/pure/logging.nim b/lib/pure/logging.nim index 830820fd1..cdff1f548 100644 --- a/lib/pure/logging.nim +++ b/lib/pure/logging.nim @@ -96,10 +96,6 @@ when not defined(js): logFiles: int # how many log files already created, e.g. basename.1, basename.2... bufSize: int # size of output buffer (-1: use system defaults, 0: unbuffered, >0: fixed buffer size) - {.deprecated: [PFileLogger: FileLogger, PRollingFileLogger: RollingFileLogger].} - -{.deprecated: [TLevel: Level, PLogger: Logger, PConsoleLogger: ConsoleLogger].} - var level {.threadvar.}: Level ## global log filter handlers {.threadvar.}: seq[Logger] ## handlers with their own log levels @@ -107,9 +103,14 @@ var proc substituteLog*(frmt: string, level: Level, args: varargs[string, `$`]): string = ## Format a log message using the ``frmt`` format string, ``level`` and varargs. ## See the module documentation for the format string syntax. + const nilString = "nil" + var msgLen = 0 for arg in args: - msgLen += arg.len + if arg.isNil: + msgLen += nilString.len + else: + msgLen += arg.len result = newStringOfCap(frmt.len + msgLen + 20) var i = 0 while i < frmt.len: @@ -121,7 +122,7 @@ proc substituteLog*(frmt: string, level: Level, args: varargs[string, `$`]): str var v = "" let app = when defined(js): "" else: getAppFilename() while frmt[i] in IdentChars: - v.add(toLower(frmt[i])) + v.add(toLowerAscii(frmt[i])) inc(i) case v of "date": result.add(getDateStr()) @@ -136,7 +137,10 @@ proc substituteLog*(frmt: string, level: Level, args: varargs[string, `$`]): str of "levelname": result.add(LevelNames[level]) else: discard for arg in args: - result.add(arg) + if arg.isNil: + result.add(nilString) + else: + result.add(arg) method log*(logger: Logger, level: Level, args: varargs[string, `$`]) {. raises: [Exception], gcsafe, @@ -361,3 +365,6 @@ when not defined(testing) and isMainModule: addHandler(L) for i in 0 .. 25: info("hello", i) + + var nilString: string + info "hello ", nilString diff --git a/lib/pure/marshal.nim b/lib/pure/marshal.nim index 6ee830786..b88c9dd85 100644 --- a/lib/pure/marshal.nim +++ b/lib/pure/marshal.nim @@ -309,7 +309,6 @@ when not defined(testing) and isMainModule: Node = object next, prev: PNode data: string - {.deprecated: [TNode: Node].} proc buildList(): PNode = new(result) diff --git a/lib/pure/matchers.nim b/lib/pure/matchers.nim index 6366fee1a..97223ed01 100644 --- a/lib/pure/matchers.nim +++ b/lib/pure/matchers.nim @@ -12,7 +12,7 @@ ## **Warning:** This module is deprecated since version 0.14.0. {.deprecated.} -{.deadCodeElim: on.} +{.deadCodeElim: on.} # dce option deprecated {.push debugger:off .} # the user does not want to trace a part # of the standard library! @@ -29,21 +29,21 @@ proc validEmailAddress*(s: string): bool {.noSideEffect, chars = Letters + Digits + {'!','#','$','%','&', '\'','*','+','/','=','?','^','_','`','{','}','|','~','-','.'} var i = 0 - if s[i] notin chars or s[i] == '.': return false - while s[i] in chars: - if s[i] == '.' and s[i+1] == '.': return false + if i >= s.len or s[i] notin chars or s[i] == '.': return false + while i < s.len and s[i] in chars: + if i+1 < s.len and s[i] == '.' and s[i+1] == '.': return false inc(i) - if s[i] != '@': return false + if i >= s.len or s[i] != '@': return false var j = len(s)-1 - if s[j] notin Letters: return false + if j >= 0 and s[j] notin Letters: return false while j >= i and s[j] in Letters: dec(j) inc(i) # skip '@' - while s[i] in {'0'..'9', 'a'..'z', '-', '.'}: inc(i) - if s[i] != '\0': return false + while i < s.len and s[i] in {'0'..'9', 'a'..'z', '-', '.'}: inc(i) + if i != s.len: return false var x = substr(s, j+1) if len(x) == 2 and x[0] in Letters and x[1] in Letters: return true - case toLower(x) + case toLowerAscii(x) of "com", "org", "net", "gov", "mil", "biz", "info", "mobi", "name", "aero", "jobs", "museum": return true else: return false diff --git a/lib/pure/math.nim b/lib/pure/math.nim index cbd04a145..8ea8ee203 100644 --- a/lib/pure/math.nim +++ b/lib/pure/math.nim @@ -21,6 +21,8 @@ include "system/inclrtl" {.push debugger:off .} # the user does not want to trace a part # of the standard library! +import bitops + proc binom*(n, k: int): int {.noSideEffect.} = ## Computes the binomial coefficient if k <= 0: return 1 @@ -29,11 +31,21 @@ proc binom*(n, k: int): int {.noSideEffect.} = for i in countup(2, k): result = (result * (n + 1 - i)) div i -proc fac*(n: int): int {.noSideEffect.} = +proc createFactTable[N: static[int]]: array[N, int] = + result[0] = 1 + for i in 1 ..< N: + result[i] = result[i - 1] * i + +proc fac*(n: int): int = ## Computes the faculty/factorial function. - result = 1 - for i in countup(2, n): - result = result * i + const factTable = + when sizeof(int) == 4: + createFactTable[13]() + else: + createFactTable[21]() + assert(n >= 0, $n & " must not be negative.") + assert(n < factTable.len, $n & " is too large to look up in the table") + factTable[n] {.push checks:off, line_dir:off, stack_trace:off.} @@ -117,8 +129,14 @@ proc sum*[T](x: openArray[T]): T {.noSideEffect.} = ## If `x` is empty, 0 is returned. for i in items(x): result = result + i +proc prod*[T](x: openArray[T]): T {.noSideEffect.} = + ## Computes the product of the elements in ``x``. + ## If ``x`` is empty, 1 is returned. + result = 1.T + for i in items(x): result = result * i + {.push noSideEffect.} -when not defined(JS): +when not defined(JS): # C proc sqrt*(x: float32): float32 {.importc: "sqrtf", header: "<math.h>".} proc sqrt*(x: float64): float64 {.importc: "sqrt", header: "<math.h>".} ## Computes the square root of `x`. @@ -132,12 +150,33 @@ when not defined(JS): proc log10*(x: float32): float32 {.importc: "log10f", header: "<math.h>".} proc log10*(x: float64): float64 {.importc: "log10", header: "<math.h>".} ## Computes the common logarithm (base 10) of `x` - proc log2*[T: float32|float64](x: T): T = return ln(x) / ln(2.0) + proc log2*(x: float32): float32 {.importc: "log2f", header: "<math.h>".} + proc log2*(x: float64): float64 {.importc: "log2", header: "<math.h>".} ## Computes the binary logarithm (base 2) of `x` proc exp*(x: float32): float32 {.importc: "expf", header: "<math.h>".} proc exp*(x: float64): float64 {.importc: "exp", header: "<math.h>".} ## Computes the exponential function of `x` (pow(E, x)) + proc sin*(x: float32): float32 {.importc: "sinf", header: "<math.h>".} + proc sin*(x: float64): float64 {.importc: "sin", header: "<math.h>".} + ## Computes the sine of `x` + proc cos*(x: float32): float32 {.importc: "cosf", header: "<math.h>".} + proc cos*(x: float64): float64 {.importc: "cos", header: "<math.h>".} + ## Computes the cosine of `x` + proc tan*(x: float32): float32 {.importc: "tanf", header: "<math.h>".} + proc tan*(x: float64): float64 {.importc: "tan", header: "<math.h>".} + ## Computes the tangent of `x` + + proc sinh*(x: float32): float32 {.importc: "sinhf", header: "<math.h>".} + proc sinh*(x: float64): float64 {.importc: "sinh", header: "<math.h>".} + ## Computes the hyperbolic sine of `x` + proc cosh*(x: float32): float32 {.importc: "coshf", header: "<math.h>".} + proc cosh*(x: float64): float64 {.importc: "cosh", header: "<math.h>".} + ## Computes the hyperbolic cosine of `x` + proc tanh*(x: float32): float32 {.importc: "tanhf", header: "<math.h>".} + proc tanh*(x: float64): float64 {.importc: "tanh", header: "<math.h>".} + ## Computes the hyperbolic tangent of `x` + proc arccos*(x: float32): float32 {.importc: "acosf", header: "<math.h>".} proc arccos*(x: float64): float64 {.importc: "acos", header: "<math.h>".} ## Computes the arc cosine of `x` @@ -154,33 +193,80 @@ when not defined(JS): ## results even when the resulting angle is near pi/2 or -pi/2 ## (`x` near 0). - proc cos*(x: float32): float32 {.importc: "cosf", header: "<math.h>".} - proc cos*(x: float64): float64 {.importc: "cos", header: "<math.h>".} - ## Computes the cosine of `x` + proc arcsinh*(x: float32): float32 {.importc: "asinhf", header: "<math.h>".} + proc arcsinh*(x: float64): float64 {.importc: "asinh", header: "<math.h>".} + ## Computes the inverse hyperbolic sine of `x` + proc arccosh*(x: float32): float32 {.importc: "acoshf", header: "<math.h>".} + proc arccosh*(x: float64): float64 {.importc: "acosh", header: "<math.h>".} + ## Computes the inverse hyperbolic cosine of `x` + proc arctanh*(x: float32): float32 {.importc: "atanhf", header: "<math.h>".} + proc arctanh*(x: float64): float64 {.importc: "atanh", header: "<math.h>".} + ## Computes the inverse hyperbolic tangent of `x` + +else: # JS + proc sqrt*(x: float32): float32 {.importc: "Math.sqrt", nodecl.} + proc sqrt*(x: float64): float64 {.importc: "Math.sqrt", nodecl.} - proc cosh*(x: float32): float32 {.importc: "coshf", header: "<math.h>".} - proc cosh*(x: float64): float64 {.importc: "cosh", header: "<math.h>".} - ## Computes the hyperbolic cosine of `x` + proc ln*(x: float32): float32 {.importc: "Math.log", nodecl.} + proc ln*(x: float64): float64 {.importc: "Math.log", nodecl.} + proc log10*(x: float32): float32 {.importc: "Math.log10", nodecl.} + proc log10*(x: float64): float64 {.importc: "Math.log10", nodecl.} + proc log2*(x: float32): float32 {.importc: "Math.log2", nodecl.} + proc log2*(x: float64): float64 {.importc: "Math.log2", nodecl.} + proc exp*(x: float32): float32 {.importc: "Math.exp", nodecl.} + proc exp*(x: float64): float64 {.importc: "Math.exp", nodecl.} + proc sin*[T: float32|float64](x: T): T {.importc: "Math.sin", nodecl.} + proc cos*[T: float32|float64](x: T): T {.importc: "Math.cos", nodecl.} + proc tan*[T: float32|float64](x: T): T {.importc: "Math.tan", nodecl.} + + proc sinh*[T: float32|float64](x: T): T {.importc: "Math.sinh", nodecl.} + proc cosh*[T: float32|float64](x: T): T {.importc: "Math.cosh", nodecl.} + proc tanh*[T: float32|float64](x: T): T {.importc: "Math.tanh", nodecl.} + + proc arcsin*[T: float32|float64](x: T): T {.importc: "Math.asin", nodecl.} + proc arccos*[T: float32|float64](x: T): T {.importc: "Math.acos", nodecl.} + proc arctan*[T: float32|float64](x: T): T {.importc: "Math.atan", nodecl.} + proc arctan2*[T: float32|float64](y, x: T): T {.importC: "Math.atan2", nodecl.} + + proc arcsinh*[T: float32|float64](x: T): T {.importc: "Math.asinh", nodecl.} + proc arccosh*[T: float32|float64](x: T): T {.importc: "Math.acosh", nodecl.} + proc arctanh*[T: float32|float64](x: T): T {.importc: "Math.atanh", nodecl.} + +proc cot*[T: float32|float64](x: T): T = 1.0 / tan(x) + ## Computes the cotangent of `x` +proc sec*[T: float32|float64](x: T): T = 1.0 / cos(x) + ## Computes the secant of `x`. +proc csc*[T: float32|float64](x: T): T = 1.0 / sin(x) + ## Computes the cosecant of `x` + +proc coth*[T: float32|float64](x: T): T = 1.0 / tanh(x) + ## Computes the hyperbolic cotangent of `x` +proc sech*[T: float32|float64](x: T): T = 1.0 / cosh(x) + ## Computes the hyperbolic secant of `x` +proc csch*[T: float32|float64](x: T): T = 1.0 / sinh(x) + ## Computes the hyperbolic cosecant of `x` + +proc arccot*[T: float32|float64](x: T): T = arctan(1.0 / x) + ## Computes the inverse cotangent of `x` +proc arcsec*[T: float32|float64](x: T): T = arccos(1.0 / x) + ## Computes the inverse secant of `x` +proc arccsc*[T: float32|float64](x: T): T = arcsin(1.0 / x) + ## Computes the inverse cosecant of `x` + +proc arccoth*[T: float32|float64](x: T): T = arctanh(1.0 / x) + ## Computes the inverse hyperbolic cotangent of `x` +proc arcsech*[T: float32|float64](x: T): T = arccosh(1.0 / x) + ## Computes the inverse hyperbolic secant of `x` +proc arccsch*[T: float32|float64](x: T): T = arcsinh(1.0 / x) + ## Computes the inverse hyperbolic cosecant of `x` + +when not defined(JS): # C proc hypot*(x, y: float32): float32 {.importc: "hypotf", header: "<math.h>".} proc hypot*(x, y: float64): float64 {.importc: "hypot", header: "<math.h>".} ## Computes the hypotenuse of a right-angle triangle with `x` and ## `y` as its base and height. Equivalent to ``sqrt(x*x + y*y)``. - proc sinh*(x: float32): float32 {.importc: "sinhf", header: "<math.h>".} - proc sinh*(x: float64): float64 {.importc: "sinh", header: "<math.h>".} - ## Computes the hyperbolic sine of `x` - proc sin*(x: float32): float32 {.importc: "sinf", header: "<math.h>".} - proc sin*(x: float64): float64 {.importc: "sin", header: "<math.h>".} - ## Computes the sine of `x` - - proc tan*(x: float32): float32 {.importc: "tanf", header: "<math.h>".} - proc tan*(x: float64): float64 {.importc: "tan", header: "<math.h>".} - ## Computes the tangent of `x` - proc tanh*(x: float32): float32 {.importc: "tanhf", header: "<math.h>".} - proc tanh*(x: float64): float64 {.importc: "tanh", header: "<math.h>".} - ## Computes the hyperbolic tangent of `x` - proc pow*(x, y: float32): float32 {.importc: "powf", header: "<math.h>".} proc pow*(x, y: float64): float64 {.importc: "pow", header: "<math.h>".} ## computes x to power raised of y. @@ -194,12 +280,18 @@ when not defined(JS): proc erfc*(x: float64): float64 {.importc: "erfc", header: "<math.h>".} ## The complementary error function + proc gamma*(x: float32): float32 {.importc: "tgammaf", header: "<math.h>".} + proc gamma*(x: float64): float64 {.importc: "tgamma", header: "<math.h>".} + ## The gamma function + proc tgamma*(x: float32): float32 + {.deprecated: "use gamma instead", importc: "tgammaf", header: "<math.h>".} + proc tgamma*(x: float64): float64 + {.deprecated: "use gamma instead", importc: "tgamma", header: "<math.h>".} + ## The gamma function + ## **Deprecated since version 0.19.0**: Use ``gamma`` instead. proc lgamma*(x: float32): float32 {.importc: "lgammaf", header: "<math.h>".} proc lgamma*(x: float64): float64 {.importc: "lgamma", header: "<math.h>".} ## Natural log of the gamma function - proc tgamma*(x: float32): float32 {.importc: "tgammaf", header: "<math.h>".} - proc tgamma*(x: float64): float64 {.importc: "tgamma", header: "<math.h>".} - ## The gamma function proc floor*(x: float32): float32 {.importc: "floorf", header: "<math.h>".} proc floor*(x: float64): float64 {.importc: "floor", header: "<math.h>".} @@ -283,57 +375,31 @@ when not defined(JS): ## .. code-block:: nim ## echo trunc(PI) # 3.0 - proc fmod*(x, y: float32): float32 {.importc: "fmodf", header: "<math.h>".} - proc fmod*(x, y: float64): float64 {.importc: "fmod", header: "<math.h>".} + proc fmod*(x, y: float32): float32 {.deprecated, importc: "fmodf", header: "<math.h>".} + proc fmod*(x, y: float64): float64 {.deprecated, importc: "fmod", header: "<math.h>".} ## Computes the remainder of `x` divided by `y` ## ## .. code-block:: nim ## echo fmod(-2.5, 0.3) ## -0.1 -else: - proc trunc*(x: float32): float32 {.importc: "Math.trunc", nodecl.} - proc trunc*(x: float64): float64 {.importc: "Math.trunc", nodecl.} + proc `mod`*(x, y: float32): float32 {.importc: "fmodf", header: "<math.h>".} + proc `mod`*(x, y: float64): float64 {.importc: "fmod", header: "<math.h>".} + ## Computes the modulo operation for float operators. +else: # JS + proc hypot*[T: float32|float64](x, y: T): T = return sqrt(x*x + y*y) + proc pow*(x, y: float32): float32 {.importC: "Math.pow", nodecl.} + proc pow*(x, y: float64): float64 {.importc: "Math.pow", nodecl.} proc floor*(x: float32): float32 {.importc: "Math.floor", nodecl.} proc floor*(x: float64): float64 {.importc: "Math.floor", nodecl.} proc ceil*(x: float32): float32 {.importc: "Math.ceil", nodecl.} proc ceil*(x: float64): float64 {.importc: "Math.ceil", nodecl.} - - proc sqrt*(x: float32): float32 {.importc: "Math.sqrt", nodecl.} - proc sqrt*(x: float64): float64 {.importc: "Math.sqrt", nodecl.} - proc ln*(x: float32): float32 {.importc: "Math.log", nodecl.} - proc ln*(x: float64): float64 {.importc: "Math.log", nodecl.} - proc log10*[T: float32|float64](x: T): T = return ln(x) / ln(10.0) - proc log2*[T: float32|float64](x: T): T = return ln(x) / ln(2.0) - - proc exp*(x: float32): float32 {.importc: "Math.exp", nodecl.} - proc exp*(x: float64): float64 {.importc: "Math.exp", nodecl.} proc round0(x: float): float {.importc: "Math.round", nodecl.} + proc trunc*(x: float32): float32 {.importc: "Math.trunc", nodecl.} + proc trunc*(x: float64): float64 {.importc: "Math.trunc", nodecl.} - proc pow*(x, y: float32): float32 {.importC: "Math.pow", nodecl.} - proc pow*(x, y: float64): float64 {.importc: "Math.pow", nodecl.} - - proc arccos*(x: float32): float32 {.importc: "Math.acos", nodecl.} - proc arccos*(x: float64): float64 {.importc: "Math.acos", nodecl.} - proc arcsin*(x: float32): float32 {.importc: "Math.asin", nodecl.} - proc arcsin*(x: float64): float64 {.importc: "Math.asin", nodecl.} - proc arctan*(x: float32): float32 {.importc: "Math.atan", nodecl.} - proc arctan*(x: float64): float64 {.importc: "Math.atan", nodecl.} - proc arctan2*(y, x: float32): float32 {.importC: "Math.atan2", nodecl.} - proc arctan2*(y, x: float64): float64 {.importc: "Math.atan2", nodecl.} - - proc cos*(x: float32): float32 {.importc: "Math.cos", nodecl.} - proc cos*(x: float64): float64 {.importc: "Math.cos", nodecl.} - proc cosh*(x: float32): float32 = return (exp(x)+exp(-x))*0.5 - proc cosh*(x: float64): float64 = return (exp(x)+exp(-x))*0.5 - proc hypot*[T: float32|float64](x, y: T): T = return sqrt(x*x + y*y) - proc sinh*[T: float32|float64](x: T): T = return (exp(x)-exp(-x))*0.5 - proc sin*(x: float32): float32 {.importc: "Math.sin", nodecl.} - proc sin*(x: float64): float64 {.importc: "Math.sin", nodecl.} - proc tan*(x: float32): float32 {.importc: "Math.tan", nodecl.} - proc tan*(x: float64): float64 {.importc: "Math.tan", nodecl.} - proc tanh*[T: float32|float64](x: T): T = - var y = exp(2.0*x) - return (y-1.0)/(y+1.0) + proc `mod`*(x, y: float32): float32 {.importcpp: "# % #".} + proc `mod`*(x, y: float64): float64 {.importcpp: "# % #".} + ## Computes the modulo operation for float operators. proc round*[T: float32|float64](x: T, places: int = 0): T = ## Round a floating point number. @@ -350,6 +416,21 @@ proc round*[T: float32|float64](x: T, places: int = 0): T = var mult = pow(10.0, places.T) result = round0(x*mult)/mult +proc floorDiv*[T: SomeInteger](x, y: T): T = + ## Floor division is conceptually defined as ``floor(x / y)``. + ## This is different from the ``div`` operator, which is defined + ## as ``trunc(x / y)``. That is, ``div`` rounds towards ``0`` and ``floorDiv`` + ## rounds down. + result = x div y + let r = x mod y + if (r > 0 and y < 0) or (r < 0 and y > 0): result.dec 1 + +proc floorMod*[T: SomeNumber](x, y: T): T = + ## Floor modulus is conceptually defined as ``x - (floorDiv(x, y) * y). + ## This proc behaves the same as the ``%`` operator in python. + result = x mod y + if (result > 0 and y < 0) or (result < 0 and y > 0): result += y + when not defined(JS): proc c_frexp*(x: float32, exponent: var int32): float32 {. importc: "frexp", header: "<math.h>".} @@ -414,15 +495,6 @@ proc sgn*[T: SomeNumber](x: T): int {.inline.} = ## `NaN`. ord(T(0) < x) - ord(x < T(0)) -proc `mod`*[T: float32|float64](x, y: T): T = - ## Computes the modulo operation for float operators. Equivalent - ## to ``x - y * floor(x/y)``. Note that the remainder will always - ## have the same sign as the divisor. - ## - ## .. code-block:: nim - ## echo (4.0 mod -3.1) # -2.2 - result = if y == 0.0: x else: x - y * (x/y).floor - {.pop.} {.pop.} @@ -445,16 +517,42 @@ proc `^`*[T](x: T, y: Natural): T = x *= x proc gcd*[T](x, y: T): T = - ## Computes the greatest common divisor of ``x`` and ``y``. + ## Computes the greatest common (positive) divisor of ``x`` and ``y``. ## Note that for floats, the result cannot always be interpreted as ## "greatest decimal `z` such that ``z*N == x and z*M == y`` ## where N and M are positive integers." - var (x,y) = (x,y) + var (x, y) = (x, y) while y != 0: x = x mod y swap x, y abs x +proc gcd*(x, y: SomeInteger): SomeInteger = + ## Computes the greatest common (positive) divisor of ``x`` and ``y``. + ## Using binary GCD (aka Stein's) algorithm. + when x is SomeSignedInt: + var x = abs(x) + else: + var x = x + when y is SomeSignedInt: + var y = abs(y) + else: + var y = y + + if x == 0: + return y + if y == 0: + return x + + let shift = countTrailingZeroBits(x or y) + y = y shr countTrailingZeroBits(y) + while x != 0: + x = x shr countTrailingZeroBits(x) + if y > x: + swap y, x + x -= y + y shl shift + proc lcm*[T](x, y: T): T = ## Computes the least common multiple of ``x`` and ``y``. x div gcd(x, y) * y @@ -465,6 +563,7 @@ when isMainModule and not defined(JS): return sqrt(num) # check gamma function + assert(gamma(5.0) == 24.0) # 4! assert($tgamma(5.0) == $24.0) # 4! assert(lgamma(1.0) == 0.0) # ln(1.0) == 0.0 assert(erf(6.0) > erf(5.0)) @@ -474,6 +573,12 @@ when isMainModule: # Function for approximate comparison of floats proc `==~`(x, y: float): bool = (abs(x-y) < 1e-9) + block: # prod + doAssert prod([1, 2, 3, 4]) == 24 + doAssert prod([1.5, 3.4]) == 5.1 + let x: seq[float] = @[] + doAssert prod(x) == 1.0 + block: # round() tests # Round to 0 decimal places doAssert round(54.652) ==~ 55.0 @@ -550,3 +655,30 @@ when isMainModule: assert sgn(Inf) == 1 assert sgn(NaN) == 0 + block: # fac() tests + try: + discard fac(-1) + except AssertionError: + discard + + doAssert fac(0) == 1 + doAssert fac(1) == 1 + doAssert fac(2) == 2 + doAssert fac(3) == 6 + doAssert fac(4) == 24 + + block: # floorMod/floorDiv + doAssert floorDiv(8, 3) == 2 + doAssert floorMod(8, 3) == 2 + + doAssert floorDiv(8, -3) == -3 + doAssert floorMod(8, -3) == -1 + + doAssert floorDiv(-8, 3) == -3 + doAssert floorMod(-8, 3) == 1 + + doAssert floorDiv(-8, -3) == 2 + doAssert floorMod(-8, -3) == -2 + + doAssert floorMod(8.0, -3.0) ==~ -1.0 + doAssert floorMod(-8.5, 3.0) ==~ 0.5 diff --git a/lib/pure/memfiles.nim b/lib/pure/memfiles.nim index 5c73381ff..bf6795b0c 100644 --- a/lib/pure/memfiles.nim +++ b/lib/pure/memfiles.nim @@ -38,8 +38,6 @@ type else: handle: cint -{.deprecated: [TMemFile: MemFile].} - proc mapMem*(m: var MemFile, mode: FileMode = fmRead, mappedSize = -1, offset = 0): pointer = ## returns a pointer to a mapped portion of MemFile `m` diff --git a/lib/pure/mersenne.nim b/lib/pure/mersenne.nim index 6ac0c4e56..b2227f114 100644 --- a/lib/pure/mersenne.nim +++ b/lib/pure/mersenne.nim @@ -12,8 +12,6 @@ type mt: array[0..623, uint32] index: int -{.deprecated: [TMersenneTwister: MersenneTwister].} - proc newMersenneTwister*(seed: uint32): MersenneTwister = result.index = 0 result.mt[0] = seed diff --git a/lib/pure/mimetypes.nim b/lib/pure/mimetypes.nim index b397ef47b..ff69ba61e 100644 --- a/lib/pure/mimetypes.nim +++ b/lib/pure/mimetypes.nim @@ -13,8 +13,6 @@ type MimeDB* = object mimes: StringTableRef -{.deprecated: [TMimeDB: MimeDB].} - const mimes* = { "ez": "application/andrew-inset", "anx": "application/annodex", diff --git a/lib/pure/nativesockets.nim b/lib/pure/nativesockets.nim index 6c8701843..5545ca2d1 100644 --- a/lib/pure/nativesockets.nim +++ b/lib/pure/nativesockets.nim @@ -31,7 +31,7 @@ else: export Sockaddr_storage, Sockaddr_un, Sockaddr_un_path_length export SocketHandle, Sockaddr_in, Addrinfo, INADDR_ANY, SockAddr, SockLen, - Sockaddr_in6, + Sockaddr_in6, Sockaddr_storage, inet_ntoa, recv, `==`, connect, send, accept, recvfrom, sendto, freeAddrInfo @@ -85,9 +85,6 @@ type length*: int addrList*: seq[string] -{.deprecated: [TPort: Port, TDomain: Domain, TType: SockType, - TProtocol: Protocol, TServent: Servent, THostent: Hostent].} - when useWinVersion: let osInvalidSocket* = winlean.INVALID_SOCKET @@ -184,20 +181,40 @@ proc toSockType*(protocol: Protocol): SockType = of IPPROTO_IP, IPPROTO_IPV6, IPPROTO_RAW, IPPROTO_ICMP: SOCK_RAW -proc newNativeSocket*(domain: Domain = AF_INET, +proc createNativeSocket*(domain: Domain = AF_INET, sockType: SockType = SOCK_STREAM, protocol: Protocol = IPPROTO_TCP): SocketHandle = - ## Creates a new socket; returns `InvalidSocket` if an error occurs. + ## Creates a new socket; returns `osInvalidSocket` if an error occurs. socket(toInt(domain), toInt(sockType), toInt(protocol)) -proc newNativeSocket*(domain: cint, sockType: cint, +proc createNativeSocket*(domain: cint, sockType: cint, protocol: cint): SocketHandle = - ## Creates a new socket; returns `InvalidSocket` if an error occurs. + ## Creates a new socket; returns `osInvalidSocket` if an error occurs. ## ## Use this overload if one of the enums specified above does ## not contain what you need. socket(domain, sockType, protocol) +proc newNativeSocket*(domain: Domain = AF_INET, + sockType: SockType = SOCK_STREAM, + protocol: Protocol = IPPROTO_TCP): SocketHandle + {.deprecated.} = + ## Creates a new socket; returns `osInvalidSocket` if an error occurs. + ## + ## **Deprecated since v0.18.0:** Use ``createNativeSocket`` instead. + createNativeSocket(domain, sockType, protocol) + +proc newNativeSocket*(domain: cint, sockType: cint, + protocol: cint): SocketHandle + {.deprecated.} = + ## Creates a new socket; returns `osInvalidSocket` if an error occurs. + ## + ## Use this overload if one of the enums specified above does + ## not contain what you need. + ## + ## **Deprecated since v0.18.0:** Use ``createNativeSocket`` instead. + createNativeSocket(domain, sockType, protocol) + proc close*(socket: SocketHandle) = ## closes a socket. when useWinVersion: @@ -375,7 +392,15 @@ proc getHostByAddr*(ip: string): Hostent {.tags: [ReadIOEffect].} = result.addrtype = AF_INET6 else: raiseOSError(osLastError(), "unknown h_addrtype") - result.addrList = cstringArrayToSeq(s.h_addr_list) + if result.addrtype == AF_INET: + result.addrlist = @[] + var i = 0 + while not isNil(s.h_addrlist[i]): + var inaddr_ptr = cast[ptr InAddr](s.h_addr_list[i]) + result.addrlist.add($inet_ntoa(inaddr_ptr[])) + inc(i) + else: + result.addrList = cstringArrayToSeq(s.h_addr_list) result.length = int(s.h_length) proc getHostByName*(name: string): Hostent {.tags: [ReadIOEffect].} = @@ -396,7 +421,15 @@ proc getHostByName*(name: string): Hostent {.tags: [ReadIOEffect].} = result.addrtype = AF_INET6 else: raiseOSError(osLastError(), "unknown h_addrtype") - result.addrList = cstringArrayToSeq(s.h_addr_list) + if result.addrtype == AF_INET: + result.addrlist = @[] + var i = 0 + while not isNil(s.h_addrlist[i]): + var inaddr_ptr = cast[ptr InAddr](s.h_addr_list[i]) + result.addrlist.add($inet_ntoa(inaddr_ptr[])) + inc(i) + else: + result.addrList = cstringArrayToSeq(s.h_addr_list) result.length = int(s.h_length) proc getHostname*(): string {.tags: [ReadIOEffect].} = @@ -580,8 +613,12 @@ proc setBlocking*(s: SocketHandle, blocking: bool) = proc timeValFromMilliseconds(timeout = 500): Timeval = if timeout != -1: var seconds = timeout div 1000 - result.tv_sec = seconds.int32 - result.tv_usec = ((timeout - seconds * 1000) * 1000).int32 + when useWinVersion: + result.tv_sec = seconds.int32 + result.tv_usec = ((timeout - seconds * 1000) * 1000).int32 + else: + result.tv_sec = seconds.Time + result.tv_usec = ((timeout - seconds * 1000) * 1000).Suseconds proc createFdSet(fd: var TFdSet, s: seq[SocketHandle], m: var int) = FD_ZERO(fd) @@ -600,7 +637,7 @@ proc pruneSocketSet(s: var seq[SocketHandle], fd: var TFdSet) = inc(i) setLen(s, L) -proc select*(readfds: var seq[SocketHandle], timeout = 500): int {.deprecated.} = +proc select*(readfds: var seq[SocketHandle], timeout = 500): int {.deprecated: "use selectRead instead".} = ## When a socket in ``readfds`` is ready to be read from then a non-zero ## value will be returned specifying the count of the sockets which can be ## read from. The sockets which can be read from will also be removed @@ -666,6 +703,19 @@ proc selectWrite*(writefds: var seq[SocketHandle], pruneSocketSet(writefds, (wr)) +proc accept*(fd: SocketHandle): (SocketHandle, string) = + ## Accepts a new client connection. + ## + ## Returns (osInvalidSocket, "") if an error occurred. + var sockAddress: Sockaddr_in + var addrLen = sizeof(sockAddress).SockLen + var sock = accept(fd, cast[ptr SockAddr](addr(sockAddress)), + addr(addrLen)) + if sock == osInvalidSocket: + return (osInvalidSocket, "") + else: + return (sock, $inet_ntoa(sockAddress.sin_addr)) + when defined(Windows): var wsa: WSAData if wsaStartup(0x0101'i16, addr wsa) != 0: raiseOSError(osLastError()) diff --git a/lib/pure/net.nim b/lib/pure/net.nim index aad6ab3e8..bf5f3f57e 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -64,8 +64,11 @@ ## socket.acceptAddr(client, address) ## echo("Client connected from: ", address) ## +## **Note:** The ``client`` variable is initialised with ``new Socket`` **not** +## ``newSocket()``. The difference is that the latter creates a new file +## descriptor. -{.deadCodeElim: on.} +{.deadCodeElim: on.} # dce option deprecated import nativesockets, os, strutils, parseutils, times, sets, options export Port, `$`, `==` export Domain, SockType, Protocol @@ -107,9 +110,6 @@ when defineSsl: serverGetPskFunc: SslServerGetPskFunc clientGetPskFunc: SslClientGetPskFunc - {.deprecated: [ESSL: SSLError, TSSLCVerifyMode: SSLCVerifyMode, - TSSLProtVersion: SSLProtVersion, PSSLContext: SSLContext, - TSSLAcceptResult: SSLAcceptResult].} else: type SslContext* = void # TODO: Workaround #4797. @@ -156,10 +156,6 @@ type Peek, SafeDisconn ## Ensures disconnection exceptions (ECONNRESET, EPIPE etc) are not thrown. -{.deprecated: [TSocketFlags: SocketFlag, ETimeout: TimeoutError, - TReadLineResult: ReadLineResult, TSOBool: SOBool, PSocket: Socket, - TSocketImpl: SocketImpl].} - type IpAddressFamily* {.pure.} = enum ## Describes the type of an IP address IPv6, ## IPv6 address @@ -173,8 +169,6 @@ type of IpAddressFamily.IPv4: address_v4*: array[0..3, uint8] ## Contains the IP address in bytes in ## case of IPv4 -{.deprecated: [TIpAddress: IpAddress].} - proc socketError*(socket: Socket, err: int = -1, async = false, lastError = (-1).OSErrorCode): void {.gcsafe.} @@ -221,7 +215,7 @@ proc newSocket*(domain, sockType, protocol: cint, buffered = true): Socket = ## Creates a new socket. ## ## If an error occurs EOS will be raised. - let fd = newNativeSocket(domain, sockType, protocol) + let fd = createNativeSocket(domain, sockType, protocol) if fd == osInvalidSocket: raiseOSError(osLastError()) result = newSocket(fd, domain.Domain, sockType.SockType, protocol.Protocol, @@ -232,7 +226,7 @@ proc newSocket*(domain: Domain = AF_INET, sockType: SockType = SOCK_STREAM, ## Creates a new socket. ## ## If an error occurs EOS will be raised. - let fd = newNativeSocket(domain, sockType, protocol) + let fd = createNativeSocket(domain, sockType, protocol) if fd == osInvalidSocket: raiseOSError(osLastError()) result = newSocket(fd, domain, sockType, protocol, buffered) @@ -411,9 +405,46 @@ proc isIpAddress*(address_str: string): bool {.tags: [].} = return false return true +proc toSockAddr*(address: IpAddress, port: Port, sa: var Sockaddr_storage, sl: var Socklen) = + ## Converts `IpAddress` and `Port` to `SockAddr` and `Socklen` + let port = htons(uint16(port)) + case address.family + of IpAddressFamily.IPv4: + sl = sizeof(Sockaddr_in).Socklen + let s = cast[ptr Sockaddr_in](addr sa) + s.sin_family = type(s.sin_family)(AF_INET) + s.sin_port = port + copyMem(addr s.sin_addr, unsafeAddr address.address_v4[0], sizeof(s.sin_addr)) + of IpAddressFamily.IPv6: + sl = sizeof(Sockaddr_in6).Socklen + let s = cast[ptr Sockaddr_in6](addr sa) + s.sin6_family = type(s.sin6_family)(AF_INET6) + s.sin6_port = port + copyMem(addr s.sin6_addr, unsafeAddr address.address_v6[0], sizeof(s.sin6_addr)) + +proc fromSockAddrAux(sa: ptr Sockaddr_storage, sl: Socklen, address: var IpAddress, port: var Port) = + if sa.ss_family.int == AF_INET.int and sl == sizeof(Sockaddr_in).Socklen: + address = IpAddress(family: IpAddressFamily.IPv4) + let s = cast[ptr Sockaddr_in](sa) + copyMem(addr address.address_v4[0], addr s.sin_addr, sizeof(address.address_v4)) + port = ntohs(s.sin_port).Port + elif sa.ss_family.int == AF_INET6.int and sl == sizeof(Sockaddr_in6).Socklen: + address = IpAddress(family: IpAddressFamily.IPv6) + let s = cast[ptr Sockaddr_in6](sa) + copyMem(addr address.address_v6[0], addr s.sin6_addr, sizeof(address.address_v6)) + port = ntohs(s.sin6_port).Port + else: + raise newException(ValueError, "Neither IPv4 nor IPv6") + +proc fromSockAddr*(sa: Sockaddr_storage | SockAddr | Sockaddr_in | Sockaddr_in6, + sl: Socklen, address: var IpAddress, port: var Port) {.inline.} = + ## Converts `SockAddr` and `Socklen` to `IpAddress` and `Port`. Raises + ## `ObjectConversionError` in case of invalid `sa` and `sl` arguments. + fromSockAddrAux(unsafeAddr sa, sl, address, port) + when defineSsl: CRYPTO_malloc_init() - SslLibraryInit() + doAssert SslLibraryInit() == 1 SslLoadErrorStrings() ErrLoadBioStrings() OpenSSL_add_all_algorithms() @@ -427,8 +458,14 @@ when defineSsl: raise newException(SSLError, "No error reported.") if err == -1: raiseOSError(osLastError()) - var errStr = ErrErrorString(err, nil) - raise newException(SSLError, $errStr) + var errStr = $ErrErrorString(err, nil) + case err + of 336032814, 336032784: + errStr = "Please upgrade your OpenSSL library, it does not support the " & + "necessary protocols. OpenSSL error is: " & errStr + else: + discard + raise newException(SSLError, errStr) proc getExtraData*(ctx: SSLContext, index: int): RootRef = ## Retrieves arbitrary data stored inside SSLContext. @@ -753,10 +790,10 @@ proc acceptAddr*(server: Socket, client: var Socket, address: var string, ## flag is specified then this error will not be raised and instead ## accept will be called again. assert(client != nil) - var sockAddress: Sockaddr_in - var addrLen = sizeof(sockAddress).SockLen - var sock = accept(server.fd, cast[ptr SockAddr](addr(sockAddress)), - addr(addrLen)) + assert client.fd.int <= 0, "Client socket needs to be initialised with " & + "`new`, not `newSocket`." + let ret = accept(server.fd) + let sock = ret[0] if sock == osInvalidSocket: let err = osLastError() @@ -764,7 +801,9 @@ proc acceptAddr*(server: Socket, client: var Socket, address: var string, acceptAddr(server, client, address, flags) raiseOSError(err) else: + address = ret[1] client.fd = sock + client.domain = getSockDomain(sock) client.isBuffered = server.isBuffered # Handle SSL. @@ -776,9 +815,6 @@ proc acceptAddr*(server: Socket, client: var Socket, address: var string, let ret = SSLAccept(client.sslHandle) socketError(client, ret, false) - # Client socket is set above. - address = $inet_ntoa(sockAddress.sin_addr) - when false: #defineSsl: proc acceptAddrSSL*(server: Socket, client: var Socket, address: var string): SSLAcceptResult {. @@ -868,6 +904,7 @@ proc close*(socket: Socket) = socket.sslHandle = nil socket.fd.close() + socket.fd = osInvalidSocket when defined(posix): from posix import TCP_NODELAY @@ -924,7 +961,7 @@ when defined(posix) and not defined(nimdoc): raise newException(ValueError, "socket path too long") copyMem(addr result.sun_path, path.cstring, path.len + 1) -when defined(posix): +when defined(posix) or defined(nimdoc): proc connectUnix*(socket: Socket, path: string) = ## Connects to Unix socket on `path`. ## This only works on Unix-style systems: Mac OS X, BSD and Linux @@ -1005,15 +1042,25 @@ proc select(readfd: Socket, timeout = 500): int = var fds = @[readfd.fd] result = select(fds, timeout) -proc readIntoBuf(socket: Socket, flags: int32): int = +proc isClosed(socket: Socket): bool = + socket.fd == osInvalidSocket + +proc uniRecv(socket: Socket, buffer: pointer, size, flags: cint): int = + ## Handles SSL and non-ssl recv in a nice package. + ## + ## In particular handles the case where socket has been closed properly + ## for both SSL and non-ssl. result = 0 + assert(not socket.isClosed, "Cannot `recv` on a closed socket") when defineSsl: - if socket.isSSL: - result = SSLRead(socket.sslHandle, addr(socket.buffer), int(socket.buffer.high)) - else: - result = recv(socket.fd, addr(socket.buffer), cint(socket.buffer.high), flags) - else: - result = recv(socket.fd, addr(socket.buffer), cint(socket.buffer.high), flags) + if socket.isSsl: + return SSLRead(socket.sslHandle, buffer, size) + + return recv(socket.fd, buffer, size, flags) + +proc readIntoBuf(socket: Socket, flags: int32): int = + result = 0 + result = uniRecv(socket, addr(socket.buffer), socket.buffer.high, flags) if result < 0: # Save it in case it gets reset (the Nim codegen occasionally may call # Win API functions which reset it). @@ -1059,16 +1106,16 @@ proc recv*(socket: Socket, data: pointer, size: int): int {.tags: [ReadIOEffect] else: when defineSsl: if socket.isSSL: - if socket.sslHasPeekChar: + if socket.sslHasPeekChar: # TODO: Merge this peek char mess into uniRecv copyMem(data, addr(socket.sslPeekChar), 1) socket.sslHasPeekChar = false if size-1 > 0: var d = cast[cstring](data) - result = SSLRead(socket.sslHandle, addr(d[1]), size-1) + 1 + result = uniRecv(socket, addr(d[1]), cint(size-1), 0'i32) + 1 else: result = 1 else: - result = SSLRead(socket.sslHandle, data, size) + result = uniRecv(socket, data, size.cint, 0'i32) else: result = recv(socket.fd, data, size.cint, 0'i32) else: @@ -1145,7 +1192,11 @@ proc recv*(socket: Socket, data: var string, size: int, timeout = -1, ## ## **Warning**: Only the ``SafeDisconn`` flag is currently supported. data.setLen(size) - result = recv(socket, cstring(data), size, timeout) + result = + if timeout == -1: + recv(socket, cstring(data), size) + else: + recv(socket, cstring(data), size, timeout) if result < 0: data.setLen(0) let lastError = getSocketError(socket) @@ -1182,7 +1233,7 @@ proc peekChar(socket: Socket, c: var char): int {.tags: [ReadIOEffect].} = when defineSsl: if socket.isSSL: if not socket.sslHasPeekChar: - result = SSLRead(socket.sslHandle, addr(socket.sslPeekChar), 1) + result = uniRecv(socket, addr(socket.sslPeekChar), 1, 0'i32) socket.sslHasPeekChar = true c = socket.sslPeekChar @@ -1316,6 +1367,7 @@ proc send*(socket: Socket, data: pointer, size: int): int {. ## ## **Note**: This is a low-level version of ``send``. You likely should use ## the version below. + assert(not socket.isClosed, "Cannot `send` on a closed socket") when defineSsl: if socket.isSSL: return SSLWrite(socket.sslHandle, cast[cstring](data), size) @@ -1360,8 +1412,8 @@ proc sendTo*(socket: Socket, address: string, port: Port, data: pointer, ## which is defined below. ## ## **Note:** This proc is not available for SSL sockets. - var aiList = getAddrInfo(address, port, af) - + assert(not socket.isClosed, "Cannot `sendTo` on a closed socket") + var aiList = getAddrInfo(address, port, af, socket.sockType, socket.protocol) # try all possibilities: var success = false var it = aiList @@ -1382,7 +1434,7 @@ proc sendTo*(socket: Socket, address: string, port: Port, ## this function will try each IP of that hostname. ## ## This is the high-level version of the above ``sendTo`` function. - result = socket.sendTo(address, port, cstring(data), data.len) + result = socket.sendTo(address, port, cstring(data), data.len, socket.domain ) proc isSsl*(socket: Socket): bool = @@ -1531,7 +1583,7 @@ proc dial*(address: string, port: Port, domain = domainOpt.unsafeGet() lastFd = fdPerDomain[ord(domain)] if lastFd == osInvalidSocket: - lastFd = newNativeSocket(domain, sockType, protocol) + lastFd = createNativeSocket(domain, sockType, protocol) if lastFd == osInvalidSocket: # we always raise if socket creation failed, because it means a # network system problem (e.g. not enough FDs), and not an unreachable @@ -1640,6 +1692,9 @@ proc connect*(socket: Socket, address: string, port = Port(0), if selectWrite(s, timeout) != 1: raise newException(TimeoutError, "Call to 'connect' timed out.") else: + let res = getSockOptInt(socket.fd, SOL_SOCKET, SO_ERROR) + if res != 0: + raiseOSError(OSErrorCode(res)) when defineSsl and not defined(nimdoc): if socket.isSSL: socket.fd.setBlocking(true) diff --git a/lib/pure/nimprof.nim b/lib/pure/nimprof.nim index 4289eb049..506c6bfaa 100644 --- a/lib/pure/nimprof.nim +++ b/lib/pure/nimprof.nim @@ -30,7 +30,6 @@ when not declared(system.StackTrace): type StackTrace = object lines: array[0..20, cstring] files: array[0..20, cstring] - {.deprecated: [TStackTrace: StackTrace].} proc `[]`*(st: StackTrace, i: int): cstring = st.lines[i] # We use a simple hash table of bounded size to keep track of the stack traces: @@ -39,7 +38,6 @@ type total: int st: StackTrace ProfileData = array[0..64*1024-1, ptr ProfileEntry] -{.deprecated: [TProfileEntry: ProfileEntry, TProfileData: ProfileData].} proc `==`(a, b: StackTrace): bool = for i in 0 .. high(a.lines): diff --git a/lib/pure/oids.nim b/lib/pure/oids.nim index 427a68964..d6369b5f9 100644 --- a/lib/pure/oids.nim +++ b/lib/pure/oids.nim @@ -23,11 +23,9 @@ type fuzz: int32 ## count: int32 ## -{.deprecated: [Toid: Oid].} - proc `==`*(oid1: Oid, oid2: Oid): bool = - ## Compare two Mongo Object IDs for equality - return (oid1.time == oid2.time) and (oid1.fuzz == oid2.fuzz) and (oid1.count == oid2.count) + ## Compare two Mongo Object IDs for equality + return (oid1.time == oid2.time) and (oid1.fuzz == oid2.fuzz) and (oid1.count == oid2.count) proc hexbyte*(hex: char): int = case hex @@ -71,7 +69,7 @@ proc genOid*(): Oid = proc rand(): cint {.importc: "rand", header: "<stdlib.h>", nodecl.} proc srand(seed: cint) {.importc: "srand", header: "<stdlib.h>", nodecl.} - var t = getTime().int32 + var t = getTime().toUnix.int32 var i = int32(atomicInc(incr)) diff --git a/lib/pure/options.nim b/lib/pure/options.nim index 6d2869bff..bd01b208a 100644 --- a/lib/pure/options.nim +++ b/lib/pure/options.nim @@ -70,26 +70,55 @@ import typetraits type + SomePointer = ref | ptr | pointer + +type Option*[T] = object ## An optional type that stores its value and state separately in a boolean. - val: T - has: bool + when T is SomePointer: + val: T + else: + val: T + has: bool + UnpackError* = ref object of ValueError proc some*[T](val: T): Option[T] = ## Returns a ``Option`` that has this value. - result.has = true + when T is SomePointer: + assert val != nil + result.val = val + else: + result.has = true + result.val = val + +proc option*[T](val: T): Option[T] = + ## Can be used to convert a pointer type to an option type. It + ## converts ``nil`` to the none-option. result.val = val + when T isnot SomePointer: + result.has = true proc none*(T: typedesc): Option[T] = - ## Returns a ``Option`` for this type that has no value. - result.has = false + ## Returns an ``Option`` for this type that has no value. + # the default is the none type + discard -proc isSome*[T](self: Option[T]): bool = - self.has +proc none*[T]: Option[T] = + ## Alias for ``none(T)``. + none(T) -proc isNone*[T](self: Option[T]): bool = - not self.has +proc isSome*[T](self: Option[T]): bool {.inline.} = + when T is SomePointer: + self.val != nil + else: + self.has + +proc isNone*[T](self: Option[T]): bool {.inline.} = + when T is SomePointer: + self.val == nil + else: + not self.has proc unsafeGet*[T](self: Option[T]): T = ## Returns the value of a ``some``. Behavior is undefined for ``none``. @@ -105,27 +134,27 @@ proc get*[T](self: Option[T]): T = proc get*[T](self: Option[T], otherwise: T): T = ## Returns the contents of this option or `otherwise` if the option is none. - if self.has: + if self.isSome: self.val else: otherwise proc map*[T](self: Option[T], callback: proc (input: T)) = ## Applies a callback to the value in this Option - if self.has: + if self.isSome: callback(self.val) proc map*[T, R](self: Option[T], callback: proc (input: T): R): Option[R] = ## Applies a callback to the value in this Option and returns an option - ## containing the new value. If this option is None, None will be returned. - if self.has: - some[R](callback(self.val)) + ## containing the new value. If this option is None, None will be returned + if self.isSome: + some[R]( callback(self.val) ) else: none(R) proc flatten*[A](self: Option[Option[A]]): Option[A] = ## Remove one level of structure in a nested Option. - if self.has: + if self.isSome: self.val else: none(A) @@ -142,7 +171,7 @@ proc filter*[T](self: Option[T], callback: proc (input: T): bool): Option[T] = ## Applies a callback to the value in this Option. If the callback returns ## `true`, the option is returned as a Some. If it returns false, it is ## returned as a None. - if self.has and not callback(self.val): + if self.isSome and not callback(self.val): none(T) else: self @@ -150,14 +179,14 @@ proc filter*[T](self: Option[T], callback: proc (input: T): bool): Option[T] = proc `==`*(a, b: Option): bool = ## Returns ``true`` if both ``Option``s are ``none``, ## or if they have equal values - (a.has and b.has and a.val == b.val) or (not a.has and not b.has) + (a.isSome and b.isSome and a.val == b.val) or (not a.isSome and not b.isSome) proc `$`*[T](self: Option[T]): string = ## Get the string representation of this option. If the option has a value, ## the result will be `Some(x)` where `x` is the string representation of the contained value. - ## If the option does not have a value, the result will be `None[T]` where `T` is the name of + ## If the option does not have a value, the result will be `None[T]` where `T` is the name of ## the type contained in the option. - if self.has: + if self.isSome: "Some(" & $self.val & ")" else: "None[" & T.name & "]" @@ -256,3 +285,16 @@ when isMainModule: check(some(1).flatMap(maybeToString).flatMap(maybeExclaim) == some("1!")) check(some(0).flatMap(maybeToString).flatMap(maybeExclaim) == none(string)) + + test "SomePointer": + var intref: ref int + check(option(intref).isNone) + intref.new + check(option(intref).isSome) + + let tmp = option(intref) + check(sizeof(tmp) == sizeof(ptr int)) + + test "none[T]": + check(none[int]().isNone) + check(none(int) == none[int]()) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index c18d03289..04afb1eff 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -10,7 +10,7 @@ ## This module contains basic operating system facilities like ## retrieving environment variables, reading command line arguments, ## working with directories, running shell commands, etc. -{.deadCodeElim: on.} +{.deadCodeElim: on.} # dce option deprecated {.push debugger: off.} @@ -23,6 +23,10 @@ when defined(windows): import winlean elif defined(posix): import posix + + proc toTime(ts: Timespec): times.Time {.inline.} = + result = initTime(ts.tv_sec.int64, ts.tv_nsec.int) + else: {.error: "OS module not ported to your operating system!".} @@ -70,7 +74,8 @@ when defined(windows): proc existsFile*(filename: string): bool {.rtl, extern: "nos$1", tags: [ReadDirEffect].} = - ## Returns true if the file exists, false otherwise. + ## Returns true if `filename` exists and is a regular file or symlink. + ## (directories, device files, named pipes and sockets return false) when defined(windows): when useWinUnicode: wrapUnary(a, getFileAttributesW, filename) @@ -139,11 +144,18 @@ proc findExe*(exe: string, followSymlinks: bool = true; ## is added the `ExeExts <#ExeExts>`_ file extensions if it has none. ## If the system supports symlinks it also resolves them until it ## meets the actual file. This behavior can be disabled if desired. - for ext in extensions: - result = addFileExt(exe, ext) - if existsFile(result): return - var path = string(getEnv("PATH")) + if exe.len == 0: return + template checkCurrentDir() = + for ext in extensions: + result = addFileExt(exe, ext) + if existsFile(result): return + when defined(posix): + if '/' in exe: checkCurrentDir() + else: + checkCurrentDir() + let path = string(getEnv("PATH")) for candidate in split(path, PathSep): + if candidate.len == 0: continue when defined(windows): var x = (if candidate[0] == '"' and candidate[^1] == '"': substr(candidate, 1, candidate.len-2) else: candidate) / @@ -178,12 +190,12 @@ proc getLastModificationTime*(file: string): times.Time {.rtl, extern: "nos$1".} when defined(posix): var res: Stat if stat(file, res) < 0'i32: raiseOSError(osLastError()) - return fromUnix(res.st_mtime.int64) + result = res.st_mtim.toTime else: var f: WIN32_FIND_DATA var h = findFirstFile(file, f) if h == -1'i32: raiseOSError(osLastError()) - result = fromUnix(winTimeToUnixTime(rdFileTime(f.ftLastWriteTime)).int64) + result = fromWinTime(rdFileTime(f.ftLastWriteTime)) findClose(h) proc getLastAccessTime*(file: string): times.Time {.rtl, extern: "nos$1".} = @@ -191,12 +203,12 @@ proc getLastAccessTime*(file: string): times.Time {.rtl, extern: "nos$1".} = when defined(posix): var res: Stat if stat(file, res) < 0'i32: raiseOSError(osLastError()) - return fromUnix(res.st_atime.int64) + result = res.st_atim.toTime else: var f: WIN32_FIND_DATA var h = findFirstFile(file, f) if h == -1'i32: raiseOSError(osLastError()) - result = fromUnix(winTimeToUnixTime(rdFileTime(f.ftLastAccessTime)).int64) + result = fromWinTime(rdFileTime(f.ftLastAccessTime)) findClose(h) proc getCreationTime*(file: string): times.Time {.rtl, extern: "nos$1".} = @@ -208,22 +220,25 @@ proc getCreationTime*(file: string): times.Time {.rtl, extern: "nos$1".} = when defined(posix): var res: Stat if stat(file, res) < 0'i32: raiseOSError(osLastError()) - return fromUnix(res.st_ctime.int64) + result = res.st_ctim.toTime else: var f: WIN32_FIND_DATA var h = findFirstFile(file, f) if h == -1'i32: raiseOSError(osLastError()) - result = fromUnix(winTimeToUnixTime(rdFileTime(f.ftCreationTime)).int64) + result = fromWinTime(rdFileTime(f.ftCreationTime)) findClose(h) proc fileNewer*(a, b: string): bool {.rtl, extern: "nos$1".} = ## Returns true if the file `a` is newer than file `b`, i.e. if `a`'s ## modification time is later than `b`'s. when defined(posix): - result = getLastModificationTime(a) - getLastModificationTime(b) >= 0 - # Posix's resolution sucks so, we use '>=' for posix. + # If we don't have access to nanosecond resolution, use '>=' + when not StatHasNanoseconds: + result = getLastModificationTime(a) >= getLastModificationTime(b) + else: + result = getLastModificationTime(a) > getLastModificationTime(b) else: - result = getLastModificationTime(a) - getLastModificationTime(b) > 0 + result = getLastModificationTime(a) > getLastModificationTime(b) proc getCurrentDir*(): string {.rtl, extern: "nos$1", tags: [].} = ## Returns the `current working directory`:idx:. @@ -324,20 +339,21 @@ proc expandFilename*(filename: string): string {.rtl, extern: "nos$1", c_free(cast[pointer](r)) when defined(Windows): - proc openHandle(path: string, followSymlink=true): Handle = + proc openHandle(path: string, followSymlink=true, writeAccess=false): Handle = var flags = FILE_FLAG_BACKUP_SEMANTICS or FILE_ATTRIBUTE_NORMAL if not followSymlink: flags = flags or FILE_FLAG_OPEN_REPARSE_POINT + let access = if writeAccess: GENERIC_WRITE else: 0'i32 when useWinUnicode: result = createFileW( - newWideCString(path), 0'i32, + newWideCString(path), access, FILE_SHARE_DELETE or FILE_SHARE_READ or FILE_SHARE_WRITE, nil, OPEN_EXISTING, flags, 0 ) else: result = createFileA( - path, 0'i32, + path, access, FILE_SHARE_DELETE or FILE_SHARE_READ or FILE_SHARE_WRITE, nil, OPEN_EXISTING, flags, 0 ) @@ -425,8 +441,6 @@ type fpOthersWrite, ## write access for others fpOthersRead ## read access for others -{.deprecated: [TFilePermission: FilePermission].} - proc getFilePermissions*(filename: string): set[FilePermission] {. rtl, extern: "nos$1", tags: [ReadDirEffect].} = ## retrieves file permissions for `filename`. `OSError` is raised in case of @@ -610,6 +624,7 @@ proc tryMoveFSObject(source, dest: string): bool = proc moveFile*(source, dest: string) {.rtl, extern: "nos$1", tags: [ReadIOEffect, WriteIOEffect].} = ## Moves a file from `source` to `dest`. If this fails, `OSError` is raised. + ## Can be used to `rename files`:idx: if not tryMoveFSObject(source, dest): when not defined(windows): # Fallback to copy & del @@ -672,7 +687,10 @@ template walkCommon(pattern: string, filter) = if dotPos < 0 or idx >= ff.len or ff[idx] == '.' or pattern[dotPos+1] == '*': yield splitFile(pattern).dir / extractFilename(ff) - if findNextFile(res, f) == 0'i32: break + if findNextFile(res, f) == 0'i32: + let errCode = getLastError() + if errCode == ERROR_NO_MORE_FILES: break + else: raiseOSError(errCode.OSErrorCode) else: # here we use glob var f: Glob @@ -720,8 +738,6 @@ type pcDir, ## path refers to a directory pcLinkToDir ## path refers to a symbolic link to a directory -{.deprecated: [TPathComponent: PathComponent].} - when defined(posix): proc getSymlinkFileKind(path: string): PathComponent = @@ -782,7 +798,10 @@ iterator walkDir*(dir: string; relative=false): tuple[kind: PathComponent, path: let xx = if relative: extractFilename(getFilename(f)) else: dir / extractFilename(getFilename(f)) yield (k, xx) - if findNextFile(h, f) == 0'i32: break + if findNextFile(h, f) == 0'i32: + let errCode = getLastError() + if errCode == ERROR_NO_MORE_FILES: break + else: raiseOSError(errCode.OSErrorCode) else: var d = opendir(dir) if d != nil: @@ -818,7 +837,7 @@ iterator walkDir*(dir: string; relative=false): tuple[kind: PathComponent, path: iterator walkDirRec*(dir: string, yieldFilter = {pcFile}, followFilter = {pcDir}): string {.tags: [ReadDirEffect].} = - ## Recursively walks over the directory `dir` and yields for each file + ## Recursively walks over the directory `dir` and yields for each file ## or directory in `dir`. ## The full path for each file or directory is returned. ## **Warning**: @@ -913,7 +932,8 @@ proc rawCreateDir(dir: string): bool = else: raiseOSError(osLastError()) -proc existsOrCreateDir*(dir: string): bool = +proc existsOrCreateDir*(dir: string): bool {.rtl, extern: "nos$1", + tags: [WriteDirEffect, ReadDirEffect].} = ## Check if a `directory`:idx: `dir` exists, and create it otherwise. ## ## Does not create parent directories (fails if parent does not exist). @@ -1046,18 +1066,17 @@ proc parseCmdLine*(c: string): seq[string] {. while true: setLen(a, 0) # eat all delimiting whitespace - while c[i] == ' ' or c[i] == '\t' or c[i] == '\l' or c[i] == '\r' : inc(i) + while i < c.len and c[i] in {' ', '\t', '\l', '\r'}: inc(i) + if i >= c.len: break when defined(windows): # parse a single argument according to the above rules: - if c[i] == '\0': break var inQuote = false - while true: + while i < c.len: case c[i] - of '\0': break of '\\': var j = i - while c[j] == '\\': inc(j) - if c[j] == '"': + while j < c.len and c[j] == '\\': inc(j) + if j < c.len and c[j] == '"': for k in 1..(j-i) div 2: a.add('\\') if (j-i) mod 2 == 0: i = j @@ -1070,7 +1089,7 @@ proc parseCmdLine*(c: string): seq[string] {. of '"': inc(i) if not inQuote: inQuote = true - elif c[i] == '"': + elif i < c.len and c[i] == '"': a.add(c[i]) inc(i) else: @@ -1088,13 +1107,12 @@ proc parseCmdLine*(c: string): seq[string] {. of '\'', '\"': var delim = c[i] inc(i) # skip ' or " - while c[i] != '\0' and c[i] != delim: + while i < c.len and c[i] != delim: add a, c[i] inc(i) - if c[i] != '\0': inc(i) - of '\0': break + if i < c.len: inc(i) else: - while c[i] > ' ': + while i < c.len and c[i] > ' ': add(a, c[i]) inc(i) add(result, a) @@ -1429,18 +1447,6 @@ proc getAppFilename*(): string {.rtl, extern: "nos$1", tags: [ReadIOEffect].} = if result.len == 0: result = getApplHeuristic() -proc getApplicationFilename*(): string {.rtl, extern: "nos$1", deprecated.} = - ## Returns the filename of the application's executable. - ## **Deprecated since version 0.8.12**: use ``getAppFilename`` - ## instead. - result = getAppFilename() - -proc getApplicationDir*(): string {.rtl, extern: "nos$1", deprecated.} = - ## Returns the directory of the application's executable. - ## **Deprecated since version 0.8.12**: use ``getAppDir`` - ## instead. - result = splitFile(getAppFilename()).dir - proc getAppDir*(): string {.rtl, extern: "nos$1", tags: [ReadIOEffect].} = ## Returns the directory of the application's executable. result = splitFile(getAppFilename()).dir @@ -1495,19 +1501,17 @@ type template rawToFormalFileInfo(rawInfo, path, formalInfo): untyped = ## Transforms the native file info structure into the one nim uses. - ## 'rawInfo' is either a 'TBY_HANDLE_FILE_INFORMATION' structure on Windows, + ## 'rawInfo' is either a 'BY_HANDLE_FILE_INFORMATION' structure on Windows, ## or a 'Stat' structure on posix when defined(Windows): - template toTime(e: FILETIME): untyped {.gensym.} = - fromUnix(winTimeToUnixTime(rdFileTime(e)).int64) # local templates default to bind semantics template merge(a, b): untyped = a or (b shl 32) formalInfo.id.device = rawInfo.dwVolumeSerialNumber formalInfo.id.file = merge(rawInfo.nFileIndexLow, rawInfo.nFileIndexHigh) formalInfo.size = merge(rawInfo.nFileSizeLow, rawInfo.nFileSizeHigh) formalInfo.linkCount = rawInfo.nNumberOfLinks - formalInfo.lastAccessTime = toTime(rawInfo.ftLastAccessTime) - formalInfo.lastWriteTime = toTime(rawInfo.ftLastWriteTime) - formalInfo.creationTime = toTime(rawInfo.ftCreationTime) + formalInfo.lastAccessTime = fromWinTime(rdFileTime(rawInfo.ftLastAccessTime)) + formalInfo.lastWriteTime = fromWinTime(rdFileTime(rawInfo.ftLastWriteTime)) + formalInfo.creationTime = fromWinTime(rdFileTime(rawInfo.ftCreationTime)) # Retrieve basic permissions if (rawInfo.dwFileAttributes and FILE_ATTRIBUTE_READONLY) != 0'i32: @@ -1523,7 +1527,6 @@ template rawToFormalFileInfo(rawInfo, path, formalInfo): untyped = if (rawInfo.dwFileAttributes and FILE_ATTRIBUTE_REPARSE_POINT) != 0'i32: formalInfo.kind = succ(result.kind) - else: template checkAndIncludeMode(rawMode, formalMode: untyped) = if (rawInfo.st_mode and rawMode) != 0'i32: @@ -1531,9 +1534,9 @@ template rawToFormalFileInfo(rawInfo, path, formalInfo): untyped = formalInfo.id = (rawInfo.st_dev, rawInfo.st_ino) formalInfo.size = rawInfo.st_size formalInfo.linkCount = rawInfo.st_Nlink.BiggestInt - formalInfo.lastAccessTime = fromUnix(rawInfo.st_atime.int64) - formalInfo.lastWriteTime = fromUnix(rawInfo.st_mtime.int64) - formalInfo.creationTime = fromUnix(rawInfo.st_ctime.int64) + formalInfo.lastAccessTime = rawInfo.st_atim.toTime + formalInfo.lastWriteTime = rawInfo.st_mtim.toTime + formalInfo.creationTime = rawInfo.st_ctim.toTime result.permissions = {} checkAndIncludeMode(S_IRUSR, fpUserRead) @@ -1641,3 +1644,20 @@ proc isHidden*(path: string): bool = result = (fileName[0] == '.') and (fileName[3] != '.') {.pop.} + +proc setLastModificationTime*(file: string, t: times.Time) = + ## Sets the `file`'s last modification time. `OSError` is raised in case of + ## an error. + when defined(posix): + let unixt = posix.Time(t.toUnix) + let micro = convert(Nanoseconds, Microseconds, t.nanosecond) + var timevals = [Timeval(tv_sec: unixt, tv_usec: micro), + Timeval(tv_sec: unixt, tv_usec: micro)] # [last access, last modification] + if utimes(file, timevals.addr) != 0: raiseOSError(osLastError()) + else: + let h = openHandle(path = file, writeAccess = true) + if h == INVALID_HANDLE_VALUE: raiseOSError(osLastError()) + var ft = t.toWinTime.toFILETIME + let res = setFileTime(h, nil, nil, ft.addr) + discard h.closeHandle + if res == 0'i32: raiseOSError(osLastError()) diff --git a/lib/pure/ospaths.nim b/lib/pure/ospaths.nim index 0d638abb9..c0d3ef512 100644 --- a/lib/pure/ospaths.nim +++ b/lib/pure/ospaths.nim @@ -29,11 +29,6 @@ type OSErrorCode* = distinct int32 ## Specifies an OS Error Code. -{.deprecated: [FReadEnv: ReadEnvEffect, FWriteEnv: WriteEnvEffect, - FReadDir: ReadDirEffect, - FWriteDir: WriteDirEffect, - TOSErrorCode: OSErrorCode -].} const doslikeFileSystem* = defined(windows) or defined(OS2) or defined(DOS) @@ -196,7 +191,7 @@ proc joinPath*(head, tail: string): string {. else: result = head & tail else: - if tail[0] in {DirSep, AltSep}: + if tail.len > 0 and tail[0] in {DirSep, AltSep}: result = head & tail else: result = head & DirSep & tail @@ -477,7 +472,7 @@ proc unixToNativePath*(path: string, drive=""): string {. var i = start while i < len(path): # ../../../ --> :::: - if path[i] == '.' and path[i+1] == '.' and path[i+2] == '/': + if i+2 < path.len and path[i] == '.' and path[i+1] == '.' and path[i+2] == '/': # parent directory when defined(macos): if result[high(result)] == ':': diff --git a/lib/pure/osproc.nim b/lib/pure/osproc.nim index 1625845d1..664446d54 100644 --- a/lib/pure/osproc.nim +++ b/lib/pure/osproc.nim @@ -61,9 +61,6 @@ type Process* = ref ProcessObj ## represents an operating system process -{.deprecated: [TProcess: ProcessObj, PProcess: Process, - TProcessOption: ProcessOption].} - const poUseShell* {.deprecated.} = poUsePath ## Deprecated alias for poUsePath. @@ -306,7 +303,7 @@ proc execProcesses*(cmds: openArray[string], raiseOSError(err) if rexit >= 0: - result = max(result, q[rexit].peekExitCode()) + result = max(result, abs(q[rexit].peekExitCode())) if afterRunEvent != nil: afterRunEvent(rexit, q[rexit]) close(q[rexit]) if i < len(cmds): @@ -331,7 +328,7 @@ proc execProcesses*(cmds: openArray[string], if beforeRunEvent != nil: beforeRunEvent(i) var p = startProcess(cmds[i], options=options + {poEvalCommand}) - result = max(waitForExit(p), result) + result = max(abs(waitForExit(p)), result) if afterRunEvent != nil: afterRunEvent(i, p) close(p) @@ -373,17 +370,15 @@ template streamAccess(p) = when defined(Windows) and not defined(useNimRtl): # We need to implement a handle stream for Windows: type - PFileHandleStream = ref FileHandleStream - FileHandleStream = object of StreamObj + FileHandleStream = ref object of StreamObj handle: Handle atTheEnd: bool - {.deprecated: [TFileHandleStream: FileHandleStream].} proc hsClose(s: Stream) = discard # nothing to do here - proc hsAtEnd(s: Stream): bool = return PFileHandleStream(s).atTheEnd + proc hsAtEnd(s: Stream): bool = return FileHandleStream(s).atTheEnd proc hsReadData(s: Stream, buffer: pointer, bufLen: int): int = - var s = PFileHandleStream(s) + var s = FileHandleStream(s) if s.atTheEnd: return 0 var br: int32 var a = winlean.readFile(s.handle, buffer, bufLen.cint, addr br, nil) @@ -395,13 +390,13 @@ when defined(Windows) and not defined(useNimRtl): result = br proc hsWriteData(s: Stream, buffer: pointer, bufLen: int) = - var s = PFileHandleStream(s) + var s = FileHandleStream(s) var bytesWritten: int32 var a = winlean.writeFile(s.handle, buffer, bufLen.cint, addr bytesWritten, nil) if a == 0: raiseOSError(osLastError()) - proc newFileHandleStream(handle: Handle): PFileHandleStream = + proc newFileHandleStream(handle: Handle): FileHandleStream = new(result) result.handle = handle result.closeImpl = hsClose @@ -494,7 +489,7 @@ when defined(Windows) and not defined(useNimRtl): sa.nLength = sizeof(SECURITY_ATTRIBUTES).cint sa.lpSecurityDescriptor = nil sa.bInheritHandle = 1 - if createPipe(rdHandle, wrHandle, sa, 1024) == 0'i32: + if createPipe(rdHandle, wrHandle, sa, 0) == 0'i32: raiseOSError(osLastError()) proc fileClose(h: Handle) {.inline.} = @@ -524,6 +519,12 @@ when defined(Windows) and not defined(useNimRtl): he = ho else: createPipeHandles(he, si.hStdError) + if setHandleInformation(he, DWORD(1), DWORD(0)) == 0'i32: + raiseOsError(osLastError()) + if setHandleInformation(hi, DWORD(1), DWORD(0)) == 0'i32: + raiseOsError(osLastError()) + if setHandleInformation(ho, DWORD(1), DWORD(0)) == 0'i32: + raiseOsError(osLastError()) else: createAllPipeHandles(si, hi, ho, he, cast[int](result)) result.inHandle = FileHandle(hi) @@ -746,14 +747,14 @@ elif not defined(useNimRtl): copyMem(result[i], addr(x[0]), x.len+1) inc(i) - type StartProcessData = object - sysCommand: string - sysArgs: cstringArray - sysEnv: cstringArray - workingDir: cstring - pStdin, pStdout, pStderr, pErrorPipe: array[0..1, cint] - options: set[ProcessOption] - {.deprecated: [TStartProcessData: StartProcessData].} + type + StartProcessData = object + sysCommand: string + sysArgs: cstringArray + sysEnv: cstringArray + workingDir: cstring + pStdin, pStdout, pStderr, pErrorPipe: array[0..1, cint] + options: set[ProcessOption] const useProcessAuxSpawn = declared(posix_spawn) and not defined(useFork) and not defined(useClone) and not defined(linux) @@ -995,13 +996,21 @@ elif not defined(useNimRtl): {.pop} proc close(p: Process) = - if p.inStream != nil: close(p.inStream) - if p.outStream != nil: close(p.outStream) - if p.errStream != nil: close(p.errStream) if poParentStreams notin p.options: - discard close(p.inHandle) - discard close(p.outHandle) - discard close(p.errHandle) + if p.inStream != nil: + close(p.inStream) + else: + discard close(p.inHandle) + + if p.outStream != nil: + close(p.outStream) + else: + discard close(p.outHandle) + + if p.errStream != nil: + close(p.errStream) + else: + discard close(p.errHandle) proc suspend(p: Process) = if kill(p.id, SIGSTOP) != 0'i32: raiseOsError(osLastError()) @@ -1275,7 +1284,7 @@ elif not defined(useNimRtl): proc select(readfds: var seq[Process], timeout = 500): int = var tv: Timeval - tv.tv_sec = 0 + tv.tv_sec = posix.Time(0) tv.tv_usec = timeout * 1000 var rd: TFdSet diff --git a/lib/pure/parsecfg.nim b/lib/pure/parsecfg.nim index 2a5dbc8f8..5fa2d8dc3 100644 --- a/lib/pure/parsecfg.nim +++ b/lib/pure/parsecfg.nim @@ -125,9 +125,6 @@ type tok: Token filename: string -{.deprecated: [TCfgEventKind: CfgEventKind, TCfgEvent: CfgEvent, - TTokKind: TokKind, TToken: Token, TCfgParser: CfgParser].} - # implementation const diff --git a/lib/pure/parsecsv.nim b/lib/pure/parsecsv.nim index 071858b7c..796114d37 100644 --- a/lib/pure/parsecsv.nim +++ b/lib/pure/parsecsv.nim @@ -32,7 +32,7 @@ ## import parsecsv ## import os ## # Prepare a file -## var content = """One,Two,Three,Four +## let content = """One,Two,Three,Four ## 1,2,3,4 ## 10,20,30,40 ## 100,200,300,400 @@ -66,8 +66,6 @@ type CsvError* = object of IOError ## exception that is raised if ## a parsing error occurs -{.deprecated: [TCsvRow: CsvRow, TCsvParser: CsvParser, EInvalidCsv: CsvError].} - proc raiseEInvalidCsv(filename: string, line, col: int, msg: string) {.noreturn.} = var e: ref CsvError @@ -123,7 +121,7 @@ proc parseField(my: var CsvParser, a: var string) = if buf[pos] == my.quote and my.quote != '\0': inc(pos) while true: - var c = buf[pos] + let c = buf[pos] if c == '\0': my.bufpos = pos # can continue after exception? error(my, pos, my.quote & " expected") @@ -153,7 +151,7 @@ proc parseField(my: var CsvParser, a: var string) = inc(pos) else: while true: - var c = buf[pos] + let c = buf[pos] if c == my.sep: break if c in {'\c', '\l', '\0'}: break add(a, c) @@ -171,9 +169,9 @@ proc readRow*(my: var CsvParser, columns = 0): bool = ## ## Blank lines are skipped. var col = 0 # current column - var oldpos = my.bufpos + let oldpos = my.bufpos while my.buf[my.bufpos] != '\0': - var oldlen = my.row.len + let oldlen = my.row.len if oldlen < col+1: setLen(my.row, col+1) my.row[col] = "" @@ -208,16 +206,16 @@ proc close*(my: var CsvParser) {.inline.} = proc readHeaderRow*(my: var CsvParser) = ## Reads the first row and creates a look-up table for column numbers ## See also `rowEntry <#rowEntry.CsvParser.string>`_. - var present = my.readRow() + let present = my.readRow() if present: my.headers = my.row -proc rowEntry*(my: var CsvParser, entry: string): string = - ## Reads a specified `entry` from the current row. +proc rowEntry*(my: var CsvParser, entry: string): var string = + ## Acceses a specified `entry` from the current row. ## ## Assumes that `readHeaderRow <#readHeaderRow.CsvParser>`_ has already been ## called. - var index = my.headers.find(entry) + let index = my.headers.find(entry) if index >= 0: result = my.row[index] @@ -237,14 +235,14 @@ when isMainModule: import os import strutils block: # Tests for reading the header row - var content = "One,Two,Three,Four\n1,2,3,4\n10,20,30,40,\n100,200,300,400\n" + let content = "One,Two,Three,Four\n1,2,3,4\n10,20,30,40,\n100,200,300,400\n" writeFile("temp.csv", content) var p: CsvParser p.open("temp.csv") p.readHeaderRow() while p.readRow(): - var zeros = repeat('0', p.currRow-2) + let zeros = repeat('0', p.currRow-2) doAssert p.rowEntry("One") == "1" & zeros doAssert p.rowEntry("Two") == "2" & zeros doAssert p.rowEntry("Three") == "3" & zeros diff --git a/lib/pure/parsejson.nim b/lib/pure/parsejson.nim new file mode 100644 index 000000000..9c53af6a6 --- /dev/null +++ b/lib/pure/parsejson.nim @@ -0,0 +1,535 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2018 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module implements a json parser. It is used +## and exported by the ``json`` standard library +## module, but can also be used in its own right. + +import + strutils, lexbase, streams, unicode + +type + JsonEventKind* = enum ## enumeration of all events that may occur when parsing + jsonError, ## an error occurred during parsing + jsonEof, ## end of file reached + jsonString, ## a string literal + jsonInt, ## an integer literal + jsonFloat, ## a float literal + jsonTrue, ## the value ``true`` + jsonFalse, ## the value ``false`` + jsonNull, ## the value ``null`` + jsonObjectStart, ## start of an object: the ``{`` token + jsonObjectEnd, ## end of an object: the ``}`` token + jsonArrayStart, ## start of an array: the ``[`` token + jsonArrayEnd ## start of an array: the ``]`` token + + TokKind* = enum # must be synchronized with TJsonEventKind! + tkError, + tkEof, + tkString, + tkInt, + tkFloat, + tkTrue, + tkFalse, + tkNull, + tkCurlyLe, + tkCurlyRi, + tkBracketLe, + tkBracketRi, + tkColon, + tkComma + + JsonError* = enum ## enumeration that lists all errors that can occur + errNone, ## no error + errInvalidToken, ## invalid token + errStringExpected, ## string expected + errColonExpected, ## ``:`` expected + errCommaExpected, ## ``,`` expected + errBracketRiExpected, ## ``]`` expected + errCurlyRiExpected, ## ``}`` expected + errQuoteExpected, ## ``"`` or ``'`` expected + errEOC_Expected, ## ``*/`` expected + errEofExpected, ## EOF expected + errExprExpected ## expr expected + + ParserState = enum + stateEof, stateStart, stateObject, stateArray, stateExpectArrayComma, + stateExpectObjectComma, stateExpectColon, stateExpectValue + + JsonParser* = object of BaseLexer ## the parser object. + a*: string + tok*: TokKind + kind: JsonEventKind + err: JsonError + state: seq[ParserState] + filename: string + rawStringLiterals: bool + + JsonKindError* = object of ValueError ## raised by the ``to`` macro if the + ## JSON kind is incorrect. + JsonParsingError* = object of ValueError ## is raised for a JSON error + +const + errorMessages*: array[JsonError, string] = [ + "no error", + "invalid token", + "string expected", + "':' expected", + "',' expected", + "']' expected", + "'}' expected", + "'\"' or \"'\" expected", + "'*/' expected", + "EOF expected", + "expression expected" + ] + tokToStr: array[TokKind, string] = [ + "invalid token", + "EOF", + "string literal", + "int literal", + "float literal", + "true", + "false", + "null", + "{", "}", "[", "]", ":", "," + ] + +proc open*(my: var JsonParser, input: Stream, filename: string; + rawStringLiterals = false) = + ## initializes the parser with an input stream. `Filename` is only used + ## for nice error messages. If `rawStringLiterals` is true, string literals + ## are kepts with their surrounding quotes and escape sequences in them are + ## left untouched too. + lexbase.open(my, input) + my.filename = filename + my.state = @[stateStart] + my.kind = jsonError + my.a = "" + my.rawStringLiterals = rawStringLiterals + +proc close*(my: var JsonParser) {.inline.} = + ## closes the parser `my` and its associated input stream. + lexbase.close(my) + +proc str*(my: JsonParser): string {.inline.} = + ## returns the character data for the events: ``jsonInt``, ``jsonFloat``, + ## ``jsonString`` + assert(my.kind in {jsonInt, jsonFloat, jsonString}) + return my.a + +proc getInt*(my: JsonParser): BiggestInt {.inline.} = + ## returns the number for the event: ``jsonInt`` + assert(my.kind == jsonInt) + return parseBiggestInt(my.a) + +proc getFloat*(my: JsonParser): float {.inline.} = + ## returns the number for the event: ``jsonFloat`` + assert(my.kind == jsonFloat) + return parseFloat(my.a) + +proc kind*(my: JsonParser): JsonEventKind {.inline.} = + ## returns the current event type for the JSON parser + return my.kind + +proc getColumn*(my: JsonParser): int {.inline.} = + ## get the current column the parser has arrived at. + result = getColNumber(my, my.bufpos) + +proc getLine*(my: JsonParser): int {.inline.} = + ## get the current line the parser has arrived at. + result = my.lineNumber + +proc getFilename*(my: JsonParser): string {.inline.} = + ## get the filename of the file that the parser processes. + result = my.filename + +proc errorMsg*(my: JsonParser): string = + ## returns a helpful error message for the event ``jsonError`` + assert(my.kind == jsonError) + result = "$1($2, $3) Error: $4" % [ + my.filename, $getLine(my), $getColumn(my), errorMessages[my.err]] + +proc errorMsgExpected*(my: JsonParser, e: string): string = + ## returns an error message "`e` expected" in the same format as the + ## other error messages + result = "$1($2, $3) Error: $4" % [ + my.filename, $getLine(my), $getColumn(my), e & " expected"] + +proc handleHexChar(c: char, x: var int): bool = + result = true # Success + case c + of '0'..'9': x = (x shl 4) or (ord(c) - ord('0')) + of 'a'..'f': x = (x shl 4) or (ord(c) - ord('a') + 10) + of 'A'..'F': x = (x shl 4) or (ord(c) - ord('A') + 10) + else: result = false # error + +proc parseEscapedUTF16*(buf: cstring, pos: var int): int = + result = 0 + #UTF-16 escape is always 4 bytes. + for _ in 0..3: + if handleHexChar(buf[pos], result): + inc(pos) + else: + return -1 + +proc parseString(my: var JsonParser): TokKind = + result = tkString + var pos = my.bufpos + 1 + var buf = my.buf + if my.rawStringLiterals: + add(my.a, '"') + while true: + case buf[pos] + of '\0': + my.err = errQuoteExpected + result = tkError + break + of '"': + if my.rawStringLiterals: + add(my.a, '"') + inc(pos) + break + of '\\': + if my.rawStringLiterals: + add(my.a, '\\') + case buf[pos+1] + of '\\', '"', '\'', '/': + add(my.a, buf[pos+1]) + inc(pos, 2) + of 'b': + add(my.a, '\b') + inc(pos, 2) + of 'f': + add(my.a, '\f') + inc(pos, 2) + of 'n': + add(my.a, '\L') + inc(pos, 2) + of 'r': + add(my.a, '\C') + inc(pos, 2) + of 't': + add(my.a, '\t') + inc(pos, 2) + of 'u': + if my.rawStringLiterals: + add(my.a, 'u') + inc(pos, 2) + var pos2 = pos + var r = parseEscapedUTF16(buf, pos) + if r < 0: + my.err = errInvalidToken + break + # Deal with surrogates + if (r and 0xfc00) == 0xd800: + if buf[pos] != '\\' or buf[pos+1] != 'u': + my.err = errInvalidToken + break + inc(pos, 2) + var s = parseEscapedUTF16(buf, pos) + if (s and 0xfc00) == 0xdc00 and s > 0: + r = 0x10000 + (((r - 0xd800) shl 10) or (s - 0xdc00)) + else: + my.err = errInvalidToken + break + if my.rawStringLiterals: + let length = pos - pos2 + for i in 1 .. length: + if buf[pos2] in {'0'..'9', 'A'..'F', 'a'..'f'}: + add(my.a, buf[pos2]) + inc pos2 + else: + break + else: + add(my.a, toUTF8(Rune(r))) + else: + # don't bother with the error + add(my.a, buf[pos]) + inc(pos) + of '\c': + pos = lexbase.handleCR(my, pos) + buf = my.buf + add(my.a, '\c') + of '\L': + pos = lexbase.handleLF(my, pos) + buf = my.buf + add(my.a, '\L') + else: + add(my.a, buf[pos]) + inc(pos) + my.bufpos = pos # store back + +proc skip(my: var JsonParser) = + var pos = my.bufpos + var buf = my.buf + while true: + case buf[pos] + of '/': + if buf[pos+1] == '/': + # skip line comment: + inc(pos, 2) + while true: + case buf[pos] + of '\0': + break + of '\c': + pos = lexbase.handleCR(my, pos) + buf = my.buf + break + of '\L': + pos = lexbase.handleLF(my, pos) + buf = my.buf + break + else: + inc(pos) + elif buf[pos+1] == '*': + # skip long comment: + inc(pos, 2) + while true: + case buf[pos] + of '\0': + my.err = errEOC_Expected + break + of '\c': + pos = lexbase.handleCR(my, pos) + buf = my.buf + of '\L': + pos = lexbase.handleLF(my, pos) + buf = my.buf + of '*': + inc(pos) + if buf[pos] == '/': + inc(pos) + break + else: + inc(pos) + else: + break + of ' ', '\t': + inc(pos) + of '\c': + pos = lexbase.handleCR(my, pos) + buf = my.buf + of '\L': + pos = lexbase.handleLF(my, pos) + buf = my.buf + else: + break + my.bufpos = pos + +proc parseNumber(my: var JsonParser) = + var pos = my.bufpos + var buf = my.buf + if buf[pos] == '-': + add(my.a, '-') + inc(pos) + if buf[pos] == '.': + add(my.a, "0.") + inc(pos) + else: + while buf[pos] in Digits: + add(my.a, buf[pos]) + inc(pos) + if buf[pos] == '.': + add(my.a, '.') + inc(pos) + # digits after the dot: + while buf[pos] in Digits: + add(my.a, buf[pos]) + inc(pos) + if buf[pos] in {'E', 'e'}: + add(my.a, buf[pos]) + inc(pos) + if buf[pos] in {'+', '-'}: + add(my.a, buf[pos]) + inc(pos) + while buf[pos] in Digits: + add(my.a, buf[pos]) + inc(pos) + my.bufpos = pos + +proc parseName(my: var JsonParser) = + var pos = my.bufpos + var buf = my.buf + if buf[pos] in IdentStartChars: + while buf[pos] in IdentChars: + add(my.a, buf[pos]) + inc(pos) + my.bufpos = pos + +proc getTok*(my: var JsonParser): TokKind = + setLen(my.a, 0) + skip(my) # skip whitespace, comments + case my.buf[my.bufpos] + of '-', '.', '0'..'9': + parseNumber(my) + if {'.', 'e', 'E'} in my.a: + result = tkFloat + else: + result = tkInt + of '"': + result = parseString(my) + of '[': + inc(my.bufpos) + result = tkBracketLe + of '{': + inc(my.bufpos) + result = tkCurlyLe + of ']': + inc(my.bufpos) + result = tkBracketRi + of '}': + inc(my.bufpos) + result = tkCurlyRi + of ',': + inc(my.bufpos) + result = tkComma + of ':': + inc(my.bufpos) + result = tkColon + of '\0': + result = tkEof + of 'a'..'z', 'A'..'Z', '_': + parseName(my) + case my.a + of "null": result = tkNull + of "true": result = tkTrue + of "false": result = tkFalse + else: result = tkError + else: + inc(my.bufpos) + result = tkError + my.tok = result + + +proc next*(my: var JsonParser) = + ## retrieves the first/next event. This controls the parser. + var tk = getTok(my) + var i = my.state.len-1 + # the following code is a state machine. If we had proper coroutines, + # the code could be much simpler. + case my.state[i] + of stateEof: + if tk == tkEof: + my.kind = jsonEof + else: + my.kind = jsonError + my.err = errEofExpected + of stateStart: + # tokens allowed? + case tk + of tkString, tkInt, tkFloat, tkTrue, tkFalse, tkNull: + my.state[i] = stateEof # expect EOF next! + my.kind = JsonEventKind(ord(tk)) + of tkBracketLe: + my.state.add(stateArray) # we expect any + my.kind = jsonArrayStart + of tkCurlyLe: + my.state.add(stateObject) + my.kind = jsonObjectStart + of tkEof: + my.kind = jsonEof + else: + my.kind = jsonError + my.err = errEofExpected + of stateObject: + case tk + of tkString, tkInt, tkFloat, tkTrue, tkFalse, tkNull: + my.state.add(stateExpectColon) + my.kind = JsonEventKind(ord(tk)) + of tkBracketLe: + my.state.add(stateExpectColon) + my.state.add(stateArray) + my.kind = jsonArrayStart + of tkCurlyLe: + my.state.add(stateExpectColon) + my.state.add(stateObject) + my.kind = jsonObjectStart + of tkCurlyRi: + my.kind = jsonObjectEnd + discard my.state.pop() + else: + my.kind = jsonError + my.err = errCurlyRiExpected + of stateArray: + case tk + of tkString, tkInt, tkFloat, tkTrue, tkFalse, tkNull: + my.state.add(stateExpectArrayComma) # expect value next! + my.kind = JsonEventKind(ord(tk)) + of tkBracketLe: + my.state.add(stateExpectArrayComma) + my.state.add(stateArray) + my.kind = jsonArrayStart + of tkCurlyLe: + my.state.add(stateExpectArrayComma) + my.state.add(stateObject) + my.kind = jsonObjectStart + of tkBracketRi: + my.kind = jsonArrayEnd + discard my.state.pop() + else: + my.kind = jsonError + my.err = errBracketRiExpected + of stateExpectArrayComma: + case tk + of tkComma: + discard my.state.pop() + next(my) + of tkBracketRi: + my.kind = jsonArrayEnd + discard my.state.pop() # pop stateExpectArrayComma + discard my.state.pop() # pop stateArray + else: + my.kind = jsonError + my.err = errBracketRiExpected + of stateExpectObjectComma: + case tk + of tkComma: + discard my.state.pop() + next(my) + of tkCurlyRi: + my.kind = jsonObjectEnd + discard my.state.pop() # pop stateExpectObjectComma + discard my.state.pop() # pop stateObject + else: + my.kind = jsonError + my.err = errCurlyRiExpected + of stateExpectColon: + case tk + of tkColon: + my.state[i] = stateExpectValue + next(my) + else: + my.kind = jsonError + my.err = errColonExpected + of stateExpectValue: + case tk + of tkString, tkInt, tkFloat, tkTrue, tkFalse, tkNull: + my.state[i] = stateExpectObjectComma + my.kind = JsonEventKind(ord(tk)) + of tkBracketLe: + my.state[i] = stateExpectObjectComma + my.state.add(stateArray) + my.kind = jsonArrayStart + of tkCurlyLe: + my.state[i] = stateExpectObjectComma + my.state.add(stateObject) + my.kind = jsonObjectStart + else: + my.kind = jsonError + my.err = errExprExpected + +proc raiseParseErr*(p: JsonParser, msg: string) {.noinline, noreturn.} = + ## raises an `EJsonParsingError` exception. + raise newException(JsonParsingError, errorMsgExpected(p, msg)) + +proc eat*(p: var JsonParser, tok: TokKind) = + if p.tok == tok: discard getTok(p) + else: raiseParseErr(p, tokToStr[tok]) diff --git a/lib/pure/parseopt.nim b/lib/pure/parseopt.nim index 23568edb9..58e1be0e4 100644 --- a/lib/pure/parseopt.nim +++ b/lib/pure/parseopt.nim @@ -11,11 +11,23 @@ ## It supports one convenience iterator over all command line options and some ## lower-level features. ## -## Supported syntax: +## Supported syntax with default empty ``shortNoVal``/``longNoVal``: ## ## 1. short options - ``-abcd``, where a, b, c, d are names ## 2. long option - ``--foo:bar``, ``--foo=bar`` or ``--foo`` ## 3. argument - everything else +## +## When ``shortNoVal``/``longNoVal`` are non-empty then the ':' and '=' above +## are still accepted, but become optional. Note that these option key sets +## must be updated along with the set of option keys taking no value, but +## keys which do take values need no special updates as their set evolves. +## +## When option values begin with ':' or '=' they need to be doubled up (as in +## ``--delim::``) or alternated (as in ``--delim=:``). +## +## The common ``--`` non-option argument delimiter appears as an empty string +## long option key. ``OptParser.cmd``, ``OptParser.pos``, and +## ``os.parseCmdLine`` may be used to complete parsing in that case. {.push debugger: off.} @@ -32,37 +44,37 @@ type cmdShortOption ## a short option ``-c`` detected OptParser* = object of RootObj ## this object implements the command line parser - cmd: string - pos: int + cmd*: string # cmd,pos exported so caller can catch "--" as.. + pos*: int # ..empty key or subcmd cmdArg & handle specially inShortState: bool + shortNoVal: set[char] + longNoVal: seq[string] kind*: CmdLineKind ## the dected command line token key*, val*: TaintedString ## key and value pair; ``key`` is the option ## or the argument, ``value`` is not "" if ## the option was given a value -{.deprecated: [TCmdLineKind: CmdLineKind, TOptParser: OptParser].} - proc parseWord(s: string, i: int, w: var string, - delim: set[char] = {'\x09', ' ', '\0'}): int = + delim: set[char] = {'\x09', ' '}): int = result = i - if s[result] == '\"': + if result < s.len and s[result] == '\"': inc(result) - while not (s[result] in {'\0', '\"'}): + while result < s.len and s[result] != '\"': add(w, s[result]) inc(result) - if s[result] == '\"': inc(result) + if result < s.len and s[result] == '\"': inc(result) else: - while not (s[result] in delim): + while result < s.len and s[result] notin delim: add(w, s[result]) inc(result) when declared(os.paramCount): proc quote(s: string): string = - if find(s, {' ', '\t'}) >= 0 and s[0] != '"': + if find(s, {' ', '\t'}) >= 0 and s.len > 0 and s[0] != '"': if s[0] == '-': result = newStringOfCap(s.len) - var i = parseWord(s, 0, result, {'\0', ' ', '\x09', ':', '='}) - if s[i] in {':','='}: + var i = parseWord(s, 0, result, {' ', '\x09', ':', '='}) + if i < s.len and s[i] in {':','='}: result.add s[i] inc i result.add '"' @@ -78,11 +90,19 @@ when declared(os.paramCount): # we cannot provide this for NimRtl creation on Posix, because we can't # access the command line arguments then! - proc initOptParser*(cmdline = ""): OptParser = + proc initOptParser*(cmdline = "", shortNoVal: set[char]={}, + longNoVal: seq[string] = @[]): OptParser = ## inits the option parser. If ``cmdline == ""``, the real command line - ## (as provided by the ``OS`` module) is taken. + ## (as provided by the ``OS`` module) is taken. If ``shortNoVal`` is + ## provided command users do not need to delimit short option keys and + ## values with a ':' or '='. If ``longNoVal`` is provided command users do + ## not need to delimit long option keys and values with a ':' or '=' + ## (though they still need at least a space). In both cases, ':' or '=' + ## may still be used if desired. They just become optional. result.pos = 0 result.inShortState = false + result.shortNoVal = shortNoVal + result.longNoVal = longNoVal if cmdline != "": result.cmd = cmdline else: @@ -94,47 +114,73 @@ when declared(os.paramCount): result.key = TaintedString"" result.val = TaintedString"" + proc initOptParser*(cmdline: seq[TaintedString], shortNoVal: set[char]={}, + longNoVal: seq[string] = @[]): OptParser = + ## inits the option parser. If ``cmdline.len == 0``, the real command line + ## (as provided by the ``OS`` module) is taken. ``shortNoVal`` and + ## ``longNoVal`` behavior is the same as for ``initOptParser(string,...)``. + result.pos = 0 + result.inShortState = false + result.shortNoVal = shortNoVal + result.longNoVal = longNoVal + result.cmd = "" + if cmdline.len != 0: + for i in 0..<cmdline.len: + result.cmd.add quote(cmdline[i].string) + result.cmd.add ' ' + else: + for i in countup(1, paramCount()): + result.cmd.add quote(paramStr(i).string) + result.cmd.add ' ' + result.kind = cmdEnd + result.key = TaintedString"" + result.val = TaintedString"" + proc handleShortOption(p: var OptParser) = var i = p.pos p.kind = cmdShortOption add(p.key.string, p.cmd[i]) inc(i) p.inShortState = true - while p.cmd[i] in {'\x09', ' '}: + while i < p.cmd.len and p.cmd[i] in {'\x09', ' '}: inc(i) p.inShortState = false - if p.cmd[i] in {':', '='}: - inc(i) + if i < p.cmd.len and p.cmd[i] in {':', '='} or + card(p.shortNoVal) > 0 and p.key.string[0] notin p.shortNoVal: + if i < p.cmd.len and p.cmd[i] in {':', '='}: + inc(i) p.inShortState = false - while p.cmd[i] in {'\x09', ' '}: inc(i) + while i < p.cmd.len and p.cmd[i] in {'\x09', ' '}: inc(i) i = parseWord(p.cmd, i, p.val.string) - if p.cmd[i] == '\0': p.inShortState = false + if i >= p.cmd.len: p.inShortState = false p.pos = i proc next*(p: var OptParser) {.rtl, extern: "npo$1".} = ## parses the first or next option; ``p.kind`` describes what token has been ## parsed. ``p.key`` and ``p.val`` are set accordingly. var i = p.pos - while p.cmd[i] in {'\x09', ' '}: inc(i) + while i < p.cmd.len and p.cmd[i] in {'\x09', ' '}: inc(i) p.pos = i setLen(p.key.string, 0) setLen(p.val.string, 0) if p.inShortState: handleShortOption(p) return - case p.cmd[i] - of '\0': + if i >= p.cmd.len: p.kind = cmdEnd - of '-': + return + if p.cmd[i] == '-': inc(i) - if p.cmd[i] == '-': - p.kind = cmdLongoption + if i < p.cmd.len and p.cmd[i] == '-': + p.kind = cmdLongOption inc(i) - i = parseWord(p.cmd, i, p.key.string, {'\0', ' ', '\x09', ':', '='}) - while p.cmd[i] in {'\x09', ' '}: inc(i) - if p.cmd[i] in {':', '='}: - inc(i) - while p.cmd[i] in {'\x09', ' '}: inc(i) + i = parseWord(p.cmd, i, p.key.string, {' ', '\x09', ':', '='}) + while i < p.cmd.len and p.cmd[i] in {'\x09', ' '}: inc(i) + if i < p.cmd.len and p.cmd[i] in {':', '='} or + len(p.longNoVal) > 0 and p.key.string notin p.longNoVal: + if i < p.cmd.len and p.cmd[i] in {':', '='}: + inc(i) + while i < p.cmd.len and p.cmd[i] in {'\x09', ' '}: inc(i) p.pos = parseWord(p.cmd, i, p.val.string) else: p.pos = i @@ -154,7 +200,7 @@ iterator getopt*(p: var OptParser): tuple[kind: CmdLineKind, key, val: TaintedSt ## Example: ## ## .. code-block:: nim - ## var p = initOptParser("--left --debug:3 -l=4 -r:2") + ## var p = initOptParser("--left --debug:3 -l -r:2") ## for kind, key, val in p.getopt(): ## case kind ## of cmdArgument: @@ -174,17 +220,30 @@ iterator getopt*(p: var OptParser): tuple[kind: CmdLineKind, key, val: TaintedSt yield (p.kind, p.key, p.val) when declared(initOptParser): - iterator getopt*(): tuple[kind: CmdLineKind, key, val: TaintedString] = - ## This is an convenience iterator for iterating over the command line arguments. - ## This create a new OptParser object. - ## See above for a more detailed example + iterator getopt*(cmdline: seq[TaintedString] = commandLineParams(), + shortNoVal: set[char]={}, longNoVal: seq[string] = @[]): + tuple[kind: CmdLineKind, key, val: TaintedString] = + ## This is an convenience iterator for iterating over command line arguments. + ## This creates a new OptParser. See the above ``getopt(var OptParser)`` + ## example for using default empty ``NoVal`` parameters. This example is + ## for the same option keys as that example but here option key-value + ## separators become optional for command users: ## ## .. code-block:: nim - ## for kind, key, val in getopt(): - ## # this will iterate over all arguments passed to the cmdline. - ## continue + ## for kind, key, val in getopt(shortNoVal = { 'l' }, + ## longNoVal = @[ "left" ]): + ## case kind + ## of cmdArgument: + ## filename = key + ## of cmdLongOption, cmdShortOption: + ## case key + ## of "help", "h": writeHelp() + ## of "version", "v": writeVersion() + ## of cmdEnd: assert(false) # cannot happen + ## if filename == "": + ## writeHelp() ## - var p = initOptParser() + var p = initOptParser(cmdline, shortNoVal=shortNoVal, longNoVal=longNoVal) while true: next(p) if p.kind == cmdEnd: break diff --git a/lib/pure/parseopt2.nim b/lib/pure/parseopt2.nim index a2ff9bf0c..b54a56c0c 100644 --- a/lib/pure/parseopt2.nim +++ b/lib/pure/parseopt2.nim @@ -17,6 +17,7 @@ ## 2. long option - ``--foo:bar``, ``--foo=bar`` or ``--foo`` ## 3. argument - everything else +{.deprecated: "Use the 'parseopt' module instead".} {.push debugger: off.} include "system/inclrtl" @@ -40,8 +41,6 @@ type ## or the argument, ``value`` is not "" if ## the option was given a value -{.deprecated: [TCmdLineKind: CmdLineKind, TOptParser: OptParser].} - proc initOptParser*(cmdline: seq[string]): OptParser {.rtl.} = ## Initalizes option parses with cmdline. cmdline should not contain ## argument 0 - program name. @@ -121,8 +120,6 @@ proc cmdLineRest*(p: OptParser): TaintedString {.rtl, extern: "npo2$1", deprecat type GetoptResult* = tuple[kind: CmdLineKind, key, val: TaintedString] -{.deprecated: [TGetoptResult: GetoptResult].} - iterator getopt*(p: var OptParser): GetoptResult = ## This is an convenience iterator for iterating over the given OptParser object. ## Example: diff --git a/lib/pure/parsesql.nim b/lib/pure/parsesql.nim index ae192ab9a..e3bab9a8d 100644 --- a/lib/pure/parsesql.nim +++ b/lib/pure/parsesql.nim @@ -11,7 +11,7 @@ ## parser. It parses PostgreSQL syntax and the SQL ANSI standard. import - hashes, strutils, lexbase, streams + hashes, strutils, lexbase # ------------------- scanner ------------------------------------------------- @@ -45,8 +45,6 @@ type SqlLexer* = object of BaseLexer ## the parser object. filename: string -{.deprecated: [TToken: Token, TSqlLexer: SqlLexer].} - const tokKindToStr: array[TokKind, string] = [ "invalid", "[EOF]", "identifier", "quoted identifier", "string constant", @@ -62,10 +60,6 @@ const "count", ] -proc open(L: var SqlLexer, input: Stream, filename: string) = - lexbase.open(L, input) - L.filename = filename - proc close(L: var SqlLexer) = lexbase.close(L) @@ -496,6 +490,7 @@ type SqlNodeKind* = enum ## kind of SQL abstract syntax tree nkNone, nkIdent, + nkQuotedIdent, nkStringLit, nkBitStringLit, nkHexStringLit, @@ -551,13 +546,18 @@ type nkCreateIndexIfNotExists, nkEnumDef +const + LiteralNodes = { + nkIdent, nkQuotedIdent, nkStringLit, nkBitStringLit, nkHexStringLit, + nkIntegerLit, nkNumericLit + } + type SqlParseError* = object of ValueError ## Invalid SQL encountered SqlNode* = ref SqlNodeObj ## an SQL abstract syntax tree node SqlNodeObj* = object ## an SQL abstract syntax tree node case kind*: SqlNodeKind ## kind of syntax tree - of nkIdent, nkStringLit, nkBitStringLit, nkHexStringLit, - nkIntegerLit, nkNumericLit: + of LiteralNodes: strVal*: string ## AST leaf: the identifier, numeric literal ## string literal, etc. else: @@ -566,21 +566,26 @@ type SqlParser* = object of SqlLexer ## SQL parser object tok: Token + {.deprecated: [EInvalidSql: SqlParseError, PSqlNode: SqlNode, TSqlNode: SqlNodeObj, TSqlParser: SqlParser, TSqlNodeKind: SqlNodeKind].} -proc newNode(k: SqlNodeKind): SqlNode = +proc newNode*(k: SqlNodeKind): SqlNode = new(result) result.kind = k -proc newNode(k: SqlNodeKind, s: string): SqlNode = +proc newNode*(k: SqlNodeKind, s: string): SqlNode = new(result) result.kind = k result.strVal = s +proc newNode*(k: SqlNodeKind, sons: seq[SqlNode]): SqlNode = + new(result) + result.kind = k + result.sons = sons + proc len*(n: SqlNode): int = - if n.kind in {nkIdent, nkStringLit, nkBitStringLit, nkHexStringLit, - nkIntegerLit, nkNumericLit}: + if n.kind in LiteralNodes: result = 0 else: result = n.sons.len @@ -630,7 +635,7 @@ proc eat(p: var SqlParser, keyw: string) = if isKeyw(p, keyw): getTok(p) else: - sqlError(p, keyw.toUpper() & " expected") + sqlError(p, keyw.toUpperAscii() & " expected") proc opt(p: var SqlParser, kind: TokKind) = if p.tok.kind == kind: getTok(p) @@ -689,7 +694,10 @@ proc parseSelect(p: var SqlParser): SqlNode proc identOrLiteral(p: var SqlParser): SqlNode = case p.tok.kind - of tkIdentifier, tkQuotedIdentifier: + of tkQuotedIdentifier: + result = newNode(nkQuotedIdent, p.tok.literal) + getTok(p) + of tkIdentifier: result = newNode(nkIdent, p.tok.literal) getTok(p) of tkStringConstant, tkEscapeConstant, tkDollarQuotedConstant: @@ -713,11 +721,15 @@ proc identOrLiteral(p: var SqlParser): SqlNode = result.add(parseExpr(p)) eat(p, tkParRi) else: - sqlError(p, "expression expected") - getTok(p) # we must consume a token here to prevend endless loops! + if p.tok.literal == "*": + result = newNode(nkIdent, p.tok.literal) + getTok(p) + else: + sqlError(p, "expression expected") + getTok(p) # we must consume a token here to prevend endless loops! proc primary(p: var SqlParser): SqlNode = - if p.tok.kind == tkOperator or isKeyw(p, "not"): + if (p.tok.kind == tkOperator and (p.tok.literal == "+" or p.tok.literal == "-")) or isKeyw(p, "not"): result = newNode(nkPrefix) result.add(newNode(nkIdent, p.tok.literal)) getTok(p) @@ -762,7 +774,7 @@ proc lowestExprAux(p: var SqlParser, v: var SqlNode, limit: int): int = result = opPred while opPred > limit: node = newNode(nkInfix) - opNode = newNode(nkIdent, p.tok.literal.toLower()) + opNode = newNode(nkIdent, p.tok.literal.toLowerAscii()) getTok(p) result = lowestExprAux(p, v2, opPred) node.add(opNode) @@ -1078,11 +1090,23 @@ proc parseSelect(p: var SqlParser): SqlNode = if p.tok.kind != tkComma: break getTok(p) result.add(g) - if isKeyw(p, "limit"): + if isKeyw(p, "order"): getTok(p) - var l = newNode(nkLimit) - l.add(parseExpr(p)) - result.add(l) + eat(p, "by") + var n = newNode(nkOrder) + while true: + var e = parseExpr(p) + if isKeyw(p, "asc"): + getTok(p) # is default + elif isKeyw(p, "desc"): + getTok(p) + var x = newNode(nkDesc) + x.add(e) + e = x + n.add(e) + if p.tok.kind != tkComma: break + getTok(p) + result.add(n) if isKeyw(p, "having"): var h = newNode(nkHaving) while true: @@ -1099,22 +1123,6 @@ proc parseSelect(p: var SqlParser): SqlNode = elif isKeyw(p, "except"): result.add(newNode(nkExcept)) getTok(p) - if isKeyw(p, "order"): - getTok(p) - eat(p, "by") - var n = newNode(nkOrder) - while true: - var e = parseExpr(p) - if isKeyw(p, "asc"): getTok(p) # is default - elif isKeyw(p, "desc"): - getTok(p) - var x = newNode(nkDesc) - x.add(e) - e = x - n.add(e) - if p.tok.kind != tkComma: break - getTok(p) - result.add(n) if isKeyw(p, "join") or isKeyw(p, "inner") or isKeyw(p, "outer") or isKeyw(p, "cross"): var join = newNode(nkJoin) result.add(join) @@ -1122,12 +1130,17 @@ proc parseSelect(p: var SqlParser): SqlNode = join.add(newNode(nkIdent, "")) getTok(p) else: - join.add(newNode(nkIdent, p.tok.literal.toLower())) + join.add(newNode(nkIdent, p.tok.literal.toLowerAscii())) getTok(p) eat(p, "join") join.add(parseFromItem(p)) eat(p, "on") join.add(parseExpr(p)) + if isKeyw(p, "limit"): + getTok(p) + var l = newNode(nkLimit) + l.add(parseExpr(p)) + result.add(l) proc parseStmt(p: var SqlParser; parent: SqlNode) = if isKeyw(p, "create"): @@ -1161,14 +1174,6 @@ proc parseStmt(p: var SqlParser; parent: SqlNode) = else: sqlError(p, "SELECT, CREATE, UPDATE or DELETE expected") -proc open(p: var SqlParser, input: Stream, filename: string) = - ## opens the parser `p` and assigns the input stream `input` to it. - ## `filename` is only used for error messages. - open(SqlLexer(p), input, filename) - p.tok.kind = tkInvalid - p.tok.literal = "" - getTok(p) - proc parse(p: var SqlParser): SqlNode = ## parses the content of `p`'s input stream and returns the SQL AST. ## Syntax errors raise an `SqlParseError` exception. @@ -1183,24 +1188,6 @@ proc close(p: var SqlParser) = ## closes the parser `p`. The associated input stream is closed too. close(SqlLexer(p)) -proc parseSQL*(input: Stream, filename: string): SqlNode = - ## parses the SQL from `input` into an AST and returns the AST. - ## `filename` is only used for error messages. - ## Syntax errors raise an `SqlParseError` exception. - var p: SqlParser - open(p, input, filename) - try: - result = parse(p) - finally: - close(p) - -proc parseSQL*(input: string, filename=""): SqlNode = - ## parses the SQL from `input` into an AST and returns the AST. - ## `filename` is only used for error messages. - ## Syntax errors raise an `SqlParseError` exception. - parseSQL(newStringStream(input), "") - - type SqlWriter = object indent: int @@ -1218,12 +1205,12 @@ proc add(s: var SqlWriter, thing: string) = proc addKeyw(s: var SqlWriter, thing: string) = var keyw = thing if s.upperCase: - keyw = keyw.toUpper() + keyw = keyw.toUpperAscii() s.add(keyw) proc addIden(s: var SqlWriter, thing: string) = var iden = thing - if iden.toLower() in reservedKeywords: + if iden.toLowerAscii() in reservedKeywords: iden = '"' & iden & '"' s.add(iden) @@ -1251,15 +1238,20 @@ proc addMulti(s: var SqlWriter, n: SqlNode, sep = ',', prefix, suffix: char) = ra(n.sons[i], s) s.add(suffix) +proc quoted(s: string): string = + "\"" & replace(s, "\"", "\"\"") & "\"" + proc ra(n: SqlNode, s: var SqlWriter) = if n == nil: return case n.kind of nkNone: discard of nkIdent: - if allCharsInSet(n.strVal, {'\33'..'\127'}) and n.strVal.toLower() notin reservedKeywords: + if allCharsInSet(n.strVal, {'\33'..'\127'}): s.add(n.strVal) else: - s.add("\"" & replace(n.strVal, "\"", "\"\"") & "\"") + s.add(quoted(n.strVal)) + of nkQuotedIdent: + s.add(quoted(n.strVal)) of nkStringLit: s.add(escape(n.strVal, "'", "'")) of nkBitStringLit: @@ -1361,18 +1353,19 @@ proc ra(n: SqlNode, s: var SqlWriter) = s.addKeyw("select") if n.kind == nkSelectDistinct: s.addKeyw("distinct") - s.addMulti(n.sons[0]) - for i in 1 .. n.len-1: + for i in 0 ..< n.len: ra(n.sons[i], s) of nkSelectColumns: - assert(false) + for i, column in n.sons: + if i > 0: s.add(',') + ra(column, s) of nkSelectPair: ra(n.sons[0], s) if n.sons.len == 2: s.addKeyw("as") ra(n.sons[1], s) of nkFromItemPair: - if n.sons[0].kind == nkIdent: + if n.sons[0].kind in {nkIdent, nkQuotedIdent}: ra(n.sons[0], s) else: assert n.sons[0].kind == nkSelect @@ -1472,3 +1465,35 @@ proc renderSQL*(n: SqlNode, upperCase=false): string = proc `$`*(n: SqlNode): string = ## an alias for `renderSQL`. renderSQL(n) + +when not defined(js): + import streams + + proc open(L: var SqlLexer, input: Stream, filename: string) = + lexbase.open(L, input) + L.filename = filename + + proc open(p: var SqlParser, input: Stream, filename: string) = + ## opens the parser `p` and assigns the input stream `input` to it. + ## `filename` is only used for error messages. + open(SqlLexer(p), input, filename) + p.tok.kind = tkInvalid + p.tok.literal = "" + getTok(p) + + proc parseSQL*(input: Stream, filename: string): SqlNode = + ## parses the SQL from `input` into an AST and returns the AST. + ## `filename` is only used for error messages. + ## Syntax errors raise an `SqlParseError` exception. + var p: SqlParser + open(p, input, filename) + try: + result = parse(p) + finally: + close(p) + + proc parseSQL*(input: string, filename=""): SqlNode = + ## parses the SQL from `input` into an AST and returns the AST. + ## `filename` is only used for error messages. + ## Syntax errors raise an `SqlParseError` exception. + parseSQL(newStringStream(input), "") diff --git a/lib/pure/parseutils.nim b/lib/pure/parseutils.nim index 57387e62e..d54f1454b 100644 --- a/lib/pure/parseutils.nim +++ b/lib/pure/parseutils.nim @@ -11,7 +11,7 @@ ## ## To unpack raw bytes look at the `streams <streams.html>`_ module. -{.deadCodeElim: on.} +{.deadCodeElim: on.} # dce option deprecated {.push debugger:off .} # the user does not want to trace a part # of the standard library! @@ -51,9 +51,9 @@ proc parseHex*(s: string, number: var int, start = 0; maxLen = 0): int {. ## upper bound. Not more than ```maxLen`` characters are parsed. var i = start var foundDigit = false - if s[i] == '0' and (s[i+1] == 'x' or s[i+1] == 'X'): inc(i, 2) - elif s[i] == '#': inc(i) let last = if maxLen == 0: s.len else: i+maxLen + if i+1 < last and s[i] == '0' and (s[i+1] == 'x' or s[i+1] == 'X'): inc(i, 2) + elif i < last and s[i] == '#': inc(i) while i < last: case s[i] of '_': discard @@ -76,8 +76,8 @@ proc parseOct*(s: string, number: var int, start = 0): int {. ## the number of the parsed characters or 0 in case of an error. var i = start var foundDigit = false - if s[i] == '0' and (s[i+1] == 'o' or s[i+1] == 'O'): inc(i, 2) - while true: + if i+1 < s.len and s[i] == '0' and (s[i+1] == 'o' or s[i+1] == 'O'): inc(i, 2) + while i < s.len: case s[i] of '_': discard of '0'..'7': @@ -93,8 +93,8 @@ proc parseBin*(s: string, number: var int, start = 0): int {. ## the number of the parsed characters or 0 in case of an error. var i = start var foundDigit = false - if s[i] == '0' and (s[i+1] == 'b' or s[i+1] == 'B'): inc(i, 2) - while true: + if i+1 < s.len and s[i] == '0' and (s[i+1] == 'b' or s[i+1] == 'B'): inc(i, 2) + while i < s.len: case s[i] of '_': discard of '0'..'1': @@ -108,9 +108,9 @@ proc parseIdent*(s: string, ident: var string, start = 0): int = ## parses an identifier and stores it in ``ident``. Returns ## the number of the parsed characters or 0 in case of an error. var i = start - if s[i] in IdentStartChars: + if i < s.len and s[i] in IdentStartChars: inc(i) - while s[i] in IdentChars: inc(i) + while i < s.len and s[i] in IdentChars: inc(i) ident = substr(s, start, i-1) result = i-start @@ -119,11 +119,9 @@ proc parseIdent*(s: string, start = 0): string = ## Returns the parsed identifier or an empty string in case of an error. result = "" var i = start - - if s[i] in IdentStartChars: + if i < s.len and s[i] in IdentStartChars: inc(i) - while s[i] in IdentChars: inc(i) - + while i < s.len and s[i] in IdentChars: inc(i) result = substr(s, start, i-1) proc parseToken*(s: string, token: var string, validChars: set[char], @@ -134,24 +132,26 @@ proc parseToken*(s: string, token: var string, validChars: set[char], ## ## **Deprecated since version 0.8.12**: Use ``parseWhile`` instead. var i = start - while s[i] in validChars: inc(i) + while i < s.len and s[i] in validChars: inc(i) result = i-start token = substr(s, start, i-1) proc skipWhitespace*(s: string, start = 0): int {.inline.} = ## skips the whitespace starting at ``s[start]``. Returns the number of ## skipped characters. - while s[start+result] in Whitespace: inc(result) + while start+result < s.len and s[start+result] in Whitespace: inc(result) proc skip*(s, token: string, start = 0): int {.inline.} = ## skips the `token` starting at ``s[start]``. Returns the length of `token` ## or 0 if there was no `token` at ``s[start]``. - while result < token.len and s[result+start] == token[result]: inc(result) + while start+result < s.len and result < token.len and + s[result+start] == token[result]: + inc(result) if result != token.len: result = 0 proc skipIgnoreCase*(s, token: string, start = 0): int = ## same as `skip` but case is ignored for token matching. - while result < token.len and + while start+result < s.len and result < token.len and toLower(s[result+start]) == toLower(token[result]): inc(result) if result != token.len: result = 0 @@ -159,18 +159,18 @@ proc skipUntil*(s: string, until: set[char], start = 0): int {.inline.} = ## Skips all characters until one char from the set `until` is found ## or the end is reached. ## Returns number of characters skipped. - while s[result+start] notin until and s[result+start] != '\0': inc(result) + while start+result < s.len and s[result+start] notin until: inc(result) proc skipUntil*(s: string, until: char, start = 0): int {.inline.} = ## Skips all characters until the char `until` is found ## or the end is reached. ## Returns number of characters skipped. - while s[result+start] != until and s[result+start] != '\0': inc(result) + while start+result < s.len and s[result+start] != until: inc(result) proc skipWhile*(s: string, toSkip: set[char], start = 0): int {.inline.} = ## Skips all characters while one char from the set `token` is found. ## Returns number of characters skipped. - while s[result+start] in toSkip and s[result+start] != '\0': inc(result) + while start+result < s.len and s[result+start] in toSkip: inc(result) proc parseUntil*(s: string, token: var string, until: set[char], start = 0): int {.inline.} = @@ -197,6 +197,9 @@ proc parseUntil*(s: string, token: var string, until: string, ## parses a token and stores it in ``token``. Returns ## the number of the parsed characters or 0 in case of an error. A token ## consists of any character that comes before the `until` token. + if until.len == 0: + token.setLen(0) + return 0 var i = start while i < s.len: if s[i] == until[0]: @@ -214,7 +217,7 @@ proc parseWhile*(s: string, token: var string, validChars: set[char], ## the number of the parsed characters or 0 in case of an error. A token ## consists of the characters in `validChars`. var i = start - while s[i] in validChars: inc(i) + while i < s.len and s[i] in validChars: inc(i) result = i-start token = substr(s, start, i-1) @@ -231,16 +234,17 @@ proc rawParseInt(s: string, b: var BiggestInt, start = 0): int = var sign: BiggestInt = -1 i = start - if s[i] == '+': inc(i) - elif s[i] == '-': - inc(i) - sign = 1 - if s[i] in {'0'..'9'}: + if i < s.len: + if s[i] == '+': inc(i) + elif s[i] == '-': + inc(i) + sign = 1 + if i < s.len and s[i] in {'0'..'9'}: b = 0 - while s[i] in {'0'..'9'}: + while i < s.len and s[i] in {'0'..'9'}: b = b * 10 - (ord(s[i]) - ord('0')) inc(i) - while s[i] == '_': inc(i) # underscores are allowed and ignored + while i < s.len and s[i] == '_': inc(i) # underscores are allowed and ignored b = b * sign result = i - start {.pop.} # overflowChecks @@ -281,17 +285,17 @@ proc parseSaturatedNatural*(s: string, b: var int, start = 0): int = ## discard parseSaturatedNatural("848", res) ## doAssert res == 848 var i = start - if s[i] == '+': inc(i) - if s[i] in {'0'..'9'}: + if i < s.len and s[i] == '+': inc(i) + if i < s.len and s[i] in {'0'..'9'}: b = 0 - while s[i] in {'0'..'9'}: + while i < s.len and s[i] in {'0'..'9'}: let c = ord(s[i]) - ord('0') if b <= (high(int) - c) div 10: b = b * 10 + c else: b = high(int) inc(i) - while s[i] == '_': inc(i) # underscores are allowed and ignored + while i < s.len and s[i] == '_': inc(i) # underscores are allowed and ignored result = i - start # overflowChecks doesn't work with BiggestUInt @@ -300,16 +304,16 @@ proc rawParseUInt(s: string, b: var BiggestUInt, start = 0): int = res = 0.BiggestUInt prev = 0.BiggestUInt i = start - if s[i] == '+': inc(i) # Allow - if s[i] in {'0'..'9'}: + if i < s.len and s[i] == '+': inc(i) # Allow + if i < s.len and s[i] in {'0'..'9'}: b = 0 - while s[i] in {'0'..'9'}: + while i < s.len and s[i] in {'0'..'9'}: prev = res res = res * 10 + (ord(s[i]) - ord('0')).BiggestUInt if prev > res: return 0 # overflowChecks emulation inc(i) - while s[i] == '_': inc(i) # underscores are allowed and ignored + while i < s.len and s[i] == '_': inc(i) # underscores are allowed and ignored b = res result = i - start @@ -364,8 +368,6 @@ type ikVar, ## ``var`` part of the interpolated string ikExpr ## ``expr`` part of the interpolated string -{.deprecated: [TInterpolatedKind: InterpolatedKind].} - iterator interpolatedFragments*(s: string): tuple[kind: InterpolatedKind, value: string] = ## Tokenizes the string `s` into substrings for interpolation purposes. @@ -389,31 +391,31 @@ iterator interpolatedFragments*(s: string): tuple[kind: InterpolatedKind, var kind: InterpolatedKind while true: var j = i - if s[j] == '$': - if s[j+1] == '{': + if j < s.len and s[j] == '$': + if j+1 < s.len and s[j+1] == '{': inc j, 2 var nesting = 0 - while true: - case s[j] - of '{': inc nesting - of '}': - if nesting == 0: - inc j - break - dec nesting - of '\0': - raise newException(ValueError, - "Expected closing '}': " & substr(s, i, s.high)) - else: discard - inc j + block curlies: + while j < s.len: + case s[j] + of '{': inc nesting + of '}': + if nesting == 0: + inc j + break curlies + dec nesting + else: discard + inc j + raise newException(ValueError, + "Expected closing '}': " & substr(s, i, s.high)) inc i, 2 # skip ${ kind = ikExpr - elif s[j+1] in IdentStartChars: + elif j+1 < s.len and s[j+1] in IdentStartChars: inc j, 2 - while s[j] in IdentChars: inc(j) + while j < s.len and s[j] in IdentChars: inc(j) inc i # skip $ kind = ikVar - elif s[j+1] == '$': + elif j+1 < s.len and s[j+1] == '$': inc j, 2 inc i # skip $ kind = ikDollar diff --git a/lib/pure/parsexml.nim b/lib/pure/parsexml.nim index 978c9c516..e0000aad3 100644 --- a/lib/pure/parsexml.nim +++ b/lib/pure/parsexml.nim @@ -98,9 +98,6 @@ type filename: string options: set[XmlParseOption] -{.deprecated: [TXmlParser: XmlParser, TXmlParseOptions: XmlParseOption, - TXmlError: XmlErrorKind, TXmlEventKind: XmlEventKind].} - const errorMessages: array[XmlErrorKind, string] = [ "no error", diff --git a/lib/pure/pegs.nim b/lib/pure/pegs.nim index 5ae2d9182..39c5790ed 100644 --- a/lib/pure/pegs.nim +++ b/lib/pure/pegs.nim @@ -74,8 +74,8 @@ type line: int ## line the symbol has been declared/used in col: int ## column the symbol has been declared/used in flags: set[NonTerminalFlag] ## the nonterminal's flags - rule: Node ## the rule that the symbol refers to - Node {.shallow.} = object + rule: Peg ## the rule that the symbol refers to + Peg* {.shallow.} = object ## type that represents a PEG case kind: PegKind of pkEmpty..pkWhitespace: nil of pkTerminal, pkTerminalIgnoreCase, pkTerminalIgnoreStyle: term: string @@ -83,13 +83,9 @@ type of pkCharChoice, pkGreedyRepSet: charChoice: ref set[char] of pkNonTerminal: nt: NonTerminal of pkBackRef..pkBackRefIgnoreStyle: index: range[0..MaxSubpatterns] - else: sons: seq[Node] + else: sons: seq[Peg] NonTerminal* = ref NonTerminalObj - Peg* = Node ## type that represents a PEG - -{.deprecated: [TPeg: Peg, TNode: Node].} - proc term*(t: string): Peg {.nosideEffect, rtl, extern: "npegs$1Str".} = ## constructs a PEG from a terminal string if t.len != 1: @@ -534,15 +530,15 @@ proc rawMatch*(s: string, p: Peg, start: int, c: var Captures): int {. case p.kind of pkEmpty: result = 0 # match of length 0 of pkAny: - if s[start] != '\0': result = 1 + if start < s.len: result = 1 else: result = -1 of pkAnyRune: - if s[start] != '\0': + if start < s.len: result = runeLenAt(s, start) else: result = -1 of pkLetter: - if s[start] != '\0': + if start < s.len: var a: Rune result = start fastRuneAt(s, result, a) @@ -551,7 +547,7 @@ proc rawMatch*(s: string, p: Peg, start: int, c: var Captures): int {. else: result = -1 of pkLower: - if s[start] != '\0': + if start < s.len: var a: Rune result = start fastRuneAt(s, result, a) @@ -560,7 +556,7 @@ proc rawMatch*(s: string, p: Peg, start: int, c: var Captures): int {. else: result = -1 of pkUpper: - if s[start] != '\0': + if start < s.len: var a: Rune result = start fastRuneAt(s, result, a) @@ -569,7 +565,7 @@ proc rawMatch*(s: string, p: Peg, start: int, c: var Captures): int {. else: result = -1 of pkTitle: - if s[start] != '\0': + if start < s.len: var a: Rune result = start fastRuneAt(s, result, a) @@ -578,7 +574,7 @@ proc rawMatch*(s: string, p: Peg, start: int, c: var Captures): int {. else: result = -1 of pkWhitespace: - if s[start] != '\0': + if start < s.len: var a: Rune result = start fastRuneAt(s, result, a) @@ -589,15 +585,15 @@ proc rawMatch*(s: string, p: Peg, start: int, c: var Captures): int {. of pkGreedyAny: result = len(s) - start of pkNewLine: - if s[start] == '\L': result = 1 - elif s[start] == '\C': - if s[start+1] == '\L': result = 2 + if start < s.len and s[start] == '\L': result = 1 + elif start < s.len and s[start] == '\C': + if start+1 < s.len and s[start+1] == '\L': result = 2 else: result = 1 else: result = -1 of pkTerminal: result = len(p.term) for i in 0..result-1: - if p.term[i] != s[start+i]: + if start+i >= s.len or p.term[i] != s[start+i]: result = -1 break of pkTerminalIgnoreCase: @@ -606,6 +602,9 @@ proc rawMatch*(s: string, p: Peg, start: int, c: var Captures): int {. a, b: Rune result = start while i < len(p.term): + if result >= s.len: + result = -1 + break fastRuneAt(p.term, i, a) fastRuneAt(s, result, b) if toLower(a) != toLower(b): @@ -618,21 +617,26 @@ proc rawMatch*(s: string, p: Peg, start: int, c: var Captures): int {. a, b: Rune result = start while i < len(p.term): - while true: + while i < len(p.term): fastRuneAt(p.term, i, a) if a != Rune('_'): break - while true: + while result < s.len: fastRuneAt(s, result, b) if b != Rune('_'): break - if toLower(a) != toLower(b): + if result >= s.len: + if i >= p.term.len: break + else: + result = -1 + break + elif toLower(a) != toLower(b): result = -1 break dec(result, start) of pkChar: - if p.ch == s[start]: result = 1 + if start < s.len and p.ch == s[start]: result = 1 else: result = -1 of pkCharChoice: - if contains(p.charChoice[], s[start]): result = 1 + if start < s.len and contains(p.charChoice[], s[start]): result = 1 else: result = -1 of pkNonTerminal: var oldMl = c.ml @@ -695,10 +699,10 @@ proc rawMatch*(s: string, p: Peg, start: int, c: var Captures): int {. of pkGreedyRepChar: result = 0 var ch = p.ch - while ch == s[start+result]: inc(result) + while start+result < s.len and ch == s[start+result]: inc(result) of pkGreedyRepSet: result = 0 - while contains(p.charChoice[], s[start+result]): inc(result) + while start+result < s.len and contains(p.charChoice[], s[start+result]): inc(result) of pkOption: result = max(0, rawMatch(s, p.sons[0], start, c)) of pkAndPredicate: @@ -1006,14 +1010,18 @@ proc replace*(s: string, sub: Peg, cb: proc( inc(m) add(result, substr(s, i)) -proc transformFile*(infile, outfile: string, - subs: varargs[tuple[pattern: Peg, repl: string]]) {. - rtl, extern: "npegs$1".} = - ## reads in the file `infile`, performs a parallel replacement (calls - ## `parallelReplace`) and writes back to `outfile`. Raises ``EIO`` if an - ## error occurs. This is supposed to be used for quick scripting. - var x = readFile(infile).string - writeFile(outfile, x.parallelReplace(subs)) +when not defined(js): + proc transformFile*(infile, outfile: string, + subs: varargs[tuple[pattern: Peg, repl: string]]) {. + rtl, extern: "npegs$1".} = + ## reads in the file `infile`, performs a parallel replacement (calls + ## `parallelReplace`) and writes back to `outfile`. Raises ``EIO`` if an + ## error occurs. This is supposed to be used for quick scripting. + ## + ## **Note**: this proc does not exist while using the JS backend. + var x = readFile(infile).string + writeFile(outfile, x.parallelReplace(subs)) + iterator split*(s: string, sep: Peg): string = ## Splits the string `s` into substrings. @@ -1117,7 +1125,7 @@ proc handleCR(L: var PegLexer, pos: int): int = assert(L.buf[pos] == '\c') inc(L.lineNumber) result = pos+1 - if L.buf[result] == '\L': inc(result) + if result < L.buf.len and L.buf[result] == '\L': inc(result) L.lineStart = result proc handleLF(L: var PegLexer, pos: int): int = @@ -1213,12 +1221,13 @@ proc getEscapedChar(c: var PegLexer, tok: var Token) = proc skip(c: var PegLexer) = var pos = c.bufpos var buf = c.buf - while true: + while pos < c.buf.len: case buf[pos] of ' ', '\t': inc(pos) of '#': - while not (buf[pos] in {'\c', '\L', '\0'}): inc(pos) + while (pos < c.buf.len) and + not (buf[pos] in {'\c', '\L', '\0'}): inc(pos) of '\c': pos = handleCR(c, pos) buf = c.buf @@ -1234,7 +1243,7 @@ proc getString(c: var PegLexer, tok: var Token) = var pos = c.bufpos + 1 var buf = c.buf var quote = buf[pos-1] - while true: + while pos < c.buf.len: case buf[pos] of '\\': c.bufpos = pos @@ -1257,7 +1266,7 @@ proc getDollar(c: var PegLexer, tok: var Token) = if buf[pos] in {'0'..'9'}: tok.kind = tkBackref tok.index = 0 - while buf[pos] in {'0'..'9'}: + while pos < c.buf.len and buf[pos] in {'0'..'9'}: tok.index = tok.index * 10 + ord(buf[pos]) - ord('0') inc(pos) else: @@ -1273,11 +1282,11 @@ proc getCharSet(c: var PegLexer, tok: var Token) = if buf[pos] == '^': inc(pos) caret = true - while true: + while pos < c.buf.len: var ch: char case buf[pos] of ']': - inc(pos) + if pos < c.buf.len: inc(pos) break of '\\': c.bufpos = pos @@ -1292,11 +1301,14 @@ proc getCharSet(c: var PegLexer, tok: var Token) = inc(pos) incl(tok.charset, ch) if buf[pos] == '-': - if buf[pos+1] == ']': + if pos+1 < c.buf.len and buf[pos+1] == ']': incl(tok.charset, '-') inc(pos) else: - inc(pos) + if pos+1 < c.buf.len: + inc(pos) + else: + break var ch2: char case buf[pos] of '\\': @@ -1308,8 +1320,11 @@ proc getCharSet(c: var PegLexer, tok: var Token) = tok.kind = tkInvalid break else: - ch2 = buf[pos] - inc(pos) + if pos+1 < c.buf.len: + ch2 = buf[pos] + inc(pos) + else: + break for i in ord(ch)+1 .. ord(ch2): incl(tok.charset, chr(i)) c.bufpos = pos @@ -1318,15 +1333,15 @@ proc getCharSet(c: var PegLexer, tok: var Token) = proc getSymbol(c: var PegLexer, tok: var Token) = var pos = c.bufpos var buf = c.buf - while true: + while pos < c.buf.len: add(tok.literal, buf[pos]) inc(pos) - if buf[pos] notin strutils.IdentChars: break + if pos < buf.len and buf[pos] notin strutils.IdentChars: break c.bufpos = pos tok.kind = tkIdentifier proc getBuiltin(c: var PegLexer, tok: var Token) = - if c.buf[c.bufpos+1] in strutils.Letters: + if c.bufpos+1 < c.buf.len and c.buf[c.bufpos+1] in strutils.Letters: inc(c.bufpos) getSymbol(c, tok) tok.kind = tkBuiltin @@ -1339,10 +1354,12 @@ proc getTok(c: var PegLexer, tok: var Token) = tok.modifier = modNone setLen(tok.literal, 0) skip(c) + case c.buf[c.bufpos] of '{': inc(c.bufpos) - if c.buf[c.bufpos] == '@' and c.buf[c.bufpos+1] == '}': + if c.buf[c.bufpos] == '@' and c.bufpos+2 < c.buf.len and + c.buf[c.bufpos+1] == '}': tok.kind = tkCurlyAt inc(c.bufpos, 2) add(tok.literal, "{@}") @@ -1375,13 +1392,11 @@ proc getTok(c: var PegLexer, tok: var Token) = getBuiltin(c, tok) of '\'', '"': getString(c, tok) of '$': getDollar(c, tok) - of '\0': - tok.kind = tkEof - tok.literal = "[EOF]" of 'a'..'z', 'A'..'Z', '\128'..'\255': getSymbol(c, tok) if c.buf[c.bufpos] in {'\'', '"'} or - c.buf[c.bufpos] == '$' and c.buf[c.bufpos+1] in {'0'..'9'}: + c.buf[c.bufpos] == '$' and c.bufpos+1 < c.buf.len and + c.buf[c.bufpos+1] in {'0'..'9'}: case tok.literal of "i": tok.modifier = modIgnoreCase of "y": tok.modifier = modIgnoreStyle @@ -1402,7 +1417,7 @@ proc getTok(c: var PegLexer, tok: var Token) = inc(c.bufpos) add(tok.literal, '+') of '<': - if c.buf[c.bufpos+1] == '-': + if c.bufpos+2 < c.buf.len and c.buf[c.bufpos+1] == '-': inc(c.bufpos, 2) tok.kind = tkArrow add(tok.literal, "<-") @@ -1437,14 +1452,17 @@ proc getTok(c: var PegLexer, tok: var Token) = inc(c.bufpos) add(tok.literal, '^') else: + if c.bufpos >= c.buf.len: + tok.kind = tkEof + tok.literal = "[EOF]" add(tok.literal, c.buf[c.bufpos]) inc(c.bufpos) proc arrowIsNextTok(c: PegLexer): bool = # the only look ahead we need var pos = c.bufpos - while c.buf[pos] in {'\t', ' '}: inc(pos) - result = c.buf[pos] == '<' and c.buf[pos+1] == '-' + while pos < c.buf.len and c.buf[pos] in {'\t', ' '}: inc(pos) + result = c.buf[pos] == '<' and (pos+1 < c.buf.len) and c.buf[pos+1] == '-' # ----------------------------- parser ---------------------------------------- @@ -1467,7 +1485,7 @@ proc pegError(p: PegParser, msg: string, line = -1, col = -1) = proc getTok(p: var PegParser) = getTok(p, p.tok) - if p.tok.kind == tkInvalid: pegError(p, "invalid token") + if p.tok.kind == tkInvalid: pegError(p, "'" & p.tok.literal & "' is invalid token") proc eat(p: var PegParser, kind: TokKind) = if p.tok.kind == kind: getTok(p) diff --git a/lib/pure/random.nim b/lib/pure/random.nim index de419b9fb..01ea9c845 100644 --- a/lib/pure/random.nim +++ b/lib/pure/random.nim @@ -115,6 +115,7 @@ proc rand*(r: var Rand; max: int): int {.benign.} = ## random number is always the same, unless `randomize` is called ## which initializes the random number generator with a "random" ## number, i.e. a tickcount. + if max == 0: return while true: let x = next(r) if x <= randMax - (randMax mod ui(max)): @@ -190,8 +191,8 @@ when not defined(nimscript): proc randomize*() {.benign.} = ## Initializes the random number generator with a "random" ## number, i.e. a tickcount. Note: Does not work for NimScript. - let time = int64(times.epochTime() * 1_000_000_000) - randomize(time) + let now = times.getTime() + randomize(convert(Seconds, Nanoseconds, now.toUnix) + now.nanosecond) {.pop.} @@ -213,4 +214,8 @@ when isMainModule: shuffle(a) doAssert a[0] == 1 doAssert a[1] == 0 + + doAssert rand(0) == 0 + doAssert rand("a") == 'a' + main() diff --git a/lib/pure/rationals.nim b/lib/pure/rationals.nim index 7907b4e6c..3946cf85b 100644 --- a/lib/pure/rationals.nim +++ b/lib/pure/rationals.nim @@ -241,6 +241,33 @@ proc abs*[T](x: Rational[T]): Rational[T] = result.num = abs x.num result.den = abs x.den +proc `div`*[T: SomeInteger](x, y: Rational[T]): T = + ## Computes the rational truncated division. + (x.num * y.den) div (y.num * x.den) + +proc `mod`*[T: SomeInteger](x, y: Rational[T]): Rational[T] = + ## Computes the rational modulo by truncated division (remainder). + ## This is same as ``x - (x div y) * y``. + result = ((x.num * y.den) mod (y.num * x.den)) // (x.den * y.den) + reduce(result) + +proc floorDiv*[T: SomeInteger](x, y: Rational[T]): T = + ## Computes the rational floor division. + ## + ## Floor division is conceptually defined as ``floor(x / y)``. + ## This is different from the ``div`` operator, which is defined + ## as ``trunc(x / y)``. That is, ``div`` rounds towards ``0`` and ``floorDiv`` + ## rounds down. + floorDiv(x.num * y.den, y.num * x.den) + +proc floorMod*[T: SomeInteger](x, y: Rational[T]): Rational[T] = + ## Computes the rational modulo by floor division (modulo). + ## + ## This is same as ``x - floorDiv(x, y) * y``. + ## This proc behaves the same as the ``%`` operator in python. + result = floorMod(x.num * y.den, y.num * x.den) // (x.den * y.den) + reduce(result) + proc hash*[T](x: Rational[T]): Hash = ## Computes hash for rational `x` # reduce first so that hash(x) == hash(y) for x == y @@ -339,3 +366,12 @@ when isMainModule: assert toRational(0.33) == 33 // 100 assert toRational(0.22) == 11 // 50 assert toRational(10.0) == 10 // 1 + + assert (1//1) div (3//10) == 3 + assert (-1//1) div (3//10) == -3 + assert (3//10) mod (1//1) == 3//10 + assert (-3//10) mod (1//1) == -3//10 + assert floorDiv(1//1, 3//10) == 3 + assert floorDiv(-1//1, 3//10) == -4 + assert floorMod(3//10, 1//1) == 3//10 + assert floorMod(-3//10, 1//1) == 7//10 diff --git a/lib/pure/ropes.nim b/lib/pure/ropes.nim index 6ddd61afa..9b9cdb52a 100644 --- a/lib/pure/ropes.nim +++ b/lib/pure/ropes.nim @@ -19,7 +19,7 @@ include "system/inclrtl" import streams -{.deadCodeElim: on.} +{.deadCodeElim: on.} # dce option deprecated {.push debugger:off .} # the user does not want to trace a part # of the standard library! @@ -37,8 +37,6 @@ type length: int data: string # != nil if a leaf -{.deprecated: [PRope: Rope].} - proc isConc(r: Rope): bool {.inline.} = return isNil(r.data) # Note that the left and right pointers are not needed for leafs. diff --git a/lib/pure/scgi.nim b/lib/pure/scgi.nim index 1ff26954e..e36803823 100644 --- a/lib/pure/scgi.nim +++ b/lib/pure/scgi.nim @@ -94,9 +94,6 @@ type disp: Dispatcher AsyncScgiState* = ref AsyncScgiStateObj -{.deprecated: [EScgi: ScgiError, TScgiState: ScgiState, - PAsyncScgiState: AsyncScgiState].} - proc recvBuffer(s: var ScgiState, L: int) = if L > s.bufLen: s.bufLen = L diff --git a/lib/pure/securehash.nim b/lib/pure/securehash.nim index 57c1f3631..c6cde599a 100644 --- a/lib/pure/securehash.nim +++ b/lib/pure/securehash.nim @@ -1,195 +1,6 @@ -# -# -# The Nim Compiler -# (c) Copyright 2015 Nim Contributors -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# -import strutils -const Sha1DigestSize = 20 +## This module is a deprecated alias for the ``sha1`` module. +{.deprecated.} -type - Sha1Digest = array[0 .. Sha1DigestSize-1, uint8] - SecureHash* = distinct Sha1Digest - -# Copyright (c) 2011, Micael Hildenborg -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of Micael Hildenborg nor the -# names of its contributors may be used to endorse or promote products -# derived from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY Micael Hildenborg ''AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL Micael Hildenborg BE LIABLE FOR ANY -# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# Ported to Nim by Erik O'Leary - -type - Sha1State* = array[0 .. 5-1, uint32] - Sha1Buffer = array[0 .. 80-1, uint32] - -template clearBuffer(w: Sha1Buffer, len = 16) = - zeroMem(addr(w), len * sizeof(uint32)) - -proc init*(result: var Sha1State) = - result[0] = 0x67452301'u32 - result[1] = 0xefcdab89'u32 - result[2] = 0x98badcfe'u32 - result[3] = 0x10325476'u32 - result[4] = 0xc3d2e1f0'u32 - -proc innerHash(state: var Sha1State, w: var Sha1Buffer) = - var - a = state[0] - b = state[1] - c = state[2] - d = state[3] - e = state[4] - - var round = 0 - - template rot(value, bits: uint32): uint32 = - (value shl bits) or (value shr (32 - bits)) - - template sha1(fun, val: uint32) = - let t = rot(a, 5) + fun + e + val + w[round] - e = d - d = c - c = rot(b, 30) - b = a - a = t - - template process(body: untyped) = - w[round] = rot(w[round - 3] xor w[round - 8] xor w[round - 14] xor w[round - 16], 1) - body - inc(round) - - template wrap(dest, value: untyped) = - let v = dest + value - dest = v - - while round < 16: - sha1((b and c) or (not b and d), 0x5a827999'u32) - inc(round) - - while round < 20: - process: - sha1((b and c) or (not b and d), 0x5a827999'u32) - - while round < 40: - process: - sha1(b xor c xor d, 0x6ed9eba1'u32) - - while round < 60: - process: - sha1((b and c) or (b and d) or (c and d), 0x8f1bbcdc'u32) - - while round < 80: - process: - sha1(b xor c xor d, 0xca62c1d6'u32) - - wrap state[0], a - wrap state[1], b - wrap state[2], c - wrap state[3], d - wrap state[4], e - -proc sha1(src: cstring; len: int): Sha1Digest = - #Initialize state - var state: Sha1State - init(state) - - #Create w buffer - var w: Sha1Buffer - - #Loop through all complete 64byte blocks. - let byteLen = len - let endOfFullBlocks = byteLen - 64 - var endCurrentBlock = 0 - var currentBlock = 0 - - while currentBlock <= endOfFullBlocks: - endCurrentBlock = currentBlock + 64 - - var i = 0 - while currentBlock < endCurrentBlock: - w[i] = uint32(src[currentBlock+3]) or - uint32(src[currentBlock+2]) shl 8'u32 or - uint32(src[currentBlock+1]) shl 16'u32 or - uint32(src[currentBlock]) shl 24'u32 - currentBlock += 4 - inc(i) - - innerHash(state, w) - - #Handle last and not full 64 byte block if existing - endCurrentBlock = byteLen - currentBlock - clearBuffer(w) - var lastBlockBytes = 0 - - while lastBlockBytes < endCurrentBlock: - - var value = uint32(src[lastBlockBytes + currentBlock]) shl - ((3'u32 - uint32(lastBlockBytes and 3)) shl 3) - - w[lastBlockBytes shr 2] = w[lastBlockBytes shr 2] or value - inc(lastBlockBytes) - - w[lastBlockBytes shr 2] = w[lastBlockBytes shr 2] or ( - 0x80'u32 shl ((3'u32 - uint32(lastBlockBytes and 3)) shl 3) - ) - - if endCurrentBlock >= 56: - innerHash(state, w) - clearBuffer(w) - - w[15] = uint32(byteLen) shl 3 - innerHash(state, w) - - # Store hash in result pointer, and make sure we get in in the correct order - # on both endian models. - for i in 0 .. Sha1DigestSize-1: - result[i] = uint8((int(state[i shr 2]) shr ((3-(i and 3)) * 8)) and 255) - -proc sha1(src: string): Sha1Digest = - ## Calculate SHA1 from input string - sha1(src, src.len) - -proc secureHash*(str: string): SecureHash = SecureHash(sha1(str)) -proc secureHashFile*(filename: string): SecureHash = secureHash(readFile(filename)) -proc `$`*(self: SecureHash): string = - result = "" - for v in Sha1Digest(self): - result.add(toHex(int(v), 2)) - -proc parseSecureHash*(hash: string): SecureHash = - for i in 0 ..< Sha1DigestSize: - Sha1Digest(result)[i] = uint8(parseHexInt(hash[i*2] & hash[i*2 + 1])) - -proc `==`*(a, b: SecureHash): bool = - # Not a constant-time comparison, but that's acceptable in this context - Sha1Digest(a) == Sha1Digest(b) - - -when isMainModule: - let hash1 = secureHash("a93tgj0p34jagp9[agjp98ajrhp9aej]") - doAssert hash1 == hash1 - doAssert parseSecureHash($hash1) == hash1 +include "../std/sha1" diff --git a/lib/pure/selectors.nim b/lib/pure/selectors.nim index 518cc4bd5..640df8282 100644 --- a/lib/pure/selectors.nim +++ b/lib/pure/selectors.nim @@ -54,9 +54,9 @@ when defined(nimdoc): Timer, ## Timer descriptor is completed Signal, ## Signal is raised Process, ## Process is finished - Vnode, ## BSD specific file change happens + Vnode, ## BSD specific file change User, ## User event is raised - Error, ## Error happens while waiting, for descriptor + Error, ## Error occurred while waiting for descriptor VnodeWrite, ## NOTE_WRITE (BSD specific, write to file occurred) VnodeDelete, ## NOTE_DELETE (BSD specific, unlink of file occurred) VnodeExtend, ## NOTE_EXTEND (BSD specific, file extended) @@ -69,6 +69,8 @@ when defined(nimdoc): ## An object which holds result for descriptor fd* : int ## file/socket descriptor events*: set[Event] ## set of events + errorCode*: OSErrorCode ## additional error code information for + ## Error events SelectEvent* = object ## An object which holds user defined event @@ -79,13 +81,14 @@ when defined(nimdoc): proc close*[T](s: Selector[T]) = ## Closes the selector. - proc registerHandle*[T](s: Selector[T], fd: SocketHandle, events: set[Event], - data: T) = + proc registerHandle*[T](s: Selector[T], fd: int | SocketHandle, + events: set[Event], data: T) = ## Registers file/socket descriptor ``fd`` to selector ``s`` ## with events set in ``events``. The ``data`` is application-defined ## data, which will be passed when an event is triggered. - proc updateHandle*[T](s: Selector[T], fd: SocketHandle, events: set[Event]) = + proc updateHandle*[T](s: Selector[T], fd: int | SocketHandle, + events: set[Event]) = ## Update file/socket descriptor ``fd``, registered in selector ## ``s`` with new events set ``event``. @@ -221,11 +224,15 @@ when defined(nimdoc): proc contains*[T](s: Selector[T], fd: SocketHandle|int): bool {.inline.} = ## Determines whether selector contains a file descriptor. + proc getFd*[T](s: Selector[T]): int = + ## Retrieves the underlying selector's file descriptor. + ## + ## For *poll* and *select* selectors ``-1`` is returned. + else: when hasThreadSupport: import locks - type SharedArray[T] = UncheckedArray[T] @@ -234,7 +241,6 @@ else: proc deallocSharedArray[T](sa: ptr SharedArray[T]) = deallocShared(cast[pointer](sa)) - type Event* {.pure.} = enum Read, Write, Timer, Signal, Process, Vnode, User, Error, Oneshot, @@ -247,6 +253,7 @@ else: ReadyKey* = object fd* : int events*: set[Event] + errorCode*: OSErrorCode SelectorKey[T] = object ident: int @@ -308,6 +315,21 @@ else: else: include ioselects/ioselectors_poll -{.deprecated: [setEvent: trigger].} -{.deprecated: [register: registerHandle].} -{.deprecated: [update: updateHandle].} +proc register*[T](s: Selector[T], fd: int | SocketHandle, + events: set[Event], data: T) {.deprecated: "use registerHandle instead".} = + ## **Deprecated since v0.18.0:** Use ``registerHandle`` instead. + s.registerHandle(fd, events, data) + +proc setEvent*(ev: SelectEvent) {.deprecated: "use trigger instead".} = + ## Trigger event ``ev``. + ## + ## **Deprecated since v0.18.0:** Use ``trigger`` instead. + ev.trigger() + +proc update*[T](s: Selector[T], fd: int | SocketHandle, + events: set[Event]) {.deprecated: "use updateHandle instead".} = + ## Update file/socket descriptor ``fd``, registered in selector + ## ``s`` with new events set ``event``. + ## + ## **Deprecated since v0.18.0:** Use ``updateHandle`` instead. + s.updateHandle() diff --git a/lib/pure/smtp.nim b/lib/pure/smtp.nim index 08e6c8112..c2c674b84 100644 --- a/lib/pure/smtp.nim +++ b/lib/pure/smtp.nim @@ -51,8 +51,6 @@ type Smtp* = SmtpBase[Socket] AsyncSmtp* = SmtpBase[AsyncSocket] -{.deprecated: [EInvalidReply: ReplyError, TMessage: Message, TSMTP: Smtp].} - proc debugSend(smtp: Smtp | AsyncSmtp, cmd: string) {.multisync.} = if smtp.debug: echo("C:" & cmd) diff --git a/lib/pure/stats.nim b/lib/pure/stats.nim index 2004337df..ce32108c2 100644 --- a/lib/pure/stats.nim +++ b/lib/pure/stats.nim @@ -1,11 +1,12 @@ # # # Nim's Runtime Library -# (c) Copyright 2015 Andreas Rumpf +# (c) Copyright 2015 Nim contributors # # See the file "copying.txt", included in this # distribution, for details about the copyright. # + ## Statistical analysis framework for performing ## basic statistical analysis of data. ## The data is analysed in a single pass, when a data value @@ -64,8 +65,6 @@ type y_stats*: RunningStat ## stats for second set of data s_xy: float ## accumulated data for combined xy -{.deprecated: [TFloatClass: FloatClass, TRunningStat: RunningStat].} - # ----------- RunningStat -------------------------- proc clear*(s: var RunningStat) = ## reset `s` @@ -181,6 +180,24 @@ proc `+`*(a, b: RunningStat): RunningStat = proc `+=`*(a: var RunningStat, b: RunningStat) {.inline.} = ## add a second RunningStats `b` to `a` a = a + b + +proc `$`*(a: RunningStat): string = + ## produces a string representation of the ``RunningStat``. The exact + ## format is currently unspecified and subject to change. Currently + ## it contains: + ## + ## - the number of probes + ## - min, max values + ## - sum, mean and standard deviation. + result = "RunningStat(\n" + result.add " number of probes: " & $a.n & "\n" + result.add " max: " & $a.max & "\n" + result.add " min: " & $a.min & "\n" + result.add " sum: " & $a.sum & "\n" + result.add " mean: " & $a.mean & "\n" + result.add " std deviation: " & $a.standardDeviation & "\n" + result.add ")" + # ---------------------- standalone array/seq stats --------------------- proc mean*[T](x: openArray[T]): float = ## computes the mean of `x` @@ -281,7 +298,7 @@ proc correlation*(r: RunningRegress): float = let t = r.x_stats.standardDeviation() * r.y_stats.standardDeviation() result = r.s_xy / ( toFloat(r.n) * t ) -proc `+`*(a, b: RunningRegress): RunningRegress = +proc `+`*(a, b: RunningRegress): RunningRegress = ## combine two `RunningRegress` objects. ## ## Useful if performing parallel analysis of data series diff --git a/lib/pure/streams.nim b/lib/pure/streams.nim index 354e07da3..a0bba05a4 100644 --- a/lib/pure/streams.nim +++ b/lib/pure/streams.nim @@ -57,8 +57,6 @@ type tags: [WriteIOEffect], gcsafe.} flushImpl*: proc (s: Stream) {.nimcall, tags: [WriteIOEffect], gcsafe.} -{.deprecated: [PStream: Stream, TStream: StreamObj].} - proc flush*(s: Stream) = ## flushes the buffers that the stream `s` might use. if not isNil(s.flushImpl): s.flushImpl(s) @@ -76,48 +74,32 @@ proc atEnd*(s: Stream): bool = ## been read. result = s.atEndImpl(s) -proc atEnd*(s, unused: Stream): bool {.deprecated.} = - ## checks if more data can be read from `f`. Returns true if all data has - ## been read. - result = s.atEndImpl(s) - proc setPosition*(s: Stream, pos: int) = ## sets the position `pos` of the stream `s`. s.setPositionImpl(s, pos) -proc setPosition*(s, unused: Stream, pos: int) {.deprecated.} = - ## sets the position `pos` of the stream `s`. - s.setPositionImpl(s, pos) - proc getPosition*(s: Stream): int = ## retrieves the current position in the stream `s`. result = s.getPositionImpl(s) -proc getPosition*(s, unused: Stream): int {.deprecated.} = - ## retrieves the current position in the stream `s`. - result = s.getPositionImpl(s) - proc readData*(s: Stream, buffer: pointer, bufLen: int): int = ## low level proc that reads data into an untyped `buffer` of `bufLen` size. result = s.readDataImpl(s, buffer, bufLen) -proc readAll*(s: Stream): string = - ## Reads all available data. - const bufferSize = 1000 - result = newString(bufferSize) - var r = 0 - while true: - let readBytes = readData(s, addr(result[r]), bufferSize) - if readBytes < bufferSize: - setLen(result, r+readBytes) - break - inc r, bufferSize - setLen(result, r+bufferSize) - -proc readData*(s, unused: Stream, buffer: pointer, - bufLen: int): int {.deprecated.} = - ## low level proc that reads data into an untyped `buffer` of `bufLen` size. - result = s.readDataImpl(s, buffer, bufLen) +when not defined(js): + proc readAll*(s: Stream): string = + ## Reads all available data. + const bufferSize = 1024 + var buffer {.noinit.}: array[bufferSize, char] + while true: + let readBytes = readData(s, addr(buffer[0]), bufferSize) + if readBytes == 0: + break + let prevLen = result.len + result.setLen(prevLen + readBytes) + copyMem(addr(result[prevLen]), addr(buffer[0]), readBytes) + if readBytes < bufferSize: + break proc peekData*(s: Stream, buffer: pointer, bufLen: int): int = ## low level proc that reads data into an untyped `buffer` of `bufLen` size @@ -151,12 +133,7 @@ proc write*(s: Stream, x: string) = when nimvm: writeData(s, cstring(x), x.len) else: - if x.len > 0: writeData(s, unsafeAddr x[0], x.len) - -proc writeLn*(s: Stream, args: varargs[string, `$`]) {.deprecated.} = - ## **Deprecated since version 0.11.4:** Use **writeLine** instead. - for str in args: write(s, str) - write(s, "\n") + if x.len > 0: writeData(s, cstring(x), x.len) proc writeLine*(s: Stream, args: varargs[string, `$`]) = ## writes one or more strings to the the stream `s` followed @@ -276,14 +253,14 @@ proc readStr*(s: Stream, length: int): TaintedString = ## reads a string of length `length` from the stream `s`. Raises `EIO` if ## an error occurred. result = newString(length).TaintedString - var L = readData(s, addr(string(result)[0]), length) + var L = readData(s, cstring(result), length) if L != length: setLen(result.string, L) proc peekStr*(s: Stream, length: int): TaintedString = ## peeks a string of length `length` from the stream `s`. Raises `EIO` if ## an error occurred. result = newString(length).TaintedString - var L = peekData(s, addr(string(result)[0]), length) + var L = peekData(s, cstring(result), length) if L != length: setLen(result.string, L) proc readLine*(s: Stream, line: var TaintedString): bool = @@ -321,6 +298,8 @@ proc readLine*(s: Stream): TaintedString = ## Reads a line from a stream `s`. Note: This is not very efficient. Raises ## `EIO` if an error occurred. result = TaintedString"" + if s.atEnd: + raise newEIO("cannot read from stream") while true: var c = readChar(s) if c == '\c': @@ -346,8 +325,6 @@ when not defined(js): data*: string pos: int - {.deprecated: [PStringStream: StringStream, TStringStream: StringStreamObj].} - proc ssAtEnd(s: Stream): bool = var s = StringStream(s) return s.pos >= s.data.len @@ -407,7 +384,6 @@ when not defined(js): FileStream* = ref FileStreamObj ## a stream that encapsulates a `File` FileStreamObj* = object of Stream f: File - {.deprecated: [PFileStream: FileStream, TFileStream: FileStreamObj].} proc fsClose(s: Stream) = if FileStream(s).f != nil: @@ -447,9 +423,19 @@ when not defined(js): ## creates a new stream from the file named `filename` with the mode `mode`. ## If the file cannot be opened, nil is returned. See the `system ## <system.html>`_ module for a list of available FileMode enums. + ## **This function returns nil in case of failure. To prevent unexpected + ## behavior and ensure proper error handling, use openFileStream instead.** var f: File if open(f, filename, mode, bufSize): result = newFileStream(f) + proc openFileStream*(filename: string, mode: FileMode = fmRead, bufSize: int = -1): FileStream = + ## creates a new stream from the file named `filename` with the mode `mode`. + ## If the file cannot be opened, an IO exception is raised. + var f: File + if open(f, filename, mode, bufSize): + return newFileStream(f) + else: + raise newEIO("cannot open file") when true: discard @@ -460,9 +446,6 @@ else: handle*: FileHandle pos: int - {.deprecated: [PFileHandleStream: FileHandleStream, - TFileHandleStream: FileHandleStreamObj].} - proc newEOS(msg: string): ref OSError = new(result) result.msg = msg diff --git a/lib/pure/strformat.nim b/lib/pure/strformat.nim index 180cbcbec..36404cdf7 100644 --- a/lib/pure/strformat.nim +++ b/lib/pure/strformat.nim @@ -11,12 +11,58 @@ String `interpolation`:idx: / `format`:idx: inspired by Python's ``f``-strings. -Examples: +``fmt`` vs. ``&`` +================= + +You can use either ``fmt`` or the unary ``&`` operator for formatting. The +difference between them is subtle but important. + +The ``fmt"{expr}"`` syntax is more aesthetically pleasing, but it hides a small +gotcha. The string is a +`generalized raw string literal <manual.html#lexical-analysis-generalized-raw-string-literals>`_. +This has some surprising effects: + +.. code-block:: nim + + import strformat + let msg = "hello" + doAssert fmt"{msg}\n" == "hello\\n" + +Because the literal is a raw string literal, the ``\n`` is not interpreted as +an escape sequence. + +There are multiple ways to get around this, including the use of the ``&`` +operator: + +.. code-block:: nim + + import strformat + let msg = "hello" + + doAssert &"{msg}\n" == "hello\n" + + doAssert fmt"{msg}{'\n'}" == "hello\n" + doAssert fmt("{msg}\n") == "hello\n" + doAssert "{msg}\n".fmt == "hello\n" + +The choice of style is up to you. + +Formatting strings +================== .. code-block:: nim - doAssert fmt"""{"abc":>4}""" == " abc" - doAssert fmt"""{"abc":<4}""" == "abc " + import strformat + + doAssert &"""{"abc":>4}""" == " abc" + doAssert &"""{"abc":<4}""" == "abc " + +Formatting floats +================= + +.. code-block:: nim + + import strformat doAssert fmt"{-12345:08}" == "-0012345" doAssert fmt"{-1:3}" == " -1" @@ -35,7 +81,10 @@ Examples: doAssert fmt"{123.456:13e}" == " 1.234560e+02" -An expression like ``fmt"{key} is {value:arg} {{z}}"`` is transformed into: +Implementation details +====================== + +An expression like ``&"{key} is {value:arg} {{z}}"`` is transformed into: .. code-block:: nim var temp = newStringOfCap(educatedCapGuess) @@ -48,13 +97,13 @@ An expression like ``fmt"{key} is {value:arg} {{z}}"`` is transformed into: Parts of the string that are enclosed in the curly braces are interpreted as Nim code, to escape an ``{`` or ``}`` double it. -``fmt`` delegates most of the work to an open overloaded set +``&`` delegates most of the work to an open overloaded set of ``format`` procs. The required signature for a type ``T`` that supports formatting is usually ``proc format(x: T; result: var string)`` for efficiency but can also be ``proc format(x: T): string``. ``add`` and ``$`` procs are used as the fallback implementation. -This is the concrete lookup algorithm that ``fmt`` uses: +This is the concrete lookup algorithm that ``&`` uses: .. code-block:: nim @@ -69,7 +118,7 @@ This is the concrete lookup algorithm that ``fmt`` uses: The subexpression after the colon -(``arg`` in ``fmt"{key} is {value:arg} {{z}}"``) is an optional argument +(``arg`` in ``&"{key} is {value:arg} {{z}}"``) is an optional argument passed to ``format``. If an optional argument is present the following lookup algorithm is used: @@ -86,8 +135,8 @@ For strings and numeric types the optional argument is a so-called "standard format specifier". -Standard format specifier -========================= +Standard format specifier for strings, integers and floats +========================================================== The general form of a standard format specifier is:: @@ -221,138 +270,15 @@ template callFormat(res, arg) {.dirty.} = template callFormatOption(res, arg, option) {.dirty.} = when compiles(format(arg, option, res)): format(arg, option, res) - else: + elif compiles(format(arg, option)): res.add format(arg, option) + else: + format($arg, option, res) -macro fmt*(pattern: string): untyped = - ## For a specification of the ``fmt`` macro, see the module level documentation. - runnableExamples: - template check(actual, expected: string) = - doAssert actual == expected - - from strutils import toUpperAscii, repeat - - # Basic tests - let s = "string" - check fmt"{0} {s}", "0 string" - check fmt"{s[0..2].toUpperAscii}", "STR" - check fmt"{-10:04}", "-010" - check fmt"{-10:<04}", "-010" - check fmt"{-10:>04}", "-010" - check fmt"0x{10:02X}", "0x0A" - - check fmt"{10:#04X}", "0x0A" - - check fmt"""{"test":#>5}""", "#test" - check fmt"""{"test":>5}""", " test" - - check fmt"""{"test":#^7}""", "#test##" - - check fmt"""{"test": <5}""", "test " - check fmt"""{"test":<5}""", "test " - check fmt"{1f:.3f}", "1.000" - check fmt"Hello, {s}!", "Hello, string!" - - # Tests for identifers without parenthesis - check fmt"{s} works{s}", "string worksstring" - check fmt"{s:>7}", " string" - doAssert(not compiles(fmt"{s_works}")) # parsed as identifier `s_works` - - # Misc general tests - check fmt"{{}}", "{}" - check fmt"{0}%", "0%" - check fmt"{0}%asdf", "0%asdf" - check fmt("\n{\"\\n\"}\n"), "\n\n\n" - check fmt"""{"abc"}s""", "abcs" - - # String tests - check fmt"""{"abc"}""", "abc" - check fmt"""{"abc":>4}""", " abc" - check fmt"""{"abc":<4}""", "abc " - check fmt"""{"":>4}""", " " - check fmt"""{"":<4}""", " " - - # Int tests - check fmt"{12345}", "12345" - check fmt"{ - 12345}", "-12345" - check fmt"{12345:6}", " 12345" - check fmt"{12345:>6}", " 12345" - check fmt"{12345:4}", "12345" - check fmt"{12345:08}", "00012345" - check fmt"{-12345:08}", "-0012345" - check fmt"{0:0}", "0" - check fmt"{0:02}", "00" - check fmt"{-1:3}", " -1" - check fmt"{-1:03}", "-01" - check fmt"{10}", "10" - check fmt"{16:#X}", "0x10" - check fmt"{16:^#7X}", " 0x10 " - check fmt"{16:^+#7X}", " +0x10 " - - # Hex tests - check fmt"{0:x}", "0" - check fmt"{-0:x}", "0" - check fmt"{255:x}", "ff" - check fmt"{255:X}", "FF" - check fmt"{-255:x}", "-ff" - check fmt"{-255:X}", "-FF" - check fmt"{255:x} uNaffeCteD CaSe", "ff uNaffeCteD CaSe" - check fmt"{255:X} uNaffeCteD CaSe", "FF uNaffeCteD CaSe" - check fmt"{255:4x}", " ff" - check fmt"{255:04x}", "00ff" - check fmt"{-255:4x}", " -ff" - check fmt"{-255:04x}", "-0ff" - - # Float tests - check fmt"{123.456}", "123.456" - check fmt"{-123.456}", "-123.456" - check fmt"{123.456:.3f}", "123.456" - check fmt"{123.456:+.3f}", "+123.456" - check fmt"{-123.456:+.3f}", "-123.456" - check fmt"{-123.456:.3f}", "-123.456" - check fmt"{123.456:1g}", "123.456" - check fmt"{123.456:.1f}", "123.5" - check fmt"{123.456:.0f}", "123." - #check fmt"{123.456:.0f}", "123." - check fmt"{123.456:>9.3f}", " 123.456" - check fmt"{123.456:9.3f}", " 123.456" - check fmt"{123.456:>9.4f}", " 123.4560" - check fmt"{123.456:>9.0f}", " 123." - check fmt"{123.456:<9.4f}", "123.4560 " - - # Float (scientific) tests - check fmt"{123.456:e}", "1.234560e+02" - check fmt"{123.456:>13e}", " 1.234560e+02" - check fmt"{123.456:<13e}", "1.234560e+02 " - check fmt"{123.456:.1e}", "1.2e+02" - check fmt"{123.456:.2e}", "1.23e+02" - check fmt"{123.456:.3e}", "1.235e+02" - - # Note: times.format adheres to the format protocol. Test that this - # works: - import times - - var nullTime: TimeInfo - check fmt"{nullTime:yyyy-mm-dd}", "0000-00-00" - - # Unicode string tests - check fmt"""{"αβγ"}""", "αβγ" - check fmt"""{"αβγ":>5}""", " αβγ" - check fmt"""{"αβγ":<5}""", "αβγ " - check fmt"""a{"a"}α{"α"}€{"€"}𐍈{"𐍈"}""", "aaαα€€𐍈𐍈" - check fmt"""a{"a":2}α{"α":2}€{"€":2}𐍈{"𐍈":2}""", "aa αα €€ 𐍈𐍈 " - # Invalid unicode sequences should be handled as plain strings. - # Invalid examples taken from: https://stackoverflow.com/a/3886015/1804173 - let invalidUtf8 = [ - "\xc3\x28", "\xa0\xa1", - "\xe2\x28\xa1", "\xe2\x82\x28", - "\xf0\x28\x8c\xbc", "\xf0\x90\x28\xbc", "\xf0\x28\x8c\x28" - ] - for s in invalidUtf8: - check fmt"{s:>5}", repeat(" ", 5-s.len) & s - +macro `&`*(pattern: string): untyped = + ## For a specification of the ``&`` macro, see the module level documentation. if pattern.kind notin {nnkStrLit..nnkTripleStrLit}: - error "fmt only works with string literals", pattern + error "string formatting (fmt(), &) only works with string literals", pattern let f = pattern.strVal var i = 0 let res = genSym(nskVar, "fmtRes") @@ -405,6 +331,11 @@ macro fmt*(pattern: string): untyped = when defined(debugFmtDsl): echo repr result +template fmt*(pattern: string): untyped = + ## An alias for ``&``. + bind `&` + &pattern + proc mkDigit(v: int, typ: char): string {.inline.} = assert(v < 26) if v < 10: @@ -444,7 +375,7 @@ type ## ``parseStandardFormatSpecifier`` returned. proc formatInt(n: SomeNumber; radix: int; spec: StandardFormatSpecifier): string = - ## Converts ``n`` to string. If ``n`` is `SomeReal`, it casts to `int64`. + ## Converts ``n`` to string. If ``n`` is `SomeFloat`, it casts to `int64`. ## Conversion is done using ``radix``. If result's length is lesser than ## ``minimumWidth``, it aligns result to the right or left (depending on ``a``) ## with ``fill`` char. @@ -558,7 +489,7 @@ proc parseStandardFormatSpecifier*(s: string; start = 0; proc format*(value: SomeInteger; specifier: string; res: var string) = ## Standard format implementation for ``SomeInteger``. It makes little ## sense to call this directly, but it is required to exist - ## by the ``fmt`` macro. + ## by the ``&`` macro. let spec = parseStandardFormatSpecifier(specifier) var radix = 10 case spec.typ @@ -572,10 +503,10 @@ proc format*(value: SomeInteger; specifier: string; res: var string) = " of 'x', 'X', 'b', 'd', 'o' but got: " & spec.typ) res.add formatInt(value, radix, spec) -proc format*(value: SomeReal; specifier: string; res: var string) = - ## Standard format implementation for ``SomeReal``. It makes little +proc format*(value: SomeFloat; specifier: string; res: var string) = + ## Standard format implementation for ``SomeFloat``. It makes little ## sense to call this directly, but it is required to exist - ## by the ``fmt`` macro. + ## by the ``&`` macro. let spec = parseStandardFormatSpecifier(specifier) var fmode = ffDefault @@ -593,8 +524,31 @@ proc format*(value: SomeReal; specifier: string; res: var string) = " of 'e', 'E', 'f', 'F', 'g', 'G' but got: " & spec.typ) var f = formatBiggestFloat(value, fmode, spec.precision) - if value >= 0.0 and spec.sign != '-': - f = spec.sign & f + var sign = false + if value >= 0.0: + if spec.sign != '-': + sign = true + if value == 0.0: + if 1.0 / value == Inf: + # only insert the sign if value != negZero + f.insert($spec.sign, 0) + else: + f.insert($spec.sign, 0) + else: + sign = true + + if spec.padWithZero: + var sign_str = "" + if sign: + sign_str = $f[0] + f = f[1..^1] + + let toFill = spec.minimumWidth - f.len - ord(sign) + if toFill > 0: + f = repeat('0', toFill) & f + if sign: + f = sign_str & f + # the default for numbers is right-alignment: let align = if spec.align == '\0': '>' else: spec.align let result = alignString(f, spec.minimumWidth, @@ -607,13 +561,148 @@ proc format*(value: SomeReal; specifier: string; res: var string) = proc format*(value: string; specifier: string; res: var string) = ## Standard format implementation for ``string``. It makes little ## sense to call this directly, but it is required to exist - ## by the ``fmt`` macro. + ## by the ``&`` macro. let spec = parseStandardFormatSpecifier(specifier) - var fmode = ffDefault + var value = value case spec.typ of 's', '\0': discard else: raise newException(ValueError, "invalid type in format string for string, expected 's', but got " & spec.typ) + if spec.precision != -1: + if spec.precision < runelen(value): + setLen(value, runeOffset(value, spec.precision)) res.add alignString(value, spec.minimumWidth, spec.align, spec.fill) + +when isMainModule: + template check(actual, expected: string) = + doAssert actual == expected + + from strutils import toUpperAscii, repeat + + # Basic tests + let s = "string" + check &"{0} {s}", "0 string" + check &"{s[0..2].toUpperAscii}", "STR" + check &"{-10:04}", "-010" + check &"{-10:<04}", "-010" + check &"{-10:>04}", "-010" + check &"0x{10:02X}", "0x0A" + + check &"{10:#04X}", "0x0A" + + check &"""{"test":#>5}""", "#test" + check &"""{"test":>5}""", " test" + + check &"""{"test":#^7}""", "#test##" + + check &"""{"test": <5}""", "test " + check &"""{"test":<5}""", "test " + check &"{1f:.3f}", "1.000" + check &"Hello, {s}!", "Hello, string!" + + # Tests for identifers without parenthesis + check &"{s} works{s}", "string worksstring" + check &"{s:>7}", " string" + doAssert(not compiles(&"{s_works}")) # parsed as identifier `s_works` + + # Misc general tests + check &"{{}}", "{}" + check &"{0}%", "0%" + check &"{0}%asdf", "0%asdf" + check &("\n{\"\\n\"}\n"), "\n\n\n" + check &"""{"abc"}s""", "abcs" + + # String tests + check &"""{"abc"}""", "abc" + check &"""{"abc":>4}""", " abc" + check &"""{"abc":<4}""", "abc " + check &"""{"":>4}""", " " + check &"""{"":<4}""", " " + + # Int tests + check &"{12345}", "12345" + check &"{ - 12345}", "-12345" + check &"{12345:6}", " 12345" + check &"{12345:>6}", " 12345" + check &"{12345:4}", "12345" + check &"{12345:08}", "00012345" + check &"{-12345:08}", "-0012345" + check &"{0:0}", "0" + check &"{0:02}", "00" + check &"{-1:3}", " -1" + check &"{-1:03}", "-01" + check &"{10}", "10" + check &"{16:#X}", "0x10" + check &"{16:^#7X}", " 0x10 " + check &"{16:^+#7X}", " +0x10 " + + # Hex tests + check &"{0:x}", "0" + check &"{-0:x}", "0" + check &"{255:x}", "ff" + check &"{255:X}", "FF" + check &"{-255:x}", "-ff" + check &"{-255:X}", "-FF" + check &"{255:x} uNaffeCteD CaSe", "ff uNaffeCteD CaSe" + check &"{255:X} uNaffeCteD CaSe", "FF uNaffeCteD CaSe" + check &"{255:4x}", " ff" + check &"{255:04x}", "00ff" + check &"{-255:4x}", " -ff" + check &"{-255:04x}", "-0ff" + + # Float tests + check &"{123.456}", "123.456" + check &"{-123.456}", "-123.456" + check &"{123.456:.3f}", "123.456" + check &"{123.456:+.3f}", "+123.456" + check &"{-123.456:+.3f}", "-123.456" + check &"{-123.456:.3f}", "-123.456" + check &"{123.456:1g}", "123.456" + check &"{123.456:.1f}", "123.5" + check &"{123.456:.0f}", "123." + #check &"{123.456:.0f}", "123." + check &"{123.456:>9.3f}", " 123.456" + check &"{123.456:9.3f}", " 123.456" + check &"{123.456:>9.4f}", " 123.4560" + check &"{123.456:>9.0f}", " 123." + check &"{123.456:<9.4f}", "123.4560 " + + # Float (scientific) tests + check &"{123.456:e}", "1.234560e+02" + check &"{123.456:>13e}", " 1.234560e+02" + check &"{123.456:<13e}", "1.234560e+02 " + check &"{123.456:.1e}", "1.2e+02" + check &"{123.456:.2e}", "1.23e+02" + check &"{123.456:.3e}", "1.235e+02" + + # Note: times.format adheres to the format protocol. Test that this + # works: + import times + + var nullTime: DateTime + check &"{nullTime:yyyy-mm-dd}", "0000-00-00" + + # Unicode string tests + check &"""{"αβγ"}""", "αβγ" + check &"""{"αβγ":>5}""", " αβγ" + check &"""{"αβγ":<5}""", "αβγ " + check &"""a{"a"}α{"α"}€{"€"}𐍈{"𐍈"}""", "aaαα€€𐍈𐍈" + check &"""a{"a":2}α{"α":2}€{"€":2}𐍈{"𐍈":2}""", "aa αα €€ 𐍈𐍈 " + # Invalid unicode sequences should be handled as plain strings. + # Invalid examples taken from: https://stackoverflow.com/a/3886015/1804173 + let invalidUtf8 = [ + "\xc3\x28", "\xa0\xa1", + "\xe2\x28\xa1", "\xe2\x82\x28", + "\xf0\x28\x8c\xbc", "\xf0\x90\x28\xbc", "\xf0\x28\x8c\x28" + ] + for s in invalidUtf8: + check &"{s:>5}", repeat(" ", 5-s.len) & s + + + import json + + doAssert fmt"{'a'} {'b'}" == "a b" + + echo("All tests ok") diff --git a/lib/pure/strmisc.nim b/lib/pure/strmisc.nim index 89ef2fcd2..d1ff920c9 100644 --- a/lib/pure/strmisc.nim +++ b/lib/pure/strmisc.nim @@ -12,7 +12,7 @@ import strutils -{.deadCodeElim: on.} +{.deadCodeElim: on.} # dce option deprecated proc expandTabs*(s: string, tabSize: int = 8): string {.noSideEffect, procvar.} = diff --git a/lib/pure/strscans.nim b/lib/pure/strscans.nim index 2bd87837f..11f182495 100644 --- a/lib/pure/strscans.nim +++ b/lib/pure/strscans.nim @@ -87,7 +87,7 @@ which we then use in our scanf pattern to help us in the matching process: proc someSep(input: string; start: int; seps: set[char] = {':','-','.'}): int = # Note: The parameters and return value must match to what ``scanf`` requires result = 0 - while input[start+result] in seps: inc result + while start+result < input.len and input[start+result] in seps: inc result if scanf(input, "$w$[someSep]$w", key, value): ... @@ -231,7 +231,7 @@ is performed. var i = start var u = 0 while true: - if s[i] == '\0' or s[i] == unless: + if i >= s.len or s[i] == unless: return 0 elif s[i] == until[0]: u = 1 @@ -315,6 +315,11 @@ macro scanf*(input: string; pattern: static[string]; results: varargs[typed]): b conds.add resLen.notZero conds.add resLen + template at(s: string; i: int): char = (if i < s.len: s[i] else: '\0') + template matchError() = + error("type mismatch between pattern '$" & pattern[p] & "' (position: " & $p & ") and " & $getType(results[i]) & + " var '" & repr(results[i]) & "'") + var i = 0 var p = 0 var idx = genSym(nskVar, "idx") @@ -336,37 +341,37 @@ macro scanf*(input: string; pattern: static[string]; results: varargs[typed]): b if i < results.len and getType(results[i]).typeKind == ntyString: matchBind "parseIdent" else: - error("no string var given for $w") + matchError inc i of 'b': if i < results.len and getType(results[i]).typeKind == ntyInt: matchBind "parseBin" else: - error("no int var given for $b") + matchError inc i of 'o': if i < results.len and getType(results[i]).typeKind == ntyInt: matchBind "parseOct" else: - error("no int var given for $o") + matchError inc i of 'i': if i < results.len and getType(results[i]).typeKind == ntyInt: matchBind "parseInt" else: - error("no int var given for $i") + matchError inc i of 'h': if i < results.len and getType(results[i]).typeKind == ntyInt: matchBind "parseHex" else: - error("no int var given for $h") + matchError inc i of 'f': if i < results.len and getType(results[i]).typeKind == ntyFloat: matchBind "parseFloat" else: - error("no float var given for $f") + matchError inc i of 's': conds.add newCall(bindSym"inc", idx, newCall(bindSym"skipWhitespace", inp, idx)) @@ -390,14 +395,14 @@ macro scanf*(input: string; pattern: static[string]; results: varargs[typed]): b conds.add newCall(bindSym"!=", resLen, newLit min) conds.add resLen else: - error("no string var given for $" & pattern[p]) + matchError inc i of '{': inc p var nesting = 0 let start = p while true: - case pattern[p] + case pattern.at(p) of '{': inc nesting of '}': if nesting == 0: break @@ -412,14 +417,14 @@ macro scanf*(input: string; pattern: static[string]; results: varargs[typed]): b conds.add newCall(bindSym"!=", resLen, newLit 0) conds.add resLen else: - error("no var given for $" & expr) + error("no var given for $" & expr & " (position: " & $p & ")") inc i of '[': inc p var nesting = 0 let start = p while true: - case pattern[p] + case pattern.at(p) of '[': inc nesting of ']': if nesting == 0: break @@ -451,10 +456,12 @@ macro scanf*(input: string; pattern: static[string]; results: varargs[typed]): b template atom*(input: string; idx: int; c: char): bool = ## Used in scanp for the matching of atoms (usually chars). - input[idx] == c + idx < input.len and input[idx] == c template atom*(input: string; idx: int; s: set[char]): bool = - input[idx] in s + idx < input.len and input[idx] in s + +template hasNxt*(input: string; idx: int): bool = idx < input.len #template prepare*(input: string): int = 0 template success*(x: int): bool = x != 0 @@ -462,7 +469,7 @@ template success*(x: int): bool = x != 0 template nxt*(input: string; idx, step: int = 1) = inc(idx, step) macro scanp*(input, idx: typed; pattern: varargs[untyped]): bool = - ## ``scanp`` is currently undocumented. + ## See top level documentation of his module of how ``scanf`` works. type StmtTriple = tuple[init, cond, action: NimNode] template interf(x): untyped = bindSym(x, brForceOpen) @@ -508,8 +515,8 @@ macro scanp*(input, idx: typed; pattern: varargs[untyped]): bool = !!newCall(interf"nxt", input, idx, resLen)) of nnkCallKinds: # *{'A'..'Z'} !! s.add(!_) - template buildWhile(init, cond, action): untyped = - while true: + template buildWhile(input, idx, init, cond, action): untyped = + while hasNxt(input, idx): init if not cond: break action @@ -528,11 +535,11 @@ macro scanp*(input, idx: typed; pattern: varargs[untyped]): bool = !!newCall(interf"nxt", input, idx, it[2])) elif it.kind == nnkPrefix and it[0].eqIdent"*": let (init, cond, action) = atm(it[1], input, idx, attached) - result = (getAst(buildWhile(init, cond, action)), + result = (getAst(buildWhile(input, idx, init, cond, action)), newEmptyNode(), newEmptyNode()) elif it.kind == nnkPrefix and it[0].eqIdent"+": # x+ is the same as xx* - result = atm(newTree(nnkPar, it[1], newTree(nnkPrefix, ident"*", it[1])), + result = atm(newTree(nnkTupleConstr, it[1], newTree(nnkPrefix, ident"*", it[1])), input, idx, attached) elif it.kind == nnkPrefix and it[0].eqIdent"?": # optional. @@ -583,18 +590,18 @@ macro scanp*(input, idx: typed; pattern: varargs[untyped]): bool = result = (newEmptyNode(), newCall(interf"atom", input, idx, it), !!newCall(interf"nxt", input, idx)) of nnkCurlyExpr: if it.len == 3 and it[1].kind == nnkIntLit and it[2].kind == nnkIntLit: - var h = newTree(nnkPar, it[0]) + var h = newTree(nnkTupleConstr, it[0]) for count in 2i64 .. it[1].intVal: h.add(it[0]) for count in it[1].intVal .. it[2].intVal-1: h.add(newTree(nnkPrefix, ident"?", it[0])) result = atm(h, input, idx, attached) elif it.len == 2 and it[1].kind == nnkIntLit: - var h = newTree(nnkPar, it[0]) + var h = newTree(nnkTupleConstr, it[0]) for count in 2i64 .. it[1].intVal: h.add(it[0]) result = atm(h, input, idx, attached) else: error("invalid pattern") - of nnkPar: - if it.len == 1: + of nnkPar, nnkTupleConstr: + if it.len == 1 and it.kind == nnkPar: result = atm(it[0], input, idx, attached) else: # concatenation: @@ -621,7 +628,7 @@ macro scanp*(input, idx: typed; pattern: varargs[untyped]): bool = when isMainModule: proc twoDigits(input: string; x: var int; start: int): int = - if input[start] == '0' and input[start+1] == '0': + if start+1 < input.len and input[start] == '0' and input[start+1] == '0': result = 2 x = 13 else: @@ -629,10 +636,10 @@ when isMainModule: proc someSep(input: string; start: int; seps: set[char] = {';',',','-','.'}): int = result = 0 - while input[start+result] in seps: inc result + while start+result < input.len and input[start+result] in seps: inc result proc demangle(s: string; res: var string; start: int): int = - while s[result+start] in {'_', '@'}: inc result + while result+start < s.len and s[result+start] in {'_', '@'}: inc result res = "" while result+start < s.len and s[result+start] > ' ' and s[result+start] != '_': res.add s[result+start] @@ -652,7 +659,7 @@ when isMainModule: var info = "" if scanp(resp, idx, *`whites`, '#', *`digits`, +`whites`, ?("0x", *`hexdigits`, " in "), demangle($input, prc, $index), *`whites`, '(', * ~ ')', ')', - *`whites`, "at ", +(~{'\C', '\L', '\0'} -> info.add($_)) ): + *`whites`, "at ", +(~{'\C', '\L'} -> info.add($_)) ): result.add prc & " " & info else: break @@ -713,7 +720,7 @@ when isMainModule: "NimMainInner c:/users/anwender/projects/nim/lib/system.nim:2605", "NimMain c:/users/anwender/projects/nim/lib/system.nim:2613", "main c:/users/anwender/projects/nim/lib/system.nim:2620"] - doAssert parseGDB(gdbOut) == result + #doAssert parseGDB(gdbOut) == result # bug #6487 var count = 0 diff --git a/lib/pure/strtabs.nim b/lib/pure/strtabs.nim index 75c5e171d..d8a23286a 100644 --- a/lib/pure/strtabs.nim +++ b/lib/pure/strtabs.nim @@ -17,9 +17,7 @@ import when defined(js): {.pragma: rtlFunc.} - {.pragma: deprecatedGetFunc.} else: - {.pragma: deprecatedGetFunc, deprecatedGet.} {.pragma: rtlFunc, rtl.} import os include "system/inclrtl" @@ -29,7 +27,7 @@ type modeCaseSensitive, ## the table is case sensitive modeCaseInsensitive, ## the table is case insensitive modeStyleInsensitive ## the table is style insensitive - KeyValuePair = tuple[key, val: string] + KeyValuePair = tuple[key, val: string, hasValue: bool] KeyValuePairSeq = seq[KeyValuePair] StringTableObj* = object of RootObj counter: int @@ -38,9 +36,6 @@ type StringTableRef* = ref StringTableObj ## use this type to declare string tables -{.deprecated: [TStringTableMode: StringTableMode, - TStringTable: StringTableObj, PStringTable: StringTableRef].} - proc len*(t: StringTableRef): int {.rtlFunc, extern: "nst$1".} = ## returns the number of keys in `t`. result = t.counter @@ -48,19 +43,19 @@ proc len*(t: StringTableRef): int {.rtlFunc, extern: "nst$1".} = iterator pairs*(t: StringTableRef): tuple[key, value: string] = ## iterates over every (key, value) pair in the table `t`. for h in 0..high(t.data): - if not isNil(t.data[h].key): + if t.data[h].hasValue: yield (t.data[h].key, t.data[h].val) iterator keys*(t: StringTableRef): string = ## iterates over every key in the table `t`. for h in 0..high(t.data): - if not isNil(t.data[h].key): + if t.data[h].hasValue: yield t.data[h].key iterator values*(t: StringTableRef): string = ## iterates over every value in the table `t`. for h in 0..high(t.data): - if not isNil(t.data[h].key): + if t.data[h].hasValue: yield t.data[h].val type @@ -73,10 +68,6 @@ type useKey ## do not replace ``$key`` if it is not found ## in the table (or in the environment) -{.deprecated: [TFormatFlag: FormatFlag].} - -# implementation - const growthFactor = 2 startSize = 64 @@ -102,7 +93,7 @@ proc nextTry(h, maxHash: Hash): Hash {.inline.} = proc rawGet(t: StringTableRef, key: string): int = var h: Hash = myhash(t, key) and high(t.data) # start with real hash value - while not isNil(t.data[h].key): + while t.data[h].hasValue: if myCmp(t, t.data[h].key, key): return h h = nextTry(h, high(t.data)) @@ -118,17 +109,12 @@ template get(t: StringTableRef, key: string) = raise newException(KeyError, "key not found") proc `[]`*(t: StringTableRef, key: string): var string {. - rtlFunc, extern: "nstTake", deprecatedGetFunc.} = + rtlFunc, extern: "nstTake".} = ## retrieves the location at ``t[key]``. If `key` is not in `t`, the ## ``KeyError`` exception is raised. One can check with ``hasKey`` whether ## the key exists. get(t, key) -proc mget*(t: StringTableRef, key: string): var string {.deprecated.} = - ## retrieves the location at ``t[key]``. If `key` is not in `t`, the - ## ``KeyError`` exception is raised. Use ```[]``` instead. - get(t, key) - proc getOrDefault*(t: StringTableRef; key: string, default: string = ""): string = var index = rawGet(t, key) if index >= 0: result = t.data[index].val @@ -144,16 +130,17 @@ proc contains*(t: StringTableRef, key: string): bool = proc rawInsert(t: StringTableRef, data: var KeyValuePairSeq, key, val: string) = var h: Hash = myhash(t, key) and high(data) - while not isNil(data[h].key): + while data[h].hasValue: h = nextTry(h, high(data)) data[h].key = key data[h].val = val + data[h].hasValue = true proc enlarge(t: StringTableRef) = var n: KeyValuePairSeq newSeq(n, len(t.data) * growthFactor) for i in countup(0, high(t.data)): - if not isNil(t.data[i].key): rawInsert(t, n, t.data[i].key, t.data[i].val) + if t.data[i].hasValue: rawInsert(t, n, t.data[i].key, t.data[i].val) swap(t.data, n) proc `[]=`*(t: StringTableRef, key, val: string) {.rtlFunc, extern: "nstPut".} = @@ -192,14 +179,14 @@ proc newStringTable*(mode: StringTableMode): StringTableRef {. result.counter = 0 newSeq(result.data, startSize) -proc clear*(s: StringTableRef, mode: StringTableMode) = +proc clear*(s: StringTableRef, mode: StringTableMode) {. + rtlFunc, extern: "nst$1".} = ## resets a string table to be empty again. s.mode = mode s.counter = 0 s.data.setLen(startSize) for i in 0..<s.data.len: - if not isNil(s.data[i].key): - s.data[i].key = nil + s.data[i].hasValue = false proc newStringTable*(keyValuePairs: varargs[string], mode: StringTableMode): StringTableRef {. diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index dbb4db781..a4fd20fdb 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -17,7 +17,7 @@ import parseutils from math import pow, round, floor, log10 from algorithm import reverse -{.deadCodeElim: on.} +{.deadCodeElim: on.} # dce option deprecated {.push debugger:off .} # the user does not want to trace a part # of the standard library! @@ -106,6 +106,12 @@ proc isUpperAscii*(c: char): bool {.noSideEffect, procvar, ## This checks ASCII characters only. return c in {'A'..'Z'} +template isImpl(call) = + if s.len == 0: return false + result = true + for c in s: + if not call(c): return false + proc isAlphaAscii*(s: string): bool {.noSideEffect, procvar, rtl, extern: "nsuIsAlphaAsciiStr".} = ## Checks whether or not `s` is alphabetical. @@ -114,12 +120,7 @@ proc isAlphaAscii*(s: string): bool {.noSideEffect, procvar, ## Returns true if all characters in `s` are ## alphabetic and there is at least one character ## in `s`. - if s.len() == 0: - return false - - result = true - for c in s: - if not c.isAlphaAscii(): return false + isImpl isAlphaAscii proc isAlphaNumeric*(s: string): bool {.noSideEffect, procvar, rtl, extern: "nsuIsAlphaNumericStr".} = @@ -129,13 +130,7 @@ proc isAlphaNumeric*(s: string): bool {.noSideEffect, procvar, ## Returns true if all characters in `s` are ## alpanumeric and there is at least one character ## in `s`. - if s.len() == 0: - return false - - result = true - for c in s: - if not c.isAlphaNumeric(): - return false + isImpl isAlphaNumeric proc isDigit*(s: string): bool {.noSideEffect, procvar, rtl, extern: "nsuIsDigitStr".} = @@ -145,13 +140,7 @@ proc isDigit*(s: string): bool {.noSideEffect, procvar, ## Returns true if all characters in `s` are ## numeric and there is at least one character ## in `s`. - if s.len() == 0: - return false - - result = true - for c in s: - if not c.isDigit(): - return false + isImpl isDigit proc isSpaceAscii*(s: string): bool {.noSideEffect, procvar, rtl, extern: "nsuIsSpaceAsciiStr".} = @@ -159,13 +148,7 @@ proc isSpaceAscii*(s: string): bool {.noSideEffect, procvar, ## ## Returns true if all characters in `s` are whitespace ## characters and there is at least one character in `s`. - if s.len() == 0: - return false - - result = true - for c in s: - if not c.isSpaceAscii(): - return false + isImpl isSpaceAscii proc isLowerAscii*(s: string): bool {.noSideEffect, procvar, rtl, extern: "nsuIsLowerAsciiStr".} = @@ -174,13 +157,7 @@ proc isLowerAscii*(s: string): bool {.noSideEffect, procvar, ## This checks ASCII characters only. ## Returns true if all characters in `s` are lower case ## and there is at least one character in `s`. - if s.len() == 0: - return false - - for c in s: - if not c.isLowerAscii(): - return false - true + isImpl isLowerAscii proc isUpperAscii*(s: string): bool {.noSideEffect, procvar, rtl, extern: "nsuIsUpperAsciiStr".} = @@ -189,13 +166,7 @@ proc isUpperAscii*(s: string): bool {.noSideEffect, procvar, ## This checks ASCII characters only. ## Returns true if all characters in `s` are upper case ## and there is at least one character in `s`. - if s.len() == 0: - return false - - for c in s: - if not c.isUpperAscii(): - return false - true + isImpl isUpperAscii proc toLowerAscii*(c: char): char {.noSideEffect, procvar, rtl, extern: "nsuToLowerAsciiChar".} = @@ -209,6 +180,11 @@ proc toLowerAscii*(c: char): char {.noSideEffect, procvar, else: result = c +template toImpl(call) = + result = newString(len(s)) + for i in 0..len(s) - 1: + result[i] = call(s[i]) + proc toLowerAscii*(s: string): string {.noSideEffect, procvar, rtl, extern: "nsuToLowerAsciiStr".} = ## Converts `s` into lower case. @@ -216,9 +192,7 @@ proc toLowerAscii*(s: string): string {.noSideEffect, procvar, ## This works only for the letters ``A-Z``. See `unicode.toLower ## <unicode.html#toLower>`_ for a version that works for any Unicode ## character. - result = newString(len(s)) - for i in 0..len(s) - 1: - result[i] = toLowerAscii(s[i]) + toImpl toLowerAscii proc toUpperAscii*(c: char): char {.noSideEffect, procvar, rtl, extern: "nsuToUpperAsciiChar".} = @@ -239,154 +213,22 @@ proc toUpperAscii*(s: string): string {.noSideEffect, procvar, ## This works only for the letters ``A-Z``. See `unicode.toUpper ## <unicode.html#toUpper>`_ for a version that works for any Unicode ## character. - result = newString(len(s)) - for i in 0..len(s) - 1: - result[i] = toUpperAscii(s[i]) + toImpl toUpperAscii proc capitalizeAscii*(s: string): string {.noSideEffect, procvar, rtl, extern: "nsuCapitalizeAscii".} = ## Converts the first character of `s` into upper case. ## ## This works only for the letters ``A-Z``. - result = toUpperAscii(s[0]) & substr(s, 1) - -proc isSpace*(c: char): bool {.noSideEffect, procvar, - rtl, deprecated, extern: "nsuIsSpaceChar".}= - ## Checks whether or not `c` is a whitespace character. - ## - ## **Deprecated since version 0.15.0**: use ``isSpaceAscii`` instead. - isSpaceAscii(c) - -proc isLower*(c: char): bool {.noSideEffect, procvar, - rtl, deprecated, extern: "nsuIsLowerChar".}= - ## Checks whether or not `c` is a lower case character. - ## - ## This checks ASCII characters only. - ## - ## **Deprecated since version 0.15.0**: use ``isLowerAscii`` instead. - isLowerAscii(c) - -proc isUpper*(c: char): bool {.noSideEffect, procvar, - rtl, deprecated, extern: "nsuIsUpperChar".}= - ## Checks whether or not `c` is an upper case character. - ## - ## This checks ASCII characters only. - ## - ## **Deprecated since version 0.15.0**: use ``isUpperAscii`` instead. - isUpperAscii(c) - -proc isAlpha*(c: char): bool {.noSideEffect, procvar, - rtl, deprecated, extern: "nsuIsAlphaChar".}= - ## Checks whether or not `c` is alphabetical. - ## - ## This checks a-z, A-Z ASCII characters only. - ## - ## **Deprecated since version 0.15.0**: use ``isAlphaAscii`` instead. - isAlphaAscii(c) - -proc isAlpha*(s: string): bool {.noSideEffect, procvar, - rtl, deprecated, extern: "nsuIsAlphaStr".}= - ## Checks whether or not `s` is alphabetical. - ## - ## This checks a-z, A-Z ASCII characters only. - ## Returns true if all characters in `s` are - ## alphabetic and there is at least one character - ## in `s`. - ## - ## **Deprecated since version 0.15.0**: use ``isAlphaAscii`` instead. - isAlphaAscii(s) - -proc isSpace*(s: string): bool {.noSideEffect, procvar, - rtl, deprecated, extern: "nsuIsSpaceStr".}= - ## Checks whether or not `s` is completely whitespace. - ## - ## Returns true if all characters in `s` are whitespace - ## characters and there is at least one character in `s`. - ## - ## **Deprecated since version 0.15.0**: use ``isSpaceAscii`` instead. - isSpaceAscii(s) - -proc isLower*(s: string): bool {.noSideEffect, procvar, - rtl, deprecated, extern: "nsuIsLowerStr".}= - ## Checks whether or not `s` contains all lower case characters. - ## - ## This checks ASCII characters only. - ## Returns true if all characters in `s` are lower case - ## and there is at least one character in `s`. - ## - ## **Deprecated since version 0.15.0**: use ``isLowerAscii`` instead. - isLowerAscii(s) - -proc isUpper*(s: string): bool {.noSideEffect, procvar, - rtl, deprecated, extern: "nsuIsUpperStr".}= - ## Checks whether or not `s` contains all upper case characters. - ## - ## This checks ASCII characters only. - ## Returns true if all characters in `s` are upper case - ## and there is at least one character in `s`. - ## - ## **Deprecated since version 0.15.0**: use ``isUpperAscii`` instead. - isUpperAscii(s) - -proc toLower*(c: char): char {.noSideEffect, procvar, - rtl, deprecated, extern: "nsuToLowerChar".} = - ## Converts `c` into lower case. - ## - ## This works only for the letters ``A-Z``. See `unicode.toLower - ## <unicode.html#toLower>`_ for a version that works for any Unicode - ## character. - ## - ## **Deprecated since version 0.15.0**: use ``toLowerAscii`` instead. - toLowerAscii(c) - -proc toLower*(s: string): string {.noSideEffect, procvar, - rtl, deprecated, extern: "nsuToLowerStr".} = - ## Converts `s` into lower case. - ## - ## This works only for the letters ``A-Z``. See `unicode.toLower - ## <unicode.html#toLower>`_ for a version that works for any Unicode - ## character. - ## - ## **Deprecated since version 0.15.0**: use ``toLowerAscii`` instead. - toLowerAscii(s) - -proc toUpper*(c: char): char {.noSideEffect, procvar, - rtl, deprecated, extern: "nsuToUpperChar".} = - ## Converts `c` into upper case. - ## - ## This works only for the letters ``A-Z``. See `unicode.toUpper - ## <unicode.html#toUpper>`_ for a version that works for any Unicode - ## character. - ## - ## **Deprecated since version 0.15.0**: use ``toUpperAscii`` instead. - toUpperAscii(c) - -proc toUpper*(s: string): string {.noSideEffect, procvar, - rtl, deprecated, extern: "nsuToUpperStr".} = - ## Converts `s` into upper case. - ## - ## This works only for the letters ``A-Z``. See `unicode.toUpper - ## <unicode.html#toUpper>`_ for a version that works for any Unicode - ## character. - ## - ## **Deprecated since version 0.15.0**: use ``toUpperAscii`` instead. - toUpperAscii(s) - -proc capitalize*(s: string): string {.noSideEffect, procvar, - rtl, deprecated, extern: "nsuCapitalize".} = - ## Converts the first character of `s` into upper case. - ## - ## This works only for the letters ``A-Z``. - ## - ## **Deprecated since version 0.15.0**: use ``capitalizeAscii`` instead. - capitalizeAscii(s) + if s.len == 0: result = "" + else: result = toUpperAscii(s[0]) & substr(s, 1) proc normalize*(s: string): string {.noSideEffect, procvar, rtl, extern: "nsuNormalize".} = ## Normalizes the string `s`. ## - ## That means to convert it to lower case and remove any '_'. This is needed - ## for Nim identifiers for example. + ## That means to convert it to lower case and remove any '_'. This + ## should NOT be used to normalize Nim identifier names. result = newString(s.len) var j = 0 for i in 0..len(s) - 1: @@ -418,8 +260,10 @@ proc cmpIgnoreCase*(a, b: string): int {.noSideEffect, proc cmpIgnoreStyle*(a, b: string): int {.noSideEffect, rtl, extern: "nsuCmpIgnoreStyle", procvar.} = - ## Compares two strings normalized (i.e. case and - ## underscores do not matter). Returns: + ## Semantically the same as ``cmp(normalize(a), normalize(b))``. It + ## is just optimized to not allocate temporary strings. This should + ## NOT be used to compare Nim identifier names. use `macros.eqIdent` + ## for that. Returns: ## ## | 0 iff a == b ## | < 0 iff a < b @@ -427,28 +271,37 @@ proc cmpIgnoreStyle*(a, b: string): int {.noSideEffect, var i = 0 var j = 0 while true: - while a[i] == '_': inc(i) - while b[j] == '_': inc(j) # BUGFIX: typo - var aa = toLowerAscii(a[i]) - var bb = toLowerAscii(b[j]) + while i < a.len and a[i] == '_': inc i + while j < b.len and b[j] == '_': inc j + var aa = if i < a.len: toLowerAscii(a[i]) else: '\0' + var bb = if j < b.len: toLowerAscii(b[j]) else: '\0' result = ord(aa) - ord(bb) - if result != 0 or aa == '\0': break - inc(i) - inc(j) - + if result != 0: return result + # the characters are identical: + if i >= a.len: + # both cursors at the end: + if j >= b.len: return 0 + # not yet at the end of 'b': + return -1 + elif j >= b.len: + return 1 + inc i + inc j proc strip*(s: string, leading = true, trailing = true, chars: set[char] = Whitespace): string {.noSideEffect, rtl, extern: "nsuStrip".} = - ## Strips `chars` from `s` and returns the resulting string. + ## Strips leading or trailing `chars` from `s` and returns + ## the resulting string. ## ## If `leading` is true, leading `chars` are stripped. ## If `trailing` is true, trailing `chars` are stripped. + ## If both are false, the string is returned unchanged. var first = 0 last = len(s)-1 if leading: - while s[first] in chars: inc(first) + while first <= last and s[first] in chars: inc(first) if trailing: while last >= 0 and s[last] in chars: dec(last) result = substr(s, first, last) @@ -464,7 +317,9 @@ proc toOctal*(c: char): string {.noSideEffect, rtl, extern: "nsuToOctal".} = result[i] = chr(val mod 8 + ord('0')) val = val div 8 -proc isNilOrEmpty*(s: string): bool {.noSideEffect, procvar, rtl, extern: "nsuIsNilOrEmpty".} = +proc isNilOrEmpty*(s: string): bool {.noSideEffect, procvar, rtl, + extern: "nsuIsNilOrEmpty", + deprecated: "use 'x.len == 0' instead".} = ## Checks if `s` is nil or empty. result = len(s) == 0 @@ -483,7 +338,6 @@ proc substrEq(s: string, pos: int, substr: string): bool = var length = substr.len while i < length and s[pos+i] == substr[i]: inc i - return i == length # --------- Private templates for different split separators ----------- @@ -517,7 +371,7 @@ template oldSplit(s, seps, maxsplit) = var splits = maxsplit assert(not ('\0' in seps)) while last < len(s): - while s[last] in seps: inc(last) + while last < len(s) and s[last] in seps: inc(last) var first = last while last < len(s) and s[last] notin seps: inc(last) if first <= last-1: @@ -568,10 +422,7 @@ iterator split*(s: string, seps: set[char] = Whitespace, ## "08" ## "08.398990" ## - when defined(nimOldSplit): - oldSplit(s, seps, maxsplit) - else: - splitCommon(s, seps, maxsplit, 1) + splitCommon(s, seps, maxsplit, 1) iterator splitWhitespace*(s: string, maxsplit: int = -1): string = ## Splits the string ``s`` at whitespace stripping leading and trailing @@ -657,7 +508,6 @@ iterator split*(s: string, sep: string, maxsplit: int = -1): string = ## "is" ## "corrupted" ## - splitCommon(s, sep, maxsplit, sep.len) template rsplitCommon(s, sep, maxsplit, sepLen) = @@ -667,29 +517,21 @@ template rsplitCommon(s, sep, maxsplit, sepLen) = first = last splits = maxsplit startPos = 0 - # go to -1 in order to get separators at the beginning while first >= -1: while first >= 0 and not stringHasSep(s, first, sep): dec(first) - if splits == 0: # No more splits means set first to the beginning first = -1 - if first == -1: startPos = 0 else: startPos = first + sepLen - yield substr(s, startPos, last) - - if splits == 0: - break - + if splits == 0: break dec(splits) dec(first) - last = first iterator rsplit*(s: string, seps: set[char] = Whitespace, @@ -709,7 +551,6 @@ iterator rsplit*(s: string, seps: set[char] = Whitespace, ## "foo" ## ## Substrings are separated from the right by the set of chars `seps` - rsplitCommon(s, seps, maxsplit, 1) iterator rsplit*(s: string, sep: char, @@ -776,14 +617,14 @@ iterator splitLines*(s: string): string = var first = 0 var last = 0 while true: - while s[last] notin {'\0', '\c', '\l'}: inc(last) + while last < s.len and s[last] notin {'\c', '\l'}: inc(last) yield substr(s, first, last-1) # skip newlines: + if last >= s.len: break if s[last] == '\l': inc(last) elif s[last] == '\c': inc(last) - if s[last] == '\l': inc(last) - else: break # was '\0' + if last < s.len and s[last] == '\l': inc(last) first = last proc splitLines*(s: string): seq[string] {.noSideEffect, @@ -808,7 +649,7 @@ proc countLines*(s: string): int {.noSideEffect, while i < s.len: case s[i] of '\c': - if s[i+1] == '\l': inc i + if i+1 < s.len and s[i+1] == '\l': inc i inc result of '\l': inc result else: discard @@ -944,6 +785,19 @@ proc toHex*[T](x: T): string = ## Shortcut for ``toHex(x, T.sizeOf * 2)`` toHex(BiggestInt(x), T.sizeOf * 2) +proc toHex*(s: string): string {.noSideEffect, rtl.} = + ## Converts a bytes string to its hexadecimal representation. + ## + ## The output is twice the input long. No prefix like + ## ``0x`` is generated. + const HexChars = "0123456789ABCDEF" + result = newString(s.len * 2) + for pos, c in s: + var n = ord(c) + result[pos * 2 + 1] = HexChars[n and 0xF] + n = n shr 4 + result[pos * 2] = HexChars[n] + proc intToStr*(x: int, minchars: Positive = 1): string {.noSideEffect, rtl, extern: "nsuIntToStr".} = ## Converts `x` to its decimal representation. @@ -1009,9 +863,9 @@ proc parseHexInt*(s: string): int {.noSideEffect, procvar, ## of the following optional prefixes: ``0x``, ``0X``, ``#``. Underscores ## within `s` are ignored. var i = 0 - if s[i] == '0' and (s[i+1] == 'x' or s[i+1] == 'X'): inc(i, 2) - elif s[i] == '#': inc(i) - while true: + if i+1 < s.len and s[i] == '0' and (s[i+1] == 'x' or s[i+1] == 'X'): inc(i, 2) + elif i < s.len and s[i] == '#': inc(i) + while i < s.len: case s[i] of '_': inc(i) of '0'..'9': @@ -1023,9 +877,45 @@ proc parseHexInt*(s: string): int {.noSideEffect, procvar, of 'A'..'F': result = result shl 4 or (ord(s[i]) - ord('A') + 10) inc(i) - of '\0': break else: raise newException(ValueError, "invalid integer: " & s) +proc generateHexCharToValueMap(): string = + ## Generate a string to map a hex digit to uint value + result = "" + for inp in 0..255: + let ch = chr(inp) + let o = + case ch: + of '0'..'9': inp - ord('0') + of 'a'..'f': inp - ord('a') + 10 + of 'A'..'F': inp - ord('A') + 10 + else: 17 # indicates an invalid hex char + result.add chr(o) + +const hexCharToValueMap = generateHexCharToValueMap() + +proc parseHexStr*(s: string): string {.noSideEffect, procvar, + rtl, extern: "nsuParseHexStr".} = + ## Convert hex-encoded string to byte string, e.g.: + ## + ## .. code-block:: nim + ## hexToStr("00ff") == "\0\255" + ## + ## Raises ``ValueError`` for an invalid hex values. The comparison is + ## case-insensitive. + if s.len mod 2 != 0: + raise newException(ValueError, "Incorrect hex string len") + result = newString(s.len div 2) + var buf = 0 + for pos, c in s: + let val = hexCharToValueMap[ord(c)].ord + if val == 17: + raise newException(ValueError, "Invalid hex char " & repr(c)) + if pos mod 2 == 0: + buf = val + else: + result[pos div 2] = chr(val + buf shl 4) + proc parseBool*(s: string): bool = ## Parses a value into a `bool`. ## @@ -1095,14 +985,6 @@ template spaces*(n: Natural): string = repeat(' ', n) ## echo text1 & spaces(max(0, width - text1.len)) & "|" ## echo text2 & spaces(max(0, width - text2.len)) & "|" -proc repeatChar*(count: Natural, c: char = ' '): string {.deprecated.} = - ## deprecated: use repeat() or spaces() - repeat(c, count) - -proc repeatStr*(count: Natural, s: string): string {.deprecated.} = - ## deprecated: use repeat(string, count) or string.repeat(count) - repeat(s, count) - proc align*(s: string, count: Natural, padding = ' '): string {. noSideEffect, rtl, extern: "nsuAlignString".} = ## Aligns a string `s` with `padding`, so that it is of length `count`. @@ -1173,7 +1055,7 @@ iterator tokenize*(s: string, seps: set[char] = Whitespace): tuple[ var i = 0 while true: var j = i - var isSep = s[j] in seps + var isSep = j < s.len and s[j] in seps while j < s.len and (s[j] in seps) == isSep: inc(j) if j > i: yield (substr(s, i, j-1), isSep) @@ -1198,7 +1080,7 @@ proc wordWrap*(s: string, maxLineWidth = 80, if len(word) > spaceLeft: if splitLongWords and len(word) > maxLineWidth: result.add(substr(word, 0, spaceLeft-1)) - var w = spaceLeft+1 + var w = spaceLeft var wordLeft = len(word) - spaceLeft while wordLeft > 0: result.add(newLine) @@ -1244,7 +1126,7 @@ proc unindent*(s: string, count: Natural, padding: string = " "): string var indentCount = 0 for j in 0..<count.int: indentCount.inc - if line[j .. j + padding.len-1] != padding: + if j + padding.len-1 >= line.len or line[j .. j + padding.len-1] != padding: indentCount = j break result.add(line[indentCount*padding.len .. ^1]) @@ -1272,13 +1154,13 @@ proc startsWith*(s, prefix: string): bool {.noSideEffect, ## If ``prefix == ""`` true is returned. var i = 0 while true: - if prefix[i] == '\0': return true - if s[i] != prefix[i]: return false + if i >= prefix.len: return true + if i >= s.len or s[i] != prefix[i]: return false inc(i) proc startsWith*(s: string, prefix: char): bool {.noSideEffect, inline.} = ## Returns true iff ``s`` starts with ``prefix``. - result = s[0] == prefix + result = s.len > 0 and s[0] == prefix proc endsWith*(s, suffix: string): bool {.noSideEffect, rtl, extern: "nsuEndsWith".} = @@ -1290,11 +1172,11 @@ proc endsWith*(s, suffix: string): bool {.noSideEffect, while i+j <% s.len: if s[i+j] != suffix[i]: return false inc(i) - if suffix[i] == '\0': return true + if i >= suffix.len: return true proc endsWith*(s: string, suffix: char): bool {.noSideEffect, inline.} = ## Returns true iff ``s`` ends with ``suffix``. - result = s[s.high] == suffix + result = s.len > 0 and s[s.high] == suffix proc continuesWith*(s, substr: string, start: Natural): bool {.noSideEffect, rtl, extern: "nsuContinuesWith".} = @@ -1303,8 +1185,8 @@ proc continuesWith*(s, substr: string, start: Natural): bool {.noSideEffect, ## If ``substr == ""`` true is returned. var i = 0 while true: - if substr[i] == '\0': return true - if s[i+start] != substr[i]: return false + if i >= substr.len: return true + if i+start >= s.len or s[i+start] != substr[i]: return false inc(i) proc addSep*(dest: var string, sep = ", ", startLen: Natural = 0) @@ -1380,21 +1262,20 @@ proc initSkipTable*(a: var SkipTable, sub: string) {.noSideEffect, rtl, extern: "nsuInitSkipTable".} = ## Preprocess table `a` for `sub`. let m = len(sub) - let m1 = m + 1 var i = 0 while i <= 0xff-7: - a[chr(i + 0)] = m1 - a[chr(i + 1)] = m1 - a[chr(i + 2)] = m1 - a[chr(i + 3)] = m1 - a[chr(i + 4)] = m1 - a[chr(i + 5)] = m1 - a[chr(i + 6)] = m1 - a[chr(i + 7)] = m1 + a[chr(i + 0)] = m + a[chr(i + 1)] = m + a[chr(i + 2)] = m + a[chr(i + 3)] = m + a[chr(i + 4)] = m + a[chr(i + 5)] = m + a[chr(i + 6)] = m + a[chr(i + 7)] = m i += 8 - for i in 0..m-1: - a[sub[i]] = m-i + for i in 0 ..< m - 1: + a[sub[i]] = m - 1 - i proc find*(a: SkipTable, s, sub: string, start: Natural = 0, last: Natural = 0): int {.noSideEffect, rtl, extern: "nsuFindStrA".} = @@ -1402,18 +1283,29 @@ proc find*(a: SkipTable, s, sub: string, start: Natural = 0, last: Natural = 0): ## If `last` is unspecified, it defaults to `s.high`. ## ## Searching is case-sensitive. If `sub` is not in `s`, -1 is returned. + let last = if last==0: s.high else: last - m = len(sub) - n = last + 1 - # search: - var j = start - while j <= n - m: - block match: - for k in 0..m-1: - if sub[k] != s[k+j]: break match - return j - inc(j, a[s[j+m]]) + sLen = last - start + 1 + subLast = sub.len - 1 + + if subLast == -1: + # this was an empty needle string, + # we count this as match in the first possible position: + return start + + # This is an implementation of the Boyer-Moore Horspool algorithms + # https://en.wikipedia.org/wiki/Boyer%E2%80%93Moore%E2%80%93Horspool_algorithm + var skip = start + + while last - skip >= subLast: + var i = subLast + while s[skip + i] == sub[i]: + if i == 0: + return skip + dec i + inc skip, a[s[skip + subLast]] + return -1 when not (defined(js) or defined(nimdoc) or defined(nimscript)): @@ -1449,12 +1341,8 @@ proc find*(s, sub: string, start: Natural = 0, last: Natural = 0): int {.noSideE ## If `last` is unspecified, it defaults to `s.high`. ## ## Searching is case-sensitive. If `sub` is not in `s`, -1 is returned. - if sub.len > s.len: - return -1 - - if sub.len == 1: - return find(s, sub[0], start, last) - + if sub.len > s.len: return -1 + if sub.len == 1: return find(s, sub[0], start, last) var a {.noinit.}: SkipTable initSkipTable(a, sub) result = find(a, s, sub, start, last) @@ -1511,18 +1399,14 @@ proc center*(s: string, width: int, fillChar: char = ' '): string {. ## ## The original string is returned if `width` is less than or equal ## to `s.len`. - if width <= s.len: - return s - + if width <= s.len: return s result = newString(width) - # Left padding will be one fillChar # smaller if there are an odd number # of characters let charsLeft = (width - s.len) leftPadding = charsLeft div 2 - for i in 0 ..< width: if i >= leftPadding and i < leftPadding + s.len: # we are where the string should be located @@ -1540,27 +1424,22 @@ proc count*(s: string, sub: string, overlapping: bool = false): int {. var i = 0 while true: i = s.find(sub, i) - if i < 0: - break - if overlapping: - inc i - else: - i += sub.len + if i < 0: break + if overlapping: inc i + else: i += sub.len inc result proc count*(s: string, sub: char): int {.noSideEffect, rtl, extern: "nsuCountChar".} = ## Count the occurrences of the character `sub` in the string `s`. for c in s: - if c == sub: - inc result + if c == sub: inc result proc count*(s: string, subs: set[char]): int {.noSideEffect, rtl, extern: "nsuCountCharSet".} = ## Count the occurrences of the group of character `subs` in the string `s`. for c in s: - if c in subs: - inc result + if c in subs: inc result proc quoteIfContainsWhite*(s: string): string {.deprecated.} = ## Returns ``'"' & s & '"'`` if `s` contains a space and does not @@ -1568,10 +1447,8 @@ proc quoteIfContainsWhite*(s: string): string {.deprecated.} = ## ## **DEPRECATED** as it was confused for shell quoting function. For this ## application use `osproc.quoteShell <osproc.html#quoteShell>`_. - if find(s, {' ', '\t'}) >= 0 and s[0] != '"': - result = '"' & s & '"' - else: - result = s + if find(s, {' ', '\t'}) >= 0 and s[0] != '"': result = '"' & s & '"' + else: result = s proc contains*(s: string, c: char): bool {.noSideEffect.} = ## Same as ``find(s, c) >= 0``. @@ -1588,19 +1465,41 @@ proc contains*(s: string, chars: set[char]): bool {.noSideEffect.} = proc replace*(s, sub: string, by = ""): string {.noSideEffect, rtl, extern: "nsuReplaceStr".} = ## Replaces `sub` in `s` by the string `by`. - var a {.noinit.}: SkipTable result = "" - initSkipTable(a, sub) - let last = s.high - var i = 0 - while true: - var j = find(a, s, sub, i, last) - if j < 0: break - add result, substr(s, i, j - 1) + let subLen = sub.len + if subLen == 0: + for c in s: + add result, by + add result, c add result, by - i = j + len(sub) - # copy the rest: - add result, substr(s, i) + return + elif subLen == 1: + # when the pattern is a single char, we use a faster + # char-based search that doesn't need a skip table: + var c = sub[0] + let last = s.high + var i = 0 + while true: + let j = find(s, c, i, last) + if j < 0: break + add result, substr(s, i, j - 1) + add result, by + i = j + subLen + # copy the rest: + add result, substr(s, i) + else: + var a {.noinit.}: SkipTable + initSkipTable(a, sub) + let last = s.high + var i = 0 + while true: + let j = find(a, s, sub, i, last) + if j < 0: break + add result, substr(s, i, j - 1) + add result, by + i = j + subLen + # copy the rest: + add result, substr(s, i) proc replace*(s: string, sub, by: char): string {.noSideEffect, rtl, extern: "nsuReplaceChar".} = @@ -1621,12 +1520,14 @@ proc replaceWord*(s, sub: string, by = ""): string {.noSideEffect, ## Each occurrence of `sub` has to be surrounded by word boundaries ## (comparable to ``\\w`` in regular expressions), otherwise it is not ## replaced. + if sub.len == 0: return s const wordChars = {'a'..'z', 'A'..'Z', '0'..'9', '_', '\128'..'\255'} var a {.noinit.}: SkipTable result = "" initSkipTable(a, sub) var i = 0 let last = s.high + let sublen = max(sub.len, 1) while true: var j = find(a, s, sub, i, last) if j < 0: break @@ -1635,7 +1536,7 @@ proc replaceWord*(s, sub: string, by = ""): string {.noSideEffect, (j+sub.len >= s.len or s[j+sub.len] notin wordChars): add result, substr(s, i, j - 1) add result, by - i = j + len(sub) + i = j + sublen else: add result, substr(s, i, j) i = j + 1 @@ -1646,9 +1547,8 @@ proc multiReplace*(s: string, replacements: varargs[(string, string)]): string { ## Same as replace, but specialized for doing multiple replacements in a single ## pass through the input string. ## - ## Calling replace multiple times after each other is inefficient and result in too many allocations - ## follwed by immediate deallocations as portions of the string gets replaced. - ## multiReplace performs all replacements in a single pass. + ## multiReplace performs all replacements in a single pass, this means it can be used + ## to swap the occurences of "a" and "b", for instance. ## ## If the resulting string is not longer than the original input string, only a single ## memory allocation is required. @@ -1695,14 +1595,13 @@ proc parseOctInt*(s: string): int {.noSideEffect, ## of the following optional prefixes: ``0o``, ``0O``. Underscores within ## `s` are ignored. var i = 0 - if s[i] == '0' and (s[i+1] == 'o' or s[i+1] == 'O'): inc(i, 2) - while true: + if i+1 < s.len and s[i] == '0' and (s[i+1] == 'o' or s[i+1] == 'O'): inc(i, 2) + while i < s.len: case s[i] of '_': inc(i) of '0'..'7': result = result shl 3 or (ord(s[i]) - ord('0')) inc(i) - of '\0': break else: raise newException(ValueError, "invalid integer: " & s) proc toOct*(x: BiggestInt, len: Positive): string {.noSideEffect, @@ -1760,20 +1659,29 @@ proc insertSep*(s: string, sep = '_', digits = 3): string {.noSideEffect, dec(L) proc escape*(s: string, prefix = "\"", suffix = "\""): string {.noSideEffect, - rtl, extern: "nsuEscape".} = + rtl, extern: "nsuEscape", deprecated.} = ## Escapes a string `s`. See `system.addEscapedChar <system.html#addEscapedChar>`_ ## for the escaping scheme. ## ## The resulting string is prefixed with `prefix` and suffixed with `suffix`. ## Both may be empty strings. + ## + ## **Warning:** This procedure is deprecated because it's to easy to missuse. result = newStringOfCap(s.len + s.len shr 2) result.add(prefix) for c in items(s): - result.addEscapedChar(c) + case c + of '\0'..'\31', '\127'..'\255': + add(result, "\\x") + add(result, toHex(ord(c), 2)) + of '\\': add(result, "\\\\") + of '\'': add(result, "\\'") + of '\"': add(result, "\\\"") + else: add(result, c) add(result, suffix) proc unescape*(s: string, prefix = "\"", suffix = "\""): string {.noSideEffect, - rtl, extern: "nsuUnescape".} = + rtl, extern: "nsuUnescape", deprecated.} = ## Unescapes a string `s`. ## ## This complements `escape <#escape>`_ as it performs the opposite @@ -1781,15 +1689,19 @@ proc unescape*(s: string, prefix = "\"", suffix = "\""): string {.noSideEffect, ## ## If `s` does not begin with ``prefix`` and end with ``suffix`` a ## ValueError exception will be raised. + ## + ## **Warning:** This procedure is deprecated because it's to easy to missuse. result = newStringOfCap(s.len) var i = prefix.len if not s.startsWith(prefix): raise newException(ValueError, - "String does not start with a prefix of: " & prefix) + "String does not start with: " & prefix) while true: - if i == s.len-suffix.len: break - case s[i] - of '\\': + if i >= s.len-suffix.len: break + if s[i] == '\\': + if i+1 >= s.len: + result.add('\\') + break case s[i+1]: of 'x': inc i, 2 @@ -1803,15 +1715,15 @@ proc unescape*(s: string, prefix = "\"", suffix = "\""): string {.noSideEffect, result.add('\'') of '\"': result.add('\"') - else: result.add("\\" & s[i+1]) - inc(i) - of '\0': break + else: + result.add("\\" & s[i+1]) + inc(i, 2) else: result.add(s[i]) - inc(i) + inc(i) if not s.endsWith(suffix): raise newException(ValueError, - "String does not end with a suffix of: " & suffix) + "String does not end in: " & suffix) proc validIdentifier*(s: string): bool {.noSideEffect, rtl, extern: "nsuValidIdentifier".} = @@ -1821,7 +1733,7 @@ proc validIdentifier*(s: string): bool {.noSideEffect, ## and is followed by any number of characters of the set `IdentChars`. runnableExamples: doAssert "abc_def08".validIdentifier - if s[0] in IdentStartChars: + if s.len > 0 and s[0] in IdentStartChars: for i in 1..s.len-1: if s[i] notin IdentChars: return false return true @@ -1840,7 +1752,7 @@ proc editDistance*(a, b: string): int {.noSideEffect, # strip common prefix: var s = 0 - while a[s] == b[s] and a[s] != '\0': + while s < len1 and a[s] == b[s]: inc(s) dec(len1) dec(len2) @@ -1913,8 +1825,6 @@ proc editDistance*(a, b: string): int {.noSideEffect, if x > c3: x = c3 row[p] = x result = row[e] - #dealloc(row) - # floating point formating: when not defined(js): @@ -1944,6 +1854,10 @@ proc formatBiggestFloat*(f: BiggestFloat, format: FloatFormatMode = ffDefault, ## ## If ``precision == -1``, it tries to format it nicely. when defined(js): + var precision = precision + if precision == -1: + # use the same default precision as c_sprintf + precision = 6 var res: cstring case format of ffDefault: @@ -1953,6 +1867,9 @@ proc formatBiggestFloat*(f: BiggestFloat, format: FloatFormatMode = ffDefault, of ffScientific: {.emit: "`res` = `f`.toExponential(`precision`);".} result = $res + if 1.0 / f == -Inf: + # JavaScript removes the "-" from negative Zero, add it back here + result = "-" & $res for i in 0 ..< result.len: # Depending on the locale either dot or comma is produced, # but nothing else is possible: @@ -2023,7 +1940,7 @@ proc trimZeros*(x: var string) {.noSideEffect.} = var spl: seq[string] if x.contains('.') or x.contains(','): if x.contains('e'): - spl= x.split('e') + spl = x.split('e') x = spl[0] while x[x.high] == '0': x.setLen(x.len-1) @@ -2092,12 +2009,13 @@ proc formatEng*(f: BiggestFloat, precision: range[0..32] = 10, trim: bool = true, siPrefix: bool = false, - unit: string = nil, - decimalSep = '.'): string {.noSideEffect.} = + unit: string = "", + decimalSep = '.', + useUnitSpace = false): string {.noSideEffect.} = ## Converts a floating point value `f` to a string using engineering notation. ## ## Numbers in of the range -1000.0<f<1000.0 will be formatted without an - ## exponent. Numbers outside of this range will be formatted as a + ## exponent. Numbers outside of this range will be formatted as a ## significand in the range -1000.0<f<1000.0 and an exponent that will always ## be an integer multiple of 3, corresponding with the SI prefix scale k, M, ## G, T etc for numbers with an absolute value greater than 1 and m, μ, n, p @@ -2105,7 +2023,7 @@ proc formatEng*(f: BiggestFloat, ## ## The default configuration (`trim=true` and `precision=10`) shows the ## **shortest** form that precisely (up to a maximum of 10 decimal places) - ## displays the value. For example, 4.100000 will be displayed as 4.1 (which + ## displays the value. For example, 4.100000 will be displayed as 4.1 (which ## is mathematically identical) whereas 4.1000003 will be displayed as ## 4.1000003. ## @@ -2125,15 +2043,15 @@ proc formatEng*(f: BiggestFloat, ## formatEng(-52731234, 2) == "-52.73e6" ## ## If `siPrefix` is set to true, the number will be displayed with the SI - ## prefix corresponding to the exponent. For example 4100 will be displayed - ## as "4.1 k" instead of "4.1e3". Note that `u` is used for micro- in place - ## of the greek letter mu (μ) as per ISO 2955. Numbers with an absolute + ## prefix corresponding to the exponent. For example 4100 will be displayed + ## as "4.1 k" instead of "4.1e3". Note that `u` is used for micro- in place + ## of the greek letter mu (μ) as per ISO 2955. Numbers with an absolute ## value outside of the range 1e-18<f<1000e18 (1a<f<1000E) will be displayed ## with an exponent rather than an SI prefix, regardless of whether ## `siPrefix` is true. ## - ## If `unit` is not nil, the provided unit will be appended to the string - ## (with a space as required by the SI standard). This behaviour is slightly + ## If `useUnitSpace` is true, the provided unit will be appended to the string + ## (with a space as required by the SI standard). This behaviour is slightly ## different to appending the unit to the result as the location of the space ## is altered depending on whether there is an exponent. ## @@ -2147,7 +2065,7 @@ proc formatEng*(f: BiggestFloat, ## formatEng(4100, siPrefix=true, unit="") == "4.1 k" ## formatEng(4100) == "4.1e3" ## formatEng(4100, unit="V") == "4.1e3 V" - ## formatEng(4100, unit="") == "4.1e3 " # Space with unit="" + ## formatEng(4100, unit="", useUnitSpace=true) == "4.1e3 " # Space with useUnitSpace=true ## ## `decimalSep` is used as the decimal separator. var @@ -2215,10 +2133,9 @@ proc formatEng*(f: BiggestFloat, if p != ' ': suffix = " " & p exponent = 0 # Exponent replaced by SI prefix - if suffix == "" and unit != nil: + if suffix == "" and useUnitSpace: suffix = " " - if unit != nil: - suffix &= unit + suffix &= unit if exponent != 0: result &= "e" & $exponent result &= suffix @@ -2241,11 +2158,10 @@ proc addf*(s: var string, formatstr: string, a: varargs[string, `$`]) {. var i = 0 var num = 0 while i < len(formatstr): - if formatstr[i] == '$': - case formatstr[i+1] # again we use the fact that strings - # are zero-terminated here + if formatstr[i] == '$' and i+1 < len(formatstr): + case formatstr[i+1] of '#': - if num >% a.high: invalidFormatString() + if num > a.high: invalidFormatString() add s, a[num] inc i, 2 inc num @@ -2257,11 +2173,11 @@ proc addf*(s: var string, formatstr: string, a: varargs[string, `$`]) {. inc(i) # skip $ var negative = formatstr[i] == '-' if negative: inc i - while formatstr[i] in Digits: + while i < formatstr.len and formatstr[i] in Digits: j = j * 10 + ord(formatstr[i]) - ord('0') inc(i) let idx = if not negative: j-1 else: a.len-j - if idx >% a.high: invalidFormatString() + if idx < 0 or idx > a.high: invalidFormatString() add s, a[idx] of '{': var j = i+2 @@ -2269,7 +2185,7 @@ proc addf*(s: var string, formatstr: string, a: varargs[string, `$`]) {. var negative = formatstr[j] == '-' if negative: inc j var isNumber = 0 - while formatstr[j] notin {'\0', '}'}: + while j < formatstr.len and formatstr[j] notin {'\0', '}'}: if formatstr[j] in Digits: k = k * 10 + ord(formatstr[j]) - ord('0') if isNumber == 0: isNumber = 1 @@ -2278,7 +2194,7 @@ proc addf*(s: var string, formatstr: string, a: varargs[string, `$`]) {. inc(j) if isNumber == 1: let idx = if not negative: k-1 else: a.len-k - if idx >% a.high: invalidFormatString() + if idx < 0 or idx > a.high: invalidFormatString() add s, a[idx] else: var x = findNormalized(substr(formatstr, i+2, j-1), a) @@ -2287,7 +2203,7 @@ proc addf*(s: var string, formatstr: string, a: varargs[string, `$`]) {. i = j+1 of 'a'..'z', 'A'..'Z', '\128'..'\255', '_': var j = i+1 - while formatstr[j] in PatternChars: inc(j) + while j < formatstr.len and formatstr[j] in PatternChars: inc(j) var x = findNormalized(substr(formatstr, i+1, j-1), a) if x >= 0 and x < high(a): add s, a[x+1] else: invalidFormatString() @@ -2446,234 +2362,243 @@ proc removePrefix*(s: var string, prefix: string) {. s.delete(0, prefix.len - 1) when isMainModule: - doAssert align("abc", 4) == " abc" - doAssert align("a", 0) == "a" - doAssert align("1232", 6) == " 1232" - doAssert align("1232", 6, '#') == "##1232" - - doAssert alignLeft("abc", 4) == "abc " - doAssert alignLeft("a", 0) == "a" - doAssert alignLeft("1232", 6) == "1232 " - doAssert alignLeft("1232", 6, '#') == "1232##" - - let - inp = """ this is a long text -- muchlongerthan10chars and here - it goes""" - outp = " this is a\nlong text\n--\nmuchlongerthan10chars\nand here\nit goes" - doAssert wordWrap(inp, 10, false) == outp - - doAssert formatBiggestFloat(1234.567, ffDecimal, -1) == "1234.567000" - doAssert formatBiggestFloat(1234.567, ffDecimal, 0) == "1235." - doAssert formatBiggestFloat(1234.567, ffDecimal, 1) == "1234.6" - doAssert formatBiggestFloat(0.00000000001, ffDecimal, 11) == "0.00000000001" - doAssert formatBiggestFloat(0.00000000001, ffScientific, 1, ',') in - ["1,0e-11", "1,0e-011"] - # bug #6589 - doAssert formatFloat(123.456, ffScientific, precision = -1) == "1.234560e+02" - - doAssert "$# $3 $# $#" % ["a", "b", "c"] == "a c b c" - doAssert "${1}12 ${-1}$2" % ["a", "b"] == "a12 bb" - - block: # formatSize tests - doAssert formatSize((1'i64 shl 31) + (300'i64 shl 20)) == "2.293GiB" - doAssert formatSize((2.234*1024*1024).int) == "2.234MiB" - doAssert formatSize(4096) == "4KiB" - doAssert formatSize(4096, prefix=bpColloquial, includeSpace=true) == "4 kB" - doAssert formatSize(4096, includeSpace=true) == "4 KiB" - doAssert formatSize(5_378_934, prefix=bpColloquial, decimalSep=',') == "5,13MB" - - doAssert "$animal eats $food." % ["animal", "The cat", "food", "fish"] == - "The cat eats fish." - - doAssert "-ld a-ldz -ld".replaceWord("-ld") == " a-ldz " - doAssert "-lda-ldz -ld abc".replaceWord("-ld") == "-lda-ldz abc" - - type MyEnum = enum enA, enB, enC, enuD, enE - doAssert parseEnum[MyEnum]("enu_D") == enuD - - doAssert parseEnum("invalid enum value", enC) == enC - - doAssert center("foo", 13) == " foo " - doAssert center("foo", 0) == "foo" - doAssert center("foo", 3, fillChar = 'a') == "foo" - doAssert center("foo", 10, fillChar = '\t') == "\t\t\tfoo\t\t\t\t" - - doAssert count("foofoofoo", "foofoo") == 1 - doAssert count("foofoofoo", "foofoo", overlapping = true) == 2 - doAssert count("foofoofoo", 'f') == 3 - doAssert count("foofoofoobar", {'f','b'}) == 4 - - doAssert strip(" foofoofoo ") == "foofoofoo" - doAssert strip("sfoofoofoos", chars = {'s'}) == "foofoofoo" - doAssert strip("barfoofoofoobar", chars = {'b', 'a', 'r'}) == "foofoofoo" - doAssert strip("stripme but don't strip this stripme", - chars = {'s', 't', 'r', 'i', 'p', 'm', 'e'}) == - " but don't strip this " - doAssert strip("sfoofoofoos", leading = false, chars = {'s'}) == "sfoofoofoo" - doAssert strip("sfoofoofoos", trailing = false, chars = {'s'}) == "foofoofoos" - - doAssert " foo\n bar".indent(4, "Q") == "QQQQ foo\nQQQQ bar" - - doAssert "abba".multiReplace(("a", "b"), ("b", "a")) == "baab" - doAssert "Hello World.".multiReplace(("ello", "ELLO"), ("World.", "PEOPLE!")) == "HELLO PEOPLE!" - doAssert "aaaa".multiReplace(("a", "aa"), ("aa", "bb")) == "aaaaaaaa" - - doAssert isAlphaAscii('r') - doAssert isAlphaAscii('A') - doAssert(not isAlphaAscii('$')) - - doAssert isAlphaAscii("Rasp") - doAssert isAlphaAscii("Args") - doAssert(not isAlphaAscii("$Tomato")) - - doAssert isAlphaNumeric('3') - doAssert isAlphaNumeric('R') - doAssert(not isAlphaNumeric('!')) - - doAssert isAlphaNumeric("34ABc") - doAssert isAlphaNumeric("Rad") - doAssert isAlphaNumeric("1234") - doAssert(not isAlphaNumeric("@nose")) - - doAssert isDigit('3') - doAssert(not isDigit('a')) - doAssert(not isDigit('%')) - - doAssert isDigit("12533") - doAssert(not isDigit("12.33")) - doAssert(not isDigit("A45b")) - - doAssert isSpaceAscii('\t') - doAssert isSpaceAscii('\l') - doAssert(not isSpaceAscii('A')) - - doAssert isSpaceAscii("\t\l \v\r\f") - doAssert isSpaceAscii(" ") - doAssert(not isSpaceAscii("ABc \td")) - - doAssert(isNilOrEmpty("")) - doAssert(isNilOrEmpty(nil)) - doAssert(not isNilOrEmpty("test")) - doAssert(not isNilOrEmpty(" ")) - - doAssert(isNilOrWhitespace("")) - doAssert(isNilOrWhitespace(nil)) - doAssert(isNilOrWhitespace(" ")) - doAssert(isNilOrWhitespace("\t\l \v\r\f")) - doAssert(not isNilOrWhitespace("ABc \td")) - - doAssert isLowerAscii('a') - doAssert isLowerAscii('z') - doAssert(not isLowerAscii('A')) - doAssert(not isLowerAscii('5')) - doAssert(not isLowerAscii('&')) - - doAssert isLowerAscii("abcd") - doAssert(not isLowerAscii("abCD")) - doAssert(not isLowerAscii("33aa")) - - doAssert isUpperAscii('A') - doAssert(not isUpperAscii('b')) - doAssert(not isUpperAscii('5')) - doAssert(not isUpperAscii('%')) - - doAssert isUpperAscii("ABC") - doAssert(not isUpperAscii("AAcc")) - doAssert(not isUpperAscii("A#$")) - - doAssert rsplit("foo bar", seps=Whitespace) == @["foo", "bar"] - doAssert rsplit(" foo bar", seps=Whitespace, maxsplit=1) == @[" foo", "bar"] - doAssert rsplit(" foo bar ", seps=Whitespace, maxsplit=1) == @[" foo bar", ""] - doAssert rsplit(":foo:bar", sep=':') == @["", "foo", "bar"] - doAssert rsplit(":foo:bar", sep=':', maxsplit=2) == @["", "foo", "bar"] - doAssert rsplit(":foo:bar", sep=':', maxsplit=3) == @["", "foo", "bar"] - doAssert rsplit("foothebar", sep="the") == @["foo", "bar"] - - doAssert(unescape(r"\x013", "", "") == "\x013") - - doAssert join(["foo", "bar", "baz"]) == "foobarbaz" - doAssert join(@["foo", "bar", "baz"], ", ") == "foo, bar, baz" - doAssert join([1, 2, 3]) == "123" - doAssert join(@[1, 2, 3], ", ") == "1, 2, 3" - - doAssert """~~!!foo + proc nonStaticTests = + doAssert formatBiggestFloat(1234.567, ffDecimal, -1) == "1234.567000" + doAssert formatBiggestFloat(1234.567, ffDecimal, 0) == "1235." + doAssert formatBiggestFloat(1234.567, ffDecimal, 1) == "1234.6" + doAssert formatBiggestFloat(0.00000000001, ffDecimal, 11) == "0.00000000001" + doAssert formatBiggestFloat(0.00000000001, ffScientific, 1, ',') in + ["1,0e-11", "1,0e-011"] + # bug #6589 + doAssert formatFloat(123.456, ffScientific, precision = -1) == "1.234560e+02" + + doAssert "$# $3 $# $#" % ["a", "b", "c"] == "a c b c" + doAssert "${1}12 ${-1}$2" % ["a", "b"] == "a12 bb" + + block: # formatSize tests + doAssert formatSize((1'i64 shl 31) + (300'i64 shl 20)) == "2.293GiB" + doAssert formatSize((2.234*1024*1024).int) == "2.234MiB" + doAssert formatSize(4096) == "4KiB" + doAssert formatSize(4096, prefix=bpColloquial, includeSpace=true) == "4 kB" + doAssert formatSize(4096, includeSpace=true) == "4 KiB" + doAssert formatSize(5_378_934, prefix=bpColloquial, decimalSep=',') == "5,13MB" + + block: # formatEng tests + doAssert formatEng(0, 2, trim=false) == "0.00" + doAssert formatEng(0, 2) == "0" + doAssert formatEng(53, 2, trim=false) == "53.00" + doAssert formatEng(0.053, 2, trim=false) == "53.00e-3" + doAssert formatEng(0.053, 4, trim=false) == "53.0000e-3" + doAssert formatEng(0.053, 4, trim=true) == "53e-3" + doAssert formatEng(0.053, 0) == "53e-3" + doAssert formatEng(52731234) == "52.731234e6" + doAssert formatEng(-52731234) == "-52.731234e6" + doAssert formatEng(52731234, 1) == "52.7e6" + doAssert formatEng(-52731234, 1) == "-52.7e6" + doAssert formatEng(52731234, 1, decimalSep=',') == "52,7e6" + doAssert formatEng(-52731234, 1, decimalSep=',') == "-52,7e6" + + doAssert formatEng(4100, siPrefix=true, unit="V") == "4.1 kV" + doAssert formatEng(4.1, siPrefix=true, unit="V", useUnitSpace=true) == "4.1 V" + doAssert formatEng(4.1, siPrefix=true) == "4.1" # Note lack of space + doAssert formatEng(4100, siPrefix=true) == "4.1 k" + doAssert formatEng(4.1, siPrefix=true, unit="", useUnitSpace=true) == "4.1 " # Includes space + doAssert formatEng(4100, siPrefix=true, unit="") == "4.1 k" + doAssert formatEng(4100) == "4.1e3" + doAssert formatEng(4100, unit="V", useUnitSpace=true) == "4.1e3 V" + doAssert formatEng(4100, unit="", useUnitSpace=true) == "4.1e3 " + # Don't use SI prefix as number is too big + doAssert formatEng(3.1e22, siPrefix=true, unit="a", useUnitSpace=true) == "31e21 a" + # Don't use SI prefix as number is too small + doAssert formatEng(3.1e-25, siPrefix=true, unit="A", useUnitSpace=true) == "310e-27 A" + + proc staticTests = + doAssert align("abc", 4) == " abc" + doAssert align("a", 0) == "a" + doAssert align("1232", 6) == " 1232" + doAssert align("1232", 6, '#') == "##1232" + + doAssert alignLeft("abc", 4) == "abc " + doAssert alignLeft("a", 0) == "a" + doAssert alignLeft("1232", 6) == "1232 " + doAssert alignLeft("1232", 6, '#') == "1232##" + + let + inp = """ this is a long text -- muchlongerthan10chars and here + it goes""" + outp = " this is a\nlong text\n--\nmuchlongerthan10chars\nand here\nit goes" + doAssert wordWrap(inp, 10, false) == outp + + let + longInp = """ThisIsOneVeryLongStringWhichWeWillSplitIntoEightSeparatePartsNow""" + longOutp = "ThisIsOn\neVeryLon\ngStringW\nhichWeWi\nllSplitI\nntoEight\nSeparate\nPartsNow" + doAssert wordWrap(longInp, 8, true) == longOutp + + doAssert "$animal eats $food." % ["animal", "The cat", "food", "fish"] == + "The cat eats fish." + + doAssert "-ld a-ldz -ld".replaceWord("-ld") == " a-ldz " + doAssert "-lda-ldz -ld abc".replaceWord("-ld") == "-lda-ldz abc" + + doAssert "-lda-ldz -ld abc".replaceWord("") == "-lda-ldz -ld abc" + doAssert "oo".replace("", "abc") == "abcoabcoabc" + + type MyEnum = enum enA, enB, enC, enuD, enE + doAssert parseEnum[MyEnum]("enu_D") == enuD + + doAssert parseEnum("invalid enum value", enC) == enC + + doAssert center("foo", 13) == " foo " + doAssert center("foo", 0) == "foo" + doAssert center("foo", 3, fillChar = 'a') == "foo" + doAssert center("foo", 10, fillChar = '\t') == "\t\t\tfoo\t\t\t\t" + + doAssert count("foofoofoo", "foofoo") == 1 + doAssert count("foofoofoo", "foofoo", overlapping = true) == 2 + doAssert count("foofoofoo", 'f') == 3 + doAssert count("foofoofoobar", {'f','b'}) == 4 + + doAssert strip(" foofoofoo ") == "foofoofoo" + doAssert strip("sfoofoofoos", chars = {'s'}) == "foofoofoo" + doAssert strip("barfoofoofoobar", chars = {'b', 'a', 'r'}) == "foofoofoo" + doAssert strip("stripme but don't strip this stripme", + chars = {'s', 't', 'r', 'i', 'p', 'm', 'e'}) == + " but don't strip this " + doAssert strip("sfoofoofoos", leading = false, chars = {'s'}) == "sfoofoofoo" + doAssert strip("sfoofoofoos", trailing = false, chars = {'s'}) == "foofoofoos" + + doAssert " foo\n bar".indent(4, "Q") == "QQQQ foo\nQQQQ bar" + + doAssert "abba".multiReplace(("a", "b"), ("b", "a")) == "baab" + doAssert "Hello World.".multiReplace(("ello", "ELLO"), ("World.", "PEOPLE!")) == "HELLO PEOPLE!" + doAssert "aaaa".multiReplace(("a", "aa"), ("aa", "bb")) == "aaaaaaaa" + + doAssert isAlphaAscii('r') + doAssert isAlphaAscii('A') + doAssert(not isAlphaAscii('$')) + + doAssert isAlphaAscii("Rasp") + doAssert isAlphaAscii("Args") + doAssert(not isAlphaAscii("$Tomato")) + + doAssert isAlphaNumeric('3') + doAssert isAlphaNumeric('R') + doAssert(not isAlphaNumeric('!')) + + doAssert isAlphaNumeric("34ABc") + doAssert isAlphaNumeric("Rad") + doAssert isAlphaNumeric("1234") + doAssert(not isAlphaNumeric("@nose")) + + doAssert isDigit('3') + doAssert(not isDigit('a')) + doAssert(not isDigit('%')) + + doAssert isDigit("12533") + doAssert(not isDigit("12.33")) + doAssert(not isDigit("A45b")) + + doAssert isSpaceAscii('\t') + doAssert isSpaceAscii('\l') + doAssert(not isSpaceAscii('A')) + + doAssert isSpaceAscii("\t\l \v\r\f") + doAssert isSpaceAscii(" ") + doAssert(not isSpaceAscii("ABc \td")) + + doAssert(isNilOrWhitespace("")) + doAssert(isNilOrWhitespace(" ")) + doAssert(isNilOrWhitespace("\t\l \v\r\f")) + doAssert(not isNilOrWhitespace("ABc \td")) + + doAssert isLowerAscii('a') + doAssert isLowerAscii('z') + doAssert(not isLowerAscii('A')) + doAssert(not isLowerAscii('5')) + doAssert(not isLowerAscii('&')) + + doAssert isLowerAscii("abcd") + doAssert(not isLowerAscii("abCD")) + doAssert(not isLowerAscii("33aa")) + + doAssert isUpperAscii('A') + doAssert(not isUpperAscii('b')) + doAssert(not isUpperAscii('5')) + doAssert(not isUpperAscii('%')) + + doAssert isUpperAscii("ABC") + doAssert(not isUpperAscii("AAcc")) + doAssert(not isUpperAscii("A#$")) + + doAssert rsplit("foo bar", seps=Whitespace) == @["foo", "bar"] + doAssert rsplit(" foo bar", seps=Whitespace, maxsplit=1) == @[" foo", "bar"] + doAssert rsplit(" foo bar ", seps=Whitespace, maxsplit=1) == @[" foo bar", ""] + doAssert rsplit(":foo:bar", sep=':') == @["", "foo", "bar"] + doAssert rsplit(":foo:bar", sep=':', maxsplit=2) == @["", "foo", "bar"] + doAssert rsplit(":foo:bar", sep=':', maxsplit=3) == @["", "foo", "bar"] + doAssert rsplit("foothebar", sep="the") == @["foo", "bar"] + + doAssert(unescape(r"\x013", "", "") == "\x013") + + doAssert join(["foo", "bar", "baz"]) == "foobarbaz" + doAssert join(@["foo", "bar", "baz"], ", ") == "foo, bar, baz" + doAssert join([1, 2, 3]) == "123" + doAssert join(@[1, 2, 3], ", ") == "1, 2, 3" + + doAssert """~~!!foo ~~!!bar ~~!!baz""".unindent(2, "~~!!") == "foo\nbar\nbaz" - doAssert """~~!!foo + doAssert """~~!!foo ~~!!bar ~~!!baz""".unindent(2, "~~!!aa") == "~~!!foo\n~~!!bar\n~~!!baz" - doAssert """~~foo + doAssert """~~foo ~~ bar ~~ baz""".unindent(4, "~") == "foo\n bar\n baz" - doAssert """foo + doAssert """foo bar baz """.unindent(4) == "foo\nbar\nbaz\n" - doAssert """foo + doAssert """foo bar baz """.unindent(2) == "foo\n bar\n baz\n" - doAssert """foo + doAssert """foo bar baz """.unindent(100) == "foo\nbar\nbaz\n" - doAssert """foo + doAssert """foo foo bar """.unindent() == "foo\nfoo\nbar\n" - let s = " this is an example " - let s2 = ":this;is;an:example;;" - - doAssert s.split() == @["", "this", "is", "an", "example", "", ""] - doAssert s2.split(seps={':', ';'}) == @["", "this", "is", "an", "example", "", ""] - doAssert s.split(maxsplit=4) == @["", "this", "is", "an", "example "] - doAssert s.split(' ', maxsplit=1) == @["", "this is an example "] - doAssert s.split(" ", maxsplit=4) == @["", "this", "is", "an", "example "] - - doAssert s.splitWhitespace() == @["this", "is", "an", "example"] - doAssert s.splitWhitespace(maxsplit=1) == @["this", "is an example "] - doAssert s.splitWhitespace(maxsplit=2) == @["this", "is", "an example "] - doAssert s.splitWhitespace(maxsplit=3) == @["this", "is", "an", "example "] - doAssert s.splitWhitespace(maxsplit=4) == @["this", "is", "an", "example"] - - block: # formatEng tests - doAssert formatEng(0, 2, trim=false) == "0.00" - doAssert formatEng(0, 2) == "0" - doAssert formatEng(53, 2, trim=false) == "53.00" - doAssert formatEng(0.053, 2, trim=false) == "53.00e-3" - doAssert formatEng(0.053, 4, trim=false) == "53.0000e-3" - doAssert formatEng(0.053, 4, trim=true) == "53e-3" - doAssert formatEng(0.053, 0) == "53e-3" - doAssert formatEng(52731234) == "52.731234e6" - doAssert formatEng(-52731234) == "-52.731234e6" - doAssert formatEng(52731234, 1) == "52.7e6" - doAssert formatEng(-52731234, 1) == "-52.7e6" - doAssert formatEng(52731234, 1, decimalSep=',') == "52,7e6" - doAssert formatEng(-52731234, 1, decimalSep=',') == "-52,7e6" - - doAssert formatEng(4100, siPrefix=true, unit="V") == "4.1 kV" - doAssert formatEng(4.1, siPrefix=true, unit="V") == "4.1 V" - doAssert formatEng(4.1, siPrefix=true) == "4.1" # Note lack of space - doAssert formatEng(4100, siPrefix=true) == "4.1 k" - doAssert formatEng(4.1, siPrefix=true, unit="") == "4.1 " # Includes space - doAssert formatEng(4100, siPrefix=true, unit="") == "4.1 k" - doAssert formatEng(4100) == "4.1e3" - doAssert formatEng(4100, unit="V") == "4.1e3 V" - doAssert formatEng(4100, unit="") == "4.1e3 " # Space with unit="" - # Don't use SI prefix as number is too big - doAssert formatEng(3.1e22, siPrefix=true, unit="a") == "31e21 a" - # Don't use SI prefix as number is too small - doAssert formatEng(3.1e-25, siPrefix=true, unit="A") == "310e-27 A" - - block: # startsWith / endsWith char tests - var s = "abcdef" - doAssert s.startsWith('a') - doAssert s.startsWith('b') == false - doAssert s.endsWith('f') - doAssert s.endsWith('a') == false - doAssert s.endsWith('\0') == false - - #echo("strutils tests passed") + let s = " this is an example " + let s2 = ":this;is;an:example;;" + + doAssert s.split() == @["", "this", "is", "an", "example", "", ""] + doAssert s2.split(seps={':', ';'}) == @["", "this", "is", "an", "example", "", ""] + doAssert s.split(maxsplit=4) == @["", "this", "is", "an", "example "] + doAssert s.split(' ', maxsplit=1) == @["", "this is an example "] + doAssert s.split(" ", maxsplit=4) == @["", "this", "is", "an", "example "] + + doAssert s.splitWhitespace() == @["this", "is", "an", "example"] + doAssert s.splitWhitespace(maxsplit=1) == @["this", "is an example "] + doAssert s.splitWhitespace(maxsplit=2) == @["this", "is", "an example "] + doAssert s.splitWhitespace(maxsplit=3) == @["this", "is", "an", "example "] + doAssert s.splitWhitespace(maxsplit=4) == @["this", "is", "an", "example"] + + block: # startsWith / endsWith char tests + var s = "abcdef" + doAssert s.startsWith('a') + doAssert s.startsWith('b') == false + doAssert s.endsWith('f') + doAssert s.endsWith('a') == false + doAssert s.endsWith('\0') == false + + #echo("strutils tests passed") + + nonStaticTests() + staticTests() + static: staticTests() + diff --git a/lib/pure/subexes.nim b/lib/pure/subexes.nim index 9d807abd4..8149c72cc 100644 --- a/lib/pure/subexes.nim +++ b/lib/pure/subexes.nim @@ -31,8 +31,6 @@ type SubexError* = object of ValueError ## exception that is raised for ## an invalid subex -{.deprecated: [EInvalidSubex: SubexError].} - proc raiseInvalidFormat(msg: string) {.noinline.} = raise newException(SubexError, "invalid format string: " & msg) @@ -44,7 +42,6 @@ type else: f: cstring num, i, lineLen: int -{.deprecated: [TFormatParser: FormatParser].} template call(x: untyped): untyped = p.i = i diff --git a/lib/pure/sugar.nim b/lib/pure/sugar.nim new file mode 100644 index 000000000..258b40191 --- /dev/null +++ b/lib/pure/sugar.nim @@ -0,0 +1,200 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2015 Dominik Picheta +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module implements nice syntactic sugar based on Nim's +## macro system. + +import macros + +proc createProcType(p, b: NimNode): NimNode {.compileTime.} = + #echo treeRepr(p) + #echo treeRepr(b) + result = newNimNode(nnkProcTy) + var formalParams = newNimNode(nnkFormalParams) + + formalParams.add b + + case p.kind + of nnkPar, nnkTupleConstr: + for i in 0 ..< p.len: + let ident = p[i] + var identDefs = newNimNode(nnkIdentDefs) + case ident.kind + of nnkExprColonExpr: + identDefs.add ident[0] + identDefs.add ident[1] + else: + identDefs.add newIdentNode("i" & $i) + identDefs.add(ident) + identDefs.add newEmptyNode() + formalParams.add identDefs + else: + var identDefs = newNimNode(nnkIdentDefs) + identDefs.add newIdentNode("i0") + identDefs.add(p) + identDefs.add newEmptyNode() + formalParams.add identDefs + + result.add formalParams + result.add newEmptyNode() + #echo(treeRepr(result)) + #echo(result.toStrLit()) + +macro `=>`*(p, b: untyped): untyped = + ## Syntax sugar for anonymous procedures. + ## + ## .. code-block:: nim + ## + ## proc passTwoAndTwo(f: (int, int) -> int): int = + ## f(2, 2) + ## + ## passTwoAndTwo((x, y) => x + y) # 4 + + #echo treeRepr(p) + #echo(treeRepr(b)) + var params: seq[NimNode] = @[newIdentNode("auto")] + + case p.kind + of nnkPar, nnkTupleConstr: + for c in children(p): + var identDefs = newNimNode(nnkIdentDefs) + case c.kind + of nnkExprColonExpr: + identDefs.add(c[0]) + identDefs.add(c[1]) + identDefs.add(newEmptyNode()) + of nnkIdent: + identDefs.add(c) + identDefs.add(newIdentNode("auto")) + identDefs.add(newEmptyNode()) + of nnkInfix: + if c[0].kind == nnkIdent and c[0].ident == !"->": + var procTy = createProcType(c[1], c[2]) + params[0] = procTy[0][0] + for i in 1 ..< procTy[0].len: + params.add(procTy[0][i]) + else: + error("Expected proc type (->) got (" & $c[0].ident & ").") + break + else: + echo treeRepr c + error("Incorrect procedure parameter list.") + params.add(identDefs) + of nnkIdent: + var identDefs = newNimNode(nnkIdentDefs) + identDefs.add(p) + identDefs.add(newIdentNode("auto")) + identDefs.add(newEmptyNode()) + params.add(identDefs) + of nnkInfix: + if p[0].kind == nnkIdent and p[0].ident == !"->": + var procTy = createProcType(p[1], p[2]) + params[0] = procTy[0][0] + for i in 1 ..< procTy[0].len: + params.add(procTy[0][i]) + else: + error("Expected proc type (->) got (" & $p[0].ident & ").") + else: + error("Incorrect procedure parameter list.") + result = newProc(params = params, body = b, procType = nnkLambda) + #echo(result.treeRepr) + #echo(result.toStrLit()) + #return result # TODO: Bug? + +macro `->`*(p, b: untyped): untyped = + ## Syntax sugar for procedure types. + ## + ## .. code-block:: nim + ## + ## proc pass2(f: (float, float) -> float): float = + ## f(2, 2) + ## + ## # is the same as: + ## + ## proc pass2(f: proc (x, y: float): float): float = + ## f(2, 2) + + result = createProcType(p, b) + +type ListComprehension = object +var lc*: ListComprehension + +macro `[]`*(lc: ListComprehension, comp, typ: untyped): untyped = + ## List comprehension, returns a sequence. `comp` is the actual list + ## comprehension, for example ``x | (x <- 1..10, x mod 2 == 0)``. `typ` is + ## the type that will be stored inside the result seq. + ## + ## .. code-block:: nim + ## + ## echo lc[x | (x <- 1..10, x mod 2 == 0), int] + ## + ## const n = 20 + ## echo lc[(x,y,z) | (x <- 1..n, y <- x..n, z <- y..n, x*x + y*y == z*z), + ## tuple[a,b,c: int]] + + expectLen(comp, 3) + expectKind(comp, nnkInfix) + expectKind(comp[0], nnkIdent) + assert($comp[0].ident == "|") + + result = newCall( + newDotExpr( + newIdentNode("result"), + newIdentNode("add")), + comp[1]) + + for i in countdown(comp[2].len-1, 0): + let x = comp[2][i] + expectMinLen(x, 1) + if x[0].kind == nnkIdent and $x[0].ident == "<-": + expectLen(x, 3) + result = newNimNode(nnkForStmt).add(x[1], x[2], result) + else: + result = newIfStmt((x, result)) + + result = newNimNode(nnkCall).add( + newNimNode(nnkPar).add( + newNimNode(nnkLambda).add( + newEmptyNode(), + newEmptyNode(), + newEmptyNode(), + newNimNode(nnkFormalParams).add( + newNimNode(nnkBracketExpr).add( + newIdentNode("seq"), + typ)), + newEmptyNode(), + newEmptyNode(), + newStmtList( + newAssignment( + newIdentNode("result"), + newNimNode(nnkPrefix).add( + newIdentNode("@"), + newNimNode(nnkBracket))), + result)))) + + +macro dump*(x: typed): untyped = + ## Dumps the content of an expression, useful for debugging. + ## It accepts any expression and prints a textual representation + ## of the tree representing the expression - as it would appear in + ## source code - together with the value of the expression. + ## + ## As an example, + ## + ## .. code-block:: nim + ## let + ## x = 10 + ## y = 20 + ## dump(x + y) + ## + ## will print ``x + y = 30``. + let s = x.toStrLit + let r = quote do: + debugEcho `s`, " = ", `x` + return r diff --git a/lib/pure/terminal.nim b/lib/pure/terminal.nim index f15cee66a..fcca4d5d7 100644 --- a/lib/pure/terminal.nim +++ b/lib/pure/terminal.nim @@ -17,6 +17,30 @@ ## ``showCursor`` before quitting. import macros +import strformat +from strutils import toLowerAscii +import colors + +const + hasThreadSupport = compileOption("threads") + +when not hasThreadSupport: + import tables + var + colorsFGCache = initTable[Color, string]() + colorsBGCache = initTable[Color, string]() + styleCache = initTable[int, string]() + +var + trueColorIsSupported: bool + trueColorIsEnabled: bool + fgSetColor: bool + +const + fgPrefix = "\x1b[38;2;" + bgPrefix = "\x1b[48;2;" + ansiResetCode* = "\e[0m" + stylePrefix = "\e[" when defined(windows): import winlean, os @@ -34,6 +58,8 @@ when defined(windows): FOREGROUND_RGB = FOREGROUND_RED or FOREGROUND_GREEN or FOREGROUND_BLUE BACKGROUND_RGB = BACKGROUND_RED or BACKGROUND_GREEN or BACKGROUND_BLUE + ENABLE_VIRTUAL_TERMINAL_PROCESSING = 0x0004 + type SHORT = int16 COORD = object @@ -124,6 +150,12 @@ when defined(windows): wAttributes: int16): WINBOOL{. stdcall, dynlib: "kernel32", importc: "SetConsoleTextAttribute".} + proc getConsoleMode(hConsoleHandle: Handle, dwMode: ptr DWORD): WINBOOL{. + stdcall, dynlib: "kernel32", importc: "GetConsoleMode".} + + proc setConsoleMode(hConsoleHandle: Handle, dwMode: DWORD): WINBOOL{. + stdcall, dynlib: "kernel32", importc: "SetConsoleMode".} + var hStdout: Handle # = createFile("CONOUT$", GENERIC_WRITE, 0, nil, # OPEN_ALWAYS, 0, 0) @@ -274,7 +306,7 @@ proc setCursorPos*(f: File, x, y: int) = let h = conHandle(f) setCursorPos(h, x, y) else: - f.write("\e[" & $y & ';' & $x & 'f') + f.write(fmt"{stylePrefix}{y};{x}f") proc setCursorXPos*(f: File, x: int) = ## Sets the terminal's cursor to the x position. @@ -289,7 +321,7 @@ proc setCursorXPos*(f: File, x: int) = if setConsoleCursorPosition(h, origin) == 0: raiseOSError(osLastError()) else: - f.write("\e[" & $x & 'G') + f.write(fmt"{stylePrefix}{x}G") when defined(windows): proc setCursorYPos*(f: File, y: int) = @@ -326,7 +358,7 @@ proc cursorDown*(f: File, count=1) = inc(p.y, count) setCursorPos(h, p.x, p.y) else: - f.write("\e[" & $count & 'B') + f.write(fmt"{stylePrefix}{count}B") proc cursorForward*(f: File, count=1) = ## Moves the cursor forward by `count` columns. @@ -336,7 +368,7 @@ proc cursorForward*(f: File, count=1) = inc(p.x, count) setCursorPos(h, p.x, p.y) else: - f.write("\e[" & $count & 'C') + f.write(fmt"{stylePrefix}{count}C") proc cursorBackward*(f: File, count=1) = ## Moves the cursor backward by `count` columns. @@ -346,7 +378,7 @@ proc cursorBackward*(f: File, count=1) = dec(p.x, count) setCursorPos(h, p.x, p.y) else: - f.write("\e[" & $count & 'D') + f.write(fmt"{stylePrefix}{count}D") when true: discard @@ -391,12 +423,11 @@ proc eraseLine*(f: File) = origin.X = 0'i16 if setConsoleCursorPosition(h, origin) == 0: raiseOSError(osLastError()) - var ht: DWORD = scrbuf.dwSize.Y - origin.Y var wt: DWORD = scrbuf.dwSize.X - origin.X - if fillConsoleOutputCharacter(h, ' ', ht*wt, + if fillConsoleOutputCharacter(h, ' ', wt, origin, addr(numwrote)) == 0: raiseOSError(osLastError()) - if fillConsoleOutputAttribute(h, scrbuf.wAttributes, ht * wt, + if fillConsoleOutputAttribute(h, scrbuf.wAttributes, wt, scrbuf.dwCursorPosition, addr(numwrote)) == 0: raiseOSError(osLastError()) else: @@ -433,7 +464,7 @@ proc resetAttributes*(f: File) = else: discard setConsoleTextAttribute(hStdout, oldStdoutAttr) else: - f.write("\e[0m") + f.write(ansiResetCode) type Style* = enum ## different styles for text output @@ -449,9 +480,25 @@ type when not defined(windows): var - # XXX: These better be thread-local - gFG = 0 - gBG = 0 + gFG {.threadvar.}: int + gBG {.threadvar.}: int + +proc ansiStyleCode*(style: int): string = + when hasThreadSupport: + result = fmt"{stylePrefix}{style}m" + else: + if styleCache.hasKey(style): + result = styleCache[style] + else: + result = fmt"{stylePrefix}{style}m" + styleCache[style] = result + +template ansiStyleCode*(style: Style): string = + ansiStyleCode(style.int) + +# The styleCache can be skipped when `style` is known at compile-time +template ansiStyleCode*(style: static[Style]): string = + (static(stylePrefix & $style.int & "m")) proc setStyle*(f: File, style: set[Style]) = ## Sets the terminal style. @@ -466,7 +513,7 @@ proc setStyle*(f: File, style: set[Style]) = discard setConsoleTextAttribute(h, old or a) else: for s in items(style): - f.write("\e[" & $ord(s) & 'm') + f.write(ansiStyleCode(s)) proc writeStyled*(txt: string, style: set[Style] = {styleBright}) = ## Writes the text `txt` in a given `style` to stdout. @@ -480,9 +527,9 @@ proc writeStyled*(txt: string, style: set[Style] = {styleBright}) = stdout.write(txt) stdout.resetAttributes() if gFG != 0: - stdout.write("\e[" & $ord(gFG) & 'm') + stdout.write(ansiStyleCode(gFG)) if gBG != 0: - stdout.write("\e[" & $ord(gBG) & 'm') + stdout.write(ansiStyleCode(gBG)) type ForegroundColor* = enum ## terminal's foreground colors @@ -513,8 +560,8 @@ proc setForegroundColor*(f: File, fg: ForegroundColor, bright=false) = when defined(windows): let h = conHandle(f) var old = getAttributes(h) and not FOREGROUND_RGB - if bright: - old = old or FOREGROUND_INTENSITY + old = if bright: old or FOREGROUND_INTENSITY + else: old and not(FOREGROUND_INTENSITY) const lookup: array[ForegroundColor, int] = [ 0, (FOREGROUND_RED), @@ -528,15 +575,15 @@ proc setForegroundColor*(f: File, fg: ForegroundColor, bright=false) = else: gFG = ord(fg) if bright: inc(gFG, 60) - f.write("\e[" & $gFG & 'm') + f.write(ansiStyleCode(gFG)) proc setBackgroundColor*(f: File, bg: BackgroundColor, bright=false) = ## Sets the terminal's background color. when defined(windows): let h = conHandle(f) var old = getAttributes(h) and not BACKGROUND_RGB - if bright: - old = old or BACKGROUND_INTENSITY + old = if bright: old or BACKGROUND_INTENSITY + else: old and not(BACKGROUND_INTENSITY) const lookup: array[BackgroundColor, int] = [ 0, (BACKGROUND_RED), @@ -550,7 +597,64 @@ proc setBackgroundColor*(f: File, bg: BackgroundColor, bright=false) = else: gBG = ord(bg) if bright: inc(gBG, 60) - f.write("\e[" & $gBG & 'm') + f.write(ansiStyleCode(gBG)) + +proc ansiForegroundColorCode*(fg: ForegroundColor, bright=false): string = + var style = ord(fg) + if bright: inc(style, 60) + return ansiStyleCode(style) + +template ansiForegroundColorCode*(fg: static[ForegroundColor], + bright: static[bool] = false): string = + ansiStyleCode(fg.int + bright.int * 60) + +proc ansiForegroundColorCode*(color: Color): string = + when hasThreadSupport: + let rgb = extractRGB(color) + result = fmt"{fgPrefix}{rgb.r};{rgb.g};{rgb.b}m" + else: + if colorsFGCache.hasKey(color): + result = colorsFGCache[color] + else: + let rgb = extractRGB(color) + result = fmt"{fgPrefix}{rgb.r};{rgb.g};{rgb.b}m" + colorsFGCache[color] = result + +template ansiForegroundColorCode*(color: static[Color]): string = + const rgb = extractRGB(color) + (static(fmt"{fgPrefix}{rgb.r};{rgb.g};{rgb.b}m")) + +proc ansiBackgroundColorCode*(color: Color): string = + when hasThreadSupport: + let rgb = extractRGB(color) + result = fmt"{bgPrefix}{rgb.r};{rgb.g};{rgb.b}m" + else: + if colorsBGCache.hasKey(color): + result = colorsBGCache[color] + else: + let rgb = extractRGB(color) + result = fmt"{bgPrefix}{rgb.r};{rgb.g};{rgb.b}m" + colorsFGCache[color] = result + +template ansiBackgroundColorCode*(color: static[Color]): string = + const rgb = extractRGB(color) + (static(fmt"{bgPrefix}{rgb.r};{rgb.g};{rgb.b}m")) + +proc setForegroundColor*(f: File, color: Color) = + ## Sets the terminal's foreground true color. + if trueColorIsEnabled: + f.write(ansiForegroundColorCode(color)) + +proc setBackgroundColor*(f: File, color: Color) = + ## Sets the terminal's background true color. + if trueColorIsEnabled: + f.write(ansiBackgroundColorCode(color)) + +proc setTrueColor(f: File, color: Color) = + if fgSetColor: + setForegroundColor(f, color) + else: + setBackgroundColor(f, color) proc isatty*(f: File): bool = ## Returns true if `f` is associated with a terminal device. @@ -565,7 +669,9 @@ proc isatty*(f: File): bool = type TerminalCmd* = enum ## commands that can be expressed as arguments - resetStyle ## reset attributes + resetStyle, ## reset attributes + fgColor, ## set foreground's true color + bgColor ## set background's true color template styledEchoProcessArg(f: File, s: string) = write f, s template styledEchoProcessArg(f: File, style: Style) = setStyle(f, {style}) @@ -574,9 +680,15 @@ template styledEchoProcessArg(f: File, color: ForegroundColor) = setForegroundColor f, color template styledEchoProcessArg(f: File, color: BackgroundColor) = setBackgroundColor f, color +template styledEchoProcessArg(f: File, color: Color) = + setTrueColor f, color template styledEchoProcessArg(f: File, cmd: TerminalCmd) = when cmd == resetStyle: resetAttributes(f) + when cmd == fgColor: + fgSetColor = true + when cmd == bgColor: + fgSetColor = false macro styledWriteLine*(f: File, m: varargs[typed]): untyped = ## Similar to ``writeLine``, but treating terminal style arguments specially. @@ -634,10 +746,7 @@ proc getch*(): char = doAssert(readConsoleInput(fd, addr(keyEvent), 1, addr(numRead)) != 0) if numRead == 0 or keyEvent.eventType != 1 or keyEvent.bKeyDown == 0: continue - if keyEvent.uChar == 0: - return char(keyEvent.wVirtualKeyCode) - else: - return char(keyEvent.uChar) + return char(keyEvent.uChar) else: let fd = getFileHandle(stdin) var oldMode: Termios @@ -646,13 +755,67 @@ proc getch*(): char = result = stdin.readChar() discard fd.tcsetattr(TCSADRAIN, addr oldMode) +when defined(windows): + from unicode import toUTF8, Rune, runeLenAt + + proc readPasswordFromStdin*(prompt: string, password: var TaintedString): + bool {.tags: [ReadIOEffect, WriteIOEffect].} = + ## Reads a `password` from stdin without printing it. `password` must not + ## be ``nil``! Returns ``false`` if the end of the file has been reached, + ## ``true`` otherwise. + password.string.setLen(0) + stdout.write(prompt) + while true: + let c = getch() + case c.char + of '\r', chr(0xA): + break + of '\b': + # ensure we delete the whole UTF-8 character: + var i = 0 + var x = 1 + while i < password.len: + x = runeLenAt(password.string, i) + inc i, x + password.string.setLen(max(password.len - x, 0)) + of chr(0x0): + # modifier key - ignore - for details see + # https://github.com/nim-lang/Nim/issues/7764 + continue + else: + password.string.add(toUTF8(c.Rune)) + stdout.write "\n" + +else: + import termios + + proc readPasswordFromStdin*(prompt: string, password: var TaintedString): + bool {.tags: [ReadIOEffect, WriteIOEffect].} = + password.string.setLen(0) + let fd = stdin.getFileHandle() + var cur, old: Termios + discard fd.tcgetattr(cur.addr) + old = cur + cur.c_lflag = cur.c_lflag and not Cflag(ECHO) + discard fd.tcsetattr(TCSADRAIN, cur.addr) + stdout.write prompt + result = stdin.readLine(password) + stdout.write "\n" + discard fd.tcsetattr(TCSADRAIN, old.addr) + +proc readPasswordFromStdin*(prompt = "password: "): TaintedString = + ## Reads a password from stdin without printing it. + result = TaintedString("") + discard readPasswordFromStdin(prompt, result) + + # Wrappers assuming output to stdout: template hideCursor*() = hideCursor(stdout) template showCursor*() = showCursor(stdout) template setCursorPos*(x, y: int) = setCursorPos(stdout, x, y) template setCursorXPos*(x: int) = setCursorXPos(stdout, x) when defined(windows): - template setCursorYPos(x: int) = setCursorYPos(stdout, x) + template setCursorYPos*(x: int) = setCursorYPos(stdout, x) template cursorUp*(count=1) = cursorUp(stdout, count) template cursorDown*(count=1) = cursorDown(stdout, count) template cursorForward*(count=1) = cursorForward(stdout, count) @@ -665,6 +828,10 @@ template setForegroundColor*(fg: ForegroundColor, bright=false) = setForegroundColor(stdout, fg, bright) template setBackgroundColor*(bg: BackgroundColor, bright=false) = setBackgroundColor(stdout, bg, bright) +template setForegroundColor*(color: Color) = + setForegroundColor(stdout, color) +template setBackgroundColor*(color: Color) = + setBackgroundColor(stdout, color) proc resetAttributes*() {.noconv.} = ## Resets all attributes on stdout. ## It is advisable to register this as a quit proc with @@ -680,3 +847,54 @@ when not defined(testing) and isMainModule: stdout.setForeGroundColor(fgBlue) stdout.writeLine("ordinary text") stdout.resetAttributes() + +proc isTrueColorSupported*(): bool = + ## Returns true if a terminal supports true color. + return trueColorIsSupported + +when defined(windows): + import os + +proc enableTrueColors*() = + ## Enable true color. + when defined(windows): + var + ver: OSVERSIONINFO + ver.dwOSVersionInfoSize = sizeof(ver).DWORD + let res = getVersionExW(addr ver) + if res == 0: + trueColorIsSupported = false + else: + trueColorIsSupported = ver.dwMajorVersion > 10 or + (ver.dwMajorVersion == 10 and (ver.dwMinorVersion > 0 or + (ver.dwMinorVersion == 0 and ver.dwBuildNumber >= 10586))) + if not trueColorIsSupported: + trueColorIsSupported = getEnv("ANSICON_DEF").len > 0 + + if trueColorIsSupported: + if getEnv("ANSICON_DEF").len == 0: + var mode: DWORD = 0 + if getConsoleMode(getStdHandle(STD_OUTPUT_HANDLE), addr(mode)) != 0: + mode = mode or ENABLE_VIRTUAL_TERMINAL_PROCESSING + if setConsoleMode(getStdHandle(STD_OUTPUT_HANDLE), mode) != 0: + trueColorIsEnabled = true + else: + trueColorIsEnabled = false + else: + trueColorIsEnabled = true + else: + trueColorIsSupported = string(getEnv("COLORTERM")).toLowerAscii() in ["truecolor", "24bit"] + trueColorIsEnabled = trueColorIsSupported + +proc disableTrueColors*() = + ## Disable true color. + when defined(windows): + if trueColorIsSupported: + if getEnv("ANSICON_DEF").len == 0: + var mode: DWORD = 0 + if getConsoleMode(getStdHandle(STD_OUTPUT_HANDLE), addr(mode)) != 0: + mode = mode and not ENABLE_VIRTUAL_TERMINAL_PROCESSING + discard setConsoleMode(getStdHandle(STD_OUTPUT_HANDLE), mode) + trueColorIsEnabled = false + else: + trueColorIsEnabled = false diff --git a/lib/pure/times.nim b/lib/pure/times.nim index 42e89e7ce..60b362665 100644 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -1,16 +1,18 @@ # # # Nim's Runtime Library -# (c) Copyright 2015 Andreas Rumpf +# (c) Copyright 2017 Nim contributors # # See the file "copying.txt", included in this # distribution, for details about the copyright. # -## This module contains routines and types for dealing with time. -## This module is available for the `JavaScript target -## <backends.html#the-javascript-target>`_. The proleptic Gregorian calendar is the only calendar supported. +## This module contains routines and types for dealing with time using a proleptic Gregorian calendar. +## It's is available for the `JavaScript target <backends.html#the-javascript-target>`_. +## +## The types uses nanosecond time resolution, but the underlying resolution used by ``getTime()`` +## depends on the platform and backend (JS is limited to millisecond precision). ## ## Examples: ## @@ -25,25 +27,48 @@ ## echo "My formatted time: ", format(now(), "d MMMM yyyy HH:mm") ## echo "Using predefined formats: ", getClockStr(), " ", getDateStr() ## -## echo "epochTime() float value: ", epochTime() -## echo "cpuTime() float value: ", cpuTime() +## echo "cpuTime() float value: ", cpuTime() ## echo "An hour from now : ", now() + 1.hours -## echo "An hour from (UTC) now: ", getTime().utc + initInterval(0,0,0,1) +## echo "An hour from (UTC) now: ", getTime().utc + initDuration(hours = 1) {.push debugger:off.} # the user does not want to trace a part # of the standard library! import - strutils, parseutils + strutils, parseutils, algorithm, math include "system/inclrtl" +# This is really bad, but overflow checks are broken badly for +# ints on the JS backend. See #6752. +when defined(JS): + {.push overflowChecks: off.} + proc `*`(a, b: int64): int64 = + system.`* `(a, b) + proc `*`(a, b: int): int = + system.`* `(a, b) + proc `+`(a, b: int64): int64 = + system.`+ `(a, b) + proc `+`(a, b: int): int = + system.`+ `(a, b) + proc `-`(a, b: int64): int64 = + system.`- `(a, b) + proc `-`(a, b: int): int = + system.`- `(a, b) + proc inc(a: var int, b: int) = + system.inc(a, b) + proc inc(a: var int64, b: int) = + system.inc(a, b) + {.pop.} + when defined(posix): import posix type CTime = posix.Time - proc posix_gettimeofday(tp: var Timeval, unused: pointer = nil) {. + var CLOCK_REALTIME {.importc: "CLOCK_REALTIME", header: "<time.h>".}: Clockid + + proc gettimeofday(tp: var Timeval, unused: pointer = nil) {. importc: "gettimeofday", header: "<sys/time.h>".} when not defined(freebsd) and not defined(netbsd) and not defined(openbsd): @@ -53,8 +78,11 @@ when defined(posix): elif defined(windows): import winlean - # newest version of Visual C++ defines time_t to be of 64 bits - type CTime {.importc: "time_t", header: "<time.h>".} = distinct int64 + when defined(i386) and defined(gcc): + type CTime {.importc: "time_t", header: "<time.h>".} = distinct int32 + else: + # newest version of Visual C++ defines time_t to be of 64 bits + type CTime {.importc: "time_t", header: "<time.h>".} = distinct int64 # visual c's c runtime exposes these under a different name var timezone {.importc: "_timezone", header: "<time.h>".}: int @@ -71,16 +99,11 @@ type MinuteRange* = range[0..59] SecondRange* = range[0..60] YeardayRange* = range[0..365] + NanosecondRange* = range[0..999_999_999] - TimeImpl = int64 - - Time* = distinct TimeImpl ## Represents a point in time. - ## This is currently implemented as a ``int64`` representing - ## seconds since ``1970-01-01T00:00:00Z``, but don't - ## rely on this knowledge because it might change - ## in the future to allow for higher precision. - ## Use the procs ``toUnix`` and ``fromUnix`` to - ## work with unix timestamps instead. + Time* = object ## Represents a point in time. + seconds: int64 + nanosecond: NanosecondRange DateTime* = object of RootObj ## Represents a time in different parts. ## Although this type can represent leap @@ -89,6 +112,8 @@ type ## but the ``DateTime``'s returned by ## procedures in this module will never have ## a leap second. + nanosecond*: NanosecondRange ## The number of nanoseconds after the second, + ## in the range 0 to 999_999_999. second*: SecondRange ## The number of seconds after the minute, ## normally in the range 0 to 59, but can ## be up to 60 to allow for a leap second. @@ -111,33 +136,57 @@ type ## of the one in a formatted offset string like ``+01:00`` ## (which would be parsed into the UTC offset ``-3600``). - TimeInterval* = object ## Represents a duration of time. Can be used to add and subtract - ## from a ``DateTime`` or ``Time``. - ## Note that a ``TimeInterval`` doesn't represent a fixed duration of time, + TimeInterval* = object ## Represents a non-fixed duration of time. Can be used to add and subtract + ## non-fixed time units from a ``DateTime`` or ``Time``. + ## ``TimeInterval`` doesn't represent a fixed duration of time, ## since the duration of some units depend on the context (e.g a year ## can be either 365 or 366 days long). The non-fixed time units are years, ## months and days. + + nanoseconds*: int ## The number of nanoseconds + microseconds*: int ## The number of microseconds milliseconds*: int ## The number of milliseconds - seconds*: int ## The number of seconds - minutes*: int ## The number of minutes - hours*: int ## The number of hours - days*: int ## The number of days - months*: int ## The number of months - years*: int ## The number of years + seconds*: int ## The number of seconds + minutes*: int ## The number of minutes + hours*: int ## The number of hours + days*: int ## The number of days + weeks*: int ## The number of weeks + months*: int ## The number of months + years*: int ## The number of years + + Duration* = object ## Represents a fixed duration of time. + ## Uses the same time resolution as ``Time``. + ## This type should be prefered over ``TimeInterval`` unless + ## non-static time units is needed. + seconds: int64 + nanosecond: NanosecondRange + + TimeUnit* = enum ## Different units of time. + Nanoseconds, Microseconds, Milliseconds, Seconds, Minutes, Hours, Days, Weeks, Months, Years + + FixedTimeUnit* = range[Nanoseconds..Weeks] ## Subrange of ``TimeUnit`` that only includes units of fixed duration. + ## These are the units that can be represented by a ``Duration``. Timezone* = object ## Timezone interface for supporting ``DateTime``'s of arbritary timezones. ## The ``times`` module only supplies implementations for the systems local time and UTC. ## The members ``zoneInfoFromUtc`` and ``zoneInfoFromTz`` should not be accessed directly ## and are only exported so that ``Timezone`` can be implemented by other modules. - zoneInfoFromUtc*: proc (time: Time): ZonedTime {.nimcall, tags: [], raises: [], benign .} - zoneInfoFromTz*: proc (adjTime: Time): ZonedTime {.nimcall, tags: [], raises: [], benign .} + zoneInfoFromUtc*: proc (time: Time): ZonedTime {.tags: [], raises: [], benign.} + zoneInfoFromTz*: proc (adjTime: Time): ZonedTime {.tags: [], raises: [], benign.} name*: string ## The name of the timezone, f.ex 'Europe/Stockholm' or 'Etc/UTC'. Used for checking equality. ## Se also: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones - ZonedTime* = object ## Represents a zooned instant in time that is not associated with any calendar. + + ZonedTime* = object ## Represents a zoned instant in time that is not associated with any calendar. ## This type is only used for implementing timezones. - adjTime*: Time ## Time adjusted to a timezone. - utcOffset*: int - isDst*: bool + adjTime*: Time ## Time adjusted to a timezone. + utcOffset*: int ## Offset from UTC in seconds. + ## The point in time represented by ``ZonedTime`` is ``adjTime + utcOffset.seconds``. + isDst*: bool ## Determines whether DST is in effect. + + DurationParts* = array[FixedTimeUnit, int64] # Array of Duration parts starts + TimeIntervalParts* = array[TimeUnit, int] # Array of Duration parts starts + + {.deprecated: [TMonth: Month, TWeekDay: WeekDay, TTime: Time, TTimeInterval: TimeInterval, TTimeInfo: DateTime, TimeInfo: DateTime].} @@ -147,14 +196,141 @@ const secondsInHour = 60*60 secondsInDay = 60*60*24 minutesInHour = 60 + rateDiff = 10000000'i64 # 100 nsecs + # The number of hectonanoseconds between 1601/01/01 (windows epoch) + # and 1970/01/01 (unix epoch). + epochDiff = 116444736000000000'i64 + +const unitWeights: array[FixedTimeUnit, int64] = [ + 1'i64, + 1000, + 1_000_000, + 1e9.int64, + secondsInMin * 1e9.int64, + secondsInHour * 1e9.int64, + secondsInDay * 1e9.int64, + 7 * secondsInDay * 1e9.int64, +] + +proc convert*[T: SomeInteger](unitFrom, unitTo: FixedTimeUnit, quantity: T): T {.inline.} = + ## Convert a quantity of some duration unit to another duration unit. + runnableExamples: + doAssert convert(Days, Hours, 2) == 48 + doAssert convert(Days, Weeks, 13) == 1 # Truncated + doAssert convert(Seconds, Milliseconds, -1) == -1000 + if unitFrom < unitTo: + (quantity div (unitWeights[unitTo] div unitWeights[unitFrom])).T + else: + ((unitWeights[unitFrom] div unitWeights[unitTo]) * quantity).T + +proc normalize[T: Duration|Time](seconds, nanoseconds: int64): T = + ## Normalize a (seconds, nanoseconds) pair and return it as either + ## a ``Duration`` or ``Time``. A normalized ``Duration|Time`` has a + ## positive nanosecond part in the range ``NanosecondRange``. + result.seconds = seconds + convert(Nanoseconds, Seconds, nanoseconds) + var nanosecond = nanoseconds mod convert(Seconds, Nanoseconds, 1) + if nanosecond < 0: + nanosecond += convert(Seconds, Nanoseconds, 1) + result.seconds -= 1 + result.nanosecond = nanosecond.int + +# Forward declarations +proc utcZoneInfoFromUtc(time: Time): ZonedTime {.tags: [], raises: [], benign .} +proc utcZoneInfoFromTz(adjTime: Time): ZonedTime {.tags: [], raises: [], benign .} +proc localZoneInfoFromUtc(time: Time): ZonedTime {.tags: [], raises: [], benign .} +proc localZoneInfoFromTz(adjTime: Time): ZonedTime {.tags: [], raises: [], benign .} +proc initTime*(unix: int64, nanosecond: NanosecondRange): Time + {.tags: [], raises: [], benign noSideEffect.} + +proc initDuration*(nanoseconds, microseconds, milliseconds, + seconds, minutes, hours, days, weeks: int64 = 0): Duration + {.tags: [], raises: [], benign noSideEffect.} + +proc nanosecond*(time: Time): NanosecondRange = + ## Get the fractional part of a ``Time`` as the number + ## of nanoseconds of the second. + time.nanosecond + + +proc weeks*(dur: Duration): int64 {.inline.} = + ## Number of whole weeks represented by the duration. + convert(Seconds, Weeks, dur.seconds) + +proc days*(dur: Duration): int64 {.inline.} = + ## Number of whole days represented by the duration. + convert(Seconds, Days, dur.seconds) + +proc minutes*(dur: Duration): int64 {.inline.} = + ## Number of whole minutes represented by the duration. + convert(Seconds, Minutes, dur.seconds) + +proc hours*(dur: Duration): int64 {.inline.} = + ## Number of whole hours represented by the duration. + convert(Seconds, Hours, dur.seconds) + +proc seconds*(dur: Duration): int64 {.inline.} = + ## Number of whole seconds represented by the duration. + dur.seconds + +proc milliseconds*(dur: Duration): int {.inline.} = + ## Number of whole milliseconds represented by the **fractional** + ## part of the duration. + runnableExamples: + let dur = initDuration(seconds = 1, milliseconds = 1) + doAssert dur.milliseconds == 1 + convert(Nanoseconds, Milliseconds, dur.nanosecond) + +proc microseconds*(dur: Duration): int {.inline.} = + ## Number of whole microseconds represented by the **fractional** + ## part of the duration. + runnableExamples: + let dur = initDuration(seconds = 1, microseconds = 1) + doAssert dur.microseconds == 1 + convert(Nanoseconds, Microseconds, dur.nanosecond) + +proc nanoseconds*(dur: Duration): int {.inline.} = + ## Number of whole nanoseconds represented by the **fractional** + ## part of the duration. + runnableExamples: + let dur = initDuration(seconds = 1, nanoseconds = 1) + doAssert dur.nanoseconds == 1 + dur.nanosecond + +proc fractional*(dur: Duration): Duration {.inline.} = + ## The fractional part of duration, as a duration. + runnableExamples: + let dur = initDuration(seconds = 1, nanoseconds = 5) + doAssert dur.fractional == initDuration(nanoseconds = 5) + initDuration(nanoseconds = dur.nanosecond) + proc fromUnix*(unix: int64): Time {.benign, tags: [], raises: [], noSideEffect.} = ## Convert a unix timestamp (seconds since ``1970-01-01T00:00:00Z``) to a ``Time``. - Time(unix) + runnableExamples: + doAssert $fromUnix(0).utc == "1970-01-01T00:00:00+00:00" + initTime(unix, 0) proc toUnix*(t: Time): int64 {.benign, tags: [], raises: [], noSideEffect.} = ## Convert ``t`` to a unix timestamp (seconds since ``1970-01-01T00:00:00Z``). - t.int64 + runnableExamples: + doAssert fromUnix(0).toUnix() == 0 + + t.seconds + +proc fromWinTime*(win: int64): Time = + ## Convert a Windows file time (100-nanosecond intervals since ``1601-01-01T00:00:00Z``) + ## to a ``Time``. + let hnsecsSinceEpoch = (win - epochDiff) + var seconds = hnsecsSinceEpoch div rateDiff + var nanos = ((hnsecsSinceEpoch mod rateDiff) * 100).int + if nanos < 0: + nanos += convert(Seconds, Nanoseconds, 1) + seconds -= 1 + result = initTime(seconds, nanos) + +proc toWinTime*(t: Time): int64 = + ## Convert ``t`` to a Windows file time (100-nanosecond intervals since ``1601-01-01T00:00:00Z``). + result = t.seconds * rateDiff + epochDiff + t.nanosecond div 100 proc isLeapYear*(year: int): bool = ## Returns true if ``year`` is a leap year. @@ -174,7 +350,7 @@ proc getDaysInYear*(year: int): int = proc assertValidDate(monthday: MonthdayRange, month: Month, year: int) {.inline.} = assert monthday <= getDaysInMonth(month, year), - $year & "-" & $ord(month) & "-" & $monthday & " is not a valid date" + $year & "-" & intToStr(ord(month), 2) & "-" & $monthday & " is not a valid date" proc toEpochDay(monthday: MonthdayRange, month: Month, year: int): int64 = ## Get the epoch day from a year/month/day date. @@ -231,53 +407,280 @@ proc getDayOfWeek*(monthday: MonthdayRange, month: Month, year: int): WeekDay {. # so we must correct for the WeekDay type. result = if wd == 0: dSun else: WeekDay(wd - 1) -# Forward declarations -proc utcZoneInfoFromUtc(time: Time): ZonedTime {.tags: [], raises: [], benign .} -proc utcZoneInfoFromTz(adjTime: Time): ZonedTime {.tags: [], raises: [], benign .} -proc localZoneInfoFromUtc(time: Time): ZonedTime {.tags: [], raises: [], benign .} -proc localZoneInfoFromTz(adjTime: Time): ZonedTime {.tags: [], raises: [], benign .} -proc `-`*(a, b: Time): int64 {. - rtl, extern: "ntDiffTime", tags: [], raises: [], noSideEffect, benign, deprecated.} = - ## Computes the difference of two calendar times. Result is in seconds. - ## This is deprecated because it will need to change when sub second time resolution is implemented. - ## Use ``a.toUnix - b.toUnix`` instead. +{. pragma: operator, rtl, noSideEffect, benign .} + +template subImpl[T: Duration|Time](a: Duration|Time, b: Duration|Time): T = + normalize[T](a.seconds - b.seconds, a.nanosecond - b.nanosecond) + +template addImpl[T: Duration|Time](a: Duration|Time, b: Duration|Time): T = + normalize[T](a.seconds + b.seconds, a.nanosecond + b.nanosecond) + +template ltImpl(a: Duration|Time, b: Duration|Time): bool = + a.seconds < b.seconds or ( + a.seconds == b.seconds and a.nanosecond < b.nanosecond) + +template lqImpl(a: Duration|Time, b: Duration|Time): bool = + a.seconds < b.seconds or ( + a.seconds == b.seconds and a.nanosecond <= b.nanosecond) + +template eqImpl(a: Duration|Time, b: Duration|Time): bool = + a.seconds == b.seconds and a.nanosecond == b.nanosecond + +proc initDuration*(nanoseconds, microseconds, milliseconds, + seconds, minutes, hours, days, weeks: int64 = 0): Duration = + runnableExamples: + let dur = initDuration(seconds = 1, milliseconds = 1) + doAssert dur.milliseconds == 1 + doAssert dur.seconds == 1 + + let seconds = convert(Weeks, Seconds, weeks) + + convert(Days, Seconds, days) + + convert(Minutes, Seconds, minutes) + + convert(Hours, Seconds, hours) + + convert(Seconds, Seconds, seconds) + + convert(Milliseconds, Seconds, milliseconds) + + convert(Microseconds, Seconds, microseconds) + + convert(Nanoseconds, Seconds, nanoseconds) + let nanoseconds = (convert(Milliseconds, Nanoseconds, milliseconds mod 1000) + + convert(Microseconds, Nanoseconds, microseconds mod 1_000_000) + + nanoseconds mod 1_000_000_000).int + # Nanoseconds might be negative so we must normalize. + result = normalize[Duration](seconds, nanoseconds) + +const DurationZero* = initDuration() ## \ + ## Zero value for durations. Useful for comparisons. ## ## .. code-block:: nim - ## let a = fromSeconds(1_000_000_000) - ## let b = fromSeconds(1_500_000_000) - ## echo initInterval(seconds=int(b - a)) - ## # (milliseconds: 0, seconds: 20, minutes: 53, hours: 0, days: 5787, months: 0, years: 0) - a.toUnix - b.toUnix - -proc `<`*(a, b: Time): bool {. - rtl, extern: "ntLtTime", tags: [], raises: [], noSideEffect, borrow.} + ## + ## doAssert initDuration(seconds = 1) > DurationZero + ## doAssert initDuration(seconds = 0) == DurationZero + +proc toParts*(dur: Duration): DurationParts = + ## Converts a duration into an array consisting of fixed time units. + ## + ## Each value in the array gives information about a specific unit of + ## time, for example ``result[Days]`` gives a count of days. + ## + ## This procedure is useful for converting ``Duration`` values to strings. + runnableExamples: + var dp = toParts(initDuration(weeks=2, days=1)) + doAssert dp[Days] == 1 + doAssert dp[Weeks] == 2 + dp = toParts(initDuration(days = -1)) + doAssert dp[Days] == -1 + + var remS = dur.seconds + var remNs = dur.nanosecond.int + + # Ensure the same sign for seconds and nanoseconds + if remS < 0 and remNs != 0: + remNs -= convert(Seconds, Nanoseconds, 1) + remS.inc 1 + + for unit in countdown(Weeks, Seconds): + let quantity = convert(Seconds, unit, remS) + remS = remS mod convert(unit, Seconds, 1) + + result[unit] = quantity + + for unit in countdown(Milliseconds, Nanoseconds): + let quantity = convert(Nanoseconds, unit, remNs) + remNs = remNs mod convert(unit, Nanoseconds, 1) + + result[unit] = quantity + +proc stringifyUnit*(value: int | int64, unit: string): string = + ## Stringify time unit with it's name, lowercased + runnableExamples: + doAssert stringifyUnit(2, "Seconds") == "2 seconds" + doAssert stringifyUnit(1, "Years") == "1 year" + result = "" + result.add($value) + result.add(" ") + if abs(value) != 1: + result.add(unit.toLowerAscii()) + else: + result.add(unit[0..^2].toLowerAscii()) + +proc humanizeParts(parts: seq[string]): string = + ## Make date string parts human-readable + + result = "" + if parts.len == 0: + result.add "0 nanoseconds" + elif parts.len == 1: + result = parts[0] + elif parts.len == 2: + result = parts[0] & " and " & parts[1] + else: + for part in parts[0..high(parts)-1]: + result.add part & ", " + result.add "and " & parts[high(parts)] + +proc `$`*(dur: Duration): string = + ## Human friendly string representation of ``Duration``. + runnableExamples: + doAssert $initDuration(seconds = 2) == "2 seconds" + doAssert $initDuration(weeks = 1, days = 2) == "1 week and 2 days" + doAssert $initDuration(hours = 1, minutes = 2, seconds = 3) == "1 hour, 2 minutes, and 3 seconds" + doAssert $initDuration(milliseconds = -1500) == "-1 second and -500 milliseconds" + var parts = newSeq[string]() + var numParts = toParts(dur) + + for unit in countdown(Weeks, Nanoseconds): + let quantity = numParts[unit] + if quantity != 0.int64: + parts.add(stringifyUnit(quantity, $unit)) + + result = humanizeParts(parts) + +proc `+`*(a, b: Duration): Duration {.operator.} = + ## Add two durations together. + runnableExamples: + doAssert initDuration(seconds = 1) + initDuration(days = 1) == + initDuration(seconds = 1, days = 1) + addImpl[Duration](a, b) + +proc `-`*(a, b: Duration): Duration {.operator.} = + ## Subtract a duration from another. + runnableExamples: + doAssert initDuration(seconds = 1, days = 1) - initDuration(seconds = 1) == + initDuration(days = 1) + subImpl[Duration](a, b) + +proc `-`*(a: Duration): Duration {.operator.} = + ## Reverse a duration. + runnableExamples: + doAssert -initDuration(seconds = 1) == initDuration(seconds = -1) + normalize[Duration](-a.seconds, -a.nanosecond) + +proc `<`*(a, b: Duration): bool {.operator.} = + ## Note that a duration can be negative, + ## so even if ``a < b`` is true ``a`` might + ## represent a larger absolute duration. + ## Use ``abs(a) < abs(b)`` to compare the absolute + ## duration. + runnableExamples: + doAssert initDuration(seconds = 1) < initDuration(seconds = 2) + doAssert initDuration(seconds = -2) < initDuration(seconds = 1) + ltImpl(a, b) + +proc `<=`*(a, b: Duration): bool {.operator.} = + lqImpl(a, b) + +proc `==`*(a, b: Duration): bool {.operator.} = + eqImpl(a, b) + +proc `*`*(a: int64, b: Duration): Duration {.operator} = + ## Multiply a duration by some scalar. + runnableExamples: + doAssert 5 * initDuration(seconds = 1) == initDuration(seconds = 5) + normalize[Duration](a * b.seconds, a * b.nanosecond) + +proc `*`*(a: Duration, b: int64): Duration {.operator} = + ## Multiply a duration by some scalar. + runnableExamples: + doAssert initDuration(seconds = 1) * 5 == initDuration(seconds = 5) + b * a + +proc `div`*(a: Duration, b: int64): Duration {.operator} = + ## Integer division for durations. + runnableExamples: + doAssert initDuration(seconds = 3) div 2 == initDuration(milliseconds = 1500) + doAssert initDuration(nanoseconds = 3) div 2 == initDuration(nanoseconds = 1) + let carryOver = convert(Seconds, Nanoseconds, a.seconds mod b) + normalize[Duration](a.seconds div b, (a.nanosecond + carryOver) div b) + +proc initTime*(unix: int64, nanosecond: NanosecondRange): Time = + ## Create a ``Time`` from a unix timestamp and a nanosecond part. + result.seconds = unix + result.nanosecond = nanosecond + +proc `-`*(a, b: Time): Duration {.operator, extern: "ntDiffTime".} = + ## Computes the duration between two points in time. + subImpl[Duration](a, b) + +proc `+`*(a: Time, b: Duration): Time {.operator, extern: "ntAddTime".} = + ## Add a duration of time to a ``Time``. + runnableExamples: + doAssert (fromUnix(0) + initDuration(seconds = 1)) == fromUnix(1) + addImpl[Time](a, b) + +proc `+=`*(a: var Time, b: Duration) {.operator.} = + ## Modify ``a`` in place by subtracting ``b``. + runnableExamples: + var tm = fromUnix(0) + tm += initDuration(seconds = 1) + doAssert tm == fromUnix(1) + + a = addImpl[Time](a, b) + +proc `-`*(a: Time, b: Duration): Time {.operator, extern: "ntSubTime".} = + ## Subtracts a duration of time from a ``Time``. + runnableExamples: + doAssert (fromUnix(0) - initDuration(seconds = 1)) == fromUnix(-1) + subImpl[Time](a, b) + +proc `-=`*(a: var Time, b: Duration) {.operator.} = + ## Modify ``a`` in place by adding ``b``. + runnableExamples: + var tm = fromUnix(0) + tm -= initDuration(seconds = 1) + doAssert tm == fromUnix(-1) + + a = subImpl[Time](a, b) + +proc `<`*(a, b: Time): bool {.operator, extern: "ntLtTime".} = ## Returns true iff ``a < b``, that is iff a happened before b. + ltImpl(a, b) -proc `<=` * (a, b: Time): bool {. - rtl, extern: "ntLeTime", tags: [], raises: [], noSideEffect, borrow.} +proc `<=` * (a, b: Time): bool {.operator, extern: "ntLeTime".} = ## Returns true iff ``a <= b``. + lqImpl(a, b) -proc `==`*(a, b: Time): bool {. - rtl, extern: "ntEqTime", tags: [], raises: [], noSideEffect, borrow.} +proc `==`*(a, b: Time): bool {.operator, extern: "ntEqTime".} = ## Returns true if ``a == b``, that is if both times represent the same point in time. + eqImpl(a, b) + +proc high*(typ: typedesc[Time]): Time = + initTime(high(int64), high(NanosecondRange)) + +proc low*(typ: typedesc[Time]): Time = + initTime(low(int64), 0) + +proc high*(typ: typedesc[Duration]): Duration = + ## Get the longest representable duration. + initDuration(seconds = high(int64), nanoseconds = high(NanosecondRange)) + +proc low*(typ: typedesc[Duration]): Duration = + ## Get the longest representable duration of negative direction. + initDuration(seconds = low(int64)) + +proc abs*(a: Duration): Duration = + runnableExamples: + doAssert initDuration(milliseconds = -1500).abs == + initDuration(milliseconds = 1500) + initDuration(seconds = abs(a.seconds), nanoseconds = -a.nanosecond) proc toTime*(dt: DateTime): Time {.tags: [], raises: [], benign.} = ## Converts a broken-down time structure to ## calendar time representation. let epochDay = toEpochday(dt.monthday, dt.month, dt.year) - result = Time(epochDay * secondsInDay) - result.inc dt.hour * secondsInHour - result.inc dt.minute * 60 - result.inc dt.second + var seconds = epochDay * secondsInDay + seconds.inc dt.hour * secondsInHour + seconds.inc dt.minute * 60 + seconds.inc dt.second # The code above ignores the UTC offset of `timeInfo`, # so we need to compensate for that here. - result.inc dt.utcOffset + seconds.inc dt.utcOffset + result = initTime(seconds, dt.nanosecond) proc initDateTime(zt: ZonedTime, zone: Timezone): DateTime = - let adjTime = zt.adjTime.int64 - let epochday = (if adjTime >= 0: adjTime else: adjTime - (secondsInDay - 1)) div secondsInDay - var rem = zt.adjTime.int64 - epochday * secondsInDay + ## Create a new ``DateTime`` using ``ZonedTime`` in the specified timezone. + let s = zt.adjTime.seconds + let epochday = (if s >= 0: s else: s - (secondsInDay - 1)) div secondsInDay + var rem = s - epochday * secondsInDay let hour = rem div secondsInHour rem = rem - hour * secondsInHour let minute = rem div secondsInMin @@ -293,6 +696,7 @@ proc initDateTime(zt: ZonedTime, zone: Timezone): DateTime = hour: hour, minute: minute, second: second, + nanosecond: zt.adjTime.nanosecond, weekday: getDayOfWeek(d, m, y), yearday: getDayOfYear(d, m, y), isDst: zt.isDst, @@ -319,10 +723,11 @@ proc `==`*(zone1, zone2: Timezone): bool = proc toAdjTime(dt: DateTime): Time = let epochDay = toEpochday(dt.monthday, dt.month, dt.year) - result = Time(epochDay * secondsInDay) - result.inc dt.hour * secondsInHour - result.inc dt.minute * secondsInMin - result.inc dt.second + var seconds = epochDay * secondsInDay + seconds.inc dt.hour * secondsInHour + seconds.inc dt.minute * secondsInMin + seconds.inc dt.second + result = initTime(seconds, dt.nanosecond) when defined(JS): type JsDate = object @@ -351,14 +756,14 @@ when defined(JS): proc setFullYear(js: JsDate, year: int): void {.tags: [], raises: [], benign, importcpp.} proc localZoneInfoFromUtc(time: Time): ZonedTime = - let jsDate = newDate(time.float * 1000) + let jsDate = newDate(time.seconds.float * 1000) let offset = jsDate.getTimezoneOffset() * secondsInMin - result.adjTime = Time(time.int64 - offset) + result.adjTime = time - initDuration(seconds = offset) result.utcOffset = offset result.isDst = false proc localZoneInfoFromTz(adjTime: Time): ZonedTime = - let utcDate = newDate(adjTime.float * 1000) + let utcDate = newDate(adjTime.seconds.float * 1000) let localDate = newDate(utcDate.getUTCFullYear(), utcDate.getUTCMonth(), utcDate.getUTCDate(), utcDate.getUTCHours(), utcDate.getUTCMinutes(), utcDate.getUTCSeconds(), 0) @@ -407,58 +812,52 @@ else: proc localtime(timer: ptr CTime): StructTmPtr {. importc: "localtime", header: "<time.h>", tags: [].} - proc toAdjTime(tm: StructTm): Time = + proc toAdjUnix(tm: StructTm): int64 = let epochDay = toEpochday(tm.monthday, (tm.month + 1).Month, tm.year.int + 1900) - result = Time(epochDay * secondsInDay) + result = epochDay * secondsInDay result.inc tm.hour * secondsInHour result.inc tm.minute * 60 result.inc tm.second - proc getStructTm(time: Time | int64): StructTm = - let timei64 = time.int64 - var a = - if timei64 < low(CTime): - CTime(low(CTime)) - elif timei64 > high(CTime): - CTime(high(CTime)) - else: - CTime(timei64) - result = localtime(addr(a))[] + proc getLocalOffsetAndDst(unix: int64): tuple[offset: int, dst: bool] = + var a = unix.CTime + let tmPtr = localtime(addr(a)) + if not tmPtr.isNil: + let tm = tmPtr[] + return ((unix - tm.toAdjUnix).int, tm.isdst > 0) + return (0, false) proc localZoneInfoFromUtc(time: Time): ZonedTime = - let tm = getStructTm(time) - let adjTime = tm.toAdjTime - result.adjTime = adjTime - result.utcOffset = (time.toUnix - adjTime.toUnix).int - result.isDst = tm.isdst > 0 + let (offset, dst) = getLocalOffsetAndDst(time.seconds) + result.adjTime = time - initDuration(seconds = offset) + result.utcOffset = offset + result.isDst = dst proc localZoneInfoFromTz(adjTime: Time): ZonedTime = - var adjTimei64 = adjTime.int64 - let past = adjTimei64 - secondsInDay - var tm = getStructTm(past) - let pastOffset = past - tm.toAdjTime.int64 + var adjUnix = adjTime.seconds + let past = adjUnix - secondsInDay + let (pastOffset, _) = getLocalOffsetAndDst(past) - let future = adjTimei64 + secondsInDay - tm = getStructTm(future) - let futureOffset = future - tm.toAdjTime.int64 + let future = adjUnix + secondsInDay + let (futureOffset, _) = getLocalOffsetAndDst(future) var utcOffset: int if pastOffset == futureOffset: utcOffset = pastOffset.int else: if pastOffset > futureOffset: - adjTimei64 -= secondsInHour + adjUnix -= secondsInHour - adjTimei64 += pastOffset - utcOffset = (adjTimei64 - getStructTm(adjTimei64).toAdjTime.int64).int + adjUnix += pastOffset + utcOffset = getLocalOffsetAndDst(adjUnix).offset # This extra roundtrip is needed to normalize any impossible datetimes # as a result of offset changes (normally due to dst) - let utcTime = adjTime.int64 + utcOffset - tm = getStructTm(utcTime) - result.adjTime = tm.toAdjTime - result.utcOffset = (utcTime - result.adjTime.int64).int - result.isDst = tm.isdst > 0 + let utcUnix = adjTime.seconds + utcOffset + let (finalOffset, dst) = getLocalOffsetAndDst(utcUnix) + result.adjTime = initTime(utcUnix - finalOffset, adjTime.nanosecond) + result.utcOffset = finalOffset + result.isDst = dst proc utcZoneInfoFromUtc(time: Time): ZonedTime = result.adjTime = time @@ -470,18 +869,16 @@ proc utcZoneInfoFromTz(adjTime: Time): ZonedTime = proc utc*(): TimeZone = ## Get the ``Timezone`` implementation for the UTC timezone. - ## - ## .. code-block:: nim - ## doAssert now().utc.timezone == utc() - ## doAssert utc().name == "Etc/UTC" + runnableExamples: + doAssert now().utc.timezone == utc() + doAssert utc().name == "Etc/UTC" Timezone(zoneInfoFromUtc: utcZoneInfoFromUtc, zoneInfoFromTz: utcZoneInfoFromTz, name: "Etc/UTC") proc local*(): TimeZone = ## Get the ``Timezone`` implementation for the local timezone. - ## - ## .. code-block:: nim - ## doAssert now().timezone == local() - ## doAssert local().name == "LOCAL" + runnableExamples: + doAssert now().timezone == local() + doAssert local().name == "LOCAL" Timezone(zoneInfoFromUtc: localZoneInfoFromUtc, zoneInfoFromTz: localZoneInfoFromTz, name: "LOCAL") proc utc*(dt: DateTime): DateTime = @@ -500,9 +897,27 @@ proc local*(t: Time): DateTime = ## Shorthand for ``t.inZone(local())``. t.inZone(local()) -proc getTime*(): Time {.tags: [TimeEffect], benign.} - ## Gets the current time as a ``Time`` with second resolution. Use epochTime for higher - ## resolution. +proc getTime*(): Time {.tags: [TimeEffect], benign.} = + ## Gets the current time as a ``Time`` with nanosecond resolution. + when defined(JS): + let millis = newDate().getTime() + let seconds = convert(Milliseconds, Seconds, millis) + let nanos = convert(Milliseconds, Nanoseconds, + millis mod convert(Seconds, Milliseconds, 1).int) + result = initTime(seconds, nanos) + # I'm not entirely certain if freebsd needs to use `gettimeofday`. + elif defined(macosx) or defined(freebsd): + var a: Timeval + gettimeofday(a) + result = initTime(a.tv_sec.int64, convert(Microseconds, Nanoseconds, a.tv_usec.int)) + elif defined(posix): + var ts: Timespec + discard clock_gettime(CLOCK_REALTIME, ts) + result = initTime(ts.tv_sec.int64, ts.tv_nsec.int) + elif defined(windows): + var f: FILETIME + getSystemTimeAsFileTime(f) + result = fromWinTime(rdFileTime(f)) proc now*(): DateTime {.tags: [TimeEffect], benign.} = ## Get the current time as a ``DateTime`` in the local timezone. @@ -510,51 +925,57 @@ proc now*(): DateTime {.tags: [TimeEffect], benign.} = ## Shorthand for ``getTime().local``. getTime().local -proc initInterval*(milliseconds, seconds, minutes, hours, days, months, - years: int = 0): TimeInterval = +proc initTimeInterval*(nanoseconds, microseconds, milliseconds, + seconds, minutes, hours, + days, weeks, months, years: int = 0): TimeInterval = ## Creates a new ``TimeInterval``. ## ## You can also use the convenience procedures called ``milliseconds``, ## ``seconds``, ``minutes``, ``hours``, ``days``, ``months``, and ``years``. ## - ## Example: - ## - ## .. code-block:: nim - ## - ## let day = initInterval(hours=24) - ## let dt = initDateTime(01, mJan, 2000, 12, 00, 00, utc()) - ## doAssert $(dt + day) == "2000-01-02T12-00-00+00:00" + runnableExamples: + let day = initTimeInterval(hours=24) + let dt = initDateTime(01, mJan, 2000, 12, 00, 00, utc()) + doAssert $(dt + day) == "2000-01-02T12:00:00+00:00" + result.nanoseconds = nanoseconds + result.microseconds = microseconds result.milliseconds = milliseconds result.seconds = seconds result.minutes = minutes result.hours = hours result.days = days + result.weeks = weeks result.months = months result.years = years proc `+`*(ti1, ti2: TimeInterval): TimeInterval = ## Adds two ``TimeInterval`` objects together. + result.nanoseconds = ti1.nanoseconds + ti2.nanoseconds + result.microseconds = ti1.microseconds + ti2.microseconds result.milliseconds = ti1.milliseconds + ti2.milliseconds result.seconds = ti1.seconds + ti2.seconds result.minutes = ti1.minutes + ti2.minutes result.hours = ti1.hours + ti2.hours result.days = ti1.days + ti2.days + result.weeks = ti1.weeks + ti2.weeks result.months = ti1.months + ti2.months result.years = ti1.years + ti2.years proc `-`*(ti: TimeInterval): TimeInterval = ## Reverses a time interval - ## - ## .. code-block:: nim - ## - ## let day = -initInterval(hours=24) - ## echo day # -> (milliseconds: 0, seconds: 0, minutes: 0, hours: -24, days: 0, months: 0, years: 0) + runnableExamples: + let day = -initTimeInterval(hours=24) + doAssert day.hours == -24 + result = TimeInterval( + nanoseconds: -ti.nanoseconds, + microseconds: -ti.microseconds, milliseconds: -ti.milliseconds, seconds: -ti.seconds, minutes: -ti.minutes, hours: -ti.hours, days: -ti.days, + weeks: -ti.weeks, months: -ti.months, years: -ti.years ) @@ -562,82 +983,13 @@ proc `-`*(ti: TimeInterval): TimeInterval = proc `-`*(ti1, ti2: TimeInterval): TimeInterval = ## Subtracts TimeInterval ``ti1`` from ``ti2``. ## - ## Time components are compared one-by-one, see output: - ## - ## .. code-block:: nim - ## let a = fromUnix(1_000_000_000) - ## let b = fromUnix(1_500_000_000) - ## echo b.toTimeInterval - a.toTimeInterval - ## # (milliseconds: 0, seconds: -40, minutes: -6, hours: 1, days: 5, months: -2, years: 16) - result = ti1 + (-ti2) - -proc evaluateInterval(dt: DateTime, interval: TimeInterval): tuple[adjDiff, absDiff: int64] = - ## Evaluates how many seconds the interval is worth - ## in the context of ``dt``. - ## The result in split into an adjusted diff and an absolute diff. - - var anew = dt - var newinterv = interval - - newinterv.months += interval.years * 12 - var curMonth = anew.month - # Subtracting - if newinterv.months < 0: - for mth in countDown(-1 * newinterv.months, 1): - if curMonth == mJan: - curMonth = mDec - anew.year.dec() - else: - curMonth.dec() - result.adjDiff -= getDaysInMonth(curMonth, anew.year) * secondsInDay - # Adding - else: - for mth in 1 .. newinterv.months: - result.adjDiff += getDaysInMonth(curMonth, anew.year) * secondsInDay - if curMonth == mDec: - curMonth = mJan - anew.year.inc() - else: - curMonth.inc() - result.adjDiff += newinterv.days * secondsInDay - result.absDiff += newinterv.hours * secondsInHour - result.absDiff += newinterv.minutes * secondsInMin - result.absDiff += newinterv.seconds - result.absDiff += newinterv.milliseconds div 1000 - -proc `+`*(dt: DateTime, interval: TimeInterval): DateTime = - ## Adds ``interval`` to ``dt``. Components from ``interval`` are added - ## in the order of their size, i.e first the ``years`` component, then the ``months`` - ## component and so on. The returned ``DateTime`` will have the same timezone as the input. - ## - ## Note that when adding months, monthday overflow is allowed. This means that if the resulting - ## month doesn't have enough days it, the month will be incremented and the monthday will be - ## set to the number of days overflowed. So adding one month to `31 October` will result in `31 November`, - ## which will overflow and result in `1 December`. - ## - ## .. code-block:: nim - ## let dt = initDateTime(30, mMar, 2017, 00, 00, 00, utc()) - ## doAssert $(dt + 1.months) == "2017-04-30T00:00:00+00:00" - ## # This is correct and happens due to monthday overflow. - ## doAssert $(dt - 1.months) == "2017-03-02T00:00:00+00:00" - let (adjDiff, absDiff) = evaluateInterval(dt, interval) - - if adjDiff.int64 != 0: - let zInfo = dt.timezone.zoneInfoFromTz(Time(dt.toAdjTime.int64 + adjDiff)) - - if absDiff != 0: - let time = Time(zInfo.adjTime.int64 + zInfo.utcOffset + absDiff) - result = initDateTime(dt.timezone.zoneInfoFromUtc(time), dt.timezone) - else: - result = initDateTime(zInfo, dt.timezone) - else: - result = initDateTime(dt.timezone.zoneInfoFromUtc(Time(dt.toTime.int64 + absDiff)), dt.timezone) + ## Time components are subtracted one-by-one, see output: + runnableExamples: + let ti1 = initTimeInterval(hours=24) + let ti2 = initTimeInterval(hours=4) + doAssert (ti1 - ti2) == initTimeInterval(hours=20) -proc `-`*(dt: DateTime, interval: TimeInterval): DateTime = - ## Subtract ``interval`` from ``dt``. Components from ``interval`` are subtracted - ## in the order of their size, i.e first the ``years`` component, then the ``months`` - ## component and so on. The returned ``DateTime`` will have the same timezone as the input. - dt + (-interval) + result = ti1 + (-ti2) proc getDateStr*(): string {.rtl, extern: "nt$1", tags: [TimeEffect].} = ## Gets the current date as a string of the format ``YYYY-MM-DD``. @@ -664,69 +1016,400 @@ proc `$`*(m: Month): string = "November", "December"] return lookup[m] -proc milliseconds*(ms: int): TimeInterval {.inline.} = - ## TimeInterval of `ms` milliseconds + +proc toParts* (ti: TimeInterval): TimeIntervalParts = + ## Converts a `TimeInterval` into an array consisting of its time units, + ## starting with nanoseconds and ending with years ## - ## Note: not all time procedures have millisecond resolution - initInterval(milliseconds = ms) + ## This procedure is useful for converting ``TimeInterval`` values to strings. + ## E.g. then you need to implement custom interval printing + runnableExamples: + var tp = toParts(initTimeInterval(years=1, nanoseconds=123)) + doAssert tp[Years] == 1 + doAssert tp[Nanoseconds] == 123 + + var index = 0 + for name, value in fieldPairs(ti): + result[index.TimeUnit()] = value + index += 1 + +proc `$`*(ti: TimeInterval): string = + ## Get string representation of `TimeInterval` + runnableExamples: + doAssert $initTimeInterval(years=1, nanoseconds=123) == "1 year and 123 nanoseconds" + doAssert $initTimeInterval() == "0 nanoseconds" + + var parts: seq[string] = @[] + var tiParts = toParts(ti) + for unit in countdown(Years, Nanoseconds): + if tiParts[unit] != 0: + parts.add(stringifyUnit(tiParts[unit], $unit)) + + result = humanizeParts(parts) + +proc nanoseconds*(nanos: int): TimeInterval {.inline.} = + ## TimeInterval of ``nanos`` nanoseconds. + initTimeInterval(nanoseconds = nanos) + +proc microseconds*(micros: int): TimeInterval {.inline.} = + ## TimeInterval of ``micros`` microseconds. + initTimeInterval(microseconds = micros) + +proc milliseconds*(ms: int): TimeInterval {.inline.} = + ## TimeInterval of ``ms`` milliseconds. + initTimeInterval(milliseconds = ms) proc seconds*(s: int): TimeInterval {.inline.} = - ## TimeInterval of `s` seconds + ## TimeInterval of ``s`` seconds. ## ## ``echo getTime() + 5.second`` - initInterval(seconds = s) + initTimeInterval(seconds = s) proc minutes*(m: int): TimeInterval {.inline.} = - ## TimeInterval of `m` minutes + ## TimeInterval of ``m`` minutes. ## ## ``echo getTime() + 5.minutes`` - initInterval(minutes = m) + initTimeInterval(minutes = m) proc hours*(h: int): TimeInterval {.inline.} = - ## TimeInterval of `h` hours + ## TimeInterval of ``h`` hours. ## ## ``echo getTime() + 2.hours`` - initInterval(hours = h) + initTimeInterval(hours = h) proc days*(d: int): TimeInterval {.inline.} = - ## TimeInterval of `d` days + ## TimeInterval of ``d`` days. ## ## ``echo getTime() + 2.days`` - initInterval(days = d) + initTimeInterval(days = d) + +proc weeks*(w: int): TimeInterval {.inline.} = + ## TimeInterval of ``w`` weeks. + ## + ## ``echo getTime() + 2.weeks`` + initTimeInterval(weeks = w) proc months*(m: int): TimeInterval {.inline.} = - ## TimeInterval of `m` months + ## TimeInterval of ``m`` months. ## ## ``echo getTime() + 2.months`` - initInterval(months = m) + initTimeInterval(months = m) proc years*(y: int): TimeInterval {.inline.} = - ## TimeInterval of `y` years + ## TimeInterval of ``y`` years. ## ## ``echo getTime() + 2.years`` - initInterval(years = y) + initTimeInterval(years = y) -proc `+=`*(time: var Time, interval: TimeInterval) = - ## Modifies `time` by adding `interval`. - time = toTime(time.local + interval) +proc evaluateInterval(dt: DateTime, interval: TimeInterval): tuple[adjDur, absDur: Duration] = + ## Evaluates how many nanoseconds the interval is worth + ## in the context of ``dt``. + ## The result in split into an adjusted diff and an absolute diff. + var months = interval.years * 12 + interval.months + var curYear = dt.year + var curMonth = dt.month + # Subtracting + if months < 0: + for mth in countDown(-1 * months, 1): + if curMonth == mJan: + curMonth = mDec + curYear.dec + else: + curMonth.dec() + let days = getDaysInMonth(curMonth, curYear) + result.adjDur = result.adjDur - initDuration(days = days) + # Adding + else: + for mth in 1 .. months: + let days = getDaysInMonth(curMonth, curYear) + result.adjDur = result.adjDur + initDuration(days = days) + if curMonth == mDec: + curMonth = mJan + curYear.inc + else: + curMonth.inc() -proc `+`*(time: Time, interval: TimeInterval): Time = - ## Adds `interval` to `time` - ## by converting to a ``DateTime`` in the local timezone, - ## adding the interval, and converting back to ``Time``. + result.adjDur = result.adjDur + initDuration( + days = interval.days, + weeks = interval.weeks) + result.absDur = initDuration( + nanoseconds = interval.nanoseconds, + microseconds = interval.microseconds, + milliseconds = interval.milliseconds, + seconds = interval.seconds, + minutes = interval.minutes, + hours = interval.hours) + + +proc initDateTime*(monthday: MonthdayRange, month: Month, year: int, + hour: HourRange, minute: MinuteRange, second: SecondRange, + nanosecond: NanosecondRange, zone: Timezone = local()): DateTime = + ## Create a new ``DateTime`` in the specified timezone. + runnableExamples: + let dt1 = initDateTime(30, mMar, 2017, 00, 00, 00, 00, utc()) + doAssert $dt1 == "2017-03-30T00:00:00+00:00" + + assertValidDate monthday, month, year + let dt = DateTime( + monthday: monthday, + year: year, + month: month, + hour: hour, + minute: minute, + second: second, + nanosecond: nanosecond + ) + result = initDateTime(zone.zoneInfoFromTz(dt.toAdjTime), zone) + +proc initDateTime*(monthday: MonthdayRange, month: Month, year: int, + hour: HourRange, minute: MinuteRange, second: SecondRange, + zone: Timezone = local()): DateTime = + ## Create a new ``DateTime`` in the specified timezone. + runnableExamples: + let dt1 = initDateTime(30, mMar, 2017, 00, 00, 00, utc()) + doAssert $dt1 == "2017-03-30T00:00:00+00:00" + initDateTime(monthday, month, year, hour, minute, second, 0, zone) + + +proc `+`*(dt: DateTime, interval: TimeInterval): DateTime = + ## Adds ``interval`` to ``dt``. Components from ``interval`` are added + ## in the order of their size, i.e first the ``years`` component, then the ``months`` + ## component and so on. The returned ``DateTime`` will have the same timezone as the input. + ## + ## Note that when adding months, monthday overflow is allowed. This means that if the resulting + ## month doesn't have enough days it, the month will be incremented and the monthday will be + ## set to the number of days overflowed. So adding one month to `31 October` will result in `31 November`, + ## which will overflow and result in `1 December`. ## - ## ``echo getTime() + 1.day`` - result = toTime(time.local + interval) + runnableExamples: + let dt = initDateTime(30, mMar, 2017, 00, 00, 00, utc()) + doAssert $(dt + 1.months) == "2017-04-30T00:00:00+00:00" + # This is correct and happens due to monthday overflow. + doAssert $(dt - 1.months) == "2017-03-02T00:00:00+00:00" + let (adjDur, absDur) = evaluateInterval(dt, interval) + + if adjDur != DurationZero: + var zInfo = dt.timezone.zoneInfoFromTz(dt.toAdjTime + adjDur) + if absDur != DurationZero: + let offsetDur = initDuration(seconds = zInfo.utcOffset) + zInfo = dt.timezone.zoneInfoFromUtc(zInfo.adjTime + offsetDur + absDur) + result = initDateTime(zInfo, dt.timezone) + else: + result = initDateTime(zInfo, dt.timezone) + else: + var zInfo = dt.timezone.zoneInfoFromUtc(dt.toTime + absDur) + result = initDateTime(zInfo, dt.timezone) -proc `-=`*(time: var Time, interval: TimeInterval) = - ## Modifies `time` by subtracting `interval`. - time = toTime(time.local - interval) +proc `-`*(dt: DateTime, interval: TimeInterval): DateTime = + ## Subtract ``interval`` from ``dt``. Components from ``interval`` are subtracted + ## in the order of their size, i.e first the ``years`` component, then the ``months`` + ## component and so on. The returned ``DateTime`` will have the same timezone as the input. + runnableExamples: + let dt = initDateTime(30, mMar, 2017, 00, 00, 00, utc()) + doAssert $(dt - 5.days) == "2017-03-25T00:00:00+00:00" + + dt + (-interval) + +proc `+`*(dt: DateTime, dur: Duration): DateTime = + runnableExamples: + let dt = initDateTime(30, mMar, 2017, 00, 00, 00, utc()) + let dur = initDuration(hours = 5) + doAssert $(dt + dur) == "2017-03-30T05:00:00+00:00" + + (dt.toTime + dur).inZone(dt.timezone) + +proc `-`*(dt: DateTime, dur: Duration): DateTime = + runnableExamples: + let dt = initDateTime(30, mMar, 2017, 00, 00, 00, utc()) + let dur = initDuration(days = 5) + doAssert $(dt - dur) == "2017-03-25T00:00:00+00:00" + + (dt.toTime - dur).inZone(dt.timezone) + +proc `-`*(dt1, dt2: DateTime): Duration = + ## Compute the duration between ``dt1`` and ``dt2``. + runnableExamples: + let dt1 = initDateTime(30, mMar, 2017, 00, 00, 00, utc()) + let dt2 = initDateTime(25, mMar, 2017, 00, 00, 00, utc()) + + doAssert dt1 - dt2 == initDuration(days = 5) + + dt1.toTime - dt2.toTime + +proc `<`*(a, b: DateTime): bool = + ## Returns true iff ``a < b``, that is iff a happened before b. + return a.toTime < b.toTime + +proc `<=` * (a, b: DateTime): bool = + ## Returns true iff ``a <= b``. + return a.toTime <= b.toTime + +proc `==`*(a, b: DateTime): bool = + ## Returns true if ``a == b``, that is if both dates represent the same point in datetime. + return a.toTime == b.toTime + + +proc isStaticInterval(interval: TimeInterval): bool = + interval.years == 0 and interval.months == 0 and + interval.days == 0 and interval.weeks == 0 + +proc evaluateStaticInterval(interval: TimeInterval): Duration = + assert interval.isStaticInterval + initDuration(nanoseconds = interval.nanoseconds, + microseconds = interval.microseconds, + milliseconds = interval.milliseconds, + seconds = interval.seconds, + minutes = interval.minutes, + hours = interval.hours) + +proc between*(startDt, endDt:DateTime): TimeInterval = + ## Evaluate difference between two dates in ``TimeInterval`` format, so, it + ## will be relative. + ## + ## **Warning:** It's not recommended to use ``between`` for ``DateTime's`` in + ## different ``TimeZone's``. + ## ``a + between(a, b) == b`` is only guaranteed when ``a`` and ``b`` are in UTC. + runnableExamples: + var a = initDateTime(year = 2018, month = Month(3), monthday = 25, + hour = 0, minute = 59, second = 59, nanosecond = 1, + zone = utc()).local + var b = initDateTime(year = 2018, month = Month(3), monthday = 25, + hour = 1, minute = 1, second = 1, nanosecond = 0, + zone = utc()).local + doAssert between(a, b) == initTimeInterval( + nanoseconds=999, milliseconds=999, microseconds=999, seconds=1, minutes=1) + + a = parse("2018-01-09T00:00:00+00:00", "yyyy-MM-dd'T'HH:mm:sszzz", utc()) + b = parse("2018-01-10T23:00:00-02:00", "yyyy-MM-dd'T'HH:mm:sszzz") + doAssert between(a, b) == initTimeInterval(hours=1, days=2) + ## Though, here correct answer should be 1 day 25 hours (cause this day in + ## this tz is actually 26 hours). That's why operating different TZ is + ## discouraged + + var startDt = startDt.utc() + var endDt = endDt.utc() + + if endDt == startDt: + return initTimeInterval() + elif endDt < startDt: + return -between(endDt, startDt) + + var coeffs: array[FixedTimeUnit, int64] = unitWeights + var timeParts: array[FixedTimeUnit, int] + for unit in Nanoseconds..Weeks: + timeParts[unit] = 0 + + for unit in Seconds..Days: + coeffs[unit] = coeffs[unit] div unitWeights[Seconds] + + var startTimepart = initTime( + nanosecond = startDt.nanosecond, + unix = startDt.hour * coeffs[Hours] + startDt.minute * coeffs[Minutes] + + startDt.second + ) + var endTimepart = initTime( + nanosecond = endDt.nanosecond, + unix = endDt.hour * coeffs[Hours] + endDt.minute * coeffs[Minutes] + + endDt.second + ) + # We wand timeParts for Seconds..Hours be positive, so we'll borrow one day + if endTimepart < startTimepart: + timeParts[Days] = -1 + + let diffTime = endTimepart - startTimepart + timeParts[Seconds] = diffTime.seconds.int() + #Nanoseconds - preliminary count + timeParts[Nanoseconds] = diffTime.nanoseconds + for unit in countdown(Milliseconds, Microseconds): + timeParts[unit] += timeParts[Nanoseconds] div coeffs[unit].int() + timeParts[Nanoseconds] -= timeParts[unit] * coeffs[unit].int() + + #Counting Seconds .. Hours - final, Days - preliminary + for unit in countdown(Days, Minutes): + timeParts[unit] += timeParts[Seconds] div coeffs[unit].int() + # Here is accounted the borrowed day + timeParts[Seconds] -= timeParts[unit] * coeffs[unit].int() + + # Set Nanoseconds .. Hours in result + result.nanoseconds = timeParts[Nanoseconds] + result.microseconds = timeParts[Microseconds] + result.milliseconds = timeParts[Milliseconds] + result.seconds = timeParts[Seconds] + result.minutes = timeParts[Minutes] + result.hours = timeParts[Hours] + + #Days + if endDt.monthday.int + timeParts[Days] < startDt.monthday.int(): + if endDt.month > 1.Month: + endDt.month -= 1.Month + else: + endDt.month = 12.Month + endDt.year -= 1 + timeParts[Days] += endDt.monthday.int() + getDaysInMonth( + endDt.month, endDt.year) - startDt.monthday.int() + else: + timeParts[Days] += endDt.monthday.int() - + startDt.monthday.int() + + result.days = timeParts[Days] + + #Months + if endDt.month < startDt.month: + result.months = endDt.month.int() + 12 - startDt.month.int() + endDt.year -= 1 + else: + result.months = endDt.month.int() - + startDt.month.int() + + # Years + result.years = endDt.year - startDt.year + +proc `+`*(time: Time, interval: TimeInterval): Time = + ## Adds `interval` to `time`. + ## If `interval` contains any years, months, weeks or days the operation + ## is performed in the local timezone. + runnableExamples: + let tm = fromUnix(0) + doAssert tm + 5.seconds == fromUnix(5) + + if interval.isStaticInterval: + time + evaluateStaticInterval(interval) + else: + toTime(time.local + interval) + +proc `+=`*(time: var Time, interval: TimeInterval) = + ## Modifies `time` by adding `interval`. + ## If `interval` contains any years, months, weeks or days the operation + ## is performed in the local timezone. + runnableExamples: + var tm = fromUnix(0) + tm += 5.seconds + doAssert tm == fromUnix(5) + + time = time + interval proc `-`*(time: Time, interval: TimeInterval): Time = ## Subtracts `interval` from Time `time`. - ## - ## ``echo getTime() - 1.day`` - result = toTime(time.local - interval) + ## If `interval` contains any years, months, weeks or days the operation + ## is performed in the local timezone. + runnableExamples: + let tm = fromUnix(5) + doAssert tm - 5.seconds == fromUnix(0) + + if interval.isStaticInterval: + time - evaluateStaticInterval(interval) + else: + toTime(time.local - interval) + +proc `-=`*(time: var Time, interval: TimeInterval) = + ## Modifies `time` by subtracting `interval`. + ## If `interval` contains any years, months, weeks or days the operation + ## is performed in the local timezone. + runnableExamples: + var tm = fromUnix(5) + tm -= 5.seconds + doAssert tm == fromUnix(0) + time = time - interval proc formatToken(dt: DateTime, token: string, buf: var string) = ## Helper of the format proc to parse individual tokens. @@ -746,12 +1429,16 @@ proc formatToken(dt: DateTime, token: string, buf: var string) = of "dddd": buf.add($dt.weekday) of "h": - buf.add($(if dt.hour > 12: dt.hour - 12 else: dt.hour)) + if dt.hour == 0: buf.add("12") + else: buf.add($(if dt.hour > 12: dt.hour - 12 else: dt.hour)) of "hh": - let amerHour = if dt.hour > 12: dt.hour - 12 else: dt.hour - if amerHour < 10: - buf.add('0') - buf.add($amerHour) + if dt.hour == 0: + buf.add("12") + else: + let amerHour = if dt.hour > 12: dt.hour - 12 else: dt.hour + if amerHour < 10: + buf.add('0') + buf.add($amerHour) of "H": buf.add($dt.hour) of "HH": @@ -843,6 +1530,12 @@ proc formatToken(dt: DateTime, token: string, buf: var string) = buf.add(':') if minutes < 10: buf.add('0') buf.add($minutes) + of "fff": + buf.add(intToStr(convert(Nanoseconds, Milliseconds, dt.nanosecond), 3)) + of "ffffff": + buf.add(intToStr(convert(Nanoseconds, Microseconds, dt.nanosecond), 6)) + of "fffffffff": + buf.add(intToStr(dt.nanosecond, 9)) of "": discard else: @@ -853,62 +1546,67 @@ proc format*(dt: DateTime, f: string): string {.tags: [].}= ## This procedure formats `dt` as specified by `f`. The following format ## specifiers are available: ## - ## ========== ================================================================================= ================================================ - ## Specifier Description Example - ## ========== ================================================================================= ================================================ - ## d Numeric value of the day of the month, it will be one or two digits long. ``1/04/2012 -> 1``, ``21/04/2012 -> 21`` - ## dd Same as above, but always two digits. ``1/04/2012 -> 01``, ``21/04/2012 -> 21`` - ## ddd Three letter string which indicates the day of the week. ``Saturday -> Sat``, ``Monday -> Mon`` - ## dddd Full string for the day of the week. ``Saturday -> Saturday``, ``Monday -> Monday`` - ## h The hours in one digit if possible. Ranging from 0-12. ``5pm -> 5``, ``2am -> 2`` - ## hh The hours in two digits always. If the hour is one digit 0 is prepended. ``5pm -> 05``, ``11am -> 11`` - ## H The hours in one digit if possible, randing from 0-24. ``5pm -> 17``, ``2am -> 2`` - ## HH The hours in two digits always. 0 is prepended if the hour is one digit. ``5pm -> 17``, ``2am -> 02`` - ## m The minutes in 1 digit if possible. ``5:30 -> 30``, ``2:01 -> 1`` - ## mm Same as above but always 2 digits, 0 is prepended if the minute is one digit. ``5:30 -> 30``, ``2:01 -> 01`` - ## M The month in one digit if possible. ``September -> 9``, ``December -> 12`` - ## MM The month in two digits always. 0 is prepended. ``September -> 09``, ``December -> 12`` - ## MMM Abbreviated three-letter form of the month. ``September -> Sep``, ``December -> Dec`` - ## MMMM Full month string, properly capitalized. ``September -> September`` - ## s Seconds as one digit if possible. ``00:00:06 -> 6`` - ## ss Same as above but always two digits. 0 is prepended. ``00:00:06 -> 06`` - ## t ``A`` when time is in the AM. ``P`` when time is in the PM. - ## tt Same as above, but ``AM`` and ``PM`` instead of ``A`` and ``P`` respectively. - ## y(yyyy) This displays the year to different digits. You most likely only want 2 or 4 'y's - ## yy Displays the year to two digits. ``2012 -> 12`` - ## yyyy Displays the year to four digits. ``2012 -> 2012`` - ## z Displays the timezone offset from UTC. ``GMT+7 -> +7``, ``GMT-5 -> -5`` - ## zz Same as above but with leading 0. ``GMT+7 -> +07``, ``GMT-5 -> -05`` - ## zzz Same as above but with ``:mm`` where *mm* represents minutes. ``GMT+7 -> +07:00``, ``GMT-5 -> -05:00`` - ## ========== ================================================================================= ================================================ + ## ============ ================================================================================= ================================================ + ## Specifier Description Example + ## ============ ================================================================================= ================================================ + ## d Numeric value of the day of the month, it will be one or two digits long. ``1/04/2012 -> 1``, ``21/04/2012 -> 21`` + ## dd Same as above, but always two digits. ``1/04/2012 -> 01``, ``21/04/2012 -> 21`` + ## ddd Three letter string which indicates the day of the week. ``Saturday -> Sat``, ``Monday -> Mon`` + ## dddd Full string for the day of the week. ``Saturday -> Saturday``, ``Monday -> Monday`` + ## h The hours in one digit if possible. Ranging from 0-12. ``5pm -> 5``, ``2am -> 2`` + ## hh The hours in two digits always. If the hour is one digit 0 is prepended. ``5pm -> 05``, ``11am -> 11`` + ## H The hours in one digit if possible, randing from 0-24. ``5pm -> 17``, ``2am -> 2`` + ## HH The hours in two digits always. 0 is prepended if the hour is one digit. ``5pm -> 17``, ``2am -> 02`` + ## m The minutes in 1 digit if possible. ``5:30 -> 30``, ``2:01 -> 1`` + ## mm Same as above but always 2 digits, 0 is prepended if the minute is one digit. ``5:30 -> 30``, ``2:01 -> 01`` + ## M The month in one digit if possible. ``September -> 9``, ``December -> 12`` + ## MM The month in two digits always. 0 is prepended. ``September -> 09``, ``December -> 12`` + ## MMM Abbreviated three-letter form of the month. ``September -> Sep``, ``December -> Dec`` + ## MMMM Full month string, properly capitalized. ``September -> September`` + ## s Seconds as one digit if possible. ``00:00:06 -> 6`` + ## ss Same as above but always two digits. 0 is prepended. ``00:00:06 -> 06`` + ## t ``A`` when time is in the AM. ``P`` when time is in the PM. + ## tt Same as above, but ``AM`` and ``PM`` instead of ``A`` and ``P`` respectively. + ## y(yyyy) This displays the year to different digits. You most likely only want 2 or 4 'y's + ## yy Displays the year to two digits. ``2012 -> 12`` + ## yyyy Displays the year to four digits. ``2012 -> 2012`` + ## z Displays the timezone offset from UTC. ``GMT+7 -> +7``, ``GMT-5 -> -5`` + ## zz Same as above but with leading 0. ``GMT+7 -> +07``, ``GMT-5 -> -05`` + ## zzz Same as above but with ``:mm`` where *mm* represents minutes. ``GMT+7 -> +07:00``, ``GMT-5 -> -05:00`` + ## fff Milliseconds display ``1000000 nanoseconds -> 1`` + ## ffffff Microseconds display ``1000000 nanoseconds -> 1000`` + ## fffffffff Nanoseconds display ``1000000 nanoseconds -> 1000000`` + ## ============ ================================================================================= ================================================ ## ## Other strings can be inserted by putting them in ``''``. For example ## ``hh'->'mm`` will give ``01->56``. The following characters can be ## inserted without quoting them: ``:`` ``-`` ``(`` ``)`` ``/`` ``[`` ``]`` ## ``,``. However you don't need to necessarily separate format specifiers, a ## unambiguous format string like ``yyyyMMddhhmmss`` is valid too. + runnableExamples: + let dt = initDateTime(01, mJan, 2000, 12, 00, 00, 01, utc()) + doAssert format(dt, "yyyy-MM-dd'T'HH:mm:ss'.'fffffffffzzz") == "2000-01-01T12:00:00.000000001+00:00" result = "" var i = 0 var currentF = "" - while true: + while i < f.len: case f[i] - of ' ', '-', '/', ':', '\'', '\0', '(', ')', '[', ']', ',': + of ' ', '-', '/', ':', '\'', '(', ')', '[', ']', ',': formatToken(dt, currentF, result) currentF = "" - if f[i] == '\0': break if f[i] == '\'': inc(i) # Skip ' - while f[i] != '\'' and f.len-1 > i: + while i < f.len-1 and f[i] != '\'': result.add(f[i]) inc(i) else: result.add(f[i]) else: # Check if the letter being added matches previous accumulated buffer. - if currentF.len < 1 or currentF[high(currentF)] == f[i]: + if currentF.len == 0 or currentF[high(currentF)] == f[i]: currentF.add(f[i]) else: formatToken(dt, currentF, result) @@ -916,10 +1614,27 @@ proc format*(dt: DateTime, f: string): string {.tags: [].}= currentF = "" inc(i) + formatToken(dt, currentF, result) + +proc format*(time: Time, f: string, zone_info: proc(t: Time): DateTime): string {.tags: [].} = + ## converts a `Time` value to a string representation. It will use format from + ## ``format(dt: DateTime, f: string)``. + runnableExamples: + var dt = initDateTime(01, mJan, 1970, 00, 00, 00, local()) + var tm = dt.toTime() + doAssert format(tm, "yyyy-MM-dd'T'HH:mm:ss", local) == "1970-01-01T00:00:00" + dt = initDateTime(01, mJan, 1970, 00, 00, 00, utc()) + tm = dt.toTime() + doAssert format(tm, "yyyy-MM-dd'T'HH:mm:ss", utc) == "1970-01-01T00:00:00" + + zone_info(time).format(f) proc `$`*(dt: DateTime): string {.tags: [], raises: [], benign.} = ## Converts a `DateTime` object to a string representation. ## It uses the format ``yyyy-MM-dd'T'HH-mm-sszzz``. + runnableExamples: + let dt = initDateTime(01, mJan, 2000, 12, 00, 00, utc()) + doAssert $dt == "2000-01-01T12:00:00+00:00" try: result = format(dt, "yyyy-MM-dd'T'HH:mm:sszzz") # todo: optimize this except ValueError: assert false # cannot happen because format string is valid @@ -927,12 +1642,24 @@ proc `$`*(dt: DateTime): string {.tags: [], raises: [], benign.} = proc `$`*(time: Time): string {.tags: [], raises: [], benign.} = ## converts a `Time` value to a string representation. It will use the local ## time zone and use the format ``yyyy-MM-dd'T'HH-mm-sszzz``. + runnableExamples: + let dt = initDateTime(01, mJan, 1970, 00, 00, 00, local()) + let tm = dt.toTime() + doAssert $tm == "1970-01-01T00:00:00" & format(dt, "zzz") $time.local {.pop.} proc parseToken(dt: var DateTime; token, value: string; j: var int) = ## Helper of the parse proc to parse individual tokens. + + # Overwrite system.`[]` to raise a ValueError on index out of bounds. + proc `[]`[T, U](s: string, x: HSlice[T, U]): string = + if x.a >= s.len or x.b >= s.len: + raise newException(ValueError, "Value is missing required tokens, got: " & + s) + return system.`[]`(s, x) + var sv: int case token of "d": @@ -1004,58 +1731,58 @@ proc parseToken(dt: var DateTime; token, value: string; j: var int) = dt.month = month.Month of "MMM": case value[j..j+2].toLowerAscii(): - of "jan": dt.month = mJan - of "feb": dt.month = mFeb - of "mar": dt.month = mMar - of "apr": dt.month = mApr - of "may": dt.month = mMay - of "jun": dt.month = mJun - of "jul": dt.month = mJul - of "aug": dt.month = mAug - of "sep": dt.month = mSep - of "oct": dt.month = mOct - of "nov": dt.month = mNov - of "dec": dt.month = mDec + of "jan": dt.month = mJan + of "feb": dt.month = mFeb + of "mar": dt.month = mMar + of "apr": dt.month = mApr + of "may": dt.month = mMay + of "jun": dt.month = mJun + of "jul": dt.month = mJul + of "aug": dt.month = mAug + of "sep": dt.month = mSep + of "oct": dt.month = mOct + of "nov": dt.month = mNov + of "dec": dt.month = mDec else: raise newException(ValueError, "Couldn't parse month (MMM), got: " & value) j += 3 of "MMMM": if value.len >= j+7 and value[j..j+6].cmpIgnoreCase("january") == 0: - dt.month = mJan + dt.month = mJan j += 7 elif value.len >= j+8 and value[j..j+7].cmpIgnoreCase("february") == 0: - dt.month = mFeb + dt.month = mFeb j += 8 elif value.len >= j+5 and value[j..j+4].cmpIgnoreCase("march") == 0: - dt.month = mMar + dt.month = mMar j += 5 elif value.len >= j+5 and value[j..j+4].cmpIgnoreCase("april") == 0: - dt.month = mApr + dt.month = mApr j += 5 elif value.len >= j+3 and value[j..j+2].cmpIgnoreCase("may") == 0: - dt.month = mMay + dt.month = mMay j += 3 elif value.len >= j+4 and value[j..j+3].cmpIgnoreCase("june") == 0: - dt.month = mJun + dt.month = mJun j += 4 elif value.len >= j+4 and value[j..j+3].cmpIgnoreCase("july") == 0: - dt.month = mJul + dt.month = mJul j += 4 elif value.len >= j+6 and value[j..j+5].cmpIgnoreCase("august") == 0: - dt.month = mAug + dt.month = mAug j += 6 elif value.len >= j+9 and value[j..j+8].cmpIgnoreCase("september") == 0: - dt.month = mSep + dt.month = mSep j += 9 elif value.len >= j+7 and value[j..j+6].cmpIgnoreCase("october") == 0: - dt.month = mOct + dt.month = mOct j += 7 elif value.len >= j+8 and value[j..j+7].cmpIgnoreCase("november") == 0: - dt.month = mNov + dt.month = mNov j += 8 elif value.len >= j+8 and value[j..j+7].cmpIgnoreCase("december") == 0: - dt.month = mDec + dt.month = mDec j += 8 else: raise newException(ValueError, @@ -1068,11 +1795,15 @@ proc parseToken(dt: var DateTime; token, value: string; j: var int) = dt.second = value[j..j+1].parseInt() j += 2 of "t": - if value[j] == 'P' and dt.hour > 0 and dt.hour < 12: + if value[j] == 'A' and dt.hour == 12: + dt.hour = 0 + elif value[j] == 'P' and dt.hour > 0 and dt.hour < 12: dt.hour += 12 j += 1 of "tt": - if value[j..j+1] == "PM" and dt.hour > 0 and dt.hour < 12: + if value[j..j+1] == "AM" and dt.hour == 12: + dt.hour = 0 + elif value[j..j+1] == "PM" and dt.hour > 0 and dt.hour < 12: dt.hour += 12 j += 2 of "yy": @@ -1086,48 +1817,56 @@ proc parseToken(dt: var DateTime; token, value: string; j: var int) = j += 4 of "z": dt.isDst = false - if value[j] == '+': + let ch = if j < value.len: value[j] else: '\0' + if ch == '+': dt.utcOffset = 0 - parseInt($value[j+1]) * secondsInHour - elif value[j] == '-': + elif ch == '-': dt.utcOffset = parseInt($value[j+1]) * secondsInHour - elif value[j] == 'Z': + elif ch == 'Z': dt.utcOffset = 0 j += 1 return else: raise newException(ValueError, - "Couldn't parse timezone offset (z), got: " & value[j]) + "Couldn't parse timezone offset (z), got: " & ch) j += 2 of "zz": dt.isDst = false - if value[j] == '+': + let ch = if j < value.len: value[j] else: '\0' + if ch == '+': dt.utcOffset = 0 - value[j+1..j+2].parseInt() * secondsInHour - elif value[j] == '-': + elif ch == '-': dt.utcOffset = value[j+1..j+2].parseInt() * secondsInHour - elif value[j] == 'Z': + elif ch == 'Z': dt.utcOffset = 0 j += 1 return else: raise newException(ValueError, - "Couldn't parse timezone offset (zz), got: " & value[j]) + "Couldn't parse timezone offset (zz), got: " & ch) j += 3 of "zzz": dt.isDst = false var factor = 0 - if value[j] == '+': factor = -1 - elif value[j] == '-': factor = 1 - elif value[j] == 'Z': + let ch = if j < value.len: value[j] else: '\0' + if ch == '+': factor = -1 + elif ch == '-': factor = 1 + elif ch == 'Z': dt.utcOffset = 0 j += 1 return else: raise newException(ValueError, - "Couldn't parse timezone offset (zzz), got: " & value[j]) + "Couldn't parse timezone offset (zzz), got: " & ch) dt.utcOffset = factor * value[j+1..j+2].parseInt() * secondsInHour j += 4 dt.utcOffset += factor * value[j..j+1].parseInt() * 60 j += 2 + of "fff", "ffffff", "fffffffff": + var numStr = "" + let n = parseWhile(value[j..len(value) - 1], numStr, {'0'..'9'}) + dt.nanosecond = parseInt(numStr) * (10 ^ (9 - n)) + j += n else: # Ignore the token and move forward in the value string by the same length j += token.len @@ -1141,39 +1880,44 @@ proc parse*(value, layout: string, zone: Timezone = local()): DateTime = ## parsed, then the input will be assumed to be specified in the `zone` timezone ## already, so no timezone conversion will be done in that case. ## - ## ========== ================================================================================= ================================================ - ## Specifier Description Example - ## ========== ================================================================================= ================================================ - ## d Numeric value of the day of the month, it will be one or two digits long. ``1/04/2012 -> 1``, ``21/04/2012 -> 21`` - ## dd Same as above, but always two digits. ``1/04/2012 -> 01``, ``21/04/2012 -> 21`` - ## ddd Three letter string which indicates the day of the week. ``Saturday -> Sat``, ``Monday -> Mon`` - ## dddd Full string for the day of the week. ``Saturday -> Saturday``, ``Monday -> Monday`` - ## h The hours in one digit if possible. Ranging from 0-12. ``5pm -> 5``, ``2am -> 2`` - ## hh The hours in two digits always. If the hour is one digit 0 is prepended. ``5pm -> 05``, ``11am -> 11`` - ## H The hours in one digit if possible, randing from 0-24. ``5pm -> 17``, ``2am -> 2`` - ## HH The hours in two digits always. 0 is prepended if the hour is one digit. ``5pm -> 17``, ``2am -> 02`` - ## m The minutes in 1 digit if possible. ``5:30 -> 30``, ``2:01 -> 1`` - ## mm Same as above but always 2 digits, 0 is prepended if the minute is one digit. ``5:30 -> 30``, ``2:01 -> 01`` - ## M The month in one digit if possible. ``September -> 9``, ``December -> 12`` - ## MM The month in two digits always. 0 is prepended. ``September -> 09``, ``December -> 12`` - ## MMM Abbreviated three-letter form of the month. ``September -> Sep``, ``December -> Dec`` - ## MMMM Full month string, properly capitalized. ``September -> September`` - ## s Seconds as one digit if possible. ``00:00:06 -> 6`` - ## ss Same as above but always two digits. 0 is prepended. ``00:00:06 -> 06`` - ## t ``A`` when time is in the AM. ``P`` when time is in the PM. - ## tt Same as above, but ``AM`` and ``PM`` instead of ``A`` and ``P`` respectively. - ## yy Displays the year to two digits. ``2012 -> 12`` - ## yyyy Displays the year to four digits. ``2012 -> 2012`` - ## z Displays the timezone offset from UTC. ``Z`` is parsed as ``+0`` ``GMT+7 -> +7``, ``GMT-5 -> -5`` - ## zz Same as above but with leading 0. ``GMT+7 -> +07``, ``GMT-5 -> -05`` - ## zzz Same as above but with ``:mm`` where *mm* represents minutes. ``GMT+7 -> +07:00``, ``GMT-5 -> -05:00`` - ## ========== ================================================================================= ================================================ + ## ======================= ================================================================================= ================================================ + ## Specifier Description Example + ## ======================= ================================================================================= ================================================ + ## d Numeric value of the day of the month, it will be one or two digits long. ``1/04/2012 -> 1``, ``21/04/2012 -> 21`` + ## dd Same as above, but always two digits. ``1/04/2012 -> 01``, ``21/04/2012 -> 21`` + ## ddd Three letter string which indicates the day of the week. ``Saturday -> Sat``, ``Monday -> Mon`` + ## dddd Full string for the day of the week. ``Saturday -> Saturday``, ``Monday -> Monday`` + ## h The hours in one digit if possible. Ranging from 0-12. ``5pm -> 5``, ``2am -> 2`` + ## hh The hours in two digits always. If the hour is one digit 0 is prepended. ``5pm -> 05``, ``11am -> 11`` + ## H The hours in one digit if possible, randing from 0-24. ``5pm -> 17``, ``2am -> 2`` + ## HH The hours in two digits always. 0 is prepended if the hour is one digit. ``5pm -> 17``, ``2am -> 02`` + ## m The minutes in 1 digit if possible. ``5:30 -> 30``, ``2:01 -> 1`` + ## mm Same as above but always 2 digits, 0 is prepended if the minute is one digit. ``5:30 -> 30``, ``2:01 -> 01`` + ## M The month in one digit if possible. ``September -> 9``, ``December -> 12`` + ## MM The month in two digits always. 0 is prepended. ``September -> 09``, ``December -> 12`` + ## MMM Abbreviated three-letter form of the month. ``September -> Sep``, ``December -> Dec`` + ## MMMM Full month string, properly capitalized. ``September -> September`` + ## s Seconds as one digit if possible. ``00:00:06 -> 6`` + ## ss Same as above but always two digits. 0 is prepended. ``00:00:06 -> 06`` + ## t ``A`` when time is in the AM. ``P`` when time is in the PM. + ## tt Same as above, but ``AM`` and ``PM`` instead of ``A`` and ``P`` respectively. + ## yy Displays the year to two digits. ``2012 -> 12`` + ## yyyy Displays the year to four digits. ``2012 -> 2012`` + ## z Displays the timezone offset from UTC. ``Z`` is parsed as ``+0`` ``GMT+7 -> +7``, ``GMT-5 -> -5`` + ## zz Same as above but with leading 0. ``GMT+7 -> +07``, ``GMT-5 -> -05`` + ## zzz Same as above but with ``:mm`` where *mm* represents minutes. ``GMT+7 -> +07:00``, ``GMT-5 -> -05:00`` + ## fff/ffffff/fffffffff for consistency with format - nanoseconds ``1 -> 1 nanosecond`` + ## ======================= ================================================================================= ================================================ ## ## Other strings can be inserted by putting them in ``''``. For example ## ``hh'->'mm`` will give ``01->56``. The following characters can be ## inserted without quoting them: ``:`` ``-`` ``(`` ``)`` ``/`` ``[`` ``]`` ## ``,``. However you don't need to necessarily separate format specifiers, a ## unambiguous format string like ``yyyyMMddhhmmss`` is valid too. + runnableExamples: + let tStr = "1970-01-01T00:00:00.0+00:00" + doAssert parse(tStr, "yyyy-MM-dd'T'HH:mm:ss.fffzzz") == fromUnix(0).utc + var i = 0 # pointer for format string var j = 0 # pointer for value string var token = "" @@ -1182,22 +1926,21 @@ proc parse*(value, layout: string, zone: Timezone = local()): DateTime = dt.hour = 0 dt.minute = 0 dt.second = 0 + dt.nanosecond = 0 dt.isDst = true # using this is flag for checking whether a timezone has \ # been read (because DST is always false when a tz is parsed) - while true: + while i < layout.len: case layout[i] - of ' ', '-', '/', ':', '\'', '\0', '(', ')', '[', ']', ',': + of ' ', '-', '/', ':', '\'', '(', ')', '[', ']', ',': if token.len > 0: parseToken(dt, token, value, j) # Reset token token = "" - # Break if at end of line - if layout[i] == '\0': break # Skip separator and everything between single quotes # These are literals in both the layout and the value string if layout[i] == '\'': inc(i) - while layout[i] != '\'' and layout.len-1 > i: + while i < layout.len-1 and layout[i] != '\'': inc(i) inc(j) inc(i) @@ -1206,13 +1949,15 @@ proc parse*(value, layout: string, zone: Timezone = local()): DateTime = inc(j) else: # Check if the letter being added matches previous accumulated buffer. - if token.len < 1 or token[high(token)] == layout[i]: + if token.len == 0 or token[high(token)] == layout[i]: token.add(layout[i]) inc(i) else: parseToken(dt, token, value, j) token = "" + if i >= layout.len and token.len > 0: + parseToken(dt, token, value, j) if dt.isDst: # No timezone parsed - assume timezone is `zone` result = initDateTime(zone.zoneInfoFromTz(dt.toAdjTime), zone) @@ -1220,6 +1965,13 @@ proc parse*(value, layout: string, zone: Timezone = local()): DateTime = # Otherwise convert to `zone` result = dt.toTime.inZone(zone) +proc parseTime*(value, layout: string, zone: Timezone): Time = + ## Simple wrapper for parsing string to time + runnableExamples: + let tStr = "1970-01-01T00:00:00+00:00" + doAssert parseTime(tStr, "yyyy-MM-dd'T'HH:mm:sszzz", local()) == fromUnix(0) + parse(value, layout, zone).toTime() + proc countLeapYears*(yearSpan: int): int = ## Returns the number of leap years spanned by a given number of years. ## @@ -1248,92 +2000,56 @@ proc toTimeInterval*(time: Time): TimeInterval = ## Converts a Time to a TimeInterval. ## ## To be used when diffing times. - ## - ## .. code-block:: nim - ## let a = fromSeconds(1_000_000_000) - ## let b = fromSeconds(1_500_000_000) - ## echo a, " ", b # real dates - ## echo a.toTimeInterval # meaningless value, don't use it by itself - ## echo b.toTimeInterval - a.toTimeInterval - ## # (milliseconds: 0, seconds: -40, minutes: -6, hours: 1, days: 5, months: -2, years: 16) - # Milliseconds not available from Time - var dt = time.local - initInterval(0, dt.second, dt.minute, dt.hour, dt.monthday, dt.month.ord - 1, dt.year) + runnableExamples: + let a = fromUnix(10) + let dt = initDateTime(01, mJan, 1970, 00, 00, 00, local()) + doAssert a.toTimeInterval() == initTimeInterval( + years=1970, days=1, seconds=10, hours=convert( + Seconds, Hours, -dt.utcOffset + ) + ) -proc initDateTime*(monthday: MonthdayRange, month: Month, year: int, - hour: HourRange, minute: MinuteRange, second: SecondRange, zone: Timezone = local()): DateTime = - ## Create a new ``DateTime`` in the specified timezone. - assertValidDate monthday, month, year - doAssert monthday <= getDaysInMonth(month, year), "Invalid date: " & $month & " " & $monthday & ", " & $year - let dt = DateTime( - monthday: monthday, - year: year, - month: month, - hour: hour, - minute: minute, - second: second - ) - result = initDateTime(zone.zoneInfoFromTz(dt.toAdjTime), zone) + var dt = time.local + initTimeInterval(dt.nanosecond, 0, 0, dt.second, dt.minute, dt.hour, + dt.monthday, 0, dt.month.ord - 1, dt.year) when not defined(JS): - proc epochTime*(): float {.rtl, extern: "nt$1", tags: [TimeEffect].} - ## gets time after the UNIX epoch (1970) in seconds. It is a float - ## because sub-second resolution is likely to be supported (depending - ## on the hardware/OS). - - proc cpuTime*(): float {.rtl, extern: "nt$1", tags: [TimeEffect].} - ## gets time spent that the CPU spent to run the current process in - ## seconds. This may be more useful for benchmarking than ``epochTime``. - ## However, it may measure the real time instead (depending on the OS). - ## The value of the result has no meaning. - ## To generate useful timing values, take the difference between - ## the results of two ``cpuTime`` calls: - ## - ## .. code-block:: nim - ## var t0 = cpuTime() - ## doWork() - ## echo "CPU time [s] ", cpuTime() - t0 - -when defined(JS): - proc getTime(): Time = - (newDate().getTime() div 1000).Time - - proc epochTime*(): float {.tags: [TimeEffect].} = - newDate().getTime() / 1000 - -else: type Clock {.importc: "clock_t".} = distinct int - proc timec(timer: ptr CTime): CTime {. - importc: "time", header: "<time.h>", tags: [].} - proc getClock(): Clock {.importc: "clock", header: "<time.h>", tags: [TimeEffect].} var clocksPerSec {.importc: "CLOCKS_PER_SEC", nodecl.}: int - proc getTime(): Time = - timec(nil).Time - - const - epochDiff = 116444736000000000'i64 - rateDiff = 10000000'i64 # 100 nsecs - - proc unixTimeToWinTime*(time: CTime): int64 = - ## converts a UNIX `Time` (``time_t``) to a Windows file time - result = int64(time) * rateDiff + epochDiff - - proc winTimeToUnixTime*(time: int64): CTime = - ## converts a Windows time to a UNIX `Time` (``time_t``) - result = CTime((time - epochDiff) div rateDiff) - when not defined(useNimRtl): - proc epochTime(): float = + proc cpuTime*(): float {.rtl, extern: "nt$1", tags: [TimeEffect].} = + ## gets time spent that the CPU spent to run the current process in + ## seconds. This may be more useful for benchmarking than ``epochTime``. + ## However, it may measure the real time instead (depending on the OS). + ## The value of the result has no meaning. + ## To generate useful timing values, take the difference between + ## the results of two ``cpuTime`` calls: + runnableExamples: + var t0 = cpuTime() + # some useless work here (calculate fibonacci) + var fib = @[0, 1, 1] + for i in 1..10: + fib.add(fib[^1] + fib[^2]) + echo "CPU time [s] ", cpuTime() - t0 + echo "Fib is [s] ", fib + result = toFloat(int(getClock())) / toFloat(clocksPerSec) + + proc epochTime*(): float {.rtl, extern: "nt$1", tags: [TimeEffect].} = + ## gets time after the UNIX epoch (1970) in seconds. It is a float + ## because sub-second resolution is likely to be supported (depending + ## on the hardware/OS). + ## + ## ``getTime`` should generally be prefered over this proc. when defined(posix): var a: Timeval - posix_gettimeofday(a) - result = toFloat(a.tv_sec) + toFloat(a.tv_usec)*0.00_0001 + gettimeofday(a) + result = toBiggestFloat(a.tv_sec.int64) + toFloat(a.tv_usec)*0.00_0001 elif defined(windows): var f: winlean.FILETIME getSystemTimeAsFileTime(f) @@ -1344,30 +2060,50 @@ else: else: {.error: "unknown OS".} - proc cpuTime(): float = - result = toFloat(int(getClock())) / toFloat(clocksPerSec) +when defined(JS): + proc epochTime*(): float {.tags: [TimeEffect].} = + newDate().getTime() / 1000 # Deprecated procs +when not defined(JS): + proc unixTimeToWinTime*(time: CTime): int64 {.deprecated: "Use toWinTime instead".} = + ## Converts a UNIX `Time` (``time_t``) to a Windows file time + ## + ## **Deprecated:** use ``toWinTime`` instead. + result = int64(time) * rateDiff + epochDiff + + proc winTimeToUnixTime*(time: int64): CTime {.deprecated: "Use fromWinTime instead".} = + ## Converts a Windows time to a UNIX `Time` (``time_t``) + ## + ## **Deprecated:** use ``fromWinTime`` instead. + result = CTime((time - epochDiff) div rateDiff) + +proc initInterval*(seconds, minutes, hours, days, months, + years: int = 0): TimeInterval {.deprecated.} = + ## **Deprecated since v0.18.0:** use ``initTimeInterval`` instead. + initTimeInterval(0, 0, 0, seconds, minutes, hours, days, 0, months, years) + proc fromSeconds*(since1970: float): Time {.tags: [], raises: [], benign, deprecated.} = ## Takes a float which contains the number of seconds since the unix epoch and ## returns a time object. ## ## **Deprecated since v0.18.0:** use ``fromUnix`` instead - Time(since1970) + let nanos = ((since1970 - since1970.int64.float) * convert(Seconds, Nanoseconds, 1).float).int + initTime(since1970.int64, nanos) proc fromSeconds*(since1970: int64): Time {.tags: [], raises: [], benign, deprecated.} = ## Takes an int which contains the number of seconds since the unix epoch and ## returns a time object. ## ## **Deprecated since v0.18.0:** use ``fromUnix`` instead - Time(since1970) + fromUnix(since1970) proc toSeconds*(time: Time): float {.tags: [], raises: [], benign, deprecated.} = ## Returns the time in seconds since the unix epoch. ## - ## **Deprecated since v0.18.0:** use ``toUnix`` instead - float(time) + ## **Deprecated since v0.18.0:** use ``fromUnix`` instead + time.seconds.float + time.nanosecond / convert(Seconds, Nanoseconds, 1) proc getLocalTime*(time: Time): DateTime {.tags: [], raises: [], benign, deprecated.} = ## Converts the calendar time `time` to broken-time representation, @@ -1378,7 +2114,7 @@ proc getLocalTime*(time: Time): DateTime {.tags: [], raises: [], benign, depreca proc getGMTime*(time: Time): DateTime {.tags: [], raises: [], benign, deprecated.} = ## Converts the calendar time `time` to broken-down time representation, - ## expressed in Coordinated Universal Time (UTC). + ## expressed in Coordinated Universal Time (UTC). ## ## **Deprecated since v0.18.0:** use ``utc`` instead time.utc @@ -1391,7 +2127,8 @@ proc getTimezone*(): int {.tags: [TimeEffect], raises: [], benign, deprecated.} when defined(JS): return newDate().getTimezoneOffset() * 60 elif defined(freebsd) or defined(netbsd) or defined(openbsd): - var a = timec(nil) + var a: CTime + discard time(a) let lt = localtime(addr(a)) # BSD stores in `gmtoff` offset east of UTC in seconds, # but posix systems using west of UTC in seconds @@ -1402,18 +2139,17 @@ proc getTimezone*(): int {.tags: [TimeEffect], raises: [], benign, deprecated.} proc timeInfoToTime*(dt: DateTime): Time {.tags: [], benign, deprecated.} = ## Converts a broken-down time structure to calendar time representation. ## - ## **Warning:** This procedure is deprecated since version 0.14.0. - ## Use ``toTime`` instead. + ## **Deprecated since v0.14.0:** use ``toTime`` instead. dt.toTime when defined(JS): - var startMilsecs = getTime() + var start = getTime() proc getStartMilsecs*(): int {.deprecated, tags: [TimeEffect], benign.} = - ## get the milliseconds from the start of the program. **Deprecated since - ## version 0.8.10.** Use ``epochTime`` or ``cpuTime`` instead. - when defined(JS): - ## get the milliseconds from the start of the program - return int(getTime() - startMilsecs) + ## get the milliseconds from the start of the program. + ## **Deprecated since v0.8.10:** use ``epochTime`` or ``cpuTime`` instead. + let dur = getTime() - start + result = (convert(Seconds, Milliseconds, dur.seconds) + + convert(Nanoseconds, Milliseconds, dur.nanosecond)).int else: proc getStartMilsecs*(): int {.deprecated, tags: [TimeEffect], benign.} = when defined(macosx): @@ -1421,60 +2157,22 @@ else: else: result = int(getClock()) div (clocksPerSec div 1000) -proc miliseconds*(t: TimeInterval): int {.deprecated.} = - t.milliseconds - proc timeToTimeInterval*(t: Time): TimeInterval {.deprecated.} = ## Converts a Time to a TimeInterval. ## - ## **Warning:** This procedure is deprecated since version 0.14.0. - ## Use ``toTimeInterval`` instead. + ## **Deprecated since v0.14.0:** use ``toTimeInterval`` instead. # Milliseconds not available from Time t.toTimeInterval() -proc timeToTimeInfo*(t: Time): DateTime {.deprecated.} = - ## Converts a Time to DateTime. - ## - ## **Warning:** This procedure is deprecated since version 0.14.0. - ## Use ``inZone`` instead. - const epochStartYear = 1970 - - let - secs = t.toSeconds().int - daysSinceEpoch = secs div secondsInDay - (yearsSinceEpoch, daysRemaining) = countYearsAndDays(daysSinceEpoch) - daySeconds = secs mod secondsInDay - - y = yearsSinceEpoch + epochStartYear - - var - mon = mJan - days = daysRemaining - daysInMonth = getDaysInMonth(mon, y) - - # calculate month and day remainder - while days > daysInMonth and mon <= mDec: - days -= daysInMonth - mon.inc - daysInMonth = getDaysInMonth(mon, y) - - let - yd = daysRemaining - m = mon # month is zero indexed enum - md = days - # NB: month is zero indexed but dayOfWeek expects 1 indexed. - wd = getDayOfWeek(days, mon, y).Weekday - h = daySeconds div secondsInHour + 1 - mi = (daySeconds mod secondsInHour) div secondsInMin - s = daySeconds mod secondsInMin - result = DateTime(year: y, yearday: yd, month: m, monthday: md, weekday: wd, hour: h, minute: mi, second: s) - proc getDayOfWeek*(day, month, year: int): WeekDay {.tags: [], raises: [], benign, deprecated.} = + ## **Deprecated since v0.18.0:** use + ## ``getDayOfWeek(monthday: MonthdayRange; month: Month; year: int)`` instead. getDayOfWeek(day, month.Month, year) proc getDayOfWeekJulian*(day, month, year: int): WeekDay {.deprecated.} = ## Returns the day of the week enum from day, month and year, ## according to the Julian calendar. + ## **Deprecated since v0.18.0:** # Day & month start from one. let a = (14 - month) div 12 diff --git a/lib/pure/unicode.nim b/lib/pure/unicode.nim index 257c620f7..bfd01be55 100644 --- a/lib/pure/unicode.nim +++ b/lib/pure/unicode.nim @@ -9,7 +9,7 @@ ## This module provides support to handle the Unicode UTF-8 encoding. -{.deadCodeElim: on.} +{.deadCodeElim: on.} # dce option deprecated include "system/inclrtl" @@ -1826,8 +1826,8 @@ when isMainModule: doAssert(runeSubStr(s, 17, 1) == "€") # echo runeStrAtPos(s, 18) # index error - doAssert(runeSubStr(s, 0) == "Hänsel ««: 10,00€") - doAssert(runeSubStr(s, -18) == "Hänsel ««: 10,00€") + doAssert(runeSubStr(s, 0) == "Hänsel ««: 10,00€") + doAssert(runeSubStr(s, -18) == "Hänsel ««: 10,00€") doAssert(runeSubStr(s, 10) == ": 10,00€") doAssert(runeSubStr(s, 18) == "") doAssert(runeSubStr(s, 0, 10) == "Hänsel ««") @@ -1840,7 +1840,7 @@ when isMainModule: doAssert(runeSubStr(s, -6, 5) == "10,00") doAssert(runeSubStr(s, -6, -1) == "10,00") - doAssert(runeSubStr(s, 0, 100) == "Hänsel ««: 10,00€") - doAssert(runeSubStr(s, -100, 100) == "Hänsel ««: 10,00€") + doAssert(runeSubStr(s, 0, 100) == "Hänsel ««: 10,00€") + doAssert(runeSubStr(s, -100, 100) == "Hänsel ««: 10,00€") doAssert(runeSubStr(s, 0, -100) == "") doAssert(runeSubStr(s, 100, -100) == "") diff --git a/lib/pure/unittest.nim b/lib/pure/unittest.nim index fbce087ff..d804ba7c8 100644 --- a/lib/pure/unittest.nim +++ b/lib/pure/unittest.nim @@ -121,9 +121,16 @@ type ConsoleOutputFormatter* = ref object of OutputFormatter colorOutput: bool ## Have test results printed in color. - ## Default is true for the non-js target - ## unless, the environment variable - ## ``NIMTEST_NO_COLOR`` is set. + ## Default is true for the non-js target, + ## for which ``stdout`` is a tty. + ## Setting the environment variable + ## ``NIMTEST_COLOR`` to ``always`` or + ## ``never`` changes the default for the + ## non-js target to true or false respectively. + ## The deprecated environment variable + ## ``NIMTEST_NO_COLOR``, when set, + ## changes the defualt to true, if + ## ``NIMTEST_COLOR`` is undefined. outputLevel: OutputLevel ## Set the verbosity of test results. ## Default is ``PRINT_ALL``, unless @@ -150,6 +157,7 @@ var checkpoints {.threadvar.}: seq[string] formatters {.threadvar.}: seq[OutputFormatter] testsFilters {.threadvar.}: HashSet[string] + disabledParamFiltering {.threadvar.}: bool when declared(stdout): abortOnError = existsEnv("NIMTEST_ABORT_ON_ERROR") @@ -185,7 +193,15 @@ proc defaultConsoleFormatter*(): ConsoleOutputFormatter = # Reading settings # On a terminal this branch is executed var envOutLvl = os.getEnv("NIMTEST_OUTPUT_LVL").string - var colorOutput = not existsEnv("NIMTEST_NO_COLOR") + var colorOutput = isatty(stdout) + if existsEnv("NIMTEST_COLOR"): + let colorEnv = getenv("NIMTEST_COLOR") + if colorEnv == "never": + colorOutput = false + elif colorEnv == "always": + colorOutput = true + elif existsEnv("NIMTEST_NO_COLOR"): + colorOutput = false var outputLevel = PRINT_ALL if envOutLvl.len > 0: for opt in countup(low(OutputLevel), high(OutputLevel)): @@ -379,7 +395,7 @@ proc ensureInitialized() = if formatters == nil: formatters = @[OutputFormatter(defaultConsoleFormatter())] - if not testsFilters.isValid: + if not disabledParamFiltering and not testsFilters.isValid: testsFilters.init() when declared(paramCount): # Read tests to run from the command line. @@ -446,6 +462,8 @@ template suite*(name, body) {.dirty.} = finally: suiteEnded() +template exceptionTypeName(e: typed): string = $e.name + template test*(name, body) {.dirty.} = ## Define a single test case identified by `name`. ## @@ -460,7 +478,7 @@ template test*(name, body) {.dirty.} = ## .. code-block:: ## ## [OK] roses are red - bind shouldRun, checkpoints, formatters, ensureInitialized, testEnded + bind shouldRun, checkpoints, formatters, ensureInitialized, testEnded, exceptionTypeName ensureInitialized() @@ -479,8 +497,10 @@ template test*(name, body) {.dirty.} = except: when not defined(js): - checkpoint("Unhandled exception: " & getCurrentExceptionMsg()) - var stackTrace {.inject.} = getCurrentException().getStackTrace() + let e = getCurrentException() + let eTypeDesc = "[" & exceptionTypeName(e) & "]" + checkpoint("Unhandled exception: " & getCurrentExceptionMsg() & " " & eTypeDesc) + var stackTrace {.inject.} = e.getStackTrace() fail() finally: @@ -701,3 +721,7 @@ macro expect*(exceptions: varargs[typed], body: untyped): untyped = errorTypes.add(exp[i]) result = getAst(expectBody(errorTypes, exp.lineinfo, body)) + +proc disableParamFiltering* = + ## disables filtering tests with the command line params + disabledParamFiltering = true diff --git a/lib/pure/uri.nim b/lib/pure/uri.nim index a651530c3..dd8040928 100644 --- a/lib/pure/uri.nim +++ b/lib/pure/uri.nim @@ -18,14 +18,12 @@ type hostname*, port*, path*, query*, anchor*: string opaque*: bool -{.deprecated: [TUrl: Url, TUri: Uri].} - {.push warning[deprecated]: off.} -proc `$`*(url: Url): string {.deprecated.} = +proc `$`*(url: Url): string {.deprecated: "use Uri instead".} = ## **Deprecated since 0.9.6**: Use ``Uri`` instead. return string(url) -proc `/`*(a, b: Url): Url {.deprecated.} = +proc `/`*(a, b: Url): Url {.deprecated: "use Uri instead".} = ## Joins two URLs together, separating them with / if needed. ## ## **Deprecated since 0.9.6**: Use ``Uri`` instead. @@ -40,39 +38,50 @@ proc `/`*(a, b: Url): Url {.deprecated.} = urlS.add(bs) result = Url(urlS) -proc add*(url: var Url, a: Url) {.deprecated.} = +proc add*(url: var Url, a: Url) {.deprecated: "use Uri instead".} = ## Appends url to url. ## ## **Deprecated since 0.9.6**: Use ``Uri`` instead. url = url / a {.pop.} -proc encodeUrl*(s: string): string = - ## Encodes a value to be HTTP safe: This means that characters in the set - ## ``{'A'..'Z', 'a'..'z', '0'..'9', '_'}`` are carried over to the result, - ## a space is converted to ``'+'`` and every other character is encoded as - ## ``'%xx'`` where ``xx`` denotes its hexadecimal value. +proc encodeUrl*(s: string, usePlus=true): string = + ## Encodes a URL according to RFC3986. + ## + ## This means that characters in the set + ## ``{'a'..'z', 'A'..'Z', '0'..'9', '-', '.', '_', '~'}`` are + ## carried over to the result. + ## All other characters are encoded as ``''%xx'`` where ``xx`` + ## denotes its hexadecimal value. + ## + ## As a special rule, when the value of ``usePlus`` is true, + ## spaces are encoded as ``'+'`` instead of ``'%20'``. result = newStringOfCap(s.len + s.len shr 2) # assume 12% non-alnum-chars - for i in 0..s.len-1: - case s[i] - of 'a'..'z', 'A'..'Z', '0'..'9', '_': add(result, s[i]) - of ' ': add(result, '+') + let fromSpace = if usePlus: "+" else: "%20" + for c in s: + case c + of 'a'..'z', 'A'..'Z', '0'..'9', '-', '.', '_', '~': add(result, c) + of ' ': add(result, fromSpace) else: add(result, '%') - add(result, toHex(ord(s[i]), 2)) - -proc decodeUrl*(s: string): string = - ## Decodes a value from its HTTP representation: This means that a ``'+'`` - ## is converted to a space, ``'%xx'`` (where ``xx`` denotes a hexadecimal - ## value) is converted to the character with ordinal number ``xx``, and + add(result, toHex(ord(c), 2)) + +proc decodeUrl*(s: string, decodePlus=true): string = + ## Decodes a URL according to RFC3986. + ## + ## This means that any ``'%xx'`` (where ``xx`` denotes a hexadecimal + ## value) are converted to the character with ordinal number ``xx``, ## and every other character is carried over. + ## + ## As a special rule, when the value of ``decodePlus`` is true, ``'+'`` + ## characters are converted to a space. proc handleHexChar(c: char, x: var int) {.inline.} = case c of '0'..'9': x = (x shl 4) or (ord(c) - ord('0')) of 'a'..'f': x = (x shl 4) or (ord(c) - ord('a') + 10) of 'A'..'F': x = (x shl 4) or (ord(c) - ord('A') + 10) else: assert(false) - + result = newString(s.len) var i = 0 var j = 0 @@ -84,7 +93,11 @@ proc decodeUrl*(s: string): string = handleHexChar(s[i+2], x) inc(i, 2) result[j] = chr(x) - of '+': result[j] = ' ' + of '+': + if decodePlus: + result[j] = ' ' + else: + result[j] = s[i] else: result[j] = s[i] inc(i) inc(j) @@ -94,7 +107,7 @@ proc parseAuthority(authority: string, result: var Uri) = var i = 0 var inPort = false var inIPv6 = false - while true: + while i < authority.len: case authority[i] of '@': swap result.password, result.port @@ -111,7 +124,6 @@ proc parseAuthority(authority: string, result: var Uri) = inIPv6 = true of ']': inIPv6 = false - of '\0': break else: if inPort: result.port.add(authority[i]) @@ -128,11 +140,11 @@ proc parsePath(uri: string, i: var int, result: var Uri) = parseAuthority(result.path, result) result.path.setLen(0) - if uri[i] == '?': + if i < uri.len and uri[i] == '?': i.inc # Skip '?' i.inc parseUntil(uri, result.query, {'#'}, i) - if uri[i] == '#': + if i < uri.len and uri[i] == '#': i.inc # Skip '#' i.inc parseUntil(uri, result.anchor, {}, i) @@ -156,7 +168,7 @@ proc parseUri*(uri: string, result: var Uri) = # Check if this is a reference URI (relative URI) let doubleSlash = uri.len > 1 and uri[1] == '/' - if uri[i] == '/': + if i < uri.len and uri[i] == '/': # Make sure ``uri`` doesn't begin with '//'. if not doubleSlash: parsePath(uri, i, result) @@ -164,7 +176,7 @@ proc parseUri*(uri: string, result: var Uri) = # Scheme i.inc parseWhile(uri, result.scheme, Letters + Digits + {'+', '-', '.'}, i) - if uri[i] != ':' and not doubleSlash: + if (i >= uri.len or uri[i] != ':') and not doubleSlash: # Assume this is a reference URI (relative URI) i = 0 result.scheme.setLen(0) @@ -174,13 +186,12 @@ proc parseUri*(uri: string, result: var Uri) = i.inc # Skip ':' # Authority - if uri[i] == '/' and uri[i+1] == '/': + if i+1 < uri.len and uri[i] == '/' and uri[i+1] == '/': i.inc(2) # Skip // var authority = "" i.inc parseUntil(uri, authority, {'/', '?', '#'}, i) - if authority == "": - raise newException(ValueError, "Expected authority got nothing.") - parseAuthority(authority, result) + if authority.len > 0: + parseAuthority(authority, result) else: result.opaque = true @@ -198,13 +209,13 @@ proc removeDotSegments(path: string): string = let endsWithSlash = path[path.len-1] == '/' var i = 0 var currentSegment = "" - while true: + while i < path.len: case path[i] of '/': collection.add(currentSegment) currentSegment = "" of '.': - if path[i+1] == '.' and path[i+2] == '/': + if i+2 < path.len and path[i+1] == '.' and path[i+2] == '/': if collection.len > 0: discard collection.pop() i.inc 3 @@ -213,13 +224,11 @@ proc removeDotSegments(path: string): string = i.inc 2 continue currentSegment.add path[i] - of '\0': - if currentSegment != "": - collection.add currentSegment - break else: currentSegment.add path[i] i.inc + if currentSegment != "": + collection.add currentSegment result = collection.join("/") if endsWithSlash: result.add '/' @@ -321,18 +330,18 @@ proc `/`*(x: Uri, path: string): Uri = result = x if result.path.len == 0: - if path[0] != '/': + if path.len == 0 or path[0] != '/': result.path = "/" result.path.add(path) return - if result.path[result.path.len-1] == '/': - if path[0] == '/': + if result.path.len > 0 and result.path[result.path.len-1] == '/': + if path.len > 0 and path[0] == '/': result.path.add(path[1 .. path.len-1]) else: result.path.add(path) else: - if path[0] != '/': + if path.len == 0 or path[0] != '/': result.path.add '/' result.path.add(path) @@ -374,7 +383,10 @@ when isMainModule: const test1 = "abc\L+def xyz" doAssert encodeUrl(test1) == "abc%0A%2Bdef+xyz" doAssert decodeUrl(encodeUrl(test1)) == test1 - + doAssert encodeUrl(test1, false) == "abc%0A%2Bdef%20xyz" + doAssert decodeUrl(encodeUrl(test1, false), false) == test1 + doAssert decodeUrl(encodeUrl(test1)) == test1 + block: let str = "http://localhost" let test = parseUri(str) @@ -466,6 +478,15 @@ when isMainModule: doAssert test.port == "dom96" doAssert test.path == "/packages" + block: + let str = "file:///foo/bar/baz.txt" + let test = parseUri(str) + doAssert test.scheme == "file" + doAssert test.username == "" + doAssert test.hostname == "" + doAssert test.port == "" + doAssert test.path == "/foo/bar/baz.txt" + # Remove dot segments tests block: doAssert removeDotSegments("/foo/bar/baz") == "/foo/bar/baz" diff --git a/lib/pure/xmldom.nim b/lib/pure/xmldom.nim index 3c891c81b..8cd47aa39 100644 --- a/lib/pure/xmldom.nim +++ b/lib/pure/xmldom.nim @@ -232,10 +232,10 @@ proc createAttributeNS*(doc: PDocument, namespaceURI: string, qualifiedName: str raise newException(EInvalidCharacterErr, "Invalid character") # Exceptions if qualifiedName.contains(':'): - let qfnamespaces = qualifiedName.toLower().split(':') + let qfnamespaces = qualifiedName.toLowerAscii().split(':') if isNil(namespaceURI): raise newException(ENamespaceErr, "When qualifiedName contains a prefix namespaceURI cannot be nil") - elif qfnamespaces[0] == "xml" and + elif qfnamespaces[0] == "xml" and namespaceURI != "http://www.w3.org/XML/1998/namespace" and qfnamespaces[1] notin stdattrnames: raise newException(ENamespaceErr, @@ -311,10 +311,10 @@ proc createElement*(doc: PDocument, tagName: string): PElement = proc createElementNS*(doc: PDocument, namespaceURI: string, qualifiedName: string): PElement = ## Creates an element of the given qualified name and namespace URI. if qualifiedName.contains(':'): - let qfnamespaces = qualifiedName.toLower().split(':') + let qfnamespaces = qualifiedName.toLowerAscii().split(':') if isNil(namespaceURI): raise newException(ENamespaceErr, "When qualifiedName contains a prefix namespaceURI cannot be nil") - elif qfnamespaces[0] == "xml" and + elif qfnamespaces[0] == "xml" and namespaceURI != "http://www.w3.org/XML/1998/namespace" and qfnamespaces[1] notin stdattrnames: raise newException(ENamespaceErr, @@ -533,13 +533,13 @@ proc `prefix=`*(n: PNode, value: string) = if isNil(n.fNamespaceURI): raise newException(ENamespaceErr, "namespaceURI cannot be nil") - elif value.toLower() == "xml" and n.fNamespaceURI != "http://www.w3.org/XML/1998/namespace": + elif value.toLowerAscii() == "xml" and n.fNamespaceURI != "http://www.w3.org/XML/1998/namespace": raise newException(ENamespaceErr, "When the namespace prefix is \"xml\" namespaceURI has to be \"http://www.w3.org/XML/1998/namespace\"") - elif value.toLower() == "xmlns" and n.fNamespaceURI != "http://www.w3.org/2000/xmlns/": + elif value.toLowerAscii() == "xmlns" and n.fNamespaceURI != "http://www.w3.org/2000/xmlns/": raise newException(ENamespaceErr, "When the namespace prefix is \"xmlns\" namespaceURI has to be \"http://www.w3.org/2000/xmlns/\"") - elif value.toLower() == "xmlns" and n.fNodeType == AttributeNode: + elif value.toLowerAscii() == "xmlns" and n.fNodeType == AttributeNode: raise newException(ENamespaceErr, "An AttributeNode cannot have a prefix of \"xmlns\"") n.fNodeName = value & ":" & n.fLocalName @@ -1069,17 +1069,15 @@ proc splitData*(textNode: PText, offset: int): PText = var newNode: PText = textNode.fOwnerDocument.createTextNode(right) return newNode - # ProcessingInstruction proc target*(pi: PProcessingInstruction): string = ## Returns the Processing Instructions target return pi.fTarget - -# --Other stuff-- -# Writer -proc addEscaped(s: string): string = +proc escapeXml*(s: string; result: var string) = + ## Prepares a string for insertion into a XML document + ## by escaping the XML special characters. result = "" for c in items(s): case c @@ -1089,11 +1087,20 @@ proc addEscaped(s: string): string = of '"': result.add(""") else: result.add(c) +proc escapeXml*(s: string): string = + ## Prepares a string for insertion into a XML document + ## by escaping the XML special characters. + result = newStringOfCap(s.len + s.len shr 4) + escapeXml(s, result) + +# --Other stuff-- +# Writer + proc nodeToXml(n: PNode, indent: int = 0): string = result = spaces(indent) & "<" & n.nodeName if not isNil(n.attributes): for i in items(n.attributes): - result.add(" " & i.name & "=\"" & addEscaped(i.value) & "\"") + result.add(" " & i.name & "=\"" & escapeXml(i.value) & "\"") if isNil(n.childNodes) or n.childNodes.len() == 0: result.add("/>") # No idea why this doesn't need a \n :O @@ -1106,7 +1113,7 @@ proc nodeToXml(n: PNode, indent: int = 0): string = result.add(nodeToXml(i, indent + 2)) of TextNode: result.add(spaces(indent * 2)) - result.add(addEscaped(i.nodeValue)) + result.add(escapeXml(i.nodeValue)) of CDataSectionNode: result.add(spaces(indent * 2)) result.add("<![CDATA[" & i.nodeValue & "]]>") diff --git a/lib/pure/xmlparser.nim b/lib/pure/xmlparser.nim index 22bd259b7..597b80eb5 100644 --- a/lib/pure/xmlparser.nim +++ b/lib/pure/xmlparser.nim @@ -12,11 +12,9 @@ import streams, parsexml, strtabs, xmltree type - XmlError* = object of ValueError ## exception that is raised - ## for invalid XML - errors*: seq[string] ## all detected parsing errors - -{.deprecated: [EInvalidXml: XmlError].} + XmlError* = object of ValueError ## Exception that is raised + ## for invalid XML. + errors*: seq[string] ## All detected parsing errors. proc raiseInvalidXml(errors: seq[string]) = var e: ref XmlError @@ -102,8 +100,8 @@ proc parse(x: var XmlParser, errors: var seq[string]): XmlNode = proc parseXml*(s: Stream, filename: string, errors: var seq[string]): XmlNode = - ## parses the XML from stream `s` and returns a ``PXmlNode``. Every - ## occurred parsing error is added to the `errors` sequence. + ## Parses the XML from stream ``s`` and returns a ``XmlNode``. Every + ## occurred parsing error is added to the ``errors`` sequence. var x: XmlParser open(x, s, filename, {reportComments}) while true: @@ -121,15 +119,20 @@ proc parseXml*(s: Stream, filename: string, close(x) proc parseXml*(s: Stream): XmlNode = - ## parses the XTML from stream `s` and returns a ``PXmlNode``. All parsing - ## errors are turned into an ``EInvalidXML`` exception. + ## Parses the XML from stream ``s`` and returns a ``XmlNode``. All parsing + ## errors are turned into an ``XmlError`` exception. var errors: seq[string] = @[] - result = parseXml(s, "unknown_html_doc", errors) + result = parseXml(s, "unknown_xml_doc", errors) if errors.len > 0: raiseInvalidXml(errors) +proc parseXml*(str: string): XmlNode = + ## Parses the XML from string ``str`` and returns a ``XmlNode``. All parsing + ## errors are turned into an ``XmlError`` exception. + parseXml(newStringStream(str)) + proc loadXml*(path: string, errors: var seq[string]): XmlNode = ## Loads and parses XML from file specified by ``path``, and returns - ## a ``PXmlNode``. Every occurred parsing error is added to the `errors` + ## a ``XmlNode``. Every occurred parsing error is added to the ``errors`` ## sequence. var s = newFileStream(path, fmRead) if s == nil: raise newException(IOError, "Unable to read file: " & path) @@ -137,7 +140,7 @@ proc loadXml*(path: string, errors: var seq[string]): XmlNode = proc loadXml*(path: string): XmlNode = ## Loads and parses XML from file specified by ``path``, and returns - ## a ``PXmlNode``. All parsing errors are turned into an ``EInvalidXML`` + ## a ``XmlNode``. All parsing errors are turned into an ``XmlError`` ## exception. var errors: seq[string] = @[] result = loadXml(path, errors) diff --git a/lib/pure/xmltree.nim b/lib/pure/xmltree.nim index 45696c80c..47658b59b 100644 --- a/lib/pure/xmltree.nim +++ b/lib/pure/xmltree.nim @@ -9,12 +9,12 @@ ## A simple XML tree. More efficient and simpler than the DOM. -import macros, strtabs +import macros, strtabs, strutils type - XmlNode* = ref XmlNodeObj ## an XML tree consists of ``PXmlNode``'s. + XmlNode* = ref XmlNodeObj ## an XML tree consists of ``XmlNode``'s. - XmlNodeKind* = enum ## different kinds of ``PXmlNode``'s + XmlNodeKind* = enum ## different kinds of ``XmlNode``'s xnText, ## a text element xnElement, ## an element with 0 or more children xnCData, ## a CDATA node @@ -33,9 +33,6 @@ type fAttr: XmlAttributes fClientData: int ## for other clients -{.deprecated: [PXmlNode: XmlNode, TXmlNodeKind: XmlNodeKind, PXmlAttributes: - XmlAttributes, TXmlNode: XmlNodeObj].} - proc newXmlNode(kind: XmlNodeKind): XmlNode = ## creates a new ``XmlNode``. new(result) @@ -155,11 +152,6 @@ proc `[]`* (n: var XmlNode, i: int): var XmlNode {.inline.} = assert n.k == xnElement result = n.s[i] -proc mget*(n: var XmlNode, i: int): var XmlNode {.inline, deprecated.} = - ## returns the `i`'th child of `n` so that it can be modified. Use ```[]``` - ## instead. - n[i] - iterator items*(n: XmlNode): XmlNode {.inline.} = ## iterates over any child of `n`. assert n.k == xnElement @@ -225,8 +217,9 @@ proc escape*(s: string): string = result = newStringOfCap(s.len) addEscaped(result, s) -proc addIndent(result: var string, indent: int) = - result.add("\n") +proc addIndent(result: var string, indent: int, addNewLines: bool) = + if addNewLines: + result.add("\n") for i in 1..indent: result.add(' ') proc noWhitespace(n: XmlNode): bool = @@ -235,7 +228,8 @@ proc noWhitespace(n: XmlNode): bool = for i in 0..n.len-1: if n[i].kind in {xnText, xnEntity}: return true -proc add*(result: var string, n: XmlNode, indent = 0, indWidth = 2) = +proc add*(result: var string, n: XmlNode, indent = 0, indWidth = 2, + addNewLines=true) = ## adds the textual representation of `n` to `result`. proc addEscapedAttr(result: var string, s: string) = @@ -268,14 +262,15 @@ proc add*(result: var string, n: XmlNode, indent = 0, indWidth = 2) = # for mixed leaves, we cannot output whitespace for readability, # because this would be wrong. For example: ``a<b>b</b>`` is # different from ``a <b>b</b>``. - for i in 0..n.len-1: result.add(n[i], indent+indWidth, indWidth) + for i in 0..n.len-1: + result.add(n[i], indent+indWidth, indWidth, addNewLines) else: for i in 0..n.len-1: - result.addIndent(indent+indWidth) - result.add(n[i], indent+indWidth, indWidth) - result.addIndent(indent) + result.addIndent(indent+indWidth, addNewLines) + result.add(n[i], indent+indWidth, indWidth, addNewLines) + result.addIndent(indent, addNewLines) else: - result.add(n[0], indent+indWidth, indWidth) + result.add(n[0], indent+indWidth, indWidth, addNewLines) result.add("</") result.add(n.fTag) result.add(">") @@ -315,18 +310,19 @@ proc newXmlTree*(tag: string, children: openArray[XmlNode], for i in 0..children.len-1: result.s[i] = children[i] result.fAttr = attributes -proc xmlConstructor(e: NimNode): NimNode {.compileTime.} = - expectLen(e, 2) - var a = e[1] +proc xmlConstructor(a: NimNode): NimNode {.compileTime.} = if a.kind == nnkCall: result = newCall("newXmlTree", toStrLit(a[0])) var attrs = newNimNode(nnkBracket, a) - var newStringTabCall = newCall("newStringTable", attrs, - newIdentNode("modeCaseSensitive")) + var newStringTabCall = newCall(bindSym"newStringTable", attrs, + bindSym"modeCaseSensitive") var elements = newNimNode(nnkBracket, a) for i in 1..a.len-1: if a[i].kind == nnkExprEqExpr: - attrs.add(toStrLit(a[i][0])) + # In order to support attributes like `data-lang` we have to + # replace whitespace because `toStrLit` gives `data - lang`. + let attrName = toStrLit(a[i][0]).strVal.replace(" ", "") + attrs.add(newStrLitNode(attrName)) attrs.add(a[i][1]) #echo repr(attrs) else: @@ -348,7 +344,6 @@ macro `<>`*(x: untyped): untyped = ## ## <a href="http://nim-lang.org">Nim rules.</a> ## - let x = callsite() result = xmlConstructor(x) proc child*(n: XmlNode, name: string): XmlNode = diff --git a/lib/std/sha1.nim b/lib/std/sha1.nim new file mode 100644 index 000000000..c0b1bffcf --- /dev/null +++ b/lib/std/sha1.nim @@ -0,0 +1,197 @@ +# +# +# The Nim Compiler +# (c) Copyright 2015 Nim Contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Note: Import ``std/sha1`` to use this module + +import strutils + +const Sha1DigestSize = 20 + +type + Sha1Digest = array[0 .. Sha1DigestSize-1, uint8] + SecureHash* = distinct Sha1Digest + +# Copyright (c) 2011, Micael Hildenborg +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Micael Hildenborg nor the +# names of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY Micael Hildenborg ''AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL Micael Hildenborg BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Ported to Nim by Erik O'Leary + +type + Sha1State* = array[0 .. 5-1, uint32] + Sha1Buffer = array[0 .. 80-1, uint32] + +template clearBuffer(w: Sha1Buffer, len = 16) = + zeroMem(addr(w), len * sizeof(uint32)) + +proc init*(result: var Sha1State) = + result[0] = 0x67452301'u32 + result[1] = 0xefcdab89'u32 + result[2] = 0x98badcfe'u32 + result[3] = 0x10325476'u32 + result[4] = 0xc3d2e1f0'u32 + +proc innerHash(state: var Sha1State, w: var Sha1Buffer) = + var + a = state[0] + b = state[1] + c = state[2] + d = state[3] + e = state[4] + + var round = 0 + + template rot(value, bits: uint32): uint32 = + (value shl bits) or (value shr (32u32 - bits)) + + template sha1(fun, val: uint32) = + let t = rot(a, 5) + fun + e + val + w[round] + e = d + d = c + c = rot(b, 30) + b = a + a = t + + template process(body: untyped) = + w[round] = rot(w[round - 3] xor w[round - 8] xor w[round - 14] xor w[round - 16], 1) + body + inc(round) + + template wrap(dest, value: untyped) = + let v = dest + value + dest = v + + while round < 16: + sha1((b and c) or (not b and d), 0x5a827999'u32) + inc(round) + + while round < 20: + process: + sha1((b and c) or (not b and d), 0x5a827999'u32) + + while round < 40: + process: + sha1(b xor c xor d, 0x6ed9eba1'u32) + + while round < 60: + process: + sha1((b and c) or (b and d) or (c and d), 0x8f1bbcdc'u32) + + while round < 80: + process: + sha1(b xor c xor d, 0xca62c1d6'u32) + + wrap state[0], a + wrap state[1], b + wrap state[2], c + wrap state[3], d + wrap state[4], e + +proc sha1(src: cstring; len: int): Sha1Digest = + #Initialize state + var state: Sha1State + init(state) + + #Create w buffer + var w: Sha1Buffer + + #Loop through all complete 64byte blocks. + let byteLen = len + let endOfFullBlocks = byteLen - 64 + var endCurrentBlock = 0 + var currentBlock = 0 + + while currentBlock <= endOfFullBlocks: + endCurrentBlock = currentBlock + 64 + + var i = 0 + while currentBlock < endCurrentBlock: + w[i] = uint32(src[currentBlock+3]) or + uint32(src[currentBlock+2]) shl 8'u32 or + uint32(src[currentBlock+1]) shl 16'u32 or + uint32(src[currentBlock]) shl 24'u32 + currentBlock += 4 + inc(i) + + innerHash(state, w) + + #Handle last and not full 64 byte block if existing + endCurrentBlock = byteLen - currentBlock + clearBuffer(w) + var lastBlockBytes = 0 + + while lastBlockBytes < endCurrentBlock: + + var value = uint32(src[lastBlockBytes + currentBlock]) shl + ((3'u32 - uint32(lastBlockBytes and 3)) shl 3) + + w[lastBlockBytes shr 2] = w[lastBlockBytes shr 2] or value + inc(lastBlockBytes) + + w[lastBlockBytes shr 2] = w[lastBlockBytes shr 2] or ( + 0x80'u32 shl ((3'u32 - uint32(lastBlockBytes and 3)) shl 3) + ) + + if endCurrentBlock >= 56: + innerHash(state, w) + clearBuffer(w) + + w[15] = uint32(byteLen) shl 3 + innerHash(state, w) + + # Store hash in result pointer, and make sure we get in in the correct order + # on both endian models. + for i in 0 .. Sha1DigestSize-1: + result[i] = uint8((int(state[i shr 2]) shr ((3-(i and 3)) * 8)) and 255) + +proc sha1(src: string): Sha1Digest = + ## Calculate SHA1 from input string + sha1(src, src.len) + +proc secureHash*(str: string): SecureHash = SecureHash(sha1(str)) +proc secureHashFile*(filename: string): SecureHash = secureHash(readFile(filename)) +proc `$`*(self: SecureHash): string = + result = "" + for v in Sha1Digest(self): + result.add(toHex(int(v), 2)) + +proc parseSecureHash*(hash: string): SecureHash = + for i in 0 ..< Sha1DigestSize: + Sha1Digest(result)[i] = uint8(parseHexInt(hash[i*2] & hash[i*2 + 1])) + +proc `==`*(a, b: SecureHash): bool = + # Not a constant-time comparison, but that's acceptable in this context + Sha1Digest(a) == Sha1Digest(b) + + +when isMainModule: + let hash1 = secureHash("a93tgj0p34jagp9[agjp98ajrhp9aej]") + doAssert hash1 == hash1 + doAssert parseSecureHash($hash1) == hash1 diff --git a/lib/std/varints.nim b/lib/std/varints.nim new file mode 100644 index 000000000..bfc1945fe --- /dev/null +++ b/lib/std/varints.nim @@ -0,0 +1,145 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2018 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Note this API is still experimental! A variable length integer +## encoding implementation inspired by SQLite. + +const + maxVarIntLen* = 9 ## the maximal number of bytes a varint can take + +proc readVu64*(z: openArray[byte]; pResult: var uint64): int = + if z[0] <= 240: + pResult = z[0] + return 1 + if z[0] <= 248: + if z.len < 2: return 0 + pResult = (uint64 z[0] - 241) * 256 + uint64 z[1] + 240 + return 2 + if z.len < int(z[0]-246): return 0 + if z[0] == 249: + pResult = 2288u64 + 256u64*z[1].uint64 + z[2].uint64 + return 3 + if z[0] == 250: + pResult = (z[1].uint64 shl 16u64) + (z[2].uint64 shl 8u64) + z[3].uint64 + return 4 + let x = (z[1].uint64 shl 24) + (z[2].uint64 shl 16) + (z[3].uint64 shl 8) + z[4].uint64 + if z[0] == 251: + pResult = x + return 5 + if z[0] == 252: + pResult = (((uint64)x) shl 8) + z[5].uint64 + return 6 + if z[0] == 253: + pResult = (((uint64)x) shl 16) + (z[5].uint64 shl 8) + z[6].uint64 + return 7 + if z[0] == 254: + pResult = (((uint64)x) shl 24) + (z[5].uint64 shl 16) + (z[6].uint64 shl 8) + z[7].uint64 + return 8 + pResult = (((uint64)x) shl 32) + + (0xffffffff'u64 and ((z[5].uint64 shl 24) + + (z[6].uint64 shl 16) + (z[7].uint64 shl 8) + z[8].uint64)) + return 9 + +proc varintWrite32(z: var openArray[byte]; y: uint32) = + z[0] = uint8(y shr 24) + z[1] = uint8(y shr 16) + z[2] = uint8(y shr 8) + z[3] = uint8(y) + +proc writeVu64*(z: var openArray[byte], x: uint64): int = + ## Write a varint into z. The buffer z must be at least 9 characters + ## long to accommodate the largest possible varint. Returns the number of + ## bytes used. + if x <= 240: + z[0] = uint8 x + return 1 + if x <= 2287: + let y = uint32(x - 240) + z[0] = uint8(y shr 8 + 241) + z[1] = uint8(y and 255) + return 2 + if x <= 67823: + let y = uint32(x - 2288) + z[0] = 249 + z[1] = uint8(y shr 8) + z[2] = uint8(y and 255) + return 3 + let y = uint32 x + let w = uint32(x shr 32) + if w == 0: + if y <= 16777215: + z[0] = 250 + z[1] = uint8(y shr 16) + z[2] = uint8(y shr 8) + z[3] = uint8(y) + return 4 + z[0] = 251 + varintWrite32(toOpenArray(z, 1, z.high-1), y) + return 5 + if w <= 255: + z[0] = 252 + z[1] = uint8 w + varintWrite32(toOpenArray(z, 2, z.high-2), y) + return 6 + if w <= 65535: + z[0] = 253 + z[1] = uint8(w shr 8) + z[2] = uint8 w + varintWrite32(toOpenArray(z, 3, z.high-3), y) + return 7 + if w <= 16777215: + z[0] = 254 + z[1] = uint8(w shr 16) + z[2] = uint8(w shr 8) + z[3] = uint8 w + varintWrite32(toOpenArray(z, 4, z.high-4), y) + return 8 + z[0] = 255 + varintWrite32(toOpenArray(z, 1, z.high-1), w) + varintWrite32(toOpenArray(z, 5, z.high-5), y) + return 9 + +proc sar(a, b: int64): int64 = + {.emit: [result, " = ", a, " >> ", b, ";"].} + +proc sal(a, b: int64): int64 = + {.emit: [result, " = ", a, " << ", b, ";"].} + +proc encodeZigzag*(x: int64): uint64 {.inline.} = + uint64(sal(x, 1)) xor uint64(sar(x, 63)) + +proc decodeZigzag*(x: uint64): int64 {.inline.} = + let casted = cast[int64](x) + result = (`shr`(casted, 1)) xor (-(casted and 1)) + +when isMainModule: + #import random + + var dest: array[50, byte] + var got: uint64 + + for test in [0xFFFF_FFFF_FFFFF_FFFFu64, 77u64, 0u64, 10_000_000u64, uint64(high(int64)), + uint64(high(int32)),uint64(low(int32)),uint64(low(int64))]: + let wrLen = writeVu64(dest, test) + let rdLen = readVu64(dest, got) + assert wrLen == rdLen + echo(if got == test: "YES" else: "NO") + echo "number is ", got + + if encodeZigzag(decodeZigzag(test)) != test: + echo "Failure for ", test, " ", encodeZigzag(decodeZigzag(test)), " ", decodeZigzag(test) + + # check this also works for floats: + for test in [0.0, 0.1, 2.0, +Inf, Nan, NegInf]: + let t = cast[uint64](test) + let wrLenB = writeVu64(dest, t) + let rdLenB = readVu64(dest, got) + assert wrLenB == rdLenB + echo rdLenB + echo(if cast[float64](got) == test: "YES" else: "NO") diff --git a/lib/system.nim b/lib/system.nim index 4d8610737..b8aa170ea 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -105,12 +105,14 @@ type ## type class matching all ordinal types; however this includes enums with ## holes. - SomeReal* = float|float32|float64 + SomeFloat* = float|float32|float64 ## type class matching all floating point number types - SomeNumber* = SomeInteger|SomeReal + SomeNumber* = SomeInteger|SomeFloat ## type class matching all number types +{.deprecated: [SomeReal: SomeFloat].} + proc defined*(x: untyped): bool {.magic: "Defined", noSideEffect, compileTime.} ## Special compile-time procedure that checks whether `x` is ## defined. @@ -128,7 +130,7 @@ when defined(nimalias): TSignedInt: SomeSignedInt, TUnsignedInt: SomeUnsignedInt, TInteger: SomeInteger, - TReal: SomeReal, + TReal: SomeFloat, TNumber: SomeNumber, TOrdinal: SomeOrdinal].} @@ -144,7 +146,7 @@ proc declared*(x: untyped): bool {.magic: "Defined", noSideEffect, compileTime.} ## # missing it. when defined(useNimRtl): - {.deadCodeElim: on.} + {.deadCodeElim: on.} # dce option deprecated proc definedInScope*(x: untyped): bool {. magic: "DefinedInScope", noSideEffect, deprecated, compileTime.} @@ -249,6 +251,10 @@ type when defined(nimHasOpt): type opt*{.magic: "Opt".}[T] +when defined(nimNewRuntime): + type sink*{.magic: "BuiltinType".}[T] + type lent*{.magic: "BuiltinType".}[T] + proc high*[T: Ordinal](x: T): T {.magic: "High", noSideEffect.} ## returns the highest possible index of an array, a sequence, a string or ## the highest possible value of an ordinal value `x`. As a special @@ -314,7 +320,7 @@ type Slice*[T] = HSlice[T, T] ## an alias for ``HSlice[T, T]`` proc `..`*[T, U](a: T, b: U): HSlice[T, U] {.noSideEffect, inline, magic: "DotDot".} = - ## `slice`:idx: operator that constructs an interval ``[a, b]``, both `a` + ## binary `slice`:idx: operator that constructs an interval ``[a, b]``, both `a` ## and `b` are inclusive. Slices can also be used in the set constructor ## and in ordinal case statements, but then they are special-cased by the ## compiler. @@ -322,7 +328,7 @@ proc `..`*[T, U](a: T, b: U): HSlice[T, U] {.noSideEffect, inline, magic: "DotDo result.b = b proc `..`*[T](b: T): HSlice[int, T] {.noSideEffect, inline, magic: "DotDot".} = - ## `slice`:idx: operator that constructs an interval ``[default(int), b]`` + ## unary `slice`:idx: operator that constructs an interval ``[default(int), b]`` result.b = b when not defined(niminheritable): @@ -479,6 +485,7 @@ type trace: string else: trace: seq[StackTraceEntry] + raise_id: uint # set when exception is raised up: ref Exception # used for stacking exceptions. Not exported! SystemError* = object of Exception ## \ @@ -613,6 +620,11 @@ type ## ## This is only raised if the ``segfaults.nim`` module was imported! +when defined(nimNewRuntime): + type + MoveError* = object of SystemError ## \ + ## Raised on attempts to re-sink an already consumed ``sink`` parameter. + {.deprecated: [TObject: RootObj, PObject: RootRef, TEffect: RootEffect, FTime: TimeEffect, FIO: IOEffect, FReadIO: ReadIOEffect, FWriteIO: WriteIOEffect, FExecIO: ExecIOEffect, @@ -639,6 +651,11 @@ type ESynch: Exception ].} +when defined(js) or defined(nimdoc): + type + JsRoot* = ref object of RootObj + ## Root type of the JavaScript object hierarchy + proc unsafeNew*[T](a: var ref T, size: Natural) {.magic: "New", noSideEffect.} ## creates a new object of type ``T`` and returns a safe (traced) ## reference to it in ``a``. This is **unsafe** as it allocates an object @@ -673,12 +690,12 @@ proc `<`*[T](x: Ordinal[T]): T {.magic: "UnaryLt", noSideEffect, deprecated.} ## write ``0 ..< 10`` instead of ``0 .. < 10`` (look at the spacing). ## For ``<x`` write ``pred(x)``. -proc succ*[T](x: Ordinal[T], y = 1): T {.magic: "Succ", noSideEffect.} +proc succ*[T: Ordinal](x: T, y = 1): T {.magic: "Succ", noSideEffect.} ## returns the ``y``-th successor of the value ``x``. ``T`` has to be ## an ordinal type. If such a value does not exist, ``EOutOfRange`` is raised ## or a compile time error occurs. -proc pred*[T](x: Ordinal[T], y = 1): T {.magic: "Pred", noSideEffect.} +proc pred*[T: Ordinal](x: T, y = 1): T {.magic: "Pred", noSideEffect.} ## returns the ``y``-th predecessor of the value ``x``. ``T`` has to be ## an ordinal type. If such a value does not exist, ``EOutOfRange`` is raised ## or a compile time error occurs. @@ -1263,15 +1280,13 @@ proc setLen*[T](s: var seq[T], newlen: Natural) {. ## sets the length of `s` to `newlen`. ## ``T`` may be any sequence type. ## If the current length is greater than the new length, - ## ``s`` will be truncated. `s` cannot be nil! To initialize a sequence with - ## a size, use ``newSeq`` instead. + ## ``s`` will be truncated. proc setLen*(s: var string, newlen: Natural) {. magic: "SetLengthStr", noSideEffect.} ## sets the length of `s` to `newlen`. ## If the current length is greater than the new length, - ## ``s`` will be truncated. `s` cannot be nil! To initialize a string with - ## a size, use ``newString`` instead. + ## ``s`` will be truncated. ## ## .. code-block:: Nim ## var myS = "Nim is great!!" @@ -1368,7 +1383,8 @@ const hostCPU* {.magic: "HostCPU".}: string = "" ## a string that describes the host CPU. Possible values: ## "i386", "alpha", "powerpc", "powerpc64", "powerpc64el", "sparc", - ## "amd64", "mips", "mipsel", "arm", "arm64", "mips64", "mips64el". + ## "amd64", "mips", "mipsel", "arm", "arm64", "mips64", "mips64el", + ## "riscv64". seqShallowFlag = low(int) strlitFlag = 1 shl (sizeof(int)*8 - 2) # later versions of the codegen \ @@ -1457,9 +1473,9 @@ when defined(nodejs): programResult = 0 else: var programResult* {.exportc: "nim_program_result".}: int - ## modify this variable to specify the exit code of the program - ## under normal circumstances. When the program is terminated - ## prematurely using ``quit``, this value is ignored. + ## modify this variable to specify the exit code of the program + ## under normal circumstances. When the program is terminated + ## prematurely using ``quit``, this value is ignored. when defined(nimdoc): proc quit*(errorcode: int = QuitSuccess) {.magic: "Exit", noreturn.} @@ -1507,7 +1523,6 @@ const hasAlloc = (hostOS != "standalone" or not defined(nogc)) and not defined(n when not defined(JS) and not defined(nimscript) and hostOS != "standalone": include "system/cgprocs" when not defined(JS) and not defined(nimscript) and hasAlloc: - proc setStackBottom(theStackBottom: pointer) {.compilerRtl, noinline, benign.} proc addChar(s: NimString, c: char): NimString {.compilerProc, benign.} proc add *[T](x: var seq[T], y: T) {.magic: "AppendSeqElem", noSideEffect.} @@ -1770,7 +1785,7 @@ when not defined(nimscript): proc createU*(T: typedesc, size = 1.Positive): ptr T {.inline, benign.} = ## allocates a new memory block with at least ``T.sizeof * size`` ## bytes. The block has to be freed with ``resize(block, 0)`` or - ## ``free(block)``. The block is not initialized, so reading + ## ``dealloc(block)``. The block is not initialized, so reading ## from it before writing to it is undefined behaviour! ## The allocated memory belongs to its allocating thread! ## Use `createSharedU` to allocate from a shared heap. @@ -1785,7 +1800,7 @@ when not defined(nimscript): proc create*(T: typedesc, size = 1.Positive): ptr T {.inline, benign.} = ## allocates a new memory block with at least ``T.sizeof * size`` ## bytes. The block has to be freed with ``resize(block, 0)`` or - ## ``free(block)``. The block is initialized with all bytes + ## ``dealloc(block)``. The block is initialized with all bytes ## containing zero, so it is somewhat safer than ``createU``. ## The allocated memory belongs to its allocating thread! ## Use `createShared` to allocate from a shared heap. @@ -1803,7 +1818,7 @@ when not defined(nimscript): ## grows or shrinks a given memory block. If p is **nil** then a new ## memory block is returned. In either way the block has at least ## ``T.sizeof * newSize`` bytes. If ``newSize == 0`` and p is not - ## **nil** ``resize`` calls ``free(p)``. In other cases the block + ## **nil** ``resize`` calls ``dealloc(p)``. In other cases the block ## has to be freed with ``free``. The allocated memory belongs to ## its allocating thread! ## Use `resizeShared` to reallocate from a shared heap. @@ -1913,7 +1928,7 @@ proc `$` *(x: float): string {.magic: "FloatToStr", noSideEffect.} proc `$` *(x: bool): string {.magic: "BoolToStr", noSideEffect.} ## The stringify operator for a boolean argument. Returns `x` ## converted to the string "false" or "true". -# + proc `$` *(x: char): string {.magic: "CharToStr", noSideEffect.} ## The stringify operator for a character argument. Returns `x` ## converted to a string. @@ -1951,13 +1966,13 @@ const ## that you cannot compare a floating point value to this value ## and expect a reasonable result - use the `classify` procedure ## in the module ``math`` for checking for NaN. - NimMajor*: int = 0 + NimMajor* {.intdefine.}: int = 0 ## is the major number of Nim's version. - NimMinor*: int = 17 + NimMinor* {.intdefine.}: int = 18 ## is the minor number of Nim's version. - NimPatch*: int = 3 + NimPatch* {.intdefine.}: int = 1 ## is the patch number of Nim's version. NimVersion*: string = $NimMajor & "." & $NimMinor & "." & $NimPatch @@ -2001,7 +2016,13 @@ iterator countdown*[T](a, b: T, step = 1): T {.inline.} = ## step count. `T` may be any ordinal type, `step` may only ## be positive. **Note**: This fails to count to ``low(int)`` if T = int for ## efficiency reasons. - when T is IntLikeForCount: + when T is (uint|uint64): + var res = a + while res >= b: + yield res + if res == b: break + dec(res, step) + elif T is IntLikeForCount: var res = int(a) while res >= int(b): yield T(res) @@ -2149,8 +2170,8 @@ proc max*[T](x, y: T): T = if y <= x: x else: y {.pop.} -proc high*(T: typedesc[SomeReal]): T = Inf -proc low*(T: typedesc[SomeReal]): T = NegInf +proc high*(T: typedesc[SomeFloat]): T = Inf +proc low*(T: typedesc[SomeFloat]): T = NegInf proc clamp*[T](x, a, b: T): T = ## limits the value ``x`` within the interval [a, b] @@ -2380,7 +2401,7 @@ proc `==` *[T](x, y: seq[T]): bool {.noSideEffect.} = if x.isNil and y.isNil: return true else: - when not defined(JS) or defined(nimphp): + when not defined(JS): proc seqToPtr[T](x: seq[T]): pointer {.inline, nosideeffect.} = result = cast[pointer](x) else: @@ -2390,8 +2411,9 @@ proc `==` *[T](x, y: seq[T]): bool {.noSideEffect.} = if seqToPtr(x) == seqToPtr(y): return true - if x.isNil or y.isNil: - return false + when not defined(nimNoNil): + if x.isNil or y.isNil: + return false if x.len != y.len: return false @@ -2616,6 +2638,11 @@ when not defined(nimscript) and hasAlloc: proc GC_unref*(x: string) {.magic: "GCunref", benign.} ## see the documentation of `GC_ref`. + when not defined(JS) and not defined(nimscript) and hasAlloc: + proc nimGC_setStackBottom*(theStackBottom: pointer) {.compilerRtl, noinline, benign.} + ## Expands operating GC stack range to `theStackBottom`. Does nothing + ## if current stack bottom is already lower than `theStackBottom`. + else: template GC_disable* = {.warning: "GC_disable is a no-op in JavaScript".} @@ -2742,17 +2769,14 @@ type when defined(JS): proc add*(x: var string, y: cstring) {.asmNoStackFrame.} = - when defined(nimphp): - asm """`x` .= `y`;""" - else: - asm """ - var len = `x`[0].length-1; - for (var i = 0; i < `y`.length; ++i) { - `x`[0][len] = `y`.charCodeAt(i); - ++len; - } - `x`[0][len] = 0 - """ + asm """ + var len = `x`[0].length-1; + for (var i = 0; i < `y`.length; ++i) { + `x`[0][len] = `y`.charCodeAt(i); + ++len; + } + `x`[0][len] = 0 + """ proc add*(x: var cstring, y: cstring) {.magic: "AppendStrStr".} elif hasAlloc: @@ -2878,16 +2902,16 @@ when not defined(JS): #and not defined(nimscript): # WARNING: This is very fragile! An array size of 8 does not work on my # Linux 64bit system. -- That's because the stack direction is the other # way round. - when declared(setStackBottom): + when declared(nimGC_setStackBottom): var locals {.volatile.}: pointer locals = addr(locals) - setStackBottom(locals) + nimGC_setStackBottom(locals) proc initStackBottomWith(locals: pointer) {.inline, compilerproc.} = # We need to keep initStackBottom around for now to avoid # bootstrapping problems. - when declared(setStackBottom): - setStackBottom(locals) + when declared(nimGC_setStackBottom): + nimGC_setStackBottom(locals) {.push profiler: off.} var @@ -3021,9 +3045,9 @@ when not defined(JS): #and not defined(nimscript): proc endOfFile*(f: File): bool {.tags: [], benign.} ## Returns true iff `f` is at the end. - proc readChar*(f: File): char {.tags: [ReadIOEffect], deprecated.} - ## Reads a single character from the stream `f`. **Deprecated** since - ## version 0.16.2. Use some variant of ``readBuffer`` instead. + proc readChar*(f: File): char {.tags: [ReadIOEffect].} + ## Reads a single character from the stream `f`. Should not be used in + ## performance sensitive code. proc flushFile*(f: File) {.tags: [WriteIOEffect].} ## Flushes `f`'s buffer. @@ -3203,13 +3227,14 @@ when not defined(JS): #and not defined(nimscript): when declared(initGC): initGC() when not defined(nimscript): - proc setControlCHook*(hook: proc () {.noconv.} not nil) + proc setControlCHook*(hook: proc () {.noconv.}) ## allows you to override the behaviour of your application when CTRL+C ## is pressed. Only one such hook is supported. - proc writeStackTrace*() {.tags: [WriteIOEffect], gcsafe.} + proc writeStackTrace*() {.tags: [], gcsafe.} ## writes the current stack trace to ``stderr``. This is only works - ## for debug builds. + ## for debug builds. Since it's usually used for debugging, this + ## is proclaimed to have no IO effect! when hostOS != "standalone": proc getStackTrace*(): string {.gcsafe.} ## gets the current stack trace. This only works for debug builds. @@ -3417,9 +3442,17 @@ elif defined(JS): when defined(nimffi): include "system/sysio" +when not defined(nimNoArrayToString): + proc `$`*[T, IDX](x: array[IDX, T]): string = + ## generic ``$`` operator for arrays that is lifted from the components + collectionToString(x, "[", ", ", "]") -proc `$`*[T, IDX](x: array[IDX, T]): string = - ## generic ``$`` operator for arrays that is lifted from the components +proc `$`*[T](x: openarray[T]): string = + ## generic ``$`` operator for openarrays that is lifted from the components + ## of `x`. Example: + ## + ## .. code-block:: nim + ## $(@[23, 45].toOpenArray(0, 1)) == "[23, 45]" collectionToString(x, "[", ", ", "]") proc quit*(errormsg: string, errorcode = QuitFailure) {.noReturn.} = @@ -3500,8 +3533,8 @@ template `..^`*(a, b: untyped): untyped = a .. ^b template `..<`*(a, b: untyped): untyped = - ## a shortcut for 'a..pred(b)'. - a .. pred(b) + ## a shortcut for 'a .. (when b is BackwardsIndex: succ(b) else: pred(b))'. + a .. (when b is BackwardsIndex: succ(b) else: pred(b)) when defined(nimNewRoof): iterator `..<`*[T](a, b: T): T = @@ -3719,7 +3752,7 @@ proc astToStr*[T](x: T): string {.magic: "AstToStr", noSideEffect.} ## for debugging. proc instantiationInfo*(index = -1, fullPaths = false): tuple[ - filename: string, line: int] {. magic: "InstantiationInfo", noSideEffect.} + filename: string, line: int, column: int] {. magic: "InstantiationInfo", noSideEffect.} ## provides access to the compiler's instantiation stack line information ## of a template. ## @@ -3764,7 +3797,6 @@ proc failedAssertImpl*(msg: string) {.raises: [], tags: [].} = # by ``assert``. type Hide = proc (msg: string) {.noinline, raises: [], noSideEffect, tags: [].} - {.deprecated: [THide: Hide].} Hide(raiseAssert)(msg) template assert*(cond: bool, msg = "") = @@ -3788,7 +3820,9 @@ template doAssert*(cond: bool, msg = "") = bind instantiationInfo {.line: instantiationInfo().}: if not cond: - raiseAssert(astToStr(cond) & ' ' & msg) + raiseAssert(astToStr(cond) & ' ' & + instantiationInfo(-1, false).fileName & '(' & + $instantiationInfo(-1, false).line & ") " & msg) iterator items*[T](a: seq[T]): T {.inline.} = ## iterates over each item of `a`. @@ -3912,29 +3946,48 @@ proc addEscapedChar*(s: var string, c: char) {.noSideEffect, inline.} = ## * replaces any ``\`` by ``\\`` ## * replaces any ``'`` by ``\'`` ## * replaces any ``"`` by ``\"`` - ## * replaces any other character in the set ``{'\0'..'\31', '\127'..'\255'}`` + ## * replaces any ``\a`` by ``\\a`` + ## * replaces any ``\b`` by ``\\b`` + ## * replaces any ``\t`` by ``\\t`` + ## * replaces any ``\n`` by ``\\n`` + ## * replaces any ``\v`` by ``\\v`` + ## * replaces any ``\f`` by ``\\f`` + ## * replaces any ``\c`` by ``\\c`` + ## * replaces any ``\e`` by ``\\e`` + ## * replaces any other character not in the set ``{'\21..'\126'} ## by ``\xHH`` where ``HH`` is its hexadecimal value. ## ## The procedure has been designed so that its output is usable for many ## different common syntaxes. ## **Note**: This is not correct for producing Ansi C code! case c - of '\0'..'\31', '\127'..'\255': - add(s, "\\x") + of '\a': s.add "\\a" # \x07 + of '\b': s.add "\\b" # \x08 + of '\t': s.add "\\t" # \x09 + of '\n': s.add "\\n" # \x0A + of '\v': s.add "\\v" # \x0B + of '\f': s.add "\\f" # \x0C + of '\c': s.add "\\c" # \x0D + of '\e': s.add "\\e" # \x1B + of '\\': s.add("\\\\") + of '\'': s.add("\\'") + of '\"': s.add("\\\"") + of {'\32'..'\126'} - {'\\', '\'', '\"'}: s.add(c) + else: + s.add("\\x") const HexChars = "0123456789ABCDEF" let n = ord(c) s.add(HexChars[int((n and 0xF0) shr 4)]) s.add(HexChars[int(n and 0xF)]) - of '\\': add(s, "\\\\") - of '\'': add(s, "\\'") - of '\"': add(s, "\\\"") - else: add(s, c) proc addQuoted*[T](s: var string, x: T) = ## Appends `x` to string `s` in place, applying quoting and escaping ## if `x` is a string or char. See ## `addEscapedChar <system.html#addEscapedChar>`_ - ## for the escaping scheme. + ## for the escaping scheme. When `x` is a string, characters in the + ## range ``{\128..\255}`` are never escaped so that multibyte UTF-8 + ## characters are untouched (note that this behavior is different from + ## ``addEscapedChar``). ## ## The Nim standard library uses this function on the elements of ## collections when producing a string representation of a collection. @@ -3953,7 +4006,12 @@ proc addQuoted*[T](s: var string, x: T) = when T is string: s.add("\"") for c in x: - s.addEscapedChar(c) + # Only ASCII chars are escaped to avoid butchering + # multibyte UTF-8 characters. + if c <= 127.char: + s.addEscapedChar(c) + else: + s.add c s.add("\"") elif T is char: s.add("'") @@ -3967,18 +4025,18 @@ proc addQuoted*[T](s: var string, x: T) = when hasAlloc: # XXX: make these the default (or implement the NilObject optimization) - proc safeAdd*[T](x: var seq[T], y: T) {.noSideEffect.} = + proc safeAdd*[T](x: var seq[T], y: T) {.noSideEffect, deprecated.} = ## Adds ``y`` to ``x`` unless ``x`` is not yet initialized; in that case, ## ``x`` becomes ``@[y]`` if x == nil: x = @[y] else: x.add(y) - proc safeAdd*(x: var string, y: char) = + proc safeAdd*(x: var string, y: char) {.noSideEffect, deprecated.} = ## Adds ``y`` to ``x``. If ``x`` is ``nil`` it is initialized to ``""`` if x == nil: x = "" x.add(y) - proc safeAdd*(x: var string, y: string) = + proc safeAdd*(x: var string, y: string) {.noSideEffect, deprecated.} = ## Adds ``y`` to ``x`` unless ``x`` is not yet initalized; in that ## case, ``x`` becomes ``y`` if x == nil: x = y @@ -4063,6 +4121,25 @@ template closureScope*(body: untyped): untyped = ## myClosure() # outputs 3 (proc() = body)() +template once*(body: untyped): untyped = + ## Executes a block of code only once (the first time the block is reached). + ## When hot code reloading is enabled, protects top-level code from being + ## re-executed on each module reload. + ## + ## .. code-block:: nim + ## proc draw(t: Triangle) = + ## once: + ## graphicsInit() + ## + ## line(t.p1, t.p2) + ## line(t.p2, t.p3) + ## line(t.p3, t.p1) + ## + var alreadyExecuted {.global.} = false + if not alreadyExecuted: + alreadyExecuted = true + body + {.pop.} #{.push warning[GcMem]: off, warning[Uninit]: off.} when defined(nimconfig): @@ -4096,14 +4173,45 @@ template doAssertRaises*(exception, code: untyped): typed = runnableExamples: doAssertRaises(ValueError): raise newException(ValueError, "Hello World") - + var wrong = false try: - block: - code - raiseAssert(astToStr(exception) & " wasn't raised by:\n" & astToStr(code)) + code + wrong = true except exception: discard except Exception as exc: raiseAssert(astToStr(exception) & " wasn't raised, another error was raised instead by:\n"& astToStr(code)) + if wrong: + raiseAssert(astToStr(exception) & " wasn't raised by:\n" & astToStr(code)) + +when defined(cpp) and appType != "lib" and + not defined(js) and not defined(nimscript) and + hostOS != "standalone" and not defined(noCppExceptions): + proc setTerminate(handler: proc() {.noconv.}) + {.importc: "std::set_terminate", header: "<exception>".} + setTerminate proc() {.noconv.} = + # Remove ourself as a handler, reinstalling the default handler. + setTerminate(nil) + + let ex = getCurrentException() + let trace = ex.getStackTrace() + stderr.write trace & "Error: unhandled exception: " & ex.msg & + " [" & $ex.name & "]\n" + quit 1 + +when not defined(js): + proc toOpenArray*[T](x: seq[T]; first, last: int): openarray[T] {. + magic: "Slice".} + proc toOpenArray*[T](x: openarray[T]; first, last: int): openarray[T] {. + magic: "Slice".} + proc toOpenArray*[I, T](x: array[I, T]; first, last: I): openarray[T] {. + magic: "Slice".} + proc toOpenArray*(x: string; first, last: int): openarray[char] {. + magic: "Slice".} + + +type + ForLoopStmt* {.compilerProc.} = object ## special type that marks a macro + ## as a `for-loop macro`:idx: diff --git a/lib/system/alloc.nim b/lib/system/alloc.nim index e274e8e0c..6aef4f411 100644 --- a/lib/system/alloc.nim +++ b/lib/system/alloc.nim @@ -29,6 +29,10 @@ const FliOffset = 6 RealFli = MaxFli - FliOffset + # size of chunks in last matrix bin + MaxBigChunkSize = 1 shl MaxFli - 1 shl (MaxFli-MaxLog2Sli-1) + HugeChunkSize = MaxBigChunkSize + 1 + type PTrunk = ptr Trunk Trunk = object @@ -104,7 +108,7 @@ type slBitmap: array[RealFli, uint32] matrix: array[RealFli, array[MaxSli, PBigChunk]] llmem: PLLChunk - currMem, maxMem, freeMem: int # memory sizes (allocated from OS) + currMem, maxMem, freeMem, occ: int # memory sizes (allocated from OS) lastSize: int # needed for the case that OS gives us pages linearly chunkStarts: IntSet root, deleted, last, freeAvlNodes: PAvlNode @@ -152,10 +156,11 @@ proc mappingSearch(r, fl, sl: var int) {.inline.} = # PageSize alignment: let t = roundup((1 shl (msbit(uint32 r) - MaxLog2Sli)), PageSize) - 1 r = r + t + r = r and not t + r = min(r, MaxBigChunkSize) fl = msbit(uint32 r) sl = (r shr (fl - MaxLog2Sli)) - MaxSli dec fl, FliOffset - r = r and not t sysAssert((r and PageMask) == 0, "mappingSearch: still not aligned") # See http://www.gii.upv.es/tlsf/files/papers/tlsf_desc.pdf for details of @@ -421,7 +426,7 @@ const nimMaxHeap {.intdefine.} = 0 proc requestOsChunks(a: var MemRegion, size: int): PBigChunk = when not defined(emscripten): if not a.blockChunkSizeIncrease: - let usedMem = a.currMem # - a.freeMem + let usedMem = a.occ #a.currMem # - a.freeMem when nimMaxHeap != 0: if usedMem > nimMaxHeap * 1024 * 1024: raiseOutOfMem() @@ -516,58 +521,63 @@ proc updatePrevSize(a: var MemRegion, c: PBigChunk, if isAccessible(a, ri): ri.prevSize = prevSize or (ri.prevSize and 1) +proc splitChunk2(a: var MemRegion, c: PBigChunk, size: int): PBigChunk = + result = cast[PBigChunk](cast[ByteAddress](c) +% size) + result.size = c.size - size + track("result.origSize", addr result.origSize, sizeof(int)) + # XXX check if these two nil assignments are dead code given + # addChunkToMatrix's implementation: + result.next = nil + result.prev = nil + # size and not used: + result.prevSize = size + sysAssert((size and 1) == 0, "splitChunk 2") + sysAssert((size and PageMask) == 0, + "splitChunk: size is not a multiple of the PageSize") + updatePrevSize(a, c, result.size) + c.size = size + incl(a, a.chunkStarts, pageIndex(result)) + +proc splitChunk(a: var MemRegion, c: PBigChunk, size: int) = + let rest = splitChunk2(a, c, size) + addChunkToMatrix(a, rest) + proc freeBigChunk(a: var MemRegion, c: PBigChunk) = var c = c sysAssert(c.size >= PageSize, "freeBigChunk") inc(a.freeMem, c.size) - when coalescRight: - var ri = cast[PChunk](cast[ByteAddress](c) +% c.size) - sysAssert((cast[ByteAddress](ri) and PageMask) == 0, "freeBigChunk 2") - if isAccessible(a, ri) and chunkUnused(ri): - sysAssert(not isSmallChunk(ri), "freeBigChunk 3") - if not isSmallChunk(ri): - removeChunkFromMatrix(a, cast[PBigChunk](ri)) - inc(c.size, ri.size) - excl(a.chunkStarts, pageIndex(ri)) + c.prevSize = c.prevSize and not 1 # set 'used' to false when coalescLeft: - let prevSize = c.prevSize and not 1 + let prevSize = c.prevSize if prevSize != 0: var le = cast[PChunk](cast[ByteAddress](c) -% prevSize) sysAssert((cast[ByteAddress](le) and PageMask) == 0, "freeBigChunk 4") if isAccessible(a, le) and chunkUnused(le): sysAssert(not isSmallChunk(le), "freeBigChunk 5") - if not isSmallChunk(le): + if not isSmallChunk(le) and le.size < MaxBigChunkSize: removeChunkFromMatrix(a, cast[PBigChunk](le)) inc(le.size, c.size) excl(a.chunkStarts, pageIndex(c)) c = cast[PBigChunk](le) - - incl(a, a.chunkStarts, pageIndex(c)) - updatePrevSize(a, c, c.size) + if c.size > MaxBigChunkSize: + let rest = splitChunk2(a, c, MaxBigChunkSize) + addChunkToMatrix(a, c) + c = rest + when coalescRight: + var ri = cast[PChunk](cast[ByteAddress](c) +% c.size) + sysAssert((cast[ByteAddress](ri) and PageMask) == 0, "freeBigChunk 2") + if isAccessible(a, ri) and chunkUnused(ri): + sysAssert(not isSmallChunk(ri), "freeBigChunk 3") + if not isSmallChunk(ri) and c.size < MaxBigChunkSize: + removeChunkFromMatrix(a, cast[PBigChunk](ri)) + inc(c.size, ri.size) + excl(a.chunkStarts, pageIndex(ri)) + if c.size > MaxBigChunkSize: + let rest = splitChunk2(a, c, MaxBigChunkSize) + addChunkToMatrix(a, rest) addChunkToMatrix(a, c) - # set 'used' to false: - c.prevSize = c.prevSize and not 1 - -proc splitChunk(a: var MemRegion, c: PBigChunk, size: int) = - var rest = cast[PBigChunk](cast[ByteAddress](c) +% size) - rest.size = c.size - size - track("rest.origSize", addr rest.origSize, sizeof(int)) - # XXX check if these two nil assignments are dead code given - # addChunkToMatrix's implementation: - rest.next = nil - rest.prev = nil - # size and not used: - rest.prevSize = size - sysAssert((size and 1) == 0, "splitChunk 2") - sysAssert((size and PageMask) == 0, - "splitChunk: size is not a multiple of the PageSize") - updatePrevSize(a, c, rest.size) - c.size = size - incl(a, a.chunkStarts, pageIndex(rest)) - addChunkToMatrix(a, rest) proc getBigChunk(a: var MemRegion, size: int): PBigChunk = - # use first fit for now: sysAssert(size > 0, "getBigChunk 2") var size = size # roundup(size, PageSize) var fl, sl: int @@ -594,6 +604,26 @@ proc getBigChunk(a: var MemRegion, size: int): PBigChunk = incl(a, a.chunkStarts, pageIndex(result)) dec(a.freeMem, size) +proc getHugeChunk(a: var MemRegion; size: int): PBigChunk = + result = cast[PBigChunk](osAllocPages(size)) + incCurrMem(a, size) + # XXX add this to the heap links. But also remove it from it later. + when false: a.addHeapLink(result, size) + sysAssert((cast[ByteAddress](result) and PageMask) == 0, "getHugeChunk") + result.next = nil + result.prev = nil + result.size = size + # set 'used' to to true: + result.prevSize = 1 + incl(a, a.chunkStarts, pageIndex(result)) + +proc freeHugeChunk(a: var MemRegion; c: PBigChunk) = + let size = c.size + sysAssert(size >= HugeChunkSize, "freeHugeChunk: invalid size") + excl(a.chunkStarts, pageIndex(c)) + decCurrMem(a, size) + osDeallocPages(c, size) + proc getSmallChunk(a: var MemRegion): PSmallChunk = var res = getBigChunk(a, PageSize) sysAssert res.prev == nil, "getSmallChunk 1" @@ -627,6 +657,85 @@ else: c = c.next result = true +when false: + var + rsizes: array[50_000, int] + rsizesLen: int + + proc trackSize(size: int) = + rsizes[rsizesLen] = size + inc rsizesLen + + proc untrackSize(size: int) = + for i in 0 .. rsizesLen-1: + if rsizes[i] == size: + rsizes[i] = rsizes[rsizesLen-1] + dec rsizesLen + return + c_fprintf(stdout, "%ld\n", size) + sysAssert(false, "untracked size!") +else: + template trackSize(x) = discard + template untrackSize(x) = discard + +when false: + # not yet used by the GCs + proc rawTryAlloc(a: var MemRegion; requestedSize: int): pointer = + sysAssert(allocInv(a), "rawAlloc: begin") + sysAssert(roundup(65, 8) == 72, "rawAlloc: roundup broken") + sysAssert(requestedSize >= sizeof(FreeCell), "rawAlloc: requested size too small") + var size = roundup(requestedSize, MemAlign) + inc a.occ, size + trackSize(size) + sysAssert(size >= requestedSize, "insufficient allocated size!") + #c_fprintf(stdout, "alloc; size: %ld; %ld\n", requestedSize, size) + if size <= SmallChunkSize-smallChunkOverhead(): + # allocate a small block: for small chunks, we use only its next pointer + var s = size div MemAlign + var c = a.freeSmallChunks[s] + if c == nil: + result = nil + else: + sysAssert c.size == size, "rawAlloc 6" + if c.freeList == nil: + sysAssert(c.acc + smallChunkOverhead() + size <= SmallChunkSize, + "rawAlloc 7") + result = cast[pointer](cast[ByteAddress](addr(c.data)) +% c.acc) + inc(c.acc, size) + else: + result = c.freeList + sysAssert(c.freeList.zeroField == 0, "rawAlloc 8") + c.freeList = c.freeList.next + dec(c.free, size) + sysAssert((cast[ByteAddress](result) and (MemAlign-1)) == 0, "rawAlloc 9") + if c.free < size: + listRemove(a.freeSmallChunks[s], c) + sysAssert(allocInv(a), "rawAlloc: end listRemove test") + sysAssert(((cast[ByteAddress](result) and PageMask) - smallChunkOverhead()) %% + size == 0, "rawAlloc 21") + sysAssert(allocInv(a), "rawAlloc: end small size") + else: + inc size, bigChunkOverhead() + var fl, sl: int + mappingSearch(size, fl, sl) + sysAssert((size and PageMask) == 0, "getBigChunk: unaligned chunk") + let c = findSuitableBlock(a, fl, sl) + if c != nil: + removeChunkFromMatrix2(a, c, fl, sl) + if c.size >= size + PageSize: + splitChunk(a, c, size) + # set 'used' to to true: + c.prevSize = 1 + incl(a, a.chunkStarts, pageIndex(c)) + dec(a.freeMem, size) + result = addr(c.data) + sysAssert((cast[ByteAddress](c) and (MemAlign-1)) == 0, "rawAlloc 13") + sysAssert((cast[ByteAddress](c) and PageMask) == 0, "rawAlloc: Not aligned on a page boundary") + if a.root == nil: a.root = getBottom(a) + add(a, a.root, cast[ByteAddress](result), cast[ByteAddress](result)+%size) + else: + result = nil + proc rawAlloc(a: var MemRegion, requestedSize: int): pointer = sysAssert(allocInv(a), "rawAlloc: begin") sysAssert(roundup(65, 8) == 72, "rawAlloc: roundup broken") @@ -676,10 +785,13 @@ proc rawAlloc(a: var MemRegion, requestedSize: int): pointer = sysAssert(((cast[ByteAddress](result) and PageMask) - smallChunkOverhead()) %% size == 0, "rawAlloc 21") sysAssert(allocInv(a), "rawAlloc: end small size") + inc a.occ, size + trackSize(c.size) else: size = requestedSize + bigChunkOverhead() # roundup(requestedSize+bigChunkOverhead(), PageSize) # allocate a large block - var c = getBigChunk(a, size) + var c = if size >= HugeChunkSize: getHugeChunk(a, size) + else: getBigChunk(a, size) sysAssert c.prev == nil, "rawAlloc 10" sysAssert c.next == nil, "rawAlloc 11" result = addr(c.data) @@ -687,9 +799,11 @@ proc rawAlloc(a: var MemRegion, requestedSize: int): pointer = sysAssert((cast[ByteAddress](c) and PageMask) == 0, "rawAlloc: Not aligned on a page boundary") if a.root == nil: a.root = getBottom(a) add(a, a.root, cast[ByteAddress](result), cast[ByteAddress](result)+%size) + inc a.occ, c.size + trackSize(c.size) sysAssert(isAccessible(a, result), "rawAlloc 14") sysAssert(allocInv(a), "rawAlloc: end") - when logAlloc: cprintf("rawAlloc: %ld %p\n", requestedSize, result) + when logAlloc: cprintf("var pointer_%p = alloc(%ld)\n", result, requestedSize) proc rawAlloc0(a: var MemRegion, requestedSize: int): pointer = result = rawAlloc(a, requestedSize) @@ -703,6 +817,9 @@ proc rawDealloc(a: var MemRegion, p: pointer) = # `p` is within a small chunk: var c = cast[PSmallChunk](c) var s = c.size + dec a.occ, s + untrackSize(s) + sysAssert a.occ >= 0, "rawDealloc: negative occupied memory (case A)" sysAssert(((cast[ByteAddress](p) and PageMask) - smallChunkOverhead()) %% s == 0, "rawDealloc 3") var f = cast[ptr FreeCell](p) @@ -733,11 +850,15 @@ proc rawDealloc(a: var MemRegion, p: pointer) = when overwriteFree: c_memset(p, -1'i32, c.size -% bigChunkOverhead()) # free big chunk var c = cast[PBigChunk](c) + dec a.occ, c.size + untrackSize(c.size) + sysAssert a.occ >= 0, "rawDealloc: negative occupied memory (case B)" a.deleted = getBottom(a) del(a, a.root, cast[int](addr(c.data))) - freeBigChunk(a, c) + if c.size >= HugeChunkSize: freeHugeChunk(a, c) + else: freeBigChunk(a, c) sysAssert(allocInv(a), "rawDealloc: end") - when logAlloc: cprintf("rawDealloc: %p\n", p) + when logAlloc: cprintf("dealloc(pointer_%p)\n", p) proc isAllocatedPtr(a: MemRegion, p: pointer): bool = if isAccessible(a, p): @@ -813,13 +934,13 @@ proc alloc0(allocator: var MemRegion, size: Natural): pointer = zeroMem(result, size) proc dealloc(allocator: var MemRegion, p: pointer) = - sysAssert(p != nil, "dealloc 0") + sysAssert(p != nil, "dealloc: p is nil") var x = cast[pointer](cast[ByteAddress](p) -% sizeof(FreeCell)) - sysAssert(x != nil, "dealloc 1") + sysAssert(x != nil, "dealloc: x is nil") sysAssert(isAccessible(allocator, x), "is not accessible") - sysAssert(cast[ptr FreeCell](x).zeroField == 1, "dealloc 2") + sysAssert(cast[ptr FreeCell](x).zeroField == 1, "dealloc: object header corrupted") rawDealloc(allocator, x) - sysAssert(not isAllocatedPtr(allocator, x), "dealloc 3") + sysAssert(not isAllocatedPtr(allocator, x), "dealloc: object still accessible") track("dealloc", p, 0) proc realloc(allocator: var MemRegion, p: pointer, newsize: Natural): pointer = @@ -851,7 +972,8 @@ proc deallocOsPages(a: var MemRegion) = proc getFreeMem(a: MemRegion): int {.inline.} = result = a.freeMem proc getTotalMem(a: MemRegion): int {.inline.} = result = a.currMem proc getOccupiedMem(a: MemRegion): int {.inline.} = - result = a.currMem - a.freeMem + result = a.occ + # a.currMem - a.freeMem # ---------------------- thread memory region ------------------------------- @@ -893,7 +1015,7 @@ template instantiateForRegion(allocator: untyped) = #sysAssert(result == countFreeMem()) proc getTotalMem(): int = return allocator.currMem - proc getOccupiedMem(): int = return getTotalMem() - getFreeMem() + proc getOccupiedMem(): int = return allocator.occ #getTotalMem() - getFreeMem() proc getMaxMem*(): int = return getMaxMem(allocator) # -------------------- shared heap region ---------------------------------- @@ -944,7 +1066,8 @@ template instantiateForRegion(allocator: untyped) = sharedMemStatsShared(sharedHeap.currMem) proc getOccupiedSharedMem(): int = - sharedMemStatsShared(sharedHeap.currMem - sharedHeap.freeMem) + sharedMemStatsShared(sharedHeap.occ) + #sharedMemStatsShared(sharedHeap.currMem - sharedHeap.freeMem) {.pop.} {.pop.} diff --git a/lib/system/assign.nim b/lib/system/assign.nim index f061c89cf..16b56aba7 100644 --- a/lib/system/assign.nim +++ b/lib/system/assign.nim @@ -74,13 +74,17 @@ proc genericAssignAux(dest, src: pointer, mt: PNimType, shallow: bool) = var dst = cast[ByteAddress](cast[PPointer](dest)[]) for i in 0..seq.len-1: genericAssignAux( - cast[pointer](dst +% i*% mt.base.size +% GenericSeqSize), + cast[pointer](dst +% i *% mt.base.size +% GenericSeqSize), cast[pointer](cast[ByteAddress](s2) +% i *% mt.base.size +% GenericSeqSize), mt.base, shallow) of tyObject: - if mt.base != nil: - genericAssignAux(dest, src, mt.base, shallow) + var it = mt.base + # don't use recursion here on the PNimType because the subtype + # check should only be done at the very end: + while it != nil: + genericAssignAux(dest, src, it.node, shallow) + it = it.base genericAssignAux(dest, src, mt.node, shallow) # we need to copy m_type field for tyObject, as it could be empty for # sequence reallocations: @@ -89,13 +93,15 @@ proc genericAssignAux(dest, src: pointer, mt: PNimType, shallow: bool) = # if p of TB: # var tbObj = TB(p) # tbObj of TC # needs to be false! + #c_fprintf(stdout, "%s %s\n", pint[].name, mt.name) + chckObjAsgn(cast[ptr PNimType](src)[], mt) pint[] = mt # cast[ptr PNimType](src)[] of tyTuple: genericAssignAux(dest, src, mt.node, shallow) of tyArray, tyArrayConstr: for i in 0..(mt.size div mt.base.size)-1: - genericAssignAux(cast[pointer](d +% i*% mt.base.size), - cast[pointer](s +% i*% mt.base.size), mt.base, shallow) + genericAssignAux(cast[pointer](d +% i *% mt.base.size), + cast[pointer](s +% i *% mt.base.size), mt.base, shallow) of tyRef: unsureAsgnRef(cast[PPointer](dest), cast[PPointer](s)[]) of tyOptAsRef: @@ -160,8 +166,8 @@ proc genericAssignOpenArray(dest, src: pointer, len: int, d = cast[ByteAddress](dest) s = cast[ByteAddress](src) for i in 0..len-1: - genericAssign(cast[pointer](d +% i*% mt.base.size), - cast[pointer](s +% i*% mt.base.size), mt.base) + genericAssign(cast[pointer](d +% i *% mt.base.size), + cast[pointer](s +% i *% mt.base.size), mt.base) proc objectInit(dest: pointer, typ: PNimType) {.compilerProc, benign.} proc objectInitAux(dest: pointer, n: ptr TNimNode) {.benign.} = @@ -229,7 +235,7 @@ proc genericReset(dest: pointer, mt: PNimType) = pint[] = nil of tyArray, tyArrayConstr: for i in 0..(mt.size div mt.base.size)-1: - genericReset(cast[pointer](d +% i*% mt.base.size), mt.base) + genericReset(cast[pointer](d +% i *% mt.base.size), mt.base) else: zeroMem(dest, mt.size) # set raw bits to zero diff --git a/lib/system/atomics.nim b/lib/system/atomics.nim index afc435638..56ebde823 100644 --- a/lib/system/atomics.nim +++ b/lib/system/atomics.nim @@ -241,7 +241,7 @@ when defined(vcc): else: {.error: "invalid CAS instruction".} -elif defined(tcc) and not defined(windows): +elif defined(tcc): when defined(amd64): {.emit:""" static int __tcc_cas(int *ptr, int oldVal, int newVal) @@ -262,7 +262,7 @@ static int __tcc_cas(int *ptr, int oldVal, int newVal) } """.} else: - assert sizeof(int) == 4 + #assert sizeof(int) == 4 {.emit:""" static int __tcc_cas(int *ptr, int oldVal, int newVal) { @@ -295,7 +295,7 @@ else: when (defined(x86) or defined(amd64)) and defined(vcc): proc cpuRelax* {.importc: "YieldProcessor", header: "<windows.h>".} -elif (defined(x86) or defined(amd64)) and someGcc: +elif (defined(x86) or defined(amd64)) and (someGcc or defined(bcc)): proc cpuRelax* {.inline.} = {.emit: """asm volatile("pause" ::: "memory");""".} elif someGcc or defined(tcc): diff --git a/lib/system/channels.nim b/lib/system/channels.nim index df6c6d41e..3c5bda4b1 100644 --- a/lib/system/channels.nim +++ b/lib/system/channels.nim @@ -32,8 +32,6 @@ type PRawChannel = ptr RawChannel LoadStoreMode = enum mStore, mLoad Channel* {.gcsafe.}[TMsg] = RawChannel ## a channel for thread communication -{.deprecated: [TRawChannel: RawChannel, TLoadStoreMode: LoadStoreMode, - TChannel: Channel].} const ChannelDeadMask = -2 diff --git a/lib/system/chcks.nim b/lib/system/chcks.nim index 69b680dbd..d3651f659 100644 --- a/lib/system/chcks.nim +++ b/lib/system/chcks.nim @@ -52,6 +52,11 @@ proc chckNil(p: pointer) = if p == nil: sysFatal(NilAccessError, "attempt to write to a nil address") +when defined(nimNewRuntime): + proc chckMove(b: bool) {.compilerproc.} = + if not b: + sysFatal(MoveError, "attempt to access an object that was moved") + proc chckNilDisp(p: pointer) {.compilerproc.} = if p == nil: sysFatal(NilAccessError, "cannot dispatch; dispatcher is nil") diff --git a/lib/system/deepcopy.nim b/lib/system/deepcopy.nim index 51e138e5e..750da00cf 100644 --- a/lib/system/deepcopy.nim +++ b/lib/system/deepcopy.nim @@ -105,7 +105,7 @@ proc genericDeepCopyAux(dest, src: pointer, mt: PNimType; tab: var PtrTable) = var dst = cast[ByteAddress](cast[PPointer](dest)[]) for i in 0..seq.len-1: genericDeepCopyAux( - cast[pointer](dst +% i*% mt.base.size +% GenericSeqSize), + cast[pointer](dst +% i *% mt.base.size +% GenericSeqSize), cast[pointer](cast[ByteAddress](s2) +% i *% mt.base.size +% GenericSeqSize), mt.base, tab) @@ -122,8 +122,8 @@ proc genericDeepCopyAux(dest, src: pointer, mt: PNimType; tab: var PtrTable) = genericDeepCopyAux(dest, src, mt.node, tab) of tyArray, tyArrayConstr: for i in 0..(mt.size div mt.base.size)-1: - genericDeepCopyAux(cast[pointer](d +% i*% mt.base.size), - cast[pointer](s +% i*% mt.base.size), mt.base, tab) + genericDeepCopyAux(cast[pointer](d +% i *% mt.base.size), + cast[pointer](s +% i *% mt.base.size), mt.base, tab) of tyRef, tyOptAsRef: let s2 = cast[PPointer](src)[] if s2 == nil: @@ -183,5 +183,5 @@ proc genericDeepCopyOpenArray(dest, src: pointer, len: int, d = cast[ByteAddress](dest) s = cast[ByteAddress](src) for i in 0..len-1: - genericDeepCopy(cast[pointer](d +% i*% mt.base.size), - cast[pointer](s +% i*% mt.base.size), mt.base) + genericDeepCopy(cast[pointer](d +% i *% mt.base.size), + cast[pointer](s +% i *% mt.base.size), mt.base) diff --git a/lib/system/dyncalls.nim b/lib/system/dyncalls.nim index c8e251d1e..f1ff307da 100644 --- a/lib/system/dyncalls.nim +++ b/lib/system/dyncalls.nim @@ -31,6 +31,13 @@ proc nimLoadLibraryError(path: string) = stderr.rawWrite("\n") when not defined(nimDebugDlOpen) and not defined(windows): stderr.rawWrite("compile with -d:nimDebugDlOpen for more information\n") + when defined(windows) and defined(guiapp): + # Because console output is not shown in GUI apps, display error as message box: + const prefix = "could not load: " + var msg: array[1000, char] + copyMem(msg[0].addr, prefix.cstring, prefix.len) + copyMem(msg[prefix.len].addr, path.cstring, min(path.len + 1, 1000 - prefix.len)) + discard MessageBoxA(0, msg[0].addr, nil, 0) quit(1) proc procAddrError(name: cstring) {.noinline.} = diff --git a/lib/system/embedded.nim b/lib/system/embedded.nim index a14f43e7e..4d453fcca 100644 --- a/lib/system/embedded.nim +++ b/lib/system/embedded.nim @@ -40,4 +40,7 @@ proc reraiseException() {.compilerRtl.} = proc writeStackTrace() = discard -proc setControlCHook(hook: proc () {.noconv.} not nil) = discard +proc setControlCHook(hook: proc () {.noconv.}) = discard + +proc closureIterSetupExc(e: ref Exception) {.compilerproc, inline.} = + sysFatal(ReraiseError, "exception handling is not available") diff --git a/lib/system/excpt.nim b/lib/system/excpt.nim index 8e42ea468..dabfe010e 100644 --- a/lib/system/excpt.nim +++ b/lib/system/excpt.nim @@ -33,6 +33,12 @@ proc showErrorMessage(data: cstring) {.gcsafe.} = else: writeToStdErr(data) +proc quitOrDebug() {.inline.} = + when not defined(endb): + quit(1) + else: + endbStep() # call the debugger + proc chckIndx(i, a, b: int): int {.inline, compilerproc, benign.} proc chckRange(i, a, b: int): int {.inline, compilerproc, benign.} proc chckRangeF(x, a, b: float): float {.inline, compilerproc, benign.} @@ -50,6 +56,8 @@ var # list of exception handlers # a global variable for the root of all try blocks currException {.threadvar.}: ref Exception + raise_counter {.threadvar.}: uint + gcFramePtr {.threadvar.}: GcFrame type @@ -108,6 +116,25 @@ proc pushCurrentException(e: ref Exception) {.compilerRtl, inl.} = proc popCurrentException {.compilerRtl, inl.} = currException = currException.up +proc popCurrentExceptionEx(id: uint) {.compilerRtl.} = + # in cpp backend exceptions can pop-up in the different order they were raised, example #5628 + if currException.raise_id == id: + currException = currException.up + else: + var cur = currException.up + var prev = currException + while cur != nil and cur.raise_id != id: + prev = cur + cur = cur.up + if cur == nil: + showErrorMessage("popCurrentExceptionEx() exception was not found in the exception stack. Aborting...") + quitOrDebug() + prev.up = cur.up + +proc closureIterSetupExc(e: ref Exception) {.compilerproc, inline.} = + if not e.isNil: + currException = e + # some platforms have native support for stack traces: const nativeStackTraceSupported* = (defined(macosx) or defined(linux)) and @@ -291,12 +318,6 @@ when hasSomeStackTrace: else: proc stackTraceAvailable*(): bool = result = false -proc quitOrDebug() {.inline.} = - when not defined(endb): - quit(1) - else: - endbStep() # call the debugger - var onUnhandledException*: (proc (errorMsg: string) {. nimcall.}) ## set this error \ ## handler to override the existing behaviour on an unhandled exception. @@ -320,7 +341,11 @@ proc raiseExceptionAux(e: ref Exception) = quitOrDebug() else: pushCurrentException(e) - {.emit: "throw NimException(`e`, `e`->name);".} + raise_counter.inc + if raise_counter == 0: + raise_counter.inc # skip zero at overflow + e.raise_id = raise_counter + {.emit: "`e`->raise();".} else: if excHandler != nil: if not excHandler.hasRaiseAction or excHandler.raiseAction(e): @@ -386,9 +411,9 @@ proc writeStackTrace() = when hasSomeStackTrace: var s = "" rawWriteStackTrace(s) - showErrorMessage(s) + cast[proc (s: cstring) {.noSideEffect, tags: [], nimcall.}](showErrorMessage)(s) else: - showErrorMessage("No stack traceback available\n") + cast[proc (s: cstring) {.noSideEffect, tags: [], nimcall.}](showErrorMessage)("No stack traceback available\n") proc getStackTrace(): string = when hasSomeStackTrace: @@ -482,7 +507,7 @@ when not defined(noSignalHandler) and not defined(useNimRtl): registerSignalHandler() # call it in initialization section -proc setControlCHook(hook: proc () {.noconv.} not nil) = +proc setControlCHook(hook: proc () {.noconv.}) = # ugly cast, but should work on all architectures: type SignalHandler = proc (sign: cint) {.noconv, benign.} c_signal(SIGINT, cast[SignalHandler](hook)) diff --git a/lib/system/gc.nim b/lib/system/gc.nim index dac06119d..425963f3f 100644 --- a/lib/system/gc.nim +++ b/lib/system/gc.nim @@ -21,10 +21,6 @@ const # reaches this threshold # this seems to be a good value withRealTime = defined(useRealtimeGC) - useMarkForDebug = defined(gcGenerational) - useBackupGc = true # use a simple M&S GC to collect - # cycles instead of the complex - # algorithm when withRealTime and not declared(getTicks): include "system/timers" @@ -92,14 +88,12 @@ type maxPause: Nanos # max allowed pause in nanoseconds; active if > 0 region: MemRegion # garbage collected region stat: GcStat - when useMarkForDebug or useBackupGc: - marked: CellSet - additionalRoots: CellSeq # dummy roots for GC_ref/unref + marked: CellSet + additionalRoots: CellSeq # dummy roots for GC_ref/unref when hasThreadSupport: toDispose: SharedList[pointer] + gcThreadId: int -{.deprecated: [TWalkOp: WalkOp, TFinalizer: Finalizer, TGcHeap: GcHeap, - TGcStat: GcStat].} var gch {.rtlThreadVar.}: GcHeap @@ -165,12 +159,12 @@ when defined(logGC): if not c.typ.name.isNil: typName = c.typ.name - when leakDetector: - c_fprintf(stdout, "[GC] %s: %p %d %s rc=%ld from %s(%ld)\n", - msg, c, kind, typName, c.refcount shr rcShift, c.filename, c.line) - else: - c_fprintf(stdout, "[GC] %s: %p %d %s rc=%ld; color=%ld\n", - msg, c, kind, typName, c.refcount shr rcShift, c.color) + when leakDetector: + c_fprintf(stdout, "[GC] %s: %p %d %s rc=%ld from %s(%ld)\n", + msg, c, kind, typName, c.refcount shr rcShift, c.filename, c.line) + else: + c_fprintf(stdout, "[GC] %s: %p %d %s rc=%ld; thread=%ld\n", + msg, c, kind, typName, c.refcount shr rcShift, gch.gcThreadId) template gcTrace(cell, state: untyped) = when traceGC: traceCell(cell, state) @@ -314,27 +308,12 @@ proc initGC() = init(gch.zct) init(gch.tempStack) init(gch.decStack) - when useMarkForDebug or useBackupGc: - init(gch.marked) - init(gch.additionalRoots) + init(gch.marked) + init(gch.additionalRoots) when hasThreadSupport: init(gch.toDispose) - -when useMarkForDebug or useBackupGc: - type - GlobalMarkerProc = proc () {.nimcall, benign.} - {.deprecated: [TGlobalMarkerProc: GlobalMarkerProc].} - var - globalMarkersLen: int - globalMarkers: array[0.. 7_000, GlobalMarkerProc] - - proc nimRegisterGlobalMarker(markerProc: GlobalMarkerProc) {.compilerProc.} = - if globalMarkersLen <= high(globalMarkers): - globalMarkers[globalMarkersLen] = markerProc - inc globalMarkersLen - else: - echo "[GC] cannot register global variable; too many global variables" - quit 1 + gch.gcThreadId = atomicInc(gHeapidGenerator) - 1 + gcAssert(gch.gcThreadId >= 0, "invalid computed thread ID") proc cellsetReset(s: var CellSet) = deinit(s) @@ -377,10 +356,10 @@ proc forAllChildrenAux(dest: pointer, mt: PNimType, op: WalkOp) = else: discard proc forAllChildren(cell: PCell, op: WalkOp) = - gcAssert(cell != nil, "forAllChildren: 1") - gcAssert(isAllocatedPtr(gch.region, cell), "forAllChildren: 2") - gcAssert(cell.typ != nil, "forAllChildren: 3") - gcAssert cell.typ.kind in {tyRef, tyOptAsRef, tySequence, tyString}, "forAllChildren: 4" + gcAssert(cell != nil, "forAllChildren: cell is nil") + gcAssert(isAllocatedPtr(gch.region, cell), "forAllChildren: pointer not part of the heap") + gcAssert(cell.typ != nil, "forAllChildren: cell.typ is nil") + gcAssert cell.typ.kind in {tyRef, tyOptAsRef, tySequence, tyString}, "forAllChildren: unknown GC'ed type" let marker = cell.typ.marker if marker != nil: marker(cellToUsr(cell), op.int) @@ -481,7 +460,7 @@ proc rawNewObj(typ: PNimType, size: int, gch: var GcHeap): pointer = release(gch) when useCellIds: inc gch.idGenerator - res.id = gch.idGenerator + res.id = gch.idGenerator * 1000_000 + gch.gcThreadId result = cellToUsr(res) sysAssert(allocInv(gch.region), "rawNewObj end") @@ -528,7 +507,7 @@ proc newObjRC1(typ: PNimType, size: int): pointer {.compilerRtl.} = release(gch) when useCellIds: inc gch.idGenerator - res.id = gch.idGenerator + res.id = gch.idGenerator * 1000_000 + gch.gcThreadId result = cellToUsr(res) zeroMem(result, size) sysAssert(allocInv(gch.region), "newObjRC1 end") @@ -556,7 +535,7 @@ proc growObj(old: pointer, newsize: int, gch: var GcHeap): pointer = var oldsize = cast[PGenericSeq](old).len*elemSize + GenericSeqSize copyMem(res, ol, oldsize + sizeof(Cell)) - zeroMem(cast[pointer](cast[ByteAddress](res)+% oldsize +% sizeof(Cell)), + zeroMem(cast[pointer](cast[ByteAddress](res) +% oldsize +% sizeof(Cell)), newsize-oldsize) sysAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "growObj: 3") # This can be wrong for intermediate temps that are nevertheless on the @@ -598,7 +577,7 @@ proc growObj(old: pointer, newsize: int, gch: var GcHeap): pointer = release(gch) when useCellIds: inc gch.idGenerator - res.id = gch.idGenerator + res.id = gch.idGenerator * 1000_000 + gch.gcThreadId result = cellToUsr(res) sysAssert(allocInv(gch.region), "growObj end") when defined(memProfiler): nimProfile(newsize-oldsize) @@ -623,29 +602,31 @@ proc freeCyclicCell(gch: var GcHeap, c: PCell) = gcAssert(c.typ != nil, "freeCyclicCell") zeroMem(c, sizeof(Cell)) -when useBackupGc: - proc sweep(gch: var GcHeap) = - for x in allObjects(gch.region): - if isCell(x): - # cast to PCell is correct here: - var c = cast[PCell](x) - if c notin gch.marked: freeCyclicCell(gch, c) - -when useMarkForDebug or useBackupGc: - proc markS(gch: var GcHeap, c: PCell) = - incl(gch.marked, c) - gcAssert gch.tempStack.len == 0, "stack not empty!" - forAllChildren(c, waMarkPrecise) - while gch.tempStack.len > 0: - dec gch.tempStack.len - var d = gch.tempStack.d[gch.tempStack.len] - if not containsOrIncl(gch.marked, d): - forAllChildren(d, waMarkPrecise) - - proc markGlobals(gch: var GcHeap) = +proc sweep(gch: var GcHeap) = + for x in allObjects(gch.region): + if isCell(x): + # cast to PCell is correct here: + var c = cast[PCell](x) + if c notin gch.marked: freeCyclicCell(gch, c) + +proc markS(gch: var GcHeap, c: PCell) = + gcAssert isAllocatedPtr(gch.region, c), "markS: foreign heap root detected A!" + incl(gch.marked, c) + gcAssert gch.tempStack.len == 0, "stack not empty!" + forAllChildren(c, waMarkPrecise) + while gch.tempStack.len > 0: + dec gch.tempStack.len + var d = gch.tempStack.d[gch.tempStack.len] + gcAssert isAllocatedPtr(gch.region, d), "markS: foreign heap root detected B!" + if not containsOrIncl(gch.marked, d): + forAllChildren(d, waMarkPrecise) + +proc markGlobals(gch: var GcHeap) = + if gch.gcThreadId == 0: for i in 0 .. globalMarkersLen-1: globalMarkers[i]() - let d = gch.additionalRoots.d - for i in 0 .. gch.additionalRoots.len-1: markS(gch, d[i]) + for i in 0 .. threadLocalMarkersLen-1: threadLocalMarkers[i]() + let d = gch.additionalRoots.d + for i in 0 .. gch.additionalRoots.len-1: markS(gch, d[i]) when logGC: var @@ -689,16 +670,9 @@ proc doOperation(p: pointer, op: WalkOp) = of waPush: add(gch.tempStack, c) of waMarkGlobal: - when useMarkForDebug or useBackupGc: - when hasThreadSupport: - # could point to a cell which we don't own and don't want to touch/trace - if isAllocatedPtr(gch.region, c): - markS(gch, c) - else: - markS(gch, c) + markS(gch, c) of waMarkPrecise: - when useMarkForDebug or useBackupGc: - add(gch.tempStack, c) + add(gch.tempStack, c) #of waDebug: debugGraph(c) proc nimGCvisit(d: pointer, op: int) {.compilerRtl.} = @@ -712,14 +686,13 @@ proc collectCycles(gch: var GcHeap) = nimGCunref(c) # ensure the ZCT 'color' is not used: while gch.zct.len > 0: discard collectZCT(gch) - when useBackupGc: - cellsetReset(gch.marked) - var d = gch.decStack.d - for i in 0..gch.decStack.len-1: - sysAssert isAllocatedPtr(gch.region, d[i]), "collectCycles" - markS(gch, d[i]) - markGlobals(gch) - sweep(gch) + cellsetReset(gch.marked) + var d = gch.decStack.d + for i in 0..gch.decStack.len-1: + sysAssert isAllocatedPtr(gch.region, d[i]), "collectCycles" + markS(gch, d[i]) + markGlobals(gch) + sweep(gch) proc gcMark(gch: var GcHeap, p: pointer) {.inline.} = # the addresses are not as cells on the stack, so turn them to cells: @@ -860,7 +833,7 @@ proc collectCT(gch: var GcHeap) = if (gch.zct.len >= stackMarkCosts or (cycleGC and getOccupiedMem(gch.region)>=gch.cycleThreshold) or alwaysGC) and gch.recGcLock == 0: - when useMarkForDebug: + when false: prepareForInteriorPointerChecking(gch.region) cellsetReset(gch.marked) markForDebug(gch) diff --git a/lib/system/gc2.nim b/lib/system/gc2.nim index d57a01dc7..283919503 100644 --- a/lib/system/gc2.nim +++ b/lib/system/gc2.nim @@ -104,6 +104,7 @@ type pDumpHeapFile: pointer # File that is used for GC_dumpHeap when hasThreadSupport: toDispose: SharedList[pointer] + gcThreadId: int var gch {.rtlThreadVar.}: GcHeap @@ -119,22 +120,6 @@ template release(gch: GcHeap) = when hasThreadSupport and hasSharedHeap: releaseSys(HeapLock) -proc initGC() = - when not defined(useNimRtl): - gch.red = (1-gch.black) - gch.cycleThreshold = InitialCycleThreshold - gch.stat.stackScans = 0 - gch.stat.completedCollections = 0 - gch.stat.maxThreshold = 0 - gch.stat.maxStackSize = 0 - gch.stat.maxStackCells = 0 - gch.stat.cycleTableSize = 0 - # init the rt - init(gch.additionalRoots) - init(gch.greyStack) - when hasThreadSupport: - init(gch.toDispose) - # Which color to use for new objects is tricky: When we're marking, # they have to be *white* so that everything is marked that is only # reachable from them. However, when we are sweeping, they have to @@ -193,7 +178,10 @@ proc writeCell(file: File; msg: cstring, c: PCell) = let id = c.id else: let id = c - when leakDetector: + when defined(nimTypeNames): + c_fprintf(file, "%s %p %d escaped=%ld color=%c of type %s\n", + msg, id, kind, didEscape(c), col, c.typ.name) + elif leakDetector: c_fprintf(file, "%s %p %d escaped=%ld color=%c from %s(%ld)\n", msg, id, kind, didEscape(c), col, c.filename, c.line) else: @@ -284,20 +272,6 @@ proc unsureAsgnRef(dest: PPointer, src: pointer) {.compilerProc.} = if not isOnStack(dest): markGrey(s) dest[] = src -type - GlobalMarkerProc = proc () {.nimcall, benign.} -var - globalMarkersLen: int - globalMarkers: array[0.. 7_000, GlobalMarkerProc] - -proc nimRegisterGlobalMarker(markerProc: GlobalMarkerProc) {.compilerProc.} = - if globalMarkersLen <= high(globalMarkers): - globalMarkers[globalMarkersLen] = markerProc - inc globalMarkersLen - else: - echo "[GC] cannot register global variable; too many global variables" - quit 1 - proc forAllSlotsAux(dest: pointer, n: ptr TNimNode, op: WalkOp) {.benign.} = var d = cast[ByteAddress](dest) case n.kind @@ -354,6 +328,24 @@ proc gcInvariant*() = include gc_common +proc initGC() = + when not defined(useNimRtl): + gch.red = (1-gch.black) + gch.cycleThreshold = InitialCycleThreshold + gch.stat.stackScans = 0 + gch.stat.completedCollections = 0 + gch.stat.maxThreshold = 0 + gch.stat.maxStackSize = 0 + gch.stat.maxStackCells = 0 + gch.stat.cycleTableSize = 0 + # init the rt + init(gch.additionalRoots) + init(gch.greyStack) + when hasThreadSupport: + init(gch.toDispose) + gch.gcThreadId = atomicInc(gHeapidGenerator) - 1 + gcAssert(gch.gcThreadId >= 0, "invalid computed thread ID") + proc rawNewObj(typ: PNimType, size: int, gch: var GcHeap): pointer = # generates a new object and sets its reference counter to 0 sysAssert(allocInv(gch.region), "rawNewObj begin") @@ -492,7 +484,9 @@ proc GC_dumpHeap*(file: File) = c_fprintf(file, "onstack %p\n", d[i]) else: c_fprintf(file, "onstack_invalid %p\n", d[i]) - for i in 0 .. globalMarkersLen-1: globalMarkers[i]() + if gch.gcThreadId == 0: + for i in 0 .. globalMarkersLen-1: globalMarkers[i]() + for i in 0 .. threadLocalMarkersLen-1: threadLocalMarkers[i]() while true: let x = allObjectsAsProc(gch.region, addr spaceIter) if spaceIter.state < 0: break @@ -579,7 +573,9 @@ proc markIncremental(gch: var GcHeap): bool = result = true proc markGlobals(gch: var GcHeap) = - for i in 0 .. globalMarkersLen-1: globalMarkers[i]() + if gch.gcThreadId == 0: + for i in 0 .. globalMarkersLen-1: globalMarkers[i]() + for i in 0 .. threadLocalMarkersLen-1: threadLocalMarkers[i]() proc doOperation(p: pointer, op: WalkOp) = if p == nil: return @@ -599,22 +595,14 @@ proc doOperation(p: pointer, op: WalkOp) = markRoot(gch, c) else: dumpRoot(gch, c) - when hasThreadSupport: - # could point to a cell which we don't own and don't want to touch/trace - if isAllocatedPtr(gch.region, c): handleRoot() - else: - #gcAssert(isAllocatedPtr(gch.region, c), "doOperation: waMarkGlobal") + handleRoot() + discard allocInv(gch.region) + of waMarkGrey: + when false: if not isAllocatedPtr(gch.region, c): - c_fprintf(stdout, "[GC] not allocated anymore: MarkGlobal %p\n", c) + c_fprintf(stdout, "[GC] not allocated anymore: MarkGrey %p\n", c) #GC_dumpHeap() sysAssert(false, "wtf") - handleRoot() - discard allocInv(gch.region) - of waMarkGrey: - if not isAllocatedPtr(gch.region, c): - c_fprintf(stdout, "[GC] not allocated anymore: MarkGrey %p\n", c) - #GC_dumpHeap() - sysAssert(false, "wtf") if c.color == 1-gch.black: c.setColor(rcGrey) add(gch.greyStack, c) diff --git a/lib/system/gc_common.nim b/lib/system/gc_common.nim index 484a4db9a..939776a58 100644 --- a/lib/system/gc_common.nim +++ b/lib/system/gc_common.nim @@ -18,12 +18,45 @@ proc protect*(x: pointer): ForeignCell = result.owner = addr(gch) when defined(nimTypeNames): + type InstancesInfo = array[400, (cstring, int, int)] + proc sortInstances(a: var InstancesInfo; n: int) = + # we use shellsort here; fast and simple + var h = 1 + while true: + h = 3 * h + 1 + if h > n: break + while true: + h = h div 3 + for i in countup(h, n - 1): + var v = a[i] + var j = i + while a[j - h][2] < v[2]: + a[j] = a[j - h] + j = j - h + if j < h: break + a[j] = v + if h == 1: break + proc dumpNumberOfInstances* = + # also add the allocated strings to the list of known types: + if strDesc.nextType == nil: + strDesc.nextType = nimTypeRoot + strDesc.name = "string" + nimTypeRoot = addr strDesc + var a: InstancesInfo + var n = 0 var it = nimTypeRoot + var totalAllocated = 0 while it != nil: - if it.instances > 0: - c_fprintf(stdout, "[Heap] %s: #%ld; bytes: %ld\n", it.name, it.instances, it.sizes) + if (it.instances > 0 or it.sizes != 0) and n < a.len: + a[n] = (it.name, it.instances, it.sizes) + inc n + inc totalAllocated, it.sizes it = it.nextType + sortInstances(a, n) + for i in 0 .. n-1: + c_fprintf(stdout, "[Heap] %s: #%ld; bytes: %ld\n", a[i][0], a[i][1], a[i][2]) + c_fprintf(stdout, "[Heap] total number of bytes: %ld\n", totalAllocated) when defined(nimGcRefLeak): proc oomhandler() = @@ -36,12 +69,12 @@ template decTypeSize(cell, t) = # XXX this needs to use atomics for multithreaded apps! when defined(nimTypeNames): if t.kind in {tyString, tySequence}: - let len = cast[PGenericSeq](cellToUsr(cell)).len - let base = if t.kind == tyString: 1 else: t.base.size - let size = addInt(mulInt(len, base), GenericSeqSize) + let cap = cast[PGenericSeq](cellToUsr(cell)).space + let size = if t.kind == tyString: cap+1+GenericSeqSize + else: addInt(mulInt(cap, t.base.size), GenericSeqSize) dec t.sizes, size+sizeof(Cell) else: - dec t.sizes, t.size+sizeof(Cell) + dec t.sizes, t.base.size+sizeof(Cell) dec t.instances template incTypeSize(typ, size) = @@ -167,7 +200,7 @@ when declared(threadType): if threadType == ThreadType.None: initAllocator() var stackTop {.volatile.}: pointer - setStackBottom(addr(stackTop)) + nimGC_setStackBottom(addr(stackTop)) initGC() threadType = ThreadType.ForeignThread @@ -224,7 +257,7 @@ when nimCoroutines: gch.activeStack.setPosition(addr(sp)) when not defined(useNimRtl): - proc setStackBottom(theStackBottom: pointer) = + proc nimGC_setStackBottom(theStackBottom: pointer) = # Initializes main stack of the thread. when nimCoroutines: if gch.stack.next == nil: @@ -393,3 +426,28 @@ proc deallocHeap*(runFinalizers = true; allowGcAfterwards = true) = zeroMem(addr gch.region, sizeof(gch.region)) if allowGcAfterwards: initGC() + +type + GlobalMarkerProc = proc () {.nimcall, benign.} +var + globalMarkersLen: int + globalMarkers: array[0.. 3499, GlobalMarkerProc] + threadLocalMarkersLen: int + threadLocalMarkers: array[0.. 3499, GlobalMarkerProc] + gHeapidGenerator: int + +proc nimRegisterGlobalMarker(markerProc: GlobalMarkerProc) {.compilerProc.} = + if globalMarkersLen <= high(globalMarkers): + globalMarkers[globalMarkersLen] = markerProc + inc globalMarkersLen + else: + echo "[GC] cannot register global variable; too many global variables" + quit 1 + +proc nimRegisterThreadLocalMarker(markerProc: GlobalMarkerProc) {.compilerProc.} = + if threadLocalMarkersLen <= high(threadLocalMarkers): + threadLocalMarkers[threadLocalMarkersLen] = markerProc + inc threadLocalMarkersLen + else: + echo "[GC] cannot register thread local variable; too many thread local variables" + quit 1 diff --git a/lib/system/gc_ms.nim b/lib/system/gc_ms.nim index 5fc48d848..75f9c6749 100644 --- a/lib/system/gc_ms.nim +++ b/lib/system/gc_ms.nim @@ -40,8 +40,6 @@ type # A ref type can have a finalizer that is called before the object's # storage is freed. - GlobalMarkerProc = proc () {.nimcall, benign.} - GcStat = object collections: int # number of performed full collections maxThreshold: int # max threshold that has been set @@ -75,9 +73,12 @@ type stat: GcStat when hasThreadSupport: toDispose: SharedList[pointer] + gcThreadId: int additionalRoots: CellSeq # dummy roots for GC_ref/unref -{.deprecated: [TWalkOp: WalkOp, TFinalizer: Finalizer, TGcStat: GcStat, - TGlobalMarkerProc: GlobalMarkerProc, TGcHeap: GcHeap].} + when defined(nimTracing): + tracing: bool + indentation: int + var gch {.rtlThreadVar.}: GcHeap @@ -119,24 +120,12 @@ proc unsureAsgnRef(dest: PPointer, src: pointer) {.inline.} = proc internRefcount(p: pointer): int {.exportc: "getRefcount".} = result = 0 -var - globalMarkersLen: int - globalMarkers: array[0.. 7_000, GlobalMarkerProc] - -proc nimRegisterGlobalMarker(markerProc: GlobalMarkerProc) {.compilerProc.} = - if globalMarkersLen <= high(globalMarkers): - globalMarkers[globalMarkersLen] = markerProc - inc globalMarkersLen - else: - echo "[GC] cannot register global variable; too many global variables" - quit 1 - # this that has to equals zero, otherwise we have to round up UnitsPerPage: when BitsPerPage mod (sizeof(int)*8) != 0: {.error: "(BitsPerPage mod BitsPerUnit) should be zero!".} # forward declarations: -proc collectCT(gch: var GcHeap) {.benign.} +proc collectCT(gch: var GcHeap; size: int) {.benign.} proc forAllChildren(cell: PCell, op: WalkOp) {.benign.} proc doOperation(p: pointer, op: WalkOp) {.benign.} proc forAllChildrenAux(dest: pointer, mt: PNimType, op: WalkOp) {.benign.} @@ -234,6 +223,8 @@ proc initGC() = init(gch.marked) when hasThreadSupport: init(gch.toDispose) + gch.gcThreadId = atomicInc(gHeapidGenerator) - 1 + gcAssert(gch.gcThreadId >= 0, "invalid computed thread ID") proc forAllSlotsAux(dest: pointer, n: ptr TNimNode, op: WalkOp) {.benign.} = var d = cast[ByteAddress](dest) @@ -286,7 +277,7 @@ proc rawNewObj(typ: PNimType, size: int, gch: var GcHeap): pointer = incTypeSize typ, size acquire(gch) gcAssert(typ.kind in {tyRef, tyOptAsRef, tyString, tySequence}, "newObj: 1") - collectCT(gch) + collectCT(gch, size + sizeof(Cell)) var res = cast[PCell](rawAlloc(gch.region, size + sizeof(Cell))) gcAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "newObj: 2") # now it is buffered in the ZCT @@ -341,7 +332,7 @@ proc newSeqRC1(typ: PNimType, len: int): pointer {.compilerRtl.} = proc growObj(old: pointer, newsize: int, gch: var GcHeap): pointer = acquire(gch) - collectCT(gch) + collectCT(gch, newsize + sizeof(Cell)) var ol = usrToCell(old) sysAssert(ol.typ != nil, "growObj: 1") gcAssert(ol.typ.kind in {tyString, tySequence}, "growObj: 2") @@ -356,12 +347,6 @@ proc growObj(old: pointer, newsize: int, gch: var GcHeap): pointer = zeroMem(cast[pointer](cast[ByteAddress](res)+% oldsize +% sizeof(Cell)), newsize-oldsize) sysAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "growObj: 3") - when false: - # this is wrong since seqs can be shared via 'shallow': - when withBitvectors: excl(gch.allocated, ol) - when reallyDealloc: rawDealloc(gch.region, ol) - else: - zeroMem(ol, sizeof(Cell)) when withBitvectors: incl(gch.allocated, res) when useCellIds: inc gch.idGenerator @@ -392,6 +377,13 @@ proc mark(gch: var GcHeap, c: PCell) = forAllChildren(d, waMarkPrecise) else: # XXX no 'if c.refCount != rcBlack' here? + when defined(nimTracing): + if gch.tracing: + for i in 1..gch.indentation: c_fprintf(stdout, " ") + c_fprintf(stdout, "start marking %p of type %s ((\n", + c, c.typ.name) + inc gch.indentation, 2 + c.refCount = rcBlack gcAssert gch.tempStack.len == 0, "stack not empty!" forAllChildren(c, waMarkPrecise) @@ -402,19 +394,24 @@ proc mark(gch: var GcHeap, c: PCell) = d.refCount = rcBlack forAllChildren(d, waMarkPrecise) + when defined(nimTracing): + if gch.tracing: + dec gch.indentation, 2 + for i in 1..gch.indentation: c_fprintf(stdout, " ") + c_fprintf(stdout, "finished marking %p of type %s))\n", + c, c.typ.name) + proc doOperation(p: pointer, op: WalkOp) = if p == nil: return var c: PCell = usrToCell(p) gcAssert(c != nil, "doOperation: 1") case op - of waMarkGlobal: - when hasThreadSupport: - # could point to a cell which we don't own and don't want to touch/trace - if isAllocatedPtr(gch.region, c): - mark(gch, c) + of waMarkGlobal: mark(gch, c) + of waMarkPrecise: + when defined(nimTracing): + if c.refcount == rcWhite: mark(gch, c) else: - mark(gch, c) - of waMarkPrecise: add(gch.tempStack, c) + add(gch.tempStack, c) proc nimGCvisit(d: pointer, op: int) {.compilerRtl.} = doOperation(d, WalkOp(op)) @@ -450,7 +447,18 @@ when false: quit 1 proc markGlobals(gch: var GcHeap) = - for i in 0 .. globalMarkersLen-1: globalMarkers[i]() + if gch.gcThreadId == 0: + when defined(nimTracing): + if gch.tracing: + c_fprintf(stdout, "------- globals marking phase:\n") + for i in 0 .. globalMarkersLen-1: globalMarkers[i]() + when defined(nimTracing): + if gch.tracing: + c_fprintf(stdout, "------- thread locals marking phase:\n") + for i in 0 .. threadLocalMarkersLen-1: threadLocalMarkers[i]() + when defined(nimTracing): + if gch.tracing: + c_fprintf(stdout, "------- additional roots marking phase:\n") let d = gch.additionalRoots.d for i in 0 .. gch.additionalRoots.len-1: mark(gch, d[i]) @@ -470,6 +478,9 @@ proc markStackAndRegisters(gch: var GcHeap) {.noinline, cdecl.} = proc collectCTBody(gch: var GcHeap) = when not nimCoroutines: gch.stat.maxStackSize = max(gch.stat.maxStackSize, stackSize()) + when defined(nimTracing): + if gch.tracing: + c_fprintf(stdout, "------- stack marking phase:\n") prepareForInteriorPointerChecking(gch.region) markStackAndRegisters(gch) markGlobals(gch) @@ -483,8 +494,9 @@ proc collectCTBody(gch: var GcHeap) = gch.stat.maxThreshold = max(gch.stat.maxThreshold, gch.cycleThreshold) sysAssert(allocInv(gch.region), "collectCT: end") -proc collectCT(gch: var GcHeap) = - if getOccupiedMem(gch.region) >= gch.cycleThreshold and gch.recGcLock == 0: +proc collectCT(gch: var GcHeap; size: int) = + if (getOccupiedMem(gch.region) >= gch.cycleThreshold or + size > getFreeMem(gch.region)) and gch.recGcLock == 0: collectCTBody(gch) when not defined(useNimRtl): @@ -511,11 +523,15 @@ when not defined(useNimRtl): gch.cycleThreshold = high(gch.cycleThreshold)-1 # set to the max value to suppress the cycle detector + when defined(nimTracing): + proc GC_logTrace*() = + gch.tracing = true + proc GC_fullCollect() = acquire(gch) var oldThreshold = gch.cycleThreshold gch.cycleThreshold = 0 # forces cycle collection - collectCT(gch) + collectCT(gch, 0) gch.cycleThreshold = oldThreshold release(gch) diff --git a/lib/system/gc_regions.nim b/lib/system/gc_regions.nim index e9efbdfb0..06fded86b 100644 --- a/lib/system/gc_regions.nim +++ b/lib/system/gc_regions.nim @@ -70,8 +70,9 @@ type bump: pointer head, tail: Chunk nextChunkSize, totalSize: int - freeLists: array[MaxSmallObject div MemAlign, FreeEntry] - holes: SizedFreeEntry + when false: + freeLists: array[MaxSmallObject div MemAlign, FreeEntry] + holes: SizedFreeEntry when hasThreadSupport: lock: SysLock @@ -144,22 +145,24 @@ proc allocSlowPath(r: var MemRegion; size: int) = r.tail = fresh r.remaining = s - sizeof(BaseChunk) -proc alloc(r: var MemRegion; size: int): pointer = - if size <= MaxSmallObject: - var it = r.freeLists[size div MemAlign] - if it != nil: - r.freeLists[size div MemAlign] = it.next - return pointer(it) - else: - var it = r.holes - var prev: SizedFreeEntry = nil - while it != nil: - if it.size >= size: - if prev != nil: prev.next = it.next - else: r.holes = it.next +proc allocFast(r: var MemRegion; size: int): pointer = + when false: + if size <= MaxSmallObject: + var it = r.freeLists[size div MemAlign] + if it != nil: + r.freeLists[size div MemAlign] = it.next return pointer(it) - prev = it - it = it.next + else: + var it = r.holes + var prev: SizedFreeEntry = nil + while it != nil: + if it.size >= size: + if prev != nil: prev.next = it.next + else: r.holes = it.next + return pointer(it) + prev = it + it = it.next + let size = roundup(size, MemAlign) if size > r.remaining: allocSlowPath(r, size) sysAssert(size <= r.remaining, "size <= r.remaining") @@ -184,15 +187,16 @@ proc dealloc(r: var MemRegion; p: pointer; size: int) = # it is benefitial to not use the free lists here: if r.bump -! size == p: dec r.bump, size - elif size <= MaxSmallObject: - let it = cast[FreeEntry](p) - it.next = r.freeLists[size div MemAlign] - r.freeLists[size div MemAlign] = it - else: - let it = cast[SizedFreeEntry](p) - it.size = size - it.next = r.holes - r.holes = it + when false: + if size <= MaxSmallObject: + let it = cast[FreeEntry](p) + it.next = r.freeLists[size div MemAlign] + r.freeLists[size div MemAlign] = it + else: + let it = cast[SizedFreeEntry](p) + it.size = size + it.next = r.holes + r.holes = it proc deallocAll(r: var MemRegion; head: Chunk) = var it = head @@ -220,9 +224,10 @@ proc setObstackPtr*(r: var MemRegion; sp: StackPtr) = if sp.current.next != nil: deallocAll(r, sp.current.next) sp.current.next = nil - # better leak this memory than be sorry: - for i in 0..high(r.freeLists): r.freeLists[i] = nil - r.holes = nil + when false: + # better leak this memory than be sorry: + for i in 0..high(r.freeLists): r.freeLists[i] = nil + r.holes = nil #else: # deallocAll(r, r.head) # r.head = nil @@ -270,7 +275,7 @@ proc isOnHeap*(r: MemRegion; p: pointer): bool = it = it.next proc rawNewObj(r: var MemRegion, typ: PNimType, size: int): pointer = - var res = cast[ptr ObjHeader](alloc(r, size + sizeof(ObjHeader))) + var res = cast[ptr ObjHeader](allocFast(r, size + sizeof(ObjHeader))) res.typ = typ if typ.finalizer != nil: res.nextFinal = r.head.head @@ -278,17 +283,19 @@ proc rawNewObj(r: var MemRegion, typ: PNimType, size: int): pointer = result = res +! sizeof(ObjHeader) proc rawNewSeq(r: var MemRegion, typ: PNimType, size: int): pointer = - var res = cast[ptr SeqHeader](alloc(r, size + sizeof(SeqHeader))) + var res = cast[ptr SeqHeader](allocFast(r, size + sizeof(SeqHeader))) res.typ = typ res.region = addr(r) result = res +! sizeof(SeqHeader) proc newObj(typ: PNimType, size: int): pointer {.compilerRtl.} = + sysAssert typ.kind notin {tySequence, tyString}, "newObj cannot be used to construct seqs" result = rawNewObj(tlRegion, typ, size) zeroMem(result, size) when defined(memProfiler): nimProfile(size) proc newObjNoInit(typ: PNimType, size: int): pointer {.compilerRtl.} = + sysAssert typ.kind notin {tySequence, tyString}, "newObj cannot be used to construct seqs" result = rawNewObj(tlRegion, typ, size) when defined(memProfiler): nimProfile(size) @@ -351,6 +358,11 @@ proc alloc0(r: var MemRegion; size: Natural): pointer = # but incorrect in general. XXX result = alloc0(size) +proc alloc(r: var MemRegion; size: Natural): pointer = + # ignore the region. That is correct for the channels module + # but incorrect in general. XXX + result = alloc(size) + proc dealloc(r: var MemRegion; p: pointer) = dealloc(p) proc allocShared(size: Natural): pointer = @@ -389,4 +401,7 @@ proc getFreeMem*(r: MemRegion): int = r.remaining proc getTotalMem*(r: MemRegion): int = result = r.totalSize -proc setStackBottom(theStackBottom: pointer) = discard +proc nimGC_setStackBottom(theStackBottom: pointer) = discard + +proc nimGCref(x: pointer) {.compilerProc.} = discard +proc nimGCunref(x: pointer) {.compilerProc.} = discard diff --git a/lib/system/jssys.nim b/lib/system/jssys.nim index 8065f2255..e12bab184 100644 --- a/lib/system/jssys.nim +++ b/lib/system/jssys.nim @@ -48,10 +48,7 @@ proc nimCharToStr(x: char): string {.compilerproc.} = result[0] = x proc isNimException(): bool {.asmNoStackFrame.} = - when defined(nimphp): - asm "return isset(`lastJSError`['m_type']);" - else: - asm "return `lastJSError`.m_type;" + asm "return `lastJSError`.m_type;" proc getCurrentException*(): ref Exception {.compilerRtl, benign.} = if isNimException(): result = cast[ref Exception](lastJSError) @@ -61,15 +58,14 @@ proc getCurrentExceptionMsg*(): string = if isNimException(): return cast[Exception](lastJSError).msg else: - when not defined(nimphp): - var msg: cstring - {.emit: """ - if (`lastJSError`.message !== undefined) { - `msg` = `lastJSError`.message; - } - """.} - if not msg.isNil: - return $msg + var msg: cstring + {.emit: """ + if (`lastJSError`.message !== undefined) { + `msg` = `lastJSError`.message; + } + """.} + if not msg.isNil: + return $msg return "" proc auxWriteStackTrace(f: PCallFrame): string = @@ -140,12 +136,9 @@ proc raiseException(e: ref Exception, ename: cstring) {. e.name = ename if excHandler == 0: unhandledException(e) - when defined(nimphp): - asm """throw new Exception($`e`["message"]);""" - else: - when NimStackTrace: - e.trace = rawWriteStackTrace() - asm "throw `e`;" + when NimStackTrace: + e.trace = rawWriteStackTrace() + asm "throw `e`;" proc reraiseException() {.compilerproc, asmNoStackFrame.} = if lastJSError == nil: @@ -173,57 +166,35 @@ proc raiseFieldError(f: string) {.compilerproc, noreturn.} = raise newException(FieldError, f & " is not accessible") proc setConstr() {.varargs, asmNoStackFrame, compilerproc.} = - when defined(nimphp): - asm """ - $args = func_get_args(); - $result = array(); - foreach ($args as $x) { - if (is_array($x)) { - for ($j = $x[0]; $j <= $x[1]; $j++) { - $result[$j] = true; - } - } else { - $result[$x] = true; - } - } - return $result; - """ - else: - asm """ - var result = {}; - for (var i = 0; i < arguments.length; ++i) { - var x = arguments[i]; - if (typeof(x) == "object") { - for (var j = x[0]; j <= x[1]; ++j) { - result[j] = true; - } - } else { - result[x] = true; + asm """ + var result = {}; + for (var i = 0; i < arguments.length; ++i) { + var x = arguments[i]; + if (typeof(x) == "object") { + for (var j = x[0]; j <= x[1]; ++j) { + result[j] = true; } + } else { + result[x] = true; } - return result; - """ - -proc makeNimstrLit(c: cstring): string {.asmNoStackFrame, compilerproc.} = - when defined(nimphp): - {.emit: """return `c`;""".} - else: - {.emit: """ - var ln = `c`.length; - var result = new Array(ln + 1); - var i = 0; - for (; i < ln; ++i) { - result[i] = `c`.charCodeAt(i); } - result[i] = 0; // terminating zero return result; - """.} + """ + +proc makeNimstrLit(c: cstring): string {.asmNoStackFrame, compilerproc.} = + {.emit: """ + var ln = `c`.length; + var result = new Array(ln + 1); + var i = 0; + for (; i < ln; ++i) { + result[i] = `c`.charCodeAt(i); + } + result[i] = 0; // terminating zero + return result; + """.} proc cstrToNimstr(c: cstring): string {.asmNoStackFrame, compilerproc.} = - when defined(nimphp): - {.emit: """return `c`;""".} - else: - {.emit: """ + {.emit: """ var ln = `c`.length; var result = new Array(ln); var r = 0; @@ -261,178 +232,108 @@ proc cstrToNimstr(c: cstring): string {.asmNoStackFrame, compilerproc.} = """.} proc toJSStr(s: string): cstring {.asmNoStackFrame, compilerproc.} = - when defined(nimphp): - {.emit: """return `s`;""".} - else: - asm """ - var len = `s`.length-1; - var asciiPart = new Array(len); - var fcc = String.fromCharCode; - var nonAsciiPart = null; - var nonAsciiOffset = 0; - for (var i = 0; i < len; ++i) { - if (nonAsciiPart !== null) { - var offset = (i - nonAsciiOffset) * 2; - var code = `s`[i].toString(16); - if (code.length == 1) { - code = "0"+code; - } - nonAsciiPart[offset] = "%"; - nonAsciiPart[offset + 1] = code; - } - else if (`s`[i] < 128) - asciiPart[i] = fcc(`s`[i]); - else { - asciiPart.length = i; - nonAsciiOffset = i; - nonAsciiPart = new Array((len - i) * 2); - --i; + asm """ + var len = `s`.length-1; + var asciiPart = new Array(len); + var fcc = String.fromCharCode; + var nonAsciiPart = null; + var nonAsciiOffset = 0; + for (var i = 0; i < len; ++i) { + if (nonAsciiPart !== null) { + var offset = (i - nonAsciiOffset) * 2; + var code = `s`[i].toString(16); + if (code.length == 1) { + code = "0"+code; } + nonAsciiPart[offset] = "%"; + nonAsciiPart[offset + 1] = code; } - asciiPart = asciiPart.join(""); - return (nonAsciiPart === null) ? - asciiPart : asciiPart + decodeURIComponent(nonAsciiPart.join("")); + else if (`s`[i] < 128) + asciiPart[i] = fcc(`s`[i]); + else { + asciiPart.length = i; + nonAsciiOffset = i; + nonAsciiPart = new Array((len - i) * 2); + --i; + } + } + asciiPart = asciiPart.join(""); + return (nonAsciiPart === null) ? + asciiPart : asciiPart + decodeURIComponent(nonAsciiPart.join("")); """ proc mnewString(len: int): string {.asmNoStackFrame, compilerproc.} = - when defined(nimphp): - asm """ - return str_repeat(chr(0),`len`); - """ - else: - asm """ - var result = new Array(`len`+1); - result[0] = 0; - result[`len`] = 0; - return result; - """ - -when defined(nimphp): - proc nimAt(x: string; i: int): string {.asmNoStackFrame, compilerproc.} = - asm """ - return `x`[`i`]; - """ - -when defined(nimphp): - proc nimSubstr(s: string; a, b: int): string {. - asmNoStackFrame, compilerproc.} = - asm """return substr(`s`,`a`,`b`-`a`+1);""" + asm """ + var result = new Array(`len`+1); + result[0] = 0; + result[`len`] = 0; + return result; + """ proc SetCard(a: int): int {.compilerproc, asmNoStackFrame.} = # argument type is a fake - when defined(nimphp): - asm """ - return count(`a`); - """ - else: - asm """ - var result = 0; - for (var elem in `a`) { ++result; } - return result; - """ + asm """ + var result = 0; + for (var elem in `a`) { ++result; } + return result; + """ proc SetEq(a, b: int): bool {.compilerproc, asmNoStackFrame.} = - when defined(nimphp): - asm """ - foreach (`a` as $elem=>$_) { if (!isset(`b`[$elem])) return false; } - foreach (`b` as $elem=>$_) { if (!isset(`a`[$elem])) return false; } - return true; - """ - else: - asm """ - for (var elem in `a`) { if (!`b`[elem]) return false; } - for (var elem in `b`) { if (!`a`[elem]) return false; } - return true; - """ + asm """ + for (var elem in `a`) { if (!`b`[elem]) return false; } + for (var elem in `b`) { if (!`a`[elem]) return false; } + return true; + """ proc SetLe(a, b: int): bool {.compilerproc, asmNoStackFrame.} = - when defined(nimphp): - asm """ - foreach (`a` as $elem=>$_) { if (!isset(`b`[$elem])) return false; } - return true; - """ - else: - asm """ - for (var elem in `a`) { if (!`b`[elem]) return false; } - return true; - """ + asm """ + for (var elem in `a`) { if (!`b`[elem]) return false; } + return true; + """ proc SetLt(a, b: int): bool {.compilerproc.} = result = SetLe(a, b) and not SetEq(a, b) proc SetMul(a, b: int): int {.compilerproc, asmNoStackFrame.} = - when defined(nimphp): - asm """ - var $result = array(); - foreach (`a` as $elem=>$_) { - if (isset(`b`[$elem])) { $result[$elem] = true; } - } - return $result; - """ - else: - asm """ - var result = {}; - for (var elem in `a`) { - if (`b`[elem]) { result[elem] = true; } - } - return result; - """ + asm """ + var result = {}; + for (var elem in `a`) { + if (`b`[elem]) { result[elem] = true; } + } + return result; + """ proc SetPlus(a, b: int): int {.compilerproc, asmNoStackFrame.} = - when defined(nimphp): - asm """ - var $result = array(); - foreach (`a` as $elem=>$_) { $result[$elem] = true; } - foreach (`b` as $elem=>$_) { $result[$elem] = true; } - return $result; - """ - else: - asm """ - var result = {}; - for (var elem in `a`) { result[elem] = true; } - for (var elem in `b`) { result[elem] = true; } - return result; - """ + asm """ + var result = {}; + for (var elem in `a`) { result[elem] = true; } + for (var elem in `b`) { result[elem] = true; } + return result; + """ proc SetMinus(a, b: int): int {.compilerproc, asmNoStackFrame.} = - when defined(nimphp): - asm """ - $result = array(); - foreach (`a` as $elem=>$_) { - if (!isset(`b`[$elem])) { $result[$elem] = true; } - } - return $result; - """ - else: - asm """ - var result = {}; - for (var elem in `a`) { - if (!`b`[elem]) { result[elem] = true; } - } - return result; - """ + asm """ + var result = {}; + for (var elem in `a`) { + if (!`b`[elem]) { result[elem] = true; } + } + return result; + """ proc cmpStrings(a, b: string): int {.asmNoStackFrame, compilerProc.} = asm """ if (`a` == `b`) return 0; if (!`a`) return -1; if (!`b`) return 1; - for (var i = 0; i < `a`.length-1; ++i) { + for (var i = 0; i < `a`.length - 1 && i < `b`.length - 1; i++) { var result = `a`[i] - `b`[i]; if (result != 0) return result; } - return 0; + return `a`.length - `b`.length; """ proc cmp(x, y: string): int = - when defined(nimphp): - asm """ - if(`x` < `y`) `result` = -1; - elseif (`x` > `y`) `result` = 1; - else `result` = 0; - """ - else: - return cmpStrings(x, y) + return cmpStrings(x, y) proc eqStrings(a, b: string): bool {.asmNoStackFrame, compilerProc.} = asm """ @@ -467,7 +368,7 @@ elif not defined(nimOldEcho): console.log(buf); """ -elif not defined(nimphp): +else: proc ewriteln(x: cstring) = var node : JSRef {.emit: "`node` = document.getElementsByTagName('body')[0];".} @@ -493,127 +394,77 @@ elif not defined(nimphp): # Arithmetic: proc addInt(a, b: int): int {.asmNoStackFrame, compilerproc.} = - when defined(nimphp): - asm """ - return `a` + `b`; - """ - else: - asm """ - var result = `a` + `b`; - if (result > 2147483647 || result < -2147483648) `raiseOverflow`(); - return result; - """ + asm """ + var result = `a` + `b`; + if (result > 2147483647 || result < -2147483648) `raiseOverflow`(); + return result; + """ proc subInt(a, b: int): int {.asmNoStackFrame, compilerproc.} = - when defined(nimphp): - asm """ - return `a` - `b`; - """ - else: - asm """ - var result = `a` - `b`; - if (result > 2147483647 || result < -2147483648) `raiseOverflow`(); - return result; - """ + asm """ + var result = `a` - `b`; + if (result > 2147483647 || result < -2147483648) `raiseOverflow`(); + return result; + """ proc mulInt(a, b: int): int {.asmNoStackFrame, compilerproc.} = - when defined(nimphp): - asm """ - return `a` * `b`; - """ - else: - asm """ - var result = `a` * `b`; - if (result > 2147483647 || result < -2147483648) `raiseOverflow`(); - return result; - """ + asm """ + var result = `a` * `b`; + if (result > 2147483647 || result < -2147483648) `raiseOverflow`(); + return result; + """ proc divInt(a, b: int): int {.asmNoStackFrame, compilerproc.} = - when defined(nimphp): - asm """ - return trunc(`a` / `b`); - """ - else: - asm """ - if (`b` == 0) `raiseDivByZero`(); - if (`b` == -1 && `a` == 2147483647) `raiseOverflow`(); - return Math.trunc(`a` / `b`); - """ + asm """ + if (`b` == 0) `raiseDivByZero`(); + if (`b` == -1 && `a` == 2147483647) `raiseOverflow`(); + return Math.trunc(`a` / `b`); + """ proc modInt(a, b: int): int {.asmNoStackFrame, compilerproc.} = - when defined(nimphp): - asm """ - return `a` % `b`; - """ - else: - asm """ - if (`b` == 0) `raiseDivByZero`(); - if (`b` == -1 && `a` == 2147483647) `raiseOverflow`(); - return Math.trunc(`a` % `b`); - """ + asm """ + if (`b` == 0) `raiseDivByZero`(); + if (`b` == -1 && `a` == 2147483647) `raiseOverflow`(); + return Math.trunc(`a` % `b`); + """ proc addInt64(a, b: int): int {.asmNoStackFrame, compilerproc.} = - when defined(nimphp): - asm """ - return `a` + `b`; - """ - else: - asm """ - var result = `a` + `b`; - if (result > 9223372036854775807 - || result < -9223372036854775808) `raiseOverflow`(); - return result; - """ + asm """ + var result = `a` + `b`; + if (result > 9223372036854775807 + || result < -9223372036854775808) `raiseOverflow`(); + return result; + """ proc subInt64(a, b: int): int {.asmNoStackFrame, compilerproc.} = - when defined(nimphp): - asm """ - return `a` - `b`; - """ - else: - asm """ - var result = `a` - `b`; - if (result > 9223372036854775807 - || result < -9223372036854775808) `raiseOverflow`(); - return result; - """ + asm """ + var result = `a` - `b`; + if (result > 9223372036854775807 + || result < -9223372036854775808) `raiseOverflow`(); + return result; + """ proc mulInt64(a, b: int): int {.asmNoStackFrame, compilerproc.} = - when defined(nimphp): - asm """ - return `a` * `b`; - """ - else: - asm """ - var result = `a` * `b`; - if (result > 9223372036854775807 - || result < -9223372036854775808) `raiseOverflow`(); - return result; - """ + asm """ + var result = `a` * `b`; + if (result > 9223372036854775807 + || result < -9223372036854775808) `raiseOverflow`(); + return result; + """ proc divInt64(a, b: int): int {.asmNoStackFrame, compilerproc.} = - when defined(nimphp): - asm """ - return trunc(`a` / `b`); - """ - else: - asm """ - if (`b` == 0) `raiseDivByZero`(); - if (`b` == -1 && `a` == 9223372036854775807) `raiseOverflow`(); - return Math.trunc(`a` / `b`); - """ + asm """ + if (`b` == 0) `raiseDivByZero`(); + if (`b` == -1 && `a` == 9223372036854775807) `raiseOverflow`(); + return Math.trunc(`a` / `b`); + """ proc modInt64(a, b: int): int {.asmNoStackFrame, compilerproc.} = - when defined(nimphp): - asm """ - return `a` % `b`; - """ - else: - asm """ - if (`b` == 0) `raiseDivByZero`(); - if (`b` == -1 && `a` == 9223372036854775807) `raiseOverflow`(); - return Math.trunc(`a` % `b`); - """ + asm """ + if (`b` == 0) `raiseDivByZero`(); + if (`b` == -1 && `a` == 9223372036854775807) `raiseOverflow`(); + return Math.trunc(`a` % `b`); + """ proc negInt(a: int): int {.compilerproc.} = result = a*(-1) @@ -767,24 +618,14 @@ proc genericReset(x: JSRef, ti: PNimType): JSRef {.compilerproc.} = else: discard -when defined(nimphp): - proc arrayConstr(len: int, value: string, typ: string): JSRef {. - asmNoStackFrame, compilerproc.} = - # types are fake - asm """ - $result = array(); - for ($i = 0; $i < `len`; $i++) $result[] = `value`; - return $result; - """ -else: - proc arrayConstr(len: int, value: JSRef, typ: PNimType): JSRef {. - asmNoStackFrame, compilerproc.} = +proc arrayConstr(len: int, value: JSRef, typ: PNimType): JSRef {. + asmNoStackFrame, compilerproc.} = # types are fake - asm """ - var result = new Array(`len`); - for (var i = 0; i < `len`; ++i) result[i] = nimCopy(null, `value`, `typ`); - return result; - """ + asm """ + var result = new Array(`len`); + for (var i = 0; i < `len`; ++i) result[i] = nimCopy(null, `value`, `typ`); + return result; + """ proc chckIndx(i, a, b: int): int {.compilerproc.} = if i >= a and i <= b: return i @@ -909,3 +750,16 @@ when defined(nodejs): else: # Deprecated. Use `alert` defined in dom.nim proc alert*(s: cstring) {.importc, nodecl, deprecated.} + +# Workaround for IE, IE up to version 11 lacks 'Math.trunc'. We produce +# 'Math.trunc' for Nim's ``div`` and ``mod`` operators: +when not defined(nodejs): + {.emit: """ + if (!Math.trunc) { + Math.trunc = function(v) { + v = +v; + if (!isFinite(v)) return v; + + return (v - v % 1) || (v < 0 ? -0 : v === 0 ? v : 0); + }; + }""".} diff --git a/lib/system/mmdisp.nim b/lib/system/mmdisp.nim index 45e0c74c0..2c9b1e502 100644 --- a/lib/system/mmdisp.nim +++ b/lib/system/mmdisp.nim @@ -146,7 +146,7 @@ when defined(boehmgc): proc getFreeMem(): int = return boehmGetFreeBytes() proc getTotalMem(): int = return boehmGetHeapSize() - proc setStackBottom(theStackBottom: pointer) = discard + proc nimGC_setStackBottom(theStackBottom: pointer) = discard proc initGC() = boehmGCinit() @@ -305,7 +305,7 @@ elif defined(gogc): goRuntime_ReadMemStats(addr mstats) result = int(mstats.sys) - proc setStackBottom(theStackBottom: pointer) = discard + proc nimGC_setStackBottom(theStackBottom: pointer) = discard proc alloc(size: Natural): pointer = result = c_malloc(size) @@ -449,7 +449,7 @@ elif defined(nogc) and defined(useMalloc): proc getFreeMem(): int = discard proc getTotalMem(): int = discard - proc setStackBottom(theStackBottom: pointer) = discard + proc nimGC_setStackBottom(theStackBottom: pointer) = discard proc initGC() = discard @@ -523,7 +523,7 @@ elif defined(nogc): proc growObj(old: pointer, newsize: int): pointer = result = realloc(old, newsize) - proc setStackBottom(theStackBottom: pointer) = discard + proc nimGC_setStackBottom(theStackBottom: pointer) = discard proc nimGCref(p: pointer) {.compilerproc, inline.} = discard proc nimGCunref(p: pointer) {.compilerproc, inline.} = discard @@ -561,13 +561,16 @@ else: when not declared(nimNewSeqOfCap): proc nimNewSeqOfCap(typ: PNimType, cap: int): pointer {.compilerproc.} = - let s = addInt(mulInt(cap, typ.base.size), GenericSeqSize) - when declared(newObjNoInit): - result = if ntfNoRefs in typ.base.flags: newObjNoInit(typ, s) else: newObj(typ, s) + when defined(gcRegions): + result = newStr(typ, cap, ntfNoRefs notin typ.base.flags) else: - result = newObj(typ, s) - cast[PGenericSeq](result).len = 0 - cast[PGenericSeq](result).reserved = cap + let s = addInt(mulInt(cap, typ.base.size), GenericSeqSize) + when declared(newObjNoInit): + result = if ntfNoRefs in typ.base.flags: newObjNoInit(typ, s) else: newObj(typ, s) + else: + result = newObj(typ, s) + cast[PGenericSeq](result).len = 0 + cast[PGenericSeq](result).reserved = cap {.pop.} diff --git a/lib/system/nimscript.nim b/lib/system/nimscript.nim index f91dae41e..7671e5962 100644 --- a/lib/system/nimscript.nim +++ b/lib/system/nimscript.nim @@ -38,13 +38,19 @@ proc removeFile(dir: string) {. tags: [ReadIOEffect, WriteIOEffect], raises: [OSError].} = builtin proc moveFile(src, dest: string) {. tags: [ReadIOEffect, WriteIOEffect], raises: [OSError].} = builtin +proc moveDir(src, dest: string) {. + tags: [ReadIOEffect, WriteIOEffect], raises: [OSError].} = builtin proc copyFile(src, dest: string) {. tags: [ReadIOEffect, WriteIOEffect], raises: [OSError].} = builtin +proc copyDir(src, dest: string) {. + tags: [ReadIOEffect, WriteIOEffect], raises: [OSError].} = builtin proc createDir(dir: string) {.tags: [WriteIOEffect], raises: [OSError].} = builtin proc getOsError: string = builtin proc setCurrentDir(dir: string) = builtin -proc getCurrentDir(): string = builtin +proc getCurrentDir*(): string = + ## Retrieves the current working directory. + builtin proc rawExec(cmd: string): int {.tags: [ExecIOEffect], raises: [OSError].} = builtin @@ -114,6 +120,10 @@ proc existsEnv*(key: string): bool {.tags: [ReadIOEffect].} = ## Checks for the existence of an environment variable named `key`. builtin +proc putEnv*(key, val: string) {.tags: [WriteIOEffect].} = + ## Sets the value of the environment variable named key to val. + builtin + proc fileExists*(filename: string): bool {.tags: [ReadIOEffect].} = ## Checks if the file exists. builtin @@ -202,12 +212,24 @@ proc mvFile*(`from`, to: string) {.raises: [OSError].} = moveFile `from`, to checkOsError() +proc mvDir*(`from`, to: string) {.raises: [OSError].} = + ## Moves the dir `from` to `to`. + log "mvDir: " & `from` & ", " & to: + moveDir `from`, to + checkOsError() + proc cpFile*(`from`, to: string) {.raises: [OSError].} = ## Copies the file `from` to `to`. log "cpFile: " & `from` & ", " & to: copyFile `from`, to checkOsError() +proc cpDir*(`from`, to: string) {.raises: [OSError].} = + ## Copies the dir `from` to `to`. + log "cpDir: " & `from` & ", " & to: + copyDir `from`, to + checkOsError() + proc exec*(command: string) = ## Executes an external process. log "exec: " & command: @@ -261,6 +283,12 @@ proc cd*(dir: string) {.raises: [OSError].} = setCurrentDir(dir) checkOsError() +proc findExe*(bin: string): string = + ## Searches for bin in the current working directory and then in directories + ## listed in the PATH environment variable. Returns "" if the exe cannot be + ## found. + builtin + template withDir*(dir: string; body: untyped): untyped = ## Changes the current directory temporarily. ## diff --git a/lib/system/osalloc.nim b/lib/system/osalloc.nim index 1ad4cf695..9609b6d39 100644 --- a/lib/system/osalloc.nim +++ b/lib/system/osalloc.nim @@ -58,7 +58,7 @@ when defined(emscripten): # Convert pointer to PageSize correct one. var new_pos = cast[ByteAddress](pos) +% (PageSize - (pos %% PageSize)) - if (new_pos-pos)< sizeof(EmscriptenMMapBlock): + if (new_pos-pos) < sizeof(EmscriptenMMapBlock): new_pos = new_pos +% PageSize result = cast[pointer](new_pos) diff --git a/lib/system/platforms.nim b/lib/system/platforms.nim index 8939615cd..97f97e8ae 100644 --- a/lib/system/platforms.nim +++ b/lib/system/platforms.nim @@ -29,8 +29,9 @@ type arm, ## ARM based processor arm64, ## ARM64 based processor vm, ## Some Virtual machine: Nim's VM or JavaScript - avr ## AVR based processor - msp430 ## TI MSP430 microcontroller + avr, ## AVR based processor + msp430, ## TI MSP430 microcontroller + riscv64 ## RISC-V 64-bit processor OsPlatform* {.pure.} = enum ## the OS this program will run on. none, dos, windows, os2, linux, morphos, skyos, solaris, @@ -84,5 +85,6 @@ const elif defined(vm): CpuPlatform.vm elif defined(avr): CpuPlatform.avr elif defined(msp430): CpuPlatform.msp430 + elif defined(riscv64): CpuPlatform.riscv64 else: CpuPlatform.none ## the CPU this program will run on. diff --git a/lib/system/reprjs.nim b/lib/system/reprjs.nim index 658220c11..d04d6e12b 100644 --- a/lib/system/reprjs.nim +++ b/lib/system/reprjs.nim @@ -35,7 +35,7 @@ proc isUndefined[T](x: T): bool {.inline.} = {.emit: "`result` = `x` === undefin proc reprEnum(e: int, typ: PNimType): string {.compilerRtl.} = if not typ.node.sons[e].isUndefined: - result = $typ.node.sons[e].name + result = makeNimstrLit(typ.node.sons[e].name) else: result = $e & " (invalid data!)" @@ -55,11 +55,11 @@ proc reprStrAux(result: var string, s: cstring, len: int) = case c of '"': add(result, "\\\"") of '\\': add(result, "\\\\") - of '\10': add(result, "\\10\"\n\"") - of '\127'..'\255', '\0'..'\9', '\11'..'\31': - add( result, "\\" & reprInt(ord(c)) ) + #of '\10': add(result, "\\10\"\n\"") + of '\127'..'\255', '\0'..'\31': + add(result, "\\" & reprInt(ord(c))) else: - add( result, reprInt(ord(c)) ) # Not sure about this. + add(result, c) add(result, "\"") proc reprStr(s: string): string {.compilerRtl.} = diff --git a/lib/system/sysio.nim b/lib/system/sysio.nim index 4348ffbb5..86b290230 100644 --- a/lib/system/sysio.nim +++ b/lib/system/sysio.nim @@ -47,10 +47,22 @@ when not declared(c_fwrite): # C routine that is used here: proc c_fread(buf: pointer, size, n: csize, f: File): csize {. importc: "fread", header: "<stdio.h>", tags: [ReadIOEffect].} -proc c_fseek(f: File, offset: clong, whence: cint): cint {. - importc: "fseek", header: "<stdio.h>", tags: [].} -proc c_ftell(f: File): clong {. - importc: "ftell", header: "<stdio.h>", tags: [].} +when defined(windows): + when not defined(amd64): + proc c_fseek(f: File, offset: int64, whence: cint): cint {. + importc: "fseek", header: "<stdio.h>", tags: [].} + proc c_ftell(f: File): int64 {. + importc: "ftell", header: "<stdio.h>", tags: [].} + else: + proc c_fseek(f: File, offset: int64, whence: cint): cint {. + importc: "_fseeki64", header: "<stdio.h>", tags: [].} + proc c_ftell(f: File): int64 {. + importc: "_ftelli64", header: "<stdio.h>", tags: [].} +else: + proc c_fseek(f: File, offset: int64, whence: cint): cint {. + importc: "fseeko", header: "<stdio.h>", tags: [].} + proc c_ftell(f: File): int64 {. + importc: "ftello", header: "<stdio.h>", tags: [].} proc c_ferror(f: File): cint {. importc: "ferror", header: "<stdio.h>", tags: [].} proc c_setvbuf(f: File, buf: pointer, mode: cint, size: csize): cint {. @@ -188,9 +200,9 @@ proc write(f: File, b: bool) = if b: write(f, "true") else: write(f, "false") proc write(f: File, r: float32) = - if c_fprintf(f, "%g", r) < 0: checkErr(f) + if c_fprintf(f, "%.16g", r) < 0: checkErr(f) proc write(f: File, r: BiggestFloat) = - if c_fprintf(f, "%g", r) < 0: checkErr(f) + if c_fprintf(f, "%.16g", r) < 0: checkErr(f) proc write(f: File, c: char) = discard c_putc(cint(c), f) proc write(f: File, a: varargs[string, `$`]) = @@ -210,12 +222,12 @@ proc readAllBuffer(file: File): string = result.add(buffer) break -proc rawFileSize(file: File): int = +proc rawFileSize(file: File): int64 = # this does not raise an error opposed to `getFileSize` var oldPos = c_ftell(file) discard c_fseek(file, 0, 2) # seek the end of the file result = c_ftell(file) - discard c_fseek(file, clong(oldPos), 0) + discard c_fseek(file, oldPos, 0) proc endOfFile(f: File): bool = var c = c_fgetc(f) @@ -223,7 +235,7 @@ proc endOfFile(f: File): bool = return c < 0'i32 #result = c_feof(f) != 0 -proc readAllFile(file: File, len: int): string = +proc readAllFile(file: File, len: int64): string = # We acquire the filesize beforehand and hope it doesn't change. # Speeds things up. result = newString(len) @@ -363,7 +375,7 @@ proc open(f: var File, filehandle: FileHandle, mode: FileMode): bool = result = f != nil proc setFilePos(f: File, pos: int64, relativeTo: FileSeekPos = fspSet) = - if c_fseek(f, clong(pos), cint(relativeTo)) != 0: + if c_fseek(f, pos, cint(relativeTo)) != 0: raiseEIO("cannot set file position") proc getFilePos(f: File): int64 = @@ -406,7 +418,8 @@ proc setStdIoUnbuffered() = when declared(stdout): proc echoBinSafe(args: openArray[string]) {.compilerProc.} = - when not defined(windows): + # flockfile deadlocks some versions of Android 5.x.x + when not defined(windows) and not defined(android): proc flockfile(f: File) {.importc, noDecl.} proc funlockfile(f: File) {.importc, noDecl.} flockfile(stdout) @@ -415,7 +428,7 @@ when declared(stdout): const linefeed = "\n" # can be 1 or more chars discard c_fwrite(linefeed.cstring, linefeed.len, 1, stdout) discard c_fflush(stdout) - when not defined(windows): + when not defined(windows) and not defined(android): funlockfile(stdout) {.pop.} diff --git a/lib/system/sysstr.nim b/lib/system/sysstr.nim index 4c5f3d9a1..7b81f54da 100644 --- a/lib/system/sysstr.nim +++ b/lib/system/sysstr.nim @@ -22,18 +22,34 @@ proc resize(old: int): int {.inline.} = proc cmpStrings(a, b: NimString): int {.inline, compilerProc.} = if a == b: return 0 - if a == nil: return -1 - if b == nil: return 1 - let minlen = min(a.len, b.len) - result = c_memcmp(addr a.data, addr b.data, minlen.csize) - if result == 0: - result = a.len - b.len + when defined(nimNoNil): + let alen = if a == nil: 0 else: a.len + let blen = if b == nil: 0 else: b.len + else: + if a == nil: return -1 + if b == nil: return 1 + let alen = a.len + let blen = b.len + let minlen = min(alen, blen) + if minlen > 0: + result = c_memcmp(addr a.data, addr b.data, minlen.csize) + if result == 0: + result = alen - blen + else: + result = alen - blen proc eqStrings(a, b: NimString): bool {.inline, compilerProc.} = if a == b: return true - if a == nil or b == nil: return false - return a.len == b.len and - equalMem(addr(a.data), addr(b.data), a.len) + when defined(nimNoNil): + let alen = if a == nil: 0 else: a.len + let blen = if b == nil: 0 else: b.len + else: + if a == nil or b == nil: return false + let alen = a.len + let blen = b.len + if alen == blen: + if alen == 0: return true + return equalMem(addr(a.data), addr(b.data), alen) when declared(allocAtomic): template allocStr(size: untyped): untyped = @@ -60,6 +76,7 @@ proc rawNewStringNoInit(space: int): NimString {.compilerProc.} = if s < 7: s = 7 result = allocStrNoInit(sizeof(TGenericSeq) + s + 1) result.reserved = s + result.len = 0 when defined(gogc): result.elemSize = 1 @@ -68,6 +85,7 @@ proc rawNewString(space: int): NimString {.compilerProc.} = if s < 7: s = 7 result = allocStr(sizeof(TGenericSeq) + s + 1) result.reserved = s + result.len = 0 when defined(gogc): result.elemSize = 1 @@ -98,9 +116,6 @@ proc cstrToNimstr(str: cstring): NimString {.compilerRtl.} = if str == nil: NimString(nil) else: toNimStr(str, str.len) -template wasMoved(x: NimString): bool = false -# (x.reserved and seqShallowFlag) != 0 - proc copyString(src: NimString): NimString {.compilerRtl.} = if src != nil: if (src.reserved and seqShallowFlag) != 0: @@ -158,14 +173,16 @@ proc hashString(s: string): int {.compilerproc.} = proc addChar(s: NimString, c: char): NimString = # is compilerproc! - result = s - if result.len >= result.space: - let r = resize(result.space) - result = cast[NimString](growObj(result, - sizeof(TGenericSeq) + r + 1)) - result.reserved = r - elif wasMoved(s): - result = newOwnedString(s, s.len) + if s == nil: + result = rawNewStringNoInit(1) + result.len = 0 + else: + result = s + if result.len >= result.space: + let r = resize(result.space) + result = cast[NimString](growObj(result, + sizeof(TGenericSeq) + r + 1)) + result.reserved = r result.data[result.len] = c result.data[result.len+1] = '\0' inc(result.len) @@ -202,7 +219,9 @@ proc addChar(s: NimString, c: char): NimString = # s = rawNewString(0); proc resizeString(dest: NimString, addlen: int): NimString {.compilerRtl.} = - if dest.len + addlen <= dest.space and not wasMoved(dest): + if dest == nil: + result = rawNewStringNoInit(addlen) + elif dest.len + addlen <= dest.space: result = dest else: # slow path: var sp = max(resize(dest.space), dest.len + addlen) @@ -213,8 +232,9 @@ proc resizeString(dest: NimString, addlen: int): NimString {.compilerRtl.} = # DO NOT UPDATE LEN YET: dest.len = newLen proc appendString(dest, src: NimString) {.compilerproc, inline.} = - copyMem(addr(dest.data[dest.len]), addr(src.data), src.len + 1) - inc(dest.len, src.len) + if src != nil: + copyMem(addr(dest.data[dest.len]), addr(src.data), src.len + 1) + inc(dest.len, src.len) proc appendChar(dest: NimString, c: char) {.compilerproc, inline.} = dest.data[dest.len] = c @@ -223,8 +243,8 @@ proc appendChar(dest: NimString, c: char) {.compilerproc, inline.} = proc setLengthStr(s: NimString, newLen: int): NimString {.compilerRtl.} = var n = max(newLen, 0) - if wasMoved(s): - result = newOwnedString(s, n) + if s == nil: + result = mnewString(newLen) elif n <= s.space: result = s else: @@ -258,6 +278,18 @@ proc incrSeqV2(seq: PGenericSeq, elemSize: int): PGenericSeq {.compilerProc.} = GenericSeqSize)) result.reserved = r +proc incrSeqV3(s: PGenericSeq, typ: PNimType): PGenericSeq {.compilerProc.} = + if s == nil: + result = cast[PGenericSeq](newSeq(typ, 1)) + result.len = 0 + else: + result = s + if result.len >= result.space: + let r = resize(result.space) + result = cast[PGenericSeq](growObj(result, typ.base.size * r + + GenericSeqSize)) + result.reserved = r + proc setLengthSeq(seq: PGenericSeq, elemSize, newLen: int): PGenericSeq {. compilerRtl, inl.} = result = seq @@ -290,7 +322,7 @@ proc setLengthSeq(seq: PGenericSeq, elemSize, newLen: int): PGenericSeq {. # XXX: zeroing out the memory can still result in crashes if a wiped-out # cell is aliased by another pointer (ie proc parameter or a let variable). - # This is a tought problem, because even if we don't zeroMem here, in the + # This is a tough problem, because even if we don't zeroMem here, in the # presence of user defined destructors, the user will expect the cell to be # "destroyed" thus creating the same problem. We can destoy the cell in the # finalizer of the sequence, but this makes destruction non-deterministic. @@ -298,6 +330,13 @@ proc setLengthSeq(seq: PGenericSeq, elemSize, newLen: int): PGenericSeq {. (newLen*%elemSize)), (result.len-%newLen) *% elemSize) result.len = newLen +proc setLengthSeqV2(s: PGenericSeq, typ: PNimType, newLen: int): PGenericSeq {. + compilerRtl.} = + if s == nil: + result = cast[PGenericSeq](newSeq(typ, newLen)) + else: + result = setLengthSeq(s, typ.base.size, newLen) + # --------------- other string routines ---------------------------------- proc add*(result: var string; x: int64) = let base = result.len @@ -386,8 +425,7 @@ proc nimParseBiggestFloat(s: string, number: var BiggestFloat, kdigits, fdigits = 0 exponent: int integer: uint64 - fraction: uint64 - frac_exponent= 0 + frac_exponent = 0 exp_sign = 1 first_digit = -1 has_sign = false @@ -418,7 +456,7 @@ proc nimParseBiggestFloat(s: string, number: var BiggestFloat, return 0 if s[i] in {'0'..'9'}: - first_digit = (s[i].ord - '0'.ord) + first_digit = (s[i].ord - '0'.ord) # Integer part? while s[i] in {'0'..'9'}: inc(kdigits) @@ -480,7 +518,8 @@ proc nimParseBiggestFloat(s: string, number: var BiggestFloat, # if integer is representable in 53 bits: fast path # max fast path integer is 1<<53 - 1 or 8999999999999999 (16 digits) - if kdigits + fdigits <= 16 and first_digit <= 8: + let digits = kdigits + fdigits + if digits <= 15 or (digits <= 16 and first_digit <= 8): # max float power of ten with set bits above the 53th bit is 10^22 if abs_exponent <= 22: if exp_negative: @@ -504,6 +543,7 @@ proc nimParseBiggestFloat(s: string, number: var BiggestFloat, result = i - start i = start # re-parse without error checking, any error should be handled by the code above. + if s[i] == '.': i.inc while s[i] in {'0'..'9','+','-'}: if ti < maxlen: t[ti] = s[i]; inc(ti) @@ -536,18 +576,3 @@ proc nimBoolToStr(x: bool): string {.compilerRtl.} = proc nimCharToStr(x: char): string {.compilerRtl.} = result = newString(1) result[0] = x - -proc binaryStrSearch(x: openArray[string], y: string): int {.compilerproc.} = - var - a = 0 - b = len(x) - while a < b: - var mid = (a + b) div 2 - if x[mid] < y: - a = mid + 1 - else: - b = mid - if a < len(x) and x[a] == y: - result = a - else: - result = -1 diff --git a/lib/system/threads.nim b/lib/system/threads.nim index f61cc4280..861bde13f 100644 --- a/lib/system/threads.nim +++ b/lib/system/threads.nim @@ -440,7 +440,7 @@ proc threadProcWrapStackFrame[TArg](thrd: ptr Thread[TArg]) = threadProcWrapDispatch[TArg] when not hasSharedHeap: # init the GC for refc/markandsweep - setStackBottom(addr(p)) + nimGC_setStackBottom(addr(p)) initGC() when declared(threadType): threadType = ThreadType.NimThread @@ -642,7 +642,10 @@ when defined(windows): elif defined(linux): proc syscall(arg: clong): clong {.varargs, importc: "syscall", header: "<unistd.h>".} - var NR_gettid {.importc: "__NR_gettid", header: "<sys/syscall.h>".}: int + when defined(amd64): + const NR_gettid = clong(186) + else: + var NR_gettid {.importc: "__NR_gettid", header: "<sys/syscall.h>".}: clong proc getThreadId*(): int = ## get the ID of the currently running thread. diff --git a/lib/upcoming/asyncdispatch.nim b/lib/upcoming/asyncdispatch.nim deleted file mode 100644 index 4e3b06173..000000000 --- a/lib/upcoming/asyncdispatch.nim +++ /dev/null @@ -1,1630 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Dominik Picheta -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -include "system/inclrtl" - -import os, tables, strutils, times, heapqueue, lists, options, asyncstreams -import asyncfutures except callSoon - -import nativesockets, net, deques - -export Port, SocketFlag -export asyncfutures, asyncstreams - -#{.injectStmt: newGcInvariant().} - -## AsyncDispatch -## ************* -## -## This module implements asynchronous IO. This includes a dispatcher, -## a ``Future`` type implementation, and an ``async`` macro which allows -## asynchronous code to be written in a synchronous style with the ``await`` -## keyword. -## -## The dispatcher acts as a kind of event loop. You must call ``poll`` on it -## (or a function which does so for you such as ``waitFor`` or ``runForever``) -## in order to poll for any outstanding events. The underlying implementation -## is based on epoll on Linux, IO Completion Ports on Windows and select on -## other operating systems. -## -## The ``poll`` function will not, on its own, return any events. Instead -## an appropriate ``Future`` object will be completed. A ``Future`` is a -## type which holds a value which is not yet available, but which *may* be -## available in the future. You can check whether a future is finished -## by using the ``finished`` function. When a future is finished it means that -## either the value that it holds is now available or it holds an error instead. -## The latter situation occurs when the operation to complete a future fails -## with an exception. You can distinguish between the two situations with the -## ``failed`` function. -## -## Future objects can also store a callback procedure which will be called -## automatically once the future completes. -## -## Futures therefore can be thought of as an implementation of the proactor -## pattern. In this -## pattern you make a request for an action, and once that action is fulfilled -## a future is completed with the result of that action. Requests can be -## made by calling the appropriate functions. For example: calling the ``recv`` -## function will create a request for some data to be read from a socket. The -## future which the ``recv`` function returns will then complete once the -## requested amount of data is read **or** an exception occurs. -## -## Code to read some data from a socket may look something like this: -## -## .. code-block::nim -## var future = socket.recv(100) -## future.callback = -## proc () = -## echo(future.read) -## -## All asynchronous functions returning a ``Future`` will not block. They -## will not however return immediately. An asynchronous function will have -## code which will be executed before an asynchronous request is made, in most -## cases this code sets up the request. -## -## In the above example, the ``recv`` function will return a brand new -## ``Future`` instance once the request for data to be read from the socket -## is made. This ``Future`` instance will complete once the requested amount -## of data is read, in this case it is 100 bytes. The second line sets a -## callback on this future which will be called once the future completes. -## All the callback does is write the data stored in the future to ``stdout``. -## The ``read`` function is used for this and it checks whether the future -## completes with an error for you (if it did it will simply raise the -## error), if there is no error however it returns the value of the future. -## -## Asynchronous procedures -## ----------------------- -## -## Asynchronous procedures remove the pain of working with callbacks. They do -## this by allowing you to write asynchronous code the same way as you would -## write synchronous code. -## -## An asynchronous procedure is marked using the ``{.async.}`` pragma. -## When marking a procedure with the ``{.async.}`` pragma it must have a -## ``Future[T]`` return type or no return type at all. If you do not specify -## a return type then ``Future[void]`` is assumed. -## -## Inside asynchronous procedures ``await`` can be used to call any -## procedures which return a -## ``Future``; this includes asynchronous procedures. When a procedure is -## "awaited", the asynchronous procedure it is awaited in will -## suspend its execution -## until the awaited procedure's Future completes. At which point the -## asynchronous procedure will resume its execution. During the period -## when an asynchronous procedure is suspended other asynchronous procedures -## will be run by the dispatcher. -## -## The ``await`` call may be used in many contexts. It can be used on the right -## hand side of a variable declaration: ``var data = await socket.recv(100)``, -## in which case the variable will be set to the value of the future -## automatically. It can be used to await a ``Future`` object, and it can -## be used to await a procedure returning a ``Future[void]``: -## ``await socket.send("foobar")``. -## -## Discarding futures -## ------------------ -## -## Futures should **never** be discarded. This is because they may contain -## errors. If you do not care for the result of a Future then you should -## use the ``asyncCheck`` procedure instead of the ``discard`` keyword. -## -## Examples -## -------- -## -## For examples take a look at the documentation for the modules implementing -## asynchronous IO. A good place to start is the -## `asyncnet module <asyncnet.html>`_. -## -## Limitations/Bugs -## ---------------- -## -## * The effect system (``raises: []``) does not work with async procedures. -## * Can't await in a ``except`` body -## * Forward declarations for async procs are broken, -## link includes workaround: https://github.com/nim-lang/Nim/issues/3182. -## * FutureVar[T] needs to be completed manually. - -# TODO: Check if yielded future is nil and throw a more meaningful exception - -type - PDispatcherBase = ref object of RootRef - timers: HeapQueue[tuple[finishAt: float, fut: Future[void]]] - callbacks: Deque[proc ()] - -proc processTimers(p: PDispatcherBase) {.inline.} = - #Process just part if timers at a step - var count = p.timers.len - let t = epochTime() - while count > 0 and t >= p.timers[0].finishAt: - p.timers.pop().fut.complete() - dec count - -proc processPendingCallbacks(p: PDispatcherBase) = - while p.callbacks.len > 0: - var cb = p.callbacks.popFirst() - cb() - -proc adjustedTimeout(p: PDispatcherBase, timeout: int): int {.inline.} = - # If dispatcher has active timers this proc returns the timeout - # of the nearest timer. Returns `timeout` otherwise. - result = timeout - if p.timers.len > 0: - let timerTimeout = p.timers[0].finishAt - let curTime = epochTime() - if timeout == -1 or (curTime + (timeout / 1000)) > timerTimeout: - result = int((timerTimeout - curTime) * 1000) - if result < 0: result = 0 - -proc callSoon(cbproc: proc ()) {.gcsafe.} - -proc initCallSoonProc = - if asyncfutures.getCallSoonProc().isNil: - asyncfutures.setCallSoonProc(callSoon) - -when defined(windows) or defined(nimdoc): - import winlean, sets, hashes - type - CompletionKey = ULONG_PTR - - CompletionData* = object - fd*: AsyncFD # TODO: Rename this. - cb*: proc (fd: AsyncFD, bytesTransferred: Dword, - errcode: OSErrorCode) {.closure,gcsafe.} - cell*: ForeignCell # we need this `cell` to protect our `cb` environment, - # when using RegisterWaitForSingleObject, because - # waiting is done in different thread. - - PDispatcher* = ref object of PDispatcherBase - ioPort: Handle - handles: HashSet[AsyncFD] - - CustomOverlapped = object of OVERLAPPED - data*: CompletionData - - PCustomOverlapped* = ref CustomOverlapped - - AsyncFD* = distinct int - - PostCallbackData = object - ioPort: Handle - handleFd: AsyncFD - waitFd: Handle - ovl: PCustomOverlapped - PostCallbackDataPtr = ptr PostCallbackData - - AsyncEventImpl = object - hEvent: Handle - hWaiter: Handle - pcd: PostCallbackDataPtr - AsyncEvent* = ptr AsyncEventImpl - - Callback = proc (fd: AsyncFD): bool {.closure,gcsafe.} - {.deprecated: [TCompletionKey: CompletionKey, TAsyncFD: AsyncFD, - TCustomOverlapped: CustomOverlapped, TCompletionData: CompletionData].} - - proc hash(x: AsyncFD): Hash {.borrow.} - proc `==`*(x: AsyncFD, y: AsyncFD): bool {.borrow.} - - proc newDispatcher*(): PDispatcher = - ## Creates a new Dispatcher instance. - new result - result.ioPort = createIoCompletionPort(INVALID_HANDLE_VALUE, 0, 0, 1) - result.handles = initSet[AsyncFD]() - result.timers.newHeapQueue() - result.callbacks = initDeque[proc ()](64) - - var gDisp{.threadvar.}: PDispatcher ## Global dispatcher - - proc setGlobalDispatcher*(disp: PDispatcher) = - if not gDisp.isNil: - assert gDisp.callbacks.len == 0 - gDisp = disp - initCallSoonProc() - - proc getGlobalDispatcher*(): PDispatcher = - if gDisp.isNil: - setGlobalDispatcher(newDispatcher()) - result = gDisp - - proc register*(fd: AsyncFD) = - ## Registers ``fd`` with the dispatcher. - let p = getGlobalDispatcher() - if createIoCompletionPort(fd.Handle, p.ioPort, - cast[CompletionKey](fd), 1) == 0: - raiseOSError(osLastError()) - p.handles.incl(fd) - - proc verifyPresence(fd: AsyncFD) = - ## Ensures that file descriptor has been registered with the dispatcher. - let p = getGlobalDispatcher() - if fd notin p.handles: - raise newException(ValueError, - "Operation performed on a socket which has not been registered with" & - " the dispatcher yet.") - - proc hasPendingOperations*(): bool = - ## Returns `true` if the global dispatcher has pending operations. - let p = getGlobalDispatcher() - p.handles.len != 0 or p.timers.len != 0 or p.callbacks.len != 0 - - proc poll*(timeout = 500) = - ## Waits for completion events and processes them. Raises ``ValueError`` - ## if there are no pending operations. - let p = getGlobalDispatcher() - if p.handles.len == 0 and p.timers.len == 0 and p.callbacks.len == 0: - raise newException(ValueError, - "No handles or timers registered in dispatcher.") - - let at = p.adjustedTimeout(timeout) - var llTimeout = - if at == -1: winlean.INFINITE - else: at.int32 - - if p.handles.len != 0: - var lpNumberOfBytesTransferred: Dword - var lpCompletionKey: ULONG_PTR - var customOverlapped: PCustomOverlapped - let res = getQueuedCompletionStatus(p.ioPort, - addr lpNumberOfBytesTransferred, addr lpCompletionKey, - cast[ptr POVERLAPPED](addr customOverlapped), llTimeout).bool - - # http://stackoverflow.com/a/12277264/492186 - # TODO: http://www.serverframework.com/handling-multiple-pending-socket-read-and-write-operations.html - if res: - # This is useful for ensuring the reliability of the overlapped struct. - assert customOverlapped.data.fd == lpCompletionKey.AsyncFD - - customOverlapped.data.cb(customOverlapped.data.fd, - lpNumberOfBytesTransferred, OSErrorCode(-1)) - - # If cell.data != nil, then system.protect(rawEnv(cb)) was called, - # so we need to dispose our `cb` environment, because it is not needed - # anymore. - if customOverlapped.data.cell.data != nil: - system.dispose(customOverlapped.data.cell) - - GC_unref(customOverlapped) - else: - let errCode = osLastError() - if customOverlapped != nil: - assert customOverlapped.data.fd == lpCompletionKey.AsyncFD - customOverlapped.data.cb(customOverlapped.data.fd, - lpNumberOfBytesTransferred, errCode) - if customOverlapped.data.cell.data != nil: - system.dispose(customOverlapped.data.cell) - GC_unref(customOverlapped) - else: - if errCode.int32 == WAIT_TIMEOUT: - # Timed out - discard - else: raiseOSError(errCode) - - # Timer processing. - processTimers(p) - # Callback queue processing - processPendingCallbacks(p) - - var acceptEx*: WSAPROC_ACCEPTEX - var connectEx*: WSAPROC_CONNECTEX - var getAcceptExSockAddrs*: WSAPROC_GETACCEPTEXSOCKADDRS - - proc initPointer(s: SocketHandle, fun: var pointer, guid: var GUID): bool = - # Ref: https://github.com/powdahound/twisted/blob/master/twisted/internet/iocpreactor/iocpsupport/winsock_pointers.c - var bytesRet: Dword - fun = nil - result = WSAIoctl(s, SIO_GET_EXTENSION_FUNCTION_POINTER, addr guid, - sizeof(GUID).Dword, addr fun, sizeof(pointer).Dword, - addr bytesRet, nil, nil) == 0 - - proc initAll() = - let dummySock = newNativeSocket() - if dummySock == INVALID_SOCKET: - raiseOSError(osLastError()) - var fun: pointer = nil - if not initPointer(dummySock, fun, WSAID_CONNECTEX): - raiseOSError(osLastError()) - connectEx = cast[WSAPROC_CONNECTEX](fun) - if not initPointer(dummySock, fun, WSAID_ACCEPTEX): - raiseOSError(osLastError()) - acceptEx = cast[WSAPROC_ACCEPTEX](fun) - if not initPointer(dummySock, fun, WSAID_GETACCEPTEXSOCKADDRS): - raiseOSError(osLastError()) - getAcceptExSockAddrs = cast[WSAPROC_GETACCEPTEXSOCKADDRS](fun) - close(dummySock) - - proc recv*(socket: AsyncFD, size: int, - flags = {SocketFlag.SafeDisconn}): Future[string] = - ## Reads **up to** ``size`` bytes from ``socket``. Returned future will - ## complete once all the data requested is read, a part of the data has been - ## read, or the socket has disconnected in which case the future will - ## complete with a value of ``""``. - ## - ## **Warning**: The ``Peek`` socket flag is not supported on Windows. - - - # Things to note: - # * When WSARecv completes immediately then ``bytesReceived`` is very - # unreliable. - # * Still need to implement message-oriented socket disconnection, - # '\0' in the message currently signifies a socket disconnect. Who - # knows what will happen when someone sends that to our socket. - verifyPresence(socket) - assert SocketFlag.Peek notin flags, "Peek not supported on Windows." - - var retFuture = newFuture[string]("recv") - var dataBuf: TWSABuf - dataBuf.buf = cast[cstring](alloc0(size)) - dataBuf.len = size.ULONG - - var bytesReceived: Dword - var flagsio = flags.toOSFlags().Dword - var ol = PCustomOverlapped() - GC_ref(ol) - ol.data = CompletionData(fd: socket, cb: - proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = - if not retFuture.finished: - if errcode == OSErrorCode(-1): - if bytesCount == 0 and dataBuf.buf[0] == '\0': - retFuture.complete("") - else: - var data = newString(bytesCount) - assert bytesCount <= size - copyMem(addr data[0], addr dataBuf.buf[0], bytesCount) - retFuture.complete($data) - else: - if flags.isDisconnectionError(errcode): - retFuture.complete("") - else: - retFuture.fail(newException(OSError, osErrorMsg(errcode))) - if dataBuf.buf != nil: - dealloc dataBuf.buf - dataBuf.buf = nil - ) - - let ret = WSARecv(socket.SocketHandle, addr dataBuf, 1, addr bytesReceived, - addr flagsio, cast[POVERLAPPED](ol), nil) - if ret == -1: - let err = osLastError() - if err.int32 != ERROR_IO_PENDING: - if dataBuf.buf != nil: - dealloc dataBuf.buf - dataBuf.buf = nil - GC_unref(ol) - if flags.isDisconnectionError(err): - retFuture.complete("") - else: - retFuture.fail(newException(OSError, osErrorMsg(err))) - elif ret == 0: - # Request completed immediately. - if bytesReceived != 0: - var data = newString(bytesReceived) - assert bytesReceived <= size - copyMem(addr data[0], addr dataBuf.buf[0], bytesReceived) - retFuture.complete($data) - else: - if hasOverlappedIoCompleted(cast[POVERLAPPED](ol)): - retFuture.complete("") - return retFuture - - proc recvInto*(socket: AsyncFD, buf: pointer, size: int, - flags = {SocketFlag.SafeDisconn}): Future[int] = - ## Reads **up to** ``size`` bytes from ``socket`` into ``buf``, which must - ## at least be of that size. Returned future will complete once all the - ## data requested is read, a part of the data has been read, or the socket - ## has disconnected in which case the future will complete with a value of - ## ``0``. - ## - ## **Warning**: The ``Peek`` socket flag is not supported on Windows. - - - # Things to note: - # * When WSARecv completes immediately then ``bytesReceived`` is very - # unreliable. - # * Still need to implement message-oriented socket disconnection, - # '\0' in the message currently signifies a socket disconnect. Who - # knows what will happen when someone sends that to our socket. - verifyPresence(socket) - assert SocketFlag.Peek notin flags, "Peek not supported on Windows." - - var retFuture = newFuture[int]("recvInto") - - #buf[] = '\0' - var dataBuf: TWSABuf - dataBuf.buf = cast[cstring](buf) - dataBuf.len = size.ULONG - - var bytesReceived: Dword - var flagsio = flags.toOSFlags().Dword - var ol = PCustomOverlapped() - GC_ref(ol) - ol.data = CompletionData(fd: socket, cb: - proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = - if not retFuture.finished: - if errcode == OSErrorCode(-1): - retFuture.complete(bytesCount) - else: - if flags.isDisconnectionError(errcode): - retFuture.complete(0) - else: - retFuture.fail(newException(OSError, osErrorMsg(errcode))) - if dataBuf.buf != nil: - dataBuf.buf = nil - ) - - let ret = WSARecv(socket.SocketHandle, addr dataBuf, 1, addr bytesReceived, - addr flagsio, cast[POVERLAPPED](ol), nil) - if ret == -1: - let err = osLastError() - if err.int32 != ERROR_IO_PENDING: - if dataBuf.buf != nil: - dataBuf.buf = nil - GC_unref(ol) - if flags.isDisconnectionError(err): - retFuture.complete(0) - else: - retFuture.fail(newException(OSError, osErrorMsg(err))) - elif ret == 0: - # Request completed immediately. - if bytesReceived != 0: - assert bytesReceived <= size - retFuture.complete(bytesReceived) - else: - if hasOverlappedIoCompleted(cast[POVERLAPPED](ol)): - retFuture.complete(bytesReceived) - return retFuture - - proc send*(socket: AsyncFD, buf: pointer, size: int, - flags = {SocketFlag.SafeDisconn}): Future[void] = - ## Sends ``size`` bytes from ``buf`` to ``socket``. The returned future - ## will complete once all data has been sent. - ## **WARNING**: Use it with caution. If ``buf`` refers to GC'ed object, - ## you must use GC_ref/GC_unref calls to avoid early freeing of the buffer. - verifyPresence(socket) - var retFuture = newFuture[void]("send") - - var dataBuf: TWSABuf - dataBuf.buf = cast[cstring](buf) - dataBuf.len = size.ULONG - - var bytesReceived, lowFlags: Dword - var ol = PCustomOverlapped() - GC_ref(ol) - ol.data = CompletionData(fd: socket, cb: - proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = - if not retFuture.finished: - if errcode == OSErrorCode(-1): - retFuture.complete() - else: - if flags.isDisconnectionError(errcode): - retFuture.complete() - else: - retFuture.fail(newException(OSError, osErrorMsg(errcode))) - ) - - let ret = WSASend(socket.SocketHandle, addr dataBuf, 1, addr bytesReceived, - lowFlags, cast[POVERLAPPED](ol), nil) - if ret == -1: - let err = osLastError() - if err.int32 != ERROR_IO_PENDING: - GC_unref(ol) - if flags.isDisconnectionError(err): - retFuture.complete() - else: - retFuture.fail(newException(OSError, osErrorMsg(err))) - else: - retFuture.complete() - # We don't deallocate ``ol`` here because even though this completed - # immediately poll will still be notified about its completion and it will - # free ``ol``. - return retFuture - - proc send*(socket: AsyncFD, data: string, - flags = {SocketFlag.SafeDisconn}): Future[void] = - ## Sends ``data`` to ``socket``. The returned future will complete once all - ## data has been sent. - verifyPresence(socket) - var retFuture = newFuture[void]("send") - - var dataBuf: TWSABuf - dataBuf.buf = data - GC_ref(data) # we need to protect data until send operation is completed - # or failed. - dataBuf.len = data.len.ULONG - - var bytesReceived, lowFlags: Dword - var ol = PCustomOverlapped() - GC_ref(ol) - ol.data = CompletionData(fd: socket, cb: - proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = - GC_unref(data) # if operation completed `data` must be released. - if not retFuture.finished: - if errcode == OSErrorCode(-1): - retFuture.complete() - else: - if flags.isDisconnectionError(errcode): - retFuture.complete() - else: - retFuture.fail(newException(OSError, osErrorMsg(errcode))) - ) - - let ret = WSASend(socket.SocketHandle, addr dataBuf, 1, addr bytesReceived, - lowFlags, cast[POVERLAPPED](ol), nil) - if ret == -1: - let err = osLastError() - if err.int32 != ERROR_IO_PENDING: - GC_unref(ol) - GC_unref(data) # if operation failed `data` must be released, because - # completion routine will not be called. - if flags.isDisconnectionError(err): - retFuture.complete() - else: - retFuture.fail(newException(OSError, osErrorMsg(err))) - else: - retFuture.complete() - # We don't deallocate ``ol`` here because even though this completed - # immediately poll will still be notified about its completion and it will - # free ``ol``. - return retFuture - - proc sendTo*(socket: AsyncFD, data: pointer, size: int, saddr: ptr SockAddr, - saddrLen: Socklen, - flags = {SocketFlag.SafeDisconn}): Future[void] = - ## Sends ``data`` to specified destination ``saddr``, using - ## socket ``socket``. The returned future will complete once all data - ## has been sent. - verifyPresence(socket) - var retFuture = newFuture[void]("sendTo") - var dataBuf: TWSABuf - dataBuf.buf = cast[cstring](data) - dataBuf.len = size.ULONG - var bytesSent = 0.Dword - var lowFlags = 0.Dword - - # we will preserve address in our stack - var staddr: array[128, char] # SOCKADDR_STORAGE size is 128 bytes - var stalen: cint = cint(saddrLen) - zeroMem(addr(staddr[0]), 128) - copyMem(addr(staddr[0]), saddr, saddrLen) - - var ol = PCustomOverlapped() - GC_ref(ol) - ol.data = CompletionData(fd: socket, cb: - proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = - if not retFuture.finished: - if errcode == OSErrorCode(-1): - retFuture.complete() - else: - retFuture.fail(newException(OSError, osErrorMsg(errcode))) - ) - - let ret = WSASendTo(socket.SocketHandle, addr dataBuf, 1, addr bytesSent, - lowFlags, cast[ptr SockAddr](addr(staddr[0])), - stalen, cast[POVERLAPPED](ol), nil) - if ret == -1: - let err = osLastError() - if err.int32 != ERROR_IO_PENDING: - GC_unref(ol) - retFuture.fail(newException(OSError, osErrorMsg(err))) - else: - retFuture.complete() - # We don't deallocate ``ol`` here because even though this completed - # immediately poll will still be notified about its completion and it will - # free ``ol``. - return retFuture - - proc recvFromInto*(socket: AsyncFD, data: pointer, size: int, - saddr: ptr SockAddr, saddrLen: ptr SockLen, - flags = {SocketFlag.SafeDisconn}): Future[int] = - ## Receives a datagram data from ``socket`` into ``buf``, which must - ## be at least of size ``size``, address of datagram's sender will be - ## stored into ``saddr`` and ``saddrLen``. Returned future will complete - ## once one datagram has been received, and will return size of packet - ## received. - verifyPresence(socket) - var retFuture = newFuture[int]("recvFromInto") - - var dataBuf = TWSABuf(buf: cast[cstring](data), len: size.ULONG) - - var bytesReceived = 0.Dword - var lowFlags = 0.Dword - - var ol = PCustomOverlapped() - GC_ref(ol) - ol.data = CompletionData(fd: socket, cb: - proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = - if not retFuture.finished: - if errcode == OSErrorCode(-1): - assert bytesCount <= size - retFuture.complete(bytesCount) - else: - # datagram sockets don't have disconnection, - # so we can just raise an exception - retFuture.fail(newException(OSError, osErrorMsg(errcode))) - ) - - let res = WSARecvFrom(socket.SocketHandle, addr dataBuf, 1, - addr bytesReceived, addr lowFlags, - saddr, cast[ptr cint](saddrLen), - cast[POVERLAPPED](ol), nil) - if res == -1: - let err = osLastError() - if err.int32 != ERROR_IO_PENDING: - GC_unref(ol) - retFuture.fail(newException(OSError, osErrorMsg(err))) - else: - # Request completed immediately. - if bytesReceived != 0: - assert bytesReceived <= size - retFuture.complete(bytesReceived) - else: - if hasOverlappedIoCompleted(cast[POVERLAPPED](ol)): - retFuture.complete(bytesReceived) - return retFuture - - proc acceptAddr*(socket: AsyncFD, flags = {SocketFlag.SafeDisconn}): - Future[tuple[address: string, client: AsyncFD]] = - ## Accepts a new connection. Returns a future containing the client socket - ## corresponding to that connection and the remote address of the client. - ## The future will complete when the connection is successfully accepted. - ## - ## The resulting client socket is automatically registered to the - ## dispatcher. - ## - ## The ``accept`` call may result in an error if the connecting socket - ## disconnects during the duration of the ``accept``. If the ``SafeDisconn`` - ## flag is specified then this error will not be raised and instead - ## accept will be called again. - verifyPresence(socket) - var retFuture = newFuture[tuple[address: string, client: AsyncFD]]("acceptAddr") - - var clientSock = newNativeSocket() - if clientSock == osInvalidSocket: raiseOSError(osLastError()) - - const lpOutputLen = 1024 - var lpOutputBuf = newString(lpOutputLen) - var dwBytesReceived: Dword - let dwReceiveDataLength = 0.Dword # We don't want any data to be read. - let dwLocalAddressLength = Dword(sizeof(Sockaddr_in6) + 16) - let dwRemoteAddressLength = Dword(sizeof(Sockaddr_in6) + 16) - - template failAccept(errcode) = - if flags.isDisconnectionError(errcode): - var newAcceptFut = acceptAddr(socket, flags) - newAcceptFut.callback = - proc () = - if newAcceptFut.failed: - retFuture.fail(newAcceptFut.readError) - else: - retFuture.complete(newAcceptFut.read) - else: - retFuture.fail(newException(OSError, osErrorMsg(errcode))) - - template completeAccept() {.dirty.} = - var listenSock = socket - let setoptRet = setsockopt(clientSock, SOL_SOCKET, - SO_UPDATE_ACCEPT_CONTEXT, addr listenSock, - sizeof(listenSock).SockLen) - if setoptRet != 0: - let errcode = osLastError() - discard clientSock.closeSocket() - failAccept(errcode) - else: - var localSockaddr, remoteSockaddr: ptr SockAddr - var localLen, remoteLen: int32 - getAcceptExSockaddrs(addr lpOutputBuf[0], dwReceiveDataLength, - dwLocalAddressLength, dwRemoteAddressLength, - addr localSockaddr, addr localLen, - addr remoteSockaddr, addr remoteLen) - try: - let address = getAddrString(remoteSockAddr) - register(clientSock.AsyncFD) - retFuture.complete((address: address, client: clientSock.AsyncFD)) - except: - # getAddrString may raise - clientSock.close() - retFuture.fail(getCurrentException()) - - var ol = PCustomOverlapped() - GC_ref(ol) - ol.data = CompletionData(fd: socket, cb: - proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = - if not retFuture.finished: - if errcode == OSErrorCode(-1): - completeAccept() - else: - failAccept(errcode) - ) - - # http://msdn.microsoft.com/en-us/library/windows/desktop/ms737524%28v=vs.85%29.aspx - let ret = acceptEx(socket.SocketHandle, clientSock, addr lpOutputBuf[0], - dwReceiveDataLength, - dwLocalAddressLength, - dwRemoteAddressLength, - addr dwBytesReceived, cast[POVERLAPPED](ol)) - - if not ret: - let err = osLastError() - if err.int32 != ERROR_IO_PENDING: - failAccept(err) - GC_unref(ol) - else: - completeAccept() - # We don't deallocate ``ol`` here because even though this completed - # immediately poll will still be notified about its completion and it will - # free ``ol``. - - return retFuture - - proc closeSocket*(socket: AsyncFD) = - ## Closes a socket and ensures that it is unregistered. - socket.SocketHandle.close() - getGlobalDispatcher().handles.excl(socket) - - proc unregister*(fd: AsyncFD) = - ## Unregisters ``fd``. - getGlobalDispatcher().handles.excl(fd) - - {.push stackTrace:off.} - proc waitableCallback(param: pointer, - timerOrWaitFired: WINBOOL): void {.stdcall.} = - var p = cast[PostCallbackDataPtr](param) - discard postQueuedCompletionStatus(p.ioPort, timerOrWaitFired.Dword, - ULONG_PTR(p.handleFd), - cast[pointer](p.ovl)) - {.pop.} - - proc registerWaitableEvent(fd: AsyncFD, cb: Callback; mask: Dword) = - let p = getGlobalDispatcher() - var flags = (WT_EXECUTEINWAITTHREAD or WT_EXECUTEONLYONCE).Dword - var hEvent = wsaCreateEvent() - if hEvent == 0: - raiseOSError(osLastError()) - var pcd = cast[PostCallbackDataPtr](allocShared0(sizeof(PostCallbackData))) - pcd.ioPort = p.ioPort - pcd.handleFd = fd - var ol = PCustomOverlapped() - GC_ref(ol) - - ol.data = CompletionData(fd: fd, cb: - proc(fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = - # we excluding our `fd` because cb(fd) can register own handler - # for this `fd` - p.handles.excl(fd) - # unregisterWait() is called before callback, because appropriate - # winsockets function can re-enable event. - # https://msdn.microsoft.com/en-us/library/windows/desktop/ms741576(v=vs.85).aspx - if unregisterWait(pcd.waitFd) == 0: - let err = osLastError() - if err.int32 != ERROR_IO_PENDING: - deallocShared(cast[pointer](pcd)) - discard wsaCloseEvent(hEvent) - raiseOSError(err) - if cb(fd): - # callback returned `true`, so we free all allocated resources - deallocShared(cast[pointer](pcd)) - if not wsaCloseEvent(hEvent): - raiseOSError(osLastError()) - # pcd.ovl will be unrefed in poll(). - else: - # callback returned `false` we need to continue - if p.handles.contains(fd): - # new callback was already registered with `fd`, so we free all - # allocated resources. This happens because in callback `cb` - # addRead/addWrite was called with same `fd`. - deallocShared(cast[pointer](pcd)) - if not wsaCloseEvent(hEvent): - raiseOSError(osLastError()) - else: - # we need to include `fd` again - p.handles.incl(fd) - # and register WaitForSingleObject again - if not registerWaitForSingleObject(addr(pcd.waitFd), hEvent, - cast[WAITORTIMERCALLBACK](waitableCallback), - cast[pointer](pcd), INFINITE, flags): - # pcd.ovl will be unrefed in poll() - let err = osLastError() - deallocShared(cast[pointer](pcd)) - discard wsaCloseEvent(hEvent) - raiseOSError(err) - else: - # we incref `pcd.ovl` and `protect` callback one more time, - # because it will be unrefed and disposed in `poll()` after - # callback finishes. - GC_ref(pcd.ovl) - pcd.ovl.data.cell = system.protect(rawEnv(pcd.ovl.data.cb)) - ) - # We need to protect our callback environment value, so GC will not free it - # accidentally. - ol.data.cell = system.protect(rawEnv(ol.data.cb)) - - # This is main part of `hacky way` is using WSAEventSelect, so `hEvent` - # will be signaled when appropriate `mask` events will be triggered. - if wsaEventSelect(fd.SocketHandle, hEvent, mask) != 0: - let err = osLastError() - GC_unref(ol) - deallocShared(cast[pointer](pcd)) - discard wsaCloseEvent(hEvent) - raiseOSError(err) - - pcd.ovl = ol - if not registerWaitForSingleObject(addr(pcd.waitFd), hEvent, - cast[WAITORTIMERCALLBACK](waitableCallback), - cast[pointer](pcd), INFINITE, flags): - let err = osLastError() - GC_unref(ol) - deallocShared(cast[pointer](pcd)) - discard wsaCloseEvent(hEvent) - raiseOSError(err) - p.handles.incl(fd) - - proc addRead*(fd: AsyncFD, cb: Callback) = - ## Start watching the file descriptor for read availability and then call - ## the callback ``cb``. - ## - ## This is not ``pure`` mechanism for Windows Completion Ports (IOCP), - ## so if you can avoid it, please do it. Use `addRead` only if really - ## need it (main usecase is adaptation of `unix like` libraries to be - ## asynchronous on Windows). - ## If you use this function, you dont need to use asyncdispatch.recv() - ## or asyncdispatch.accept(), because they are using IOCP, please use - ## nativesockets.recv() and nativesockets.accept() instead. - ## - ## Be sure your callback ``cb`` returns ``true``, if you want to remove - ## watch of `read` notifications, and ``false``, if you want to continue - ## receiving notifies. - registerWaitableEvent(fd, cb, FD_READ or FD_ACCEPT or FD_OOB or FD_CLOSE) - - proc addWrite*(fd: AsyncFD, cb: Callback) = - ## Start watching the file descriptor for write availability and then call - ## the callback ``cb``. - ## - ## This is not ``pure`` mechanism for Windows Completion Ports (IOCP), - ## so if you can avoid it, please do it. Use `addWrite` only if really - ## need it (main usecase is adaptation of `unix like` libraries to be - ## asynchronous on Windows). - ## If you use this function, you dont need to use asyncdispatch.send() - ## or asyncdispatch.connect(), because they are using IOCP, please use - ## nativesockets.send() and nativesockets.connect() instead. - ## - ## Be sure your callback ``cb`` returns ``true``, if you want to remove - ## watch of `write` notifications, and ``false``, if you want to continue - ## receiving notifies. - registerWaitableEvent(fd, cb, FD_WRITE or FD_CONNECT or FD_CLOSE) - - template registerWaitableHandle(p, hEvent, flags, pcd, timeout, - handleCallback) = - let handleFD = AsyncFD(hEvent) - pcd.ioPort = p.ioPort - pcd.handleFd = handleFD - var ol = PCustomOverlapped() - GC_ref(ol) - ol.data.fd = handleFD - ol.data.cb = handleCallback - # We need to protect our callback environment value, so GC will not free it - # accidentally. - ol.data.cell = system.protect(rawEnv(ol.data.cb)) - - pcd.ovl = ol - if not registerWaitForSingleObject(addr(pcd.waitFd), hEvent, - cast[WAITORTIMERCALLBACK](waitableCallback), - cast[pointer](pcd), timeout.Dword, flags): - let err = osLastError() - GC_unref(ol) - deallocShared(cast[pointer](pcd)) - discard closeHandle(hEvent) - raiseOSError(err) - p.handles.incl(handleFD) - - template closeWaitable(handle: untyped) = - let waitFd = pcd.waitFd - deallocShared(cast[pointer](pcd)) - p.handles.excl(fd) - if unregisterWait(waitFd) == 0: - let err = osLastError() - if err.int32 != ERROR_IO_PENDING: - discard closeHandle(handle) - raiseOSError(err) - if closeHandle(handle) == 0: - raiseOSError(osLastError()) - - proc addTimer*(timeout: int, oneshot: bool, cb: Callback) = - ## Registers callback ``cb`` to be called when timer expired. - ## ``timeout`` - timeout value in milliseconds. - ## ``oneshot`` - `true`, to generate only one timeout event, `false`, to - ## generate timeout events periodically. - - doAssert(timeout > 0) - let p = getGlobalDispatcher() - - var hEvent = createEvent(nil, 1, 0, nil) - if hEvent == INVALID_HANDLE_VALUE: - raiseOSError(osLastError()) - - var pcd = cast[PostCallbackDataPtr](allocShared0(sizeof(PostCallbackData))) - var flags = WT_EXECUTEINWAITTHREAD.Dword - if oneshot: flags = flags or WT_EXECUTEONLYONCE - - proc timercb(fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = - let res = cb(fd) - if res or oneshot: - closeWaitable(hEvent) - else: - # if callback returned `false`, then it wants to be called again, so - # we need to ref and protect `pcd.ovl` again, because it will be - # unrefed and disposed in `poll()`. - GC_ref(pcd.ovl) - pcd.ovl.data.cell = system.protect(rawEnv(pcd.ovl.data.cb)) - - registerWaitableHandle(p, hEvent, flags, pcd, timeout, timercb) - - proc addProcess*(pid: int, cb: Callback) = - ## Registers callback ``cb`` to be called when process with pid ``pid`` - ## exited. - let p = getGlobalDispatcher() - let procFlags = SYNCHRONIZE - var hProcess = openProcess(procFlags, 0, pid.Dword) - if hProcess == INVALID_HANDLE_VALUE: - raiseOSError(osLastError()) - - var pcd = cast[PostCallbackDataPtr](allocShared0(sizeof(PostCallbackData))) - var flags = WT_EXECUTEINWAITTHREAD.Dword - - proc proccb(fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = - closeWaitable(hProcess) - discard cb(fd) - - registerWaitableHandle(p, hProcess, flags, pcd, INFINITE, proccb) - - proc newAsyncEvent*(): AsyncEvent = - ## Creates new ``AsyncEvent`` object. - ## New ``AsyncEvent`` object is not automatically registered with - ## dispatcher like ``AsyncSocket``. - var sa = SECURITY_ATTRIBUTES( - nLength: sizeof(SECURITY_ATTRIBUTES).cint, - bInheritHandle: 1 - ) - var event = createEvent(addr(sa), 0'i32, 0'i32, nil) - if event == INVALID_HANDLE_VALUE: - raiseOSError(osLastError()) - result = cast[AsyncEvent](allocShared0(sizeof(AsyncEventImpl))) - result.hEvent = event - - proc setEvent*(ev: AsyncEvent) = - ## Set event ``ev`` to signaled state. - if setEvent(ev.hEvent) == 0: - raiseOSError(osLastError()) - - proc unregister*(ev: AsyncEvent) = - ## Unregisters event ``ev``. - doAssert(ev.hWaiter != 0, "Event is not registered in the queue!") - let p = getGlobalDispatcher() - p.handles.excl(AsyncFD(ev.hEvent)) - if unregisterWait(ev.hWaiter) == 0: - let err = osLastError() - if err.int32 != ERROR_IO_PENDING: - raiseOSError(err) - ev.hWaiter = 0 - - proc close*(ev: AsyncEvent) = - ## Closes event ``ev``. - let res = closeHandle(ev.hEvent) - deallocShared(cast[pointer](ev)) - if res == 0: - raiseOSError(osLastError()) - - proc addEvent*(ev: AsyncEvent, cb: Callback) = - ## Registers callback ``cb`` to be called when ``ev`` will be signaled - doAssert(ev.hWaiter == 0, "Event is already registered in the queue!") - - let p = getGlobalDispatcher() - let hEvent = ev.hEvent - - var pcd = cast[PostCallbackDataPtr](allocShared0(sizeof(PostCallbackData))) - var flags = WT_EXECUTEINWAITTHREAD.Dword - - proc eventcb(fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = - if ev.hWaiter != 0: - if cb(fd): - # we need this check to avoid exception, if `unregister(event)` was - # called in callback. - deallocShared(cast[pointer](pcd)) - if ev.hWaiter != 0: - unregister(ev) - else: - # if callback returned `false`, then it wants to be called again, so - # we need to ref and protect `pcd.ovl` again, because it will be - # unrefed and disposed in `poll()`. - GC_ref(pcd.ovl) - pcd.ovl.data.cell = system.protect(rawEnv(pcd.ovl.data.cb)) - else: - # if ev.hWaiter == 0, then event was unregistered before `poll()` call. - deallocShared(cast[pointer](pcd)) - - registerWaitableHandle(p, hEvent, flags, pcd, INFINITE, eventcb) - ev.hWaiter = pcd.waitFd - - initAll() -else: - import ioselectors - from posix import EINTR, EAGAIN, EINPROGRESS, EWOULDBLOCK, MSG_PEEK, - MSG_NOSIGNAL - const - InitCallbackListSize = 4 # initial size of callbacks sequence, - # associated with file/socket descriptor. - InitDelayedCallbackListSize = 64 # initial size of delayed callbacks - # queue. - type - AsyncFD* = distinct cint - Callback = proc (fd: AsyncFD): bool {.closure,gcsafe.} - - AsyncData = object - readList: seq[Callback] - writeList: seq[Callback] - - AsyncEvent* = distinct SelectEvent - - PDispatcher* = ref object of PDispatcherBase - selector: Selector[AsyncData] - {.deprecated: [TAsyncFD: AsyncFD, TCallback: Callback].} - - proc `==`*(x, y: AsyncFD): bool {.borrow.} - proc `==`*(x, y: AsyncEvent): bool {.borrow.} - - template newAsyncData(): AsyncData = - AsyncData( - readList: newSeqOfCap[Callback](InitCallbackListSize), - writeList: newSeqOfCap[Callback](InitCallbackListSize) - ) - - proc newDispatcher*(): PDispatcher = - new result - result.selector = newSelector[AsyncData]() - result.timers.newHeapQueue() - result.callbacks = initDeque[proc ()](InitDelayedCallbackListSize) - - var gDisp{.threadvar.}: PDispatcher ## Global dispatcher - - proc setGlobalDispatcher*(disp: PDispatcher) = - if not gDisp.isNil: - assert gDisp.callbacks.len == 0 - gDisp = disp - initCallSoonProc() - - proc getGlobalDispatcher*(): PDispatcher = - if gDisp.isNil: - setGlobalDispatcher(newDispatcher()) - result = gDisp - - proc register*(fd: AsyncFD) = - let p = getGlobalDispatcher() - var data = newAsyncData() - p.selector.registerHandle(fd.SocketHandle, {}, data) - - proc closeSocket*(sock: AsyncFD) = - let disp = getGlobalDispatcher() - disp.selector.unregister(sock.SocketHandle) - sock.SocketHandle.close() - - proc unregister*(fd: AsyncFD) = - getGlobalDispatcher().selector.unregister(fd.SocketHandle) - - proc unregister*(ev: AsyncEvent) = - getGlobalDispatcher().selector.unregister(SelectEvent(ev)) - - proc addRead*(fd: AsyncFD, cb: Callback) = - let p = getGlobalDispatcher() - var newEvents = {Event.Read} - withData(p.selector, fd.SocketHandle, adata) do: - adata.readList.add(cb) - newEvents.incl(Event.Read) - if len(adata.writeList) != 0: newEvents.incl(Event.Write) - do: - raise newException(ValueError, "File descriptor not registered.") - p.selector.updateHandle(fd.SocketHandle, newEvents) - - proc addWrite*(fd: AsyncFD, cb: Callback) = - let p = getGlobalDispatcher() - var newEvents = {Event.Write} - withData(p.selector, fd.SocketHandle, adata) do: - adata.writeList.add(cb) - newEvents.incl(Event.Write) - if len(adata.readList) != 0: newEvents.incl(Event.Read) - do: - raise newException(ValueError, "File descriptor not registered.") - p.selector.updateHandle(fd.SocketHandle, newEvents) - - proc hasPendingOperations*(): bool = - let p = getGlobalDispatcher() - not p.selector.isEmpty() or p.timers.len != 0 or p.callbacks.len != 0 - - template processBasicCallbacks(ident, rwlist: untyped) = - # Process pending descriptor's and AsyncEvent callbacks. - # Invoke every callback stored in `rwlist`, until first one - # returned `false`, which means callback wants to stay - # alive. In such case all remaining callbacks will be added - # to `rwlist` again, in the order they have been inserted. - # - # `rwlist` associated with file descriptor MUST BE emptied before - # dispatching callback (See https://github.com/nim-lang/Nim/issues/5128), - # or it can be possible to fall into endless cycle. - var curList: seq[Callback] - - withData(p.selector, ident, adata) do: - shallowCopy(curList, adata.rwlist) - adata.rwlist = newSeqOfCap[Callback](InitCallbackListSize) - - let newLength = max(len(curList), InitCallbackListSize) - var newList = newSeqOfCap[Callback](newLength) - - for cb in curList: - if len(newList) > 0: - newList.add(cb) - else: - if not cb(fd.AsyncFD): - newList.add(cb) - - withData(p.selector, ident, adata) do: - # descriptor still present in queue. - adata.rwlist = newList & adata.rwlist - rLength = len(adata.readList) - wLength = len(adata.writeList) - do: - # descriptor was unregistered in callback via `unregister()`. - rLength = -1 - wLength = -1 - - template processCustomCallbacks(ident: untyped) = - # Process pending custom event callbacks. Custom events are - # {Event.Timer, Event.Signal, Event.Process, Event.Vnode}. - # There can be only one callback registered with one descriptor, - # so there no need to iterate over list. - var curList: seq[Callback] - - withData(p.selector, ident, adata) do: - shallowCopy(curList, adata.readList) - adata.readList = newSeqOfCap[Callback](InitCallbackListSize) - - let newLength = len(curList) - var newList = newSeqOfCap[Callback](newLength) - - var cb = curList[0] - if not cb(fd.AsyncFD): - newList.add(cb) - - withData(p.selector, ident, adata) do: - # descriptor still present in queue. - adata.readList = newList & adata.readList - if len(adata.readList) == 0: - # if no callbacks registered with descriptor, unregister it. - p.selector.unregister(fd) - do: - # descriptor was unregistered in callback via `unregister()`. - discard - - proc poll*(timeout = 500) = - var keys: array[64, ReadyKey] - - let p = getGlobalDispatcher() - when ioselSupportedPlatform: - let customSet = {Event.Timer, Event.Signal, Event.Process, - Event.Vnode} - - if p.selector.isEmpty() and p.timers.len == 0 and p.callbacks.len == 0: - raise newException(ValueError, - "No handles or timers registered in dispatcher.") - - if not p.selector.isEmpty(): - var count = p.selector.selectInto(p.adjustedTimeout(timeout), keys) - var i = 0 - while i < count: - var custom = false - let fd = keys[i].fd - let events = keys[i].events - var rLength = 0 # len(data.readList) after callback - var wLength = 0 # len(data.writeList) after callback - - if Event.Read in events or events == {Event.Error}: - processBasicCallbacks(fd, readList) - - if Event.Write in events or events == {Event.Error}: - processBasicCallbacks(fd, writeList) - - if Event.User in events or events == {Event.Error}: - processBasicCallbacks(fd, readList) - custom = true - if rLength == 0: - p.selector.unregister(fd) - - when ioselSupportedPlatform: - if (customSet * events) != {}: - custom = true - processCustomCallbacks(fd) - - # because state `data` can be modified in callback we need to update - # descriptor events with currently registered callbacks. - if not custom: - var newEvents: set[Event] = {} - if rLength != -1 and wLength != -1: - if rLength > 0: incl(newEvents, Event.Read) - if wLength > 0: incl(newEvents, Event.Write) - p.selector.updateHandle(SocketHandle(fd), newEvents) - inc(i) - - # Timer processing. - processTimers(p) - # Callback queue processing - processPendingCallbacks(p) - - proc recv*(socket: AsyncFD, size: int, - flags = {SocketFlag.SafeDisconn}): Future[string] = - var retFuture = newFuture[string]("recv") - - var readBuffer = newString(size) - - proc cb(sock: AsyncFD): bool = - result = true - let res = recv(sock.SocketHandle, addr readBuffer[0], size.cint, - flags.toOSFlags()) - if res < 0: - let lastError = osLastError() - if lastError.int32 notin {EINTR, EWOULDBLOCK, EAGAIN}: - if flags.isDisconnectionError(lastError): - retFuture.complete("") - else: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) - else: - result = false # We still want this callback to be called. - elif res == 0: - # Disconnected - retFuture.complete("") - else: - readBuffer.setLen(res) - retFuture.complete(readBuffer) - # TODO: The following causes a massive slowdown. - #if not cb(socket): - addRead(socket, cb) - return retFuture - - proc recvInto*(socket: AsyncFD, buf: pointer, size: int, - flags = {SocketFlag.SafeDisconn}): Future[int] = - var retFuture = newFuture[int]("recvInto") - - proc cb(sock: AsyncFD): bool = - result = true - let res = recv(sock.SocketHandle, buf, size.cint, - flags.toOSFlags()) - if res < 0: - let lastError = osLastError() - if lastError.int32 notin {EINTR, EWOULDBLOCK, EAGAIN}: - if flags.isDisconnectionError(lastError): - retFuture.complete(0) - else: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) - else: - result = false # We still want this callback to be called. - else: - retFuture.complete(res) - # TODO: The following causes a massive slowdown. - #if not cb(socket): - addRead(socket, cb) - return retFuture - - proc send*(socket: AsyncFD, buf: pointer, size: int, - flags = {SocketFlag.SafeDisconn}): Future[void] = - var retFuture = newFuture[void]("send") - - var written = 0 - - proc cb(sock: AsyncFD): bool = - result = true - let netSize = size-written - var d = cast[cstring](buf) - let res = send(sock.SocketHandle, addr d[written], netSize.cint, - MSG_NOSIGNAL) - if res < 0: - let lastError = osLastError() - if lastError.int32 notin {EINTR, EWOULDBLOCK, EAGAIN}: - if flags.isDisconnectionError(lastError): - retFuture.complete() - else: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) - else: - result = false # We still want this callback to be called. - else: - written.inc(res) - if res != netSize: - result = false # We still have data to send. - else: - retFuture.complete() - # TODO: The following causes crashes. - #if not cb(socket): - addWrite(socket, cb) - return retFuture - - proc send*(socket: AsyncFD, data: string, - flags = {SocketFlag.SafeDisconn}): Future[void] = - var retFuture = newFuture[void]("send") - - var written = 0 - - proc cb(sock: AsyncFD): bool = - result = true - let netSize = data.len-written - var d = data.cstring - let res = send(sock.SocketHandle, addr d[written], netSize.cint, - MSG_NOSIGNAL) - if res < 0: - let lastError = osLastError() - if lastError.int32 notin {EINTR, EWOULDBLOCK, EAGAIN}: - if flags.isDisconnectionError(lastError): - retFuture.complete() - else: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) - else: - result = false # We still want this callback to be called. - else: - written.inc(res) - if res != netSize: - result = false # We still have data to send. - else: - retFuture.complete() - # TODO: The following causes crashes. - #if not cb(socket): - addWrite(socket, cb) - return retFuture - - proc sendTo*(socket: AsyncFD, data: pointer, size: int, saddr: ptr SockAddr, - saddrLen: SockLen, - flags = {SocketFlag.SafeDisconn}): Future[void] = - ## Sends ``data`` of size ``size`` in bytes to specified destination - ## (``saddr`` of size ``saddrLen`` in bytes, using socket ``socket``. - ## The returned future will complete once all data has been sent. - var retFuture = newFuture[void]("sendTo") - - # we will preserve address in our stack - var staddr: array[128, char] # SOCKADDR_STORAGE size is 128 bytes - var stalen = saddrLen - zeroMem(addr(staddr[0]), 128) - copyMem(addr(staddr[0]), saddr, saddrLen) - - proc cb(sock: AsyncFD): bool = - result = true - let res = sendto(sock.SocketHandle, data, size, MSG_NOSIGNAL, - cast[ptr SockAddr](addr(staddr[0])), stalen) - if res < 0: - let lastError = osLastError() - if lastError.int32 notin {EINTR, EWOULDBLOCK, EAGAIN}: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) - else: - result = false # We still want this callback to be called. - else: - retFuture.complete() - - addWrite(socket, cb) - return retFuture - - proc recvFromInto*(socket: AsyncFD, data: pointer, size: int, - saddr: ptr SockAddr, saddrLen: ptr SockLen, - flags = {SocketFlag.SafeDisconn}): Future[int] = - ## Receives a datagram data from ``socket`` into ``data``, which must - ## be at least of size ``size`` in bytes, address of datagram's sender - ## will be stored into ``saddr`` and ``saddrLen``. Returned future will - ## complete once one datagram has been received, and will return size - ## of packet received. - var retFuture = newFuture[int]("recvFromInto") - proc cb(sock: AsyncFD): bool = - result = true - let res = recvfrom(sock.SocketHandle, data, size.cint, flags.toOSFlags(), - saddr, saddrLen) - if res < 0: - let lastError = osLastError() - if lastError.int32 notin {EINTR, EWOULDBLOCK, EAGAIN}: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) - else: - result = false - else: - retFuture.complete(res) - addRead(socket, cb) - return retFuture - - proc acceptAddr*(socket: AsyncFD, flags = {SocketFlag.SafeDisconn}): - Future[tuple[address: string, client: AsyncFD]] = - var retFuture = newFuture[tuple[address: string, - client: AsyncFD]]("acceptAddr") - proc cb(sock: AsyncFD): bool = - result = true - var sockAddress: Sockaddr_storage - var addrLen = sizeof(sockAddress).Socklen - var client = accept(sock.SocketHandle, - cast[ptr SockAddr](addr(sockAddress)), addr(addrLen)) - if client == osInvalidSocket: - let lastError = osLastError() - assert lastError.int32 notin {EWOULDBLOCK, EAGAIN} - if lastError.int32 == EINTR: - return false - else: - if flags.isDisconnectionError(lastError): - return false - else: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) - else: - try: - let address = getAddrString(cast[ptr SockAddr](addr sockAddress)) - register(client.AsyncFD) - retFuture.complete((address, client.AsyncFD)) - except: - # getAddrString may raise - client.close() - retFuture.fail(getCurrentException()) - addRead(socket, cb) - return retFuture - - when ioselSupportedPlatform: - - proc addTimer*(timeout: int, oneshot: bool, cb: Callback) = - ## Start watching for timeout expiration, and then call the - ## callback ``cb``. - ## ``timeout`` - time in milliseconds, - ## ``oneshot`` - if ``true`` only one event will be dispatched, - ## if ``false`` continuous events every ``timeout`` milliseconds. - let p = getGlobalDispatcher() - var data = newAsyncData() - data.readList.add(cb) - p.selector.registerTimer(timeout, oneshot, data) - - proc addSignal*(signal: int, cb: Callback) = - ## Start watching signal ``signal``, and when signal appears, call the - ## callback ``cb``. - let p = getGlobalDispatcher() - var data = newAsyncData() - data.readList.add(cb) - p.selector.registerSignal(signal, data) - - proc addProcess*(pid: int, cb: Callback) = - ## Start watching for process exit with pid ``pid``, and then call - ## the callback ``cb``. - let p = getGlobalDispatcher() - var data = newAsyncData() - data.readList.add(cb) - p.selector.registerProcess(pid, data) - - proc newAsyncEvent*(): AsyncEvent = - ## Creates new ``AsyncEvent``. - result = AsyncEvent(newSelectEvent()) - - proc setEvent*(ev: AsyncEvent) = - ## Sets new ``AsyncEvent`` to signaled state. - setEvent(SelectEvent(ev)) - - proc close*(ev: AsyncEvent) = - ## Closes ``AsyncEvent`` - close(SelectEvent(ev)) - - proc addEvent*(ev: AsyncEvent, cb: Callback) = - ## Start watching for event ``ev``, and call callback ``cb``, when - ## ev will be set to signaled state. - let p = getGlobalDispatcher() - var data = newAsyncData() - data.readList.add(cb) - p.selector.registerEvent(SelectEvent(ev), data) - -# Common procedures between current and upcoming asyncdispatch -include includes.asynccommon - -proc sleepAsync*(ms: int): Future[void] = - ## Suspends the execution of the current async procedure for the next - ## ``ms`` milliseconds. - var retFuture = newFuture[void]("sleepAsync") - let p = getGlobalDispatcher() - p.timers.push((epochTime() + (ms / 1000), retFuture)) - return retFuture - -proc withTimeout*[T](fut: Future[T], timeout: int): Future[bool] = - ## Returns a future which will complete once ``fut`` completes or after - ## ``timeout`` milliseconds has elapsed. - ## - ## If ``fut`` completes first the returned future will hold true, - ## otherwise, if ``timeout`` milliseconds has elapsed first, the returned - ## future will hold false. - - var retFuture = newFuture[bool]("asyncdispatch.`withTimeout`") - var timeoutFuture = sleepAsync(timeout) - fut.callback = - proc () = - if not retFuture.finished: retFuture.complete(true) - timeoutFuture.callback = - proc () = - if not retFuture.finished: retFuture.complete(false) - return retFuture - -proc accept*(socket: AsyncFD, - flags = {SocketFlag.SafeDisconn}): Future[AsyncFD] = - ## Accepts a new connection. Returns a future containing the client socket - ## corresponding to that connection. - ## The future will complete when the connection is successfully accepted. - var retFut = newFuture[AsyncFD]("accept") - var fut = acceptAddr(socket, flags) - fut.callback = - proc (future: Future[tuple[address: string, client: AsyncFD]]) = - assert future.finished - if future.failed: - retFut.fail(future.error) - else: - retFut.complete(future.read.client) - return retFut - -# -- Await Macro -include asyncmacro - -proc readAll*(future: FutureStream[string]): Future[string] {.async.} = - ## Returns a future that will complete when all the string data from the - ## specified future stream is retrieved. - result = "" - while true: - let (hasValue, value) = await future.read() - if hasValue: - result.add(value) - else: - break - -proc recvLine*(socket: AsyncFD): Future[string] {.async.} = - ## Reads a line of data from ``socket``. Returned future will complete once - ## a full line is read or an error occurs. - ## - ## If a full line is read ``\r\L`` is not - ## added to ``line``, however if solely ``\r\L`` is read then ``line`` - ## will be set to it. - ## - ## If the socket is disconnected, ``line`` will be set to ``""``. - ## - ## If the socket is disconnected in the middle of a line (before ``\r\L`` - ## is read) then line will be set to ``""``. - ## The partial line **will be lost**. - ## - ## **Warning**: This assumes that lines are delimited by ``\r\L``. - ## - ## **Note**: This procedure is mostly used for testing. You likely want to - ## use ``asyncnet.recvLine`` instead. - - template addNLIfEmpty(): typed = - if result.len == 0: - result.add("\c\L") - - result = "" - var c = "" - while true: - c = await recv(socket, 1) - if c.len == 0: - return "" - if c == "\r": - c = await recv(socket, 1) - assert c == "\l" - addNLIfEmpty() - return - elif c == "\L": - addNLIfEmpty() - return - add(result, c) - -proc callSoon(cbproc: proc ()) = - ## Schedule `cbproc` to be called as soon as possible. - ## The callback is called when control returns to the event loop. - getGlobalDispatcher().callbacks.addLast(cbproc) - -proc runForever*() = - ## Begins a never ending global dispatcher poll loop. - while true: - poll() - -proc waitFor*[T](fut: Future[T]): T = - ## **Blocks** the current thread until the specified future completes. - while not fut.finished: - poll() - - fut.read diff --git a/lib/windows/winlean.nim b/lib/windows/winlean.nim index a833377e5..7e22f98c7 100644 --- a/lib/windows/winlean.nim +++ b/lib/windows/winlean.nim @@ -10,7 +10,7 @@ ## This module implements a small wrapper for some needed Win API procedures, ## so that the Nim compiler does not depend on the huge Windows module. -{.deadCodeElim:on.} +{.deadCodeElim: on.} # dce option deprecated import dynlib @@ -481,6 +481,13 @@ type sin6_flowinfo*: int32 # unsigned sin6_addr*: In6_addr + Sockaddr_storage* {.importc: "SOCKADDR_STORAGE", + header: "winsock2.h".} = object + ss_family*: int16 + ss_pad1: array[6, byte] + ss_align: int64 + ss_pad2: array[112, byte] + Servent* = object s_name*: cstring s_aliases*: cstringArray @@ -666,6 +673,7 @@ const CREATE_ALWAYS* = 2'i32 CREATE_NEW* = 1'i32 OPEN_EXISTING* = 3'i32 + OPEN_ALWAYS* = 4'i32 FILE_BEGIN* = 0'i32 INVALID_SET_FILE_POINTER* = -1'i32 NO_ERROR* = 0'i32 @@ -686,6 +694,7 @@ const ERROR_FILE_NOT_FOUND* = 2 ERROR_PATH_NOT_FOUND* = 3 ERROR_ACCESS_DENIED* = 5 + ERROR_NO_MORE_FILES* = 18 ERROR_HANDLE_EOF* = 38 ERROR_BAD_ARGUMENTS* = 165 @@ -695,6 +704,11 @@ proc duplicateHandle*(hSourceProcessHandle: HANDLE, hSourceHandle: HANDLE, dwDesiredAccess: DWORD, bInheritHandle: WINBOOL, dwOptions: DWORD): WINBOOL{.stdcall, dynlib: "kernel32", importc: "DuplicateHandle".} + +proc setHandleInformation*(hObject: HANDLE, dwMask: DWORD, + dwFlags: DWORD): WINBOOL {.stdcall, + dynlib: "kernel32", importc: "SetHandleInformation".} + proc getCurrentProcess*(): HANDLE{.stdcall, dynlib: "kernel32", importc: "GetCurrentProcess".} @@ -1072,3 +1086,14 @@ proc ConvertThreadToFiberEx*(param: pointer, flags: int32): pointer {.stdcall, d proc DeleteFiber*(fiber: pointer): void {.stdcall, discardable, dynlib: "kernel32", importc.} proc SwitchToFiber*(fiber: pointer): void {.stdcall, discardable, dynlib: "kernel32", importc.} proc GetCurrentFiber*(): pointer {.stdcall, importc, header: "Windows.h".} + +proc toFILETIME*(t: int64): FILETIME = + ## Convert the Windows file time timestamp ``t`` to ``FILETIME``. + result = FILETIME(dwLowDateTime: cast[DWORD](t), dwHighDateTime: DWORD(t shr 32)) + +type + LPFILETIME* = ptr FILETIME + +proc setFileTime*(hFile: HANDLE, lpCreationTime: LPFILETIME, + lpLastAccessTime: LPFILETIME, lpLastWriteTime: LPFILETIME): WINBOOL + {.stdcall, dynlib: "kernel32", importc: "SetFileTime".} diff --git a/lib/wrappers/iup.nim b/lib/wrappers/iup.nim index d910173ca..dee2e14c7 100644 --- a/lib/wrappers/iup.nim +++ b/lib/wrappers/iup.nim @@ -34,7 +34,7 @@ # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # **************************************************************************** -{.deadCodeElim: on.} +{.deadCodeElim: on.} # dce option deprecated when defined(windows): const dllname = "iup(|30|27|26|25|24).dll" diff --git a/lib/wrappers/joyent_http_parser.nim b/lib/wrappers/joyent_http_parser.nim deleted file mode 100644 index f7412d2b8..000000000 --- a/lib/wrappers/joyent_http_parser.nim +++ /dev/null @@ -1,93 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -type - csize = int - - HttpDataProc* = proc (a2: ptr HttpParser, at: cstring, length: csize): cint {.cdecl.} - HttpProc* = proc (a2: ptr HttpParser): cint {.cdecl.} - - HttpMethod* = enum - HTTP_DELETE = 0, HTTP_GET, HTTP_HEAD, HTTP_POST, HTTP_PUT, HTTP_CONNECT, - HTTP_OPTIONS, HTTP_TRACE, HTTP_COPY, HTTP_LOCK, HTTP_MKCOL, HTTP_MOVE, - HTTP_PROPFIND, HTTP_PROPPATCH, HTTP_UNLOCK, HTTP_REPORT, HTTP_MKACTIVITY, - HTTP_CHECKOUT, HTTP_MERGE, HTTP_MSEARCH, HTTP_NOTIFY, HTTP_SUBSCRIBE, - HTTP_UNSUBSCRIBE, HTTP_PATCH - - HttpParserType* = enum - HTTP_REQUEST, HTTP_RESPONSE, HTTP_BOTH - - ParserFlag* = enum - F_CHUNKED = 1 shl 0, - F_CONNECTION_KEEP_ALIVE = 1 shl 1, - F_CONNECTION_CLOSE = 1 shl 2, - F_TRAILING = 1 shl 3, - F_UPGRADE = 1 shl 4, - F_SKIPBODY = 1 shl 5 - - HttpErrNo* = enum - HPE_OK, HPE_CB_message_begin, HPE_CB_path, HPE_CB_query_string, HPE_CB_url, - HPE_CB_fragment, HPE_CB_header_field, HPE_CB_header_value, - HPE_CB_headers_complete, HPE_CB_body, HPE_CB_message_complete, - HPE_INVALID_EOF_STATE, HPE_HEADER_OVERFLOW, HPE_CLOSED_CONNECTION, - HPE_INVALID_VERSION, HPE_INVALID_STATUS, HPE_INVALID_METHOD, - HPE_INVALID_URL, HPE_INVALID_HOST, HPE_INVALID_PORT, HPE_INVALID_PATH, - HPE_INVALID_QUERY_STRING, HPE_INVALID_FRAGMENT, HPE_LF_EXPECTED, - HPE_INVALID_HEADER_TOKEN, HPE_INVALID_CONTENT_LENGTH, - HPE_INVALID_CHUNK_SIZE, HPE_INVALID_CONSTANT, HPE_INVALID_INTERNAL_STATE, - HPE_STRICT, HPE_UNKNOWN - - HttpParser*{.pure, final, importc: "http_parser", header: "http_parser.h".} = object - typ {.importc: "type".}: char - flags {.importc: "flags".}: char - state*{.importc: "state".}: char - header_state*{.importc: "header_state".}: char - index*{.importc: "index".}: char - nread*{.importc: "nread".}: cint - content_length*{.importc: "content_length".}: int64 - http_major*{.importc: "http_major".}: cshort - http_minor*{.importc: "http_minor".}: cshort - status_code*{.importc: "status_code".}: cshort - http_method*{.importc: "method".}: cshort - http_errno_bits {.importc: "http_errno".}: char - upgrade {.importc: "upgrade".}: bool - data*{.importc: "data".}: pointer - - HttpParserSettings*{.pure, final, importc: "http_parser_settings", header: "http_parser.h".} = object - on_message_begin*{.importc: "on_message_begin".}: HttpProc - on_url*{.importc: "on_url".}: HttpDataProc - on_header_field*{.importc: "on_header_field".}: HttpDataProc - on_header_value*{.importc: "on_header_value".}: HttpDataProc - on_headers_complete*{.importc: "on_headers_complete".}: HttpProc - on_body*{.importc: "on_body".}: HttpDataProc - on_message_complete*{.importc: "on_message_complete".}: HttpProc -{.deprecated: [THttpMethod: HttpMethod, THttpParserType: HttpParserType, - TParserFlag: ParserFlag, THttpErrNo: HttpErrNo, - THttpParser: HttpParser, THttpParserSettings: HttpParserSettings].} - -proc http_parser_init*(parser: var HttpParser, typ: HttpParserType){. - importc: "http_parser_init", header: "http_parser.h".} - -proc http_parser_execute*(parser: var HttpParser, - settings: var HttpParserSettings, data: cstring, - len: csize): csize {. - importc: "http_parser_execute", header: "http_parser.h".} - -proc http_should_keep_alive*(parser: var HttpParser): cint{. - importc: "http_should_keep_alive", header: "http_parser.h".} - -proc http_method_str*(m: HttpMethod): cstring{. - importc: "http_method_str", header: "http_parser.h".} - -proc http_errno_name*(err: HttpErrNo): cstring{. - importc: "http_errno_name", header: "http_parser.h".} - -proc http_errno_description*(err: HttpErrNo): cstring{. - importc: "http_errno_description", header: "http_parser.h".} - diff --git a/lib/wrappers/libsvm.nim b/lib/wrappers/libsvm.nim deleted file mode 100644 index ac5889410..000000000 --- a/lib/wrappers/libsvm.nim +++ /dev/null @@ -1,117 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2012 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## This module is a low level wrapper for `libsvm`:idx:. - -{.deadCodeElim: on.} -const - LIBSVM_VERSION* = 312 - -when defined(windows): - const svmdll* = "libsvm.dll" -elif defined(macosx): - const svmdll* = "libsvm.dylib" -else: - const svmdll* = "libsvm.so" - -type - Node*{.pure, final.} = object - index*: cint - value*: cdouble - - Problem*{.pure, final.} = object - L*: cint - y*: ptr cdouble - x*: ptr ptr Node - - Type*{.size: sizeof(cint).} = enum - C_SVC, NU_SVC, ONE_CLASS, EPSILON_SVR, NU_SVR - - KernelType*{.size: sizeof(cint).} = enum - LINEAR, POLY, RBF, SIGMOID, PRECOMPUTED - - Parameter*{.pure, final.} = object - typ*: Type - kernelType*: KernelType - degree*: cint # for poly - gamma*: cdouble # for poly/rbf/sigmoid - coef0*: cdouble # for poly/sigmoid - # these are for training only - cache_size*: cdouble # in MB - eps*: cdouble # stopping criteria - C*: cdouble # for C_SVC, EPSILON_SVR and NU_SVR - nr_weight*: cint # for C_SVC - weight_label*: ptr cint # for C_SVC - weight*: ptr cdouble # for C_SVC - nu*: cdouble # for NU_SVC, ONE_CLASS, and NU_SVR - p*: cdouble # for EPSILON_SVR - shrinking*: cint # use the shrinking heuristics - probability*: cint # do probability estimates -{.deprecated: [Tnode: Node, Tproblem: Problem, Ttype: Type, - TKernelType: KernelType, Tparameter: Parameter].} - -# -# svm_model -# - -type - Model*{.pure, final.} = object - param*: Parameter # parameter - nr_class*: cint # number of classes, = 2 in regression/one class svm - L*: cint # total #SV - SV*: ptr ptr Node # SVs (SV[l]) - sv_coef*: ptr ptr cdouble # coefficients for SVs in decision functions (sv_coef[k-1][l]) - rho*: ptr cdouble # constants in decision functions (rho[k*(k-1)/2]) - probA*: ptr cdouble # pariwise probability information - probB*: ptr cdouble # for classification only - label*: ptr cint # label of each class (label[k]) - nSV*: ptr cint # number of SVs for each class (nSV[k]) - # nSV[0] + nSV[1] + ... + nSV[k-1] = l - # XXX - free_sv*: cint # 1 if svm_model is created by svm_load_model - # 0 if svm_model is created by svm_train -{.deprecated: [TModel: Model].} - -proc train*(prob: ptr Problem, param: ptr Parameter): ptr Model{.cdecl, - importc: "svm_train", dynlib: svmdll.} -proc cross_validation*(prob: ptr Problem, param: ptr Parameter, nr_fold: cint, - target: ptr cdouble){.cdecl, - importc: "svm_cross_validation", dynlib: svmdll.} -proc save_model*(model_file_name: cstring, model: ptr Model): cint{.cdecl, - importc: "svm_save_model", dynlib: svmdll.} -proc load_model*(model_file_name: cstring): ptr Model{.cdecl, - importc: "svm_load_model", dynlib: svmdll.} -proc get_svm_type*(model: ptr Model): cint{.cdecl, importc: "svm_get_svm_type", - dynlib: svmdll.} -proc get_nr_class*(model: ptr Model): cint{.cdecl, importc: "svm_get_nr_class", - dynlib: svmdll.} -proc get_labels*(model: ptr Model, label: ptr cint){.cdecl, - importc: "svm_get_labels", dynlib: svmdll.} -proc get_svr_probability*(model: ptr Model): cdouble{.cdecl, - importc: "svm_get_svr_probability", dynlib: svmdll.} -proc predict_values*(model: ptr Model, x: ptr Node, dec_values: ptr cdouble): cdouble{. - cdecl, importc: "svm_predict_values", dynlib: svmdll.} -proc predict*(model: ptr Model, x: ptr Node): cdouble{.cdecl, - importc: "svm_predict", dynlib: svmdll.} -proc predict_probability*(model: ptr Model, x: ptr Node, - prob_estimates: ptr cdouble): cdouble{.cdecl, - importc: "svm_predict_probability", dynlib: svmdll.} -proc free_model_content*(model_ptr: ptr Model){.cdecl, - importc: "svm_free_model_content", dynlib: svmdll.} -proc free_and_destroy_model*(model_ptr_ptr: ptr ptr Model){.cdecl, - importc: "svm_free_and_destroy_model", dynlib: svmdll.} -proc destroy_param*(param: ptr Parameter){.cdecl, importc: "svm_destroy_param", - dynlib: svmdll.} -proc check_parameter*(prob: ptr Problem, param: ptr Parameter): cstring{. - cdecl, importc: "svm_check_parameter", dynlib: svmdll.} -proc check_probability_model*(model: ptr Model): cint{.cdecl, - importc: "svm_check_probability_model", dynlib: svmdll.} - -proc set_print_string_function*(print_func: proc (arg: cstring) {.cdecl.}){. - cdecl, importc: "svm_set_print_string_function", dynlib: svmdll.} diff --git a/lib/wrappers/mysql.nim b/lib/wrappers/mysql.nim index 6dbed23b3..06c663822 100644 --- a/lib/wrappers/mysql.nim +++ b/lib/wrappers/mysql.nim @@ -7,19 +7,19 @@ # distribution, for details about the copyright. # -{.deadCodeElim: on.} +{.deadCodeElim: on.} # dce option deprecated {.push, callconv: cdecl.} when defined(Unix): when defined(macosx): const - lib = "libmysqlclient.(15|16|17|18|19|20).dylib" + lib = "(libmysqlclient|libmariadbclient)(|.20|.19|.18|.17|.16|.15).dylib" else: const - lib = "libmysqlclient.so.(15|16|17|18|19|20)" + lib = "(libmysqlclient|libmariadbclient).so(|.20|.19|.18|.17|.16|.15)" when defined(Windows): const - lib = "libmysql.dll" + lib = "(libmysql.dll|libmariadb.dll)" type my_bool* = bool Pmy_bool* = ptr my_bool diff --git a/lib/wrappers/odbcsql.nim b/lib/wrappers/odbcsql.nim index 1b2544ec0..43d1c504d 100644 --- a/lib/wrappers/odbcsql.nim +++ b/lib/wrappers/odbcsql.nim @@ -7,7 +7,7 @@ # distribution, for details about the copyright. # -{.deadCodeElim: on.} +{.deadCodeElim: on.} # dce option deprecated when not defined(ODBCVER): const diff --git a/lib/wrappers/openssl.nim b/lib/wrappers/openssl.nim index 55b0bc3f8..a24575d11 100644 --- a/lib/wrappers/openssl.nim +++ b/lib/wrappers/openssl.nim @@ -8,33 +8,46 @@ # ## OpenSSL support - -{.deadCodeElim: on.} +## +## When OpenSSL is dynamically linked, the wrapper provides partial forward and backward +## compatibility for OpenSSL versions above and below 1.1.0 +## +## OpenSSL can also be statically linked using ``--dynlibOverride:ssl`` for OpenSSL >= 1.1.0. +## If you want to statically link against OpenSSL 1.0.x, you now have to +## define the ``openssl10`` symbol via ``-d:openssl10``. +## +## Build and test examples: +## +## .. code-block:: +## ./bin/nim c -d:ssl -p:. -r tests/untestable/tssl.nim +## ./bin/nim c -d:ssl -p:. --dynlibOverride:ssl --passL:-lcrypto --passL:-lssl -r tests/untestable/tssl.nim + +{.deadCodeElim: on.} # dce option deprecated const useWinVersion = defined(Windows) or defined(nimdoc) when useWinVersion: when not defined(nimOldDlls) and defined(cpu64): const - DLLSSLName = "(ssleay64|libssl64).dll" - DLLUtilName = "libeay64.dll" + DLLSSLName* = "(libssl-1_1-x64|ssleay64|libssl64).dll" + DLLUtilName* = "(libcrypto-1_1-x64|libeay64).dll" else: const - DLLSSLName = "(ssleay32|libssl32).dll" - DLLUtilName = "libeay32.dll" + DLLSSLName* = "(libssl-1_1|ssleay32|libssl32).dll" + DLLUtilName* = "(libcrypto-1_1|libeay32).dll" from winlean import SocketHandle else: - const - versions = "(|.38|.39|.41|.43|.10|.1.0.2|.1.0.1|.1.0.0|.0.9.9|.0.9.8)" + const versions = "(.1.1|.38|.39|.41|.43|.44|.45|.10|.1.0.2|.1.0.1|.1.0.0|.0.9.9|.0.9.8|)" + when defined(macosx): const - DLLSSLName = "libssl" & versions & ".dylib" - DLLUtilName = "libcrypto" & versions & ".dylib" + DLLSSLName* = "libssl" & versions & ".dylib" + DLLUtilName* = "libcrypto" & versions & ".dylib" else: const - DLLSSLName = "libssl.so" & versions - DLLUtilName = "libcrypto.so" & versions + DLLSSLName* = "libssl.so" & versions + DLLUtilName* = "libcrypto.so" & versions from posix import SocketHandle import dynlib @@ -140,6 +153,7 @@ const SSL_OP_NO_SSLv2* = 0x01000000 SSL_OP_NO_SSLv3* = 0x02000000 SSL_OP_NO_TLSv1* = 0x04000000 + SSL_OP_NO_TLSv1_1* = 0x08000000 SSL_OP_ALL* = 0x000FFFFF SSL_VERIFY_NONE* = 0x00000000 SSL_VERIFY_PEER* = 0x00000001 @@ -191,16 +205,45 @@ const proc TLSv1_method*(): PSSL_METHOD{.cdecl, dynlib: DLLSSLName, importc.} +# TLS_method(), TLS_server_method(), TLS_client_method() are introduced in 1.1.0 +# and support SSLv3, TLSv1, TLSv1.1 and TLSv1.2 +# SSLv23_method(), SSLv23_server_method(), SSLv23_client_method() are removed in 1.1.0 + when compileOption("dynlibOverride", "ssl"): - proc SSL_library_init*(): cint {.cdecl, dynlib: DLLSSLName, importc, discardable.} - proc SSL_load_error_strings*() {.cdecl, dynlib: DLLSSLName, importc.} - proc SSLv23_client_method*(): PSSL_METHOD {.cdecl, dynlib: DLLSSLName, importc.} + # Static linking + + when defined(openssl10): + proc SSL_library_init*(): cint {.cdecl, dynlib: DLLSSLName, importc, discardable.} + proc SSL_load_error_strings*() {.cdecl, dynlib: DLLSSLName, importc.} + proc SSLv23_method*(): PSSL_METHOD {.cdecl, dynlib: DLLSSLName, importc.} + else: + proc OPENSSL_init_ssl*(opts: uint64, settings: uint8): cint {.cdecl, dynlib: DLLSSLName, importc, discardable.} + proc SSL_library_init*(): cint {.discardable.} = + ## Initialize SSL using OPENSSL_init_ssl for OpenSSL >= 1.1.0 + return OPENSSL_init_ssl(0.uint64, 0.uint8) + + proc TLS_method*(): PSSL_METHOD {.cdecl, dynlib: DLLSSLName, importc.} + proc SSLv23_method*(): PSSL_METHOD = + TLS_method() + + proc OpenSSL_version_num(): culong {.cdecl, dynlib: DLLSSLName, importc.} + + proc getOpenSSLVersion*(): culong = + ## Return OpenSSL version as unsigned long + OpenSSL_version_num() - proc SSLv23_method*(): PSSL_METHOD {.cdecl, dynlib: DLLSSLName, importc.} + proc SSL_load_error_strings*() = + ## Removed from OpenSSL 1.1.0 + # This proc prevents breaking existing code calling SslLoadErrorStrings + # Static linking against OpenSSL < 1.1.0 is not supported + discard + + template OpenSSL_add_all_algorithms*() = discard + + proc SSLv23_client_method*(): PSSL_METHOD {.cdecl, dynlib: DLLSSLName, importc.} proc SSLv2_method*(): PSSL_METHOD {.cdecl, dynlib: DLLSSLName, importc.} proc SSLv3_method*(): PSSL_METHOD {.cdecl, dynlib: DLLSSLName, importc.} - template OpenSSL_add_all_algorithms*() = discard else: # Here we're trying to stay compatible with openssl 1.0.* and 1.1.*. Some # symbols are loaded dynamically and we don't use them if not found. @@ -223,38 +266,58 @@ else: if not dl.isNil: result = symAddr(dl, name) + proc loadPSSLMethod(method1, method2: string): PSSL_METHOD = + ## Load <method1> from OpenSSL if available, otherwise <method2> + let m1 = cast[proc(): PSSL_METHOD {.cdecl, gcsafe.}](sslSym(method1)) + if not m1.isNil: + return m1() + cast[proc(): PSSL_METHOD {.cdecl, gcsafe.}](sslSym(method2))() + proc SSL_library_init*(): cint {.discardable.} = - let theProc = cast[proc(): cint {.cdecl.}](sslSym("SSL_library_init")) - if not theProc.isNil: result = theProc() + ## Initialize SSL using OPENSSL_init_ssl for OpenSSL >= 1.1.0 otherwise + ## SSL_library_init + let theProc = cast[proc(opts: uint64, settings: uint8): cint {.cdecl.}](sslSym("OPENSSL_init_ssl")) + if not theProc.isNil: + return theProc(0, 0) + let olderProc = cast[proc(): cint {.cdecl.}](sslSym("SSL_library_init")) + if not olderProc.isNil: result = olderProc() proc SSL_load_error_strings*() = let theProc = cast[proc() {.cdecl.}](sslSym("SSL_load_error_strings")) if not theProc.isNil: theProc() proc SSLv23_client_method*(): PSSL_METHOD = - let theProc = cast[proc(): PSSL_METHOD {.cdecl, gcsafe.}](sslSym("SSLv23_client_method")) - if not theProc.isNil: result = theProc() - else: result = TLSv1_method() + loadPSSLMethod("SSLv23_client_method", "TLS_client_method") proc SSLv23_method*(): PSSL_METHOD = - let theProc = cast[proc(): PSSL_METHOD {.cdecl, gcsafe.}](sslSym("SSLv23_method")) - if not theProc.isNil: result = theProc() - else: result = TLSv1_method() + loadPSSLMethod("SSLv23_method", "TLS_method") proc SSLv2_method*(): PSSL_METHOD = - let theProc = cast[proc(): PSSL_METHOD {.cdecl, gcsafe.}](sslSym("SSLv2_method")) - if not theProc.isNil: result = theProc() - else: result = TLSv1_method() + loadPSSLMethod("SSLv2_method", "TLS_method") proc SSLv3_method*(): PSSL_METHOD = - let theProc = cast[proc(): PSSL_METHOD {.cdecl, gcsafe.}](sslSym("SSLv3_method")) - if not theProc.isNil: result = theProc() - else: result = TLSv1_method() + loadPSSLMethod("SSLv3_method", "TLS_method") + + proc TLS_method*(): PSSL_METHOD = + loadPSSLMethod("TLS_method", "SSLv23_method") + + proc TLS_client_method*(): PSSL_METHOD = + loadPSSLMethod("TLS_client_method", "SSLv23_client_method") + + proc TLS_server_method*(): PSSL_METHOD = + loadPSSLMethod("TLS_server_method", "SSLv23_server_method") proc OpenSSL_add_all_algorithms*() = let theProc = cast[proc() {.cdecl.}](sslSym("OPENSSL_add_all_algorithms_conf")) if not theProc.isNil: theProc() + proc getOpenSSLVersion*(): culong = + ## Return OpenSSL version as unsigned long or 0 if not available + let theProc = cast[proc(): culong {.cdecl.}](sslSym("OpenSSL_version_num")) + result = + if theProc.isNil: 0.culong + else: theProc() + proc ERR_load_BIO_strings*(){.cdecl, dynlib: DLLUtilName, importc.} proc SSL_new*(context: SslCtx): SslPtr{.cdecl, dynlib: DLLSSLName, importc.} @@ -327,7 +390,7 @@ proc ERR_peek_last_error*(): cInt{.cdecl, dynlib: DLLUtilName, importc.} proc OPENSSL_config*(configName: cstring){.cdecl, dynlib: DLLSSLName, importc.} -when not useWinVersion and not defined(macosx) and not defined(android): +when not useWinVersion and not defined(macosx) and not defined(android) and not defined(nimNoAllocForSSL): proc CRYPTO_set_mem_functions(a,b,c: pointer){.cdecl, dynlib: DLLUtilName, importc.} @@ -341,7 +404,7 @@ when not useWinVersion and not defined(macosx) and not defined(android): if p != nil: dealloc(p) proc CRYPTO_malloc_init*() = - when not useWinVersion and not defined(macosx) and not defined(android): + when not useWinVersion and not defined(macosx) and not defined(android) and not defined(nimNoAllocForSSL): CRYPTO_set_mem_functions(allocWrapper, reallocWrapper, deallocWrapper) proc SSL_CTX_ctrl*(ctx: SslCtx, cmd: cInt, larg: int, parg: pointer): int{. diff --git a/lib/wrappers/pcre.nim b/lib/wrappers/pcre.nim index 6c7088bbf..e9e11960c 100644 --- a/lib/wrappers/pcre.nim +++ b/lib/wrappers/pcre.nim @@ -7,7 +7,7 @@ # distribution, for details about the copyright. # -{.deadCodeElim: on.} +{.deadCodeElim: on.} # dce option deprecated # The current PCRE version information. diff --git a/lib/wrappers/postgres.nim b/lib/wrappers/postgres.nim index f9a10dccb..7cb816f68 100644 --- a/lib/wrappers/postgres.nim +++ b/lib/wrappers/postgres.nim @@ -15,7 +15,7 @@ # connection-protocol. # -{.deadCodeElim: on.} +{.deadCodeElim: on.} # dce option deprecated when defined(windows): const diff --git a/lib/wrappers/sqlite3.nim b/lib/wrappers/sqlite3.nim index a12945832..0276a0a65 100644 --- a/lib/wrappers/sqlite3.nim +++ b/lib/wrappers/sqlite3.nim @@ -7,7 +7,7 @@ # distribution, for details about the copyright. # -{.deadCodeElim: on.} +{.deadCodeElim: on.} # dce option deprecated when defined(windows): when defined(nimOldDlls): const Lib = "sqlite3.dll" |